gremling/KytyPS5
1
0 Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-29 16:51:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

graphics: refactoring #2

Browse Source
This commit is contained in:
nmzik committed 2026-07-21 07:37:52 +02:00
1 parent b51d4322cc
commit 0a5f78c727
137 files changed
+10230 -10893
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/graphics/shader/recompiler/BindingLayout.cpp
+43 -44
View File
@@ -58,21 +58,21 @@ bool ImageBinding(const ImageResource& image, DescriptorBindingKind& result) {
}
}
bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>* visited,
std::set<uint32_t>* registers) {
bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>& visited,
std::set<uint32_t>& registers) {
if (id <= ScalarProvenance::Unknown) {
return true;
}
if (id >= provenance.values.size()) {
return false;
}
if ((*visited)[id] != 0) {
if (visited[id] != 0) {
return true;
}
(*visited)[id] = 1;
visited[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::UserData) {
registers->insert(value.imm);
registers.insert(value.imm);
return true;
}
if (value.op == ScalarValueOp::Phi) {
@@ -92,7 +92,7 @@ bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector<u
}
bool CollectSource(const Program& program, uint32_t source, bool allow_unknown,
std::vector<uint8_t>* visited, std::set<uint32_t>* registers) {
std::vector<uint8_t>& visited, std::set<uint32_t>& registers) {
if (allow_unknown && source == ScalarProvenance::Unknown) {
return true;
}
@@ -112,49 +112,49 @@ bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
std::set<uint32_t> registers;
std::vector<uint8_t> visited(program.provenance.values.size());
for (const auto& buffer: program.info.buffers) {
if (!CollectSource(program, buffer.source, false, &visited, &registers)) {
if (!CollectSource(program, buffer.source, false, visited, registers)) {
return false;
}
}
for (const auto& image: program.info.images) {
if (!CollectSource(program, image.source, false, &visited, &registers)) {
if (!CollectSource(program, image.source, false, visited, registers)) {
return false;
}
}
for (const auto& sampler: program.info.samplers) {
if (!CollectSource(program, sampler.source, false, &visited, &registers)) {
if (!CollectSource(program, sampler.source, false, visited, registers)) {
return false;
}
}
for (const auto& address: program.info.addresses) {
if (!CollectSource(program, address.source, true, &visited, &registers)) {
if (!CollectSource(program, address.source, true, visited, registers)) {
return false;
}
}
for (const auto& read: program.srt.reads) {
if (!CollectValue(program.provenance, read.value, &visited, &registers)) {
if (!CollectValue(program.provenance, read.value, visited, registers)) {
return false;
}
}
for (const auto read: program.srt.dynamic_reads) {
if (!CollectValue(program.provenance, read, &visited, &registers)) {
if (!CollectValue(program.provenance, read, visited, registers)) {
return false;
}
}
for (const auto source: program.srt.dynamic_sources) {
if (!CollectSource(program, source, true, &visited, &registers)) {
if (!CollectSource(program, source, true, visited, registers)) {
return false;
}
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::SLoadDword && inst.memory.kind == ResourceKind::ScalarBuffer &&
!CollectValue(program.provenance, inst.scalar_value, &visited, &registers)) {
!CollectValue(program.provenance, inst.scalar_value, visited, registers)) {
return false;
}
for (uint32_t i = 0; i < inst.src_count; i++) {
if (!CollectValue(program.provenance, inst.scalar_sources[i], &visited,
&registers)) {
if (!CollectValue(program.provenance, inst.scalar_sources[i], visited,
registers)) {
return false;
}
}
@@ -164,10 +164,10 @@ bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
return true;
}
void AddBinding(BindingLayout* layout, DescriptorBindingKind kind,
void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
std::vector<uint32_t> resources = {}) {
layout->descriptors.push_back(
{kind, static_cast<uint32_t>(layout->descriptors.size()), std::move(resources)});
layout.descriptors.push_back(
{kind, static_cast<uint32_t>(layout.descriptors.size()), std::move(resources)});
}
bool UsesGds(const Program& program) {
@@ -183,12 +183,11 @@ bool UsesGds(const Program& program) {
} // namespace
bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std::string* error) {
if (program == nullptr || !program->shader_info_complete || program->binding_layout_complete) {
bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std::string* error) {
if (!program.shader_info_complete || program.binding_layout_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid binding-layout program"
: !program->shader_info_complete ? "shader info is not ready"
: "binding layout already allocated";
*error = !program.shader_info_complete ? "shader info is not ready"
: "binding layout already allocated";
}
return false;
}
@@ -208,25 +207,25 @@ bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std
BindingLayout next;
next.descriptor_set = options.descriptor_set;
next.push_constant_offset = options.push_constant_offset;
if (!CollectUserData(*program, next.user_data_registers)) {
if (!CollectUserData(program, next.user_data_registers)) {
if (error != nullptr) {
*error = "shader plan contains an invalid scalar provenance reference";
}
return false;
}
if (!program->info.buffers.empty()) {
std::vector<uint32_t> resources(program->info.buffers.size());
if (!program.info.buffers.empty()) {
std::vector<uint32_t> resources(program.info.buffers.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(&next, DescriptorBindingKind::Buffers, std::move(resources));
AddBinding(next, DescriptorBindingKind::Buffers, std::move(resources));
}
std::array<std::vector<uint32_t>, ImageBindingKinds.size()> image_groups;
for (uint32_t i = 0; i < program->info.images.size(); i++) {
for (uint32_t i = 0; i < program.info.images.size(); i++) {
DescriptorBindingKind kind;
if (!ImageBinding(program->info.images[i], kind)) {
if (!ImageBinding(program.info.images[i], kind)) {
if (error != nullptr) {
*error = "shader info contains an invalid image binding class";
}
@@ -243,29 +242,29 @@ bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std
}
for (uint32_t i = 0; i < image_groups.size(); i++) {
if (!image_groups[i].empty()) {
AddBinding(&next, ImageBindingKinds[i], std::move(image_groups[i]));
AddBinding(next, ImageBindingKinds[i], std::move(image_groups[i]));
}
}
if (!program->info.samplers.empty()) {
std::vector<uint32_t> resources(program->info.samplers.size());
if (!program.info.samplers.empty()) {
std::vector<uint32_t> resources(program.info.samplers.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(&next, DescriptorBindingKind::Samplers, std::move(resources));
AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
}
if (UsesGds(*program)) {
AddBinding(&next, DescriptorBindingKind::Gds);
if (UsesGds(program)) {
AddBinding(next, DescriptorBindingKind::Gds);
}
if (!program->info.addresses.empty()) {
std::vector<uint32_t> resources(program->info.addresses.size());
if (!program.info.addresses.empty()) {
std::vector<uint32_t> resources(program.info.addresses.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(&next, DescriptorBindingKind::AddressMemory, std::move(resources));
AddBinding(next, DescriptorBindingKind::AddressMemory, std::move(resources));
}
if (!program->srt.reads.empty()) {
AddBinding(&next, DescriptorBindingKind::FlattenedSrt);
if (!program.srt.reads.empty()) {
AddBinding(next, DescriptorBindingKind::FlattenedSrt);
}
const auto available_push_dwords = (MaxPushConstantBytes - options.push_constant_offset) / 4u;
@@ -273,11 +272,11 @@ bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std
if (next.user_data_registers.size() <= push_limit) {
next.push_constant_size = static_cast<uint32_t>(next.user_data_registers.size() * 4u);
} else {
AddBinding(&next, DescriptorBindingKind::UserData);
AddBinding(next, DescriptorBindingKind::UserData);
}
program->bindings = std::move(next);
program->binding_layout_complete = true;
program.bindings = std::move(next);
program.binding_layout_complete = true;
return true;
}
src/graphics/shader/recompiler/BindingLayout.h
+1 -1
View File
@@ -11,7 +11,7 @@ struct BindingLayoutOptions {
uint32_t max_push_dwords = 32;
};
bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std::string* error);
bool AllocateBindings(Program& program, const BindingLayoutOptions& options, std::string* error);
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
src/graphics/shader/recompiler/ExportOps.cpp
+20 -20
View File
@@ -4,7 +4,7 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -18,33 +18,33 @@ bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
const uint32_t target = (word0 >> 4u) & 0x3fu;
const uint32_t en = word0 & 0xfu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->family = Family::EXP;
inst->opcode_id = target;
inst->opcode = Opcode::Exp;
inst->exp.target = target;
inst->exp.en = en;
inst->exp.compr = ((word0 >> 10u) & 1u) != 0;
inst->exp.done = ((word0 >> 11u) & 1u) != 0;
inst->exp.vm = ((word0 >> 12u) & 1u) != 0;
inst.pc = pc;
inst.word = word0;
inst.word_count = 2;
inst.family = Family::EXP;
inst.opcode_id = target;
inst.opcode = Opcode::Exp;
inst.exp.target = target;
inst.exp.en = en;
inst.exp.compr = ((word0 >> 10u) & 1u) != 0;
inst.exp.done = ((word0 >> 11u) & 1u) != 0;
inst.exp.vm = ((word0 >> 12u) & 1u) != 0;
SetRawWords(inst, code, word_index, 2);
if (!DecodeVectorGpr(word1 & 0xffu, &inst->src0, error) ||
!DecodeVectorGpr((word1 >> 8u) & 0xffu, &inst->src1, error) ||
!DecodeVectorGpr((word1 >> 16u) & 0xffu, &inst->src2, error) ||
!DecodeVectorGpr((word1 >> 24u) & 0xffu, &inst->src3, error)) {
if (!DecodeVectorGpr(word1 & 0xffu, inst.src0, error) ||
!DecodeVectorGpr((word1 >> 8u) & 0xffu, inst.src1, error) ||
!DecodeVectorGpr((word1 >> 16u) & 0xffu, inst.src2, error) ||
!DecodeVectorGpr((word1 >> 24u) & 0xffu, inst.src3, error)) {
return false;
}
if (target == 0x14u && inst->exp.done && en == 0x1u) {
inst->src_count = 1u;
if (target == 0x14u && inst.exp.done && en == 0x1u) {
inst.src_count = 1u;
} else {
inst->src_count = inst->exp.compr ? 2u : 4u;
inst.src_count = inst.exp.compr ? 2u : 4u;
}
if (en == 0u) {
inst->src_count = 0;
inst.src_count = 0;
}
return true;
src/graphics/shader/recompiler/ExportOps.h
+1 -1
View File
@@ -5,7 +5,7 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/ImageOps.cpp
+23 -23
View File
@@ -226,7 +226,7 @@ bool IsSingleDmaskBit(uint32_t dmask) {
} // namespace
bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -257,12 +257,12 @@ bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
const auto* gather = LookupGather(opcode);
const auto* atomic = LookupAtomic(opcode);
inst->pc = pc;
inst->word = word0;
inst->word_count = word_count;
inst->family = Family::MIMG;
inst->opcode_id = opcode;
inst->opcode = (sample != nullptr ? Opcode::ImageSample
inst.pc = pc;
inst.word = word0;
inst.word_count = word_count;
inst.family = Family::MIMG;
inst.opcode_id = opcode;
inst.opcode = (sample != nullptr ? Opcode::ImageSample
: gather != nullptr ? gather->decoded
: atomic != nullptr ? atomic->decoded
: opcode == 0x00u ? Opcode::ImageLoad
@@ -272,20 +272,20 @@ bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
: opcode == 0x0eu ? Opcode::ImageGetResinfo
: opcode == 0x60u ? Opcode::ImageGetLod
: Opcode::Unsupported);
inst->dmask = (word0 >> 8u) & 0xfu;
inst->data_dwords = gather != nullptr ? 4u : CountDmaskComponents(inst->dmask);
inst->glc = ((word0 >> 13u) & 1u) != 0;
inst->slc = ((word0 >> 25u) & 1u) != 0;
inst->image_sample_flags = (sample != nullptr ? sample->flags
inst.dmask = (word0 >> 8u) & 0xfu;
inst.data_dwords = gather != nullptr ? 4u : CountDmaskComponents(inst.dmask);
inst.glc = ((word0 >> 13u) & 1u) != 0;
inst.slc = ((word0 >> 25u) & 1u) != 0;
inst.image_sample_flags = (sample != nullptr ? sample->flags
: gather != nullptr ? gather->flags
: 0u) |
(a16 ? ImageSampleFlagA16 : 0u);
inst->image_dimension = dimension;
inst->image_nsa_dwords = nsa_dwords;
inst.image_dimension = dimension;
inst.image_nsa_dwords = nsa_dwords;
for (uint32_t i = 0; i < nsa_dwords * 4u && i < MaxImageNsaAddressComponents; i++) {
inst->image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
inst.image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
}
inst->image_address_components =
inst.image_address_components =
sample != nullptr ? ImageSampleAddressComponents(sample->flags, dimension)
: gather != nullptr ? ImageSampleAddressComponents(gather->flags, dimension)
: atomic != nullptr ? 3u
@@ -297,19 +297,19 @@ bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
: 0u);
SetRawWords(inst, code, word_index, word_count);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode is not implemented");
}
if (gather != nullptr && !IsSingleDmaskBit(inst->dmask)) {
if (gather != nullptr && !IsSingleDmaskBit(inst.dmask)) {
SetUnsupported(inst, Family::MIMG, opcode,
"MIMG image gather requires exactly one dmask bit");
}
DecodeVectorGpr(vdata, &inst->dst, nullptr);
DecodeVectorGpr(vaddr, &inst->src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr);
DecodeScalarSource(ssamp * 4u, pc, &inst->src2, nullptr);
inst->src_count = 3;
DecodeVectorGpr(vdata, inst.dst, nullptr);
DecodeVectorGpr(vaddr, inst.src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
DecodeScalarSource(ssamp * 4u, pc, inst.src2, nullptr);
inst.src_count = 3;
return true;
}
src/graphics/shader/recompiler/ImageOps.h
+1 -1
View File
@@ -5,7 +5,7 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
const char* MimgSampleOpcodeName(uint32_t opcode);
src/graphics/shader/recompiler/MemoryOps.cpp
+114 -114
View File
@@ -125,24 +125,24 @@ uint32_t SignExtendU32(uint32_t value, uint32_t bits) {
return (value ^ sign) - sign;
}
void MarkMemoryUnsupported(Instruction* inst, Family family, uint32_t opcode, const char* reason) {
inst->family = family;
inst->opcode_id = opcode;
inst->opcode = Opcode::Unsupported;
void MarkMemoryUnsupported(Instruction& inst, Family family, uint32_t opcode, const char* reason) {
inst.family = family;
inst.opcode_id = opcode;
inst.opcode = Opcode::Unsupported;
SetUnsupported(inst, family, opcode, reason);
}
void ApplyMemoryInfo(Instruction* inst, const MemoryOpcodeInfo* info) {
void ApplyMemoryInfo(Instruction& inst, const MemoryOpcodeInfo* info) {
if (info == nullptr) {
inst->opcode = Opcode::Unsupported;
inst.opcode = Opcode::Unsupported;
return;
}
inst->opcode = info->decoded;
inst->data_dwords = info->data_dwords;
inst->data_bits = info->data_bits;
inst->data_signed = info->data_signed;
inst->typed = info->typed;
inst->formatted = info->formatted;
inst.opcode = info->decoded;
inst.data_dwords = info->data_dwords;
inst.data_bits = info->data_bits;
inst.data_signed = info->data_signed;
inst.typed = info->typed;
inst.formatted = info->formatted;
}
bool IsDsWriteOpcode(Opcode opcode) {
@@ -215,7 +215,7 @@ bool IsFlatStoreOpcode(Opcode opcode) {
} // namespace
bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -231,32 +231,32 @@ bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
const uint32_t sbase = word0 & 0x3fu;
const uint32_t soffset = (word1 >> 25u) & 0x7fu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = SignExtendU32(word1 & 0x1fffffu, 21u);
inst->glc = ((word0 >> 16u) & 1u) != 0;
inst->family = Family::SMEM;
inst->opcode_id = opcode;
inst.pc = pc;
inst.word = word0;
inst.word_count = 2;
inst.offset = SignExtendU32(word1 & 0x1fffffu, 21u);
inst.glc = ((word0 >> 16u) & 1u) != 0;
inst.family = Family::SMEM;
inst.opcode_id = opcode;
const auto* info =
LookupMemoryOpcode(SMEM_OPS, static_cast<uint32_t>(std::size(SMEM_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
}
DecodeScalarDestination(sdst, pc, &inst->dst, nullptr);
DecodeScalarDestination(sdst, pc, inst.dst, nullptr);
// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
// pair index; their descriptor operand consumes four SGPRs from that base.
DecodeScalarSource(sbase * 2u, pc, &inst->src0, nullptr);
DecodeScalarSource(soffset, pc, &inst->src1, nullptr);
inst->src_count = 2;
DecodeScalarSource(sbase * 2u, pc, inst.src0, nullptr);
DecodeScalarSource(soffset, pc, inst.src1, nullptr);
inst.src_count = 2;
return true;
}
bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated MUBUF instruction at pc 0x{:08x}", pc);
@@ -272,34 +272,34 @@ bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
const uint32_t srsrc = (word1 >> 16u) & 0x1fu;
const uint32_t soffset = (word1 >> 24u) & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = word0 & 0xfffu;
inst->idxen = ((word0 >> 13u) & 1u) != 0;
inst->offen = ((word0 >> 12u) & 1u) != 0;
inst->glc = ((word0 >> 14u) & 1u) != 0;
inst->slc = ((word1 >> 22u) & 1u) != 0;
inst->family = Family::MUBUF;
inst->opcode_id = opcode;
inst.pc = pc;
inst.word = word0;
inst.word_count = 2;
inst.offset = word0 & 0xfffu;
inst.idxen = ((word0 >> 13u) & 1u) != 0;
inst.offen = ((word0 >> 12u) & 1u) != 0;
inst.glc = ((word0 >> 14u) & 1u) != 0;
inst.slc = ((word1 >> 22u) & 1u) != 0;
inst.family = Family::MUBUF;
inst.opcode_id = opcode;
const auto* info =
LookupMemoryOpcode(MUBUF_OPS, static_cast<uint32_t>(std::size(MUBUF_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented");
}
DecodeVectorGpr(vdata, &inst->dst, nullptr);
DecodeVectorGpr(vaddr, &inst->src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr);
DecodeScalarSource(soffset, pc, &inst->src2, nullptr);
inst->src_count = 3;
DecodeVectorGpr(vdata, inst.dst, nullptr);
DecodeVectorGpr(vaddr, inst.src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
DecodeScalarSource(soffset, pc, inst.src2, nullptr);
inst.src_count = 3;
return true;
}
bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated MTBUF instruction at pc 0x{:08x}", pc);
@@ -317,35 +317,35 @@ bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
const uint32_t srsrc = (word1 >> 16u) & 0x1fu;
const uint32_t soffset = (word1 >> 24u) & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = word0 & 0xfffu;
inst->idxen = ((word0 >> 13u) & 1u) != 0;
inst->offen = ((word0 >> 12u) & 1u) != 0;
inst->glc = ((word0 >> 14u) & 1u) != 0;
inst->slc = ((word1 >> 22u) & 1u) != 0;
inst->family = Family::MTBUF;
inst->opcode_id = opcode;
inst->data_format = dfmt;
inst->number_format = nfmt;
inst.pc = pc;
inst.word = word0;
inst.word_count = 2;
inst.offset = word0 & 0xfffu;
inst.idxen = ((word0 >> 13u) & 1u) != 0;
inst.offen = ((word0 >> 12u) & 1u) != 0;
inst.glc = ((word0 >> 14u) & 1u) != 0;
inst.slc = ((word1 >> 22u) & 1u) != 0;
inst.family = Family::MTBUF;
inst.opcode_id = opcode;
inst.data_format = dfmt;
inst.number_format = nfmt;
const auto* info =
LookupMemoryOpcode(MTBUF_OPS, static_cast<uint32_t>(std::size(MTBUF_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::MTBUF, opcode, "MTBUF opcode is not implemented");
}
DecodeVectorGpr(vdata, &inst->dst, nullptr);
DecodeVectorGpr(vaddr, &inst->src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr);
DecodeScalarSource(soffset, pc, &inst->src2, nullptr);
inst->src_count = 3;
DecodeVectorGpr(vdata, inst.dst, nullptr);
DecodeVectorGpr(vaddr, inst.src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
DecodeScalarSource(soffset, pc, inst.src2, nullptr);
inst.src_count = 3;
return true;
}
bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -366,43 +366,43 @@ bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
const uint32_t data = (word1 >> 8u) & 0xffu;
const uint32_t addr = word1 & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
inst->glc = ((word0 >> 16u) & 1u) != 0;
inst->slc = ((word0 >> 17u) & 1u) != 0;
inst->family = Family::FLAT;
inst->opcode_id = opcode;
inst->memory_segment = seg;
inst.pc = pc;
inst.word = word0;
inst.word_count = 2;
inst.offset = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
inst.glc = ((word0 >> 16u) & 1u) != 0;
inst.slc = ((word0 >> 17u) & 1u) != 0;
inst.family = Family::FLAT;
inst.opcode_id = opcode;
inst.memory_segment = seg;
const auto* info =
LookupMemoryOpcode(FLAT_OPS, static_cast<uint32_t>(std::size(FLAT_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (dlc != 0 || lds != 0 || inst->glc || inst->slc || seg == 3u) {
if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) {
SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
return true;
}
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::FLAT, opcode, "FLAT opcode is not implemented");
return true;
}
DecodeVectorGpr(IsFlatStoreOpcode(inst->opcode) ? data : vdst, &inst->dst, nullptr);
DecodeVectorGpr(addr, &inst->src0, nullptr);
inst->src_count = 1;
DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst, nullptr);
DecodeVectorGpr(addr, inst.src0, nullptr);
inst.src_count = 1;
if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
DecodeVectorGpr(addr + 1u, &inst->src1, nullptr);
inst->src_count = 2;
DecodeVectorGpr(addr + 1u, inst.src1, nullptr);
inst.src_count = 2;
} else {
DecodeScalarSource(saddr, pc, &inst->src1, nullptr);
inst->src_count = 2;
DecodeScalarSource(saddr, pc, inst.src1, nullptr);
inst.src_count = 2;
}
return true;
}
bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -421,59 +421,59 @@ bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
const uint32_t data0 = (word1 >> 8u) & 0xffu;
const uint32_t addr = word1 & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = offset0 | (offset1 << 8u);
inst->gds = ((word0 >> 17u) & 1u) != 0u;
inst->family = Family::DS;
inst->opcode_id = opcode;
inst.pc = pc;
inst.word = word0;
inst.word_count = 2;
inst.offset = offset0 | (offset1 << 8u);
inst.gds = ((word0 >> 17u) & 1u) != 0u;
inst.family = Family::DS;
inst.opcode_id = opcode;
const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
}
if (inst->opcode == Opcode::DsSwizzleB32 && inst->offset >= 0xe000u) {
if (inst.opcode == Opcode::DsSwizzleB32 && inst.offset >= 0xe000u) {
SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
}
if (inst->gds &&
(inst->opcode == Opcode::DsSwizzleB32 || inst->opcode == Opcode::DsWriteAddtidB32 ||
inst->opcode == Opcode::DsReadAddtidB32)) {
if (inst.gds &&
(inst.opcode == Opcode::DsSwizzleB32 || inst.opcode == Opcode::DsWriteAddtidB32 ||
inst.opcode == Opcode::DsReadAddtidB32)) {
SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
}
if (inst->gds && (inst->opcode == Opcode::DsAppend || inst->opcode == Opcode::DsConsume) &&
inst->offset != 0u) {
if (inst.gds && (inst.opcode == Opcode::DsAppend || inst.opcode == Opcode::DsConsume) &&
inst.offset != 0u) {
SetUnsupported(inst, Family::DS, opcode,
"GDS append/consume requires a zero instruction offset");
}
if (inst->opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
SetUnsupported(inst, Family::DS, opcode,
"DS write addtid data1 operand is not implemented");
}
if (inst->opcode == Opcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) {
if (inst.opcode == Opcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) {
SetUnsupported(inst, Family::DS, opcode,
"DS read addtid data operands are not implemented");
}
if (inst->opcode == Opcode::DsWrite2B32 || opcode == 0x37u) {
inst->offset = offset0 * 4u;
inst->secondary_offset = offset1 * 4u;
} else if (inst->opcode == Opcode::DsWrite2St64B32 || opcode == 0x38u) {
inst->offset = offset0 * 256u;
inst->secondary_offset = offset1 * 256u;
} else if (inst->opcode == Opcode::DsWrite2B64 || inst->opcode == Opcode::DsRead2B64) {
inst->offset = offset0 * 8u;
inst->secondary_offset = offset1 * 8u;
} else if (inst->opcode == Opcode::DsWrite2St64B64 || inst->opcode == Opcode::DsRead2St64B64) {
inst->offset = offset0 * 512u;
inst->secondary_offset = offset1 * 512u;
if (inst.opcode == Opcode::DsWrite2B32 || opcode == 0x37u) {
inst.offset = offset0 * 4u;
inst.secondary_offset = offset1 * 4u;
} else if (inst.opcode == Opcode::DsWrite2St64B32 || opcode == 0x38u) {
inst.offset = offset0 * 256u;
inst.secondary_offset = offset1 * 256u;
} else if (inst.opcode == Opcode::DsWrite2B64 || inst.opcode == Opcode::DsRead2B64) {
inst.offset = offset0 * 8u;
inst.secondary_offset = offset1 * 8u;
} else if (inst.opcode == Opcode::DsWrite2St64B64 || inst.opcode == Opcode::DsRead2St64B64) {
inst.offset = offset0 * 512u;
inst.secondary_offset = offset1 * 512u;
}
DecodeVectorGpr(vdst, &inst->dst, nullptr);
DecodeVectorGpr(addr, &inst->src0, nullptr);
DecodeVectorGpr(data0, &inst->src1, nullptr);
DecodeVectorGpr(data1, &inst->src2, nullptr);
inst->src_count = DsSourceCount(inst->opcode);
DecodeVectorGpr(vdst, inst.dst, nullptr);
DecodeVectorGpr(addr, inst.src0, nullptr);
DecodeVectorGpr(data0, inst.src1, nullptr);
DecodeVectorGpr(data1, inst.src2, nullptr);
inst.src_count = DsSourceCount(inst.opcode);
return true;
}
src/graphics/shader/recompiler/MemoryOps.h
+5 -5
View File
@@ -5,15 +5,15 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
Instruction& inst, std::string* error);
bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
Instruction& inst, std::string* error);
bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/ResourceMaterialization.cpp
+19 -20
View File
@@ -5,6 +5,7 @@
#include <algorithm>
#include <fmt/format.h>
#include <functional>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
@@ -200,11 +201,10 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
}
bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot* snapshot, std::string* error) {
if (snapshot == nullptr || !program.resource_tracking_complete) {
ResourceSnapshot& snapshot, std::string* error) {
if (!program.resource_tracking_complete) {
if (error != nullptr) {
*error = snapshot == nullptr ? "invalid resource snapshot"
: "shader resources were not tracked";
*error = "shader resources were not tracked";
}
return false;
}
@@ -229,7 +229,7 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
std::vector<DescriptorValue> values;
std::vector<uint32_t> flattened_srt;
if (!EvaluateRuntimeSources(program, requests, runtime, &values, &flattened_srt, error)) {
if (!EvaluateRuntimeSources(program, requests, runtime, values, flattened_srt, error)) {
return false;
}
@@ -280,25 +280,24 @@ bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
if (!ValidateResourceSnapshot(program, next, error)) {
return false;
}
*snapshot = std::move(next);
snapshot = std::move(next);
return true;
}
bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std::string* error) {
if (program == nullptr || !program->resource_tracking_complete ||
program->shader_info_complete || program->binding_layout_complete) {
bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std::string* error) {
if (!program.resource_tracking_complete || program.shader_info_complete ||
program.binding_layout_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid resource specialization program"
: !program->resource_tracking_complete ? "shader resources were not tracked"
: "resource specialization is too late";
*error = !program.resource_tracking_complete ? "shader resources were not tracked"
: "resource specialization is too late";
}
return false;
}
if (!ValidateResourceSnapshot(*program, snapshot, error)) {
if (!ValidateResourceSnapshot(program, snapshot, error)) {
return false;
}
auto next = program->info;
auto next = program.info;
for (uint32_t i = 0; i < next.buffers.size(); i++) {
ShaderBufferResource descriptor;
if (!DecodeBufferDescriptor(snapshot.buffers[i], descriptor)) {
@@ -347,12 +346,12 @@ bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std
}
}
struct ImagePatch {
Instruction* inst;
std::reference_wrapper<Instruction> inst;
ResourceKind kind;
Decoder::ImageDimension dimension;
};
std::vector<ImagePatch> patches;
for (auto& block: program->blocks) {
for (auto& block: program.blocks) {
for (auto& inst: block.instructions) {
if (inst.memory.kind != ResourceKind::Image &&
inst.memory.kind != ResourceKind::ImageUint &&
@@ -368,13 +367,13 @@ bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std
return false;
}
const auto& image = next.images[inst.memory.resource];
patches.push_back({&inst, image.kind, image.dimension});
patches.push_back({std::ref(inst), image.kind, image.dimension});
}
}
program->info = std::move(next);
program.info = std::move(next);
for (const auto& patch: patches) {
patch.inst->memory.kind = patch.kind;
patch.inst->memory.image_dimension = patch.dimension;
patch.inst.get().memory.kind = patch.kind;
patch.inst.get().memory.image_dimension = patch.dimension;
}
return true;
}
src/graphics/shader/recompiler/ResourceMaterialization.h
+2 -2
View File
@@ -31,11 +31,11 @@ bool ValidateResourceSpecialization(const Program& program, const ResourceSnapsh
// Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot.
// On failure the destination is unchanged.
bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot* snapshot, std::string* error);
ResourceSnapshot& snapshot, std::string* error);
// Applies runtime descriptor shape/format facts to a copied dense topology before layout and
// emission. On failure the program is unchanged.
bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std::string* error);
bool SpecializeResources(Program& program, const ResourceSnapshot& snapshot, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ResourceTracking.cpp
+51 -57
View File
@@ -4,6 +4,7 @@
#include <algorithm>
#include <fmt/format.h>
#include <functional>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
@@ -82,17 +83,17 @@ uint32_t ByteExtent(const Instruction& inst) {
return end > UINT32_MAX ? UINT32_MAX : static_cast<uint32_t>(end);
}
bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>* visited,
std::vector<uint32_t>* path) {
path->push_back(id);
bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>& visited,
std::vector<uint32_t>& path) {
path.push_back(id);
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return true;
}
if ((*visited)[id] != 0) {
path->pop_back();
if (visited[id] != 0) {
path.pop_back();
return false;
}
(*visited)[id] = 1;
visited[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
@@ -100,7 +101,7 @@ bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vecto
return true;
}
}
path->pop_back();
path.pop_back();
return false;
}
const auto args = ScalarValueArgCount(value.op);
@@ -109,19 +110,19 @@ bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vecto
return true;
}
}
path->pop_back();
path.pop_back();
return false;
}
bool IsLoopInvariantValue(const ScalarProvenance& provenance, uint32_t id,
std::vector<uint8_t>* visiting) {
std::vector<uint8_t>& visiting) {
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return false;
}
if ((*visiting)[id] != 0) {
if (visiting[id] != 0) {
return true;
}
(*visiting)[id] = 1;
visiting[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
uint32_t invariant = ScalarProvenance::Undefined;
@@ -150,7 +151,7 @@ bool IsLoopInvariantDescriptor(const ScalarProvenance& provenance,
const DescriptorValue& descriptor) {
std::vector<uint8_t> visiting(provenance.values.size());
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (!IsLoopInvariantValue(provenance, descriptor.dwords[i], &visiting)) {
if (!IsLoopInvariantValue(provenance, descriptor.dwords[i], visiting)) {
return false;
}
}
@@ -173,7 +174,7 @@ public:
}
for (auto& block: m_program.blocks) {
for (auto& inst: block.instructions) {
if (!Collect(&inst, error)) {
if (!Collect(inst, error)) {
return false;
}
}
@@ -181,10 +182,11 @@ public:
LinkImageAliases();
m_program.info = std::move(m_info);
for (const auto& patch: m_patches) {
patch.inst->memory.resource = patch.resource;
patch.inst->memory.sampler = patch.sampler;
patch.inst->memory.resource_source = ScalarProvenance::Undefined;
patch.inst->memory.sampler_source = ScalarProvenance::Undefined;
auto& inst = patch.inst.get();
inst.memory.resource = patch.resource;
inst.memory.sampler = patch.sampler;
inst.memory.resource_source = ScalarProvenance::Undefined;
inst.memory.sampler_source = ScalarProvenance::Undefined;
}
m_program.resource_tracking_complete = true;
return true;
@@ -192,9 +194,9 @@ public:
private:
struct Patch {
Instruction* inst = nullptr;
uint32_t resource = 0;
uint32_t sampler = 0;
std::reference_wrapper<Instruction> inst;
uint32_t resource = 0;
uint32_t sampler = 0;
};
bool Fail(uint32_t pc, std::string* error, const std::string& reason) const {
@@ -214,7 +216,7 @@ private:
std::vector<uint8_t> visited(m_program.provenance.values.size());
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
std::vector<uint32_t> path;
if (ContainsUnknown(m_program.provenance, descriptor->dwords[i], &visited, &path)) {
if (ContainsUnknown(m_program.provenance, descriptor->dwords[i], visited, path)) {
const auto value = descriptor->dwords[i];
std::string chain;
for (const auto id: path) {
@@ -400,50 +402,49 @@ private:
return true;
}
bool Collect(Instruction* inst, std::string* error) {
if (IsAddress(*inst)) {
const bool unbased = inst->memory.resource_source == ScalarProvenance::Unknown;
if ((!unbased && !ValidateSource(inst->memory.resource_source, 2, inst->pc, error)) ||
(unbased && inst->memory.kind != ResourceKind::Flat &&
inst->memory.kind != ResourceKind::Global &&
inst->memory.kind != ResourceKind::Scratch)) {
return unbased ? Fail(inst->pc, error, "scalar memory base is unresolved") : false;
bool Collect(Instruction& inst, std::string* error) {
if (IsAddress(inst)) {
const bool unbased = inst.memory.resource_source == ScalarProvenance::Unknown;
if ((!unbased && !ValidateSource(inst.memory.resource_source, 2, inst.pc, error)) ||
(unbased && inst.memory.kind != ResourceKind::Flat &&
inst.memory.kind != ResourceKind::Global &&
inst.memory.kind != ResourceKind::Scratch)) {
return unbased ? Fail(inst.pc, error, "scalar memory base is unresolved") : false;
}
const auto resource = AddAddress(*inst);
const auto resource = AddAddress(inst);
if (resource == UINT32_MAX) {
return Fail(inst->pc, error, "address resource limit exceeded");
return Fail(inst.pc, error, "address resource limit exceeded");
}
m_patches.push_back({inst, resource, 0});
m_patches.push_back({std::ref(inst), resource, 0});
return true;
}
if (!IsBuffer(*inst) && !IsImage(*inst)) {
if (!IsBuffer(inst) && !IsImage(inst)) {
return true;
}
if (!ValidateSource(inst->memory.resource_source, IsBuffer(*inst) ? 4u : 8u, inst->pc,
if (!ValidateSource(inst.memory.resource_source, IsBuffer(inst) ? 4u : 8u, inst.pc,
error)) {
return false;
}
Patch patch;
patch.inst = inst;
patch.resource = IsBuffer(*inst) ? AddBuffer(*inst) : AddImage(*inst);
if (patch.resource == UINT32_MAX) {
return Fail(inst->pc, error,
IsBuffer(*inst) ? "buffer resource limit exceeded"
: "image resource limit exceeded");
const auto resource = IsBuffer(inst) ? AddBuffer(inst) : AddImage(inst);
if (resource == UINT32_MAX) {
return Fail(inst.pc, error,
IsBuffer(inst) ? "buffer resource limit exceeded"
: "image resource limit exceeded");
}
if (NeedsSampler(inst->op)) {
if (!ValidateSource(inst->memory.sampler_source, 4, inst->pc, error)) {
uint32_t sampler = 0;
if (NeedsSampler(inst.op)) {
if (!ValidateSource(inst.memory.sampler_source, 4, inst.pc, error)) {
return false;
}
patch.sampler = AddSampler(*inst);
if (patch.sampler == UINT32_MAX) {
return Fail(inst->pc, error, "sampler resource limit exceeded");
sampler = AddSampler(inst);
if (sampler == UINT32_MAX) {
return Fail(inst.pc, error, "sampler resource limit exceeded");
}
if (!AddSampledPair(patch.resource, patch.sampler, inst->pc, error)) {
if (!AddSampledPair(resource, sampler, inst.pc, error)) {
return false;
}
}
m_patches.push_back(patch);
m_patches.push_back({std::ref(inst), resource, sampler});
return true;
}
@@ -454,15 +455,8 @@ private:
} // namespace
bool TrackResources(Program* program, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error = "shader resource tracking: hash=0x0000000000000000 stage=unknown "
"pc=0x00000000 invalid program";
}
return false;
}
return Tracker(*program).Run(error);
bool TrackResources(Program& program, std::string* error) {
return Tracker(program).Run(error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ResourceTracking.h
+1 -1
View File
@@ -7,7 +7,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
// Collects immutable resource topology, then replaces descriptor operands with dense indices.
// On failure the program is left unpatched.
bool TrackResources(Program* program, std::string* error);
bool TrackResources(Program& program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ScalarAluOps.cpp
+81 -81
View File
@@ -89,59 +89,59 @@ Opcode Lookup(const OpcodeMap* ops, uint32_t count, uint32_t opcode) {
}
bool DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) {
if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error) ||
!DecodeScalarSource(ssrc1, pc, &inst->src1, error)) {
Instruction& inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) {
if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
!DecodeScalarSource(ssrc1, pc, inst.src1, error)) {
return false;
}
inst->src_count = 2;
inst.src_count = 2;
return ReadLiteralOperands(code, word_index, inst, error);
}
} // namespace
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu;
const uint32_t sdst = (word >> 16u) & 0x7fu;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOP1;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOP1_OPS, static_cast<uint32_t>(std::size(SOP1_OPS)), opcode);
inst.pc = pc;
inst.word = word;
inst.family = Family::SOP1;
inst.opcode_id = opcode;
inst.opcode = Lookup(SOP1_OPS, static_cast<uint32_t>(std::size(SOP1_OPS)), opcode);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented");
return true;
}
switch (inst->opcode) {
switch (inst.opcode) {
case Opcode::SGetpcB64:
inst->src_count = 0;
return DecodeScalarDestination(sdst, pc, &inst->dst, error);
inst.src_count = 0;
return DecodeScalarDestination(sdst, pc, inst.dst, error);
case Opcode::SSetpcB64:
inst->src_count = 1;
inst->dst.kind = OperandKind::Null;
if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error)) {
inst.src_count = 1;
inst.dst.kind = OperandKind::Null;
if (!DecodeScalarSource(ssrc0, pc, inst.src0, error)) {
return false;
}
return ReadLiteralOperands(code, word_index, inst, error);
default: break;
}
if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error) ||
!DecodeScalarDestination(sdst, pc, &inst->dst, error)) {
if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
!DecodeScalarDestination(sdst, pc, inst.dst, error)) {
return false;
}
inst->src_count = 1;
inst.src_count = 1;
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 23u) & 0x7fu;
@@ -149,25 +149,25 @@ bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
const uint32_t ssrc0 = word & 0xffu;
const uint32_t sdst = (word >> 16u) & 0x7fu;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOP2;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOP2_OPS, static_cast<uint32_t>(std::size(SOP2_OPS)), opcode);
inst.pc = pc;
inst.word = word;
inst.family = Family::SOP2;
inst.opcode_id = opcode;
inst.opcode = Lookup(SOP2_OPS, static_cast<uint32_t>(std::size(SOP2_OPS)), opcode);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented");
return true;
}
if (!DecodeScalarDestination(sdst, pc, &inst->dst, error)) {
if (!DecodeScalarDestination(sdst, pc, inst.dst, error)) {
return false;
}
return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
}
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 23u) & 0x1fu;
@@ -176,72 +176,72 @@ bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
(void)error;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOPK;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOPK_OPS, static_cast<uint32_t>(std::size(SOPK_OPS)), opcode);
inst->src0.kind = OperandKind::IntegerInlineConstant;
inst->src0.signed_val = imm;
inst->src0.value = static_cast<uint32_t>(imm);
inst->src_count = 1;
inst.pc = pc;
inst.word = word;
inst.family = Family::SOPK;
inst.opcode_id = opcode;
inst.opcode = Lookup(SOPK_OPS, static_cast<uint32_t>(std::size(SOPK_OPS)), opcode);
inst.src0.kind = OperandKind::IntegerInlineConstant;
inst.src0.signed_val = imm;
inst.src0.value = static_cast<uint32_t>(imm);
inst.src_count = 1;
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented");
return true;
}
switch (inst->opcode) {
case Opcode::SMovkI32: return DecodeScalarDestination(sdst, pc, &inst->dst, error);
switch (inst.opcode) {
case Opcode::SMovkI32: return DecodeScalarDestination(sdst, pc, inst.dst, error);
case Opcode::SWaitcnt: {
const uint32_t waitcnt = word & 0xffffu;
inst->dst.kind = OperandKind::Null;
inst->src0.signed_val = static_cast<int32_t>(waitcnt);
inst->src0.value = waitcnt;
inst->src_count = 1;
inst.dst.kind = OperandKind::Null;
inst.src0.signed_val = static_cast<int32_t>(waitcnt);
inst.src0.value = waitcnt;
inst.src_count = 1;
return true;
}
case Opcode::SSetregB32:
inst->dst.kind = OperandKind::Null;
inst->src1.kind = OperandKind::LiteralConstant;
inst->src1.value = word & 0xffffu;
inst->src1.signed_val = static_cast<int32_t>(imm);
inst->src_count = 2;
return DecodeScalarSource(sdst, pc, &inst->src0, error);
inst.dst.kind = OperandKind::Null;
inst.src1.kind = OperandKind::LiteralConstant;
inst.src1.value = word & 0xffffu;
inst.src1.signed_val = static_cast<int32_t>(imm);
inst.src_count = 2;
return DecodeScalarSource(sdst, pc, inst.src0, error);
default: break;
}
inst->src1 = inst->src0;
if (!DecodeScalarSource(sdst, pc, &inst->src0, error)) {
inst.src1 = inst.src0;
if (!DecodeScalarSource(sdst, pc, inst.src0, error)) {
return false;
}
if (inst->opcode == Opcode::SAddI32 || inst->opcode == Opcode::SMulkI32) {
inst->src_count = 2;
return DecodeScalarDestination(sdst, pc, &inst->dst, error);
if (inst.opcode == Opcode::SAddI32 || inst.opcode == Opcode::SMulkI32) {
inst.src_count = 2;
return DecodeScalarDestination(sdst, pc, inst.dst, error);
}
inst->dst.kind = OperandKind::Scc;
inst->src_count = 2;
inst.dst.kind = OperandKind::Scc;
inst.src_count = 2;
return true;
}
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t ssrc1 = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu;
const uint32_t opcode = (word >> 16u) & 0x7fu;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOPC;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOPC_OPS, static_cast<uint32_t>(std::size(SOPC_OPS)), opcode);
inst->dst.kind = OperandKind::Scc;
inst.pc = pc;
inst.word = word;
inst.family = Family::SOPC;
inst.opcode_id = opcode;
inst.opcode = Lookup(SOPC_OPS, static_cast<uint32_t>(std::size(SOPC_OPS)), opcode);
inst.dst.kind = OperandKind::Scc;
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented");
return true;
}
@@ -249,7 +249,7 @@ bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
}
bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 16u) & 0x7fu;
@@ -257,24 +257,24 @@ bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
(void)error;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOPP;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOPP_OPS, static_cast<uint32_t>(std::size(SOPP_OPS)), opcode);
inst->src0.kind = OperandKind::LiteralConstant;
inst->src0.value = simm;
inst->src0.signed_val = static_cast<int16_t>(simm);
inst->src_count = (inst->opcode == Opcode::SNop || inst->opcode == Opcode::SWaitcnt ||
inst->opcode == Opcode::SSleep || inst->opcode == Opcode::SSendmsg ||
inst->opcode == Opcode::STtraceData || inst->opcode == Opcode::SInstPrefetch)
inst.pc = pc;
inst.word = word;
inst.family = Family::SOPP;
inst.opcode_id = opcode;
inst.opcode = Lookup(SOPP_OPS, static_cast<uint32_t>(std::size(SOPP_OPS)), opcode);
inst.src0.kind = OperandKind::LiteralConstant;
inst.src0.value = simm;
inst.src0.signed_val = static_cast<int16_t>(simm);
inst.src_count = (inst.opcode == Opcode::SNop || inst.opcode == Opcode::SWaitcnt ||
inst.opcode == Opcode::SSleep || inst.opcode == Opcode::SSendmsg ||
inst.opcode == Opcode::STtraceData || inst.opcode == Opcode::SInstPrefetch)
? 1
: 0;
inst->branch_offset = static_cast<int32_t>(static_cast<int16_t>(simm)) * 4;
inst->branch_target = pc + 4u + static_cast<uint32_t>(inst->branch_offset);
inst.branch_offset = static_cast<int32_t>(static_cast<int16_t>(simm)) * 4;
inst.branch_target = pc + 4u + static_cast<uint32_t>(inst.branch_offset);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOPP, opcode, "SOPP control-flow opcode is not implemented");
}
return true;
src/graphics/shader/recompiler/ScalarAluOps.h
+5 -5
View File
@@ -5,15 +5,15 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/ScalarProvenance.cpp
+104 -113
View File
@@ -140,20 +140,20 @@ bool ScalarMemoryLoad(Opcode op) {
return op == Opcode::SLoadDword || op == Opcode::SBufferLoadDword;
}
bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>* visited) {
bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>& visited) {
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return false;
}
if ((*visited)[id] == 1) {
if (visited[id] == 1) {
return false;
}
if ((*visited)[id] == 2) {
if (visited[id] == 2) {
return true;
}
if ((*visited)[id] == 3) {
if (visited[id] == 3) {
return false;
}
(*visited)[id] = 1;
visited[id] = 1;
const auto& value = provenance.values[id];
bool resolved = true;
if (value.op == ScalarValueOp::Phi) {
@@ -167,13 +167,13 @@ bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector<
resolved = ValueResolved(provenance, value.args[i], visited);
}
}
(*visited)[id] = resolved ? 2 : 3;
visited[id] = resolved ? 2 : 3;
return resolved;
}
class Builder {
public:
explicit Builder(Program* program): m_program(program), m_graph(program->provenance) {}
explicit Builder(Program& program): m_program(program), m_graph(program.provenance) {}
bool Run(std::string* error) {
m_graph = {};
@@ -181,7 +181,7 @@ public:
m_graph.values[ScalarProvenance::Undefined].op = ScalarValueOp::Undefined;
m_graph.values[ScalarProvenance::Unknown].op = ScalarValueOp::Unknown;
const auto block_count = m_program->blocks.size();
const auto block_count = m_program.blocks.size();
m_entry.resize(block_count);
m_exit.resize(block_count);
m_phi.resize(block_count);
@@ -200,8 +200,8 @@ public:
m_user_data.Fill(ScalarProvenance::Undefined);
for (uint32_t reg = 0; reg < ScalarRegisters; reg++) {
m_user_data.regs[reg] =
reg >= m_program->user_data_base &&
reg - m_program->user_data_base < m_program->user_data_count
reg >= m_program.user_data_base &&
reg - m_program.user_data_base < m_program.user_data_count
? InternValue({ScalarValueOp::UserData, 0, reg})
: ScalarProvenance::Unknown;
}
@@ -211,7 +211,7 @@ public:
if (block_count == 0) {
return true;
}
for (const auto& block: m_program->blocks) {
for (const auto& block: m_program.blocks) {
for (const auto predecessor: block.predecessors) {
if (predecessor >= block_count) {
return Fail(error, fmt::format("scalar provenance has invalid predecessor {}",
@@ -238,20 +238,20 @@ public:
const bool entry_changed = next_entry != m_entry[block_index];
m_entry[block_index] = next_entry;
auto next_exit = Execute(m_program->blocks[block_index], next_entry, false);
auto next_exit = Execute(m_program.blocks[block_index], next_entry, false);
const bool was_ready = m_exit_ready[block_index];
if (was_ready && !entry_changed && next_exit == m_exit[block_index]) {
continue;
}
m_exit[block_index] = next_exit;
m_exit_ready[block_index] = true;
for (const auto successor: m_program->blocks[block_index].successors) {
for (const auto successor: m_program.blocks[block_index].successors) {
Queue(successor);
}
}
for (size_t block = 0; block < block_count; block++) {
Execute(m_program->blocks[block], m_entry[block], true);
Execute(m_program.blocks[block], m_entry[block], true);
}
return true;
}
@@ -366,40 +366,37 @@ private:
return InternValue(std::move(node));
}
bool ConstantOperand(const Operand& operand, const ScalarState& state, uint32_t* value) {
if (value == nullptr) {
return false;
}
bool ConstantOperand(const Operand& operand, const ScalarState& state, uint32_t& value) {
const auto id = OperandValue(operand, state);
if (id >= m_graph.values.size() || m_graph.values[id].op != ScalarValueOp::Constant) {
return false;
}
*value = m_graph.values[id].imm;
value = m_graph.values[id].imm;
return true;
}
uint32_t ReadVectorLane(const Instruction& inst, const ScalarState& state) {
if (inst.src_count < 2 || inst.src[0].kind != OperandKind::Register ||
inst.src[0].reg.file != RegisterFile::Vector ||
(m_program->wave_size != 32 && m_program->wave_size != 64)) {
(m_program.wave_size != 32 && m_program.wave_size != 64)) {
return ScalarProvenance::Unknown;
}
uint32_t lane = 0;
if (!ConstantOperand(inst.src[1], state, &lane)) {
if (!ConstantOperand(inst.src[1], state, lane)) {
return ScalarProvenance::Unknown;
}
const auto found = state.vector_lanes.find(
VectorLaneKey(inst.src[0].reg.index, lane % m_program->wave_size));
VectorLaneKey(inst.src[0].reg.index, lane % m_program.wave_size));
return found != state.vector_lanes.end() ? found->second : ScalarProvenance::Unknown;
}
void ClearVectorLanes(uint32_t reg, ScalarState* state) {
const auto first = state->vector_lanes.lower_bound(VectorLaneKey(reg, 0));
const auto last = state->vector_lanes.lower_bound((static_cast<uint64_t>(reg) + 1u) << 32u);
state->vector_lanes.erase(first, last);
void ClearVectorLanes(uint32_t reg, ScalarState& state) {
const auto first = state.vector_lanes.lower_bound(VectorLaneKey(reg, 0));
const auto last = state.vector_lanes.lower_bound((static_cast<uint64_t>(reg) + 1u) << 32u);
state.vector_lanes.erase(first, last);
}
void WriteVectorDestination(const Instruction& inst, ScalarState* state) {
void WriteVectorDestination(const Instruction& inst, ScalarState& state) {
if (inst.dst.kind != OperandKind::Register || inst.dst.reg.file != RegisterFile::Vector) {
return;
}
@@ -410,8 +407,8 @@ private:
for (uint32_t i = 0; i < inst.src_count; i++) {
uint32_t src_reg = 0;
if (VectorRegister(inst.src[i], src_reg) &&
state->address_bases[src_reg] > ScalarProvenance::Unknown) {
address_base = state->address_bases[src_reg];
state.address_bases[src_reg] > ScalarProvenance::Unknown) {
address_base = state.address_bases[src_reg];
break;
}
}
@@ -419,16 +416,16 @@ private:
for (uint32_t i = 0; i < inst.src_count; i++) {
uint32_t src_reg = 0;
if (ScalarRegister(inst.src[i], src_reg) &&
state->regs[src_reg] > ScalarProvenance::Unknown) {
address_base = state->regs[src_reg];
state.regs[src_reg] > ScalarProvenance::Unknown) {
address_base = state.regs[src_reg];
break;
}
}
}
}
state->address_bases[reg] = ScalarProvenance::Unknown;
state.address_bases[reg] = ScalarProvenance::Unknown;
if (inst.op == Opcode::MoveRelDestU32) {
state->vector_lanes.clear();
state.vector_lanes.clear();
return;
}
if (inst.op != Opcode::WriteLaneU32) {
@@ -438,24 +435,24 @@ private:
}
for (uint32_t i = 0; i < dwords && reg <= UINT32_MAX - i; i++) {
ClearVectorLanes(reg + i, state);
if (reg + i < state->address_bases.size()) {
state->address_bases[reg + i] = ScalarProvenance::Unknown;
if (reg + i < state.address_bases.size()) {
state.address_bases[reg + i] = ScalarProvenance::Unknown;
}
}
state->address_bases[reg] = address_base;
state.address_bases[reg] = address_base;
return;
}
if (inst.src_count < 2 || (m_program->wave_size != 32 && m_program->wave_size != 64)) {
if (inst.src_count < 2 || (m_program.wave_size != 32 && m_program.wave_size != 64)) {
ClearVectorLanes(reg, state);
return;
}
uint32_t lane = 0;
if (!ConstantOperand(inst.src[1], *state, &lane)) {
if (!ConstantOperand(inst.src[1], state, lane)) {
ClearVectorLanes(reg, state);
return;
}
state->vector_lanes[VectorLaneKey(reg, lane % m_program->wave_size)] =
OperandValue(inst.src[0], *state);
state.vector_lanes[VectorLaneKey(reg, lane % m_program.wave_size)] =
OperandValue(inst.src[0], state);
}
uint32_t ReadConst(const Instruction& inst, const ScalarState& state, bool buffer) {
@@ -514,31 +511,31 @@ private:
}
}
void UpdateScc(const Instruction& inst, const ScalarState& before, ScalarState* state) {
void UpdateScc(const Instruction& inst, const ScalarState& before, ScalarState& state) {
switch (inst.op) {
case Opcode::ScalarAddCarryU32:
state->scc = Define(inst, ScalarValueOp::Carry, before);
state.scc = Define(inst, ScalarValueOp::Carry, before);
break;
case Opcode::ScalarSubBorrowU32:
case Opcode::ScalarSubBorrowCarryU32: state->scc = DefineBorrow(inst, before); break;
case Opcode::ScalarSubBorrowCarryU32: state.scc = DefineBorrow(inst, before); break;
case Opcode::ScalarShiftLeftAddCarryU32:
state->scc = Define(inst, ScalarValueOp::ShiftLeftAddCarry, before);
state.scc = Define(inst, ScalarValueOp::ShiftLeftAddCarry, before);
break;
default: break;
}
}
void WriteDestination(const Instruction& inst, ScalarState* state) {
void WriteDestination(const Instruction& inst, ScalarState& state) {
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Scc) {
state->scc = inst.op == Opcode::MoveU32 && inst.src_count != 0
? OperandValue(inst.src[0], *state)
state.scc = inst.op == Opcode::MoveU32 && inst.src_count != 0
? OperandValue(inst.src[0], state)
: ScalarProvenance::Unknown;
return;
}
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::M0) {
const auto before = *state;
const auto before = state;
const auto op = Operation(inst.op);
state->m0 = inst.op == Opcode::MoveU32 && inst.src_count != 0
state.m0 = inst.op == Opcode::MoveU32 && inst.src_count != 0
? OperandValue(inst.src[0], before)
: op != ScalarValueOp::Unknown ? Define(inst, op, before)
: ScalarProvenance::Unknown;
@@ -551,10 +548,10 @@ private:
return;
}
const auto before = *state;
state->regs[dst] = ScalarProvenance::Unknown;
const auto before = state;
state.regs[dst] = ScalarProvenance::Unknown;
if (PairDwordOpcode(inst.op) && dst + 1 < ScalarRegisters) {
state->regs[dst + 1] = ScalarProvenance::Unknown;
state.regs[dst + 1] = ScalarProvenance::Unknown;
}
uint32_t value = ScalarProvenance::Unknown;
@@ -582,10 +579,10 @@ private:
if (inst.src_count != 0 && dst + 1 < ScalarRegisters) {
if (ScalarRegister(inst.src[0], src) && src + 1 < ScalarRegisters) {
value = before.regs[src];
state->regs[dst + 1] = before.regs[src + 1];
state.regs[dst + 1] = before.regs[src + 1];
} else {
value = OperandValue(inst.src[0], before);
state->regs[dst + 1] = Constant(
state.regs[dst + 1] = Constant(
inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].sext_64
? UINT32_MAX
: 0u);
@@ -596,7 +593,7 @@ private:
case Opcode::BitFieldMaskU64:
value = Define(inst, ScalarValueOp::BitFieldMaskU64Low, before);
if (dst + 1 < ScalarRegisters) {
state->regs[dst + 1] = Define(inst, ScalarValueOp::BitFieldMaskU64High, before);
state.regs[dst + 1] = Define(inst, ScalarValueOp::BitFieldMaskU64High, before);
}
break;
case Opcode::SLoadDword: value = ReadConst(inst, before, false); break;
@@ -610,12 +607,12 @@ private:
break;
}
}
state->regs[dst] = value;
state.regs[dst] = value;
UpdateScc(inst, before, state);
uint32_t dst2 = 0;
if (ScalarRegister(inst.dst2, dst2)) {
state->regs[dst2] =
state.regs[dst2] =
inst.src_count > 1 ? OperandValue(inst.src[1], before) : ScalarProvenance::Unknown;
}
}
@@ -663,57 +660,57 @@ private:
return AddDescriptor(descriptor);
}
void AttachSources(Instruction* inst, const ScalarState& state) {
inst->memory.resource_source = 0;
inst->memory.sampler_source = 0;
if (inst->op == Opcode::SLoadDword) {
inst->memory.resource_source = AddDescriptor(state, inst->memory.resource, 2);
void AttachSources(Instruction& inst, const ScalarState& state) {
inst.memory.resource_source = 0;
inst.memory.sampler_source = 0;
if (inst.op == Opcode::SLoadDword) {
inst.memory.resource_source = AddDescriptor(state, inst.memory.resource, 2);
return;
}
if (inst->memory.kind == ResourceKind::Flat || inst->memory.kind == ResourceKind::Global ||
inst->memory.kind == ResourceKind::Scratch) {
inst->memory.resource_source = ScalarProvenance::Unknown;
if (inst->memory.kind == ResourceKind::Flat) {
inst->memory.resource_source = AddFlatAddressDescriptor(*inst, state);
if (inst.memory.kind == ResourceKind::Flat || inst.memory.kind == ResourceKind::Global ||
inst.memory.kind == ResourceKind::Scratch) {
inst.memory.resource_source = ScalarProvenance::Unknown;
if (inst.memory.kind == ResourceKind::Flat) {
inst.memory.resource_source = AddFlatAddressDescriptor(inst, state);
}
for (uint32_t i = 0; i < inst->src_count; i++) {
const auto& source = inst->src[i];
if (inst->memory.resource_source == ScalarProvenance::Unknown &&
for (uint32_t i = 0; i < inst.src_count; i++) {
const auto& source = inst.src[i];
if (inst.memory.resource_source == ScalarProvenance::Unknown &&
source.kind == OperandKind::Register &&
source.reg.file == RegisterFile::Scalar) {
inst->memory.resource_source = AddDescriptor(state, source.reg.index, 2);
inst.memory.resource_source = AddDescriptor(state, source.reg.index, 2);
break;
}
}
return;
}
if (inst->memory.kind == ResourceKind::Buffer ||
(inst->memory.kind == ResourceKind::ScalarBuffer &&
inst->op == Opcode::SBufferLoadDword)) {
const auto base = inst->memory.resource * 4u;
inst->memory.resource_source = AddDescriptor(state, base, 4);
if (inst.memory.kind == ResourceKind::Buffer ||
(inst.memory.kind == ResourceKind::ScalarBuffer &&
inst.op == Opcode::SBufferLoadDword)) {
const auto base = inst.memory.resource * 4u;
inst.memory.resource_source = AddDescriptor(state, base, 4);
return;
}
if (inst->memory.kind == ResourceKind::Image ||
inst->memory.kind == ResourceKind::ImageUint ||
inst->memory.kind == ResourceKind::StorageImage ||
inst->memory.kind == ResourceKind::StorageImageUint) {
inst->memory.resource_source = AddDescriptor(state, inst->memory.resource * 4u, 8);
if (inst->op == Opcode::ImageSample || inst->op == Opcode::ImageGather4 ||
inst->op == Opcode::ImageGetLod) {
inst->memory.sampler_source = AddDescriptor(state, inst->memory.sampler * 4u, 4);
if (inst.memory.kind == ResourceKind::Image ||
inst.memory.kind == ResourceKind::ImageUint ||
inst.memory.kind == ResourceKind::StorageImage ||
inst.memory.kind == ResourceKind::StorageImageUint) {
inst.memory.resource_source = AddDescriptor(state, inst.memory.resource * 4u, 8);
if (inst.op == Opcode::ImageSample || inst.op == Opcode::ImageGather4 ||
inst.op == Opcode::ImageGetLod) {
inst.memory.sampler_source = AddDescriptor(state, inst.memory.sampler * 4u, 4);
}
}
}
void AttachScalarSources(Instruction* inst, const ScalarState& state) {
std::fill(std::begin(inst->scalar_sources), std::end(inst->scalar_sources),
void AttachScalarSources(Instruction& inst, const ScalarState& state) {
std::fill(std::begin(inst.scalar_sources), std::end(inst.scalar_sources),
ScalarProvenance::Undefined);
for (uint32_t i = 0; i < inst->src_count; i++) {
const auto& source = inst->src[i];
for (uint32_t i = 0; i < inst.src_count; i++) {
const auto& source = inst.src[i];
if (source.kind == OperandKind::Register && source.reg.file != RegisterFile::Vector &&
source.reg.file != RegisterFile::Exec) {
inst->scalar_sources[i] = OperandValue(source, state);
inst.scalar_sources[i] = OperandValue(source, state);
}
}
}
@@ -729,11 +726,11 @@ private:
auto& component_inst = block.instructions[index + component];
if (attach_sources) {
component_inst.scalar_value = ScalarProvenance::Undefined;
AttachScalarSources(&component_inst, before);
AttachSources(&component_inst, before);
AttachScalarSources(component_inst, before);
AttachSources(component_inst, before);
}
auto component_state = before;
WriteDestination(component_inst, &component_state);
WriteDestination(component_inst, component_state);
uint32_t dst = 0;
ScalarRegister(component_inst.dst, dst);
state.regs[dst] = component_state.regs[dst];
@@ -746,11 +743,11 @@ private:
}
if (attach_sources) {
inst.scalar_value = ScalarProvenance::Undefined;
AttachScalarSources(&inst, state);
AttachSources(&inst, state);
AttachScalarSources(inst, state);
AttachSources(inst, state);
}
WriteVectorDestination(inst, &state);
WriteDestination(inst, &state);
WriteVectorDestination(inst, state);
WriteDestination(inst, state);
if (attach_sources) {
uint32_t dst = 0;
if (ScalarRegister(inst.dst, dst)) {
@@ -765,7 +762,7 @@ private:
ScalarState MergeEntry(size_t block_index) {
ScalarState result;
result.Fill(ScalarProvenance::Undefined);
const auto& block = m_program->blocks[block_index];
const auto& block = m_program.blocks[block_index];
const auto merge = [&](uint32_t initial, auto predecessor_value, uint32_t* phi) {
std::vector<uint32_t> incoming;
if (initial != ScalarProvenance::Undefined) {
@@ -845,7 +842,7 @@ private:
}
}
Program* m_program;
Program& m_program;
ScalarProvenance& m_graph;
ScalarState m_user_data;
std::vector<ScalarState> m_entry;
@@ -860,40 +857,34 @@ private:
} // namespace
bool BuildScalarProvenance(Program* program, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error = "invalid scalar provenance program";
}
return false;
}
if (program->resource_tracking_complete) {
bool BuildScalarProvenance(Program& program, std::string* error) {
if (program.resource_tracking_complete) {
if (error != nullptr) {
*error = "cannot rebuild scalar provenance after resource tracking";
}
return false;
}
if (program->user_data_count > 64) {
if (program.user_data_count > 64) {
if (error != nullptr) {
*error = "scalar provenance user-data count exceeds 64 SGPRs";
}
return false;
}
if (program->user_data_base > ScalarRegisters ||
program->user_data_count > ScalarRegisters - program->user_data_base) {
if (program.user_data_base > ScalarRegisters ||
program.user_data_count > ScalarRegisters - program.user_data_base) {
if (error != nullptr) {
*error = "scalar provenance user-data window is out of range";
}
return false;
}
if (program->srt_patching_complete) {
if (program.srt_patching_complete) {
if (error != nullptr) {
*error = "cannot rebuild scalar provenance after SRT patching";
}
return false;
}
program->srt = {};
program->srt_plan_complete = false;
program.srt = {};
program.srt_plan_complete = false;
return Builder(program).Run(error);
}
@@ -911,7 +902,7 @@ bool DescriptorSourceResolved(const Program& program, uint32_t source) {
}
std::vector<uint8_t> visited(program.provenance.values.size());
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!ValueResolved(program.provenance, descriptor->dwords[i], &visited)) {
if (!ValueResolved(program.provenance, descriptor->dwords[i], visited)) {
return false;
}
}
src/graphics/shader/recompiler/ScalarProvenance.h
+1 -1
View File
@@ -6,7 +6,7 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
// Builds scalar reaching definitions and attaches descriptor sources to individual memory uses.
bool BuildScalarProvenance(Program* program, std::string* error);
bool BuildScalarProvenance(Program& program, std::string* error);
uint32_t ScalarValueArgCount(ScalarValueOp op);
src/graphics/shader/recompiler/ShaderCFG.cpp
+239 -269
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/ShaderCFG.h
+2 -2
View File
@@ -105,8 +105,8 @@ struct Graph {
uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
};
bool BuildGraph(const Decoder::Program& program, Graph* graph, std::string* error);
bool Structurize(Graph* graph, std::string* error);
bool BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error);
bool Structurize(Graph& graph, std::string* error);
std::string BranchConditionToString(BranchCondition condition);
std::string FailureKindToString(FailureKind kind);
std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage,
src/graphics/shader/recompiler/ShaderDecoder.cpp
+87 -101
View File
@@ -44,10 +44,10 @@ bool IsControlFlowBranch(Opcode opcode) {
}
}
void ApplyLiteral(Operand* operand, uint32_t literal) {
if (operand->kind == OperandKind::LiteralConstant) {
operand->value = literal;
operand->signed_val = static_cast<int32_t>(literal);
void ApplyLiteral(Operand& operand, uint32_t literal) {
if (operand.kind == OperandKind::LiteralConstant) {
operand.value = literal;
operand.signed_val = static_cast<int32_t>(literal);
}
}
@@ -196,36 +196,31 @@ std::string FormatExp(const Instruction& inst) {
} // namespace
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::string* error) {
if (operand == nullptr) {
SetError(error, "internal decoder error: null source operand");
return false;
}
*operand = {};
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error) {
operand = {};
if (code <= 105u) {
operand->kind = OperandKind::Sgpr;
operand->reg = code;
operand.kind = OperandKind::Sgpr;
operand.reg = code;
return true;
}
if (code >= 128u && code <= 192u) {
operand->kind = OperandKind::IntegerInlineConstant;
operand->signed_val = static_cast<int32_t>(code - 128u);
operand->value = static_cast<uint32_t>(operand->signed_val);
operand.kind = OperandKind::IntegerInlineConstant;
operand.signed_val = static_cast<int32_t>(code - 128u);
operand.value = static_cast<uint32_t>(operand.signed_val);
return true;
}
if (code >= 193u && code <= 208u) {
operand->kind = OperandKind::IntegerInlineConstant;
operand->signed_val = 192 - static_cast<int32_t>(code);
operand->value = static_cast<uint32_t>(operand->signed_val);
operand.kind = OperandKind::IntegerInlineConstant;
operand.signed_val = 192 - static_cast<int32_t>(code);
operand.value = static_cast<uint32_t>(operand.signed_val);
return true;
}
if (code >= 240u && code <= 247u) {
constexpr float values[] = {0.5f, -0.5f, 1.0f, -1.0f, 2.0f, -2.0f, 4.0f, -4.0f};
operand->kind = OperandKind::FloatInlineConstant;
operand->float_val = values[code - 240u];
operand->value = FloatBits(operand->float_val);
operand.kind = OperandKind::FloatInlineConstant;
operand.float_val = values[code - 240u];
operand.value = FloatBits(operand.float_val);
return true;
}
if (code >= 256u && code <= 511u) {
@@ -233,22 +228,22 @@ bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::strin
}
switch (code) {
case 106u: operand->kind = OperandKind::VccLo; return true;
case 107u: operand->kind = OperandKind::VccHi; return true;
case 124u: operand->kind = OperandKind::M0; return true;
case 125u: operand->kind = OperandKind::Null; return true;
case 126u: operand->kind = OperandKind::ExecLo; return true;
case 127u: operand->kind = OperandKind::ExecHi; return true;
case 239u: operand->kind = OperandKind::PopsExitingWaveId; return true;
case 106u: operand.kind = OperandKind::VccLo; return true;
case 107u: operand.kind = OperandKind::VccHi; return true;
case 124u: operand.kind = OperandKind::M0; return true;
case 125u: operand.kind = OperandKind::Null; return true;
case 126u: operand.kind = OperandKind::ExecLo; return true;
case 127u: operand.kind = OperandKind::ExecHi; return true;
case 239u: operand.kind = OperandKind::PopsExitingWaveId; return true;
case 248u:
operand->kind = OperandKind::FloatInlineConstant;
operand->float_val = 0.15915494309189535f;
operand->value = FloatBits(operand->float_val);
operand.kind = OperandKind::FloatInlineConstant;
operand.float_val = 0.15915494309189535f;
operand.value = FloatBits(operand.float_val);
return true;
case 251u: operand->kind = OperandKind::VccZ; return true;
case 252u: operand->kind = OperandKind::ExecZ; return true;
case 253u: operand->kind = OperandKind::Scc; return true;
case 255u: operand->kind = OperandKind::LiteralConstant; return true;
case 251u: operand.kind = OperandKind::VccZ; return true;
case 252u: operand.kind = OperandKind::ExecZ; return true;
case 253u: operand.kind = OperandKind::Scc; return true;
case 255u: operand.kind = OperandKind::LiteralConstant; return true;
default:
if (error != nullptr) {
*error = fmt::format("unsupported scalar source operand 0x{:08x} at pc 0x{:08x}",
@@ -258,27 +253,22 @@ bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::strin
}
}
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand* operand, std::string* error) {
if (operand == nullptr) {
SetError(error, "internal decoder error: null destination operand");
return false;
}
*operand = {};
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error) {
operand = {};
if (code <= 105u) {
operand->kind = OperandKind::Sgpr;
operand->reg = code;
operand.kind = OperandKind::Sgpr;
operand.reg = code;
return true;
}
switch (code) {
case 106u: operand->kind = OperandKind::VccLo; return true;
case 107u: operand->kind = OperandKind::VccHi; return true;
case 124u: operand->kind = OperandKind::M0; return true;
case 125u: operand->kind = OperandKind::Null; return true;
case 126u: operand->kind = OperandKind::ExecLo; return true;
case 127u: operand->kind = OperandKind::ExecHi; return true;
case 106u: operand.kind = OperandKind::VccLo; return true;
case 107u: operand.kind = OperandKind::VccHi; return true;
case 124u: operand.kind = OperandKind::M0; return true;
case 125u: operand.kind = OperandKind::Null; return true;
case 126u: operand.kind = OperandKind::ExecLo; return true;
case 127u: operand.kind = OperandKind::ExecHi; return true;
default:
if (error != nullptr) {
*error = fmt::format(
@@ -288,67 +278,63 @@ bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand* operand, std::
}
}
bool DecodeVectorGpr(uint32_t reg, Operand* operand, std::string* error) {
if (operand == nullptr) {
SetError(error, "internal decoder error: null VGPR operand");
return false;
}
bool DecodeVectorGpr(uint32_t reg, Operand& operand, std::string* error) {
if (reg > 255u) {
SetError(error, "VGPR index is out of range");
return false;
}
*operand = {};
operand->kind = OperandKind::Vgpr;
operand->reg = reg;
operand = {};
operand.kind = OperandKind::Vgpr;
operand.reg = reg;
return true;
}
bool ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (!HasLiteral(*inst)) {
if (!HasLiteral(inst)) {
return true;
}
if (word_index + inst->word_count >= code.size()) {
if (word_index + inst.word_count >= code.size()) {
if (error != nullptr) {
*error = fmt::format("missing literal constant at pc 0x{:08x}", inst->pc);
*error = fmt::format("missing literal constant at pc 0x{:08x}", inst.pc);
}
return false;
}
const auto literal = code[word_index + inst->word_count];
ApplyLiteral(&inst->src0, literal);
ApplyLiteral(&inst->src1, literal);
ApplyLiteral(&inst->src2, literal);
ApplyLiteral(&inst->src3, literal);
inst->word_count++;
SetRawWords(inst, code, word_index, inst->word_count);
const auto literal = code[word_index + inst.word_count];
ApplyLiteral(inst.src0, literal);
ApplyLiteral(inst.src1, literal);
ApplyLiteral(inst.src2, literal);
ApplyLiteral(inst.src3, literal);
inst.word_count++;
SetRawWords(inst, code, word_index, inst.word_count);
return true;
}
void SetRawWords(Instruction* inst, std::span<const uint32_t> code, uint32_t word_index,
void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index,
uint32_t word_count) {
inst->word_count = word_count;
inst->raw_count = std::min<uint32_t>(word_count, MaxInstructionRawWords);
for (uint32_t i = 0; i < inst->raw_count; i++) {
inst->raw[i] = code[word_index + i];
inst.word_count = word_count;
inst.raw_count = std::min<uint32_t>(word_count, MaxInstructionRawWords);
for (uint32_t i = 0; i < inst.raw_count; i++) {
inst.raw[i] = code[word_index + i];
}
}
void SetUnsupported(Instruction* inst, Family family, uint32_t opcode_id, const char* reason) {
inst->opcode = Opcode::Unsupported;
inst->family = family;
inst->opcode_id = opcode_id;
inst->unsupported_reason = reason;
void SetUnsupported(Instruction& inst, Family family, uint32_t opcode_id, const char* reason) {
inst.opcode = Opcode::Unsupported;
inst.family = family;
inst.opcode_id = opcode_id;
inst.unsupported_reason = reason;
}
bool DecodeProgram(std::span<const uint32_t> code, Program* program, std::string* error) {
if (code.empty() || code.size() > UINT32_MAX / sizeof(uint32_t) || program == nullptr) {
bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error) {
if (code.empty() || code.size() > UINT32_MAX / sizeof(uint32_t)) {
SetError(error, "invalid shader decoder input");
return false;
}
program->instructions.clear();
program->code = code;
program.instructions.clear();
program.code = code;
std::set<uint32_t> branch_targets;
for (uint32_t word_index = 0; word_index < code.size();) {
@@ -358,30 +344,30 @@ bool DecodeProgram(std::span<const uint32_t> code, Program* program, std::string
Instruction inst;
bool ok = false;
if ((word & 0x80000000u) == 0u) {
ok = DecodeVop2(pc, code, word_index, &inst, error);
ok = DecodeVop2(pc, code, word_index, inst, error);
} else if ((word & 0xc0000000u) == 0x80000000u) {
const auto opcode = (word >> 23u) & 0x7fu;
switch (opcode) {
case 0x7du: ok = DecodeSop1(pc, code, word_index, &inst, error); break;
case 0x7eu: ok = DecodeSopc(pc, code, word_index, &inst, error); break;
case 0x7fu: ok = DecodeSopp(pc, code, word_index, &inst, error); break;
case 0x7du: ok = DecodeSop1(pc, code, word_index, inst, error); break;
case 0x7eu: ok = DecodeSopc(pc, code, word_index, inst, error); break;
case 0x7fu: ok = DecodeSopp(pc, code, word_index, inst, error); break;
default:
ok = opcode >= 0x60u ? DecodeSopk(pc, code, word_index, &inst, error)
: DecodeSop2(pc, code, word_index, &inst, error);
ok = opcode >= 0x60u ? DecodeSopk(pc, code, word_index, inst, error)
: DecodeSop2(pc, code, word_index, inst, error);
break;
}
} else {
switch (word >> 26u) {
case 0x32u: ok = DecodeVintrp(pc, code, word_index, &inst, error); break;
case 0x33u: ok = DecodeVop3p(pc, code, word_index, &inst, error); break;
case 0x35u: ok = DecodeVop3(pc, code, word_index, &inst, error); break;
case 0x36u: ok = DecodeDs(pc, code, word_index, &inst, error); break;
case 0x37u: ok = DecodeFlat(pc, code, word_index, &inst, error); break;
case 0x38u: ok = DecodeMubuf(pc, code, word_index, &inst, error); break;
case 0x3au: ok = DecodeMtbuf(pc, code, word_index, &inst, error); break;
case 0x3cu: ok = DecodeMimg(pc, code, word_index, &inst, error); break;
case 0x3du: ok = DecodeSmem(pc, code, word_index, &inst, error); break;
case 0x3eu: ok = DecodeExp(pc, code, word_index, &inst, error); break;
case 0x32u: ok = DecodeVintrp(pc, code, word_index, inst, error); break;
case 0x33u: ok = DecodeVop3p(pc, code, word_index, inst, error); break;
case 0x35u: ok = DecodeVop3(pc, code, word_index, inst, error); break;
case 0x36u: ok = DecodeDs(pc, code, word_index, inst, error); break;
case 0x37u: ok = DecodeFlat(pc, code, word_index, inst, error); break;
case 0x38u: ok = DecodeMubuf(pc, code, word_index, inst, error); break;
case 0x3au: ok = DecodeMtbuf(pc, code, word_index, inst, error); break;
case 0x3cu: ok = DecodeMimg(pc, code, word_index, inst, error); break;
case 0x3du: ok = DecodeSmem(pc, code, word_index, inst, error); break;
case 0x3eu: ok = DecodeExp(pc, code, word_index, inst, error); break;
default:
if (error != nullptr) {
*error = fmt::format(
@@ -396,7 +382,7 @@ bool DecodeProgram(std::span<const uint32_t> code, Program* program, std::string
return false;
}
program->instructions.push_back(inst);
program.instructions.push_back(inst);
word_index += inst.word_count;
if (IsControlFlowBranch(inst.opcode)) {
src/graphics/shader/recompiler/ShaderDecoder.h
+7 -7
View File
@@ -656,16 +656,16 @@ struct Program {
std::vector<Instruction> instructions;
};
bool DecodeProgram(std::span<const uint32_t> code, Program* program, std::string* error);
bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error);
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::string* error);
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand* operand, std::string* error);
bool DecodeVectorGpr(uint32_t reg, Operand* operand, std::string* error);
bool ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error);
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error);
bool DecodeVectorGpr(uint32_t reg, Operand& operand, std::string* error);
bool ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
void SetRawWords(Instruction* inst, std::span<const uint32_t> code, uint32_t word_index,
void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index,
uint32_t word_count);
void SetUnsupported(Instruction* inst, Family family, uint32_t opcode_id, const char* reason);
void SetUnsupported(Instruction& inst, Family family, uint32_t opcode_id, const char* reason);
std::string FamilyToString(Family family);
std::string OpcodeToString(Opcode opcode);
std::string OperandToString(const Operand& operand);
src/graphics/shader/recompiler/ShaderIR.h
+1 -1
View File
@@ -753,7 +753,7 @@ struct Program {
};
bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage,
uint32_t wave_size, Program* program, std::string* error);
uint32_t wave_size, Program& program, std::string* error);
std::string RegisterToString(Register reg);
std::string OperandToString(const Operand& operand);
src/graphics/shader/recompiler/ShaderInfoCollection.cpp
+23 -25
View File
@@ -12,10 +12,10 @@ bool HasInput(const ShaderInfo& info, StageInputKind kind, uint32_t location) {
});
}
void AddInput(ShaderInfo* info, StageInputKind kind, uint32_t location, uint32_t components,
void AddInput(ShaderInfo& info, StageInputKind kind, uint32_t location, uint32_t components,
std::string name) {
if (!HasInput(*info, kind, location)) {
info->inputs.push_back({kind, location, components, std::move(name)});
if (!HasInput(info, kind, location)) {
info.inputs.push_back({kind, location, components, std::move(name)});
}
}
@@ -25,10 +25,10 @@ bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) {
});
}
void AddOutput(ShaderInfo* info, StageOutputKind kind, uint32_t index, uint32_t location,
void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t location,
std::string name) {
if (!HasOutput(*info, kind, index)) {
info->outputs.push_back({kind, index, location, std::move(name)});
if (!HasOutput(info, kind, index)) {
info.outputs.push_back({kind, index, location, std::move(name)});
}
}
@@ -87,7 +87,7 @@ bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, s
}
void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex,
ShaderInfo* info) {
ShaderInfo& info) {
AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex");
AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex");
if (vertex == nullptr) {
@@ -114,7 +114,7 @@ void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* ve
}
}
void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo* info) {
void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
if (pixel == nullptr) {
return;
}
@@ -130,7 +130,7 @@ void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo* info) {
}
void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo* compute,
ShaderInfo* info) {
ShaderInfo& info) {
if (compute != nullptr) {
if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) {
AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID");
@@ -150,7 +150,7 @@ void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo*
}
}
void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo* info) {
void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != Opcode::Export) {
@@ -193,32 +193,30 @@ void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, S
} // namespace
bool CollectShaderInfo(Program* program, const ShaderInfoOptions& options, std::string* error) {
if (program == nullptr || !program->resource_tracking_complete ||
program->shader_info_complete) {
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error) {
if (!program.resource_tracking_complete || program.shader_info_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid shader info program"
: !program->resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected";
*error = !program.resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected";
}
return false;
}
if (!ValidateOptions(*program, options, error)) {
if (!ValidateOptions(program, options, error)) {
return false;
}
auto next = program->info;
auto next = program.info;
next.inputs.clear();
next.outputs.clear();
switch (program->stage) {
case ShaderType::Vertex: CollectVertexInputs(*program, options.vertex, &next); break;
case ShaderType::Pixel: CollectPixelInputs(options.pixel, &next); break;
case ShaderType::Compute: CollectComputeInputs(*program, options.compute, &next); break;
switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break;
case ShaderType::Pixel: CollectPixelInputs(options.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(program, options.compute, next); break;
default: return false;
}
CollectOutputs(*program, options.pixel, &next);
program->info = std::move(next);
program->shader_info_complete = true;
CollectOutputs(program, options.pixel, next);
program.info = std::move(next);
program.shader_info_complete = true;
return true;
}
src/graphics/shader/recompiler/ShaderInfoCollection.h
+1 -1
View File
@@ -13,7 +13,7 @@ struct ShaderInfoOptions {
// Completes the immutable shader interface after resource tracking. On failure Program::info and
// all completion state remain unchanged.
bool CollectShaderInfo(Program* program, const ShaderInfoOptions& options, std::string* error);
bool CollectShaderInfo(Program& program, const ShaderInfoOptions& options, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ShaderRecompiler.cpp
+30 -31
View File
@@ -289,12 +289,11 @@ int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) {
return static_cast<int>((raw_offset + static_cast<uint32_t>(dword) * 4u) / 16u);
}
void AppendUniquePcs(std::vector<uint32_t>* dst, const std::vector<uint32_t>& src) {
EXIT_IF(dst == nullptr);
dst->reserve(dst->size() + src.size());
void AppendUniquePcs(std::vector<uint32_t>& dst, const std::vector<uint32_t>& src) {
dst.reserve(dst.size() + src.size());
for (auto pc: src) {
if (std::find(dst->begin(), dst->end(), pc) == dst->end()) {
dst->push_back(pc);
if (std::find(dst.begin(), dst.end(), pc) == dst.end()) {
dst.push_back(pc);
}
}
}
@@ -578,21 +577,21 @@ int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
return -1;
}
uint32_t RewriteEmbeddedVertexFetches(IR::Program* ir, const ShaderVertexInputInfo* input_info,
uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputInfo* input_info,
const std::vector<EmbeddedFetchLoad>& loads) {
if (ir == nullptr || input_info == nullptr || loads.empty()) {
if (input_info == nullptr || loads.empty()) {
return 0;
}
std::vector<uint32_t> prolog_pcs;
prolog_pcs.reserve(loads.size());
for (const auto& load: loads) {
AppendUniquePcs(&prolog_pcs, load.prolog_loads);
AppendUniquePcs(prolog_pcs, load.prolog_loads);
}
auto* mutable_input_info = const_cast<ShaderVertexInputInfo*>(input_info);
uint32_t rewritten = 0;
for (auto& block: ir->blocks) {
for (auto& block: ir.blocks) {
for (auto& inst: block.instructions) {
if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) {
auto pc = inst.pc;
@@ -637,8 +636,8 @@ uint32_t RewriteEmbeddedVertexFetches(IR::Program* ir, const ShaderVertexInputIn
} // namespace
bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
CompileResult* result, std::string* error) {
if (code.empty() || result == nullptr) {
CompileResult& result, std::string* error) {
if (code.empty()) {
if (error != nullptr) {
*error = "invalid shader recompiler input";
}
@@ -664,7 +663,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
static_cast<uint64_t>(code.size()));
Decoder::Program decoded;
if (!Decoder::DecodeProgram(code, &decoded, error)) {
if (!Decoder::DecodeProgram(code, decoded, error)) {
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64
@@ -683,7 +682,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
CFG::Graph cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash);
if (!CFG::BuildGraph(decoded, &cfg, error)) {
if (!CFG::BuildGraph(decoded, cfg, error)) {
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
@@ -703,7 +702,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
const auto unstructured_cfg = cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!CFG::Structurize(&cfg, &structure_error)) {
if (!CFG::Structurize(cfg, &structure_error)) {
const auto diagnostic = FormatCfgFailure(cfg, options, structure_error);
LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str());
dispatcher_fallback = true;
@@ -729,7 +728,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
IR::Program ir;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash);
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, &ir, error)) {
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, ir, error)) {
return false;
}
ir.lane_mask_mode = options.lane_mask_mode;
@@ -747,18 +746,18 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
DetectEmbeddedVertexFetch(decoded, options.vertex_input_info, ir.user_data_base,
ir.user_data_count, options.wave_size);
auto rewritten =
RewriteEmbeddedVertexFetches(&ir, options.vertex_input_info, embedded_fetch.loads);
RewriteEmbeddedVertexFetches(ir, options.vertex_input_info, embedded_fetch.loads);
if (rewritten > 0 || !embedded_fetch.loads.empty()) {
LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n",
GetDumpLabel(options), static_cast<uint64_t>(embedded_fetch.loads.size()),
rewritten);
}
}
if (!IR::BuildScalarProvenance(&ir, error)) {
if (!IR::BuildScalarProvenance(ir, error)) {
return false;
}
std::string srt_error;
if (!IR::BuildSrtPlan(&ir, &srt_error)) {
if (!IR::BuildSrtPlan(ir, &srt_error)) {
LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str());
if (error != nullptr) {
*error = std::move(srt_error);
@@ -770,7 +769,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
ir.fallback_reason = dispatcher_reason;
}
if (!IR::PatchSrtReads(&ir, error) || !IR::TrackResources(&ir, error)) {
if (!IR::PatchSrtReads(ir, error) || !IR::TrackResources(ir, error)) {
return false;
}
if (options.stage == ShaderType::Vertex) {
@@ -810,11 +809,11 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
: nullptr;
runtime.userdata = options.read_memory_data;
runtime.flat_memory_base = options.flat_memory_base;
if (!IR::MaterializeResources(ir, runtime, &resources, error)) {
if (!IR::MaterializeResources(ir, runtime, resources, error)) {
return false;
}
}
if (!IR::SpecializeResources(&ir, resources, error)) {
if (!IR::SpecializeResources(ir, resources, error)) {
return false;
}
@@ -828,13 +827,13 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel;
info_options.compute =
options.compute_input_info != nullptr ? options.compute_input_info : &default_compute;
if (!IR::CollectShaderInfo(&ir, info_options, error)) {
if (!IR::CollectShaderInfo(ir, info_options, error)) {
return false;
}
IR::BindingLayoutOptions layout_options;
layout_options.descriptor_set = options.descriptor_set;
layout_options.push_constant_offset = options.push_constant_offset;
if (!IR::AllocateBindings(&ir, layout_options, error)) {
if (!IR::AllocateBindings(ir, layout_options, error)) {
return false;
}
std::string ir_dump;
@@ -850,7 +849,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info,
options.compute_input_info, &spirv, &emit_error)) {
options.compute_input_info, spirv, &emit_error)) {
if (dispatcher_fallback && error != nullptr) {
*error = fmt::format("dispatcher fallback failed after {}: {}",
dispatcher_reason.c_str(), emit_error.c_str());
@@ -868,15 +867,15 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str());
}
result->spirv = std::move(spirv);
result->program = std::move(ir);
result->resources = std::move(resources);
result.spirv = std::move(spirv);
result.program = std::move(ir);
result.resources = std::move(resources);
if (options.dump_ir) {
result->decoded_dump = std::move(decoded_dump);
result->ir_dump = std::move(ir_dump);
result.decoded_dump = std::move(decoded_dump);
result.ir_dump = std::move(ir_dump);
} else {
result->decoded_dump.clear();
result->ir_dump.clear();
result.decoded_dump.clear();
result.ir_dump.clear();
}
return true;
}
src/graphics/shader/recompiler/ShaderRecompiler.h
+1 -1
View File
@@ -44,7 +44,7 @@ struct CompileResult {
};
bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
CompileResult* result, std::string* error);
CompileResult& result, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler
src/graphics/shader/recompiler/SpirvBuilder.cpp
+26 -26
View File
@@ -8,15 +8,15 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv {
static constexpr size_t InitialSpirvSectionReserve = 4096;
static constexpr size_t InitialSpirvFunctionSectionReserve = 450000;
static void AppendInstructionWords(std::vector<uint32_t>* section, const uint32_t* words,
static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_t* words,
size_t words_num) {
if (words_num == 0) {
return;
}
const auto opcode = words[0];
const auto word_count = static_cast<uint32_t>(words_num);
section->push_back((word_count << 16u) | opcode);
section->insert(section->end(), words + 1, words + words_num);
section.push_back((word_count << 16u) | opcode);
section.insert(section.end(), words + 1, words + words_num);
}
Builder::Builder() {
@@ -30,7 +30,7 @@ uint32_t Builder::AllocateId() {
return m_next_id++;
}
void Builder::AppendString(std::vector<uint32_t>* words, const char* text) {
void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
const auto len = text != nullptr ? std::strlen(text) : 0;
const auto word_count = (len + 1u + 3u) / 4u;
for (size_t i = 0; i < word_count; i++) {
@@ -42,76 +42,76 @@ void Builder::AppendString(std::vector<uint32_t>* words, const char* text) {
<< (byte * 8u);
}
}
words->push_back(word);
words.push_back(word);
}
}
void Builder::AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
const std::vector<uint32_t>& operands) {
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
section->push_back((word_count << 16u) | opcode);
section->insert(section->end(), operands.begin(), operands.end());
section.push_back((word_count << 16u) | opcode);
section.insert(section.end(), operands.begin(), operands.end());
}
void Builder::AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
std::initializer_list<uint32_t> operands) {
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
section->push_back((word_count << 16u) | opcode);
section->insert(section->end(), operands.begin(), operands.end());
section.push_back((word_count << 16u) | opcode);
section.insert(section.end(), operands.begin(), operands.end());
}
void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
AppendInstruction(&m_capabilities, 17u, operands);
AppendInstruction(m_capabilities, 17u, operands);
}
void Builder::AddExtension(const char* name) {
std::vector<uint32_t> operands;
AppendString(&operands, name);
AppendInstruction(&m_extensions, 10u, operands);
AppendString(operands, name);
AppendInstruction(m_extensions, 10u, operands);
}
void Builder::AddExtInstImport(uint32_t id, const char* name) {
std::vector<uint32_t> operands = {id};
AppendString(&operands, name);
AppendInstruction(&m_ext_inst_imports, 11u, operands);
AppendString(operands, name);
AppendInstruction(m_ext_inst_imports, 11u, operands);
}
void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
AppendInstruction(&m_memory_model, 14u, operands);
AppendInstruction(m_memory_model, 14u, operands);
}
void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
const std::vector<uint32_t>& interfaces) {
std::vector<uint32_t> operands = {execution_model, entry_point};
AppendString(&operands, name);
AppendString(operands, name);
operands.insert(operands.end(), interfaces.begin(), interfaces.end());
AppendInstruction(&m_entry_points, 15u, operands);
AppendInstruction(m_entry_points, 15u, operands);
}
void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
AppendInstruction(&m_execution_modes, 16u, operands);
AppendInstruction(m_execution_modes, 16u, operands);
}
void Builder::AddName(uint32_t target, const char* name) {
std::vector<uint32_t> operands = {target};
AppendString(&operands, name);
AppendInstruction(&m_debug, 5u, operands);
AppendString(operands, name);
AppendInstruction(m_debug, 5u, operands);
}
void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
AppendInstructionWords(&m_annotations, words.begin(), words.size());
AppendInstructionWords(m_annotations, words.begin(), words.size());
}
void Builder::AddType(std::initializer_list<uint32_t> words) {
AppendInstructionWords(&m_types, words.begin(), words.size());
AppendInstructionWords(m_types, words.begin(), words.size());
}
void Builder::AddFunction(std::initializer_list<uint32_t> words) {
AppendInstructionWords(&m_functions, words.begin(), words.size());
AppendInstructionWords(m_functions, words.begin(), words.size());
}
void Builder::AddFunction(const std::vector<uint32_t>& words) {
AppendInstructionWords(&m_functions, words.data(), words.size());
AppendInstructionWords(m_functions, words.data(), words.size());
}
std::vector<uint32_t> Builder::Build() const {
src/graphics/shader/recompiler/SpirvBuilder.h
+3 -3
View File
@@ -32,11 +32,11 @@ public:
[[nodiscard]] std::vector<uint32_t> Build() const;
private:
static void AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
const std::vector<uint32_t>& operands);
static void AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
std::initializer_list<uint32_t> operands);
static void AppendString(std::vector<uint32_t>* words, const char* text);
static void AppendString(std::vector<uint32_t>& words, const char* text);
uint32_t m_next_id = 1;
std::vector<uint32_t> m_capabilities;
src/graphics/shader/recompiler/SpirvEmitter.cpp
+18 -24
View File
@@ -339,7 +339,7 @@ static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruc
return false;
}
uint32_t count = 0;
return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], &count) &&
return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], count) &&
(count & 63u) == 0u;
}
@@ -408,16 +408,12 @@ bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
}
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>* spirv,
std::string* error) {
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
std::string* error) {
using namespace Emitter;
if (spirv == nullptr) {
SetError(error, "invalid SPIR-V output");
return false;
}
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) {
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
@@ -438,9 +434,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
return false;
}
EmitterState state;
state.program = &program;
state.resources = &resources;
EmitterState state(program, resources);
state.vertex_input_info = vertex_input_info;
state.pixel_input_info = pixel_input_info;
state.compute_input_info = compute_input_info;
@@ -454,8 +448,8 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
state.outputs.reserve(InitialEmitterVectorReserve);
state.interface_variables.reserve(InitialEmitterVectorReserve);
state.reachable_blocks.reserve(InitialEmitterVectorReserve);
CollectRegisters(program, &state.registers);
CopyProgramInputsAndOutputs(&state, program);
CollectRegisters(program, state.registers);
CopyProgramInputsAndOutputs(state, program);
state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program);
state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
@@ -463,15 +457,15 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program);
state.needs_function_lds = ProgramNeedsFunctionLds(program);
state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program);
ComputeReachableBlocks(&state, program);
AllocateInputVariables(&state);
AllocateOutputVariables(&state);
AllocateDescriptorVariables(&state);
EmitHeaderAndTypes(&state);
AllocateRegisterVariables(&state);
AllocateBlockLabels(&state, program);
AllocateDispatcherState(&state, program);
EmitFunction(&state, program);
ComputeReachableBlocks(state, program);
AllocateInputVariables(state);
AllocateOutputVariables(state);
AllocateDescriptorVariables(state);
EmitHeaderAndTypes(state);
AllocateRegisterVariables(state);
AllocateBlockLabels(state, program);
AllocateDispatcherState(state, program);
EmitFunction(state, program);
ReportReserveExceeded(program, "registers", state.registers.size());
ReportReserveExceeded(program, "inputs", state.inputs.size());
@@ -484,7 +478,7 @@ bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resourc
SetError(error, "SPIR-V builder returned an empty module");
return false;
}
*spirv = std::move(binary);
spirv = std::move(binary);
return true;
}
src/graphics/shader/recompiler/SpirvEmitter.h
+3 -3
View File
@@ -12,9 +12,9 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv {
bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>* spirv,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/SrtPatcher.cpp
+31 -30
View File
@@ -1,19 +1,19 @@
#include "graphics/shader/recompiler/SrtPatcher.h"
#include <fmt/format.h>
#include <functional>
#include <map>
#include <set>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
bool PatchSrtReads(Program* program, std::string* error) {
if (program == nullptr || !program->srt_plan_complete || program->srt_patching_complete ||
program->resource_tracking_complete) {
bool PatchSrtReads(Program& program, std::string* error) {
if (!program.srt_plan_complete || program.srt_patching_complete ||
program.resource_tracking_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid SRT patch program"
: !program->srt_plan_complete ? "SRT plan is not ready"
: program->srt_patching_complete ? "SRT reads already patched"
*error = !program.srt_plan_complete ? "SRT plan is not ready"
: program.srt_patching_complete ? "SRT reads already patched"
: "resources already tracked";
}
return false;
@@ -21,11 +21,11 @@ bool PatchSrtReads(Program* program, std::string* error) {
std::map<uint32_t, uint32_t> offsets;
std::map<uint32_t, uint32_t> values;
for (const auto& read: program->srt.reads) {
if (read.value >= program->provenance.values.size() ||
read.flat_offset >= program->srt.reads.size() ||
(program->provenance.values[read.value].op != ScalarValueOp::ReadConst &&
program->provenance.values[read.value].op != ScalarValueOp::ReadConstBuffer)) {
for (const auto& read: program.srt.reads) {
if (read.value >= program.provenance.values.size() ||
read.flat_offset >= program.srt.reads.size() ||
(program.provenance.values[read.value].op != ScalarValueOp::ReadConst &&
program.provenance.values[read.value].op != ScalarValueOp::ReadConstBuffer)) {
if (error != nullptr) {
*error = fmt::format("invalid SRT read value={} flat_offset={}", read.value,
read.flat_offset);
@@ -47,7 +47,7 @@ bool PatchSrtReads(Program* program, std::string* error) {
return false;
}
}
if (offsets.size() != program->srt.reads.size() || values.size() != program->srt.reads.size()) {
if (offsets.size() != program.srt.reads.size() || values.size() != program.srt.reads.size()) {
if (error != nullptr) {
*error = "SRT reads do not form a dense value-to-offset bijection";
}
@@ -55,10 +55,10 @@ bool PatchSrtReads(Program* program, std::string* error) {
}
std::set<uint32_t> dynamic_reads;
for (const auto value: program->srt.dynamic_reads) {
if (value >= program->provenance.values.size() || offsets.contains(value) ||
(program->provenance.values[value].op != ScalarValueOp::ReadConst &&
program->provenance.values[value].op != ScalarValueOp::ReadConstBuffer) ||
for (const auto value: program.srt.dynamic_reads) {
if (value >= program.provenance.values.size() || offsets.contains(value) ||
(program.provenance.values[value].op != ScalarValueOp::ReadConst &&
program.provenance.values[value].op != ScalarValueOp::ReadConstBuffer) ||
!dynamic_reads.insert(value).second) {
if (error != nullptr) {
*error = fmt::format("invalid dynamic SRT read value={}", value);
@@ -67,10 +67,10 @@ bool PatchSrtReads(Program* program, std::string* error) {
}
}
std::set<uint32_t> dynamic_sources;
for (const auto source: program->srt.dynamic_sources) {
for (const auto source: program.srt.dynamic_sources) {
if (source == ScalarProvenance::Undefined ||
(source > ScalarProvenance::Unknown &&
source - 2u >= program->provenance.descriptors.size()) ||
source - 2u >= program.provenance.descriptors.size()) ||
!dynamic_sources.insert(source).second) {
if (error != nullptr) {
*error = fmt::format("invalid dynamic descriptor source={}", source);
@@ -80,21 +80,21 @@ bool PatchSrtReads(Program* program, std::string* error) {
}
struct Patch {
Instruction* inst = nullptr;
uint32_t flat_offset = 0;
std::reference_wrapper<Instruction> inst;
uint32_t flat_offset = 0;
};
std::vector<Patch> patches;
std::map<uint32_t, bool> matched;
for (const auto& [value, offset]: offsets) {
matched.emplace(value, false);
}
for (auto& block: program->blocks) {
for (auto& block: program.blocks) {
for (auto& inst: block.instructions) {
if (inst.op != Opcode::SLoadDword && inst.op != Opcode::SBufferLoadDword) {
continue;
}
if (const auto found = offsets.find(inst.scalar_value); found != offsets.end()) {
const auto value_op = program->provenance.values[found->first].op;
const auto value_op = program.provenance.values[found->first].op;
if ((inst.op == Opcode::SLoadDword) != (value_op == ScalarValueOp::ReadConst)) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has the wrong scalar-load producer",
@@ -102,7 +102,7 @@ bool PatchSrtReads(Program* program, std::string* error) {
}
return false;
}
patches.push_back({&inst, found->second});
patches.push_back({std::ref(inst), found->second});
matched[found->first] = true;
}
}
@@ -117,16 +117,17 @@ bool PatchSrtReads(Program* program, std::string* error) {
}
for (const auto& patch: patches) {
patch.inst->op = Opcode::LoadSrtDword;
patch.inst->src_count = 1;
for (auto& src: patch.inst->src) {
auto& inst = patch.inst.get();
inst.op = Opcode::LoadSrtDword;
inst.src_count = 1;
for (auto& src: inst.src) {
src = {};
}
patch.inst->src[0].kind = OperandKind::ImmediateU32;
patch.inst->src[0].imm = patch.flat_offset;
patch.inst->memory = {};
inst.src[0].kind = OperandKind::ImmediateU32;
inst.src[0].imm = patch.flat_offset;
inst.memory = {};
}
program->srt_patching_complete = true;
program.srt_patching_complete = true;
return true;
}
src/graphics/shader/recompiler/SrtPatcher.h
+1 -1
View File
@@ -7,7 +7,7 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
// Replaces each immediate ReadConst producer selected by the SRT plan with a dense flat-buffer
// dword load. Dynamic-offset reads remain explicit. On failure no instruction is changed.
bool PatchSrtReads(Program* program, std::string* error);
bool PatchSrtReads(Program& program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/SrtWalker.cpp
+96 -128
View File
@@ -3,6 +3,7 @@
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm>
#include <array>
#include <cstring>
#include <fmt/format.h>
@@ -26,8 +27,8 @@ std::string Diagnostic(const Program& program, uint32_t pc, const std::string& m
StageName(program.stage), pc, message);
}
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t* result) {
if (result == nullptr || base > AddressMask) {
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
if (base > AddressMask) {
return false;
}
if (offset < 0) {
@@ -35,69 +36,66 @@ bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t* result) {
if (magnitude > base) {
return false;
}
*result = base - magnitude;
result = base - magnitude;
return true;
}
const auto magnitude = static_cast<uint64_t>(offset);
if (magnitude > AddressMask - base) {
return false;
}
*result = base + magnitude;
result = base + magnitude;
return true;
}
bool ApplyOperation(ScalarValueOp op, const uint32_t args[3], uint32_t* result) {
if (result == nullptr) {
return false;
}
bool ApplyOperation(ScalarValueOp op, const std::array<uint32_t, 3>& args, uint32_t& result) {
const auto shift = args[1] & 31u;
switch (op) {
case ScalarValueOp::Add: *result = args[0] + args[1]; break;
case ScalarValueOp::AddCarry: *result = args[0] + args[1] + (args[2] & 1u); break;
case ScalarValueOp::Add: result = args[0] + args[1]; break;
case ScalarValueOp::AddCarry: result = args[0] + args[1] + (args[2] & 1u); break;
case ScalarValueOp::Carry:
*result = static_cast<uint32_t>(
result = static_cast<uint32_t>(
(static_cast<uint64_t>(args[0]) + args[1] + (args[2] & 1u)) >> 32u);
break;
case ScalarValueOp::Sub: *result = args[0] - args[1]; break;
case ScalarValueOp::SubBorrow: *result = args[0] - args[1] - (args[2] & 1u); break;
case ScalarValueOp::Sub: result = args[0] - args[1]; break;
case ScalarValueOp::SubBorrow: result = args[0] - args[1] - (args[2] & 1u); break;
case ScalarValueOp::Borrow:
*result = static_cast<uint64_t>(args[1]) + (args[2] & 1u) > args[0] ? 1u : 0u;
result = static_cast<uint64_t>(args[1]) + (args[2] & 1u) > args[0] ? 1u : 0u;
break;
case ScalarValueOp::Mul: *result = args[0] * args[1]; break;
case ScalarValueOp::And: *result = args[0] & args[1]; break;
case ScalarValueOp::AndNot: *result = args[0] & ~args[1]; break;
case ScalarValueOp::Or: *result = args[0] | args[1]; break;
case ScalarValueOp::OrNot: *result = args[0] | ~args[1]; break;
case ScalarValueOp::Xor: *result = args[0] ^ args[1]; break;
case ScalarValueOp::Not: *result = ~args[0]; break;
case ScalarValueOp::ShiftLeft: *result = args[0] << shift; break;
case ScalarValueOp::ShiftRight: *result = args[0] >> shift; break;
case ScalarValueOp::Mul: result = args[0] * args[1]; break;
case ScalarValueOp::And: result = args[0] & args[1]; break;
case ScalarValueOp::AndNot: result = args[0] & ~args[1]; break;
case ScalarValueOp::Or: result = args[0] | args[1]; break;
case ScalarValueOp::OrNot: result = args[0] | ~args[1]; break;
case ScalarValueOp::Xor: result = args[0] ^ args[1]; break;
case ScalarValueOp::Not: result = ~args[0]; break;
case ScalarValueOp::ShiftLeft: result = args[0] << shift; break;
case ScalarValueOp::ShiftRight: result = args[0] >> shift; break;
case ScalarValueOp::ShiftRightArithmetic:
*result = static_cast<uint32_t>(static_cast<int32_t>(args[0]) >> shift);
result = static_cast<uint32_t>(static_cast<int32_t>(args[0]) >> shift);
break;
case ScalarValueOp::BitFieldMaskU32: {
const auto count = args[0] & 31u;
const auto offset = args[1] & 31u;
*result = ((uint32_t {1} << count) - 1u) << offset;
result = ((uint32_t {1} << count) - 1u) << offset;
break;
}
case ScalarValueOp::BitFieldMaskU64Low:
case ScalarValueOp::BitFieldMaskU64High: {
const auto count = args[0] & 63u;
const auto value = ((uint64_t {1} << count) - 1u) << (args[1] & 63u);
*result = op == ScalarValueOp::BitFieldMaskU64Low ? static_cast<uint32_t>(value)
: static_cast<uint32_t>(value >> 32u);
result = op == ScalarValueOp::BitFieldMaskU64Low ? static_cast<uint32_t>(value)
: static_cast<uint32_t>(value >> 32u);
break;
}
case ScalarValueOp::Add3: *result = args[0] + args[1] + args[2]; break;
case ScalarValueOp::ShiftLeftAdd: *result = (args[0] << shift) + args[2]; break;
case ScalarValueOp::Add3: result = args[0] + args[1] + args[2]; break;
case ScalarValueOp::ShiftLeftAdd: result = (args[0] << shift) + args[2]; break;
case ScalarValueOp::ShiftLeftAddCarry:
*result =
result =
static_cast<uint32_t>(((static_cast<uint64_t>(args[0]) << shift) + args[2]) >> 32u);
break;
case ScalarValueOp::AddShiftLeft: *result = (args[0] + args[1]) << (args[2] & 31u); break;
case ScalarValueOp::XorAdd: *result = (args[0] ^ args[1]) + args[2]; break;
case ScalarValueOp::ShiftLeftOr: *result = (args[0] << shift) | args[2]; break;
case ScalarValueOp::AddShiftLeft: result = (args[0] + args[1]) << (args[2] & 31u); break;
case ScalarValueOp::XorAdd: result = (args[0] ^ args[1]) + args[2]; break;
case ScalarValueOp::ShiftLeftOr: result = (args[0] << shift) | args[2]; break;
default: return false;
}
return true;
@@ -138,13 +136,13 @@ public:
if (count == 0 || count > 3 || value.op == ScalarValueOp::ReadConst) {
return Dynamic(id);
}
uint32_t args[3] = {};
std::array<uint32_t, 3> args {};
for (uint32_t i = 0; i < count; i++) {
if (!Fold(value.args[i], args[i])) {
return Dynamic(id);
}
}
if (!ApplyOperation(value.op, args, &out)) {
if (!ApplyOperation(value.op, args, out)) {
return Dynamic(id);
}
break;
@@ -169,13 +167,13 @@ private:
class PlanBuilder {
public:
explicit PlanBuilder(Program* program): m_program(program), m_folder(program->provenance) {
m_state.resize(program->provenance.values.size());
explicit PlanBuilder(Program& program): m_program(program), m_folder(program.provenance) {
m_state.resize(program.provenance.values.size());
}
bool Run(std::string* error) {
m_program->srt = {};
for (const auto& block: m_program->blocks) {
m_program.srt = {};
for (const auto& block: m_program.blocks) {
for (const auto& inst: block.instructions) {
if (!CollectDescriptor(inst.memory.resource_source, inst.pc, error) ||
!CollectDescriptor(inst.memory.sampler_source, inst.pc, error)) {
@@ -183,11 +181,11 @@ public:
}
}
}
for (const auto& block: m_program->blocks) {
for (const auto& block: m_program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::SLoadDword && inst.scalar_value < m_state.size() &&
m_state[inst.scalar_value] == 0) {
const auto& value = m_program->provenance.values[inst.scalar_value];
const auto& value = m_program.provenance.values[inst.scalar_value];
uint32_t ignored = 0;
if (value.op == ScalarValueOp::ReadConst &&
m_folder.Fold(value.args[2], ignored) &&
@@ -203,20 +201,20 @@ public:
private:
bool Fail(uint32_t pc, std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(*m_program, pc, message);
*error = Diagnostic(m_program, pc, message);
}
return false;
}
bool CollectDescriptor(uint32_t source, uint32_t use_pc, std::string* error) {
const auto* descriptor = GetDescriptorSource(*m_program, source);
const auto* descriptor = GetDescriptorSource(m_program, source);
if (descriptor == nullptr) {
if (source <= ScalarProvenance::Unknown) {
return source == ScalarProvenance::Undefined || AddDynamicSource(source);
}
return Fail(use_pc, error, fmt::format("invalid descriptor source {}", source));
}
if (!DescriptorSourceResolved(*m_program, source) ||
if (!DescriptorSourceResolved(m_program, source) ||
DescriptorNeedsControlFlow(*descriptor)) {
MarkDescriptor(*descriptor);
return AddDynamicSource(source);
@@ -240,7 +238,7 @@ private:
return;
}
m_state[id] = 2;
const auto& value = m_program->provenance.values[id];
const auto& value = m_program.provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
MarkValue(arg);
@@ -253,21 +251,21 @@ private:
}
bool DescriptorNeedsControlFlow(const DescriptorValue& descriptor) {
std::vector<uint8_t> visited(m_program->provenance.values.size());
std::vector<uint8_t> visited(m_program.provenance.values.size());
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (ValueNeedsControlFlow(descriptor.dwords[i], &visited)) {
if (ValueNeedsControlFlow(descriptor.dwords[i], visited)) {
return true;
}
}
return false;
}
bool ValueNeedsControlFlow(uint32_t id, std::vector<uint8_t>* visited) {
if (id >= m_program->provenance.values.size() || (*visited)[id] != 0) {
bool ValueNeedsControlFlow(uint32_t id, std::vector<uint8_t>& visited) {
if (id >= m_program.provenance.values.size() || visited[id] != 0) {
return false;
}
(*visited)[id] = 1;
const auto& value = m_program->provenance.values[id];
visited[id] = 1;
const auto& value = m_program.provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
uint32_t ignored = 0;
return !m_folder.Fold(id, ignored);
@@ -281,15 +279,15 @@ private:
}
bool AddDynamicSource(uint32_t source) {
if (std::find(m_program->srt.dynamic_sources.begin(), m_program->srt.dynamic_sources.end(),
source) == m_program->srt.dynamic_sources.end()) {
m_program->srt.dynamic_sources.push_back(source);
if (std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(),
source) == m_program.srt.dynamic_sources.end()) {
m_program.srt.dynamic_sources.push_back(source);
}
return true;
}
bool CollectValue(uint32_t id, uint32_t use_pc, std::string* error) {
if (id >= m_program->provenance.values.size()) {
if (id >= m_program.provenance.values.size()) {
return Fail(use_pc, error, fmt::format("invalid scalar value {}", id));
}
if (m_state[id] == 2) {
@@ -299,7 +297,7 @@ private:
return Fail(use_pc, error, fmt::format("cyclic scalar value {}", id));
}
m_state[id] = 1;
const auto& value = m_program->provenance.values[id];
const auto& value = m_program.provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (!CollectValue(arg, use_pc, error)) {
@@ -317,17 +315,17 @@ private:
const auto offset_arg = value.op == ScalarValueOp::ReadConst ? 2u : 4u;
uint32_t ignored = 0;
if (m_folder.Fold(value.args[offset_arg], ignored)) {
m_program->srt.reads.push_back(
{id, static_cast<uint32_t>(m_program->srt.reads.size()), use_pc});
m_program.srt.reads.push_back(
{id, static_cast<uint32_t>(m_program.srt.reads.size()), use_pc});
} else {
m_program->srt.dynamic_reads.push_back(id);
m_program.srt.dynamic_reads.push_back(id);
}
}
m_state[id] = 2;
return true;
}
Program* m_program;
Program& m_program;
ConstantFolder m_folder;
std::vector<uint8_t> m_state;
};
@@ -395,7 +393,7 @@ public:
}
case ScalarValueOp::ReadConst:
case ScalarValueOp::ReadConstBuffer:
if (!Read(value, &out, error)) {
if (!Read(value, out, error)) {
return false;
}
break;
@@ -403,7 +401,7 @@ public:
case ScalarValueOp::Unknown:
return Fail(error, fmt::format("scalar value {} is unresolved", id));
default:
if (!EvaluateOperation(value, &out, error)) {
if (!EvaluateOperation(value, out, error)) {
return false;
}
break;
@@ -422,13 +420,13 @@ private:
return false;
}
bool EvaluateOperation(const ScalarValue& value, uint32_t* result, std::string* error) {
bool EvaluateOperation(const ScalarValue& value, uint32_t& result, std::string* error) {
const auto count = ScalarValueArgCount(value.op);
if (count == 0 || count > 3) {
return Fail(error, fmt::format("unsupported scalar operation {}",
static_cast<uint32_t>(value.op)));
}
uint32_t args[3] = {};
std::array<uint32_t, 3> args {};
for (uint32_t i = 0; i < count; i++) {
if (!Evaluate(value.args[i], args[i], error)) {
return false;
@@ -439,7 +437,7 @@ private:
static_cast<uint32_t>(value.op)));
}
bool Read(const ScalarValue& value, uint32_t* result, std::string* error) {
bool Read(const ScalarValue& value, uint32_t& result, std::string* error) {
const bool buffer = value.op == ScalarValueOp::ReadConstBuffer;
uint32_t lo = 0;
uint32_t hi = 0;
@@ -478,7 +476,7 @@ private:
const auto base_aligned = base & ~uint64_t {3};
const auto immediate_aligned = immediate & ~int64_t {3};
const auto relative = immediate_aligned + static_cast<int64_t>(offset & ~uint32_t {3});
if (!AddSignedAddress(base_aligned, relative, &address)) {
if (!AddSignedAddress(base_aligned, relative, address)) {
return Fail(error,
fmt::format("ReadConst pc=0x{:08x} address base=0x{:016x} offset={} "
"is outside the 48-bit address space",
@@ -486,12 +484,12 @@ private:
}
}
if (m_runtime.read_memory != nullptr &&
!m_runtime.read_memory(m_runtime.userdata, address, result)) {
!m_runtime.read_memory(m_runtime.userdata, address, &result)) {
return Fail(
error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address));
}
if (m_runtime.read_memory == nullptr) {
std::memcpy(result, reinterpret_cast<const void*>(address), sizeof(*result));
std::memcpy(&result, reinterpret_cast<const void*>(address), sizeof(result));
}
return true;
}
@@ -505,71 +503,53 @@ private:
} // namespace
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t* result) {
if (result == nullptr) {
return false;
}
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t& result) {
ConstantFolder folder(provenance);
return folder.Fold(value, *result);
return folder.Fold(value, result);
}
bool BuildSrtPlan(Program* program, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error = "invalid SRT program";
}
return false;
}
if (program->resource_tracking_complete) {
bool BuildSrtPlan(Program& program, std::string* error) {
if (program.resource_tracking_complete) {
if (error != nullptr) {
*error = "cannot rebuild SRT plan after resource tracking";
}
return false;
}
if (program->srt_patching_complete) {
if (program.srt_patching_complete) {
if (error != nullptr) {
*error = "cannot rebuild SRT plan after SRT patching";
}
return false;
}
program->srt_plan_complete = false;
program.srt_plan_complete = false;
if (!PlanBuilder(program).Run(error)) {
return false;
}
program->srt_plan_complete = true;
program.srt_plan_complete = true;
return true;
}
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue* result,
std::string* error) {
if (result == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, use_pc, "invalid descriptor result");
}
return false;
}
const SrtRuntime& runtime, DescriptorValue& result,
std::string* error) {
const DescriptorSourceRequest request {source, use_pc};
std::vector<DescriptorValue> results;
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, &results, error)) {
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, results, error)) {
return false;
}
*result = results[0];
result = results[0];
return true;
}
static bool EvaluateRuntimeSourcesImpl(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime,
std::vector<DescriptorValue>* results,
std::vector<uint32_t>* flat, bool evaluate_flat,
std::string* error) {
if (results == nullptr || flat == nullptr || !program.srt_plan_complete) {
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime,
std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, bool evaluate_flat,
std::string* error) {
if (!program.srt_plan_complete) {
if (error != nullptr) {
*error = Diagnostic(program, 0,
results == nullptr || flat == nullptr
? "invalid runtime evaluation output"
: "SRT plan is not ready");
*error = Diagnostic(program, 0, "SRT plan is not ready");
}
return false;
}
@@ -614,44 +594,32 @@ static bool EvaluateRuntimeSourcesImpl(const Program&
}
}
}
*results = std::move(evaluated);
results = std::move(evaluated);
if (evaluate_flat) {
*flat = std::move(flattened);
flat = std::move(flattened);
}
return true;
}
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::string* error) {
if (results == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "invalid descriptor results");
}
return false;
}
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::string* error) {
std::vector<uint32_t> ignored;
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, &ignored, false, error);
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, ignored, false, error);
}
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::vector<uint32_t>* flat, std::string* error) {
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::string* error) {
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error);
}
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>* flat,
std::string* error) {
if (flat == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "invalid SRT walker output");
}
return false;
}
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
std::string* error) {
std::vector<DescriptorValue> ignored;
return EvaluateRuntimeSources(program, {}, runtime, &ignored, flat, error);
return EvaluateRuntimeSources(program, {}, runtime, ignored, flat, error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/SrtWalker.h
+9 -9
View File
@@ -23,31 +23,31 @@ struct DescriptorSourceRequest {
uint32_t use_pc = 0;
};
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t* result);
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t& result);
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
// dynamic offsets remain explicit and are never assigned a fake slot.
bool BuildSrtPlan(Program* program, std::string* error);
bool BuildSrtPlan(Program& program, std::string* error);
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue* result,
const SrtRuntime& runtime, DescriptorValue& result,
std::string* error);
// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by
// several descriptors are memoized once across the batch.
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::string* error);
// Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk.
// On failure neither destination is changed.
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::vector<uint32_t>* flat, std::string* error);
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>& results,
std::vector<uint32_t>& flat, std::string* error);
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>* flat,
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>& flat,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/VectorAluOps.cpp
+281 -281
View File
@@ -319,9 +319,9 @@ bool IsNativeVop3B16BinaryOpcode(Opcode opcode) {
}
}
void ApplyDefaultVop2F16Destination(Instruction* inst) {
if (inst != nullptr && IsVop2LowHalfF16Opcode(inst->opcode)) {
inst->dst.sdwa_sel = 4;
void ApplyDefaultVop2F16Destination(Instruction& inst) {
if (IsVop2LowHalfF16Opcode(inst.opcode)) {
inst.dst.sdwa_sel = 4;
}
}
@@ -476,7 +476,7 @@ bool HasUnsupportedVop1SdwaSourceModifiers(Opcode opcode, bool src_neg, bool src
}
}
bool ValidateVop1Sdwa(Instruction* inst, uint32_t opcode, uint32_t modifier) {
bool ValidateVop1Sdwa(Instruction& inst, uint32_t opcode, uint32_t modifier) {
const auto dst_sel = (modifier >> 8u) & 0x7u;
const auto dst_u = (modifier >> 11u) & 0x3u;
const auto clamp = (modifier >> 13u) & 0x1u;
@@ -489,20 +489,20 @@ bool ValidateVop1Sdwa(Instruction* inst, uint32_t opcode, uint32_t modifier) {
SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA selector is invalid");
return false;
}
if ((clamp != 0u || omod != 0u) && !IsVop1FloatResultOpcode(inst->opcode)) {
if ((clamp != 0u || omod != 0u) && !IsVop1FloatResultOpcode(inst.opcode)) {
SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA output modifiers are not supported");
return false;
}
if (!IsVop1SdwaDestinationSupported(inst->opcode, dst_sel, dst_u, src0_sel)) {
if (!IsVop1SdwaDestinationSupported(inst.opcode, dst_sel, dst_u, src0_sel)) {
SetUnsupported(inst, Family::VOP1, opcode,
"VOP1 SDWA destination selector is not supported");
return false;
}
if (HasUnsupportedVop1SdwaSourceModifiers(inst->opcode, src0_neg != 0u, src0_abs != 0u)) {
if (HasUnsupportedVop1SdwaSourceModifiers(inst.opcode, src0_neg != 0u, src0_abs != 0u)) {
SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source modifiers are not supported");
return false;
}
if (!IsVop1SdwaSourceSupported(inst->opcode, src0_sel, src0_neg != 0u, src0_abs != 0u)) {
if (!IsVop1SdwaSourceSupported(inst.opcode, src0_sel, src0_neg != 0u, src0_abs != 0u)) {
SetUnsupported(inst, Family::VOP1, opcode, "VOP1 SDWA source selector is not supported");
return false;
}
@@ -510,7 +510,7 @@ bool ValidateVop1Sdwa(Instruction* inst, uint32_t opcode, uint32_t modifier) {
}
bool DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, Instruction* inst, std::string* error) {
uint32_t opcode, uint32_t vdst, Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated VOP1 SDWA instruction at pc 0x{:08x}", pc);
@@ -535,26 +535,26 @@ bool DecodeVop1Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
return true;
}
const bool scalar_dst = UsesScalarDestination(inst->opcode);
if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, &inst->dst, error)
: DecodeVectorGpr(vdst, &inst->dst, error)) ||
!DecodeScalarSource(src0 + (s0 == 0u ? 256u : 0u), pc, &inst->src0, error)) {
const bool scalar_dst = UsesScalarDestination(inst.opcode);
if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, inst.dst, error)
: DecodeVectorGpr(vdst, inst.dst, error)) ||
!DecodeScalarSource(src0 + (s0 == 0u ? 256u : 0u), pc, inst.src0, error)) {
return false;
}
inst->dst.sdwa_sel = dst_sel;
inst->dst.sdwa_dst_unused = dst_u;
inst->dst.clamp = clamp != 0u;
inst->dst.omod = omod;
inst->src0.sdwa_sel = src0_sel;
inst->src0.sdwa_sext = src0_sext != 0u;
inst->src0.negate = src0_neg != 0u;
inst->src0.absolute = src0_abs != 0u;
inst->src_count = 1;
inst.dst.sdwa_sel = dst_sel;
inst.dst.sdwa_dst_unused = dst_u;
inst.dst.clamp = clamp != 0u;
inst.dst.omod = omod;
inst.src0.sdwa_sel = src0_sel;
inst.src0.sdwa_sext = src0_sext != 0u;
inst.src0.negate = src0_neg != 0u;
inst.src0.absolute = src0_abs != 0u;
inst.src_count = 1;
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, Instruction* inst, std::string* error) {
uint32_t opcode, uint32_t vdst, Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated VOP1 DPP instruction at pc 0x{:08x}", pc);
@@ -566,23 +566,23 @@ bool DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
const auto src0 = modifier & 0xffu;
SetRawWords(inst, code, word_index, 2);
const bool scalar_dst = UsesScalarDestination(inst->opcode);
if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, &inst->dst, error)
: DecodeVectorGpr(vdst, &inst->dst, error)) ||
!DecodeScalarSource(src0 + 256u, pc, &inst->src0, error)) {
const bool scalar_dst = UsesScalarDestination(inst.opcode);
if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, inst.dst, error)
: DecodeVectorGpr(vdst, inst.dst, error)) ||
!DecodeScalarSource(src0 + 256u, pc, inst.src0, error)) {
return false;
}
inst->src0.negate = ((modifier >> 20u) & 0x1u) != 0u;
inst->src0.absolute = ((modifier >> 21u) & 0x1u) != 0u;
inst->src0.dpp = true;
inst->src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu;
inst->src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
inst->src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u;
inst->src0.dpp_bank_mask = (modifier >> 24u) & 0xfu;
inst->src0.dpp_row_mask = (modifier >> 28u) & 0xfu;
inst->src_count = 1;
inst.src0.negate = ((modifier >> 20u) & 0x1u) != 0u;
inst.src0.absolute = ((modifier >> 21u) & 0x1u) != 0u;
inst.src0.dpp = true;
inst.src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu;
inst.src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
inst.src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u;
inst.src0.dpp_bank_mask = (modifier >> 24u) & 0xfu;
inst.src0.dpp_row_mask = (modifier >> 28u) & 0xfu;
inst.src_count = 1;
if (!IsVop1FloatSourceOpcode(inst->opcode) && (inst->src0.negate || inst->src0.absolute)) {
if (!IsVop1FloatSourceOpcode(inst.opcode) && (inst.src0.negate || inst.src0.absolute)) {
SetUnsupported(inst, Family::VOP1, opcode,
"VOP1 DPP integer source modifiers are not supported");
return true;
@@ -592,7 +592,7 @@ bool DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
using Vop1ModifierDecodeFn = bool (*)(uint32_t pc, std::span<const uint32_t> code,
uint32_t word_index, uint32_t opcode, uint32_t vdst,
Instruction* inst, std::string* error);
Instruction& inst, std::string* error);
struct Vop1ModifierDecoder {
uint32_t escape = 0;
@@ -605,7 +605,7 @@ constexpr Vop1ModifierDecoder VOP1_MODIFIER_DECODERS[] = {
};
bool TryDecodeVop1Modifier(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t src0, uint32_t opcode, uint32_t vdst, Instruction* inst,
uint32_t src0, uint32_t opcode, uint32_t vdst, Instruction& inst,
std::string* error, bool* handled) {
if (handled != nullptr) {
*handled = false;
@@ -857,17 +857,17 @@ bool IsFullWidthVop2Sdwa(const Vop2SdwaFields& fields) {
fields.src1_sel == 6u;
}
bool ValidateVop2Sdwa(Instruction* inst, uint32_t opcode, const Vop2SdwaFields& fields) {
bool ValidateVop2Sdwa(Instruction& inst, uint32_t opcode, const Vop2SdwaFields& fields) {
if (fields.src0_sel > 6u || fields.src1_sel > 6u || fields.dst_sel > 6u) {
SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA selector is invalid");
return false;
}
if ((fields.clamp != 0u || fields.omod != 0u) && !IsVop2FloatResultOpcode(inst->opcode)) {
if ((fields.clamp != 0u || fields.omod != 0u) && !IsVop2FloatResultOpcode(inst.opcode)) {
SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA output modifiers are not supported");
return false;
}
if (IsFullWidthVop2Sdwa(fields)) {
if (!IsVop2FloatSourceOpcode(inst->opcode) && inst->opcode != Opcode::VCndmaskB32 &&
if (!IsVop2FloatSourceOpcode(inst.opcode) && inst.opcode != Opcode::VCndmaskB32 &&
(fields.src0_neg != 0u || fields.src0_abs != 0u || fields.src1_neg != 0u ||
fields.src1_abs != 0u)) {
SetUnsupported(inst, Family::VOP2, opcode,
@@ -877,7 +877,7 @@ bool ValidateVop2Sdwa(Instruction* inst, uint32_t opcode, const Vop2SdwaFields&
return true;
}
const auto* rule = FindVop2SdwaRule(inst->opcode);
const auto* rule = FindVop2SdwaRule(inst.opcode);
if (rule == nullptr) {
SetUnsupported(inst, Family::VOP2, opcode,
"VOP2 SDWA modifier is not supported for opcode");
@@ -896,7 +896,7 @@ bool ValidateVop2Sdwa(Instruction* inst, uint32_t opcode, const Vop2SdwaFields&
const bool has_source_modifiers = (fields.src0_neg != 0u || fields.src0_abs != 0u ||
fields.src1_neg != 0u || fields.src1_abs != 0u);
const bool cndmask_float_modifiers =
inst->opcode == Opcode::VCndmaskB32 && fields.src0_sel == 6u && fields.src1_sel == 6u;
inst.opcode == Opcode::VCndmaskB32 && fields.src0_sel == 6u && fields.src1_sel == 6u;
if (!rule->source_modifiers && has_source_modifiers && !cndmask_float_modifiers) {
SetUnsupported(inst, Family::VOP2, opcode, "VOP2 SDWA source modifiers are not supported");
return false;
@@ -905,34 +905,34 @@ bool ValidateVop2Sdwa(Instruction* inst, uint32_t opcode, const Vop2SdwaFields&
}
bool FinalizeVop2Instruction(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
Instruction& inst, std::string* error) {
(void)pc;
switch (inst->opcode) {
switch (inst.opcode) {
case Opcode::VMadmkF32:
case Opcode::VFmamkF16:
inst->src2 = inst->src1;
inst->src1 = {};
inst->src1.kind = OperandKind::LiteralConstant;
inst->src_count = 3;
inst.src2 = inst.src1;
inst.src1 = {};
inst.src1.kind = OperandKind::LiteralConstant;
inst.src_count = 3;
break;
case Opcode::VMadakF32:
case Opcode::VFmaakF16:
inst->src2 = {};
inst->src2.kind = OperandKind::LiteralConstant;
inst->src_count = 3;
inst.src2 = {};
inst.src2.kind = OperandKind::LiteralConstant;
inst.src_count = 3;
break;
case Opcode::VAddcU32:
inst->dst2.kind = OperandKind::VccLo;
inst->src2.kind = OperandKind::VccLo;
inst->src_count = 3;
inst.dst2.kind = OperandKind::VccLo;
inst.src2.kind = OperandKind::VccLo;
inst.src_count = 3;
break;
default: inst->src_count = 2; break;
default: inst.src_count = 2; break;
}
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeVop2Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction* inst,
uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -948,28 +948,28 @@ bool DecodeVop2Sdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
return true;
}
if (!DecodeVectorGpr(vdst, &inst->dst, error) ||
!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, &inst->src0, error) ||
!DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, &inst->src1, error)) {
if (!DecodeVectorGpr(vdst, inst.dst, error) ||
!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, inst.src0, error) ||
!DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1, error)) {
return false;
}
inst->dst.sdwa_sel = fields.dst_sel;
inst->dst.sdwa_dst_unused = fields.dst_u;
inst->dst.clamp = fields.clamp != 0u;
inst->dst.omod = fields.omod;
inst->src0.sdwa_sel = fields.src0_sel;
inst->src0.sdwa_sext = fields.src0_sext != 0u;
inst->src0.negate = fields.src0_neg != 0u;
inst->src0.absolute = fields.src0_abs != 0u;
inst->src1.sdwa_sel = fields.src1_sel;
inst->src1.sdwa_sext = fields.src1_sext != 0u;
inst->src1.negate = fields.src1_neg != 0u;
inst->src1.absolute = fields.src1_abs != 0u;
inst.dst.sdwa_sel = fields.dst_sel;
inst.dst.sdwa_dst_unused = fields.dst_u;
inst.dst.clamp = fields.clamp != 0u;
inst.dst.omod = fields.omod;
inst.src0.sdwa_sel = fields.src0_sel;
inst.src0.sdwa_sext = fields.src0_sext != 0u;
inst.src0.negate = fields.src0_neg != 0u;
inst.src0.absolute = fields.src0_abs != 0u;
inst.src1.sdwa_sel = fields.src1_sel;
inst.src1.sdwa_sext = fields.src1_sext != 0u;
inst.src1.negate = fields.src1_neg != 0u;
inst.src1.absolute = fields.src1_abs != 0u;
return FinalizeVop2Instruction(pc, code, word_index, inst, error);
}
bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction* inst,
uint32_t opcode, uint32_t vdst, uint32_t vsrc1, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -982,25 +982,25 @@ bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
const auto src0 = modifier & 0xffu;
SetRawWords(inst, code, word_index, 2);
if (!DecodeVectorGpr(vdst, &inst->dst, error) ||
!DecodeScalarSource(src0 + 256u, pc, &inst->src0, error) ||
!DecodeVectorGpr(vsrc1, &inst->src1, error)) {
if (!DecodeVectorGpr(vdst, inst.dst, error) ||
!DecodeScalarSource(src0 + 256u, pc, inst.src0, error) ||
!DecodeVectorGpr(vsrc1, inst.src1, error)) {
return false;
}
ApplyDefaultVop2F16Destination(inst);
inst->src0.negate = ((modifier >> 20u) & 0x1u) != 0u;
inst->src0.absolute = ((modifier >> 21u) & 0x1u) != 0u;
inst->src0.dpp = true;
inst->src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu;
inst->src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
inst->src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u;
inst->src0.dpp_bank_mask = (modifier >> 24u) & 0xfu;
inst->src0.dpp_row_mask = (modifier >> 28u) & 0xfu;
inst->src1.negate = ((modifier >> 22u) & 0x1u) != 0u;
inst->src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
inst.src0.negate = ((modifier >> 20u) & 0x1u) != 0u;
inst.src0.absolute = ((modifier >> 21u) & 0x1u) != 0u;
inst.src0.dpp = true;
inst.src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu;
inst.src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
inst.src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u;
inst.src0.dpp_bank_mask = (modifier >> 24u) & 0xfu;
inst.src0.dpp_row_mask = (modifier >> 28u) & 0xfu;
inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u;
inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
if (!IsVop2FloatSourceOpcode(inst->opcode) &&
(inst->src0.negate || inst->src0.absolute || inst->src1.negate || inst->src1.absolute)) {
if (!IsVop2FloatSourceOpcode(inst.opcode) &&
(inst.src0.negate || inst.src0.absolute || inst.src1.negate || inst.src1.absolute)) {
SetUnsupported(inst, Family::VOP2, opcode,
"VOP2 DPP integer source modifiers are not supported");
return true;
@@ -1010,7 +1010,7 @@ bool DecodeVop2Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
using Vop2ModifierDecodeFn = bool (*)(uint32_t pc, std::span<const uint32_t> code,
uint32_t word_index, uint32_t opcode, uint32_t vdst,
uint32_t vsrc1, Instruction* inst, std::string* error);
uint32_t vsrc1, Instruction& inst, std::string* error);
struct Vop2ModifierDecoder {
uint32_t escape = 0;
@@ -1024,7 +1024,7 @@ constexpr Vop2ModifierDecoder VOP2_MODIFIER_DECODERS[] = {
bool TryDecodeVop2Modifier(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t src0, uint32_t opcode, uint32_t vdst, uint32_t vsrc1,
Instruction* inst, std::string* error, bool* handled) {
Instruction& inst, std::string* error, bool* handled) {
if (handled != nullptr) {
*handled = false;
}
@@ -1078,7 +1078,7 @@ bool SupportsVopcSdwa(Opcode opcode) {
}
bool DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vsrc1, Instruction* inst, std::string* error) {
uint32_t opcode, uint32_t vsrc1, Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated VOPC SDWA instruction at pc 0x{:08x}", pc);
@@ -1093,37 +1093,37 @@ bool DecodeVopcSdwa(uint32_t pc, std::span<const uint32_t> code, uint32_t word_i
SetUnsupported(inst, Family::VOPC, opcode, "VOPC SDWA selector is invalid");
return true;
}
if (!SupportsVopcSdwa(inst->opcode)) {
if (!SupportsVopcSdwa(inst.opcode)) {
SetUnsupported(inst, Family::VOPC, opcode,
"VOPC SDWA modifier is not supported for opcode");
return true;
}
if (!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, &inst->src0, error) ||
!DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, &inst->src1, error)) {
if (!DecodeScalarSource(fields.src0 + (fields.s0 == 0u ? 256u : 0u), pc, inst.src0, error) ||
!DecodeScalarSource(vsrc1 + (fields.s1 == 0u ? 256u : 0u), pc, inst.src1, error)) {
return false;
}
if (IsVopcCompareExec(inst->opcode)) {
inst->dst.kind = OperandKind::ExecLo;
if (IsVopcCompareExec(inst.opcode)) {
inst.dst.kind = OperandKind::ExecLo;
} else if (fields.sd == 0u) {
inst->dst.kind = OperandKind::VccLo;
} else if (!DecodeScalarDestination(fields.sdst, pc, &inst->dst, error)) {
inst.dst.kind = OperandKind::VccLo;
} else if (!DecodeScalarDestination(fields.sdst, pc, inst.dst, error)) {
return false;
}
inst->src0.sdwa_sel = fields.src0_sel;
inst->src0.sdwa_sext = fields.src0_sext != 0u;
inst->src0.negate = fields.src0_neg != 0u;
inst->src0.absolute = fields.src0_abs != 0u;
inst->src1.sdwa_sel = fields.src1_sel;
inst->src1.sdwa_sext = fields.src1_sext != 0u;
inst->src1.negate = fields.src1_neg != 0u;
inst->src1.absolute = fields.src1_abs != 0u;
inst->src_count = 2;
inst.src0.sdwa_sel = fields.src0_sel;
inst.src0.sdwa_sext = fields.src0_sext != 0u;
inst.src0.negate = fields.src0_neg != 0u;
inst.src0.absolute = fields.src0_abs != 0u;
inst.src1.sdwa_sel = fields.src1_sel;
inst.src1.sdwa_sext = fields.src1_sext != 0u;
inst.src1.negate = fields.src1_neg != 0u;
inst.src1.absolute = fields.src1_abs != 0u;
inst.src_count = 2;
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
uint32_t opcode, uint32_t vsrc1, Instruction* inst, std::string* error) {
uint32_t opcode, uint32_t vsrc1, Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated VOPC DPP instruction at pc 0x{:08x}", pc);
@@ -1134,26 +1134,26 @@ bool DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
const auto modifier = code[word_index + 1u];
const auto src0 = modifier & 0xffu;
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented");
return true;
}
if (!DecodeScalarSource(src0 + 256u, pc, &inst->src0, error) ||
!DecodeVectorGpr(vsrc1, &inst->src1, error)) {
if (!DecodeScalarSource(src0 + 256u, pc, inst.src0, error) ||
!DecodeVectorGpr(vsrc1, inst.src1, error)) {
return false;
}
inst->dst.kind = IsVopcCompareExec(inst->opcode) ? OperandKind::ExecLo : OperandKind::VccLo;
inst->src0.negate = ((modifier >> 20u) & 0x1u) != 0u;
inst->src0.absolute = ((modifier >> 21u) & 0x1u) != 0u;
inst->src0.dpp = true;
inst->src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu;
inst->src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
inst->src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u;
inst->src0.dpp_bank_mask = (modifier >> 24u) & 0xfu;
inst->src0.dpp_row_mask = (modifier >> 28u) & 0xfu;
inst->src1.negate = ((modifier >> 22u) & 0x1u) != 0u;
inst->src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
inst->src_count = 2;
inst.dst.kind = IsVopcCompareExec(inst.opcode) ? OperandKind::ExecLo : OperandKind::VccLo;
inst.src0.negate = ((modifier >> 20u) & 0x1u) != 0u;
inst.src0.absolute = ((modifier >> 21u) & 0x1u) != 0u;
inst.src0.dpp = true;
inst.src0.dpp_ctrl = (modifier >> 8u) & 0x1ffu;
inst.src0.dpp_fetch_inactive = ((modifier >> 18u) & 0x1u) != 0u;
inst.src0.dpp_bound_ctrl = ((modifier >> 19u) & 0x1u) != 0u;
inst.src0.dpp_bank_mask = (modifier >> 24u) & 0xfu;
inst.src0.dpp_row_mask = (modifier >> 28u) & 0xfu;
inst.src1.negate = ((modifier >> 22u) & 0x1u) != 0u;
inst.src1.absolute = ((modifier >> 23u) & 0x1u) != 0u;
inst.src_count = 2;
return ReadLiteralOperands(code, word_index, inst, error);
}
@@ -1229,9 +1229,9 @@ bool IsMadMixF16(Opcode opcode) {
return opcode == Opcode::VMadMixloF16 || opcode == Opcode::VMadMixhiF16;
}
void ApplyVop3pSourceModifiers(Instruction* inst, uint32_t op_sel, uint32_t op_sel_hi, uint32_t neg,
void ApplyVop3pSourceModifiers(Instruction& inst, uint32_t op_sel, uint32_t op_sel_hi, uint32_t neg,
uint32_t neg_hi) {
Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2};
Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2};
for (uint32_t i = 0; i < 3u; i++) {
sources[i]->op_sel = ((op_sel >> i) & 1u) != 0;
sources[i]->op_sel_hi = ((op_sel_hi >> i) & 1u) != 0;
@@ -1240,8 +1240,8 @@ void ApplyVop3pSourceModifiers(Instruction* inst, uint32_t op_sel, uint32_t op_s
}
}
void ApplyVop3pMixAbsModifiers(Instruction* inst) {
Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2};
void ApplyVop3pMixAbsModifiers(Instruction& inst) {
Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2};
for (auto* source: sources) {
// RDNA2 MIX opcodes reuse the VOP3P NEG_HI bits as source absolute modifiers.
source->absolute = source->negate_hi;
@@ -1249,44 +1249,44 @@ void ApplyVop3pMixAbsModifiers(Instruction* inst) {
}
}
void ApplyNativeVop3MadMixModifiers(Instruction* inst, uint32_t op_sel, uint32_t abs,
void ApplyNativeVop3MadMixModifiers(Instruction& inst, uint32_t op_sel, uint32_t abs,
uint32_t neg) {
Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2};
Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2};
for (uint32_t i = 0; i < 3u; i++) {
sources[i]->op_sel = ((op_sel >> i) & 1u) != 0;
sources[i]->op_sel_hi = true;
sources[i]->negate = ((neg >> i) & 1u) != 0;
sources[i]->absolute = ((abs >> i) & 1u) != 0;
}
inst->dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u;
inst.dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u;
}
void ApplyNativeVop3F16TernaryModifiers(Instruction* inst, uint32_t op_sel, uint32_t abs,
void ApplyNativeVop3F16TernaryModifiers(Instruction& inst, uint32_t op_sel, uint32_t abs,
uint32_t neg) {
Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2};
Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2};
for (uint32_t i = 0; i < 3u; i++) {
sources[i]->op_sel = ((op_sel >> i) & 1u) != 0;
sources[i]->op_sel_hi = true;
sources[i]->negate = ((neg >> i) & 1u) != 0;
sources[i]->absolute = ((abs >> i) & 1u) != 0;
}
inst->dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u;
inst.dst.sdwa_sel = ((op_sel & 0x8u) != 0) ? 5u : 4u;
}
void ApplyNativeVop3B16BinaryModifiers(Instruction* inst, uint32_t op_sel) {
inst->src0.op_sel = (op_sel & 0x1u) != 0;
inst->src1.op_sel = (op_sel & 0x2u) != 0;
inst->dst.sdwa_sel = (op_sel & 0x8u) != 0 ? 5u : 4u;
void ApplyNativeVop3B16BinaryModifiers(Instruction& inst, uint32_t op_sel) {
inst.src0.op_sel = (op_sel & 0x1u) != 0;
inst.src1.op_sel = (op_sel & 0x2u) != 0;
inst.dst.sdwa_sel = (op_sel & 0x8u) != 0 ? 5u : 4u;
}
void ApplyNativeVop3PackB32F16Modifiers(Instruction* inst, uint32_t op_sel, uint32_t abs,
void ApplyNativeVop3PackB32F16Modifiers(Instruction& inst, uint32_t op_sel, uint32_t abs,
uint32_t neg) {
inst->src0.op_sel = (op_sel & 0x1u) != 0;
inst->src1.op_sel = (op_sel & 0x2u) != 0;
inst->src0.negate = (neg & 0x1u) != 0;
inst->src1.negate = (neg & 0x2u) != 0;
inst->src0.absolute = (abs & 0x1u) != 0;
inst->src1.absolute = (abs & 0x2u) != 0;
inst.src0.op_sel = (op_sel & 0x1u) != 0;
inst.src1.op_sel = (op_sel & 0x2u) != 0;
inst.src0.negate = (neg & 0x1u) != 0;
inst.src1.negate = (neg & 0x2u) != 0;
inst.src0.absolute = (abs & 0x1u) != 0;
inst.src1.absolute = (abs & 0x2u) != 0;
}
bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
@@ -1390,9 +1390,9 @@ bool HasUnsupportedNativeVop3Modifiers(Opcode opcode, bool permlane, bool mad_mi
return abs != 0u || op_sel != 0u || clamp != 0u || omod != 0u || neg != 0u;
}
void ApplyNativeVop3SourceModifiers(Instruction* inst, uint32_t abs, uint32_t neg) {
Operand* sources[] = {&inst->src0, &inst->src1, &inst->src2};
for (uint32_t i = 0; i < inst->src_count && i < 3u; i++) {
void ApplyNativeVop3SourceModifiers(Instruction& inst, uint32_t abs, uint32_t neg) {
Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2};
for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) {
sources[i]->absolute = ((abs >> i) & 1u) != 0;
sources[i]->negate = ((neg >> i) & 1u) != 0;
}
@@ -1437,7 +1437,7 @@ bool IsVopcCompareExec(Opcode opcode) {
} // namespace
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 25u) & 0x3fu;
@@ -1445,11 +1445,11 @@ bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
const uint32_t src0 = word & 0x1ffu;
const uint32_t vsrc1 = (word >> 9u) & 0xffu;
inst->pc = pc;
inst->word = word;
inst->family = Family::VOP2;
inst->opcode_id = opcode;
inst->opcode = Lookup(VOP2_OPS, static_cast<uint32_t>(std::size(VOP2_OPS)), opcode);
inst.pc = pc;
inst.word = word;
inst.family = Family::VOP2;
inst.opcode_id = opcode;
inst.opcode = Lookup(VOP2_OPS, static_cast<uint32_t>(std::size(VOP2_OPS)), opcode);
SetRawWords(inst, code, word_index, 1);
switch (opcode) {
@@ -1457,7 +1457,7 @@ bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
case 0x3fu: return DecodeVop1(pc, code, word_index, inst, error);
default: break;
}
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VOP2, opcode, "VOP2 opcode is not implemented");
return true;
}
@@ -1470,36 +1470,36 @@ bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
return true;
}
if (!DecodeVectorGpr(vdst, &inst->dst, error) ||
!DecodeScalarSource(src0, pc, &inst->src0, error) ||
!DecodeVectorGpr(vsrc1, &inst->src1, error)) {
if (!DecodeVectorGpr(vdst, inst.dst, error) ||
!DecodeScalarSource(src0, pc, inst.src0, error) ||
!DecodeVectorGpr(vsrc1, inst.src1, error)) {
return false;
}
ApplyDefaultVop2F16Destination(inst);
return FinalizeVop2Instruction(pc, code, word_index, inst, error);
}
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 9u) & 0xffu;
const uint32_t src0 = word & 0x1ffu;
const uint32_t vdst = (word >> 17u) & 0xffu;
inst->pc = pc;
inst->word = word;
inst->family = Family::VOP1;
inst->opcode_id = opcode;
inst->opcode = Lookup(VOP1_OPS, static_cast<uint32_t>(std::size(VOP1_OPS)), opcode);
inst.pc = pc;
inst.word = word;
inst.family = Family::VOP1;
inst.opcode_id = opcode;
inst.opcode = Lookup(VOP1_OPS, static_cast<uint32_t>(std::size(VOP1_OPS)), opcode);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VOP1, opcode, "VOP1 opcode is not implemented");
return true;
}
if (inst->opcode == Opcode::VNop) {
inst->dst.kind = OperandKind::Null;
inst->src_count = 0;
if (inst.opcode == Opcode::VNop) {
inst.dst.kind = OperandKind::Null;
inst.src_count = 0;
return true;
}
bool modifier_handled = false;
@@ -1510,29 +1510,29 @@ bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
if (modifier_handled) {
return true;
}
const bool scalar_dst = UsesScalarDestination(inst->opcode);
if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, &inst->dst, error)
: DecodeVectorGpr(vdst, &inst->dst, error)) ||
!DecodeScalarSource(src0, pc, &inst->src0, error)) {
const bool scalar_dst = UsesScalarDestination(inst.opcode);
if (!(scalar_dst ? DecodeScalarDestination(vdst, pc, inst.dst, error)
: DecodeVectorGpr(vdst, inst.dst, error)) ||
!DecodeScalarSource(src0, pc, inst.src0, error)) {
return false;
}
inst->src_count = 1;
inst.src_count = 1;
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 17u) & 0xffu;
const uint32_t src0 = word & 0x1ffu;
const uint32_t vsrc1 = (word >> 9u) & 0xffu;
inst->pc = pc;
inst->word = word;
inst->family = Family::VOPC;
inst->opcode_id = opcode;
inst->opcode = Lookup(VOPC_OPS, static_cast<uint32_t>(std::size(VOPC_OPS)), opcode);
inst->dst.kind = IsVopcCompareExec(inst->opcode) ? OperandKind::ExecLo : OperandKind::VccLo;
inst.pc = pc;
inst.word = word;
inst.family = Family::VOPC;
inst.opcode_id = opcode;
inst.opcode = Lookup(VOPC_OPS, static_cast<uint32_t>(std::size(VOPC_OPS)), opcode);
inst.dst.kind = IsVopcCompareExec(inst.opcode) ? OperandKind::ExecLo : OperandKind::VccLo;
SetRawWords(inst, code, word_index, 1);
switch (src0) {
@@ -1540,19 +1540,19 @@ bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
case 250u: return DecodeVopcDpp(pc, code, word_index, opcode, vsrc1, inst, error);
default: break;
}
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VOPC, opcode, "VOPC opcode is not implemented");
return true;
}
if (!DecodeScalarSource(src0, pc, &inst->src0, error) ||
!DecodeVectorGpr(vsrc1, &inst->src1, error)) {
if (!DecodeScalarSource(src0, pc, inst.src0, error) ||
!DecodeVectorGpr(vsrc1, inst.src1, error)) {
return false;
}
inst->src_count = 2;
inst.src_count = 2;
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
@@ -1575,30 +1575,30 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
const uint32_t omod = (word1 >> 27u) & 0x3u;
const uint32_t neg = (word1 >> 29u) & 0x7u;
inst->pc = pc;
inst->word = word0;
inst->family = Family::VOP3;
inst->opcode_id = opcode;
inst->opcode = LookupVop3Opcode(opcode);
inst.pc = pc;
inst.word = word0;
inst.family = Family::VOP3;
inst.opcode_id = opcode;
inst.opcode = LookupVop3Opcode(opcode);
SetRawWords(inst, code, word_index, 2);
const bool addc = inst->opcode == Opcode::VAddcU32 && opcode == 0x128u;
const bool vop3b_carry_out = IsVop3BCarryOutOpcode(inst->opcode);
const bool vop3b_mad_u64 = IsVop3BMadU64Opcode(inst->opcode);
const bool addc = inst.opcode == Opcode::VAddcU32 && opcode == 0x128u;
const bool vop3b_carry_out = IsVop3BCarryOutOpcode(inst.opcode);
const bool vop3b_mad_u64 = IsVop3BMadU64Opcode(inst.opcode);
const bool vop3b_uses_sdst = addc || vop3b_carry_out || vop3b_mad_u64;
const bool mad_mix = false;
const bool f16_ternary = IsNativeVop3F16TernaryOpcode(inst->opcode);
const bool b16_binary = IsNativeVop3B16BinaryOpcode(inst->opcode);
const bool pack_b32_f16 = inst->opcode == Opcode::VPackB32F16;
const bool permlane = IsPermlaneOpcode(inst->opcode);
const bool f16_ternary = IsNativeVop3F16TernaryOpcode(inst.opcode);
const bool b16_binary = IsNativeVop3B16BinaryOpcode(inst.opcode);
const bool pack_b32_f16 = inst.opcode == Opcode::VPackB32F16;
const bool permlane = IsPermlaneOpcode(inst.opcode);
const bool vop3_vopc = IsVop3EncodedVopc(opcode);
const bool compare_exec = vop3_vopc && IsVopcCompareExec(inst->opcode);
const bool scalar_dst = vop3_vopc || UsesScalarDestination(inst->opcode);
const bool scalar_modifier_limits = UsesScalarDestination(inst->opcode);
const bool native_source_modifiers = SupportsNativeVop3SourceModifiers(inst->opcode);
const bool native_result_modifiers = SupportsNativeVop3ResultModifiers(inst->opcode);
const bool compare_exec = vop3_vopc && IsVopcCompareExec(inst.opcode);
const bool scalar_dst = vop3_vopc || UsesScalarDestination(inst.opcode);
const bool scalar_modifier_limits = UsesScalarDestination(inst.opcode);
const bool native_source_modifiers = SupportsNativeVop3SourceModifiers(inst.opcode);
const bool native_result_modifiers = SupportsNativeVop3ResultModifiers(inst.opcode);
const bool modifiers =
HasUnsupportedNativeVop3Modifiers(inst->opcode, permlane, mad_mix, vop3b_uses_sdst,
HasUnsupportedNativeVop3Modifiers(inst.opcode, permlane, mad_mix, vop3b_uses_sdst,
scalar_modifier_limits, abs, op_sel, clamp, omod, neg);
if (modifiers) {
SetUnsupported(inst, Family::VOP3, opcode,
@@ -1608,98 +1608,98 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
: "VOP3 source modifiers are not implemented");
return true;
}
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VOP3, opcode, "VOP3 opcode is not implemented");
return true;
}
if (inst->opcode == Opcode::VNop) {
inst->dst.kind = OperandKind::Null;
inst->src_count = 0;
if (inst.opcode == Opcode::VNop) {
inst.dst.kind = OperandKind::Null;
inst.src_count = 0;
return true;
}
bool dst_ok = true;
if (compare_exec) {
inst->dst.kind = OperandKind::ExecLo;
inst.dst.kind = OperandKind::ExecLo;
} else if (scalar_dst) {
// VOP3A uses VDST for VOPC and the scalar-destination lane-read opcodes.
dst_ok = DecodeScalarDestination(vdst, pc, &inst->dst, error);
dst_ok = DecodeScalarDestination(vdst, pc, inst.dst, error);
} else {
dst_ok = DecodeVectorGpr(vdst, &inst->dst, error);
dst_ok = DecodeVectorGpr(vdst, inst.dst, error);
}
if (!dst_ok || !DecodeScalarSource(src0, pc, &inst->src0, error)) {
if (!dst_ok || !DecodeScalarSource(src0, pc, inst.src0, error)) {
return false;
}
inst->dst.clamp = native_result_modifiers && clamp != 0u;
inst->dst.omod = native_result_modifiers ? omod : 0u;
inst.dst.clamp = native_result_modifiers && clamp != 0u;
inst.dst.omod = native_result_modifiers ? omod : 0u;
if (permlane) {
inst->dst.op_sel = (op_sel & 0x1u) != 0u;
inst->dst.op_sel_hi = (op_sel & 0x2u) != 0u;
inst.dst.op_sel = (op_sel & 0x1u) != 0u;
inst.dst.op_sel_hi = (op_sel & 0x2u) != 0u;
}
if (vop3_vopc) {
if (!DecodeScalarSource(src1, pc, &inst->src1, error)) {
if (!DecodeScalarSource(src1, pc, inst.src1, error)) {
return false;
}
inst->src_count = 2;
inst.src_count = 2;
if (native_source_modifiers) {
ApplyNativeVop3SourceModifiers(inst, abs, neg);
}
return ReadLiteralOperands(code, word_index, inst, error);
}
if (IsVop3EncodedVop1(opcode)) {
inst->src_count = 1;
inst.src_count = 1;
if (native_source_modifiers) {
ApplyNativeVop3SourceModifiers(inst, abs, neg);
}
return ReadLiteralOperands(code, word_index, inst, error);
}
if (addc) {
if (!DecodeScalarDestination(sdst, pc, &inst->dst2, error) ||
!DecodeScalarSource(src1, pc, &inst->src1, error) ||
!DecodeScalarSource(src2, pc, &inst->src2, error)) {
if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) ||
!DecodeScalarSource(src1, pc, inst.src1, error) ||
!DecodeScalarSource(src2, pc, inst.src2, error)) {
return false;
}
inst->src_count = 3;
inst.src_count = 3;
return ReadLiteralOperands(code, word_index, inst, error);
}
if (vop3b_carry_out) {
if (!DecodeScalarDestination(sdst, pc, &inst->dst2, error) ||
!DecodeScalarSource(src1, pc, &inst->src1, error)) {
if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) ||
!DecodeScalarSource(src1, pc, inst.src1, error)) {
return false;
}
inst->src_count = 2;
inst.src_count = 2;
return ReadLiteralOperands(code, word_index, inst, error);
}
if (vop3b_mad_u64) {
if (!DecodeScalarDestination(sdst, pc, &inst->dst2, error) ||
!DecodeScalarSource(src1, pc, &inst->src1, error) ||
!DecodeScalarSource(src2, pc, &inst->src2, error)) {
if (!DecodeScalarDestination(sdst, pc, inst.dst2, error) ||
!DecodeScalarSource(src1, pc, inst.src1, error) ||
!DecodeScalarSource(src2, pc, inst.src2, error)) {
return false;
}
inst->src_count = 3;
inst.src_count = 3;
return ReadLiteralOperands(code, word_index, inst, error);
}
if (IsVop3EncodedVop2(opcode)) {
if (!DecodeScalarSource(src1, pc, &inst->src1, error)) {
if (!DecodeScalarSource(src1, pc, inst.src1, error)) {
return false;
}
if (inst->opcode == Opcode::VCndmaskB32) {
if (!DecodeScalarSource(src2, pc, &inst->src2, error)) {
if (inst.opcode == Opcode::VCndmaskB32) {
if (!DecodeScalarSource(src2, pc, inst.src2, error)) {
return false;
}
inst->src_count = 3;
inst.src_count = 3;
} else {
inst->src_count = 2;
inst.src_count = 2;
}
if (native_source_modifiers) {
ApplyNativeVop3SourceModifiers(inst, abs, neg);
}
return ReadLiteralOperands(code, word_index, inst, error);
}
if (!DecodeScalarSource(src1, pc, &inst->src1, error)) {
if (!DecodeScalarSource(src1, pc, inst.src1, error)) {
return false;
}
inst->src_count = NativeVop3SourceCount(inst->opcode);
if (inst->src_count > 2u && !DecodeScalarSource(src2, pc, &inst->src2, error)) {
inst.src_count = NativeVop3SourceCount(inst.opcode);
if (inst.src_count > 2u && !DecodeScalarSource(src2, pc, inst.src2, error)) {
return false;
}
if (mad_mix) {
@@ -1717,7 +1717,7 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
}
bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
Instruction& inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated VOP3P instruction at pc 0x{:08x}", pc);
@@ -1739,51 +1739,51 @@ bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
const uint32_t op_sel_hi = ((word1 >> 27u) & 0x3u) | (op_sel_hi_2 << 2u);
const uint32_t neg = (word1 >> 29u) & 0x7u;
inst->pc = pc;
inst->word = word0;
inst->family = Family::VOP3P;
inst->opcode_id = opcode;
inst->opcode = Lookup(VOP3P_OPS, static_cast<uint32_t>(std::size(VOP3P_OPS)), opcode);
inst.pc = pc;
inst.word = word0;
inst.family = Family::VOP3P;
inst.opcode_id = opcode;
inst.opcode = Lookup(VOP3P_OPS, static_cast<uint32_t>(std::size(VOP3P_OPS)), opcode);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P opcode is not implemented");
return true;
}
inst->src_count = Vop3pSourceCount(inst->opcode);
if (!DecodeVectorGpr(vdst, &inst->dst, error) ||
!DecodeScalarSource(src0, pc, &inst->src0, error) ||
!DecodeScalarSource(src1, pc, &inst->src1, error)) {
inst.src_count = Vop3pSourceCount(inst.opcode);
if (!DecodeVectorGpr(vdst, inst.dst, error) ||
!DecodeScalarSource(src0, pc, inst.src0, error) ||
!DecodeScalarSource(src1, pc, inst.src1, error)) {
return false;
}
if (inst->src_count > 2u && !DecodeScalarSource(src2, pc, &inst->src2, error)) {
if (inst.src_count > 2u && !DecodeScalarSource(src2, pc, inst.src2, error)) {
return false;
}
if (inst->opcode == Opcode::VFmaF32) {
inst->dst.clamp = clamp != 0u;
} else if (IsMadMixF16(inst->opcode)) {
inst->dst.clamp = clamp != 0u;
} else if (IsPackedVop3p(inst->opcode)) {
inst->dst.clamp = clamp != 0u;
if (inst.opcode == Opcode::VFmaF32) {
inst.dst.clamp = clamp != 0u;
} else if (IsMadMixF16(inst.opcode)) {
inst.dst.clamp = clamp != 0u;
} else if (IsPackedVop3p(inst.opcode)) {
inst.dst.clamp = clamp != 0u;
} else if (clamp != 0u) {
SetUnsupported(inst, Family::VOP3P, opcode, "VOP3P integer clamp is not implemented");
return true;
}
ApplyVop3pSourceModifiers(inst, op_sel, op_sel_hi, neg, neg_hi);
if (inst->opcode == Opcode::VMadMixloF16) {
if (inst.opcode == Opcode::VMadMixloF16) {
ApplyVop3pMixAbsModifiers(inst);
inst->dst.sdwa_sel = 4;
} else if (inst->opcode == Opcode::VMadMixhiF16) {
inst.dst.sdwa_sel = 4;
} else if (inst.opcode == Opcode::VMadMixhiF16) {
ApplyVop3pMixAbsModifiers(inst);
inst->dst.sdwa_sel = 5;
} else if (inst->opcode == Opcode::VFmaF32) {
inst.dst.sdwa_sel = 5;
} else if (inst.opcode == Opcode::VFmaF32) {
ApplyVop3pMixAbsModifiers(inst);
}
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
Instruction& inst, std::string* error) {
if (word_index >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated VINTRP instruction at pc 0x{:08x}", pc);
@@ -1798,35 +1798,35 @@ bool DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_ind
const uint32_t chan = (word >> 8u) & 0x3u;
const uint32_t vsrc = word & 0xffu;
inst->pc = pc;
inst->word = word;
inst->word_count = 1;
inst->family = Family::VINTRP;
inst->opcode_id = opcode;
inst->opcode = LookupVintrpOpcode(opcode);
inst.pc = pc;
inst.word = word;
inst.word_count = 1;
inst.family = Family::VINTRP;
inst.opcode_id = opcode;
inst.opcode = LookupVintrpOpcode(opcode);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
if (inst.opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::VINTRP, opcode, "VINTRP opcode is not implemented");
return true;
}
if (!DecodeVectorGpr(vdst, &inst->dst, error)) {
if (!DecodeVectorGpr(vdst, inst.dst, error)) {
return false;
}
if (inst->opcode == Opcode::VInterpMovF32) {
inst->src0.kind = OperandKind::IntegerInlineConstant;
inst->src0.value = vsrc & 0x3u;
inst->src0.signed_val = static_cast<int32_t>(inst->src0.value);
} else if (!DecodeVectorGpr(vsrc, &inst->src0, error)) {
if (inst.opcode == Opcode::VInterpMovF32) {
inst.src0.kind = OperandKind::IntegerInlineConstant;
inst.src0.value = vsrc & 0x3u;
inst.src0.signed_val = static_cast<int32_t>(inst.src0.value);
} else if (!DecodeVectorGpr(vsrc, inst.src0, error)) {
return false;
}
inst->src1.kind = OperandKind::IntegerInlineConstant;
inst->src1.value = attr;
inst->src1.signed_val = static_cast<int32_t>(attr);
inst->src2.kind = OperandKind::IntegerInlineConstant;
inst->src2.value = chan;
inst->src2.signed_val = static_cast<int32_t>(chan);
inst->src_count = 3;
inst.src1.kind = OperandKind::IntegerInlineConstant;
inst.src1.value = attr;
inst.src1.signed_val = static_cast<int32_t>(attr);
inst.src2.kind = OperandKind::IntegerInlineConstant;
inst.src2.value = chan;
inst.src2.signed_val = static_cast<int32_t>(chan);
inst.src_count = 3;
return true;
}
src/graphics/shader/recompiler/VectorAluOps.h
+6 -6
View File
@@ -5,18 +5,18 @@
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
std::string* error);
bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
Instruction& inst, std::string* error);
bool DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
Instruction& inst, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/shaderIR/ShaderIR.cpp
+207 -229
View File
@@ -32,17 +32,17 @@ Operand MakePcRelativeU32(uint32_t value) {
return operand;
}
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock* block,
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
if (!ScalarResultWritesSccNonZero(decoded.opcode)) {
return true;
}
if (block == nullptr || block->instructions.empty()) {
if (block.instructions.empty()) {
SetError(error, "internal IR error: missing scalar result for SCC update");
return false;
}
const auto result = block->instructions.back().dst;
const auto result = block.instructions.back().dst;
if (result.kind != OperandKind::Register) {
SetError(error, "internal IR error: scalar SCC update without register result");
return false;
@@ -58,7 +58,7 @@ bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBloc
scc_update.src[0] = result;
scc_update.src[1] = MakeImmediateU32(0);
scc_update.src_count = 2;
block->instructions.push_back(scc_update);
block.instructions.push_back(scc_update);
return true;
}
@@ -90,81 +90,76 @@ void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand) {
operand.dpp = decoded.dpp;
}
bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error) {
if (operand == nullptr) {
SetError(error, "internal IR error: null operand");
return false;
}
*operand = {};
CopyOperandModifiers(decoded, *operand);
bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error) {
operand = {};
CopyOperandModifiers(decoded, operand);
switch (decoded.kind) {
case Decoder::OperandKind::Sgpr:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Scalar;
operand->reg.index = decoded.reg;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Scalar;
operand.reg.index = decoded.reg;
return true;
case Decoder::OperandKind::Vgpr:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Vector;
operand->reg.index = decoded.reg;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Vector;
operand.reg.index = decoded.reg;
return true;
case Decoder::OperandKind::VccLo:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Vcc;
operand->reg.index = 0;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Vcc;
operand.reg.index = 0;
return true;
case Decoder::OperandKind::VccHi:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Vcc;
operand->reg.index = 1;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Vcc;
operand.reg.index = 1;
return true;
case Decoder::OperandKind::ExecLo:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Exec;
operand->reg.index = 0;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Exec;
operand.reg.index = 0;
return true;
case Decoder::OperandKind::ExecHi:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Exec;
operand->reg.index = 1;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Exec;
operand.reg.index = 1;
return true;
case Decoder::OperandKind::Scc:
case Decoder::OperandKind::VccZ:
case Decoder::OperandKind::ExecZ:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::Scc;
operand->reg.index = static_cast<uint32_t>(decoded.kind);
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::Scc;
operand.reg.index = static_cast<uint32_t>(decoded.kind);
return true;
case Decoder::OperandKind::M0:
operand->kind = OperandKind::Register;
operand->reg.file = RegisterFile::M0;
operand->reg.index = 0;
operand.kind = OperandKind::Register;
operand.reg.file = RegisterFile::M0;
operand.reg.index = 0;
return true;
case Decoder::OperandKind::Null: operand->kind = OperandKind::Null; return true;
case Decoder::OperandKind::Null: operand.kind = OperandKind::Null; return true;
default: SetError(error, "decoded operand cannot be used as an IR register"); return false;
}
}
bool LowerSourceOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error) {
bool LowerSourceOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error) {
switch (decoded.kind) {
case Decoder::OperandKind::LiteralConstant:
case Decoder::OperandKind::IntegerInlineConstant:
case Decoder::OperandKind::FloatInlineConstant:
*operand = {};
CopyOperandModifiers(decoded, *operand);
operand->kind = OperandKind::ImmediateU32;
operand->imm = decoded.value;
operand->sext_64 = decoded.kind == Decoder::OperandKind::IntegerInlineConstant &&
decoded.signed_val < 0;
operand = {};
CopyOperandModifiers(decoded, operand);
operand.kind = OperandKind::ImmediateU32;
operand.imm = decoded.value;
operand.sext_64 = decoded.kind == Decoder::OperandKind::IntegerInlineConstant &&
decoded.signed_val < 0;
return true;
case Decoder::OperandKind::Null:
case Decoder::OperandKind::PopsExitingWaveId:
*operand = {};
CopyOperandModifiers(decoded, *operand);
operand->kind = OperandKind::ImmediateU32;
operand->imm = 0;
operand = {};
CopyOperandModifiers(decoded, operand);
operand.kind = OperandKind::ImmediateU32;
operand.imm = 0;
return true;
default: return LowerRegisterOperand(decoded, operand, error);
}
@@ -284,7 +279,7 @@ bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
// dword index so s_buffer_load_dwordx2 vcc_lo writes vcc_lo, then vcc_hi.
uint32_t code = 0;
return TryGetScalarDestinationCode(operand, code) &&
Decoder::DecodeScalarDestination(code + index, 0, &decoded, nullptr);
Decoder::DecodeScalarDestination(code + index, 0, decoded, nullptr);
}
Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index) {
@@ -347,16 +342,16 @@ uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded) {
}
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock* block, std::string* error) {
BasicBlock& block, std::string* error) {
Instruction inst;
inst.pc = pc;
inst.op = Opcode::MoveU32;
inst.src[0] = MakeImmediateU32(value);
inst.src_count = 1;
if (!LowerRegisterOperand(dst, &inst.dst, error)) {
if (!LowerRegisterOperand(dst, inst.dst, error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -405,7 +400,7 @@ Decoder::Operand M0Operand() {
namespace {
bool LowerScalarSelect(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerScalarSelect(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
Decoder::Operand scc;
scc.kind = Decoder::OperandKind::Scc;
@@ -413,17 +408,17 @@ bool LowerScalarSelect(const Decoder::Instruction& decoded, BasicBlock* block, s
inst.pc = decoded.pc;
inst.op = Opcode::SelectU32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerRegisterOperand(scc, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerRegisterOperand(scc, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error) ||
!LowerSourceOperand(decoded.src1, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerScalarSelect64(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarSelect64(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Decoder::Operand scc;
scc.kind = Decoder::OperandKind::Scc;
@@ -432,13 +427,13 @@ bool LowerScalarSelect64(const Decoder::Instruction& decoded, BasicBlock* block,
inst.pc = decoded.pc;
inst.op = Opcode::SelectU64;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerRegisterOperand(scc, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerRegisterOperand(scc, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error) ||
!LowerSourceOperand(decoded.src1, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -452,7 +447,7 @@ uint32_t ScalarShiftLeftAddAmount(Decoder::Opcode opcode) {
}
}
bool LowerScalarShiftLeftAdd(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarShiftLeftAdd(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
const auto shift = ScalarShiftLeftAddAmount(decoded.opcode);
if (shift == 0) {
@@ -466,12 +461,12 @@ bool LowerScalarShiftLeftAdd(const Decoder::Instruction& decoded, BasicBlock* bl
inst.src_count = 3;
inst.src[1].kind = OperandKind::ImmediateU32;
inst.src[1].imm = shift;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -495,33 +490,33 @@ bool IsScalarMinMaxOpcode(Decoder::Opcode opcode) {
}
}
bool LowerScalarMinMaxWithScc(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarMinMaxWithScc(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction result;
result.pc = decoded.pc;
result.op = LookupIrOpcode(decoded.opcode);
result.src_count = 2;
if (!LowerRegisterOperand(decoded.dst, &result.dst, error) ||
!LowerSourceOperand(decoded.src0, &result.src[0], error) ||
!LowerSourceOperand(decoded.src1, &result.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, result.dst, error) ||
!LowerSourceOperand(decoded.src0, result.src[0], error) ||
!LowerSourceOperand(decoded.src1, result.src[1], error)) {
return false;
}
block->instructions.push_back(result);
block.instructions.push_back(result);
Instruction scc;
scc.pc = decoded.pc;
scc.op = ScalarMinMaxSccCompareOpcode(decoded.opcode);
scc.dst = MakeSccOperand();
scc.src_count = 2;
if (!LowerSourceOperand(decoded.src0, &scc.src[0], error) ||
!LowerSourceOperand(decoded.src1, &scc.src[1], error)) {
if (!LowerSourceOperand(decoded.src0, scc.src[0], error) ||
!LowerSourceOperand(decoded.src1, scc.src[1], error)) {
return false;
}
block->instructions.push_back(scc);
block.instructions.push_back(scc);
return true;
}
bool LowerScalarAddCarry(const Decoder::Instruction& decoded, BasicBlock* block, bool carry_in,
bool LowerScalarAddCarry(const Decoder::Instruction& decoded, BasicBlock& block, bool carry_in,
std::string* error) {
Decoder::Operand scc;
scc.kind = Decoder::OperandKind::Scc;
@@ -530,24 +525,24 @@ bool LowerScalarAddCarry(const Decoder::Instruction& decoded, BasicBlock* block,
inst.pc = decoded.pc;
inst.op = Opcode::ScalarAddCarryU32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error)) {
return false;
}
if (carry_in) {
if (!LowerRegisterOperand(scc, &inst.src[2], error)) {
if (!LowerRegisterOperand(scc, inst.src[2], error)) {
return false;
}
} else {
inst.src[2].kind = OperandKind::ImmediateU32;
inst.src[2].imm = 0;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerScalarSubBorrowCarry(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarSubBorrowCarry(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Decoder::Operand scc;
scc.kind = Decoder::OperandKind::Scc;
@@ -556,62 +551,62 @@ bool LowerScalarSubBorrowCarry(const Decoder::Instruction& decoded, BasicBlock*
inst.pc = decoded.pc;
inst.op = Opcode::ScalarSubBorrowCarryU32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerRegisterOperand(scc, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerRegisterOperand(scc, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerScalarBinaryScc(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerScalarBinaryScc(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.src_count = 2;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerScalarBitset0B32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarBitset0B32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::BitClearU32;
inst.src_count = 2;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerScalarBitset1B32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarBitset1B32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::BitSetU32;
inst.src_count = 2;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorCndmask(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorCndmask(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Decoder::Operand vcc;
vcc.kind = Decoder::OperandKind::VccLo;
@@ -624,32 +619,32 @@ bool LowerVectorCndmask(const Decoder::Instruction& decoded, BasicBlock* block,
? Opcode::SelectMaskF32Bits
: Opcode::SelectMaskU32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(mask, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(mask, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerSourceOperand(decoded.src0, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorMoveB32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorMoveB32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = decoded.src0.negate || decoded.src0.absolute ? Opcode::MoveF32Bits : Opcode::MoveU32;
inst.src_count = 1;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
ApplyDppDestinationMask(decoded, inst.dst);
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorMoveRelSource(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorMoveRelSource(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
if (decoded.dst.kind != Decoder::OperandKind::Vgpr ||
decoded.src0.kind != Decoder::OperandKind::Vgpr) {
@@ -668,16 +663,16 @@ bool LowerVectorMoveRelSource(const Decoder::Instruction& decoded, BasicBlock* b
inst.op = Opcode::MoveRelSourceU32;
inst.src_count = 2;
const auto m0 = M0Operand();
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(m0, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(m0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorMoveRelDestination(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorMoveRelDestination(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
if (decoded.dst.kind != Decoder::OperandKind::Vgpr) {
SetError(error, "V_MOVRELD_B32 requires VGPR destination");
@@ -695,33 +690,33 @@ bool LowerVectorMoveRelDestination(const Decoder::Instruction& decoded, BasicBlo
inst.op = Opcode::MoveRelDestU32;
inst.src_count = 2;
const auto m0 = M0Operand();
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(m0, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(m0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorAddCarry(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorAddCarry(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::IAddCarryU32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerRegisterOperand(decoded.dst2, &inst.dst2, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerRegisterOperand(decoded.dst2, inst.dst2, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -730,100 +725,100 @@ bool LowerVectorCarryOut(const Decoder::Instruction& decoded, BasicBlock* block,
inst.src_count = decoded.opcode == Decoder::Opcode::VAddI32 ? 3u : 2u;
const auto& src0 = decoded.opcode == Decoder::Opcode::VSubrevI32 ? decoded.src1 : decoded.src0;
const auto& src1 = decoded.opcode == Decoder::Opcode::VSubrevI32 ? decoded.src0 : decoded.src1;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerRegisterOperand(decoded.dst2, &inst.dst2, error) ||
!LowerSourceOperand(src0, &inst.src[0], error) ||
!LowerSourceOperand(src1, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerRegisterOperand(decoded.dst2, inst.dst2, error) ||
!LowerSourceOperand(src0, inst.src[0], error) ||
!LowerSourceOperand(src1, inst.src[1], error)) {
return false;
}
if (decoded.opcode == Decoder::Opcode::VAddI32) {
inst.src[2] = MakeImmediateU32(0);
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorMadU64U32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorMadU64U32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::UMadU64U32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerRegisterOperand(decoded.dst2, &inst.dst2, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerRegisterOperand(decoded.dst2, inst.dst2, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorMacF32(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerVectorMacF32(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::FMadF32;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerSourceOperand(decoded.dst, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerSourceOperand(decoded.dst, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorPkFmacF16(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorPkFmacF16(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::PackedFmaF16;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerSourceOperand(decoded.dst, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerSourceOperand(decoded.dst, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorFmacF16(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorFmacF16(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::FmaF16;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error) ||
!LowerSourceOperand(decoded.dst, &inst.src[2], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error) ||
!LowerSourceOperand(decoded.dst, inst.src[2], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorDot2cF32F16(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorDot2cF32F16(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::Dot2AccF32F16;
inst.src_count = 3;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, &inst.src[1], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error) ||
!LowerSourceOperand(decoded.src1, inst.src[1], error)) {
return false;
}
inst.src[2] = inst.dst;
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVectorByteConvert(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVectorByteConvert(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
const auto byte_index = VectorByteConvertIndex(decoded.opcode);
if (byte_index > 3u) {
@@ -837,23 +832,23 @@ bool LowerVectorByteConvert(const Decoder::Instruction& decoded, BasicBlock* blo
inst.src_count = 2;
inst.src[1].kind = OperandKind::ImmediateU32;
inst.src[1].imm = byte_index;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVInterpP1F32(const Decoder::Instruction& decoded, BasicBlock* block) {
bool LowerVInterpP1F32(const Decoder::Instruction& decoded, BasicBlock& block) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::ControlNop;
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
if (decoded.opcode == Decoder::Opcode::VInterpMovF32 && decoded.src0.value != 2u) {
if (error != nullptr) {
@@ -868,17 +863,15 @@ bool LowerVInterpLoadF32(const Decoder::Instruction& decoded, BasicBlock* block,
inst.op = Opcode::LoadInputF32;
inst.input_info.attr = decoded.src1.value;
inst.input_info.chan = decoded.src2.value;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t* index) {
if (index != nullptr) {
*index = 0;
}
ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t& index) {
index = 0;
switch (target) {
case 0x08u: return ExportTargetKind::MrtZ;
case 0x09u: return ExportTargetKind::Null;
@@ -886,30 +879,24 @@ ExportTargetKind ExportTargetKindFromTarget(uint32_t target, uint32_t* index) {
default: break;
}
if (target <= 0x07u) {
if (index != nullptr) {
*index = target;
}
index = target;
return ExportTargetKind::Mrt;
}
if (target >= 0x0cu && target <= 0x0fu) {
if (index != nullptr) {
*index = target - 0x0cu;
}
index = target - 0x0cu;
return ExportTargetKind::Position;
}
if (target >= 0x20u && target <= 0x3fu) {
if (index != nullptr) {
*index = target - 0x20u;
}
index = target - 0x20u;
return ExportTargetKind::Parameter;
}
return ExportTargetKind::Unknown;
}
bool LowerExportInstruction(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerExportInstruction(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
uint32_t index = 0;
const auto kind = ExportTargetKindFromTarget(decoded.exp.target, &index);
const auto kind = ExportTargetKindFromTarget(decoded.exp.target, index);
if (kind == ExportTargetKind::Unknown) {
if (error != nullptr) {
*error = fmt::format("unsupported EXP target 0x{:02x} at pc 0x{:08x}",
@@ -934,30 +921,30 @@ bool LowerExportInstruction(const Decoder::Instruction& decoded, BasicBlock* blo
const Decoder::Operand* sources[] = {&decoded.src0, &decoded.src1, &decoded.src2,
&decoded.src3};
for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
if (!LowerSourceOperand(*sources[i], &inst.src[i], error)) {
if (!LowerSourceOperand(*sources[i], inst.src[i], error)) {
return false;
}
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerControlMarker(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerControlMarker(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
bool has_imm, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.dst.kind = OperandKind::Null;
inst.src_count = has_imm ? 1u : 0u;
if (has_imm && !LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (has_imm && !LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerControlInstruction(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerControlInstruction(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
switch (decoded.opcode) {
case Decoder::Opcode::SNop:
@@ -979,23 +966,18 @@ bool LowerControlInstruction(const Decoder::Instruction& decoded, BasicBlock* bl
}
}
bool LowerScalarGetpcB64(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarGetpcB64(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
if (block == nullptr) {
SetError(error, "internal IR error: missing block for s_getpc_b64");
return false;
}
for (uint32_t i = 0; i < 2u; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::MoveU32;
inst.src[0] = i == 0u ? MakePcRelativeU32(decoded.pc + 4u) : MakeImmediateU32(0u);
inst.src_count = 1;
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error)) {
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
@@ -1020,7 +1002,7 @@ SaveexecMode ScalarSaveexecMode(Decoder::Opcode opcode) {
}
}
bool LowerScalarSaveexec(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerScalarSaveexec(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -1030,11 +1012,11 @@ bool LowerScalarSaveexec(const Decoder::Instruction& decoded, BasicBlock* block,
: Opcode::SaveexecB64;
inst.saveexec_mode = ScalarSaveexecMode(decoded.opcode);
inst.src_count = 1;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -1067,9 +1049,9 @@ bool IsTerminatorOpcode(Decoder::Opcode opcode) {
}
}
bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error);
bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error);
bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock* block,
bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock& block,
std::string* error) {
switch (inst.opcode) {
case Decoder::Opcode::Unsupported:
@@ -1144,47 +1126,43 @@ bool LowerDecodedInstruction(const Decoder::Instruction& inst, BasicBlock* block
return LowerImplemented(inst, block, error);
}
bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerImplemented(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = LookupIrOpcode(decoded.opcode);
inst.src_count = decoded.src_count;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) {
return false;
}
ApplyDppDestinationMask(decoded, inst.dst);
for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
const auto src_index =
(IsReversedBinary(decoded.opcode) && decoded.src_count == 2u) ? 1u - i : i;
if (!LowerSourceOperand(DecodedSourceAt(decoded, src_index), &inst.src[i], error)) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, src_index), inst.src[i], error)) {
return false;
}
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return AppendScalarResultSccNonZero(decoded, block, error);
}
} // namespace
bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage,
uint32_t wave_size, Program* program, std::string* error) {
if (program == nullptr) {
SetError(error, "invalid IR output");
return false;
}
uint32_t wave_size, Program& program, std::string* error) {
if (wave_size != 32u && wave_size != 64u) {
EXIT("invalid shader wave size: %u\n", wave_size);
}
*program = {};
program->stage = stage;
program->wave_size = wave_size;
program->dispatcher_fallback = cfg.irreducible;
program->cfg_failure_kind = cfg.failure_kind;
program->fallback_reason = cfg.unsupported_reason;
program->blocks.clear();
program = {};
program.stage = stage;
program.wave_size = wave_size;
program.dispatcher_fallback = cfg.irreducible;
program.cfg_failure_kind = cfg.failure_kind;
program.fallback_reason = cfg.unsupported_reason;
program.blocks.clear();
if (cfg.blocks.empty()) {
SetError(error, "cannot lower empty CFG");
@@ -1211,12 +1189,12 @@ bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, Shader
decoded.instructions[i].pc) != cfg.code_table_load_pcs.end()) {
continue;
}
if (!LowerDecodedInstruction(decoded.instructions[i], &block, error)) {
if (!LowerDecodedInstruction(decoded.instructions[i], block, error)) {
return false;
}
}
program->blocks.push_back(std::move(block));
program.blocks.push_back(std::move(block));
}
return true;
src/graphics/shader/recompiler/shaderIR/ShaderIRInternal.h
+6 -6
View File
@@ -17,12 +17,12 @@ bool ScalarResultIs64Bit(Decoder::Opcode opcode);
Operand MakeSccOperand();
Operand MakeImmediateU32(uint32_t value);
Operand MakePcRelativeU32(uint32_t value);
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock* block,
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error);
uint32_t VectorByteConvertIndex(Decoder::Opcode opcode);
void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand);
bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error);
bool LowerSourceOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error);
bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error);
bool LowerSourceOperand(const Decoder::Operand& decoded, Operand& operand, std::string* error);
void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst);
uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand);
MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind);
@@ -30,7 +30,7 @@ uint32_t RawScalarLoadBase(const Decoder::Operand& operand);
void ClearRegisterOffsetModifiers(Decoder::Operand& operand);
bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code);
bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
Decoder::Operand* result);
Decoder::Operand& result);
Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t dword_index);
@@ -38,7 +38,7 @@ uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& dec
uint32_t component_index);
uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded);
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock* block, std::string* error);
BasicBlock& block, std::string* error);
MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded, uint32_t component_index);
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset);
@@ -47,7 +47,7 @@ const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uin
Decoder::Operand M0Operand();
bool IsMemoryOpcode(Decoder::Opcode opcode);
bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/shaderIR/ShaderIRMemory.cpp
+88 -88
View File
@@ -10,7 +10,7 @@
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
bool LowerScalarBufferLoadDword(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerScalarBufferLoadDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
std::string* error) {
for (uint32_t i = 0; i < decoded.data_dwords; i++) {
Instruction inst;
@@ -22,38 +22,38 @@ bool LowerScalarBufferLoadDword(const Decoder::Instruction& decoded, BasicBlock*
// descriptor resource index.
inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
: ResourceIndexFromOperand(decoded.src0);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) ||
!LowerSourceOperand(decoded.src1, &inst.src[0], error)) {
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
!LowerSourceOperand(decoded.src1, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction* inst,
bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction& inst,
uint32_t first_src, std::string* error) {
uint32_t src = first_src;
if (decoded.idxen) {
if (!LowerSourceOperand(decoded.src0, &inst->src[src++], error)) {
if (!LowerSourceOperand(decoded.src0, inst.src[src++], error)) {
return false;
}
}
if (decoded.offen) {
const auto offset_source =
decoded.idxen ? OffsetDecodedRegister(decoded.src0, 1) : decoded.src0;
if (!LowerSourceOperand(offset_source, &inst->src[src++], error)) {
if (!LowerSourceOperand(offset_source, inst.src[src++], error)) {
return false;
}
}
if (!LowerSourceOperand(decoded.src2, &inst->src[src++], error)) {
if (!LowerSourceOperand(decoded.src2, inst.src[src++], error)) {
return false;
}
inst->src_count = src;
inst.src_count = src;
return true;
}
bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op =
decoded.data_bits == 8u
? (decoded.data_signed ? Opcode::BufferLoadSbyte : Opcode::BufferLoadUbyte)
@@ -75,16 +75,16 @@ bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock* block, std
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) ||
!LowerBufferAddressSources(decoded, &inst, 0, error)) {
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
!LowerBufferAddressSources(decoded, inst, 0, error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = decoded.data_bits == 8u ? Opcode::BufferStoreByte
: decoded.data_bits == 16u ? Opcode::BufferStoreShort
: Opcode::BufferStoreDword;
@@ -101,28 +101,28 @@ bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock* block, st
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), &inst.src[0], error) ||
!LowerBufferAddressSources(decoded, &inst, 1, error)) {
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), inst.src[0], error) ||
!LowerBufferAddressSources(decoded, inst, 1, error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerBufferAtomicDword(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerBufferAtomicDword(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
inst.dst.kind = OperandKind::Null;
if ((decoded.glc && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) ||
!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerBufferAddressSources(decoded, &inst, 1, error)) {
if ((decoded.glc && !LowerRegisterOperand(decoded.dst, inst.dst, error)) ||
!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerBufferAddressSources(decoded, inst, 1, error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -130,7 +130,7 @@ ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) {
return decoded.gds ? ResourceKind::Gds : ResourceKind::Lds;
}
bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = decoded.data_bits == 8u
? (decoded.data_signed ? Opcode::DsReadSbyte : Opcode::DsReadUbyte)
: decoded.data_bits == 16u
@@ -143,16 +143,16 @@ bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock* block, std::st
inst.op = ir_op;
inst.src_count = 1;
inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock* block, uint32_t dwords_per_read,
bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock& block, uint32_t dwords_per_read,
std::string* error) {
const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
for (uint32_t read = 0; read < 2u; read++) {
@@ -164,18 +164,18 @@ bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock* block, uint32
inst.src_count = 1;
inst.memory =
ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), &inst.dst,
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), inst.dst,
error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
}
return true;
}
bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
bool return_old, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -183,31 +183,31 @@ bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block, Op
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.dst.kind = return_old ? OperandKind::Register : OperandKind::Null;
if ((return_old && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) ||
!LowerSourceOperand(decoded.src1, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
if ((return_old && !LowerRegisterOperand(decoded.dst, inst.dst, error)) ||
!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerDsSwizzleB32(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerDsSwizzleB32(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsSwizzleB32;
inst.src_count = 2;
inst.src[1].kind = OperandKind::ImmediateU32;
inst.src[1].imm = decoded.offset & 0xffffu;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -216,17 +216,17 @@ bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock* bloc
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.resource = 0;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.src1, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, &inst.src[2], error)) {
if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, inst.src[2], error)) {
return false;
}
inst.src_count = 3;
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -236,15 +236,15 @@ bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock* bloc
inst.memory.resource = 0;
inst.dst.kind = OperandKind::Null;
const auto m0 = M0Operand();
if (!LowerSourceOperand(decoded.src1, &inst.src[0], error) ||
!LowerSourceOperand(m0, &inst.src[1], error)) {
if (!LowerSourceOperand(decoded.src1, inst.src[0], error) ||
!LowerSourceOperand(m0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -253,15 +253,15 @@ bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.resource = 0;
const auto m0 = M0Operand();
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(m0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(m0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock& block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -269,11 +269,11 @@ bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock* block
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
const auto m0 = M0Operand();
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(m0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(m0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -289,7 +289,7 @@ Opcode ImageAtomicIrOpcode(Decoder::Opcode opcode) {
}
}
bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -298,16 +298,16 @@ bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block,
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImageUint);
inst.memory.sampler = 0;
inst.dst.kind = OperandKind::Null;
if ((decoded.glc && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) ||
!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
if ((decoded.glc && !LowerRegisterOperand(decoded.dst, inst.dst, error)) ||
!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto kind = FlatSegmentResourceKind(decoded.memory_segment);
const auto ir_op = decoded.data_bits == 8u
? (decoded.data_signed ? Opcode::FlatLoadSbyte : Opcode::FlatLoadUbyte)
@@ -322,15 +322,15 @@ bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::
inst.src_count = decoded.src_count;
inst.memory = MemoryInfoFromDecoded(decoded, kind);
inst.memory.resource = 0;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error)) {
return false;
}
for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i], error)) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i], error)) {
return false;
}
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -341,20 +341,20 @@ bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::
inst.src_count = decoded.src_count;
inst.memory = OffsetMemoryInfo(decoded, kind, dword);
inst.memory.resource = 0;
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, dword), &inst.dst, error)) {
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, dword), inst.dst, error)) {
return false;
}
for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i], error)) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i], error)) {
return false;
}
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto kind = FlatSegmentResourceKind(decoded.memory_segment);
const auto ir_op = decoded.data_bits == 8u ? Opcode::FlatStoreByte
: decoded.data_bits == 16u ? Opcode::FlatStoreShort
@@ -368,20 +368,20 @@ bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock* block, std:
inst.memory = OffsetMemoryInfo(decoded, kind, dword);
inst.memory.resource = 0;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, dword), &inst.src[0], error)) {
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, dword), inst.src[0], error)) {
return false;
}
for (uint32_t i = 0; i < decoded.src_count && i + 1u < 3u; i++) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i + 1u], error)) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), inst.src[i + 1u], error)) {
return false;
}
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock& block, std::string* error) {
const auto ir_op = decoded.data_bits == 8u ? Opcode::DsWriteByte
: decoded.data_bits == 16u ? Opcode::DsWriteShort
: Opcode::DsWriteB32;
@@ -393,16 +393,16 @@ bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock* block, std::s
inst.src_count = 2;
inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
return true;
}
bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock& block,
uint32_t dwords_per_write, std::string* error) {
const Decoder::Operand* data[] = {&decoded.src1, &decoded.src2};
const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
@@ -415,18 +415,18 @@ bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock* block,
inst.memory =
ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), &inst.src[0],
if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), inst.src[0],
error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
}
}
return true;
}
bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
@@ -437,11 +437,11 @@ bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock* block,
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImage);
inst.memory.sampler = 0;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
if (!LowerSourceOperand(decoded.dst, inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
@@ -461,17 +461,17 @@ bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock* block,
: Opcode::ImageSample;
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Image);
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
if (!LowerRegisterOperand(decoded.dst, inst.dst, error) ||
!LowerSourceOperand(decoded.src0, inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
block.instructions.push_back(inst);
return true;
}
} // namespace
bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock* block,
bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock& block,
std::string* error) {
switch (decoded.opcode) {
case Decoder::Opcode::SLoadDword:
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAluOps.cpp
+1162 -1181
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAnalysis.cpp
+46 -46
View File
@@ -60,12 +60,12 @@ void SetError(std::string* error, const char* message) {
}
}
void CollectRegister(std::vector<RegisterBinding>* registers, IR::Register reg) {
if (std::any_of(registers->begin(), registers->end(),
void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg) {
if (std::any_of(registers.begin(), registers.end(),
[reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
return;
}
registers->push_back({reg, 0});
registers.push_back({reg, 0});
}
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
@@ -98,7 +98,7 @@ bool IsSccOperand(const IR::Operand& operand) {
bool IsCompareOpcode(IR::Opcode op);
void CollectMaskStateRegisters(std::vector<RegisterBinding>* registers) {
void CollectMaskStateRegisters(std::vector<RegisterBinding>& registers) {
CollectRegister(registers, {IR::RegisterFile::Exec, 0});
CollectRegister(registers, {IR::RegisterFile::Exec, 1});
CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
@@ -106,7 +106,7 @@ void CollectMaskStateRegisters(std::vector<RegisterBinding>* registers) {
CollectRegister(registers, SccRegister());
}
void CollectSequentialRegisters(std::vector<RegisterBinding>* registers, const IR::Operand& base,
void CollectSequentialRegisters(std::vector<RegisterBinding>& registers, const IR::Operand& base,
uint32_t count) {
if (base.kind != IR::OperandKind::Register) {
return;
@@ -129,8 +129,8 @@ uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& regis
}
void CollectMoveRelSourceRegisters(const IR::Program& program,
std::vector<RegisterBinding>* registers) {
const auto max_vector_end = MaxCollectedVectorRegisterEnd(*registers);
std::vector<RegisterBinding>& registers) {
const auto max_vector_end = MaxCollectedVectorRegisterEnd(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != IR::Opcode::MoveRelSourceU32 ||
@@ -184,7 +184,7 @@ uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
}
}
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>* registers) {
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>& registers) {
CollectMaskStateRegisters(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
@@ -267,16 +267,16 @@ bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind ki
});
}
void CopyProgramInputsAndOutputs(EmitterState* state, const IR::Program& program) {
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
for (const auto& input: program.info.inputs) {
state->inputs.push_back(
state.inputs.push_back(
{input.kind, input.location, input.component_count, 0, input.debug_name});
}
for (const auto& output: program.info.outputs) {
if (HasOutput(state->outputs, output.kind, output.index)) {
if (HasOutput(state.outputs, output.kind, output.index)) {
continue;
}
state->outputs.push_back(
state.outputs.push_back(
{output.kind, output.index, output.location, 0, output.debug_name});
}
}
@@ -528,23 +528,23 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
return 0;
}
uint32_t ConstantU32(EmitterState* state, uint32_t value);
void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value);
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state);
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane);
uint32_t ConstantU32(EmitterState& state, uint32_t value);
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* reason) {
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
state.program->shader_hash, static_cast<unsigned>(state.stage), resource,
state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
static_cast<unsigned>(kind), reason);
std::abort();
}
DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
IR::DescriptorBindingKind kind, uint32_t resource) {
const auto* descriptor = IR::FindBinding(state.program->bindings, kind);
const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
if (descriptor == nullptr) {
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
}
@@ -557,15 +557,15 @@ DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
}
uint32_t DescriptorElementPointer(EmitterState* state, uint32_t result_ptr_type,
uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name) {
if (variable_id == 0) {
ExitDescriptorBindingFailure(*state, kind, resource, variable_name);
ExitDescriptorBindingFailure(state, kind, resource, variable_name);
}
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction(
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
return pointer;
}
@@ -673,71 +673,71 @@ IR::DescriptorBindingKind StorageBindingKind(bool uint_image, ImageViewKind view
}
}
uint32_t LoadSampledImageDescriptor(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
ImageViewKind view) {
(void)use_pc;
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
const auto kind = SampledBindingKind(integer, view);
const auto binding = ResourceForDescriptor(*state, kind, mem.resource);
const auto& descriptors = state->sampled_images[SampledImageIndex(integer, view)];
const auto binding = ResourceForDescriptor(state, kind, mem.resource);
const auto& descriptors = state.sampled_images[SampledImageIndex(integer, view)];
const auto pointer = DescriptorElementPointer(
state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
mem.resource, "sampled image descriptor array was not emitted");
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, ImageViewImageType(*state, view, integer), image, pointer});
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
return image;
}
uint32_t LoadSamplerDescriptor(EmitterState* state, uint32_t sampler, uint32_t use_pc) {
uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
(void)use_pc;
const auto binding =
ResourceForDescriptor(*state, IR::DescriptorBindingKind::Samplers, sampler);
ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
const auto pointer = DescriptorElementPointer(
state, state->ptr_uniform_sampler, state->sampler_variable, binding.array_index,
state, state.ptr_uniform_sampler, state.sampler_variable, binding.array_index,
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
const auto sampler_id = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->sampler_type, sampler_id, pointer});
const auto sampler_id = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.sampler_type, sampler_id, pointer});
return sampler_id;
}
uint32_t MakeSampledImage(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
ImageViewKind view) {
const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view);
const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
if (image == 0 || sampler == 0) {
ExitDescriptorBindingFailure(
*state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
"sampled image or sampler descriptor load failed");
}
const auto sampled_image = state->builder.AllocateId();
state->builder.AddFunction(
const auto sampled_image = state.builder.AllocateId();
state.builder.AddFunction(
{OpSampledImage,
ImageViewSampledImageType(*state, view, mem.kind == IR::ResourceKind::ImageUint),
ImageViewSampledImageType(state, view, mem.kind == IR::ResourceKind::ImageUint),
sampled_image, image, sampler});
return sampled_image;
}
uint32_t StorageImageDescriptorPointer(EmitterState* state, uint32_t resource, bool uint_image,
uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
uint32_t use_pc, ImageViewKind view) {
(void)use_pc;
const auto kind = StorageBindingKind(uint_image, view);
const auto binding = ResourceForDescriptor(*state, kind, resource);
const auto ptr_type = StorageImagePointerType(*state, uint_image, view);
const auto variable = StorageImageVariable(*state, uint_image, view);
const auto binding = ResourceForDescriptor(state, kind, resource);
const auto ptr_type = StorageImagePointerType(state, uint_image, view);
const auto variable = StorageImageVariable(state, uint_image, view);
return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
"storage image descriptor array was not emitted");
}
uint32_t LoadStorageImageDescriptor(EmitterState* state, uint32_t resource, bool uint_image,
uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
uint32_t use_pc, ImageViewKind view) {
const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
if (pointer == 0) {
ExitDescriptorBindingFailure(*state, StorageBindingKind(uint_image, view), resource,
ExitDescriptorBindingFailure(state, StorageBindingKind(uint_image, view), resource,
"storage image descriptor pointer creation failed");
}
const auto type = StorageImageType(*state, uint_image, view);
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, type, image, pointer});
const auto type = StorageImageType(state, uint_image, view);
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, image, pointer});
return image;
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp
+212 -212
View File
@@ -2,8 +2,8 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
void ComputeReachableBlocks(EmitterState* state, const IR::Program& program) {
state->reachable_blocks.assign(program.blocks.size(), false);
void ComputeReachableBlocks(EmitterState& state, const IR::Program& program) {
state.reachable_blocks.assign(program.blocks.size(), false);
if (program.blocks.empty()) {
return;
}
@@ -13,38 +13,38 @@ void ComputeReachableBlocks(EmitterState* state, const IR::Program& program) {
while (!stack.empty()) {
const auto block_id = stack.back();
stack.pop_back();
if (block_id >= program.blocks.size() || state->reachable_blocks[block_id]) {
if (block_id >= program.blocks.size() || state.reachable_blocks[block_id]) {
continue;
}
state->reachable_blocks[block_id] = true;
state.reachable_blocks[block_id] = true;
for (auto succ: program.blocks[block_id].successors) {
if (succ < program.blocks.size() && !state->reachable_blocks[succ]) {
if (succ < program.blocks.size() && !state.reachable_blocks[succ]) {
stack.push_back(succ);
}
}
}
}
void AllocateBlockLabels(EmitterState* state, const IR::Program& program) {
void AllocateBlockLabels(EmitterState& state, const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.id < state->reachable_blocks.size() && state->reachable_blocks[block.id]) {
state->block_labels.emplace(block.id, state->builder.AllocateId());
if (block.id < state.reachable_blocks.size() && state.reachable_blocks[block.id]) {
state.block_labels.emplace(block.id, state.builder.AllocateId());
}
}
}
void AllocateDispatcherState(EmitterState* state, const IR::Program& program) {
state->dispatcher_fallback = program.dispatcher_fallback;
if (!state->dispatcher_fallback) {
void AllocateDispatcherState(EmitterState& state, const IR::Program& program) {
state.dispatcher_fallback = program.dispatcher_fallback;
if (!state.dispatcher_fallback) {
return;
}
state->dispatch_pc_variable = state->builder.AllocateId();
state->dispatch_header_label = state->builder.AllocateId();
state->dispatch_select_label = state->builder.AllocateId();
state->dispatch_default_label = state->builder.AllocateId();
state->dispatch_after_switch_label = state->builder.AllocateId();
state->dispatch_continue_label = state->builder.AllocateId();
state->dispatch_merge_label = state->builder.AllocateId();
state.dispatch_pc_variable = state.builder.AllocateId();
state.dispatch_header_label = state.builder.AllocateId();
state.dispatch_select_label = state.builder.AllocateId();
state.dispatch_default_label = state.builder.AllocateId();
state.dispatch_after_switch_label = state.builder.AllocateId();
state.dispatch_continue_label = state.builder.AllocateId();
state.dispatch_merge_label = state.builder.AllocateId();
}
uint32_t BlockLabel(const EmitterState& state, uint32_t block_id) {
@@ -52,7 +52,7 @@ uint32_t BlockLabel(const EmitterState& state, uint32_t block_id) {
return it == state.block_labels.end() ? 0 : it->second;
}
uint32_t EmitBranchCondition(EmitterState* state, CFG::BranchCondition condition) {
uint32_t EmitBranchCondition(EmitterState& state, CFG::BranchCondition condition) {
switch (condition) {
case CFG::BranchCondition::SccZero: return EmitSccBool(state, false);
case CFG::BranchCondition::SccNonZero: return EmitSccBool(state, true);
@@ -68,29 +68,29 @@ uint32_t EmitBranchCondition(EmitterState* state, CFG::BranchCondition condition
}
}
void EmitStructuredPrefix(EmitterState* state, const CFG::Terminator& term) {
void EmitStructuredPrefix(EmitterState& state, const CFG::Terminator& term) {
if (term.loop_header) {
const auto merge_label = BlockLabel(*state, term.merge_block);
const auto continue_label = BlockLabel(*state, term.continue_block);
const auto merge_label = BlockLabel(state, term.merge_block);
const auto continue_label = BlockLabel(state, term.continue_block);
if (merge_label != 0 && continue_label != 0) {
state->builder.AddFunction({OpLoopMerge, merge_label, continue_label, LoopControlNone});
state.builder.AddFunction({OpLoopMerge, merge_label, continue_label, LoopControlNone});
}
return;
}
if (term.kind == CFG::TerminatorKind::ConditionalBranch && term.merge_block != UINT32_MAX) {
const auto merge_label = BlockLabel(*state, term.merge_block);
const auto merge_label = BlockLabel(state, term.merge_block);
if (merge_label != 0) {
state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
}
}
}
void EmitReturn(EmitterState* state) {
void EmitReturn(EmitterState& state) {
EmitKillIfPixelValidMaskInactive(state);
state->builder.AddFunction({OpReturn});
state.builder.AddFunction({OpReturn});
}
void EmitTerminator(EmitterState* state, const CFG::Terminator& term) {
void EmitTerminator(EmitterState& state, const CFG::Terminator& term) {
const auto condition = term.kind == CFG::TerminatorKind::ConditionalBranch
? EmitBranchCondition(state, term.condition)
: 0;
@@ -98,19 +98,19 @@ void EmitTerminator(EmitterState* state, const CFG::Terminator& term) {
switch (term.kind) {
case CFG::TerminatorKind::Branch: {
const auto label = BlockLabel(*state, term.true_block);
const auto label = BlockLabel(state, term.true_block);
if (label != 0) {
state->builder.AddFunction({OpBranch, label});
state.builder.AddFunction({OpBranch, label});
} else {
EmitReturn(state);
}
break;
}
case CFG::TerminatorKind::ConditionalBranch: {
const auto true_label = BlockLabel(*state, term.true_block);
const auto false_label = BlockLabel(*state, term.false_block);
const auto true_label = BlockLabel(state, term.true_block);
const auto false_label = BlockLabel(state, term.false_block);
if (true_label != 0 && false_label != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpBranchConditional, condition, true_label, false_label});
} else {
EmitReturn(state);
@@ -122,10 +122,10 @@ void EmitTerminator(EmitterState* state, const CFG::Terminator& term) {
}
bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
if (state.program == nullptr || reg.file != IR::RegisterFile::Scalar) {
if (reg.file != IR::RegisterFile::Scalar) {
return false;
}
const auto& registers = state.program->bindings.user_data_registers;
const auto& registers = state.program.bindings.user_data_registers;
const auto found = std::lower_bound(registers.begin(), registers.end(), reg.index);
if (found != registers.end() && *found == reg.index) {
dword_index = static_cast<uint32_t>(found - registers.begin());
@@ -134,24 +134,24 @@ bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& d
return false;
}
uint32_t EmitVsharpDwordLoad(EmitterState* state, uint32_t dword_index) {
if (state->push_constant_variable != 0) {
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_push_constant_uint, pointer,
state->push_constant_variable, ConstantU32(state, 0),
uint32_t EmitVsharpDwordLoad(EmitterState& state, uint32_t dword_index) {
if (state.push_constant_variable != 0) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index / 4u),
ConstantU32(state, dword_index % 4u)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
if (state->vsharp_storage_variable != 0) {
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_storage_buffer_uint, pointer,
state->vsharp_storage_variable, ConstantU32(state, 0),
if (state.vsharp_storage_variable != 0) {
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
state.vsharp_storage_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
return ConstantU32(state, 0);
@@ -170,52 +170,52 @@ uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
return 0;
}
uint32_t EmitInitialRegisterValue(EmitterState* state, IR::Register reg) {
uint32_t EmitInitialRegisterValue(EmitterState& state, IR::Register reg) {
uint32_t dword_index = 0;
if (UserDataDwordIndex(*state, reg, dword_index)) {
if (UserDataDwordIndex(state, reg, dword_index)) {
return EmitVsharpDwordLoad(state, dword_index);
}
return ConstantU32(state, InitialRegisterValue(*state, reg));
return ConstantU32(state, InitialRegisterValue(state, reg));
}
void EmitRegisterVariables(EmitterState* state) {
if (state->needs_function_lds) {
state->builder.AddFunction(
{OpVariable, state->ptr_workgroup_array, state->lds_variable, StorageClassFunction});
void EmitRegisterVariables(EmitterState& state) {
if (state.needs_function_lds) {
state.builder.AddFunction(
{OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
}
for (const auto& binding: state->registers) {
state->builder.AddFunction(
{OpVariable, state->ptr_func_uint, binding.pointer_id, StorageClassFunction});
for (const auto& binding: state.registers) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
}
if (state->dispatcher_fallback) {
state->builder.AddFunction(
{OpVariable, state->ptr_func_uint, state->dispatch_pc_variable, StorageClassFunction});
if (state.dispatcher_fallback) {
state.builder.AddFunction(
{OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
}
if (state->pixel_valid_mask_variable != 0) {
state->builder.AddFunction({OpVariable, state->ptr_func_uint,
state->pixel_valid_mask_variable, StorageClassFunction});
state->builder.AddFunction(
{OpStore, state->pixel_valid_mask_variable, ConstantU32(state, 1)});
if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable, state.ptr_func_uint,
state.pixel_valid_mask_variable, StorageClassFunction});
state.builder.AddFunction(
{OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
}
for (const auto& binding: state->registers) {
state->builder.AddFunction(
for (const auto& binding: state.registers) {
state.builder.AddFunction(
{OpStore, binding.pointer_id, EmitInitialRegisterValue(state, binding.reg)});
}
}
void EmitComputeInputRegisters(EmitterState* state) {
if (state->stage != ShaderType::Compute || state->compute_input_info == nullptr) {
void EmitComputeInputRegisters(EmitterState& state) {
if (state.stage != ShaderType::Compute || state.compute_input_info == nullptr) {
return;
}
const auto* cs = state->compute_input_info;
const auto* cs = state.compute_input_info;
const auto thread_ids = cs->thread_ids_num > 0
? std::min<uint32_t>(static_cast<uint32_t>(cs->thread_ids_num), 3u)
: 0u;
for (uint32_t i = 0; i < thread_ids; i++) {
const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, i});
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, i});
if (pointer != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpStore, pointer,
EmitInputComponentU32(state, IR::StageInputKind::LocalInvocationId, i)});
}
@@ -227,10 +227,10 @@ void EmitComputeInputRegisters(EmitterState* state) {
continue;
}
const auto pointer = PointerForRegister(
*state,
state,
{IR::RegisterFile::Scalar, static_cast<uint32_t>(cs->workgroup_register) + reg_offset});
if (pointer != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpStore, pointer,
EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, i)});
}
@@ -239,7 +239,7 @@ void EmitComputeInputRegisters(EmitterState* state) {
if (cs->tg_size_en) {
const auto pointer = PointerForRegister(
*state,
state,
{IR::RegisterFile::Scalar, static_cast<uint32_t>(cs->workgroup_register) + reg_offset});
if (pointer != 0) {
const uint32_t wave_size = cs->wave_size != 0 ? cs->wave_size : 64u;
@@ -250,66 +250,66 @@ void EmitComputeInputRegisters(EmitterState* state) {
std::min<uint32_t>((total_threads + wave_size - 1u) / wave_size, 0x3fu);
const auto local_index = EmitLocalInvocationIndex(state);
const auto wave_id = state->builder.AllocateId();
const auto wave_bits = state->builder.AllocateId();
const auto is_first = state->builder.AllocateId();
const auto first_bit = state->builder.AllocateId();
const auto base = state->builder.AllocateId();
const auto packed = state->builder.AllocateId();
state->builder.AddFunction(
{OpUDiv, state->uint_type, wave_id, local_index, ConstantU32(state, wave_size)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, wave_bits, wave_id, ConstantU32(state, 20)});
state->builder.AddFunction(
{OpIEqual, state->bool_type, is_first, wave_id, ConstantU32(state, 0)});
state->builder.AddFunction({OpSelect, state->uint_type, first_bit, is_first,
const auto wave_id = state.builder.AllocateId();
const auto wave_bits = state.builder.AllocateId();
const auto is_first = state.builder.AllocateId();
const auto first_bit = state.builder.AllocateId();
const auto base = state.builder.AllocateId();
const auto packed = state.builder.AllocateId();
state.builder.AddFunction(
{OpUDiv, state.uint_type, wave_id, local_index, ConstantU32(state, wave_size)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, wave_bits, wave_id, ConstantU32(state, 20)});
state.builder.AddFunction(
{OpIEqual, state.bool_type, is_first, wave_id, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, state.uint_type, first_bit, is_first,
ConstantU32(state, 0x80000000u), ConstantU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseOr, state->uint_type, base, wave_bits, ConstantU32(state, waves)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, packed, base, first_bit});
state->builder.AddFunction({OpStore, pointer, packed});
state.builder.AddFunction(
{OpBitwiseOr, state.uint_type, base, wave_bits, ConstantU32(state, waves)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, packed, base, first_bit});
state.builder.AddFunction({OpStore, pointer, packed});
}
}
}
void EmitPixelFragCoordComponent(EmitterState* state, uint32_t component, uint32_t vgpr) {
const auto input = InputVariableForKind(*state, IR::StageInputKind::FragCoord);
const auto dst = PointerForRegister(*state, {IR::RegisterFile::Vector, vgpr});
void EmitPixelFragCoordComponent(EmitterState& state, uint32_t component, uint32_t vgpr) {
const auto input = InputVariableForKind(state, IR::StageInputKind::FragCoord);
const auto dst = PointerForRegister(state, {IR::RegisterFile::Vector, vgpr});
if (input == 0 || dst == 0) {
return;
}
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction(
{OpAccessChain, state->ptr_input_float, pointer, input, ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, state->float_type, value, pointer});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, value});
state->builder.AddFunction({OpStore, dst, bits});
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_input_float, pointer, input, ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
state.builder.AddFunction({OpStore, dst, bits});
}
void EmitPixelFrontFacingRegister(EmitterState* state, uint32_t vgpr) {
const auto input = InputVariableForKind(*state, IR::StageInputKind::FrontFacing);
const auto dst = PointerForRegister(*state, {IR::RegisterFile::Vector, vgpr});
void EmitPixelFrontFacingRegister(EmitterState& state, uint32_t vgpr) {
const auto input = InputVariableForKind(state, IR::StageInputKind::FrontFacing);
const auto dst = PointerForRegister(state, {IR::RegisterFile::Vector, vgpr});
if (input == 0 || dst == 0) {
return;
}
const auto value = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->bool_type, value, input});
state->builder.AddFunction(
{OpSelect, state->uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, dst, bits});
const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.bool_type, value, input});
state.builder.AddFunction(
{OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, dst, bits});
}
void EmitPixelInputRegisters(EmitterState* state) {
if (state->stage != ShaderType::Pixel || state->pixel_input_info == nullptr) {
void EmitPixelInputRegisters(EmitterState& state) {
if (state.stage != ShaderType::Pixel || state.pixel_input_info == nullptr) {
return;
}
const auto* ps = state->pixel_input_info;
const auto* ps = state.pixel_input_info;
uint32_t reg = ps->ps_system_input_base;
if (ps->HasPositionInput()) {
if (ps->ps_pos_x) {
@@ -334,37 +334,37 @@ void EmitPixelInputRegisters(EmitterState* state) {
}
}
uint32_t EmitInputScalarU32(EmitterState* state, IR::StageInputKind kind) {
const auto variable = InputVariableForKind(*state, kind);
uint32_t EmitInputScalarU32(EmitterState& state, IR::StageInputKind kind) {
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto raw = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->int_type, raw, variable});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw});
const auto raw = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.int_type, raw, variable});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
return bits;
}
void EmitVertexInputRegisters(EmitterState* state) {
if (state->stage != ShaderType::Vertex) {
void EmitVertexInputRegisters(EmitterState& state) {
if (state.stage != ShaderType::Vertex) {
return;
}
const auto vertex_index = PointerForRegister(*state, {IR::RegisterFile::Vector, 5});
const auto vertex_index = PointerForRegister(state, {IR::RegisterFile::Vector, 5});
if (vertex_index != 0) {
state->builder.AddFunction(
state.builder.AddFunction(
{OpStore, vertex_index, EmitInputScalarU32(state, IR::StageInputKind::VertexIndex)});
}
const auto instance_index = PointerForRegister(*state, {IR::RegisterFile::Vector, 8});
const auto instance_index = PointerForRegister(state, {IR::RegisterFile::Vector, 8});
if (instance_index != 0) {
state->builder.AddFunction({OpStore, instance_index,
state.builder.AddFunction({OpStore, instance_index,
EmitInputScalarU32(state, IR::StageInputKind::InstanceIndex)});
}
}
void EmitInstruction(EmitterState* state, const IR::Instruction& inst) {
void EmitInstruction(EmitterState& state, const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::ControlNop: EmitControlNop(state, inst); break;
case IR::Opcode::Waitcnt: EmitWaitcnt(state, inst); break;
@@ -846,7 +846,7 @@ void EmitInstruction(EmitterState* state, const IR::Instruction& inst) {
case IR::Opcode::ImageSample: EmitImageSample(state, inst); break;
case IR::Opcode::ImageGather4: EmitImageGather4(state, inst); break;
case IR::Opcode::Export:
if (ExportUsesPixelValidMask(*state, inst)) {
if (ExportUsesPixelValidMask(state, inst)) {
EmitUpdatePixelValidMask(state);
}
EmitGuardedByExec(state, [&]() { EmitExport(state, inst); });
@@ -855,16 +855,16 @@ void EmitInstruction(EmitterState* state, const IR::Instruction& inst) {
}
}
uint32_t DispatcherTargetValue(EmitterState* state, uint32_t block_id) {
return BlockLabel(*state, block_id) != 0 ? block_id : UINT32_MAX;
uint32_t DispatcherTargetValue(EmitterState& state, uint32_t block_id) {
return BlockLabel(state, block_id) != 0 ? block_id : UINT32_MAX;
}
void EmitDispatcherStoreTarget(EmitterState* state, uint32_t block_id) {
state->builder.AddFunction({OpStore, state->dispatch_pc_variable,
void EmitDispatcherStoreTarget(EmitterState& state, uint32_t block_id) {
state.builder.AddFunction({OpStore, state.dispatch_pc_variable,
ConstantU32(state, DispatcherTargetValue(state, block_id))});
}
void EmitDispatcherExit(EmitterState* state);
void EmitDispatcherExit(EmitterState& state);
bool RegisterForScalarCode(uint32_t code, IR::Register& reg) {
if (code < 106u) {
@@ -881,21 +881,21 @@ bool RegisterForScalarCode(uint32_t code, IR::Register& reg) {
}
}
bool EmitDispatcherLoadScalarCode(EmitterState* state, uint32_t code, uint32_t& value) {
bool EmitDispatcherLoadScalarCode(EmitterState& state, uint32_t code, uint32_t& value) {
IR::Register reg;
if (!RegisterForScalarCode(code, reg)) {
return false;
}
const auto pointer = PointerForRegister(*state, reg);
const auto pointer = PointerForRegister(state, reg);
if (pointer == 0) {
return false;
}
value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return true;
}
void EmitDispatcherStoreSelectorTarget(EmitterState* state, const CFG::Terminator& term) {
void EmitDispatcherStoreSelectorTarget(EmitterState& state, const CFG::Terminator& term) {
if (term.indirect_selector_code == UINT32_MAX || term.indirect_selector_values.empty() ||
term.indirect_selector_values.size() != term.indirect_selector_targets.size()) {
EmitDispatcherExit(state);
@@ -910,21 +910,21 @@ void EmitDispatcherStoreSelectorTarget(EmitterState* state, const CFG::Terminato
uint32_t selected = ConstantU32(state, UINT32_MAX);
for (uint32_t i = 0; i < term.indirect_selector_values.size(); i++) {
const auto match = state->builder.AllocateId();
state->builder.AddFunction({OpIEqual, state->bool_type, match, selector_value,
const auto match = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, match, selector_value,
ConstantU32(state, term.indirect_selector_values[i])});
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, next, match,
const auto next = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, next, match,
ConstantU32(state, DispatcherTargetValue(state, term.indirect_selector_targets[i])),
selected});
selected = next;
}
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
}
void EmitDispatcherStoreIndirectTarget(EmitterState* state, const CFG::Terminator& term) {
void EmitDispatcherStoreIndirectTarget(EmitterState& state, const CFG::Terminator& term) {
if (term.indirect_selector_code != UINT32_MAX) {
EmitDispatcherStoreSelectorTarget(state, term);
return;
@@ -936,55 +936,55 @@ void EmitDispatcherStoreIndirectTarget(EmitterState* state, const CFG::Terminato
}
const auto pointer =
PointerForRegister(*state, IR::Register {IR::RegisterFile::Scalar, term.indirect_pc_sgpr});
PointerForRegister(state, IR::Register {IR::RegisterFile::Scalar, term.indirect_pc_sgpr});
if (pointer == 0) {
EmitDispatcherExit(state);
return;
}
const auto pc_value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, pc_value, pointer});
const auto pc_value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, pc_value, pointer});
uint32_t selected = ConstantU32(state, UINT32_MAX);
const auto count = std::min(term.indirect_target_pcs.size(), term.indirect_targets.size());
for (uint32_t i = 0; i < count; i++) {
const auto match = state->builder.AllocateId();
state->builder.AddFunction({OpIEqual, state->bool_type, match, pc_value,
const auto match = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, state.bool_type, match, pc_value,
ConstantU32(state, term.indirect_target_pcs[i])});
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, next, match,
const auto next = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, next, match,
ConstantU32(state, DispatcherTargetValue(state, term.indirect_targets[i])), selected});
selected = next;
}
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
}
void EmitDispatcherExit(EmitterState* state) {
state->builder.AddFunction(
{OpStore, state->dispatch_pc_variable, ConstantU32(state, UINT32_MAX)});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
void EmitDispatcherExit(EmitterState& state) {
state.builder.AddFunction(
{OpStore, state.dispatch_pc_variable, ConstantU32(state, UINT32_MAX)});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
}
void EmitDispatcherTerminator(EmitterState* state, const CFG::Terminator& term) {
void EmitDispatcherTerminator(EmitterState& state, const CFG::Terminator& term) {
switch (term.kind) {
case CFG::TerminatorKind::Branch:
if (BlockLabel(*state, term.true_block) == 0) {
if (BlockLabel(state, term.true_block) == 0) {
EmitDispatcherExit(state);
return;
}
EmitDispatcherStoreTarget(state, term.true_block);
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
return;
case CFG::TerminatorKind::ConditionalBranch: {
const auto condition = EmitBranchCondition(state, term.condition);
const auto selected = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, selected, condition,
const auto selected = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, selected, condition,
ConstantU32(state, DispatcherTargetValue(state, term.true_block)),
ConstantU32(state, DispatcherTargetValue(state, term.false_block))});
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, selected});
state->builder.AddFunction({OpBranch, state->dispatch_after_switch_label});
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, selected});
state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
return;
}
case CFG::TerminatorKind::IndirectBranch:
@@ -994,44 +994,44 @@ void EmitDispatcherTerminator(EmitterState* state, const CFG::Terminator& term)
}
}
void EmitDispatcherSwitch(EmitterState* state, const IR::Program& program) {
state->builder.AddFunction({OpLabel, state->dispatch_header_label});
const auto pc_value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, pc_value, state->dispatch_pc_variable});
const auto done = state->builder.AllocateId();
state->builder.AddFunction(
{OpIEqual, state->bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)});
state->builder.AddFunction({OpLoopMerge, state->dispatch_merge_label,
state->dispatch_continue_label, LoopControlNone});
state->builder.AddFunction(
{OpBranchConditional, done, state->dispatch_merge_label, state->dispatch_select_label});
void EmitDispatcherSwitch(EmitterState& state, const IR::Program& program) {
state.builder.AddFunction({OpLabel, state.dispatch_header_label});
const auto pc_value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, pc_value, state.dispatch_pc_variable});
const auto done = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, state.bool_type, done, pc_value, ConstantU32(state, UINT32_MAX)});
state.builder.AddFunction({OpLoopMerge, state.dispatch_merge_label,
state.dispatch_continue_label, LoopControlNone});
state.builder.AddFunction(
{OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
state->builder.AddFunction({OpLabel, state->dispatch_select_label});
state->builder.AddFunction(
{OpSelectionMerge, state->dispatch_after_switch_label, SelectionControlNone});
std::vector<uint32_t> switch_words = {OpSwitch, pc_value, state->dispatch_default_label};
state.builder.AddFunction({OpLabel, state.dispatch_select_label});
state.builder.AddFunction(
{OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
std::vector<uint32_t> switch_words = {OpSwitch, pc_value, state.dispatch_default_label};
for (const auto& block: program.blocks) {
if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
const auto label = BlockLabel(*state, block.id);
const auto label = BlockLabel(state, block.id);
if (label != 0) {
switch_words.push_back(block.id);
switch_words.push_back(label);
}
}
state->builder.AddFunction(switch_words);
state.builder.AddFunction(switch_words);
state->builder.AddFunction({OpLabel, state->dispatch_default_label});
state.builder.AddFunction({OpLabel, state.dispatch_default_label});
EmitDispatcherExit(state);
}
void EmitDispatcherBlocks(EmitterState* state, const IR::Program& program) {
void EmitDispatcherBlocks(EmitterState& state, const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
state->builder.AddFunction({OpLabel, BlockLabel(*state, block.id)});
state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
for (const auto& inst: block.instructions) {
EmitInstruction(state, inst);
}
@@ -1039,57 +1039,57 @@ void EmitDispatcherBlocks(EmitterState* state, const IR::Program& program) {
}
}
void EmitDispatcherLoopTail(EmitterState* state) {
state->builder.AddFunction({OpLabel, state->dispatch_after_switch_label});
state->builder.AddFunction({OpBranch, state->dispatch_continue_label});
state->builder.AddFunction({OpLabel, state->dispatch_continue_label});
state->builder.AddFunction({OpBranch, state->dispatch_header_label});
state->builder.AddFunction({OpLabel, state->dispatch_merge_label});
void EmitDispatcherLoopTail(EmitterState& state) {
state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
state.builder.AddFunction({OpBranch, state.dispatch_header_label});
state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
EmitReturn(state);
}
void EmitDispatcherFunction(EmitterState* state, const IR::Program& program) {
state->builder.AddFunction({OpStore, state->dispatch_pc_variable, ConstantU32(state, 0)});
state->builder.AddFunction({OpBranch, state->dispatch_header_label});
void EmitDispatcherFunction(EmitterState& state, const IR::Program& program) {
state.builder.AddFunction({OpStore, state.dispatch_pc_variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpBranch, state.dispatch_header_label});
EmitDispatcherSwitch(state, program);
EmitDispatcherBlocks(state, program);
EmitDispatcherLoopTail(state);
}
void EmitFunction(EmitterState* state, const IR::Program& program) {
state->builder.AddFunction(
{OpFunction, state->void_type, state->main_func, FunctionControlNone, state->func_type});
state->builder.AddFunction({OpLabel, state->entry_label});
void EmitFunction(EmitterState& state, const IR::Program& program) {
state.builder.AddFunction(
{OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
state.builder.AddFunction({OpLabel, state.entry_label});
EmitRegisterVariables(state);
EmitComputeInputRegisters(state);
EmitPixelInputRegisters(state);
EmitVertexInputRegisters(state);
if (state->dispatcher_fallback) {
if (state.dispatcher_fallback) {
EmitDispatcherFunction(state, program);
state->builder.AddFunction({OpFunctionEnd});
state.builder.AddFunction({OpFunctionEnd});
return;
}
const auto entry_block_label = BlockLabel(*state, 0);
const auto entry_block_label = BlockLabel(state, 0);
if (entry_block_label != 0) {
state->builder.AddFunction({OpBranch, entry_block_label});
state.builder.AddFunction({OpBranch, entry_block_label});
} else {
EmitReturn(state);
}
for (const auto& block: program.blocks) {
if (block.id >= state->reachable_blocks.size() || !state->reachable_blocks[block.id]) {
if (block.id >= state.reachable_blocks.size() || !state.reachable_blocks[block.id]) {
continue;
}
state->builder.AddFunction({OpLabel, BlockLabel(*state, block.id)});
state.builder.AddFunction({OpLabel, BlockLabel(state, block.id)});
for (const auto& inst: block.instructions) {
EmitInstruction(state, inst);
}
EmitTerminator(state, block.terminator);
}
state->builder.AddFunction({OpFunctionEnd});
state.builder.AddFunction({OpFunctionEnd});
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterExport.cpp
+59 -59
View File
@@ -2,7 +2,7 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
uint32_t component) {
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
if (!enabled || component >= inst.src_count || component >= 4u) {
@@ -11,7 +11,7 @@ uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst
return EmitFloatLoad(state, inst.src[component]);
}
uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) {
if (inst.export_info.compr) {
uint32_t components[4] = {
ConstantF32(state, 0u),
@@ -22,21 +22,21 @@ uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
pair_index++) {
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
const auto unpacked = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked,
state->glsl_std450, GlslUnpackHalf2x16, raw});
const auto unpacked = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
state.glsl_std450, GlslUnpackHalf2x16, raw});
for (uint32_t lane = 0; lane < 2u; lane++) {
const auto component = pair_index * 2u + lane;
if (((inst.export_info.en >> component) & 1u) == 0) {
continue;
}
components[component] = state->builder.AllocateId();
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, components[component], unpacked, lane});
components[component] = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, components[component], unpacked, lane});
}
}
const auto vec = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec,
const auto vec = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec,
components[0], components[1], components[2], components[3]});
return vec;
}
@@ -45,24 +45,24 @@ uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
const auto y = EmitExportComponentF32(state, inst, 1);
const auto z = EmitExportComponentF32(state, inst, 2);
const auto w = EmitExportComponentF32(state, inst, 3);
const auto vec = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec, x, y, z, w});
const auto vec = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w});
return vec;
}
uint32_t ApplyMrtExportMapping(EmitterState* state, const IR::Instruction& inst, uint32_t value) {
if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state->pixel_input_info == nullptr ||
inst.export_info.index >= state->pixel_input_info->target_export_mapping.size()) {
uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value) {
if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state.pixel_input_info == nullptr ||
inst.export_info.index >= state.pixel_input_info->target_export_mapping.size()) {
return value;
}
const auto mapping = state->pixel_input_info->target_export_mapping[inst.export_info.index];
const auto mapping = state.pixel_input_info->target_export_mapping[inst.export_info.index];
if (mapping.IsIdentity()) {
return value;
}
const auto mapped = state->builder.AllocateId();
state->builder.AddFunction({OpVectorShuffle, state->vec4_float_type, mapped, value, value,
const auto mapped = state.builder.AllocateId();
state.builder.AddFunction({OpVectorShuffle, state.vec4_float_type, mapped, value, value,
mapping.Map(0), mapping.Map(1), mapping.Map(2), mapping.Map(3)});
return mapped;
}
@@ -76,25 +76,25 @@ bool ExportWritesData(const IR::Instruction& inst) {
}
}
void EmitMrtZExport(EmitterState* state, const IR::Instruction& inst) {
if ((inst.export_info.en & 0x1u) != 0 && state->depth_variable != 0) {
state->builder.AddFunction(
{OpStore, state->depth_variable, EmitExportComponentF32(state, inst, 0)});
void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) {
if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) {
state.builder.AddFunction(
{OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)});
}
if ((inst.export_info.en & 0x4u) != 0 && state->sample_mask_variable != 0) {
if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) {
const auto raw =
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
const auto mask = state->builder.AllocateId();
const auto ptr = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, mask, raw});
state->builder.AddFunction({OpAccessChain, state->ptr_output_int, ptr,
state->sample_mask_variable, ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, ptr, mask});
const auto mask = state.builder.AllocateId();
const auto ptr = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, mask, raw});
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr,
state.sample_mask_variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, ptr, mask});
}
}
void EmitExport(EmitterState* state, const IR::Instruction& inst) {
void EmitExport(EmitterState& state, const IR::Instruction& inst) {
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
return;
@@ -109,62 +109,62 @@ void EmitExport(EmitterState* state, const IR::Instruction& inst) {
return;
}
const auto variable = OutputVariableForExport(*state, inst.export_info);
const auto variable = OutputVariableForExport(state, inst.export_info);
if (variable == 0) {
return;
}
const auto value = ApplyMrtExportMapping(state, inst, EmitExportVec4F32(state, inst));
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_output_vec4_float, pointer, variable,
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer, variable,
ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, pointer, value});
state.builder.AddFunction({OpStore, pointer, value});
return;
}
state->builder.AddFunction({OpStore, variable, value});
state.builder.AddFunction({OpStore, variable, value});
}
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
}
void EmitKillIfBoolFalse(EmitterState* state, uint32_t active) {
const auto kill_label = state->builder.AllocateId();
const auto merge_label = state->builder.AllocateId();
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state->builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
state->builder.AddFunction({OpLabel, kill_label});
state->builder.AddFunction({OpKill});
state->builder.AddFunction({OpLabel, merge_label});
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
const auto kill_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId();
const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
state.builder.AddFunction({OpLabel, kill_label});
state.builder.AddFunction({OpKill});
state.builder.AddFunction({OpLabel, merge_label});
}
void EmitUpdatePixelValidMask(EmitterState* state) {
if (state->pixel_valid_mask_variable == 0) {
void EmitUpdatePixelValidMask(EmitterState& state) {
if (state.pixel_valid_mask_variable == 0) {
return;
}
const auto active = EmitExecActiveBool(state);
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, state->pixel_valid_mask_variable, value});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
}
void EmitKillIfPixelValidMaskInactive(EmitterState* state) {
if (state->pixel_valid_mask_variable == 0) {
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
if (state.pixel_valid_mask_variable == 0) {
return;
}
const auto mask_value = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpLoad, state->uint_type, mask_value, state->pixel_valid_mask_variable});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, mask_value, ConstantU32(state, 0)});
const auto mask_value = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
EmitKillIfBoolFalse(state, active);
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterFlowOps.cpp
+158 -158
View File
@@ -2,15 +2,15 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
void EmitMoveU32(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
EmitStoreU32(state, inst.dst, value);
}
void EmitMoveF32Bits(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitFloatLoad(state, inst.src[0]);
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, value});
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
EmitStoreU32(state, inst.dst, bits);
}
@@ -24,7 +24,7 @@ uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
return std::min(limit, 256u);
}
void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) {
if (inst.src[0].kind != IR::OperandKind::Register ||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
EmitMoveU32(state, inst);
@@ -34,20 +34,20 @@ void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) {
const auto base_reg = inst.src[0].reg;
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
auto selected = EmitRegisterLoad(state, base_reg);
const auto limit = MoveRelRegisterLimit(*state, base_reg);
const auto limit = MoveRelRegisterLimit(state, base_reg);
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
const auto match = state->builder.AllocateId();
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state->builder.AddFunction({OpSelect, state->uint_type, next, match, candidate, selected});
const auto match = state.builder.AllocateId();
const auto next = state.builder.AllocateId();
state.builder.AddFunction(
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected});
selected = next;
}
EmitStoreU32(state, inst.dst, selected);
}
void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) {
void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) {
if (inst.dst.kind != IR::OperandKind::Register ||
inst.dst.reg.file != IR::RegisterFile::Vector) {
EmitMoveU32(state, inst);
@@ -58,27 +58,27 @@ void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
const auto exec_active = EmitExecActiveBool(state);
const auto limit = MoveRelRegisterLimit(*state, base_reg);
const auto limit = MoveRelRegisterLimit(state, base_reg);
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, reg});
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg});
if (pointer == 0) {
continue;
}
const auto old_value = state->builder.AllocateId();
const auto match = state->builder.AllocateId();
const auto write = state->builder.AllocateId();
const auto merged = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer});
state->builder.AddFunction(
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state->builder.AddFunction({OpLogicalAnd, state->bool_type, write, exec_active, match});
state->builder.AddFunction({OpSelect, state->uint_type, merged, write, value, old_value});
state->builder.AddFunction({OpStore, pointer, merged});
const auto old_value = state.builder.AllocateId();
const auto match = state.builder.AllocateId();
const auto write = state.builder.AllocateId();
const auto merged = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
state.builder.AddFunction(
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match});
state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value});
state.builder.AddFunction({OpStore, pointer, merged});
}
}
void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) {
if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
IsMaskRegisterFile(inst.dst.reg.file)) {
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
return;
@@ -89,28 +89,28 @@ void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
}
uint32_t EmitWqmLaneU32(EmitterState* state, uint32_t src) {
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
uint32_t ret = ConstantU32(state, 0);
for (uint32_t i = 0; i < 8u; i++) {
const auto mask = 0x0fu << (i * 4u);
const auto masked = state->builder.AllocateId();
const auto non_zero = state->builder.AllocateId();
const auto expanded = state->builder.AllocateId();
const auto combined = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, masked, src, ConstantU32(state, mask)});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, non_zero, masked, ConstantU32(state, 0)});
state->builder.AddFunction({OpSelect, state->uint_type, expanded, non_zero,
const auto masked = state.builder.AllocateId();
const auto non_zero = state.builder.AllocateId();
const auto expanded = state.builder.AllocateId();
const auto combined = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
ConstantU32(state, mask), ConstantU32(state, 0)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, ret, expanded});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
ret = combined;
}
return ret;
}
void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) {
if (!state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) {
if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
@@ -120,44 +120,44 @@ void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
return;
}
const auto ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup),
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
const auto low = state->builder.AllocateId();
const auto high = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, ballot, 0});
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, ballot, 1});
const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
const auto wqm_low = EmitWqmLaneU32(state, low);
const auto wqm_high = EmitWqmLaneU32(state, high);
const auto lane = EmitSubgroupLocalInvocationId(state);
const auto high_half = state->builder.AllocateId();
const auto mask = state->builder.AllocateId();
const auto bit_index = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
const auto hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, high_half, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, mask, high_half, wqm_high, wqm_low});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, bit_index, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), bit_index});
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
if (state->per_invocation_masks) {
const auto high_half = state.builder.AllocateId();
const auto mask = state.builder.AllocateId();
const auto bit_index = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index});
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
if (state.per_invocation_masks) {
EmitPerInvocationMask(state, inst.dst, active);
} else {
const auto result = state->builder.AllocateId();
state->builder.AddFunction({OpSelect, state->uint_type, result, active,
const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, state.uint_type, result, active,
ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, inst.dst, result);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
}
}
void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst) {
void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) {
const auto old_exec = EmitExecActiveBool(state);
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
EmitPerInvocationMask(state, inst.dst, old_exec);
@@ -179,8 +179,8 @@ void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst)
EmitStoreSccBool(state, result);
}
void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks) {
void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) {
if (state.per_invocation_masks) {
EmitSaveexecPerInvocation(state, inst);
return;
}
@@ -191,29 +191,29 @@ void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) {
uint32_t rhs_low = src_low;
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
rhs_low = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
rhs_low = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
}
const auto new_low = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, new_low, rhs_low, old_low});
const auto new_low = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low});
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
if (state->wave_size == 32u) {
if (state.wave_size == 32u) {
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
}
const auto cond = state->builder.AllocateId();
const auto scc = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, cond, new_low, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
const auto cond = state.builder.AllocateId();
const auto scc = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, SccOperand(), scc);
}
void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks) {
void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) {
if (state.per_invocation_masks) {
EmitSaveexecPerInvocation(state, inst);
return;
}
@@ -231,150 +231,150 @@ void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) {
uint32_t rhs_high = src_high;
uint32_t opcode = OpBitwiseAnd;
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
rhs_low = state->builder.AllocateId();
rhs_high = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
state->builder.AddFunction({OpNot, state->uint_type, rhs_high, src_high});
rhs_low = state.builder.AllocateId();
rhs_high = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high});
}
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
lhs_low = state->builder.AllocateId();
lhs_high = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, lhs_low, old_low});
state->builder.AddFunction({OpNot, state->uint_type, lhs_high, old_high});
lhs_low = state.builder.AllocateId();
lhs_high = state.builder.AllocateId();
state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low});
state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high});
opcode = OpBitwiseOr;
}
const auto new_low = state->builder.AllocateId();
const auto new_high = state->builder.AllocateId();
state->builder.AddFunction({opcode, state->uint_type, new_low, lhs_low, rhs_low});
state->builder.AddFunction({opcode, state->uint_type, new_high, lhs_high, rhs_high});
const auto new_low = state.builder.AllocateId();
const auto new_high = state.builder.AllocateId();
state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low});
state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high});
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
const auto active_new_high = state->wave_size == 32u ? ConstantU32(state, 0) : new_high;
const auto mask = state->builder.AllocateId();
const auto cond = state->builder.AllocateId();
const auto scc = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, mask, new_low, active_new_high});
state->builder.AddFunction({OpINotEqual, state->bool_type, cond, mask, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high;
const auto mask = state.builder.AllocateId();
const auto cond = state.builder.AllocateId();
const auto scc = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high});
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, SccOperand(), scc);
}
void EmitReadFirstLaneU32(EmitterState* state, const IR::Instruction& inst) {
void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) {
const auto src = EmitValueLoad(state, inst.src[0]);
const auto active = EmitExecActiveBool(state);
const auto ballot = state->builder.AllocateId();
const auto first_lane = state->builder.AllocateId();
const auto first_value = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
const auto ballot = state.builder.AllocateId();
const auto first_lane = state.builder.AllocateId();
const auto first_value = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), active});
state->builder.AddFunction({OpGroupNonUniformBallotFindLSB, state->uint_type, first_lane,
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane,
ConstantU32(state, ScopeSubgroup), ballot});
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, first_value,
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value,
ConstantU32(state, ScopeSubgroup), src, first_lane});
EmitStoreU32(state, inst.dst, first_value);
}
uint32_t EmitLaneIndex(EmitterState* state, const IR::Operand& operand) {
const auto lane = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, lane, EmitValueLoad(state, operand),
uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) {
const auto lane = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand),
ConstantU32(state, 63)});
return lane;
}
void EmitReadLaneU32(EmitterState* state, const IR::Instruction& inst) {
void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) {
const auto src = EmitValueLoad(state, inst.src[0]);
const auto lane = EmitLaneIndex(state, inst.src[1]);
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, value,
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
ConstantU32(state, ScopeSubgroup), src, lane});
EmitStoreU32(state, inst.dst, value);
}
void EmitWriteLaneU32(EmitterState* state, const IR::Instruction& inst) {
void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
const auto lane = EmitLaneIndex(state, inst.src[1]);
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto hit = state->builder.AllocateId();
const auto hit = state.builder.AllocateId();
const auto pointer =
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(*state, inst.dst.reg) : 0;
const auto old = pointer != 0 ? state->builder.AllocateId() : ConstantU32(state, 0);
const auto merged = state->builder.AllocateId();
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0;
const auto old = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0);
const auto merged = state.builder.AllocateId();
if (pointer != 0) {
state->builder.AddFunction({OpLoad, state->uint_type, old, pointer});
state.builder.AddFunction({OpLoad, state.uint_type, old, pointer});
}
state->builder.AddFunction({OpIEqual, state->bool_type, hit, subid, lane});
state->builder.AddFunction({OpSelect, state->uint_type, merged, hit, value, old});
state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane});
state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old});
if (pointer != 0) {
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
state->builder.AddFunction({OpStore, pointer, merged});
state.builder.AddFunction({OpStore, pointer, merged});
}
}
void EmitPermlaneB32(EmitterState* state, const IR::Instruction& inst, bool x16) {
void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto lane8 = state->builder.AllocateId();
const auto shift = state->builder.AllocateId();
const auto upper = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto index0 = state->builder.AllocateId();
const auto index1 = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
const auto shuffled = state->builder.AllocateId();
const auto row = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto lane8 = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto upper = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto index0 = state.builder.AllocateId();
const auto index1 = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
const auto shuffled = state.builder.AllocateId();
const auto value = EmitValueLoad(state, inst.src[0]);
const auto src1 = EmitValueLoad(state, inst.src[1]);
const auto src2 = EmitValueLoad(state, inst.src[2]);
uint32_t row_value = row;
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
if (x16) {
row_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseXor, state->uint_type, row_value, row, ConstantU32(state, 16)});
row_value = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
}
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane8, lane, ConstantU32(state, 7)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, shift, lane8, ConstantU32(state, 2)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, upper, lane, ConstantU32(state, 8)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, upper, src2, src1});
state->builder.AddFunction({OpShiftRightLogical, state->uint_type, index0, selected, shift});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, index1, index0, ConstantU32(state, 15)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row_value, index1});
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1});
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1});
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), value, target});
uint32_t ret = shuffled;
if (!inst.dst.op_sel) {
const auto source_active = EmitLaneIndexActiveBool(state, target);
ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
}
EmitStoreU32(state, inst.dst, ret);
}
void EmitControlNop(EmitterState* state, const IR::Instruction& inst) {
void EmitControlNop(EmitterState& state, const IR::Instruction& inst) {
(void)state;
(void)inst;
}
void EmitWaitcnt(EmitterState* state, const IR::Instruction& inst) {
void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) {
(void)state;
(void)inst;
}
void EmitBarrier(EmitterState* state, const IR::Instruction& inst) {
void EmitBarrier(EmitterState& state, const IR::Instruction& inst) {
(void)inst;
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
state->builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageHelpers.cpp
+86 -86
View File
@@ -33,25 +33,25 @@ ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKi
return layout;
}
uint32_t EmitImageCoordF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, ImageViewKind view) {
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
const auto y = inst.memory.image_address_components > layout.coord + 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
: EmitZeroF32(state);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > layout.coord + 2u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
: EmitZeroF32(state);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
@@ -60,7 +60,7 @@ uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
}
uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
return EmitZeroF32(state);
@@ -68,7 +68,7 @@ uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
}
uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
return EmitZeroF32(state);
@@ -76,7 +76,7 @@ uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
}
uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
uint32_t first_component) {
const auto x = inst.memory.image_address_components > first_component
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
@@ -84,37 +84,37 @@ uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst,
const auto y = inst.memory.image_address_components > first_component + 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
: EmitZeroF32(state);
const auto grad = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, grad, x, y});
const auto grad = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
return grad;
}
uint32_t EmitImagePackedOffset2I32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImagePackedOffset2I32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.offset == NoImageComponent ||
inst.memory.image_address_components <= layout.offset) {
const auto zero = ConstantI32(state, 0);
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec2_int_type, ret, zero, zero});
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
return ret;
}
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
const auto packed_i32 = state->builder.AllocateId();
const auto offset_x = state->builder.AllocateId();
const auto offset_y = state->builder.AllocateId();
const auto offset = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, packed_i32, packed_bits});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_x, packed_i32,
const auto packed_i32 = state.builder.AllocateId();
const auto offset_x = state.builder.AllocateId();
const auto offset_y = state.builder.AllocateId();
const auto offset = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
ConstantI32(state, 0), ConstantI32(state, 6)});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_y, packed_i32,
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
ConstantI32(state, 8), ConstantI32(state, 6)});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec2_int_type, offset, offset_x, offset_y});
state.builder.AddFunction(
{OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y});
return offset;
}
uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageOffsetCoordF32(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, uint32_t sampled_image,
uint32_t coord, ImageViewKind view) {
if (!HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
@@ -122,92 +122,92 @@ uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& ins
}
const auto offset_i32 = EmitImagePackedOffset2I32(state, inst, layout);
const auto offset_f32 = state->builder.AllocateId();
const auto image = state->builder.AllocateId();
const auto size_u32 = state->builder.AllocateId();
const auto offset_f32 = state.builder.AllocateId();
const auto image = state.builder.AllocateId();
const auto size_u32 = state.builder.AllocateId();
const auto size_xy_u32 =
ImageViewCoordinateComponents(view) == 3u ? state->builder.AllocateId() : size_u32;
const auto size_f32 = state->builder.AllocateId();
const auto normalized = state->builder.AllocateId();
const auto adjusted = state->builder.AllocateId();
state->builder.AddFunction({OpConvertSToF, state->vec2_float_type, offset_f32, offset_i32});
state->builder.AddFunction(
{OpImage, ImageViewImageType(*state, view, inst.memory.kind == IR::ResourceKind::ImageUint),
ImageViewCoordinateComponents(view) == 3u ? state.builder.AllocateId() : size_u32;
const auto size_f32 = state.builder.AllocateId();
const auto normalized = state.builder.AllocateId();
const auto adjusted = state.builder.AllocateId();
state.builder.AddFunction({OpConvertSToF, state.vec2_float_type, offset_f32, offset_i32});
state.builder.AddFunction(
{OpImage, ImageViewImageType(state, view, inst.memory.kind == IR::ResourceKind::ImageUint),
image, sampled_image});
state->builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(*state, view), size_u32,
state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(state, view), size_u32,
image, ConstantU32(state, 0)});
if (ImageViewCoordinateComponents(view) == 3u) {
const auto width = state->builder.AllocateId();
const auto height = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size_u32, 0});
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size_u32, 1});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec2_uint_type, size_xy_u32, width, height});
const auto width = state.builder.AllocateId();
const auto height = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size_u32, 0});
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size_u32, 1});
state.builder.AddFunction(
{OpCompositeConstruct, state.vec2_uint_type, size_xy_u32, width, height});
}
state->builder.AddFunction({OpConvertUToF, state->vec2_float_type, size_f32, size_xy_u32});
state->builder.AddFunction({OpFDiv, state->vec2_float_type, normalized, offset_f32, size_f32});
state.builder.AddFunction({OpConvertUToF, state.vec2_float_type, size_f32, size_xy_u32});
state.builder.AddFunction({OpFDiv, state.vec2_float_type, normalized, offset_f32, size_f32});
if (ImageViewCoordinateComponents(view) == 2u) {
state->builder.AddFunction({OpFAdd, state->vec2_float_type, adjusted, coord, normalized});
state.builder.AddFunction({OpFAdd, state.vec2_float_type, adjusted, coord, normalized});
return adjusted;
}
const auto x = state->builder.AllocateId();
const auto y = state->builder.AllocateId();
const auto z = state->builder.AllocateId();
const auto normalized_x = state->builder.AllocateId();
const auto normalized_y = state->builder.AllocateId();
const auto adjusted_x = state->builder.AllocateId();
const auto adjusted_y = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->float_type, x, coord, 0});
state->builder.AddFunction({OpCompositeExtract, state->float_type, y, coord, 1});
state->builder.AddFunction({OpCompositeExtract, state->float_type, z, coord, 2});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, normalized_x, normalized, 0});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, normalized_y, normalized, 1});
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_x, x, normalized_x});
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_y, y, normalized_y});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec3_float_type, adjusted, adjusted_x, adjusted_y, z});
const auto x = state.builder.AllocateId();
const auto y = state.builder.AllocateId();
const auto z = state.builder.AllocateId();
const auto normalized_x = state.builder.AllocateId();
const auto normalized_y = state.builder.AllocateId();
const auto adjusted_x = state.builder.AllocateId();
const auto adjusted_y = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.float_type, x, coord, 0});
state.builder.AddFunction({OpCompositeExtract, state.float_type, y, coord, 1});
state.builder.AddFunction({OpCompositeExtract, state.float_type, z, coord, 2});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, normalized_x, normalized, 0});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, normalized_y, normalized, 1});
state.builder.AddFunction({OpFAdd, state.float_type, adjusted_x, x, normalized_x});
state.builder.AddFunction({OpFAdd, state.float_type, adjusted_y, y, normalized_y});
state.builder.AddFunction(
{OpCompositeConstruct, state.vec3_float_type, adjusted, adjusted_x, adjusted_y, z});
return adjusted;
}
uint32_t EmitImageCoordU32(EmitterState* state, const IR::Instruction& inst, ImageViewKind view) {
uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) {
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
: ConstantU32(state, 0);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
: ConstantU32(state, 0);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageLoadCoordU32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
ImageViewKind view) {
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
: ConstantU32(state, 0);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
: ConstantU32(state, 0);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
const IR::Operand& address, ImageViewKind view) {
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
return ConstantU32(state, 0);
@@ -219,20 +219,20 @@ uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst,
return EmitImageAddressValueLoad(state, inst, address, lod_component);
}
uint32_t EmitImageQueryCoordF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
ImageViewKind view) {
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
: EmitZeroF32(state);
const auto coord = state->builder.AllocateId();
const auto coord = state.builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
: EmitZeroF32(state);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
}
return coord;
}
@@ -245,7 +245,7 @@ uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
return index;
}
uint32_t EmitImageStoreComponentF32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
uint32_t component) {
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
if (((dmask >> component) & 1u) == 0) {
@@ -265,14 +265,14 @@ uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
}
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
if (state.program == nullptr || inst.memory.resource >= state.program->info.images.size()) {
if (inst.memory.resource >= state.program.info.images.size()) {
EXIT("storage image instruction has no specialized swizzle\n");
}
return state.program->info.images[inst.memory.resource].storage_swizzle;
return state.program.info.images[inst.memory.resource].storage_swizzle;
}
uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(*state, inst);
uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(state, inst);
const auto component_value = [&](uint32_t component) {
const auto source = InverseStorageSwizzleComponent(swizzle, component);
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
@@ -281,12 +281,12 @@ uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst
const auto y = component_value(1);
const auto z = component_value(2);
const auto w = component_value(3);
const auto texel = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, texel, x, y, z, w});
const auto texel = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w});
return texel;
}
uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst,
uint32_t component) {
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
if (((dmask >> component) & 1u) == 0) {
@@ -295,8 +295,8 @@ uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction&
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
}
uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(*state, inst);
uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(state, inst);
const auto component_value = [&](uint32_t component) {
const auto source = InverseStorageSwizzleComponent(swizzle, component);
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
@@ -306,8 +306,8 @@ uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst
const auto y = component_value(1);
const auto z = component_value(2);
const auto w = component_value(3);
const auto texel = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_uint_type, texel, x, y, z, w});
const auto texel = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w});
return texel;
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageOps.cpp
+106 -115
View File
@@ -4,54 +4,54 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t ConstantVec2I32(EmitterState* state, int32_t x, int32_t y) {
const auto value = state->builder.AllocateId();
state->builder.AddType({OpConstantComposite, state->vec2_int_type, value, ConstantI32(state, x),
ConstantI32(state, y)});
uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) {
const auto value = state.builder.AllocateId();
state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x),
ConstantI32(state, y)});
return value;
}
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState* state) {
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state) {
const auto offset0 = ConstantVec2I32(state, -1, 0);
const auto offset1 = ConstantVec2I32(state, 0, 0);
const auto offset2 = ConstantVec2I32(state, 1, 0);
const auto offset3 = ConstantVec2I32(state, 2, 0);
const auto type = state->builder.AllocateId();
state->builder.AddType({OpTypeArray, type, state->vec2_int_type, ConstantU32(state, 4)});
const auto value = state->builder.AllocateId();
state->builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
const auto type = state.builder.AllocateId();
state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)});
const auto value = state.builder.AllocateId();
state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
return value;
}
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resource,
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
uint32_t array_index, bool uint_image,
ImageViewKind view) {
uint32_t pointer_type =
uint_image ? state->ptr_uniform_storage_image_uint : state->ptr_uniform_storage_image;
uint_image ? state.ptr_uniform_storage_image_uint : state.ptr_uniform_storage_image;
uint32_t variable =
uint_image ? state->storage_image_uint_variable : state->storage_image_variable;
uint32_t image_type = uint_image ? state->storage_image_uint_type : state->storage_image_type;
uint_image ? state.storage_image_uint_variable : state.storage_image_variable;
uint32_t image_type = uint_image ? state.storage_image_uint_type : state.storage_image_type;
IR::DescriptorBindingKind kind = uint_image ? IR::DescriptorBindingKind::StorageUint2D
: IR::DescriptorBindingKind::Storage2D;
switch (view) {
case ImageViewKind::Dim2D: break;
case ImageViewKind::Dim2DArray:
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_2d_array
: state->ptr_uniform_storage_image_2d_array;
variable = uint_image ? state->storage_image_uint_2d_array_variable
: state->storage_image_2d_array_variable;
image_type = uint_image ? state->storage_image_uint_2d_array_type
: state->storage_image_2d_array_type;
pointer_type = uint_image ? state.ptr_uniform_storage_image_uint_2d_array
: state.ptr_uniform_storage_image_2d_array;
variable = uint_image ? state.storage_image_uint_2d_array_variable
: state.storage_image_2d_array_variable;
image_type = uint_image ? state.storage_image_uint_2d_array_type
: state.storage_image_2d_array_type;
kind = uint_image ? IR::DescriptorBindingKind::StorageUint2DArray
: IR::DescriptorBindingKind::Storage2DArray;
break;
case ImageViewKind::Dim3D:
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_3d
: state->ptr_uniform_storage_image_3d;
variable = uint_image ? state->storage_image_uint_3d_variable
: state->storage_image_3d_variable;
pointer_type = uint_image ? state.ptr_uniform_storage_image_uint_3d
: state.ptr_uniform_storage_image_3d;
variable =
uint_image ? state.storage_image_uint_3d_variable : state.storage_image_3d_variable;
image_type =
uint_image ? state->storage_image_uint_3d_type : state->storage_image_3d_type;
uint_image ? state.storage_image_uint_3d_type : state.storage_image_3d_type;
kind = uint_image ? IR::DescriptorBindingKind::StorageUint3D
: IR::DescriptorBindingKind::Storage3D;
break;
@@ -59,50 +59,50 @@ uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resourc
const auto pointer =
DescriptorElementPointer(state, pointer_type, variable, array_index, kind, resource,
"storage image descriptor array was not emitted");
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, image_type, image, pointer});
const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, image_type, image, pointer});
return image;
}
} // namespace
uint32_t EmitImageGetResinfoComponent(EmitterState* state, uint32_t image, uint32_t size,
uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
uint32_t component_index, ImageViewKind view) {
switch (component_index) {
case 0: {
const auto width = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size, 0});
const auto width = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0});
return width;
}
case 1: {
const auto height = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size, 1});
const auto height = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1});
return height;
}
case 2:
if (ImageViewCoordinateComponents(view) == 3u) {
const auto depth = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, depth, size, 2});
const auto depth = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2});
return depth;
}
return ConstantU32(state, 1);
case 3: {
const auto levels = state->builder.AllocateId();
state->builder.AddFunction({OpImageQueryLevels, state->uint_type, levels, image});
const auto levels = state.builder.AllocateId();
state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image});
return levels;
}
default: return ConstantU32(state, 0);
}
}
void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
const auto mip_level = EmitValueLoad(state, inst.src[0]);
const auto size = state->builder.AllocateId();
state->builder.AddFunction(
{OpImageQuerySizeLod, ImageViewSizeType(*state, view), size, image, mip_level});
const auto size = state.builder.AllocateId();
state.builder.AddFunction(
{OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
@@ -116,13 +116,13 @@ void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) {
}
}
void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto lod = state->builder.AllocateId();
state->builder.AddFunction({OpImageQueryLod, state->vec2_float_type, lod, sampled_image,
EmitImageQueryCoordF32(state, inst, view)});
const auto lod = state.builder.AllocateId();
state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image,
EmitImageQueryCoordF32(state, inst, view)});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
@@ -130,25 +130,25 @@ void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, component, lod, component_index});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
const auto component = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, component, lod, component_index});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto color = state->builder.AllocateId();
state->builder.AddFunction(
{OpImageFetch, integer ? state->vec4_uint_type : state->vec4_float_type, color, image,
EmitImageLoadCoordU32(state, inst, view), ImageOperandsLodMask,
EmitImageMipLodU32(state, inst, inst.src[0], view)});
const auto color = state.builder.AllocateId();
state.builder.AddFunction({OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
color, image, EmitImageLoadCoordU32(state, inst, view),
ImageOperandsLodMask,
EmitImageMipLodU32(state, inst, inst.src[0], view)});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
@@ -156,23 +156,22 @@ void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
color, component_index});
const auto bits = integer ? component : state->builder.AllocateId();
const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
component, color, component_index});
const auto bits = integer ? component : state.builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
void EmitImageStore(EmitterState* state, const IR::Instruction& inst) {
void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
const auto view = StorageImageViewKind(*state, inst.memory, uint_image, inst.pc);
const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
const auto binding = ResourceForDescriptor(
*state,
state,
uint_image
? (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::StorageUint2DArray
: view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::StorageUint3D
@@ -184,16 +183,16 @@ void EmitImageStore(EmitterState* state, const IR::Instruction& inst) {
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
binding.array_index, uint_image, view);
state->builder.AddFunction(
state.builder.AddFunction(
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
}
void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uint32_t sample,
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
bool dref, bool integer) {
if (dref) {
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, sample});
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample});
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
}
@@ -206,13 +205,12 @@ void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uin
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
sample, component_index});
const auto bits = integer ? component : state->builder.AllocateId();
const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
component, sample, component_index});
const auto bits = integer ? component : state.builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
@@ -231,12 +229,9 @@ bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instructio
return state.stage != ShaderType::Pixel;
}
void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst,
void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, bool explicit_lod,
std::vector<uint32_t>* words) {
if (words == nullptr) {
return;
}
std::vector<uint32_t>& words) {
uint32_t mask = 0;
std::vector<uint32_t> operands;
@@ -255,8 +250,8 @@ void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst,
operands.push_back(EmitImageBiasF32(state, inst, layout));
}
if (mask != 0) {
words->push_back(mask);
words->insert(words->end(), operands.begin(), operands.end());
words.push_back(mask);
words.insert(words.end(), operands.begin(), operands.end());
}
}
@@ -270,20 +265,20 @@ uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& ins
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
}
void EmitImageSample(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageSample(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto layout = MakeImageSampleLayout(inst, view);
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
const auto sample = state->builder.AllocateId();
const auto sample = state.builder.AllocateId();
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto result_type = dref ? state->float_type
: integer ? state->vec4_uint_type
: state->vec4_float_type;
const auto explicit_lod = ImageSampleNeedsExplicitLod(*state, inst);
const auto opcode = ImageSampleOpcode(*state, inst);
const auto result_type = dref ? state.float_type
: integer ? state.vec4_uint_type
: state.vec4_float_type;
const auto explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
const auto opcode = ImageSampleOpcode(state, inst);
const auto coord =
EmitImageOffsetCoordF32(state, inst, layout, sampled_image, base_coord, view);
@@ -291,8 +286,8 @@ void EmitImageSample(EmitterState* state, const IR::Instruction& inst) {
if (dref) {
words.push_back(EmitImageDrefF32(state, inst, layout));
}
AddImageSampleOperands(state, inst, layout, explicit_lod, &words);
state->builder.AddFunction(words);
AddImageSampleOperands(state, inst, layout, explicit_lod, words);
state.builder.AddFunction(words);
EmitImageSampleResult(state, inst, sample, dref, integer);
}
@@ -305,35 +300,32 @@ uint32_t ImageGatherComponent(uint32_t dmask) {
}
}
void AddImageGatherOperands(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout, std::vector<uint32_t>* words) {
if (words == nullptr) {
return;
}
void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst,
const ImageSampleLayout& layout, std::vector<uint32_t>& words) {
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
words->push_back(ImageOperandsConstOffsetsMask);
words->push_back(ConstantImageGatherHorizontalOffsets(state));
words.push_back(ImageOperandsConstOffsetsMask);
words.push_back(ConstantImageGatherHorizontalOffsets(state));
return;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
words->push_back(ImageOperandsOffsetMask);
words->push_back(EmitImagePackedOffset2I32(state, inst, layout));
words.push_back(ImageOperandsOffsetMask);
words.push_back(EmitImagePackedOffset2I32(state, inst, layout));
}
}
void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto layout = MakeImageSampleLayout(inst, view);
const auto coord = EmitImageCoordF32(state, inst, layout, view);
const auto texels = state->builder.AllocateId();
const auto texels = state.builder.AllocateId();
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto result_type = integer ? state->vec4_uint_type : state->vec4_float_type;
const auto result_type = integer ? state.vec4_uint_type : state.vec4_float_type;
std::vector<uint32_t> words =
dref ? std::vector<uint32_t> {OpImageDrefGather,
state->vec4_float_type,
state.vec4_float_type,
texels,
sampled_image,
coord,
@@ -342,17 +334,16 @@ void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) {
OpImageGather, result_type,
texels, sampled_image,
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
AddImageGatherOperands(state, inst, layout, &words);
state->builder.AddFunction(words);
AddImageGatherOperands(state, inst, layout, words);
state.builder.AddFunction(words);
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
texels, i});
const auto bits = integer ? component : state->builder.AllocateId();
const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
component, texels, i});
const auto bits = integer ? component : state.builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
}
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h
+410 -407
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterMemory.cpp
+499 -499
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp
+525 -525
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterValues.cpp
+332 -332
View File
@@ -21,7 +21,7 @@ bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
return false;
}
uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
return value;
}
@@ -33,117 +33,117 @@ uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uin
}
if (operand.sdwa_sext) {
const auto signed_value = state->builder.AllocateId();
const auto extracted = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, signed_value, value});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_value,
const auto signed_value = state.builder.AllocateId();
const auto extracted = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value});
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value,
ConstantU32(state, offset), ConstantU32(state, width)});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, extracted});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted});
return bits;
}
const auto extracted = state->builder.AllocateId();
state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, extracted, value,
const auto extracted = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value,
ConstantU32(state, offset), ConstantU32(state, width)});
return extracted;
}
uint32_t EmitTrueBool(EmitterState* state) {
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
uint32_t EmitTrueBool(EmitterState& state) {
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
return ret;
}
DppTargetLane EmitDppQuadPermTargetLane(EmitterState* state, uint32_t subid, uint32_t control) {
const auto quad_base = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto shift = state->builder.AllocateId();
const auto selected0 = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 3)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, shift, lane, ConstantU32(state, 1)});
state->builder.AddFunction(
{OpShiftRightLogical, state->uint_type, selected0, ConstantU32(state, control), shift});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, selected, selected0, ConstantU32(state, 3)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, quad_base, selected});
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
const auto quad_base = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto shift = state.builder.AllocateId();
const auto selected0 = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
state.builder.AddFunction(
{OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppRowShiftTargetLane(EmitterState* state, uint32_t subid, uint32_t amount,
DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
bool left) {
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto lane_shifted = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
const auto valid = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
const auto row = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto lane_shifted = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
const auto valid = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
if (left) {
state->builder.AddFunction(
{OpIAdd, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpULessThan, state->bool_type, valid, lane, ConstantU32(state, 16u - amount)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
} else {
state->builder.AddFunction(
{OpISub, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, valid, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
}
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, lane_shifted});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
return {target, valid};
}
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState* state, uint32_t subid,
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
uint32_t amount) {
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto in_high = state->builder.AllocateId();
const auto minus = state->builder.AllocateId();
const auto plus = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, amount)});
state->builder.AddFunction({OpISub, state->uint_type, minus, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpIAdd, state->uint_type, plus, lane, ConstantU32(state, 16u - amount)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, minus, plus});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, selected});
const auto row = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId();
const auto minus = state.builder.AllocateId();
const auto plus = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppMirrorTargetLane(EmitterState* state, uint32_t subid, bool half_row) {
DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
const auto lane_mask = half_row ? 7u : 15u;
const auto base = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto mirrored = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, base_mask)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, lane_mask)});
state->builder.AddFunction(
{OpISub, state->uint_type, mirrored, ConstantU32(state, lane_mask), lane});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, mirrored});
const auto base = state.builder.AllocateId();
const auto lane = state.builder.AllocateId();
const auto mirrored = state.builder.AllocateId();
const auto target = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
state.builder.AddFunction(
{OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) {
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
const auto subid = EmitSubgroupLocalInvocationId(state);
if (control <= 0xffu) {
return EmitDppQuadPermTargetLane(state, subid, control);
@@ -166,42 +166,42 @@ DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) {
return {subid, EmitTrueBool(state)};
}
uint32_t EmitDppValueU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
if (!operand.dpp) {
return value;
}
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
const auto shuffled = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), value, target.lane});
if (operand.dpp_fetch_inactive) {
return shuffled;
}
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
const auto can_fetch = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpLogicalAnd, state->bool_type, can_fetch, target.valid, source_active});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
const auto can_fetch = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
uint32_t value = 0;
switch (operand.kind) {
case IR::OperandKind::ImmediateU32:
case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
case IR::OperandKind::Register: {
const auto pointer = PointerForRegister(*state, operand.reg);
const auto pointer = PointerForRegister(state, operand.reg);
if (pointer == 0) {
value = ConstantU32(state, 0);
break;
}
value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
break;
}
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
@@ -211,160 +211,160 @@ uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) {
return EmitSdwaExtractU32(state, operand, value);
}
uint32_t EmitRegisterLoad(EmitterState* state, IR::Register reg) {
if (IsInactiveWave32ExecHigh(*state, reg)) {
uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) {
if (IsInactiveWave32ExecHigh(state, reg)) {
return ConstantU32(state, 0);
}
const auto pointer = PointerForRegister(*state, reg);
const auto pointer = PointerForRegister(state, reg);
if (pointer == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
uint32_t WaveMaskHighLoad(EmitterState* state, IR::RegisterFile file) {
if (state->wave_size == 32u && IsMaskRegisterFile(file)) {
uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) {
if (state.wave_size == 32u && IsMaskRegisterFile(file)) {
return ConstantU32(state, 0);
}
return EmitRegisterLoad(state, {file, 1});
}
uint32_t EmitLaneMaskPartU32(EmitterState* state, uint32_t part) {
uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto lane = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
const auto lane = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
if (part == 0) {
const auto in_low = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpULessThan, state->bool_type, in_low, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_low, bit, ConstantU32(state, 0)});
const auto in_low = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)});
return ret;
}
if (state->wave_size == 32u) {
if (state.wave_size == 32u) {
return ConstantU32(state, 0);
}
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, bit, ConstantU32(state, 0)});
const auto in_high = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitThreadMaskBelowPartU32(EmitterState* state, uint32_t part) {
uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto lane = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
const auto below = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
state->builder.AddFunction({OpISub, state->uint_type, below, bit, ConstantU32(state, 1)});
const auto lane = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
const auto below = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)});
if (part == 0) {
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
const auto in_high = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
return ret;
}
if (state->wave_size == 32u) {
if (state.wave_size == 32u) {
return ConstantU32(state, 0);
}
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, below, ConstantU32(state, 0)});
const auto in_high = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
state.builder.AddFunction(
{OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitMaskActiveBool(EmitterState* state, IR::RegisterFile file) {
uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) {
const auto low = EmitRegisterLoad(state, {file, 0});
const auto condition = state->builder.AllocateId();
if (state->per_invocation_masks) {
const auto combined = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseOr, state->uint_type, combined, low, WaveMaskHighLoad(state, file)});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, combined, ConstantU32(state, 0)});
const auto condition = state.builder.AllocateId();
if (state.per_invocation_masks) {
const auto combined = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)});
return condition;
}
if (state->exact_subgroup_operations) {
if (state.exact_subgroup_operations) {
const auto high = WaveMaskHighLoad(state, file);
const auto low_hit = state->builder.AllocateId();
const auto high_hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, active, low_hit, high_hit});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, active, ConstantU32(state, 0)});
const auto low_hit = state.builder.AllocateId();
const auto high_hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)});
return condition;
}
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, low, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)});
return condition;
}
uint32_t EmitMaskZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
const auto active = EmitMaskActiveBool(state, file);
if (!zero) {
return active;
}
const auto cond = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, cond, active});
const auto cond = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active});
return cond;
}
uint32_t EmitMaskSummaryZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
if (state->per_invocation_masks) {
uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
if (state.per_invocation_masks) {
return EmitMaskZeroBool(state, file, zero);
}
if (state->exact_subgroup_operations) {
if (state.exact_subgroup_operations) {
const auto low = EmitRegisterLoad(state, {file, 0});
const auto high = WaveMaskHighLoad(state, file);
const auto combined = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
const auto combined = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
if (!zero) {
return active;
}
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
return inactive;
}
const auto active = EmitMaskActiveBool(state, file);
if (!zero) {
return active;
}
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
return inactive;
}
uint32_t EmitExecActiveBool(EmitterState* state) {
uint32_t EmitExecActiveBool(EmitterState& state) {
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
}
uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::Register) {
if (operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc) {
@@ -372,19 +372,19 @@ uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) {
}
}
const auto cond_value = EmitValueLoad(state, operand);
const auto cond = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, cond, cond_value, ConstantU32(state, 0)});
const auto cond = state.builder.AllocateId();
state.builder.AddFunction(
{OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)});
return cond;
}
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state) {
if (state->subgroup_local_invocation_id_variable == 0) {
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
if (state.subgroup_local_invocation_id_variable == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpLoad, state->uint_type, value, state->subgroup_local_invocation_id_variable});
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
return value;
}
@@ -415,30 +415,30 @@ const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t
return nullptr;
}
uint32_t EmitInputComponentU32(EmitterState* state, IR::StageInputKind kind, uint32_t component) {
const auto variable = InputVariableForKind(*state, kind);
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
const auto variable = InputVariableForKind(state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpAccessChain, state->ptr_input_uint, pointer, variable, ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId();
state.builder.AddFunction(
{OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
return value;
}
uint32_t EmitLocalInvocationIndex(EmitterState* state) {
const auto variable = InputVariableForKind(*state, IR::StageInputKind::LocalInvocationIndex);
uint32_t EmitLocalInvocationIndex(EmitterState& state) {
const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, variable});
const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
return value;
}
uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKind kind,
uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
uint32_t component) {
if ((component & 3u) != 3u) {
return ConstantU32(state, 0);
@@ -446,105 +446,105 @@ uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKi
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
}
uint32_t EmitVertexParameterComponentU32(EmitterState* state, const InputBinding& input,
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
uint32_t component) {
const auto count = VertexParameterComponentCount(*state, input);
const auto kind = VertexParameterScalarKind(*state, input.location);
const auto count = VertexParameterComponentCount(state, input);
const auto kind = VertexParameterScalarKind(state, input.location);
if (component >= count) {
return VertexInputDefaultComponentU32(state, kind, component);
}
const auto scalar_type = VertexParameterScalarType(*state, kind);
uint32_t raw = state->builder.AllocateId();
const auto scalar_type = VertexParameterScalarType(state, kind);
uint32_t raw = state.builder.AllocateId();
if (count == 1u) {
state->builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
} else {
const auto pointer_type = VertexParameterScalarPointerType(*state, kind);
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
const auto pointer_type = VertexParameterScalarPointerType(state, kind);
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, scalar_type, raw, pointer});
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
}
if (kind == VertexInputScalarKind::Uint) {
return raw;
}
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw});
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
return bits;
}
void EmitLoadInputF32(EmitterState* state, const IR::Instruction& inst) {
const auto* input = InputBindingForParameter(*state, inst.input_info.attr);
void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) {
const auto* input = InputBindingForParameter(state, inst.input_info.attr);
if (input == nullptr || input->variable_id == 0) {
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
return;
}
if (state->stage == ShaderType::Vertex) {
if (state.stage == ShaderType::Vertex) {
EmitStoreU32(state, inst.dst,
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
return;
}
const auto input_value = state->builder.AllocateId();
const auto component = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->vec4_float_type, input_value, input->variable_id});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, component, input_value, inst.input_info.chan & 3u});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
const auto input_value = state.builder.AllocateId();
const auto component = state.builder.AllocateId();
const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u});
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
EmitStoreU32(state, inst.dst, bits);
}
uint32_t EmitSccBool(EmitterState* state, bool non_zero) {
uint32_t EmitSccBool(EmitterState& state, bool non_zero) {
const auto value = EmitRegisterLoad(state, SccRegister());
const auto cond = state->builder.AllocateId();
state->builder.AddFunction(
{non_zero ? OpINotEqual : OpIEqual, state->bool_type, cond, value, ConstantU32(state, 0)});
const auto cond = state.builder.AllocateId();
state.builder.AddFunction(
{non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)});
return cond;
}
uint32_t EmitFloatLoad(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) {
const auto uint_value = EmitValueLoad(state, operand);
auto float_value = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, float_value, uint_value});
auto float_value = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value});
if (operand.absolute) {
const auto abs_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, float_value});
const auto abs_value = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value});
float_value = abs_value;
}
if (operand.negate) {
const auto neg_value = state->builder.AllocateId();
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, float_value});
const auto neg_value = state.builder.AllocateId();
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value});
float_value = neg_value;
}
return float_value;
}
uint32_t ApplyResultModifiersF32(EmitterState* state, uint32_t value, const IR::Operand& dst) {
uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) {
uint32_t result = value;
if (dst.omod != 0) {
const uint32_t bits = dst.omod == 1u ? 0x40000000u
: dst.omod == 2u ? 0x40800000u
: 0x3f000000u;
const auto scaled = state->builder.AllocateId();
state->builder.AddFunction(
{OpFMul, state->float_type, scaled, result, ConstantF32(state, bits)});
const auto scaled = state.builder.AllocateId();
state.builder.AddFunction(
{OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)});
result = scaled;
}
if (!dst.clamp) {
return result;
}
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->float_type, ret, state->glsl_std450, GlslFClamp,
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp,
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
return ret;
}
uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) {
auto value_operand = operand;
value_operand.negate_hi = false;
@@ -562,21 +562,21 @@ uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) {
raw.dpp_bound_ctrl = false;
const auto bits = EmitValueLoad(state, raw);
const auto unpacked = state->builder.AllocateId();
auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450,
const auto unpacked = state.builder.AllocateId();
auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450,
GlslUnpackHalf2x16, bits});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
state.builder.AddFunction(
{OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
if (operand.absolute) {
const auto abs_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32});
const auto abs_value = state.builder.AllocateId();
state.builder.AddFunction(
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32});
f32 = abs_value;
}
if (value_operand.negate) {
const auto neg_value = state->builder.AllocateId();
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32});
const auto neg_value = state.builder.AllocateId();
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32});
f32 = neg_value;
}
return f32;
@@ -611,10 +611,10 @@ IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
return ret;
}
uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& operand) {
uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) {
const auto low = EmitValueLoad(state, operand);
const auto active = state->builder.AllocateId();
if (state->per_invocation_masks || state->exact_subgroup_operations) {
const auto active = state.builder.AllocateId();
if (state.per_invocation_masks || state.exact_subgroup_operations) {
uint32_t high = ConstantU32(state, 0);
if (operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Scalar ||
@@ -624,102 +624,102 @@ uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& o
? WaveMaskHighLoad(state, operand.reg.file)
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
}
if (state->per_invocation_masks) {
const auto combined = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
if (state.per_invocation_masks) {
const auto combined = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
return active;
}
const auto low_hit = state->builder.AllocateId();
const auto high_hit = state->builder.AllocateId();
const auto hit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, hit, low_hit, high_hit});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
const auto low_hit = state.builder.AllocateId();
const auto high_hit = state.builder.AllocateId();
const auto hit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit});
state.builder.AddFunction(
{OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
return active;
}
state->builder.AddFunction({OpINotEqual, state->bool_type, active, low, ConstantU32(state, 0)});
state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)});
return active;
}
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane) {
if (state->per_invocation_masks) {
const auto ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) {
if (state.per_invocation_masks) {
const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
return EmitBallotLaneActiveBool(state, ballot, lane);
}
const auto lane_low = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
const auto lane_low = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
if (state->wave_size == 64u) {
if (state.wave_size == 64u) {
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
const auto in_high = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, mask});
const auto in_high = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask});
mask = selected;
}
const auto hit = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, ret, hit, ConstantU32(state, 0)});
const auto hit = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitSubgroupLaneActiveBool(EmitterState* state, uint32_t lane) {
const auto active_ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, active_ballot,
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
const auto active_ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
return EmitBallotLaneActiveBool(state, active_ballot, lane);
}
uint32_t EmitBallotLaneActiveBool(EmitterState* state, uint32_t active_ballot, uint32_t lane) {
const auto low = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, active_ballot, 0});
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
const auto low = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
uint32_t mask = low;
if (state->wave_size == 64u) {
const auto high = state->builder.AllocateId();
const auto in_high = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, active_ballot, 1});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, low});
if (state.wave_size == 64u) {
const auto high = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId();
const auto selected = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
state.builder.AddFunction(
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
mask = selected;
}
const auto lane_low = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
const auto lane_low = state.builder.AllocateId();
const auto bit = state.builder.AllocateId();
state.builder.AddFunction(
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
state.builder.AddFunction(
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
const auto hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
const auto in_range = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpULessThan, state->bool_type, in_range, lane, ConstantU32(state, state->wave_size)});
state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, active, in_range});
const auto hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId();
const auto in_range = state.builder.AllocateId();
const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
state.builder.AddFunction(
{OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
return ret;
}
uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
}
@@ -729,10 +729,10 @@ uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand
return ConstantU32(state, 0);
}
uint32_t EmitSequentialFloatLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
const auto value = EmitSequentialValueLoad(state, operand, offset);
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
return f32;
}
@@ -803,7 +803,7 @@ IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand&
return OffsetRegisterOperand(base, packed_component);
}
uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
const IR::Operand& base, uint32_t component) {
const auto operand = ImageAddressOperand(inst, base, component);
const auto packed_component = ImageAddressPackedComponent(inst, component);
@@ -820,20 +820,20 @@ uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& i
return ConstantU32(state, 0);
}
uint32_t EmitImageAddressFloatLoad(EmitterState* state, const IR::Instruction& inst,
uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
const IR::Operand& base, uint32_t component) {
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
if (ImageAddressComponentUsesA16(inst, component)) {
return EmitF16BitsToF32(state, value);
}
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
return f32;
}
uint32_t EmitZeroF32(EmitterState* state) {
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, ConstantU32(state, 0)});
uint32_t EmitZeroF32(EmitterState& state) {
const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)});
return f32;
}
src/graphics/shader/shader.cpp
+283 -322
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/shader.h
+24 -24
View File
@@ -42,8 +42,8 @@ struct ShaderStageRuntime {
// Resolves an immutable native shader plan against current user data. The prior stage is preserved
// if any ReadConst, snapshot, or specialization check fails.
bool ShaderMaterializeStageRuntime(std::shared_ptr<const ShaderRecompiler::IR::Program> program,
std::span<const uint32_t> user_data, uint64_t shader_base,
ShaderStageRuntime* stage, std::string* error);
std::span<const uint32_t> user_data, uint64_t shader_base,
ShaderStageRuntime& stage, std::string* error);
struct ShaderId {
uint32_t hash0 = 0;
@@ -216,33 +216,33 @@ struct ShaderMappedData {
void ShaderInit();
void ShaderMapUserData(uint64_t addr, const ShaderMappedData& data);
void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo* info);
void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo* info);
void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo* info);
ShaderId ShaderGetIdVS(const HW::VertexShaderInfo* regs, const ShaderVertexInputInfo* input_info,
void ShaderDbgDumpInputInfo(const ShaderVertexInputInfo& info);
void ShaderDbgDumpInputInfo(const ShaderPixelInputInfo& info);
void ShaderDbgDumpInputInfo(const ShaderComputeInputInfo& info);
ShaderId ShaderGetIdVS(const HW::VertexShaderInfo& regs, const ShaderVertexInputInfo& input_info,
bool include_bind_specialization);
ShaderId ShaderGetIdPS(const HW::PixelShaderInfo* regs, const ShaderPixelInputInfo* input_info,
ShaderId ShaderGetIdPS(const HW::PixelShaderInfo& regs, const ShaderPixelInputInfo& input_info,
bool include_bind_specialization);
ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo* regs, const ShaderComputeInputInfo* input_info,
ShaderId ShaderGetIdCS(const HW::ComputeShaderInfo& regs, const ShaderComputeInputInfo& input_info,
bool include_bind_specialization);
// Returned SPIR-V spans are read-only views backed by the shader program cache.
bool ShaderCompileInfoVS(const HW::VertexShaderInfo* regs, const HW::ShaderRegisters* sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo* input_info,
std::span<const uint32_t>* spirv);
bool ShaderCompileInfoPS(const HW::PixelShaderInfo* regs, const HW::ShaderRegisters* sh,
ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo* vs_info,
bool ShaderCompileInfoVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::span<const uint32_t>& spirv);
bool ShaderCompileInfoPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, const ShaderVertexInputInfo& vs_info,
std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
ShaderPixelInputInfo* input_info, std::span<const uint32_t>* spirv);
bool ShaderCompileInfoCS(const HW::ComputeShaderInfo* regs, const HW::ShaderRegisters* sh,
ShaderComputeInputInfo* input_info, std::span<const uint32_t>* spirv);
bool ShaderCompileSpirvVS(const HW::VertexShaderInfo* regs, const HW::ShaderRegisters* sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo* input_info,
std::vector<uint32_t>* spirv);
bool ShaderCompileSpirvPS(const HW::PixelShaderInfo* regs, const HW::ShaderRegisters* sh,
ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo* input_info,
std::vector<uint32_t>* spirv);
bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo* regs, const HW::ShaderRegisters* sh,
ShaderComputeInputInfo* input_info, std::vector<uint32_t>* spirv);
ShaderPixelInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileInfoCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderComputeInputInfo& input_info, std::span<const uint32_t>& spirv);
bool ShaderCompileSpirvVS(const HW::VertexShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderVertexInputInfo& input_info,
std::vector<uint32_t>& spirv);
bool ShaderCompileSpirvPS(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderLaneMaskMode lane_mask_mode, ShaderPixelInputInfo& input_info,
std::vector<uint32_t>& spirv);
bool ShaderCompileSpirvCS(const HW::ComputeShaderInfo& regs, const HW::ShaderRegisters& sh,
ShaderComputeInputInfo& input_info, std::vector<uint32_t>& spirv);
bool ShaderAddressValid(uint64_t addr);
} // namespace Libs::Graphics
src/graphics/shader/shaderStageRuntime.cpp
+6 -7
View File
@@ -6,12 +6,11 @@
namespace Libs::Graphics {
bool ShaderMaterializeStageRuntime(std::shared_ptr<const ShaderRecompiler::IR::Program> program,
std::span<const uint32_t> user_data, uint64_t shader_base,
ShaderStageRuntime* stage, std::string* error) {
if (program == nullptr || stage == nullptr) {
std::span<const uint32_t> user_data, uint64_t shader_base,
ShaderStageRuntime& stage, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error =
program == nullptr ? "missing native shader plan" : "missing shader stage output";
*error = "missing native shader plan";
}
return false;
}
@@ -19,13 +18,13 @@ bool ShaderMaterializeStageRuntime(std::shared_ptr<const ShaderRecompiler::IR::P
runtime.user_data = user_data;
runtime.shader_base = shader_base;
ShaderRecompiler::IR::ResourceSnapshot snapshot;
if (!ShaderRecompiler::IR::MaterializeResources(*program, runtime, &snapshot, error) ||
if (!ShaderRecompiler::IR::MaterializeResources(*program, runtime, snapshot, error) ||
!ShaderRecompiler::IR::ValidateResourceSpecialization(*program, snapshot, error)) {
return false;
}
auto resources =
std::make_shared<const ShaderRecompiler::IR::ResourceSnapshot>(std::move(snapshot));
*stage = {std::move(program), std::move(resources)};
stage = {std::move(program), std::move(resources)};
return true;
}
src/graphics/shader/shaderVertexMetadata.cpp
+3 -6
View File
@@ -17,10 +17,7 @@ bool Fail(std::string* error, const char* message) {
} // namespace
bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
ShaderVertexMetadata* metadata, std::string* error) {
if (metadata == nullptr) {
return Fail(error, "missing vertex metadata output");
}
ShaderVertexMetadata& metadata, std::string* error) {
if (data.user_data == nullptr) {
return Fail(error, "missing AGC user-data header");
}
@@ -61,7 +58,7 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
const bool embedded = next.vertex_buffer_reg >= 0 || next.vertex_attrib_reg >= 0;
if (!embedded) {
*metadata = next;
metadata = next;
return true;
}
if (next.vertex_buffer_reg < 0 || next.vertex_attrib_reg < 0) {
@@ -83,7 +80,7 @@ bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sg
}
std::memcpy(next.input_semantics.data(), data.input_semantics, semantic_size);
next.input_semantics_count = data.num_input_semantics;
*metadata = next;
metadata = next;
return true;
}
src/graphics/shader/shaderVertexMetadata.h
+1 -1
View File
@@ -17,7 +17,7 @@ struct ShaderVertexMetadata {
// Copies the small AGC metadata subset used by the vertex path after validating every guest range.
bool ShaderReadVertexMetadata(const ShaderMappedData& data, uint32_t max_user_sgprs,
ShaderVertexMetadata* metadata, std::string* error);
ShaderVertexMetadata& metadata, std::string* error);
} // namespace Libs::Graphics