mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-28 08:13:05 +00:00
refactor(graphics): split renderer and recompiler into focused modules
This commit is contained in:
nmzik committed
1 parent
2b9cba457e
commit
a6b61d7aa0
108 files changed
+245
-237
No files matched your search
@@ -0,0 +1,160 @@
|
||||
#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
static constexpr size_t InitialSpirvSectionReserve = 4096;
|
||||
static constexpr size_t InitialSpirvFunctionSectionReserve = 450000;
|
||||
|
||||
static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_t* words,
|
||||
size_t words_num) {
|
||||
if (words_num == 0) {
|
||||
return;
|
||||
}
|
||||
const auto opcode = words[0];
|
||||
const auto word_count = static_cast<uint32_t>(words_num);
|
||||
section.push_back((word_count << 16u) | opcode);
|
||||
section.insert(section.end(), words + 1, words + words_num);
|
||||
}
|
||||
|
||||
Builder::Builder() {
|
||||
m_debug.reserve(InitialSpirvSectionReserve);
|
||||
m_annotations.reserve(InitialSpirvSectionReserve);
|
||||
m_types.reserve(InitialSpirvSectionReserve);
|
||||
m_functions.reserve(InitialSpirvFunctionSectionReserve);
|
||||
}
|
||||
|
||||
uint32_t Builder::AllocateId() {
|
||||
return m_next_id++;
|
||||
}
|
||||
|
||||
void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
|
||||
const auto len = text != nullptr ? std::strlen(text) : 0;
|
||||
const auto word_count = (len + 1u + 3u) / 4u;
|
||||
for (size_t i = 0; i < word_count; i++) {
|
||||
uint32_t word = 0;
|
||||
for (size_t byte = 0; byte < 4; byte++) {
|
||||
const auto index = i * 4u + byte;
|
||||
if (index < len) {
|
||||
word |= static_cast<uint32_t>(static_cast<unsigned char>(text[index]))
|
||||
<< (byte * 8u);
|
||||
}
|
||||
}
|
||||
words.push_back(word);
|
||||
}
|
||||
}
|
||||
|
||||
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
const std::vector<uint32_t>& operands) {
|
||||
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
|
||||
section.push_back((word_count << 16u) | opcode);
|
||||
section.insert(section.end(), operands.begin(), operands.end());
|
||||
}
|
||||
|
||||
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
std::initializer_list<uint32_t> operands) {
|
||||
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
|
||||
section.push_back((word_count << 16u) | opcode);
|
||||
section.insert(section.end(), operands.begin(), operands.end());
|
||||
}
|
||||
|
||||
void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
|
||||
AppendInstruction(m_capabilities, 17u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddExtension(const char* name) {
|
||||
std::vector<uint32_t> operands;
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_extensions, 10u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddExtInstImport(uint32_t id, const char* name) {
|
||||
std::vector<uint32_t> operands = {id};
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_ext_inst_imports, 11u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
|
||||
AppendInstruction(m_memory_model, 14u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
|
||||
const std::vector<uint32_t>& interfaces) {
|
||||
std::vector<uint32_t> operands = {execution_model, entry_point};
|
||||
AppendString(operands, name);
|
||||
operands.insert(operands.end(), interfaces.begin(), interfaces.end());
|
||||
AppendInstruction(m_entry_points, 15u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
|
||||
AppendInstruction(m_execution_modes, 16u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddName(uint32_t target, const char* name) {
|
||||
std::vector<uint32_t> operands = {target};
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_debug, 5u, operands);
|
||||
}
|
||||
|
||||
void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
|
||||
AppendInstructionWords(m_annotations, words.begin(), words.size());
|
||||
}
|
||||
|
||||
void Builder::AddType(std::initializer_list<uint32_t> words) {
|
||||
AppendInstructionWords(m_types, words.begin(), words.size());
|
||||
}
|
||||
|
||||
void Builder::AddFunction(std::initializer_list<uint32_t> words) {
|
||||
AppendInstructionWords(m_functions, words.begin(), words.size());
|
||||
}
|
||||
|
||||
void Builder::AddFunction(const std::vector<uint32_t>& words) {
|
||||
AppendInstructionWords(m_functions, words.data(), words.size());
|
||||
}
|
||||
|
||||
std::vector<uint32_t> Builder::Build() const {
|
||||
if (m_debug.size() > InitialSpirvSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=debug size=%zu reserve=%zu\n",
|
||||
m_debug.size(), InitialSpirvSectionReserve);
|
||||
}
|
||||
if (m_annotations.size() > InitialSpirvSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=annotations size=%zu reserve=%zu\n",
|
||||
m_annotations.size(), InitialSpirvSectionReserve);
|
||||
}
|
||||
if (m_types.size() > InitialSpirvSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=types size=%zu reserve=%zu\n",
|
||||
m_types.size(), InitialSpirvSectionReserve);
|
||||
}
|
||||
if (m_functions.size() > InitialSpirvFunctionSectionReserve) {
|
||||
std::printf("SPIR-V builder reserve exceeded: section=functions size=%zu reserve=%zu\n",
|
||||
m_functions.size(), InitialSpirvFunctionSectionReserve);
|
||||
}
|
||||
|
||||
std::vector<uint32_t> module;
|
||||
module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() +
|
||||
m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() +
|
||||
m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size());
|
||||
|
||||
module.push_back(0x07230203u);
|
||||
module.push_back(0x00010300u);
|
||||
module.push_back(0u);
|
||||
module.push_back(m_next_id);
|
||||
module.push_back(0u);
|
||||
|
||||
module.insert(module.end(), m_capabilities.begin(), m_capabilities.end());
|
||||
module.insert(module.end(), m_extensions.begin(), m_extensions.end());
|
||||
module.insert(module.end(), m_ext_inst_imports.begin(), m_ext_inst_imports.end());
|
||||
module.insert(module.end(), m_memory_model.begin(), m_memory_model.end());
|
||||
module.insert(module.end(), m_entry_points.begin(), m_entry_points.end());
|
||||
module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end());
|
||||
module.insert(module.end(), m_debug.begin(), m_debug.end());
|
||||
module.insert(module.end(), m_annotations.begin(), m_annotations.end());
|
||||
module.insert(module.end(), m_types.begin(), m_types.end());
|
||||
module.insert(module.end(), m_functions.begin(), m_functions.end());
|
||||
|
||||
return module;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
@@ -0,0 +1,56 @@
|
||||
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
|
||||
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
|
||||
|
||||
#include "common/common.h"
|
||||
|
||||
#include <initializer_list>
|
||||
#include <vector>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
class Builder {
|
||||
public:
|
||||
Builder();
|
||||
~Builder() = default;
|
||||
KYTY_CLASS_DEFAULT_COPY(Builder);
|
||||
|
||||
uint32_t AllocateId();
|
||||
|
||||
void AddCapability(std::initializer_list<uint32_t> operands);
|
||||
void AddExtension(const char* name);
|
||||
void AddExtInstImport(uint32_t id, const char* name);
|
||||
void AddMemoryModel(std::initializer_list<uint32_t> operands);
|
||||
void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
|
||||
const std::vector<uint32_t>& interfaces);
|
||||
void AddExecutionMode(std::initializer_list<uint32_t> operands);
|
||||
void AddName(uint32_t target, const char* name);
|
||||
void AddAnnotation(std::initializer_list<uint32_t> words);
|
||||
void AddType(std::initializer_list<uint32_t> words);
|
||||
void AddFunction(std::initializer_list<uint32_t> words);
|
||||
void AddFunction(const std::vector<uint32_t>& words);
|
||||
|
||||
[[nodiscard]] std::vector<uint32_t> Build() const;
|
||||
|
||||
private:
|
||||
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
const std::vector<uint32_t>& operands);
|
||||
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
|
||||
std::initializer_list<uint32_t> operands);
|
||||
static void AppendString(std::vector<uint32_t>& words, const char* text);
|
||||
|
||||
uint32_t m_next_id = 1;
|
||||
std::vector<uint32_t> m_capabilities;
|
||||
std::vector<uint32_t> m_extensions;
|
||||
std::vector<uint32_t> m_ext_inst_imports;
|
||||
std::vector<uint32_t> m_memory_model;
|
||||
std::vector<uint32_t> m_entry_points;
|
||||
std::vector<uint32_t> m_execution_modes;
|
||||
std::vector<uint32_t> m_debug;
|
||||
std::vector<uint32_t> m_annotations;
|
||||
std::vector<uint32_t> m_types;
|
||||
std::vector<uint32_t> m_functions;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ */
|
||||
@@ -0,0 +1,500 @@
|
||||
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
|
||||
|
||||
#include "graphics/shader/recompiler/ir/SrtWalker.h"
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cinttypes>
|
||||
#include <cstdio>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
namespace {
|
||||
|
||||
bool Fail(std::string* error, const std::string& message) {
|
||||
if (error != nullptr) {
|
||||
*error = message;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
|
||||
using Kind = IR::DescriptorBindingKind;
|
||||
using Dim = Decoder::ImageDimension;
|
||||
if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
|
||||
const bool integer = image.kind == IR::ResourceKind::ImageUint;
|
||||
switch (image.dimension) {
|
||||
case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
|
||||
case Dim::Dim1DArray:
|
||||
kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
|
||||
return true;
|
||||
case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
|
||||
case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
|
||||
case Dim::Dim2DArray:
|
||||
kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
|
||||
return true;
|
||||
case Dim::Unknown: return false;
|
||||
}
|
||||
}
|
||||
const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
|
||||
if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
|
||||
return false;
|
||||
}
|
||||
switch (image.dimension) {
|
||||
case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
|
||||
case Dim::Dim1DArray:
|
||||
kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
|
||||
return true;
|
||||
case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
|
||||
case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
|
||||
case Dim::Dim2DArray:
|
||||
kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
|
||||
return true;
|
||||
case Dim::Unknown: return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool NeedsSampler(IR::Opcode op) {
|
||||
return op == IR::Opcode::ImageSample || op == IR::Opcode::ImageGather4 ||
|
||||
op == IR::Opcode::ImageGetLod;
|
||||
}
|
||||
|
||||
bool IsBufferLoad(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::BufferLoadUbyte:
|
||||
case IR::Opcode::BufferLoadSbyte:
|
||||
case IR::Opcode::BufferLoadUshort:
|
||||
case IR::Opcode::BufferLoadSshort:
|
||||
case IR::Opcode::BufferLoadDword: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool IsBufferStore(IR::Opcode op) {
|
||||
return op == IR::Opcode::BufferStoreByte || op == IR::Opcode::BufferStoreShort ||
|
||||
op == IR::Opcode::BufferStoreDword;
|
||||
}
|
||||
|
||||
bool IsFlatLoad(IR::Opcode op) {
|
||||
return op == IR::Opcode::FlatLoadUbyte || op == IR::Opcode::FlatLoadSbyte ||
|
||||
op == IR::Opcode::FlatLoadUshort || op == IR::Opcode::FlatLoadSshort ||
|
||||
op == IR::Opcode::FlatLoadDword;
|
||||
}
|
||||
|
||||
bool IsFlatStore(IR::Opcode op) {
|
||||
return op == IR::Opcode::FlatStoreByte || op == IR::Opcode::FlatStoreShort ||
|
||||
op == IR::Opcode::FlatStoreDword;
|
||||
}
|
||||
|
||||
bool IsDsKind(IR::ResourceKind kind) {
|
||||
return kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds;
|
||||
}
|
||||
|
||||
bool IsDsRead(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::DsReadUbyte:
|
||||
case IR::Opcode::DsReadSbyte:
|
||||
case IR::Opcode::DsReadUshort:
|
||||
case IR::Opcode::DsReadSshort:
|
||||
case IR::Opcode::DsReadB32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool IsDsWrite(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::DsWriteByte:
|
||||
case IR::Opcode::DsWriteShort:
|
||||
case IR::Opcode::DsWriteB32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool IsAtomic(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::AtomicSwapU32:
|
||||
case IR::Opcode::AtomicAddU32:
|
||||
case IR::Opcode::AtomicSubU32:
|
||||
case IR::Opcode::AtomicSMinI32:
|
||||
case IR::Opcode::AtomicUMinU32:
|
||||
case IR::Opcode::AtomicSMaxI32:
|
||||
case IR::Opcode::AtomicUMaxU32:
|
||||
case IR::Opcode::AtomicAndU32:
|
||||
case IR::Opcode::AtomicOrU32:
|
||||
case IR::Opcode::AtomicXorU32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t BufferAddressOperandCount(const IR::Instruction& inst) {
|
||||
return 1u + (inst.memory.idxen ? 1u : 0u) + (inst.memory.offen ? 1u : 0u);
|
||||
}
|
||||
|
||||
bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error) {
|
||||
if (inst.src_count > std::size(inst.src)) {
|
||||
return Fail(error, "instruction source count exceeds fixed IR storage");
|
||||
}
|
||||
const auto kind = inst.memory.kind;
|
||||
const auto flat_kind = kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global ||
|
||||
kind == IR::ResourceKind::Scratch;
|
||||
const auto buffer_address_count = BufferAddressOperandCount(inst);
|
||||
if ((IsBufferLoad(inst.op) &&
|
||||
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count)) ||
|
||||
(IsBufferStore(inst.op) &&
|
||||
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) ||
|
||||
(inst.op == IR::Opcode::SLoadDword &&
|
||||
(kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
|
||||
(inst.op == IR::Opcode::SBufferLoadDword &&
|
||||
(kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
|
||||
(IsFlatLoad(inst.op) && (!flat_kind || inst.src_count != 2)) ||
|
||||
(IsFlatStore(inst.op) && (!flat_kind || inst.src_count != 3))) {
|
||||
return Fail(error, "memory instruction opcode/kind/operand contract is invalid");
|
||||
}
|
||||
const auto ds_kind = IsDsKind(kind);
|
||||
const auto ds_resource = !ds_kind || inst.memory.resource == 0;
|
||||
if ((IsDsRead(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 1 ||
|
||||
inst.dst.kind != IR::OperandKind::Register)) ||
|
||||
(IsDsWrite(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 2 ||
|
||||
inst.dst.kind != IR::OperandKind::Null)) ||
|
||||
((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
|
||||
(!ds_kind || !ds_resource || inst.src_count != 1 ||
|
||||
inst.dst.kind != IR::OperandKind::Register ||
|
||||
(kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) ||
|
||||
((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
|
||||
(!ds_kind || !ds_resource || inst.src_count != 3 ||
|
||||
inst.dst.kind != IR::OperandKind::Null)) ||
|
||||
(inst.op == IR::Opcode::DsWriteAddtidB32 &&
|
||||
(kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 2 ||
|
||||
inst.dst.kind != IR::OperandKind::Null)) ||
|
||||
(inst.op == IR::Opcode::DsReadAddtidB32 &&
|
||||
(kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 1 ||
|
||||
inst.dst.kind != IR::OperandKind::Register)) ||
|
||||
(inst.op == IR::Opcode::DsSwizzleB32 &&
|
||||
(kind != IR::ResourceKind::None || inst.src_count != 2 ||
|
||||
inst.dst.kind != IR::OperandKind::Register))) {
|
||||
return Fail(error, "DS instruction has an invalid resource/operand contract");
|
||||
}
|
||||
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4 ||
|
||||
inst.op == IR::Opcode::ImageGetLod || inst.op == IR::Opcode::ImageLoad ||
|
||||
inst.op == IR::Opcode::ImageGetResinfo) &&
|
||||
((kind != IR::ResourceKind::Image && kind != IR::ResourceKind::ImageUint) ||
|
||||
inst.src_count != 1)) {
|
||||
return Fail(error, "sampled image instruction has an invalid resource class");
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageGather4 &&
|
||||
(inst.memory.image_dimension == Decoder::ImageDimension::Dim1D ||
|
||||
inst.memory.image_dimension == Decoder::ImageDimension::Dim1DArray)) {
|
||||
return Fail(error, "1D image gather is not supported by SPIR-V");
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageStore &&
|
||||
((kind != IR::ResourceKind::StorageImage && kind != IR::ResourceKind::StorageImageUint) ||
|
||||
inst.src_count != 2)) {
|
||||
return Fail(error, "storage image instruction has an invalid resource class");
|
||||
}
|
||||
if (IsAtomic(inst.op) &&
|
||||
!((kind == IR::ResourceKind::Buffer && inst.src_count == buffer_address_count + 1u) ||
|
||||
((kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds) &&
|
||||
inst.memory.resource == 0 && inst.src_count == 2 &&
|
||||
(inst.dst.kind == IR::OperandKind::Null ||
|
||||
inst.dst.kind == IR::OperandKind::Register)) ||
|
||||
(kind == IR::ResourceKind::StorageImageUint && inst.src_count == 2))) {
|
||||
return Fail(error, "atomic instruction has an invalid resource/operand contract");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
|
||||
using Kind = IR::DescriptorBindingKind;
|
||||
constexpr auto KindCount = static_cast<size_t>(Kind::Count);
|
||||
std::array<std::vector<uint32_t>, KindCount> expected;
|
||||
std::array<bool, KindCount> present {};
|
||||
const auto Dense = [](size_t size) {
|
||||
std::vector<uint32_t> values(size);
|
||||
for (uint32_t i = 0; i < values.size(); i++) {
|
||||
values[i] = i;
|
||||
}
|
||||
return values;
|
||||
};
|
||||
auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
|
||||
const auto index = static_cast<size_t>(kind);
|
||||
present[index] = true;
|
||||
expected[index] = std::move(resources);
|
||||
};
|
||||
if (!program.info.buffers.empty()) {
|
||||
Expect(Kind::Buffers, Dense(program.info.buffers.size()));
|
||||
}
|
||||
for (uint32_t i = 0; i < program.info.images.size(); i++) {
|
||||
Kind kind;
|
||||
if (!ImageBinding(program.info.images[i], kind)) {
|
||||
return Fail(error, "native shader plan has an invalid image class");
|
||||
}
|
||||
present[static_cast<size_t>(kind)] = true;
|
||||
expected[static_cast<size_t>(kind)].push_back(i);
|
||||
}
|
||||
if (!program.info.samplers.empty()) {
|
||||
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
|
||||
}
|
||||
const auto uses_gds =
|
||||
std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) {
|
||||
return std::any_of(
|
||||
block.instructions.begin(), block.instructions.end(),
|
||||
[](const auto& inst) { return inst.memory.kind == IR::ResourceKind::Gds; });
|
||||
});
|
||||
if (uses_gds) {
|
||||
Expect(Kind::Gds);
|
||||
}
|
||||
if (!program.info.addresses.empty()) {
|
||||
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
|
||||
}
|
||||
if (!program.srt.reads.empty()) {
|
||||
Expect(Kind::FlattenedSrt);
|
||||
}
|
||||
if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
|
||||
Expect(Kind::UserData);
|
||||
}
|
||||
|
||||
std::array<bool, KindCount> seen {};
|
||||
for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
|
||||
const auto& binding = program.bindings.descriptors[i];
|
||||
const auto kind = static_cast<size_t>(binding.kind);
|
||||
if (kind >= KindCount || seen[kind] || !present[kind] ||
|
||||
binding.resources != expected[kind]) {
|
||||
return Fail(error, "native descriptor groups do not match shader topology");
|
||||
}
|
||||
for (uint32_t j = 0; j < i; j++) {
|
||||
if (program.bindings.descriptors[j].binding == binding.binding) {
|
||||
return Fail(error, "native descriptor binding numbers are not unique");
|
||||
}
|
||||
}
|
||||
seen[kind] = true;
|
||||
}
|
||||
for (size_t i = 0; i < KindCount; i++) {
|
||||
if (present[i] != seen[i]) {
|
||||
return Fail(error, "native shader plan is missing a required descriptor group");
|
||||
}
|
||||
}
|
||||
const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
|
||||
if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
|
||||
program.bindings.buffer_offset_count != program.info.buffers.size() ||
|
||||
(!has_user_storage &&
|
||||
program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
|
||||
(has_user_storage && program.bindings.push_constant_size != 0) ||
|
||||
!std::is_sorted(program.bindings.user_data_registers.begin(),
|
||||
program.bindings.user_data_registers.end()) ||
|
||||
std::adjacent_find(program.bindings.user_data_registers.begin(),
|
||||
program.bindings.user_data_registers.end()) !=
|
||||
program.bindings.user_data_registers.end()) {
|
||||
return Fail(error, "native user-data layout is inconsistent");
|
||||
}
|
||||
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (!ValidateInstructionContract(inst, error)) {
|
||||
return false;
|
||||
}
|
||||
const auto memory = inst.memory.kind;
|
||||
if ((memory == IR::ResourceKind::Buffer || (memory == IR::ResourceKind::ScalarBuffer &&
|
||||
inst.op == IR::Opcode::SBufferLoadDword)) &&
|
||||
inst.memory.resource >= program.info.buffers.size()) {
|
||||
return Fail(error, "buffer instruction has an invalid dense resource");
|
||||
}
|
||||
if ((memory == IR::ResourceKind::Image || memory == IR::ResourceKind::ImageUint ||
|
||||
memory == IR::ResourceKind::StorageImage ||
|
||||
memory == IR::ResourceKind::StorageImageUint) &&
|
||||
(inst.memory.resource >= program.info.images.size() ||
|
||||
program.info.images[inst.memory.resource].kind != memory ||
|
||||
program.info.images[inst.memory.resource].dimension !=
|
||||
inst.memory.image_dimension)) {
|
||||
return Fail(error, "image instruction has an invalid dense resource");
|
||||
}
|
||||
const bool address =
|
||||
inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat ||
|
||||
memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch;
|
||||
if (address && (inst.memory.resource >= program.info.addresses.size() ||
|
||||
program.info.addresses[inst.memory.resource].kind != memory)) {
|
||||
return Fail(error, "address instruction has an invalid dense resource");
|
||||
}
|
||||
if (NeedsSampler(inst.op) && inst.memory.sampler >= program.info.samplers.size()) {
|
||||
return Fail(error, "sample instruction has an invalid dense sampler");
|
||||
}
|
||||
if (inst.op == IR::Opcode::LoadSrtDword &&
|
||||
(inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32 ||
|
||||
inst.src[0].imm >= program.srt.reads.size())) {
|
||||
return Fail(error, "flattened SRT instruction has an invalid dense slot");
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
static constexpr size_t InitialEmitterVectorReserve = 4096;
|
||||
|
||||
static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
|
||||
size_t size) {
|
||||
if (size <= InitialEmitterVectorReserve) {
|
||||
return;
|
||||
}
|
||||
std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
|
||||
" stage=%u vector=%s size=%zu reserve=%zu\n",
|
||||
program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
|
||||
InitialEmitterVectorReserve);
|
||||
}
|
||||
|
||||
static bool IsExecDestination(const IR::Operand& operand) {
|
||||
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Exec;
|
||||
}
|
||||
|
||||
static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruction& inst) {
|
||||
if (inst.op != IR::Opcode::BitFieldMaskU64 || !IsExecDestination(inst.dst) ||
|
||||
inst.src_count < 1u) {
|
||||
return false;
|
||||
}
|
||||
uint32_t count = 0;
|
||||
return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], count) &&
|
||||
(count & 63u) == 0u;
|
||||
}
|
||||
|
||||
static bool ProducesLaneMask(const IR::Instruction& inst) {
|
||||
switch (inst.op) {
|
||||
case IR::Opcode::IAddCarryU32:
|
||||
case IR::Opcode::ISubBorrowU32:
|
||||
case IR::Opcode::UMadU64U32: return inst.dst2.kind != IR::OperandKind::Null;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
if (block.terminator.kind == CFG::TerminatorKind::ConditionalBranch) {
|
||||
switch (block.terminator.condition) {
|
||||
case CFG::BranchCondition::VccZero:
|
||||
case CFG::BranchCondition::VccNonZero:
|
||||
case CFG::BranchCondition::ExecZero:
|
||||
case CFG::BranchCondition::ExecNonZero: return true;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (Emitter::InstructionHasDppSource(inst)) {
|
||||
return true;
|
||||
}
|
||||
if (Emitter::IsCompareOpcode(inst.op) && !Emitter::IsSccOperand(inst.dst)) {
|
||||
return true;
|
||||
}
|
||||
if ((IsExecDestination(inst.dst) || IsExecDestination(inst.dst2)) &&
|
||||
!IsUniformZeroExecWrite(program, inst)) {
|
||||
return true;
|
||||
}
|
||||
if (ProducesLaneMask(inst)) {
|
||||
return true;
|
||||
}
|
||||
if (inst.op == IR::Opcode::SelectMaskU32 || inst.op == IR::Opcode::SelectMaskF32Bits) {
|
||||
return true;
|
||||
}
|
||||
if (inst.op == IR::Opcode::SaveexecB32 || inst.op == IR::Opcode::SaveexecB64) {
|
||||
return true;
|
||||
}
|
||||
switch (inst.op) {
|
||||
case IR::Opcode::WqmB64:
|
||||
if (inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file != IR::RegisterFile::Scalar) {
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
case IR::Opcode::MaskedBitCountLowU32:
|
||||
case IR::Opcode::MaskedBitCountHighU32:
|
||||
case IR::Opcode::ReadFirstLaneU32:
|
||||
case IR::Opcode::ReadLaneU32:
|
||||
case IR::Opcode::WriteLaneU32:
|
||||
case IR::Opcode::Permlane16B32:
|
||||
case IR::Opcode::Permlanex16B32:
|
||||
case IR::Opcode::DsSwizzleB32:
|
||||
case IR::Opcode::DsConsume:
|
||||
case IR::Opcode::DsAppend: return true;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
|
||||
const ShaderVertexInputInfo* vertex_input_info,
|
||||
const ShaderPixelInputInfo* pixel_input_info,
|
||||
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
|
||||
std::string* error) {
|
||||
using namespace Emitter;
|
||||
|
||||
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
|
||||
program.stage != ShaderType::Pixel) {
|
||||
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
|
||||
return false;
|
||||
}
|
||||
if (!program.srt_patching_complete || !program.resource_tracking_complete ||
|
||||
!program.shader_info_complete || !program.binding_layout_complete) {
|
||||
SetError(error, "SPIR-V emitter requires a fully planned native shader program");
|
||||
return false;
|
||||
}
|
||||
if (!IR::ValidateResourceSnapshot(program, resources, error)) {
|
||||
return false;
|
||||
}
|
||||
if (!IR::ValidateResourceSpecialization(program, resources, error)) {
|
||||
return false;
|
||||
}
|
||||
if (!ValidateNativeProgram(program, error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
EmitterState state(program, resources);
|
||||
state.vertex_input_info = vertex_input_info;
|
||||
state.pixel_input_info = pixel_input_info;
|
||||
state.compute_input_info = compute_input_info;
|
||||
state.stage = program.stage;
|
||||
state.wave_size = program.wave_size;
|
||||
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
|
||||
state.exact_subgroup_operations =
|
||||
!state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
|
||||
state.registers.reserve(InitialEmitterVectorReserve);
|
||||
state.inputs.reserve(InitialEmitterVectorReserve);
|
||||
state.outputs.reserve(InitialEmitterVectorReserve);
|
||||
state.interface_variables.reserve(InitialEmitterVectorReserve);
|
||||
state.reachable_blocks.reserve(InitialEmitterVectorReserve);
|
||||
CollectRegisters(program, state.registers);
|
||||
CopyProgramInputsAndOutputs(state, program);
|
||||
state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program);
|
||||
state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
|
||||
state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
|
||||
state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program);
|
||||
state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program);
|
||||
state.needs_function_lds = ProgramNeedsFunctionLds(program);
|
||||
state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program);
|
||||
ComputeReachableBlocks(state, program);
|
||||
AllocateInputVariables(state);
|
||||
AllocateOutputVariables(state);
|
||||
AllocateDescriptorVariables(state);
|
||||
EmitHeaderAndTypes(state);
|
||||
AllocateRegisterVariables(state);
|
||||
AllocateBlockLabels(state, program);
|
||||
AllocateDispatcherState(state, program);
|
||||
EmitFunction(state, program);
|
||||
|
||||
ReportReserveExceeded(program, "registers", state.registers.size());
|
||||
ReportReserveExceeded(program, "inputs", state.inputs.size());
|
||||
ReportReserveExceeded(program, "outputs", state.outputs.size());
|
||||
ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
|
||||
ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
|
||||
|
||||
auto binary = state.builder.Build();
|
||||
if (binary.empty()) {
|
||||
SetError(error, "SPIR-V builder returned an empty module");
|
||||
return false;
|
||||
}
|
||||
spirv = std::move(binary);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
@@ -0,0 +1,22 @@
|
||||
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_
|
||||
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_
|
||||
|
||||
#include "common/common.h"
|
||||
#include "common/stringUtils.h"
|
||||
#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv {
|
||||
|
||||
bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
|
||||
|
||||
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
|
||||
const ShaderVertexInputInfo* vertex_input_info,
|
||||
const ShaderPixelInputInfo* pixel_input_info,
|
||||
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
|
||||
std::string* error);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_ */
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,725 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
|
||||
const auto* ps = state.pixel_input_info;
|
||||
if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
|
||||
return attr;
|
||||
}
|
||||
// VINTRP ATTR selects the PS input slot. SPI_PS_INPUT_CNTL maps that slot to a
|
||||
// VS parameter export, which is the SPIR-V location we must link against.
|
||||
return ps->interpolator_settings[attr] & PsInputOffsetMask;
|
||||
}
|
||||
|
||||
uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
|
||||
bool used_locations[32] = {};
|
||||
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind != IR::StageInputKind::Parameter) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto location = PixelParameterMappedLocation(state, input.location);
|
||||
if (location < std::size(used_locations) && used_locations[location]) {
|
||||
auto fallback_location = input.location;
|
||||
while (fallback_location < std::size(used_locations) &&
|
||||
used_locations[fallback_location]) {
|
||||
fallback_location++;
|
||||
}
|
||||
EXIT_NOT_IMPLEMENTED(fallback_location >= std::size(used_locations));
|
||||
location = fallback_location;
|
||||
}
|
||||
|
||||
if (input.location == attr) {
|
||||
return location;
|
||||
}
|
||||
|
||||
if (location < std::size(used_locations)) {
|
||||
used_locations[location] = true;
|
||||
}
|
||||
}
|
||||
|
||||
return PixelParameterMappedLocation(state, attr);
|
||||
}
|
||||
|
||||
bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
|
||||
const auto* ps = state.pixel_input_info;
|
||||
if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
|
||||
return false;
|
||||
}
|
||||
return (ps->interpolator_settings[attr] & PsInputFlatShade) != 0;
|
||||
}
|
||||
|
||||
void SetError(std::string* error, const char* message) {
|
||||
if (error != nullptr) {
|
||||
*error = message;
|
||||
}
|
||||
}
|
||||
|
||||
void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg) {
|
||||
if (std::any_of(registers.begin(), registers.end(),
|
||||
[reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
|
||||
return;
|
||||
}
|
||||
registers.push_back({reg, 0});
|
||||
}
|
||||
|
||||
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
|
||||
IR::Operand operand;
|
||||
operand.kind = IR::OperandKind::Register;
|
||||
operand.reg = {file, index};
|
||||
return operand;
|
||||
}
|
||||
|
||||
IR::Register SccRegister() {
|
||||
return {IR::RegisterFile::Scc, static_cast<uint32_t>(Decoder::OperandKind::Scc)};
|
||||
}
|
||||
|
||||
IR::Operand SccOperand() {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Scc,
|
||||
static_cast<uint32_t>(Decoder::OperandKind::Scc));
|
||||
}
|
||||
|
||||
bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
|
||||
return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
|
||||
}
|
||||
|
||||
bool IsMaskRegisterFile(IR::RegisterFile file) {
|
||||
return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc;
|
||||
}
|
||||
|
||||
bool IsSccOperand(const IR::Operand& operand) {
|
||||
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
|
||||
}
|
||||
|
||||
bool IsCompareOpcode(IR::Opcode op);
|
||||
|
||||
void CollectMaskStateRegisters(std::vector<RegisterBinding>& registers) {
|
||||
CollectRegister(registers, {IR::RegisterFile::Exec, 0});
|
||||
CollectRegister(registers, {IR::RegisterFile::Exec, 1});
|
||||
CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
|
||||
CollectRegister(registers, {IR::RegisterFile::Vcc, 1});
|
||||
CollectRegister(registers, SccRegister());
|
||||
}
|
||||
|
||||
void CollectSequentialRegisters(std::vector<RegisterBinding>& registers, const IR::Operand& base,
|
||||
uint32_t count) {
|
||||
if (base.kind != IR::OperandKind::Register) {
|
||||
return;
|
||||
}
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
auto reg = base.reg;
|
||||
reg.index += i;
|
||||
CollectRegister(registers, reg);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& registers) {
|
||||
uint32_t max_end = 0;
|
||||
for (const auto& binding: registers) {
|
||||
if (binding.reg.file == IR::RegisterFile::Vector) {
|
||||
max_end = std::max(max_end, binding.reg.index + 1u);
|
||||
}
|
||||
}
|
||||
return std::min(max_end, 256u);
|
||||
}
|
||||
|
||||
void CollectMoveRelSourceRegisters(const IR::Program& program,
|
||||
std::vector<RegisterBinding>& registers) {
|
||||
const auto max_vector_end = MaxCollectedVectorRegisterEnd(registers);
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op != IR::Opcode::MoveRelSourceU32 ||
|
||||
inst.src[0].kind != IR::OperandKind::Register ||
|
||||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
|
||||
continue;
|
||||
}
|
||||
const auto base_index = inst.src[0].reg.index;
|
||||
const auto limit = std::min(std::max(max_vector_end, base_index + 1u), 256u);
|
||||
CollectSequentialRegisters(registers, inst.src[0], limit - base_index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool IsPairDwordOpcode(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::MoveU64:
|
||||
case IR::Opcode::WqmB64:
|
||||
case IR::Opcode::SaveexecB64:
|
||||
case IR::Opcode::BitwiseAndU64:
|
||||
case IR::Opcode::BitwiseAndNotU64:
|
||||
case IR::Opcode::BitwiseOrU64:
|
||||
case IR::Opcode::BitwiseOrNotU64:
|
||||
case IR::Opcode::BitwiseXorU64:
|
||||
case IR::Opcode::BitwiseNandU64:
|
||||
case IR::Opcode::BitwiseNorU64:
|
||||
case IR::Opcode::BitwiseXnorU64:
|
||||
case IR::Opcode::BitwiseNotU64:
|
||||
case IR::Opcode::BitFieldMaskU64:
|
||||
case IR::Opcode::BitFieldExtractU64:
|
||||
case IR::Opcode::BitReplicateB64B32:
|
||||
case IR::Opcode::ShiftLeftLogicalU64:
|
||||
case IR::Opcode::ShiftRightLogicalU64:
|
||||
case IR::Opcode::SelectU64: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
|
||||
switch (op) {
|
||||
case IR::Opcode::BitFieldMaskU64: return 0;
|
||||
case IR::Opcode::BitReplicateB64B32: return 0;
|
||||
case IR::Opcode::BitFieldExtractU64: return 1;
|
||||
case IR::Opcode::ShiftLeftLogicalU64:
|
||||
case IR::Opcode::ShiftRightLogicalU64:
|
||||
case IR::Opcode::BitwiseNotU64:
|
||||
case IR::Opcode::SaveexecB64:
|
||||
case IR::Opcode::WqmB64:
|
||||
case IR::Opcode::MoveU64: return std::min<uint32_t>(src_count, 1u);
|
||||
default: return src_count;
|
||||
}
|
||||
}
|
||||
|
||||
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>& registers) {
|
||||
CollectMaskStateRegisters(registers);
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.dst.kind == IR::OperandKind::Register) {
|
||||
CollectRegister(registers, inst.dst.reg);
|
||||
}
|
||||
if (inst.dst2.kind == IR::OperandKind::Register) {
|
||||
CollectRegister(registers, inst.dst2.reg);
|
||||
}
|
||||
for (uint32_t i = 0; i < inst.src_count; i++) {
|
||||
if (inst.src[i].kind == IR::OperandKind::Register) {
|
||||
CollectRegister(registers, inst.src[i].reg);
|
||||
}
|
||||
}
|
||||
if (IsPairDwordOpcode(inst.op)) {
|
||||
CollectSequentialRegisters(registers, inst.dst, 2);
|
||||
const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u;
|
||||
const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count);
|
||||
for (uint32_t i = first_pair_src; i < pair_src_count; i++) {
|
||||
CollectSequentialRegisters(registers, inst.src[i], 2);
|
||||
}
|
||||
}
|
||||
if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) {
|
||||
CollectSequentialRegisters(registers, inst.src[0], 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::CompareNeU64) {
|
||||
CollectSequentialRegisters(registers, inst.src[0], 2);
|
||||
CollectSequentialRegisters(registers, inst.src[1], 2);
|
||||
}
|
||||
if (IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file != IR::RegisterFile::Scc) {
|
||||
CollectSequentialRegisters(registers, inst.dst, 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) {
|
||||
CollectSequentialRegisters(registers, inst.dst2, 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::UMadU64U32) {
|
||||
CollectSequentialRegisters(registers, inst.dst, 2);
|
||||
CollectSequentialRegisters(registers, inst.dst2, 2);
|
||||
CollectSequentialRegisters(registers, inst.src[2], 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::SBufferLoadDword &&
|
||||
inst.memory.kind == IR::ResourceKind::ScalarBuffer) {
|
||||
CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u});
|
||||
}
|
||||
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) &&
|
||||
inst.memory.image_address_components > 1u) {
|
||||
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[0],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageGetLod) {
|
||||
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[0],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageLoad) {
|
||||
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[0],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageStore) {
|
||||
CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[1],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.memory.kind == IR::ResourceKind::StorageImageUint) {
|
||||
CollectSequentialRegisters(registers, inst.src[1],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
}
|
||||
}
|
||||
CollectMoveRelSourceRegisters(program, registers);
|
||||
}
|
||||
|
||||
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
|
||||
uint32_t index) {
|
||||
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
|
||||
return binding.kind == kind && binding.index == index;
|
||||
});
|
||||
}
|
||||
|
||||
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
|
||||
for (const auto& input: program.info.inputs) {
|
||||
state.inputs.push_back(
|
||||
{input.kind, input.location, input.component_count, 0, input.debug_name});
|
||||
}
|
||||
for (const auto& output: program.info.outputs) {
|
||||
if (HasOutput(state.outputs, output.kind, output.index)) {
|
||||
continue;
|
||||
}
|
||||
state.outputs.push_back(
|
||||
{output.kind, output.index, output.location, 0, output.debug_name});
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
|
||||
if (exp.kind == IR::ExportTargetKind::Position) {
|
||||
return state.per_vertex_variable;
|
||||
}
|
||||
if (exp.kind == IR::ExportTargetKind::MrtZ) {
|
||||
return state.depth_variable;
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
const auto expected_kind = exp.kind == IR::ExportTargetKind::Mrt
|
||||
? IR::StageOutputKind::Mrt
|
||||
: IR::StageOutputKind::Parameter;
|
||||
if (binding.kind == expected_kind && binding.index == exp.index) {
|
||||
return binding.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::ImageGetLod) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::ImageGather4) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IsLdsOpcode(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::DsReadUbyte:
|
||||
case IR::Opcode::DsReadSbyte:
|
||||
case IR::Opcode::DsReadUshort:
|
||||
case IR::Opcode::DsReadSshort:
|
||||
case IR::Opcode::DsReadB32:
|
||||
case IR::Opcode::DsWriteByte:
|
||||
case IR::Opcode::DsWriteShort:
|
||||
case IR::Opcode::DsWriteB32:
|
||||
case IR::Opcode::DsMinF32:
|
||||
case IR::Opcode::DsMaxF32:
|
||||
case IR::Opcode::DsSwizzleB32:
|
||||
case IR::Opcode::DsWriteAddtidB32:
|
||||
case IR::Opcode::DsReadAddtidB32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool ProgramNeedsFunctionLds(const IR::Program& program) {
|
||||
if (program.stage == ShaderType::Compute) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (IsLdsOpcode(inst.op)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsPixelValidMask(const IR::Program& program) {
|
||||
if (program.stage != ShaderType::Pixel) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::Export && inst.export_info.vm) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool InstructionHasDppSource(const IR::Instruction& inst) {
|
||||
for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) {
|
||||
if (inst.src[i].dpp) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
|
||||
if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::WqmB64) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ProgramRequiresExactSubgroupSize(program);
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 ||
|
||||
inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 ||
|
||||
inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume ||
|
||||
inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IsCompareOpcode(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::CompareFalse:
|
||||
case IR::Opcode::CompareTrue:
|
||||
case IR::Opcode::CompareEqU32:
|
||||
case IR::Opcode::CompareNeU32:
|
||||
case IR::Opcode::CompareGtU32:
|
||||
case IR::Opcode::CompareGeU32:
|
||||
case IR::Opcode::CompareLtU32:
|
||||
case IR::Opcode::CompareLeU32:
|
||||
case IR::Opcode::CompareNeU64:
|
||||
case IR::Opcode::CompareMaskEqU32:
|
||||
case IR::Opcode::CompareMaskNeU32:
|
||||
case IR::Opcode::CompareMaskGtU32:
|
||||
case IR::Opcode::CompareMaskGeU32:
|
||||
case IR::Opcode::CompareMaskLtU32:
|
||||
case IR::Opcode::CompareMaskLeU32:
|
||||
case IR::Opcode::CompareEqI32:
|
||||
case IR::Opcode::CompareNeI32:
|
||||
case IR::Opcode::CompareGtI32:
|
||||
case IR::Opcode::CompareGeI32:
|
||||
case IR::Opcode::CompareLtI32:
|
||||
case IR::Opcode::CompareLeI32:
|
||||
case IR::Opcode::CompareEqI16:
|
||||
case IR::Opcode::CompareNeI16:
|
||||
case IR::Opcode::CompareGtI16:
|
||||
case IR::Opcode::CompareGeI16:
|
||||
case IR::Opcode::CompareLtI16:
|
||||
case IR::Opcode::CompareLeI16:
|
||||
case IR::Opcode::CompareMaskEqI32:
|
||||
case IR::Opcode::CompareMaskNeI32:
|
||||
case IR::Opcode::CompareMaskGtI32:
|
||||
case IR::Opcode::CompareMaskGeI32:
|
||||
case IR::Opcode::CompareMaskLtI32:
|
||||
case IR::Opcode::CompareMaskLeI32:
|
||||
case IR::Opcode::CompareEqU16:
|
||||
case IR::Opcode::CompareNeU16:
|
||||
case IR::Opcode::CompareGtU16:
|
||||
case IR::Opcode::CompareGeU16:
|
||||
case IR::Opcode::CompareLtU16:
|
||||
case IR::Opcode::CompareLeU16:
|
||||
case IR::Opcode::CompareEqF32:
|
||||
case IR::Opcode::CompareNeF32:
|
||||
case IR::Opcode::CompareGtF32:
|
||||
case IR::Opcode::CompareGeF32:
|
||||
case IR::Opcode::CompareLtF32:
|
||||
case IR::Opcode::CompareLeF32:
|
||||
case IR::Opcode::CompareOrderedF32:
|
||||
case IR::Opcode::CompareUnorderedF32:
|
||||
case IR::Opcode::CompareUnordEqF32:
|
||||
case IR::Opcode::CompareUnordNeF32:
|
||||
case IR::Opcode::CompareUnordGtF32:
|
||||
case IR::Opcode::CompareUnordGeF32:
|
||||
case IR::Opcode::CompareUnordLtF32:
|
||||
case IR::Opcode::CompareUnordLeF32:
|
||||
case IR::Opcode::CompareClassF32:
|
||||
case IR::Opcode::CompareEqF16:
|
||||
case IR::Opcode::CompareNeF16:
|
||||
case IR::Opcode::CompareGtF16:
|
||||
case IR::Opcode::CompareGeF16:
|
||||
case IR::Opcode::CompareLtF16:
|
||||
case IR::Opcode::CompareLeF16:
|
||||
case IR::Opcode::CompareUnordNeF16:
|
||||
case IR::Opcode::CompareMaskEqF16:
|
||||
case IR::Opcode::CompareMaskNeF16:
|
||||
case IR::Opcode::CompareMaskGtF16:
|
||||
case IR::Opcode::CompareMaskGeF16:
|
||||
case IR::Opcode::CompareMaskLtF16:
|
||||
case IR::Opcode::CompareMaskLeF16:
|
||||
case IR::Opcode::CompareMaskUnordNeF16:
|
||||
case IR::Opcode::CompareMaskUnordGeF16:
|
||||
case IR::Opcode::CompareMaskEqF32:
|
||||
case IR::Opcode::CompareMaskNeF32:
|
||||
case IR::Opcode::CompareMaskGtF32:
|
||||
case IR::Opcode::CompareMaskGeF32:
|
||||
case IR::Opcode::CompareMaskLtF32:
|
||||
case IR::Opcode::CompareMaskLeF32:
|
||||
case IR::Opcode::CompareMaskUnordEqF32:
|
||||
case IR::Opcode::CompareMaskUnordNeF32:
|
||||
case IR::Opcode::CompareMaskUnordGtF32:
|
||||
case IR::Opcode::CompareMaskUnordGeF32:
|
||||
case IR::Opcode::CompareMaskUnordLtF32:
|
||||
case IR::Opcode::CompareMaskUnordLeF32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 ||
|
||||
inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend ||
|
||||
inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 ||
|
||||
inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export ||
|
||||
InstructionHasDppSource(inst)) {
|
||||
return true;
|
||||
}
|
||||
if (IsCompareOpcode(inst.op)) {
|
||||
return true;
|
||||
}
|
||||
if (inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file == IR::RegisterFile::Vector) {
|
||||
return true;
|
||||
}
|
||||
if (inst.dst2.kind == IR::OperandKind::Register &&
|
||||
inst.dst2.reg.file == IR::RegisterFile::Vector) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg == reg) {
|
||||
return binding.pointer_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState& state, uint32_t value);
|
||||
void EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
|
||||
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* reason) {
|
||||
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
|
||||
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
|
||||
state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
|
||||
static_cast<unsigned>(kind), reason);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource) {
|
||||
const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
|
||||
if (descriptor == nullptr) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
|
||||
}
|
||||
const auto found =
|
||||
std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
|
||||
if (found == descriptor->resources.end()) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource,
|
||||
"resource is absent from descriptor group");
|
||||
}
|
||||
return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
|
||||
}
|
||||
|
||||
uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
|
||||
uint32_t variable_id, uint32_t array_index,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* variable_name) {
|
||||
if (variable_id == 0) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource, variable_name);
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
ImageViewKind ImageViewKindFromDimension(Decoder::ImageDimension dimension) {
|
||||
switch (dimension) {
|
||||
case Decoder::ImageDimension::Dim1D: return ImageViewKind::Dim1D;
|
||||
case Decoder::ImageDimension::Dim1DArray: return ImageViewKind::Dim1DArray;
|
||||
case Decoder::ImageDimension::Dim2DArray: return ImageViewKind::Dim2DArray;
|
||||
case Decoder::ImageDimension::Dim3D: return ImageViewKind::Dim3D;
|
||||
default: return ImageViewKind::Dim2D;
|
||||
}
|
||||
}
|
||||
|
||||
ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc) {
|
||||
(void)state;
|
||||
(void)use_pc;
|
||||
return ImageViewKindFromDimension(mem.image_dimension);
|
||||
}
|
||||
|
||||
ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
bool uint_image, uint32_t use_pc) {
|
||||
(void)state;
|
||||
(void)uint_image;
|
||||
(void)use_pc;
|
||||
return ImageViewKindFromDimension(mem.image_dimension);
|
||||
}
|
||||
|
||||
uint32_t ImageViewCoordinateComponents(ImageViewKind view) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim1D: return 1u;
|
||||
case ImageViewKind::Dim1DArray:
|
||||
case ImageViewKind::Dim2D: return 2u;
|
||||
case ImageViewKind::Dim2DArray:
|
||||
case ImageViewKind::Dim3D: return 3u;
|
||||
default: return 0u;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageViewSpatialComponents(ImageViewKind view) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim1D:
|
||||
case ImageViewKind::Dim1DArray: return 1u;
|
||||
case ImageViewKind::Dim2D:
|
||||
case ImageViewKind::Dim2DArray: return 2u;
|
||||
case ImageViewKind::Dim3D: return 3u;
|
||||
default: return 0u;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) {
|
||||
return state.sampled_images[SampledImageIndex(integer, view)].image_type;
|
||||
}
|
||||
|
||||
uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) {
|
||||
return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type;
|
||||
}
|
||||
|
||||
uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) {
|
||||
switch (ImageViewCoordinateComponents(view)) {
|
||||
case 1u: return state.uint_type;
|
||||
case 2u: return state.vec2_uint_type;
|
||||
case 3u: return state.vec3_uint_type;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
return state.storage_images[StorageImageIndex(uint_image, view)].image_type;
|
||||
}
|
||||
|
||||
uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
return state.storage_images[StorageImageIndex(uint_image, view)].pointer_type;
|
||||
}
|
||||
|
||||
uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
return state.storage_images[StorageImageIndex(uint_image, view)].variable;
|
||||
}
|
||||
|
||||
uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
|
||||
ImageViewKind view) {
|
||||
(void)use_pc;
|
||||
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
|
||||
const auto kind = SampledBindingKind(integer, view);
|
||||
const auto binding = ResourceForDescriptor(state, kind, mem.resource);
|
||||
const auto& descriptors = state.sampled_images[SampledImageIndex(integer, view)];
|
||||
const auto pointer = DescriptorElementPointer(
|
||||
state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
|
||||
mem.resource, "sampled image descriptor array was not emitted");
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
|
||||
(void)use_pc;
|
||||
const auto binding =
|
||||
ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
|
||||
const auto pointer = DescriptorElementPointer(
|
||||
state, state.ptr_uniform_sampler, state.sampler_variable, binding.array_index,
|
||||
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
|
||||
const auto sampler_id = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.sampler_type, sampler_id, pointer});
|
||||
return sampler_id;
|
||||
}
|
||||
|
||||
uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
|
||||
ImageViewKind view) {
|
||||
const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view);
|
||||
const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
|
||||
if (image == 0 || sampler == 0) {
|
||||
ExitDescriptorBindingFailure(
|
||||
state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
|
||||
"sampled image or sampler descriptor load failed");
|
||||
}
|
||||
const auto sampled_image = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSampledImage,
|
||||
ImageViewSampledImageType(state, view, mem.kind == IR::ResourceKind::ImageUint),
|
||||
sampled_image, image, sampler});
|
||||
return sampled_image;
|
||||
}
|
||||
|
||||
uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
|
||||
uint32_t use_pc, ImageViewKind view) {
|
||||
(void)use_pc;
|
||||
const auto kind = StorageBindingKind(uint_image, view);
|
||||
const auto binding = ResourceForDescriptor(state, kind, resource);
|
||||
const auto ptr_type = StorageImagePointerType(state, uint_image, view);
|
||||
const auto variable = StorageImageVariable(state, uint_image, view);
|
||||
return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
|
||||
"storage image descriptor array was not emitted");
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool uint_image,
|
||||
uint32_t use_pc, ImageViewKind view) {
|
||||
const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
|
||||
if (pointer == 0) {
|
||||
ExitDescriptorBindingFailure(state, StorageBindingKind(uint_image, view), resource,
|
||||
"storage image descriptor pointer creation failed");
|
||||
}
|
||||
const auto type = StorageImageType(state, uint_image, view);
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
uint32_t ExecutionModelForStage(ShaderType stage) {
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex: return ExecutionModelVertex;
|
||||
case ShaderType::Pixel: return ExecutionModelFragment;
|
||||
default: return ExecutionModelGLCompute;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,171 @@
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
|
||||
if (!enabled || component >= inst.src_count || component >= 4u) {
|
||||
return ConstantF32(state, component == 3u ? 0x3f800000u : 0u);
|
||||
}
|
||||
return EmitFloatLoad(state, inst.src[component]);
|
||||
}
|
||||
|
||||
uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.compr) {
|
||||
uint32_t components[4] = {
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0x3f800000u),
|
||||
};
|
||||
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
|
||||
pair_index++) {
|
||||
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
|
||||
state.glsl_std450, GlslUnpackHalf2x16, raw});
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair_index * 2u + lane;
|
||||
if (((inst.export_info.en >> component) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
components[component] = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, components[component], unpacked, lane});
|
||||
}
|
||||
}
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec,
|
||||
components[0], components[1], components[2], components[3]});
|
||||
return vec;
|
||||
}
|
||||
|
||||
const auto x = EmitExportComponentF32(state, inst, 0);
|
||||
const auto y = EmitExportComponentF32(state, inst, 1);
|
||||
const auto z = EmitExportComponentF32(state, inst, 2);
|
||||
const auto w = EmitExportComponentF32(state, inst, 3);
|
||||
const auto vec = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w});
|
||||
return vec;
|
||||
}
|
||||
|
||||
uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value) {
|
||||
if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state.pixel_input_info == nullptr ||
|
||||
inst.export_info.index >= state.pixel_input_info->target_export_mapping.size()) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto mapping = state.pixel_input_info->target_export_mapping[inst.export_info.index];
|
||||
if (mapping.IsIdentity()) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto mapped = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpVectorShuffle, state.vec4_float_type, mapped, value, value,
|
||||
mapping.Map(0), mapping.Map(1), mapping.Map(2), mapping.Map(3)});
|
||||
return mapped;
|
||||
}
|
||||
|
||||
bool ExportWritesData(const IR::Instruction& inst) {
|
||||
switch (inst.export_info.kind) {
|
||||
case IR::ExportTargetKind::Null:
|
||||
case IR::ExportTargetKind::Primitive: return false;
|
||||
case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0;
|
||||
default: return inst.export_info.en != 0;
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) {
|
||||
if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) {
|
||||
state.builder.AddFunction(
|
||||
{OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)});
|
||||
}
|
||||
|
||||
if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) {
|
||||
const auto raw =
|
||||
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto ptr = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, mask, raw});
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr,
|
||||
state.sample_mask_variable, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, ptr, mask});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitExport(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
|
||||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ExportWritesData(inst)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) {
|
||||
EmitMrtZExport(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto variable = OutputVariableForExport(state, inst.export_info);
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto value = ApplyMrtExportMapping(state, inst, EmitExportVec4F32(state, inst));
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer, variable,
|
||||
ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, pointer, value});
|
||||
return;
|
||||
}
|
||||
|
||||
state.builder.AddFunction({OpStore, variable, value});
|
||||
}
|
||||
|
||||
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
|
||||
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
|
||||
}
|
||||
|
||||
void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
|
||||
const auto kill_label = state.builder.AllocateId();
|
||||
const auto merge_label = state.builder.AllocateId();
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
|
||||
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
|
||||
state.builder.AddFunction({OpLabel, kill_label});
|
||||
state.builder.AddFunction({OpKill});
|
||||
state.builder.AddFunction({OpLabel, merge_label});
|
||||
}
|
||||
|
||||
void EmitUpdatePixelValidMask(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
|
||||
}
|
||||
|
||||
void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
|
||||
if (state.pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto mask_value = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
|
||||
EmitKillIfBoolFalse(state, active);
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,381 @@
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitFloatLoad(state, inst.src[0]);
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
|
||||
uint32_t limit = base.index + 1u;
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg.file == IR::RegisterFile::Vector) {
|
||||
limit = std::max(limit, binding.reg.index + 1u);
|
||||
}
|
||||
}
|
||||
return std::min(limit, 256u);
|
||||
}
|
||||
|
||||
void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.src[0].kind != IR::OperandKind::Register ||
|
||||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.src[0].reg;
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
auto selected = EmitRegisterLoad(state, base_reg);
|
||||
const auto limit = MoveRelRegisterLimit(state, base_reg);
|
||||
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
|
||||
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
|
||||
const auto match = state.builder.AllocateId();
|
||||
const auto next = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected});
|
||||
selected = next;
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, selected);
|
||||
}
|
||||
|
||||
void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.dst.kind != IR::OperandKind::Register ||
|
||||
inst.dst.reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.dst.reg;
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
const auto exec_active = EmitExecActiveBool(state);
|
||||
const auto limit = MoveRelRegisterLimit(state, base_reg);
|
||||
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
|
||||
const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg});
|
||||
if (pointer == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto old_value = state.builder.AllocateId();
|
||||
const auto match = state.builder.AllocateId();
|
||||
const auto write = state.builder.AllocateId();
|
||||
const auto merged = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
|
||||
state.builder.AddFunction(
|
||||
{OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value});
|
||||
state.builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
IsMaskRegisterFile(inst.dst.reg.file)) {
|
||||
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
|
||||
return;
|
||||
}
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
EmitStoreU32(state, inst.dst, low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
|
||||
}
|
||||
|
||||
uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
|
||||
uint32_t ret = ConstantU32(state, 0);
|
||||
for (uint32_t i = 0; i < 8u; i++) {
|
||||
const auto mask = 0x0fu << (i * 4u);
|
||||
const auto masked = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto expanded = state.builder.AllocateId();
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
|
||||
ConstantU32(state, mask), ConstantU32(state, 0)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
|
||||
ret = combined;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file == IR::RegisterFile::Scalar) {
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
const auto ret_low = EmitWqmLaneU32(state, low);
|
||||
const auto ret_high = EmitWqmLaneU32(state, high);
|
||||
EmitStoreU32(state, inst.dst, ret_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
|
||||
return;
|
||||
}
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup),
|
||||
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
|
||||
const auto low = state.builder.AllocateId();
|
||||
const auto high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
|
||||
const auto wqm_low = EmitWqmLaneU32(state, low);
|
||||
const auto wqm_high = EmitWqmLaneU32(state, high);
|
||||
const auto lane = EmitSubgroupLocalInvocationId(state);
|
||||
const auto high_half = state.builder.AllocateId();
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto bit_index = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
if (state.per_invocation_masks) {
|
||||
EmitPerInvocationMask(state, inst.dst, active);
|
||||
} else {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, result, active,
|
||||
ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, inst.dst, result);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto old_exec = EmitExecActiveBool(state);
|
||||
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
|
||||
EmitPerInvocationMask(state, inst.dst, old_exec);
|
||||
|
||||
uint32_t lhs = old_exec;
|
||||
uint32_t rhs = src;
|
||||
bool use_or = false;
|
||||
switch (inst.saveexec_mode) {
|
||||
case IR::SaveexecMode::And: break;
|
||||
case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break;
|
||||
case IR::SaveexecMode::Orn2:
|
||||
lhs = EmitLogicalNotBool(state, old_exec);
|
||||
use_or = true;
|
||||
break;
|
||||
}
|
||||
const auto result =
|
||||
use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs);
|
||||
EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result);
|
||||
EmitStoreSccBool(state, result);
|
||||
}
|
||||
|
||||
void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto src_low = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
|
||||
uint32_t rhs_low = src_low;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
|
||||
}
|
||||
|
||||
const auto new_low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
|
||||
if (state.wave_size == 32u) {
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
|
||||
}
|
||||
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto scc = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high);
|
||||
|
||||
uint32_t lhs_low = old_low;
|
||||
uint32_t lhs_high = old_high;
|
||||
uint32_t rhs_low = src_low;
|
||||
uint32_t rhs_high = src_high;
|
||||
uint32_t opcode = OpBitwiseAnd;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state.builder.AllocateId();
|
||||
rhs_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
|
||||
state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high});
|
||||
}
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
|
||||
lhs_low = state.builder.AllocateId();
|
||||
lhs_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low});
|
||||
state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high});
|
||||
opcode = OpBitwiseOr;
|
||||
}
|
||||
|
||||
const auto new_low = state.builder.AllocateId();
|
||||
const auto new_high = state.builder.AllocateId();
|
||||
state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low});
|
||||
state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
|
||||
|
||||
const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high;
|
||||
const auto mask = state.builder.AllocateId();
|
||||
const auto cond = state.builder.AllocateId();
|
||||
const auto scc = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
const auto first_lane = state.builder.AllocateId();
|
||||
const auto first_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), active});
|
||||
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane,
|
||||
ConstantU32(state, ScopeSubgroup), ballot});
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value,
|
||||
ConstantU32(state, ScopeSubgroup), src, first_lane});
|
||||
EmitStoreU32(state, inst.dst, first_value);
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto lane = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand),
|
||||
ConstantU32(state, 63)});
|
||||
return lane;
|
||||
}
|
||||
|
||||
void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
|
||||
ConstantU32(state, ScopeSubgroup), src, lane});
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto pointer =
|
||||
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0;
|
||||
const auto old = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0);
|
||||
const auto merged = state.builder.AllocateId();
|
||||
if (pointer != 0) {
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, old, pointer});
|
||||
}
|
||||
state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old});
|
||||
if (pointer != 0) {
|
||||
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
|
||||
state.builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane8 = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto upper = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto index0 = state.builder.AllocateId();
|
||||
const auto index1 = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto src1 = EmitValueLoad(state, inst.src[1]);
|
||||
const auto src2 = EmitValueLoad(state, inst.src[2]);
|
||||
uint32_t row_value = row;
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
if (x16) {
|
||||
row_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
|
||||
}
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1});
|
||||
state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1});
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target});
|
||||
uint32_t ret = shuffled;
|
||||
if (!inst.dst.op_sel) {
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target);
|
||||
ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, ret);
|
||||
}
|
||||
|
||||
void EmitControlNop(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitBarrier(EmitterState& state, const IR::Instruction& inst) {
|
||||
(void)inst;
|
||||
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
|
||||
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
|
||||
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,306 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
|
||||
return (inst.memory.image_sample_flags & flag) != 0;
|
||||
}
|
||||
|
||||
ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) {
|
||||
ImageSampleLayout layout;
|
||||
uint32_t cursor = 0;
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
layout.offset = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
layout.bias = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) {
|
||||
layout.dref = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
layout.grad_x = cursor;
|
||||
cursor += components;
|
||||
layout.grad_y = cursor;
|
||||
cursor += components;
|
||||
}
|
||||
layout.coord = cursor;
|
||||
cursor += ImageViewCoordinateComponents(view);
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) {
|
||||
layout.lod = cursor++;
|
||||
}
|
||||
return layout;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > layout.coord + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
|
||||
: EmitZeroF32(state);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > layout.coord + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
|
||||
: EmitZeroF32(state);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
|
||||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
|
||||
}
|
||||
|
||||
uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
|
||||
}
|
||||
|
||||
uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
|
||||
}
|
||||
|
||||
uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t first_component, ImageViewKind view) {
|
||||
const auto x = inst.memory.image_address_components > first_component
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
|
||||
: EmitZeroF32(state);
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > first_component + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
|
||||
: EmitZeroF32(state);
|
||||
const auto grad = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > first_component + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0],
|
||||
first_component + 2u)
|
||||
: EmitZeroF32(state);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
|
||||
}
|
||||
return grad;
|
||||
}
|
||||
|
||||
uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
const auto components = ImageViewSpatialComponents(view);
|
||||
if (layout.offset == NoImageComponent ||
|
||||
inst.memory.image_address_components <= layout.offset) {
|
||||
const auto zero = ConstantI32(state, 0);
|
||||
if (components == 1u) {
|
||||
return zero;
|
||||
}
|
||||
const auto ret = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
|
||||
const auto packed_i32 = state.builder.AllocateId();
|
||||
const auto offset_x = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
|
||||
ConstantI32(state, 0), ConstantI32(state, 6)});
|
||||
if (components == 1u) {
|
||||
return offset_x;
|
||||
}
|
||||
const auto offset_y = state.builder.AllocateId();
|
||||
const auto offset = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
|
||||
ConstantI32(state, 8), ConstantI32(state, 6)});
|
||||
if (components == 3u) {
|
||||
const auto offset_z = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32,
|
||||
ConstantI32(state, 16), ConstantI32(state, 6)});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y});
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, ImageViewKind view) {
|
||||
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto lod_component = ImageViewCoordinateComponents(view);
|
||||
if (inst.memory.image_address_components <= lod_component) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitImageAddressValueLoad(state, inst, address, lod_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
if (components == 1u) {
|
||||
return x;
|
||||
}
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
|
||||
: EmitZeroF32(state);
|
||||
const auto coord = state.builder.AllocateId();
|
||||
if (components == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
|
||||
: EmitZeroF32(state);
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
|
||||
} else {
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
|
||||
uint32_t index = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
index += (dmask >> i) & 1u;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
|
||||
const auto target = 4u + component;
|
||||
for (uint32_t source = 0; source < 4u; source++) {
|
||||
if (((swizzle >> (source * 3u)) & 0x7u) == target) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
|
||||
if (inst.memory.resource >= state.program.info.images.size()) {
|
||||
EXIT("storage image instruction has no specialized swizzle\n");
|
||||
}
|
||||
return state.program.info.images[inst.memory.resource].storage_swizzle;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
|
||||
: ConstantU32(state, 0);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,334 @@
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
namespace {
|
||||
|
||||
uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x),
|
||||
ConstantI32(state, y)});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state, ImageViewKind view) {
|
||||
if (ImageViewSpatialComponents(view) == 1u) {
|
||||
const auto type = state.builder.AllocateId();
|
||||
state.builder.AddType({OpTypeArray, type, state.int_type, ConstantU32(state, 4)});
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, type, value, ConstantI32(state, -1),
|
||||
ConstantI32(state, 0), ConstantI32(state, 1),
|
||||
ConstantI32(state, 2)});
|
||||
return value;
|
||||
}
|
||||
const auto offset0 = ConstantVec2I32(state, -1, 0);
|
||||
const auto offset1 = ConstantVec2I32(state, 0, 0);
|
||||
const auto offset2 = ConstantVec2I32(state, 1, 0);
|
||||
const auto offset3 = ConstantVec2I32(state, 2, 0);
|
||||
const auto type = state.builder.AllocateId();
|
||||
state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)});
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
|
||||
uint32_t array_index, bool uint_image,
|
||||
ImageViewKind view) {
|
||||
const auto kind = StorageBindingKind(uint_image, view);
|
||||
const auto& descriptors = state.storage_images[StorageImageIndex(uint_image, view)];
|
||||
const auto pointer =
|
||||
DescriptorElementPointer(state, descriptors.pointer_type, descriptors.variable, array_index,
|
||||
kind, resource, "storage image descriptor array was not emitted");
|
||||
const auto image = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, descriptors.image_type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
|
||||
uint32_t component_index, ImageViewKind view) {
|
||||
const auto components = ImageViewCoordinateComponents(view);
|
||||
switch (component_index) {
|
||||
case 0: {
|
||||
if (components == 1u) {
|
||||
return size;
|
||||
}
|
||||
const auto width = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0});
|
||||
return width;
|
||||
}
|
||||
case 1: {
|
||||
if (components < 2u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto height = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1});
|
||||
return height;
|
||||
}
|
||||
case 2:
|
||||
if (components == 3u) {
|
||||
const auto depth = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2});
|
||||
return depth;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
case 3: {
|
||||
const auto levels = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image});
|
||||
return levels;
|
||||
}
|
||||
default: return ConstantU32(state, 0);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const auto mip_level = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
const auto size = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component =
|
||||
EmitImageGetResinfoComponent(state, image, size, component_index, view);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto lod = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image,
|
||||
EmitImageQueryCoordF32(state, inst, view)});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 2u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, lod, component_index});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
|
||||
const auto color = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
|
||||
color, image, EmitImageLoadCoordU32(state, inst, view),
|
||||
ImageOperandsLodMask,
|
||||
EmitImageMipLodU32(state, inst, inst.src[0], view)});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, color, component_index});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
|
||||
const auto view = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
|
||||
const auto binding = ResourceForDescriptor(state, StorageBindingKind(uint_image, view),
|
||||
inst.memory.resource);
|
||||
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
|
||||
binding.array_index, uint_image, view);
|
||||
|
||||
state.builder.AddFunction(
|
||||
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
|
||||
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
|
||||
}
|
||||
|
||||
void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
|
||||
bool dref, bool integer) {
|
||||
if (dref) {
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample});
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, sample, component_index});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) {
|
||||
// RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it
|
||||
// to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information.
|
||||
// SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need
|
||||
// an explicit operand even for the unsuffixed sample opcode.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) {
|
||||
return true;
|
||||
}
|
||||
return state.stage != ShaderType::Pixel;
|
||||
}
|
||||
|
||||
void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, bool explicit_lod,
|
||||
std::vector<uint32_t>& words) {
|
||||
uint32_t mask = 0;
|
||||
std::vector<uint32_t> operands;
|
||||
|
||||
// Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod
|
||||
// operands belong to explicit-lod instructions; bias belongs to implicit-lod
|
||||
// sampling because it modifies the hardware-computed LOD.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
mask |= ImageOperandsGradMask;
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x, view));
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y, view));
|
||||
} else if (explicit_lod) {
|
||||
mask |= ImageOperandsLodMask;
|
||||
operands.push_back(EmitImageLodF32(state, inst, layout));
|
||||
} else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
mask |= ImageOperandsBiasMask;
|
||||
operands.push_back(EmitImageBiasF32(state, inst, layout));
|
||||
}
|
||||
if (mask != 0) {
|
||||
words.push_back(mask);
|
||||
words.insert(words.end(), operands.begin(), operands.end());
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto flags = inst.memory.image_sample_flags;
|
||||
const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0;
|
||||
const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
if (explicit_lod) {
|
||||
return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
|
||||
}
|
||||
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
|
||||
}
|
||||
|
||||
void EmitImageSample(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto sample = state.builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
uint32_t result_type = state.vec4_float_type;
|
||||
if (dref) {
|
||||
result_type = state.float_type;
|
||||
} else if (integer) {
|
||||
result_type = state.vec4_uint_type;
|
||||
}
|
||||
const auto explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
const auto opcode = ImageSampleOpcode(state, inst);
|
||||
std::vector<uint32_t> words = {opcode, result_type, sample, sampled_image, base_coord};
|
||||
if (dref) {
|
||||
words.push_back(EmitImageDrefF32(state, inst, layout));
|
||||
}
|
||||
AddImageSampleOperands(state, inst, layout, explicit_lod, words);
|
||||
state.builder.AddFunction(words);
|
||||
EmitImageSampleResult(state, inst, sample, dref, integer);
|
||||
}
|
||||
|
||||
uint32_t ImageGatherComponent(uint32_t dmask) {
|
||||
switch (dmask) {
|
||||
case 0x2u: return 1;
|
||||
case 0x4u: return 2;
|
||||
case 0x8u: return 3;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view,
|
||||
std::vector<uint32_t>& words) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
|
||||
words.push_back(ImageOperandsConstOffsetsMask);
|
||||
words.push_back(ConstantImageGatherHorizontalOffsets(state, view));
|
||||
return;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
words.push_back(ImageOperandsOffsetMask);
|
||||
words.push_back(EmitImagePackedOffsetI32(state, inst, layout, view));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto texels = state.builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
const auto result_type = integer ? state.vec4_uint_type : state.vec4_float_type;
|
||||
std::vector<uint32_t> words =
|
||||
dref ? std::vector<uint32_t> {OpImageDrefGather,
|
||||
state.vec4_float_type,
|
||||
texels,
|
||||
sampled_image,
|
||||
coord,
|
||||
EmitImageDrefF32(state, inst, layout)}
|
||||
: std::vector<uint32_t> {
|
||||
OpImageGather, result_type,
|
||||
texels, sampled_image,
|
||||
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
|
||||
AddImageGatherOperands(state, inst, layout, view, words);
|
||||
state.builder.AddFunction(words);
|
||||
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
const auto component = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
|
||||
component, texels, i});
|
||||
const auto bits = integer ? component : state.builder.AllocateId();
|
||||
if (!integer) {
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,810 @@
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind) {
|
||||
return IR::FindBinding(state.program.bindings, kind);
|
||||
}
|
||||
|
||||
uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind) {
|
||||
const auto* binding = DescriptorBinding(state, kind);
|
||||
return binding != nullptr ? static_cast<uint32_t>(binding->resources.size()) : 0;
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState& state, uint32_t value) {
|
||||
if (auto it = state.constants.find(value); it != state.constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.constants.emplace(value, id);
|
||||
state.builder.AddType({OpConstant, state.uint_type, id, value});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantI32(EmitterState& state, int32_t value) {
|
||||
const auto bits = static_cast<uint32_t>(value);
|
||||
if (auto it = state.signed_constants.find(bits); it != state.signed_constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.signed_constants.emplace(bits, id);
|
||||
state.builder.AddType({OpConstant, state.int_type, id, bits});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantF32(EmitterState& state, uint32_t bits) {
|
||||
if (auto it = state.float_constants.find(bits); it != state.float_constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state.builder.AllocateId();
|
||||
state.float_constants.emplace(bits, id);
|
||||
state.builder.AddType({OpConstant, state.float_type, id, bits});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t FloatBits(float value) {
|
||||
uint32_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
return bits;
|
||||
}
|
||||
|
||||
uint32_t ConstantF32Value(EmitterState& state, float value) {
|
||||
return ConstantF32(state, FloatBits(value));
|
||||
}
|
||||
|
||||
VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
|
||||
if (state.stage != ShaderType::Vertex || state.vertex_input_info == nullptr ||
|
||||
location >= ShaderVertexInputInfo::RES_MAX ||
|
||||
location >= static_cast<uint32_t>(state.vertex_input_info->resources_num)) {
|
||||
return VertexInputScalarKind::Float;
|
||||
}
|
||||
|
||||
switch (state.vertex_input_info->resources[location].Format()) {
|
||||
case 5:
|
||||
case 11:
|
||||
case 18:
|
||||
case 20:
|
||||
case 27:
|
||||
case 60:
|
||||
case 62:
|
||||
case 69:
|
||||
case 72:
|
||||
case 75: return VertexInputScalarKind::Uint;
|
||||
case 6:
|
||||
case 12:
|
||||
case 19:
|
||||
case 21:
|
||||
case 28:
|
||||
case 61:
|
||||
case 63:
|
||||
case 70:
|
||||
case 73:
|
||||
case 76: return VertexInputScalarKind::Sint;
|
||||
default: return VertexInputScalarKind::Float;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) {
|
||||
uint32_t count = input.component_count;
|
||||
if (state.stage == ShaderType::Vertex && state.vertex_input_info != nullptr &&
|
||||
input.location < ShaderVertexInputInfo::RES_MAX &&
|
||||
input.location < static_cast<uint32_t>(state.vertex_input_info->resources_num) &&
|
||||
state.vertex_input_info->resources_dst[input.location].registers_num > 0) {
|
||||
count = static_cast<uint32_t>(
|
||||
state.vertex_input_info->resources_dst[input.location].registers_num);
|
||||
}
|
||||
return std::clamp(count, 1u, 4u);
|
||||
}
|
||||
|
||||
uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint: return state.int_type;
|
||||
case VertexInputScalarKind::Uint: return state.uint_type;
|
||||
case VertexInputScalarKind::Float:
|
||||
default: return state.float_type;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint: return state.ptr_input_int;
|
||||
case VertexInputScalarKind::Uint: return state.ptr_input_uint;
|
||||
case VertexInputScalarKind::Float:
|
||||
default: return state.ptr_input_float;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t components) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint:
|
||||
switch (components) {
|
||||
case 1: return state.int_type;
|
||||
case 2: return state.vec2_int_type;
|
||||
case 3: return state.vec3_int_type;
|
||||
default: return state.vec4_int_type;
|
||||
}
|
||||
case VertexInputScalarKind::Uint:
|
||||
switch (components) {
|
||||
case 1: return state.uint_type;
|
||||
case 2: return state.vec2_uint_type;
|
||||
case 3: return state.vec3_uint_type;
|
||||
default: return state.vec4_uint_type;
|
||||
}
|
||||
case VertexInputScalarKind::Float:
|
||||
default:
|
||||
switch (components) {
|
||||
case 1: return state.float_type;
|
||||
case 2: return state.vec2_float_type;
|
||||
case 3: return state.vec3_float_type;
|
||||
default: return state.vec4_float_type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t components) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_int;
|
||||
case 2: return state.ptr_input_vec2_int;
|
||||
case 3: return state.ptr_input_vec3_int;
|
||||
default: return state.ptr_input_vec4_int;
|
||||
}
|
||||
case VertexInputScalarKind::Uint:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_uint;
|
||||
case 2: return state.ptr_input_vec2_uint;
|
||||
case 3: return state.ptr_input_vec3_uint;
|
||||
default: return state.ptr_input_vec4_uint;
|
||||
}
|
||||
case VertexInputScalarKind::Float:
|
||||
default:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_float;
|
||||
case 2: return state.ptr_input_vec2_float;
|
||||
case 3: return state.ptr_input_vec3_float;
|
||||
default: return state.ptr_input_vec4_float;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AllocateInputVariables(EmitterState& state) {
|
||||
for (auto& binding: state.inputs) {
|
||||
binding.variable_id = state.builder.AllocateId();
|
||||
state.interface_variables.push_back(binding.variable_id);
|
||||
}
|
||||
if (state.needs_subgroup_local_invocation_id) {
|
||||
state.subgroup_local_invocation_id_variable = state.builder.AllocateId();
|
||||
state.interface_variables.push_back(state.subgroup_local_invocation_id_variable);
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t AllocateInterfaceVariable(EmitterState& state) {
|
||||
const auto variable = state.builder.AllocateId();
|
||||
state.interface_variables.push_back(variable);
|
||||
return variable;
|
||||
}
|
||||
|
||||
static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& variable) {
|
||||
if (variable == 0) {
|
||||
variable = AllocateInterfaceVariable(state);
|
||||
}
|
||||
return variable;
|
||||
}
|
||||
|
||||
void AllocateOutputVariables(EmitterState& state) {
|
||||
for (auto& binding: state.outputs) {
|
||||
switch (binding.kind) {
|
||||
case IR::StageOutputKind::Position:
|
||||
binding.variable_id =
|
||||
AllocateSharedOutputVariable(state, state.per_vertex_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::Depth:
|
||||
binding.variable_id = AllocateSharedOutputVariable(state, state.depth_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::SampleMask:
|
||||
binding.variable_id =
|
||||
AllocateSharedOutputVariable(state, state.sample_mask_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::Parameter:
|
||||
case IR::StageOutputKind::Mrt:
|
||||
binding.variable_id = AllocateInterfaceVariable(state);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t BuiltInForInput(IR::StageInputKind kind) {
|
||||
switch (kind) {
|
||||
case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex;
|
||||
case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex;
|
||||
case IR::StageInputKind::FragCoord: return BuiltInFragCoord;
|
||||
case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing;
|
||||
case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId;
|
||||
case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId;
|
||||
case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex;
|
||||
case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId;
|
||||
default: return UINT32_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
void AddInputAnnotationsAndNames(EmitterState& state) {
|
||||
if (state.subgroup_local_invocation_id_variable != 0) {
|
||||
state.builder.AddName(state.subgroup_local_invocation_id_variable,
|
||||
"gl_SubgroupInvocationID");
|
||||
state.builder.AddAnnotation({OpDecorate, state.subgroup_local_invocation_id_variable,
|
||||
DecorationBuiltIn, BuiltInSubgroupLocalInvocationId});
|
||||
if (state.stage == ShaderType::Pixel) {
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.subgroup_local_invocation_id_variable, DecorationFlat});
|
||||
}
|
||||
}
|
||||
for (const auto& input: state.inputs) {
|
||||
state.builder.AddName(input.variable_id, input.debug_name.c_str());
|
||||
if (input.kind == IR::StageInputKind::Parameter) {
|
||||
const auto flat = PixelParameterIsFlat(state, input.location);
|
||||
if (flat) {
|
||||
state.builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
|
||||
}
|
||||
if (state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr &&
|
||||
state.pixel_input_info->ps_no_perspective && !flat) {
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationNoPerspective});
|
||||
}
|
||||
const auto location = PixelParameterLocation(state, input.location);
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationLocation, location});
|
||||
continue;
|
||||
}
|
||||
const auto builtin = BuiltInForInput(input.kind);
|
||||
if (builtin != UINT32_MAX) {
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationBuiltIn, builtin});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AddOutputAnnotationsAndNames(EmitterState& state) {
|
||||
if (state.per_vertex_variable != 0) {
|
||||
state.builder.AddName(state.per_vertex_type, "gl_PerVertex");
|
||||
state.builder.AddName(state.per_vertex_variable, "outPerVertex");
|
||||
state.builder.AddAnnotation(
|
||||
{OpMemberDecorate, state.per_vertex_type, 0, DecorationBuiltIn, BuiltInPosition});
|
||||
state.builder.AddAnnotation({OpDecorate, state.per_vertex_type, DecorationBlock});
|
||||
}
|
||||
if (state.depth_variable != 0) {
|
||||
state.builder.AddName(state.depth_variable, "gl_FragDepth");
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.depth_variable, DecorationBuiltIn, BuiltInFragDepth});
|
||||
}
|
||||
if (state.sample_mask_variable != 0) {
|
||||
state.builder.AddName(state.sample_mask_variable, "gl_SampleMask");
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask});
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
if (binding.kind == IR::StageOutputKind::Parameter ||
|
||||
binding.kind == IR::StageOutputKind::Mrt) {
|
||||
state.builder.AddName(binding.variable_id, binding.debug_name.c_str());
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, binding.variable_id, DecorationLocation, binding.location});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name, uint32_t set,
|
||||
uint32_t binding) {
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
state.builder.AddName(variable, name);
|
||||
state.builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, set});
|
||||
state.builder.AddAnnotation({OpDecorate, variable, DecorationBinding, binding});
|
||||
}
|
||||
|
||||
void AddDescriptorAnnotationsAndNames(EmitterState& state) {
|
||||
const auto set = state.program.bindings.descriptor_set;
|
||||
auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) {
|
||||
const auto* binding = DescriptorBinding(state, kind);
|
||||
if (binding != nullptr) {
|
||||
DecorateDescriptor(state, variable, name, set, binding->binding);
|
||||
}
|
||||
};
|
||||
if (state.storage_buffer_variable != 0) {
|
||||
Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
|
||||
}
|
||||
if (state.address_memory_variable != 0) {
|
||||
Decorate(state.address_memory_variable, "address_memory",
|
||||
IR::DescriptorBindingKind::AddressMemory);
|
||||
}
|
||||
constexpr const char* SampledNames[] = {
|
||||
"sampled_1d", "sampled_1d_array", "sampled_2d", "sampled_2d_array",
|
||||
"sampled_3d", "sampled_uint_1d", "sampled_uint_1d_array",
|
||||
"sampled_uint_2d", "sampled_uint_2d_array", "sampled_uint_3d"};
|
||||
for (uint32_t i = 0; i < state.sampled_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % ImageViewKindCount);
|
||||
Decorate(state.sampled_images[i].variable, SampledNames[i],
|
||||
SampledBindingKind(i >= ImageViewKindCount, view));
|
||||
}
|
||||
constexpr const char* StorageNames[] = {
|
||||
"storage_1d", "storage_1d_array", "storage_2d", "storage_2d_array",
|
||||
"storage_3d", "storage_uint_1d", "storage_uint_1d_array",
|
||||
"storage_uint_2d", "storage_uint_2d_array", "storage_uint_3d"};
|
||||
for (uint32_t i = 0; i < state.storage_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % ImageViewKindCount);
|
||||
Decorate(state.storage_images[i].variable, StorageNames[i],
|
||||
StorageBindingKind(i >= ImageViewKindCount, view));
|
||||
}
|
||||
if (state.sampler_variable != 0) {
|
||||
Decorate(state.sampler_variable, "samplers", IR::DescriptorBindingKind::Samplers);
|
||||
}
|
||||
if (state.gds_variable != 0) {
|
||||
Decorate(state.gds_variable, "gds", IR::DescriptorBindingKind::Gds);
|
||||
}
|
||||
if (state.flattened_srt_variable != 0) {
|
||||
Decorate(state.flattened_srt_variable, "flattened_srt",
|
||||
IR::DescriptorBindingKind::FlattenedSrt);
|
||||
}
|
||||
}
|
||||
|
||||
void AddVsharpAnnotationsAndNames(EmitterState& state) {
|
||||
const auto& bind = state.program.bindings;
|
||||
if (state.push_constant_variable != 0) {
|
||||
state.builder.AddName(state.push_constant_block_type, "BufferResource");
|
||||
state.builder.AddName(state.push_constant_variable, "vsharp");
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.push_constant_array_type, DecorationArrayStride, 4});
|
||||
state.builder.AddAnnotation({OpMemberDecorate, state.push_constant_block_type, 0,
|
||||
DecorationOffset, bind.push_constant_offset});
|
||||
state.builder.AddAnnotation({OpDecorate, state.push_constant_block_type, DecorationBlock});
|
||||
}
|
||||
if (state.vsharp_storage_variable != 0) {
|
||||
const auto* storage = DescriptorBinding(state, IR::DescriptorBindingKind::UserData);
|
||||
if (storage != nullptr) {
|
||||
DecorateDescriptor(state, state.vsharp_storage_variable, "user_data",
|
||||
bind.descriptor_set, storage->binding);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void EmitHeaderAndTypes(EmitterState& state) {
|
||||
state.void_type = state.builder.AllocateId();
|
||||
state.bool_type = state.builder.AllocateId();
|
||||
state.uint_type = state.builder.AllocateId();
|
||||
state.uint_pair_type = state.builder.AllocateId();
|
||||
state.int_type = state.builder.AllocateId();
|
||||
state.int_pair_type = state.builder.AllocateId();
|
||||
state.float_type = state.builder.AllocateId();
|
||||
state.vec2_uint_type = state.builder.AllocateId();
|
||||
state.vec3_uint_type = state.builder.AllocateId();
|
||||
state.vec4_uint_type = state.builder.AllocateId();
|
||||
state.vec2_int_type = state.builder.AllocateId();
|
||||
state.vec3_int_type = state.builder.AllocateId();
|
||||
state.vec4_int_type = state.builder.AllocateId();
|
||||
state.vec2_float_type = state.builder.AllocateId();
|
||||
state.vec3_float_type = state.builder.AllocateId();
|
||||
state.vec4_float_type = state.builder.AllocateId();
|
||||
state.ptr_func_uint = state.builder.AllocateId();
|
||||
state.ptr_input_float = state.builder.AllocateId();
|
||||
state.ptr_input_bool = state.builder.AllocateId();
|
||||
state.ptr_input_int = state.builder.AllocateId();
|
||||
state.ptr_input_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec2_float = state.builder.AllocateId();
|
||||
state.ptr_input_vec3_float = state.builder.AllocateId();
|
||||
state.ptr_input_vec2_int = state.builder.AllocateId();
|
||||
state.ptr_input_vec3_int = state.builder.AllocateId();
|
||||
state.ptr_input_vec4_int = state.builder.AllocateId();
|
||||
state.ptr_input_vec2_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec3_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec4_uint = state.builder.AllocateId();
|
||||
state.ptr_input_vec4_float = state.builder.AllocateId();
|
||||
state.sample_mask_array_type = state.builder.AllocateId();
|
||||
state.ptr_output_int = state.builder.AllocateId();
|
||||
state.ptr_output_sample_mask_array = state.builder.AllocateId();
|
||||
state.ptr_output_float = state.builder.AllocateId();
|
||||
state.ptr_output_vec4_float = state.builder.AllocateId();
|
||||
state.per_vertex_type = state.builder.AllocateId();
|
||||
state.ptr_output_per_vertex = state.builder.AllocateId();
|
||||
state.storage_runtime_array_type = state.builder.AllocateId();
|
||||
state.storage_buffer_type = state.builder.AllocateId();
|
||||
state.ptr_storage_buffer = state.builder.AllocateId();
|
||||
state.ptr_storage_buffer_uint = state.builder.AllocateId();
|
||||
state.storage_buffer_array_type = state.builder.AllocateId();
|
||||
state.ptr_storage_buffer_array = state.builder.AllocateId();
|
||||
if (state.address_memory_variable != 0) {
|
||||
state.address_memory_array_type = state.builder.AllocateId();
|
||||
state.ptr_address_memory_array = state.builder.AllocateId();
|
||||
}
|
||||
if (state.push_constant_variable != 0) {
|
||||
state.push_constant_array_type = state.builder.AllocateId();
|
||||
state.push_constant_block_type = state.builder.AllocateId();
|
||||
state.ptr_push_constant_block = state.builder.AllocateId();
|
||||
state.ptr_push_constant_uint = state.builder.AllocateId();
|
||||
}
|
||||
state.lds_array_type = state.builder.AllocateId();
|
||||
state.ptr_workgroup_array = state.builder.AllocateId();
|
||||
state.ptr_workgroup_uint = state.builder.AllocateId();
|
||||
state.lds_variable = state.builder.AllocateId();
|
||||
for (auto& image: state.sampled_images) {
|
||||
image.image_type = state.builder.AllocateId();
|
||||
image.sampled_image_type = state.builder.AllocateId();
|
||||
image.pointer_type = state.builder.AllocateId();
|
||||
image.array_type = state.builder.AllocateId();
|
||||
image.array_pointer_type = state.builder.AllocateId();
|
||||
}
|
||||
for (auto& image: state.storage_images) {
|
||||
image.image_type = state.builder.AllocateId();
|
||||
image.pointer_type = state.builder.AllocateId();
|
||||
image.array_type = state.builder.AllocateId();
|
||||
image.array_pointer_type = state.builder.AllocateId();
|
||||
}
|
||||
state.sampler_type = state.builder.AllocateId();
|
||||
state.sampler_array_type = state.builder.AllocateId();
|
||||
state.ptr_uniform_sampler = state.builder.AllocateId();
|
||||
state.ptr_uniform_sampler_array = state.builder.AllocateId();
|
||||
state.ptr_image_uint = state.builder.AllocateId();
|
||||
state.func_type = state.builder.AllocateId();
|
||||
state.main_func = state.builder.AllocateId();
|
||||
state.entry_label = state.builder.AllocateId();
|
||||
state.glsl_std450 = state.builder.AllocateId();
|
||||
|
||||
state.builder.AddCapability({CapabilityShader});
|
||||
state.builder.AddCapability({CapabilitySampled1D});
|
||||
state.builder.AddCapability({CapabilityImage1D});
|
||||
state.builder.AddCapability({CapabilityImageQuery});
|
||||
if (state.needs_image_gather_extended) {
|
||||
state.builder.AddCapability({CapabilityImageGatherExtended});
|
||||
}
|
||||
if (std::any_of(state.storage_images.begin(),
|
||||
state.storage_images.begin() + ImageViewKindCount,
|
||||
[](const auto& image) { return image.variable != 0; })) {
|
||||
state.builder.AddCapability({CapabilityStorageImageReadWithoutFormat});
|
||||
state.builder.AddCapability({CapabilityStorageImageWriteWithoutFormat});
|
||||
}
|
||||
if (state.needs_subgroup_ballot || state.needs_subgroup_shuffle ||
|
||||
state.needs_subgroup_local_invocation_id) {
|
||||
state.builder.AddCapability({CapabilityGroupNonUniform});
|
||||
}
|
||||
if (state.needs_subgroup_ballot) {
|
||||
state.builder.AddCapability({CapabilityGroupNonUniformBallot});
|
||||
}
|
||||
if (state.needs_subgroup_shuffle) {
|
||||
state.builder.AddCapability({CapabilityGroupNonUniformShuffle});
|
||||
}
|
||||
if (state.needs_compute_derivatives && state.stage == ShaderType::Compute) {
|
||||
state.builder.AddCapability({CapabilityComputeDerivativeGroupQuadsKHR});
|
||||
state.builder.AddExtension("SPV_KHR_compute_shader_derivatives");
|
||||
}
|
||||
state.builder.AddExtInstImport(state.glsl_std450, "GLSL.std.450");
|
||||
state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
|
||||
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
|
||||
state.interface_variables);
|
||||
if (state.stage == ShaderType::Compute) {
|
||||
uint32_t local_x = state.needs_compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_y = state.needs_compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_z = 1u;
|
||||
if (state.compute_input_info != nullptr) {
|
||||
const auto* cs = state.compute_input_info;
|
||||
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
|
||||
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
|
||||
local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
|
||||
}
|
||||
state.builder.AddExecutionMode(
|
||||
{state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
|
||||
}
|
||||
if (state.stage == ShaderType::Pixel) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeOriginUpperLeft});
|
||||
if (state.depth_variable != 0) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeDepthReplacing});
|
||||
}
|
||||
if (state.pixel_input_info != nullptr && state.pixel_input_info->ps_early_z &&
|
||||
!state.pixel_input_info->ps_pixel_kill_enable &&
|
||||
!state.pixel_input_info->ps_depth_export_enable &&
|
||||
!state.pixel_input_info->ps_sample_mask_export_enable) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeEarlyFragmentTests});
|
||||
}
|
||||
}
|
||||
if (state.needs_compute_derivatives && state.stage == ShaderType::Compute) {
|
||||
state.builder.AddExecutionMode({state.main_func, ExecutionModeDerivativeGroupQuadsKHR});
|
||||
}
|
||||
state.builder.AddName(state.main_func, "main");
|
||||
if (state.stage == ShaderType::Compute || state.needs_function_lds) {
|
||||
state.builder.AddName(state.lds_variable, "lds_dwords");
|
||||
}
|
||||
AddInputAnnotationsAndNames(state);
|
||||
AddOutputAnnotationsAndNames(state);
|
||||
AddDescriptorAnnotationsAndNames(state);
|
||||
AddVsharpAnnotationsAndNames(state);
|
||||
state.builder.AddAnnotation(
|
||||
{OpDecorate, state.storage_runtime_array_type, DecorationArrayStride, 4});
|
||||
state.builder.AddAnnotation(
|
||||
{OpMemberDecorate, state.storage_buffer_type, 0, DecorationOffset, 0});
|
||||
state.builder.AddAnnotation({OpDecorate, state.storage_buffer_type, DecorationBlock});
|
||||
|
||||
state.builder.AddType({OpTypeVoid, state.void_type});
|
||||
state.builder.AddType({OpTypeBool, state.bool_type});
|
||||
state.builder.AddType({OpTypeInt, state.uint_type, 32, 0});
|
||||
state.builder.AddType({OpTypeStruct, state.uint_pair_type, state.uint_type, state.uint_type});
|
||||
state.builder.AddType({OpTypeInt, state.int_type, 32, 1});
|
||||
state.builder.AddType({OpTypeStruct, state.int_pair_type, state.int_type, state.int_type});
|
||||
state.builder.AddType({OpTypeFloat, state.float_type, 32});
|
||||
state.builder.AddType({OpTypeVector, state.vec2_uint_type, state.uint_type, 2});
|
||||
state.builder.AddType({OpTypeVector, state.vec3_uint_type, state.uint_type, 3});
|
||||
state.builder.AddType({OpTypeVector, state.vec4_uint_type, state.uint_type, 4});
|
||||
state.builder.AddType({OpTypeVector, state.vec2_int_type, state.int_type, 2});
|
||||
state.builder.AddType({OpTypeVector, state.vec3_int_type, state.int_type, 3});
|
||||
state.builder.AddType({OpTypeVector, state.vec4_int_type, state.int_type, 4});
|
||||
state.builder.AddType({OpTypeVector, state.vec2_float_type, state.float_type, 2});
|
||||
state.builder.AddType({OpTypeVector, state.vec3_float_type, state.float_type, 3});
|
||||
state.builder.AddType({OpTypeVector, state.vec4_float_type, state.float_type, 4});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_func_uint, StorageClassFunction, state.uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_float, StorageClassInput, state.float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_bool, StorageClassInput, state.bool_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_input_int, StorageClassInput, state.int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_uint, StorageClassInput, state.uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec2_float, StorageClassInput, state.vec2_float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec3_float, StorageClassInput, state.vec3_float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec2_int, StorageClassInput, state.vec2_int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec3_int, StorageClassInput, state.vec3_int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec4_int, StorageClassInput, state.vec4_int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec2_uint, StorageClassInput, state.vec2_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec3_uint, StorageClassInput, state.vec3_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec4_uint, StorageClassInput, state.vec4_uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_input_vec4_float, StorageClassInput, state.vec4_float_type});
|
||||
if (state.subgroup_local_invocation_id_variable != 0) {
|
||||
state.builder.AddType({OpVariable, state.ptr_input_uint,
|
||||
state.subgroup_local_invocation_id_variable, StorageClassInput});
|
||||
}
|
||||
for (const auto& input: state.inputs) {
|
||||
uint32_t ptr_type = state.ptr_input_uint;
|
||||
switch (input.kind) {
|
||||
case IR::StageInputKind::VertexIndex:
|
||||
case IR::StageInputKind::InstanceIndex: ptr_type = state.ptr_input_int; break;
|
||||
case IR::StageInputKind::WorkgroupId:
|
||||
case IR::StageInputKind::LocalInvocationId:
|
||||
case IR::StageInputKind::GlobalInvocationId:
|
||||
ptr_type = state.ptr_input_vec3_uint;
|
||||
break;
|
||||
case IR::StageInputKind::FragCoord: ptr_type = state.ptr_input_vec4_float; break;
|
||||
case IR::StageInputKind::FrontFacing: ptr_type = state.ptr_input_bool; break;
|
||||
case IR::StageInputKind::Parameter:
|
||||
if (state.stage == ShaderType::Vertex) {
|
||||
const auto kind = VertexParameterScalarKind(state, input.location);
|
||||
const auto components = VertexParameterComponentCount(state, input);
|
||||
ptr_type = VertexParameterInputPointerType(state, kind, components);
|
||||
} else {
|
||||
ptr_type = state.ptr_input_vec4_float;
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
state.builder.AddType({OpVariable, ptr_type, input.variable_id, StorageClassInput});
|
||||
}
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_output_float, StorageClassOutput, state.float_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_output_int, StorageClassOutput, state.int_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_output_vec4_float, StorageClassOutput, state.vec4_float_type});
|
||||
if (state.per_vertex_variable != 0) {
|
||||
state.builder.AddType({OpTypeStruct, state.per_vertex_type, state.vec4_float_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_output_per_vertex, StorageClassOutput,
|
||||
state.per_vertex_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_output_per_vertex, state.per_vertex_variable,
|
||||
StorageClassOutput});
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
if (binding.kind == IR::StageOutputKind::Parameter ||
|
||||
binding.kind == IR::StageOutputKind::Mrt) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_output_vec4_float, binding.variable_id, StorageClassOutput});
|
||||
}
|
||||
}
|
||||
if (state.depth_variable != 0) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_output_float, state.depth_variable, StorageClassOutput});
|
||||
}
|
||||
if (state.sample_mask_variable != 0) {
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.sample_mask_array_type, state.int_type, ConstantU32(state, 1)});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_output_sample_mask_array,
|
||||
StorageClassOutput, state.sample_mask_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_output_sample_mask_array,
|
||||
state.sample_mask_variable, StorageClassOutput});
|
||||
}
|
||||
state.builder.AddType({OpTypeRuntimeArray, state.storage_runtime_array_type, state.uint_type});
|
||||
state.builder.AddType(
|
||||
{OpTypeStruct, state.storage_buffer_type, state.storage_runtime_array_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_storage_buffer, StorageClassStorageBuffer,
|
||||
state.storage_buffer_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_storage_buffer_uint, StorageClassStorageBuffer, state.uint_type});
|
||||
if (state.storage_buffer_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Buffers));
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.storage_buffer_array_type, state.storage_buffer_type, count});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_storage_buffer_array,
|
||||
StorageClassStorageBuffer, state.storage_buffer_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_storage_buffer_array,
|
||||
state.storage_buffer_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.gds_variable != 0) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_storage_buffer, state.gds_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.push_constant_variable != 0) {
|
||||
const auto dword_count =
|
||||
ConstantU32(state, state.program.bindings.push_constant_size / sizeof(uint32_t));
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.push_constant_array_type, state.uint_type, dword_count});
|
||||
state.builder.AddType(
|
||||
{OpTypeStruct, state.push_constant_block_type, state.push_constant_array_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_push_constant_block,
|
||||
StorageClassPushConstant, state.push_constant_block_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_push_constant_uint,
|
||||
StorageClassPushConstant, state.uint_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_push_constant_block,
|
||||
state.push_constant_variable, StorageClassPushConstant});
|
||||
}
|
||||
if (state.vsharp_storage_variable != 0) {
|
||||
state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.vsharp_storage_variable,
|
||||
StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.address_memory_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::AddressMemory));
|
||||
state.builder.AddType(
|
||||
{OpTypeArray, state.address_memory_array_type, state.storage_buffer_type, count});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_address_memory_array,
|
||||
StorageClassStorageBuffer, state.address_memory_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_address_memory_array,
|
||||
state.address_memory_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.flattened_srt_variable != 0) {
|
||||
state.builder.AddType({OpVariable, state.ptr_storage_buffer, state.flattened_srt_variable,
|
||||
StorageClassStorageBuffer});
|
||||
}
|
||||
if (state.stage == ShaderType::Compute || state.needs_function_lds) {
|
||||
const auto storage_class =
|
||||
state.needs_function_lds ? StorageClassFunction : StorageClassWorkgroup;
|
||||
const auto lds_size = ConstantU32(state, state.needs_function_lds ? 8192u : 1024u);
|
||||
state.builder.AddType({OpTypeArray, state.lds_array_type, state.uint_type, lds_size});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_workgroup_array, storage_class, state.lds_array_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_workgroup_uint, storage_class, state.uint_type});
|
||||
if (state.stage == ShaderType::Compute) {
|
||||
state.builder.AddType(
|
||||
{OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassWorkgroup});
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < state.sampled_images.size(); i++) {
|
||||
auto& image = state.sampled_images[i];
|
||||
const auto view = static_cast<ImageViewKind>(i % ImageViewKindCount);
|
||||
const bool integer = i >= ImageViewKindCount;
|
||||
const auto component = integer ? state.uint_type : state.float_type;
|
||||
state.builder.AddType({OpTypeImage, image.image_type, component,
|
||||
ImageSpirvDimension(view), 0, ImageSpirvArrayed(view), 0, 1,
|
||||
ImageFormatUnknown});
|
||||
state.builder.AddType({OpTypeSampledImage, image.sampled_image_type, image.image_type});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type});
|
||||
if (image.variable != 0) {
|
||||
const auto kind = SampledBindingKind(integer, view);
|
||||
const auto count = ConstantU32(state, DescriptorCount(state, kind));
|
||||
state.builder.AddType({OpTypeArray, image.array_type, image.image_type, count});
|
||||
state.builder.AddType({OpTypePointer, image.array_pointer_type,
|
||||
StorageClassUniformConstant, image.array_type});
|
||||
state.builder.AddType({OpVariable, image.array_pointer_type, image.variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
}
|
||||
state.builder.AddType({OpTypeSampler, state.sampler_type});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_uniform_sampler, StorageClassUniformConstant,
|
||||
state.sampler_type});
|
||||
if (state.sampler_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Samplers));
|
||||
state.builder.AddType({OpTypeArray, state.sampler_array_type, state.sampler_type, count});
|
||||
state.builder.AddType({OpTypePointer, state.ptr_uniform_sampler_array,
|
||||
StorageClassUniformConstant, state.sampler_array_type});
|
||||
state.builder.AddType({OpVariable, state.ptr_uniform_sampler_array, state.sampler_variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
for (uint32_t i = 0; i < state.storage_images.size(); i++) {
|
||||
auto& image = state.storage_images[i];
|
||||
const auto view = static_cast<ImageViewKind>(i % ImageViewKindCount);
|
||||
const bool integer = i >= ImageViewKindCount;
|
||||
const auto component = integer ? state.uint_type : state.float_type;
|
||||
const auto format = integer ? ImageFormatR32ui : ImageFormatUnknown;
|
||||
state.builder.AddType({OpTypeImage, image.image_type, component,
|
||||
ImageSpirvDimension(view), 0, ImageSpirvArrayed(view), 0, 2,
|
||||
format});
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type});
|
||||
if (image.variable != 0) {
|
||||
const auto kind = StorageBindingKind(integer, view);
|
||||
const auto count = ConstantU32(state, DescriptorCount(state, kind));
|
||||
state.builder.AddType({OpTypeArray, image.array_type, image.image_type, count});
|
||||
state.builder.AddType({OpTypePointer, image.array_pointer_type,
|
||||
StorageClassUniformConstant, image.array_type});
|
||||
state.builder.AddType({OpVariable, image.array_pointer_type, image.variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
}
|
||||
state.builder.AddType(
|
||||
{OpTypePointer, state.ptr_image_uint, StorageClassImage, state.uint_type});
|
||||
state.builder.AddType({OpTypeFunction, state.func_type, state.void_type});
|
||||
}
|
||||
|
||||
void AllocateRegisterVariables(EmitterState& state) {
|
||||
if (state.stage == ShaderType::Pixel && state.needs_pixel_valid_mask) {
|
||||
state.pixel_valid_mask_variable = state.builder.AllocateId();
|
||||
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
|
||||
}
|
||||
for (auto& binding: state.registers) {
|
||||
binding.pointer_id = state.builder.AllocateId();
|
||||
const auto name = IR::RegisterToString(binding.reg);
|
||||
state.builder.AddName(binding.pointer_id, name.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
void AllocateDescriptorVariables(EmitterState& state) {
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::Buffers) != nullptr) {
|
||||
state.storage_buffer_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::AddressMemory) != nullptr) {
|
||||
state.address_memory_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (state.program.bindings.push_constant_size != 0) {
|
||||
state.push_constant_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::UserData) != nullptr) {
|
||||
state.vsharp_storage_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
|
||||
state.flattened_srt_variable = state.builder.AllocateId();
|
||||
}
|
||||
for (uint32_t i = 0; i < state.sampled_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % ImageViewKindCount);
|
||||
if (DescriptorBinding(state, SampledBindingKind(i >= ImageViewKindCount, view)) !=
|
||||
nullptr) {
|
||||
state.sampled_images[i].variable = state.builder.AllocateId();
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < state.storage_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % ImageViewKindCount);
|
||||
if (DescriptorBinding(state, StorageBindingKind(i >= ImageViewKindCount, view)) !=
|
||||
nullptr) {
|
||||
state.storage_images[i].variable = state.builder.AllocateId();
|
||||
}
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::Samplers) != nullptr) {
|
||||
state.sampler_variable = state.builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(state, IR::DescriptorBindingKind::Gds) != nullptr) {
|
||||
state.gds_variable = state.builder.AllocateId();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,838 @@
|
||||
#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
|
||||
if (sel <= 3u) {
|
||||
offset = sel * 8u;
|
||||
width = 8u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 4u || sel == 5u) {
|
||||
offset = sel == 5u ? 16u : 0u;
|
||||
width = 16u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 6u) {
|
||||
offset = 0;
|
||||
width = 32u;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
|
||||
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t offset = 0;
|
||||
uint32_t width = 0;
|
||||
if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (operand.sdwa_sext) {
|
||||
const auto signed_value = state.builder.AllocateId();
|
||||
const auto extracted = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value});
|
||||
state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted});
|
||||
return bits;
|
||||
}
|
||||
|
||||
const auto extracted = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
return extracted;
|
||||
}
|
||||
|
||||
uint32_t EmitTrueBool(EmitterState& state) {
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
|
||||
const auto quad_base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto shift = state.builder.AllocateId();
|
||||
const auto selected0 = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
|
||||
bool left) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto lane_shifted = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
const auto valid = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
if (left) {
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
|
||||
} else {
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
|
||||
}
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
|
||||
return {target, valid};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
|
||||
uint32_t amount) {
|
||||
const auto row = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto minus = state.builder.AllocateId();
|
||||
const auto plus = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
|
||||
state.builder.AddFunction(
|
||||
{OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
|
||||
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
|
||||
const auto lane_mask = half_row ? 7u : 15u;
|
||||
const auto base = state.builder.AllocateId();
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto mirrored = state.builder.AllocateId();
|
||||
const auto target = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
|
||||
state.builder.AddFunction(
|
||||
{OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
if (control <= 0xffu) {
|
||||
return EmitDppQuadPermTargetLane(state, subid, control);
|
||||
}
|
||||
if (control >= 0x101u && control <= 0x10fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
|
||||
}
|
||||
if (control >= 0x111u && control <= 0x11fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
|
||||
}
|
||||
if (control >= 0x121u && control <= 0x12fu) {
|
||||
return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
|
||||
}
|
||||
if (control == 0x140u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, false);
|
||||
}
|
||||
if (control == 0x141u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, true);
|
||||
}
|
||||
return {subid, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
|
||||
if (!operand.dpp) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
|
||||
const auto shuffled = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target.lane});
|
||||
if (operand.dpp_fetch_inactive) {
|
||||
return shuffled;
|
||||
}
|
||||
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
|
||||
const auto can_fetch = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
|
||||
uint32_t value = 0;
|
||||
switch (operand.kind) {
|
||||
case IR::OperandKind::ImmediateU32:
|
||||
case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
|
||||
case IR::OperandKind::Register: {
|
||||
const auto pointer = PointerForRegister(state, operand.reg);
|
||||
if (pointer == 0) {
|
||||
value = ConstantU32(state, 0);
|
||||
break;
|
||||
}
|
||||
value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
break;
|
||||
}
|
||||
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
|
||||
default: value = ConstantU32(state, 0); break;
|
||||
}
|
||||
value = EmitDppValueU32(state, operand, value);
|
||||
return EmitSdwaExtractU32(state, operand, value);
|
||||
}
|
||||
|
||||
uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) {
|
||||
if (IsInactiveWave32ExecHigh(state, reg)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = PointerForRegister(state, reg);
|
||||
if (pointer == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) {
|
||||
if (state.wave_size == 32u && IsMaskRegisterFile(file)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitRegisterLoad(state, {file, 1});
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
|
||||
if (part == 0) {
|
||||
const auto in_low = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
if (state.wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
const auto below = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
|
||||
state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)});
|
||||
|
||||
if (part == 0) {
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
|
||||
return ret;
|
||||
}
|
||||
if (state.wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction(
|
||||
{OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto condition = state.builder.AllocateId();
|
||||
if (state.per_invocation_masks) {
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
if (state.exact_subgroup_operations) {
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto low_hit = state.builder.AllocateId();
|
||||
const auto high_hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
|
||||
if (state.per_invocation_masks) {
|
||||
return EmitMaskZeroBool(state, file, zero);
|
||||
}
|
||||
if (state.exact_subgroup_operations) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto combined = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
|
||||
uint32_t EmitExecActiveBool(EmitterState& state) {
|
||||
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
|
||||
}
|
||||
|
||||
uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) {
|
||||
if (operand.kind == IR::OperandKind::Register) {
|
||||
if (operand.reg.file == IR::RegisterFile::Exec ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc) {
|
||||
return EmitMaskActiveBool(state, operand.reg.file);
|
||||
}
|
||||
}
|
||||
const auto cond_value = EmitValueLoad(state, operand);
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
|
||||
if (state.subgroup_local_invocation_id_variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == kind) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return &input;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
|
||||
const auto variable = InputVariableForKind(state, kind);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t EmitLocalInvocationIndex(EmitterState& state) {
|
||||
const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t component) {
|
||||
if ((component & 3u) != 3u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
|
||||
}
|
||||
|
||||
uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
|
||||
uint32_t component) {
|
||||
const auto count = VertexParameterComponentCount(state, input);
|
||||
const auto kind = VertexParameterScalarKind(state, input.location);
|
||||
if (component >= count) {
|
||||
return VertexInputDefaultComponentU32(state, kind, component);
|
||||
}
|
||||
|
||||
const auto scalar_type = VertexParameterScalarType(state, kind);
|
||||
uint32_t raw = state.builder.AllocateId();
|
||||
if (count == 1u) {
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
|
||||
} else {
|
||||
const auto pointer_type = VertexParameterScalarPointerType(state, kind);
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
|
||||
ConstantU32(state, component)});
|
||||
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
|
||||
}
|
||||
|
||||
if (kind == VertexInputScalarKind::Uint) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
|
||||
return bits;
|
||||
}
|
||||
|
||||
void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto* input = InputBindingForParameter(state, inst.input_info.attr);
|
||||
if (input == nullptr || input->variable_id == 0) {
|
||||
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.stage == ShaderType::Vertex) {
|
||||
EmitStoreU32(state, inst.dst,
|
||||
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
|
||||
return;
|
||||
}
|
||||
|
||||
const auto input_value = state.builder.AllocateId();
|
||||
const auto component = state.builder.AllocateId();
|
||||
const auto bits = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u});
|
||||
state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t EmitSccBool(EmitterState& state, bool non_zero) {
|
||||
const auto value = EmitRegisterLoad(state, SccRegister());
|
||||
const auto cond = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto uint_value = EmitValueLoad(state, operand);
|
||||
auto float_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value});
|
||||
float_value = abs_value;
|
||||
}
|
||||
if (operand.negate) {
|
||||
const auto neg_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value});
|
||||
float_value = neg_value;
|
||||
}
|
||||
return float_value;
|
||||
}
|
||||
|
||||
uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) {
|
||||
uint32_t result = value;
|
||||
if (dst.omod != 0) {
|
||||
uint32_t bits = 0x3f000000u;
|
||||
switch (dst.omod) {
|
||||
case 1u: bits = 0x40000000u; break;
|
||||
case 2u: bits = 0x40800000u; break;
|
||||
default: break;
|
||||
}
|
||||
const auto scaled = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)});
|
||||
result = scaled;
|
||||
}
|
||||
if (!dst.clamp) {
|
||||
return result;
|
||||
}
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp,
|
||||
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) {
|
||||
auto value_operand = operand;
|
||||
value_operand.negate_hi = false;
|
||||
|
||||
if (operand.op_sel_hi) {
|
||||
auto raw = operand;
|
||||
raw.sdwa_sel = 6;
|
||||
raw.sdwa_sext = false;
|
||||
raw.op_sel = false;
|
||||
raw.op_sel_hi = false;
|
||||
raw.negate = false;
|
||||
raw.negate_hi = false;
|
||||
raw.absolute = false;
|
||||
raw.dpp = false;
|
||||
raw.dpp_fetch_inactive = false;
|
||||
raw.dpp_bound_ctrl = false;
|
||||
|
||||
const auto bits = EmitValueLoad(state, raw);
|
||||
const auto unpacked = state.builder.AllocateId();
|
||||
auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450,
|
||||
GlslUnpackHalf2x16, bits});
|
||||
state.builder.AddFunction(
|
||||
{OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32});
|
||||
f32 = abs_value;
|
||||
}
|
||||
if (value_operand.negate) {
|
||||
const auto neg_value = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32});
|
||||
f32 = neg_value;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
|
||||
value_operand.op_sel = false;
|
||||
value_operand.op_sel_hi = false;
|
||||
return EmitFloatLoad(state, value_operand);
|
||||
}
|
||||
|
||||
IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind != IR::OperandKind::Register) {
|
||||
return operand;
|
||||
}
|
||||
auto ret = operand;
|
||||
ret.reg.index += offset;
|
||||
ret.sdwa_sel = 6;
|
||||
ret.omod = 0;
|
||||
ret.sdwa_sext = false;
|
||||
ret.op_sel = false;
|
||||
ret.op_sel_hi = false;
|
||||
ret.negate = false;
|
||||
ret.negate_hi = false;
|
||||
ret.absolute = false;
|
||||
ret.clamp = false;
|
||||
ret.dpp_ctrl = 0;
|
||||
ret.dpp_row_mask = 0xf;
|
||||
ret.dpp_bank_mask = 0xf;
|
||||
ret.dpp_fetch_inactive = false;
|
||||
ret.dpp_bound_ctrl = false;
|
||||
ret.dpp = false;
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) {
|
||||
const auto low = EmitValueLoad(state, operand);
|
||||
const auto active = state.builder.AllocateId();
|
||||
if (state.per_invocation_masks || state.exact_subgroup_operations) {
|
||||
uint32_t high = ConstantU32(state, 0);
|
||||
if (operand.kind == IR::OperandKind::Register &&
|
||||
(operand.reg.file == IR::RegisterFile::Scalar ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc ||
|
||||
operand.reg.file == IR::RegisterFile::Exec)) {
|
||||
high = IsMaskRegisterFile(operand.reg.file)
|
||||
? WaveMaskHighLoad(state, operand.reg.file)
|
||||
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
|
||||
}
|
||||
if (state.per_invocation_masks) {
|
||||
const auto combined = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
const auto low_hit = state.builder.AllocateId();
|
||||
const auto high_hit = state.builder.AllocateId();
|
||||
const auto hit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit});
|
||||
state.builder.AddFunction(
|
||||
{OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) {
|
||||
if (state.per_invocation_masks) {
|
||||
const auto ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, ballot, lane);
|
||||
}
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
|
||||
const auto active_ballot = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, active_ballot, lane);
|
||||
}
|
||||
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
|
||||
const auto low = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
|
||||
uint32_t mask = low;
|
||||
if (state.wave_size == 64u) {
|
||||
const auto high = state.builder.AllocateId();
|
||||
const auto in_high = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
|
||||
state.builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto lane_low = state.builder.AllocateId();
|
||||
const auto bit = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
{OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state.builder.AddFunction(
|
||||
{OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
const auto hit = state.builder.AllocateId();
|
||||
const auto active = state.builder.AllocateId();
|
||||
const auto in_range = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
|
||||
state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
|
||||
state.builder.AddFunction(
|
||||
{OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
|
||||
state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
|
||||
}
|
||||
if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) {
|
||||
return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u);
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
|
||||
const auto value = EmitSequentialValueLoad(state, operand, offset);
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
bool ImageAddressUsesA16(const IR::Instruction& inst) {
|
||||
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u;
|
||||
}
|
||||
|
||||
bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t cursor = 0;
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) {
|
||||
if (component == cursor) {
|
||||
return false;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) {
|
||||
if (component == cursor) {
|
||||
return true;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u &&
|
||||
component == cursor) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return component * 2u;
|
||||
}
|
||||
|
||||
uint32_t half_component = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u;
|
||||
if (half_width == 2u && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
half_component += half_width;
|
||||
}
|
||||
if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
return half_component;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
return ImageAddressHalfComponent(inst, component) / 2u;
|
||||
}
|
||||
|
||||
IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
|
||||
uint32_t component) {
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) {
|
||||
const auto nsa_index = packed_component - 1u;
|
||||
const auto nsa_count = std::min<uint32_t>(inst.memory.image_nsa_dwords * 4u,
|
||||
Decoder::MaxImageNsaAddressComponents);
|
||||
if (nsa_index < nsa_count) {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Vector,
|
||||
inst.memory.image_nsa_addr[nsa_index]);
|
||||
}
|
||||
}
|
||||
return OffsetRegisterOperand(base, packed_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto operand = ImageAddressOperand(inst, base, component);
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
auto value = EmitValueLoad(state, operand);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) {
|
||||
value = EmitShiftRightConstant(state, value, 16u);
|
||||
}
|
||||
value = EmitAndConstant(state, value, 0xffffu);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
return EmitF16BitsToF32(state, value);
|
||||
}
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
uint32_t EmitZeroF32(EmitterState& state) {
|
||||
const auto f32 = state.builder.AllocateId();
|
||||
state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)});
|
||||
return f32;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
Reference in new issue
Block a user