shader: use Khronos definitions and simplify compiler ownership

This commit is contained in:
nmzik committed 2026-09-12 13:42:15 +04:00
1 parent 47b26e497f
commit 1d1f00bdef
28 files changed
+1992 -2097

No files matched your search

+1
View File
@@ -342,6 +342,7 @@ set(inc_headers
${KYTY_THIRD_PARTY_DIR}/SDL2/include ${KYTY_THIRD_PARTY_DIR}/SDL2/include
${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include ${KYTY_THIRD_PARTY_DIR}/VulkanMemoryAllocator/include
${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include ${KYTY_THIRD_PARTY_DIR}/SPIRV-Tools/include
${KYTY_THIRD_PARTY_DIR}/SPIRV-Headers/include
${KYTY_THIRD_PARTY_DIR}/fmt/include ${KYTY_THIRD_PARTY_DIR}/fmt/include
${KYTY_THIRD_PARTY_DIR}/xxhash ${KYTY_THIRD_PARTY_DIR}/xxhash
${KYTY_THIRD_PARTY_DIR}/cpuinfo/include ${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
@@ -338,13 +338,12 @@ struct PipelineCache::ProgramCache {
options.early_dump = options.dump_ir; options.early_dump = options.dump_ir;
options.dump_label = label; options.dump_label = label;
options.input_info = stage_input; options.input_info = stage_input;
options.scratch_dwords = input_info.scratch_size_dwords;
if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) { if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
options.user_data_base = 8; options.user_data_base = 8;
if (stage == ShaderType::Mesh) { if (stage == ShaderType::Mesh) {
options.user_data_base = 0; options.user_data_base = 0;
options.wave_size = input_info.mesh.wave_size; options.wave_size = input_info.mesh.wave_size;
options.scratch_dwords = input_info.mesh.scratch_size_dwords;
} }
} else if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) { } else if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
options.wave_size = input_info.wave_size; options.wave_size = input_info.wave_size;
@@ -582,25 +582,12 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms()); static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
} }
const ShaderVertexInputInfo* vertex = nullptr;
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
switch (options.stage) {
case ShaderType::Vertex:
case ShaderType::Mesh: vertex = options.input_info.vertex; break;
case ShaderType::Pixel:
pixel = options.input_info.pixel;
break;
case ShaderType::Compute:
compute = options.input_info.compute;
break;
default: break;
}
EmbeddedFetchData embedded_fetch; EmbeddedFetchData embedded_fetch;
if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) { if (options.stage == ShaderType::Vertex && options.input_info.vertex != nullptr &&
embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base, options.input_info.vertex->fetch_embedded) {
static_cast<uint32_t>(options.user_data.size()), embedded_fetch = DetectEmbeddedVertexFetch(
options.wave_size); decoded, options.input_info.vertex, options.user_data_base,
static_cast<uint32_t>(options.user_data.size()), options.wave_size);
if (!embedded_fetch.loads.empty()) { if (!embedded_fetch.loads.empty()) {
LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options), LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
static_cast<uint64_t>(embedded_fetch.loads.size())); static_cast<uint64_t>(embedded_fetch.loads.size()));
@@ -612,14 +599,10 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
.shader_hash = options.shader_hash, .shader_hash = options.shader_hash,
.user_data_base = options.user_data_base, .user_data_base = options.user_data_base,
.user_data_count = static_cast<uint32_t>(options.user_data.size()), .user_data_count = static_cast<uint32_t>(options.user_data.size()),
.scratch_dwords = options.scratch_dwords,
.dispatcher_fallback = dispatcher_fallback, .dispatcher_fallback = dispatcher_fallback,
.cfg_failure_kind = cfg.failure_kind, .cfg_failure_kind = cfg.failure_kind,
.fallback_reason = dispatcher_reason.empty() ? cfg.unsupported_reason .fallback_reason = dispatcher_reason.empty() ? cfg.unsupported_reason : dispatcher_reason,
: dispatcher_reason, .input_info = options.input_info,
.vertex = vertex,
.pixel = pixel,
.compute = compute,
.embedded_fetch = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch, .embedded_fetch = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
}; };
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n", LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
@@ -665,24 +648,8 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
IR::RemoveIdentities(ir.blocks); IR::RemoveIdentities(ir.blocks);
IR::EliminateDeadCode(ir.blocks); IR::EliminateDeadCode(ir.blocks);
const ShaderVertexInputInfo* vertex = nullptr; IR::CollectShaderInfo(ir, options.input_info);
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
switch (options.stage) {
case ShaderType::Vertex:
case ShaderType::Mesh: vertex = options.input_info.vertex; break;
case ShaderType::Pixel: pixel = options.input_info.pixel; break;
case ShaderType::Compute: compute = options.input_info.compute; break;
default: EXIT("invalid shader stage\n");
}
IR::ShaderInfoOptions info_options;
info_options.vertex = vertex;
info_options.pixel = pixel;
info_options.compute = compute;
IR::CollectShaderInfo(ir, info_options);
IR::AllocateBindings(ir, push_data_start_dword); IR::AllocateBindings(ir, push_data_start_dword);
Spirv::AnalyzeProgramRequirements(ir);
std::string ir_dump; std::string ir_dump;
if (options.dump_ir) { if (options.dump_ir) {
ir_dump = MakeIrDump(translated.cfg_dump, ir); ir_dump = MakeIrDump(translated.cfg_dump, ir);
@@ -692,11 +659,11 @@ CompileResult CompileProgram(TranslateResult translated, const CompileOptions& o
} }
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n", LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash); GetDumpLabel(options), StageName(ir.stage), ir.shader_hash);
auto spirv = Spirv::EmitProgram(ir, options.input_info); auto spirv = Spirv::EmitProgram(ir, options.input_info);
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64 LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
" elapsed_ms=%" PRIu64 "\n", " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash, GetDumpLabel(options), StageName(ir.stage), ir.shader_hash,
static_cast<uint64_t>(spirv.size()), static_cast<uint64_t>(spirv.size()),
static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>( static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - emit_begin) std::chrono::steady_clock::now() - emit_begin)
@@ -15,7 +15,6 @@ struct CompileOptions {
ShaderType stage = ShaderType::Compute; ShaderType stage = ShaderType::Compute;
uint32_t wave_size = 64; uint32_t wave_size = 64;
uint32_t user_data_base = 0; uint32_t user_data_base = 0;
uint32_t scratch_dwords = 0;
uint64_t shader_hash = 0; uint64_t shader_hash = 0;
bool dump_ir = true; bool dump_ir = true;
bool early_dump = false; bool early_dump = false;
@@ -17,7 +17,7 @@ static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_
} }
const auto opcode = words[0]; const auto opcode = words[0];
const auto word_count = static_cast<uint32_t>(words_num); const auto word_count = static_cast<uint32_t>(words_num);
section.push_back((word_count << 16u) | opcode); section.push_back((word_count << spv::WordCountShift) | opcode);
section.insert(section.end(), words + 1, words + words_num); section.insert(section.end(), words + 1, words + words_num);
} }
@@ -36,15 +36,17 @@ void Builder::RequireVersion(uint32_t version) {
m_version = std::max(m_version, version); m_version = std::max(m_version, version);
} }
void Builder::RequireCapability(uint32_t capability) { void Builder::RequireCapability(spv::Capability capability) {
if (m_required_capabilities.insert(capability).second) { if (m_required_capabilities.insert(capability).second) {
AddCapability({capability}); AppendInstruction(m_capabilities, spv::OpCapability, {capability});
} }
} }
void Builder::RequireExtension(const char* name) { void Builder::RequireExtension(const char* name) {
if (m_required_extensions.emplace(name).second) { if (m_required_extensions.emplace(name).second) {
AddExtension(name); std::vector<uint32_t> operands;
AppendString(operands, name);
AppendInstruction(m_extensions, spv::OpExtension, operands);
} }
} }
@@ -54,15 +56,17 @@ uint32_t Builder::Import(const char* name) {
} }
const auto id = AllocateId(); const auto id = AllocateId();
m_import_ids.emplace(name, id); m_import_ids.emplace(name, id);
AddExtInstImport(id, name); std::vector<uint32_t> operands = {id};
AppendString(operands, name);
AppendInstruction(m_ext_inst_imports, spv::OpExtInstImport, operands);
return id; return id;
} }
uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) { uint32_t Builder::Type(spv::Op opcode, std::initializer_list<uint32_t> operands) {
return Type(opcode, std::vector<uint32_t>(operands)); return Type(opcode, std::vector<uint32_t>(operands));
} }
uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) { uint32_t Builder::Type(spv::Op opcode, const std::vector<uint32_t>& operands) {
std::vector<uint32_t> key; std::vector<uint32_t> key;
key.reserve(operands.size() + 2u); key.reserve(operands.size() + 2u);
key.push_back(opcode); key.push_back(opcode);
@@ -79,7 +83,7 @@ uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
return id; return id;
} }
uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands, uint32_t Builder::DecoratedType(spv::Op opcode, std::initializer_list<uint32_t> operands,
std::initializer_list<TypeAnnotation> annotations) { std::initializer_list<TypeAnnotation> annotations) {
if (annotations.size() == 0) { if (annotations.size() == 0) {
return Type(opcode, operands); return Type(opcode, operands);
@@ -108,12 +112,12 @@ uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t>
return id; return id;
} }
uint32_t Builder::Constant(uint32_t opcode, uint32_t type, uint32_t Builder::Constant(spv::Op opcode, uint32_t type,
std::initializer_list<uint32_t> operands) { std::initializer_list<uint32_t> operands) {
return Constant(opcode, type, std::vector<uint32_t>(operands)); return Constant(opcode, type, std::vector<uint32_t>(operands));
} }
uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) { uint32_t Builder::Constant(spv::Op opcode, uint32_t type, const std::vector<uint32_t>& operands) {
std::vector<uint32_t> key; std::vector<uint32_t> key;
key.reserve(operands.size() + 2u); key.reserve(operands.size() + 2u);
key.push_back(opcode); key.push_back(opcode);
@@ -130,14 +134,15 @@ uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uin
return id; return id;
} }
uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) { uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class) {
const auto id = AllocateId(); const auto id = AllocateId();
DefineGlobalVariable(id, pointer_type, storage_class); DefineGlobalVariable(id, pointer_type, storage_class);
return id; return id;
} }
void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) { void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type,
AddType({59u, pointer_type, id, storage_class}); spv::StorageClass storage_class) {
AppendInstruction(m_declarations, spv::OpVariable, {pointer_type, id, storage_class});
} }
void Builder::AppendString(std::vector<uint32_t>& words, const char* text) { void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
@@ -156,49 +161,33 @@ void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
} }
} }
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode, void Builder::AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
const std::vector<uint32_t>& operands) { const std::vector<uint32_t>& operands) {
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u); const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
section.push_back((word_count << 16u) | opcode); section.push_back((word_count << spv::WordCountShift) | opcode);
section.insert(section.end(), operands.begin(), operands.end()); section.insert(section.end(), operands.begin(), operands.end());
} }
void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode, void Builder::AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
std::initializer_list<uint32_t> operands) { std::initializer_list<uint32_t> operands) {
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u); const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
section.push_back((word_count << 16u) | opcode); section.push_back((word_count << spv::WordCountShift) | opcode);
section.insert(section.end(), operands.begin(), operands.end()); section.insert(section.end(), operands.begin(), operands.end());
} }
void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
AppendInstruction(m_capabilities, 17u, operands);
}
void Builder::AddExtension(const char* name) {
std::vector<uint32_t> operands;
AppendString(operands, name);
AppendInstruction(m_extensions, 10u, operands);
}
void Builder::AddExtInstImport(uint32_t id, const char* name) {
std::vector<uint32_t> operands = {id};
AppendString(operands, name);
AppendInstruction(m_ext_inst_imports, 11u, operands);
}
void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) { void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
AppendInstruction(m_memory_model, 14u, operands); AppendInstruction(m_memory_model, spv::OpMemoryModel, operands);
} }
void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name, void Builder::AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point,
const std::vector<uint32_t>& interfaces) { const char* name, const std::vector<uint32_t>& interfaces) {
std::vector<uint32_t> operands = {execution_model, entry_point}; std::vector<uint32_t> operands = {execution_model, entry_point};
AppendString(operands, name); AppendString(operands, name);
operands.insert(operands.end(), interfaces.begin(), interfaces.end()); operands.insert(operands.end(), interfaces.begin(), interfaces.end());
if (m_version >= 0x00010400u) { if (m_version >= 0x00010400u) {
for (size_t offset = 0; offset < m_declarations.size();) { for (size_t offset = 0; offset < m_declarations.size();) {
const auto count = m_declarations[offset] >> 16u; const auto count = m_declarations[offset] >> spv::WordCountShift;
if ((m_declarations[offset] & 0xffffu) == 59u) { if ((m_declarations[offset] & spv::OpCodeMask) == spv::OpVariable) {
const auto id = m_declarations[offset + 2u]; const auto id = m_declarations[offset + 2u];
if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) { if (std::find(interfaces.begin(), interfaces.end(), id) == interfaces.end()) {
operands.push_back(id); operands.push_back(id);
@@ -207,27 +196,23 @@ void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, cons
offset += count; offset += count;
} }
} }
AppendInstruction(m_entry_points, 15u, operands); AppendInstruction(m_entry_points, spv::OpEntryPoint, operands);
} }
void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) { void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
AppendInstruction(m_execution_modes, 16u, operands); AppendInstruction(m_execution_modes, spv::OpExecutionMode, operands);
} }
void Builder::AddName(uint32_t target, const char* name) { void Builder::AddName(uint32_t target, const char* name) {
std::vector<uint32_t> operands = {target}; std::vector<uint32_t> operands = {target};
AppendString(operands, name); AppendString(operands, name);
AppendInstruction(m_debug, 5u, operands); AppendInstruction(m_debug, spv::OpName, operands);
} }
void Builder::AddAnnotation(std::initializer_list<uint32_t> words) { void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
AppendInstructionWords(m_annotations, words.begin(), words.size()); AppendInstructionWords(m_annotations, words.begin(), words.size());
} }
void Builder::AddType(std::initializer_list<uint32_t> words) {
AppendInstructionWords(m_declarations, words.begin(), words.size());
}
void Builder::AddFunction(std::initializer_list<uint32_t> words) { void Builder::AddFunction(std::initializer_list<uint32_t> words) {
AppendInstructionWords(m_functions, words.begin(), words.size()); AppendInstructionWords(m_functions, words.begin(), words.size());
} }
@@ -237,16 +222,18 @@ void Builder::AddFunction(const std::vector<uint32_t>& words) {
} }
DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) { DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
std::vector<uint32_t> words {245u, type, result}; std::vector<uint32_t> words {spv::OpPhi, type, result};
words.resize(words.size() + incoming_count * 2u); words.resize(words.size() + incoming_count * 2u);
const DeferredPhi phi {m_functions.size(), incoming_count}; const DeferredPhi phi {m_functions.size()};
AddFunction(words); AddFunction(words);
m_unpatched_phi_incomings += incoming_count; m_unpatched_phi_incomings += incoming_count;
return phi; return phi;
} }
void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) { void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0); const auto incoming_count =
((m_functions.at(phi.word_offset) >> spv::WordCountShift) - 3u) / 2u;
EXIT_IF(incoming >= incoming_count || value == 0 || parent == 0);
const auto value_word = phi.word_offset + 3u + incoming * 2u; const auto value_word = phi.word_offset + 3u + incoming * 2u;
const auto parent_word = value_word + 1u; const auto parent_word = value_word + 1u;
EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0); EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
@@ -264,7 +251,7 @@ std::vector<uint32_t> Builder::Build() const {
m_debug.size() + m_annotations.size() + m_declarations.size() + m_debug.size() + m_annotations.size() + m_declarations.size() +
m_functions.size()); m_functions.size());
module.push_back(0x07230203u); module.push_back(spv::MagicNumber);
module.push_back(m_version); module.push_back(m_version);
module.push_back(0u); module.push_back(0u);
module.push_back(m_next_id); module.push_back(m_next_id);
@@ -6,19 +6,19 @@
#include <initializer_list> #include <initializer_list>
#include <map> #include <map>
#include <set> #include <set>
#include <spirv/unified1/spirv.hpp>
#include <string> #include <string>
#include <vector> #include <vector>
namespace Libs::Graphics::ShaderRecompiler::Spirv { namespace Libs::Graphics::ShaderRecompiler::Spirv {
struct TypeAnnotation { struct TypeAnnotation {
uint32_t opcode = 0; spv::Op opcode = spv::OpNop;
std::vector<uint32_t> operands; std::vector<uint32_t> operands;
}; };
struct DeferredPhi { struct DeferredPhi {
size_t word_offset = 0; size_t word_offset = 0;
size_t incoming_count = 0;
}; };
class Builder { class Builder {
@@ -29,21 +29,20 @@ public:
uint32_t AllocateId(); uint32_t AllocateId();
void RequireVersion(uint32_t version); void RequireVersion(uint32_t version);
void RequireCapability(uint32_t capability); void RequireCapability(spv::Capability capability);
void RequireExtension(const char* name); void RequireExtension(const char* name);
uint32_t Import(const char* name); uint32_t Import(const char* name);
uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {}); uint32_t Type(spv::Op opcode, std::initializer_list<uint32_t> operands = {});
uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands); uint32_t Type(spv::Op opcode, const std::vector<uint32_t>& operands);
uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands, uint32_t DecoratedType(spv::Op opcode, std::initializer_list<uint32_t> operands,
std::initializer_list<TypeAnnotation> annotations); std::initializer_list<TypeAnnotation> annotations);
uint32_t Constant(uint32_t opcode, uint32_t type, uint32_t Constant(spv::Op opcode, uint32_t type, std::initializer_list<uint32_t> operands = {});
std::initializer_list<uint32_t> operands = {}); uint32_t Constant(spv::Op opcode, uint32_t type, const std::vector<uint32_t>& operands);
uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands); uint32_t DefineGlobalVariable(uint32_t pointer_type, spv::StorageClass storage_class);
uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class); void DefineGlobalVariable(uint32_t id, uint32_t pointer_type, spv::StorageClass storage_class);
void DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
void AddMemoryModel(std::initializer_list<uint32_t> operands); void AddMemoryModel(std::initializer_list<uint32_t> operands);
void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name, void AddEntryPoint(spv::ExecutionModel execution_model, uint32_t entry_point, const char* name,
const std::vector<uint32_t>& interfaces); const std::vector<uint32_t>& interfaces);
void AddExecutionMode(std::initializer_list<uint32_t> operands); void AddExecutionMode(std::initializer_list<uint32_t> operands);
void AddName(uint32_t target, const char* name); void AddName(uint32_t target, const char* name);
@@ -56,15 +55,11 @@ public:
[[nodiscard]] std::vector<uint32_t> Build() const; [[nodiscard]] std::vector<uint32_t> Build() const;
private: private:
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode, static void AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
const std::vector<uint32_t>& operands); const std::vector<uint32_t>& operands);
static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode, static void AppendInstruction(std::vector<uint32_t>& section, spv::Op opcode,
std::initializer_list<uint32_t> operands); std::initializer_list<uint32_t> operands);
static void AppendString(std::vector<uint32_t>& words, const char* text); static void AppendString(std::vector<uint32_t>& words, const char* text);
void AddCapability(std::initializer_list<uint32_t> operands);
void AddExtension(const char* name);
void AddExtInstImport(uint32_t id, const char* name);
void AddType(std::initializer_list<uint32_t> words);
uint32_t m_next_id = 1; uint32_t m_next_id = 1;
uint32_t m_version = 0; uint32_t m_version = 0;
@@ -78,7 +73,7 @@ private:
std::vector<uint32_t> m_annotations; std::vector<uint32_t> m_annotations;
std::vector<uint32_t> m_declarations; std::vector<uint32_t> m_declarations;
std::vector<uint32_t> m_functions; std::vector<uint32_t> m_functions;
std::set<uint32_t> m_required_capabilities; std::set<spv::Capability> m_required_capabilities;
std::set<std::string> m_required_extensions; std::set<std::string> m_required_extensions;
std::map<std::string, uint32_t> m_import_ids; std::map<std::string, uint32_t> m_import_ids;
std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids; std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
@@ -183,10 +183,8 @@ void ValidateNativeProgram(const IR::Program& program) {
} // namespace } // namespace
void AnalyzeProgramRequirements(IR::Program& program) { Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program& program) {
program.spirv_requirements.reset(); SpirvRequirements requirements {};
IR::SpirvRequirements requirements {};
const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic && if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
@@ -242,17 +240,17 @@ void AnalyzeProgramRequirements(IR::Program& program) {
} }
if (shared_access == IR::SharedAccess::Append || if (shared_access == IR::SharedAccess::Append ||
shared_access == IR::SharedAccess::Consume) { shared_access == IR::SharedAccess::Consume) {
MarkBallot(); requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true; requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true; requirements.subgroup_local_invocation_id = true;
} }
} }
switch (inst.GetOpcode()) { switch (inst.GetOpcode()) {
case IR::ValueOpcode::Ballot: MarkBallot(); break; case IR::ValueOpcode::Ballot: requirements.subgroup_ballot = true; break;
case IR::ValueOpcode::DppMoveU32: case IR::ValueOpcode::DppMoveU32:
case IR::ValueOpcode::ReadFirstLane: case IR::ValueOpcode::ReadFirstLane:
case IR::ValueOpcode::ReadLane: { case IR::ValueOpcode::ReadLane: {
MarkBallot(); requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true; requirements.subgroup_shuffle = true;
if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) { if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
requirements.subgroup_local_invocation_id = true; requirements.subgroup_local_invocation_id = true;
@@ -261,19 +259,19 @@ void AnalyzeProgramRequirements(IR::Program& program) {
} }
case IR::ValueOpcode::DppUpdateU32: case IR::ValueOpcode::DppUpdateU32:
case IR::ValueOpcode::WriteLane: { case IR::ValueOpcode::WriteLane: {
MarkBallot(); requirements.subgroup_ballot = true;
requirements.subgroup_local_invocation_id = true; requirements.subgroup_local_invocation_id = true;
break; break;
} }
case IR::ValueOpcode::Permlane16U32: { case IR::ValueOpcode::Permlane16U32: {
MarkBallot(); requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true; requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true; requirements.subgroup_local_invocation_id = true;
break; break;
} }
case IR::ValueOpcode::SwizzleU32: case IR::ValueOpcode::SwizzleU32:
case IR::ValueOpcode::BpermuteU32: { case IR::ValueOpcode::BpermuteU32: {
MarkBallot(); requirements.subgroup_ballot = true;
requirements.subgroup_shuffle = true; requirements.subgroup_shuffle = true;
requirements.subgroup_local_invocation_id = true; requirements.subgroup_local_invocation_id = true;
break; break;
@@ -300,7 +298,7 @@ void AnalyzeProgramRequirements(IR::Program& program) {
} }
} }
} }
program.spirv_requirements.emplace(requirements); return requirements;
} }
std::vector<uint32_t> EmitProgram(const IR::Program& program, std::vector<uint32_t> EmitProgram(const IR::Program& program,
@@ -312,14 +310,12 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders"); Fail(program, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
} }
if (!program.srt_plan_complete || !program.resource_tracking_complete || if (!program.srt_plan_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete || !program.shader_info_complete || !program.binding_layout_complete) {
!program.spirv_requirements.has_value()) {
Fail(program, "SPIR-V emitter requires a fully planned native shader program"); Fail(program, "SPIR-V emitter requires a fully planned native shader program");
} }
ValidateNativeProgram(program); ValidateNativeProgram(program);
IR::ValidateProgram(program, true); IR::ValidateProgram(program, true);
EmitterState state(program, input_info); EmitterState state(program, input_info);
state.stage = program.stage;
const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info); const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
state.lane_count = state.lane_count =
workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u workgroup != nullptr && program.wave_size == 64u && workgroup->host_subgroup_size == 32u
@@ -333,14 +329,10 @@ std::vector<uint32_t> EmitProgram(const IR::Program& program,
AllocateOutputVariables(state); AllocateOutputVariables(state);
DefineModule(state); DefineModule(state);
EmitProgram(state); EmitProgram(state);
state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main", state.builder.AddEntryPoint(ExecutionModelForStage(state.program.stage), state.main_func,
state.interface_variables); "main", state.interface_variables);
auto binary = state.builder.Build(); return state.builder.Build();
if (binary.empty()) {
Fail(program, "SPIR-V builder returned an empty module");
}
return binary;
} }
} // namespace Libs::Graphics::ShaderRecompiler::Spirv } // namespace Libs::Graphics::ShaderRecompiler::Spirv
@@ -9,8 +9,6 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv { namespace Libs::Graphics::ShaderRecompiler::Spirv {
void AnalyzeProgramRequirements(IR::Program& program);
std::vector<uint32_t> EmitProgram(const IR::Program& program, std::vector<uint32_t> EmitProgram(const IR::Program& program,
ShaderStageInputInfo input_info); ShaderStageInputInfo input_info);
@@ -12,33 +12,33 @@ struct Pair {
Pair ExtractPair(EmitterState& state, uint32_t value) { Pair ExtractPair(EmitterState& state, uint32_t value) {
Pair result {state.builder.AllocateId(), state.builder.AllocateId()}; Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.low, value, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), result.low, value, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.high, value, 1}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), result.high, value, 1});
return result; return result;
} }
uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) { uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU64(state), result, low, high}); state.builder.AddFunction({spv::OpCompositeConstruct, TypeU64(state), result, low, high});
return result; return result;
} }
uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
bool not_equal) { bool not_equal) {
const auto compare = Binary(state, not_equal ? OpINotEqual : OpIEqual, const auto compare = Binary(state, not_equal ? spv::OpINotEqual : spv::OpIEqual,
TypeBoolVector(state, 2), lhs_value, rhs_value); TypeBoolVector(state, 2), lhs_value, rhs_value);
return Unary(state, not_equal ? OpAny : OpAll, TypeBool(state), compare); return Unary(state, not_equal ? spv::OpAny : spv::OpAll, TypeBool(state), compare);
} }
uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value, uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
uint32_t high_compare, uint32_t low_compare) { uint32_t high_compare, uint32_t low_compare) {
const auto lhs = ExtractPair(state, lhs_value); const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value); const auto rhs = ExtractPair(state, rhs_value);
const auto high_equal = Binary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high); const auto high_equal = Binary(state, spv::OpIEqual, TypeBool(state), lhs.high, rhs.high);
const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high); const auto high_result = Binary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
const auto low_result = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low); const auto low_result = Binary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
const auto low_path = Binary(state, OpLogicalAnd, TypeBool(state), high_equal, low_result); const auto low_path = Binary(state, spv::OpLogicalAnd, TypeBool(state), high_equal, low_result);
return Binary(state, OpLogicalOr, TypeBool(state), high_result, low_path); return Binary(state, spv::OpLogicalOr, TypeBool(state), high_result, low_path);
} }
uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
@@ -47,23 +47,23 @@ uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value)
const auto low_pair = state.builder.AllocateId(); const auto low_pair = state.builder.AllocateId();
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
const auto carry = state.builder.AllocateId(); const auto carry = state.builder.AllocateId();
state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low}); state.builder.AddFunction({spv::OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, low_pair, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, low_pair, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry, low_pair, 1}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
const auto high0 = Binary(state, OpIAdd, TypeU32(state), lhs.high, rhs.high); const auto high0 = Binary(state, spv::OpIAdd, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, OpIAdd, TypeU32(state), high0, carry); const auto high = Binary(state, spv::OpIAdd, TypeU32(state), high0, carry);
return MakePair(state, low, high); return MakePair(state, low, high);
} }
uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value); const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value); const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, OpISub, TypeU32(state), lhs.low, rhs.low); const auto low = Binary(state, spv::OpISub, TypeU32(state), lhs.low, rhs.low);
const auto borrow = Binary(state, OpULessThan, TypeBool(state), lhs.low, rhs.low); const auto borrow = Binary(state, spv::OpULessThan, TypeBool(state), lhs.low, rhs.low);
const auto borrow_u32 = const auto borrow_u32 =
Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0)); Select(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
const auto high0 = Binary(state, OpISub, TypeU32(state), lhs.high, rhs.high); const auto high0 = Binary(state, spv::OpISub, TypeU32(state), lhs.high, rhs.high);
const auto high = Binary(state, OpISub, TypeU32(state), high0, borrow_u32); const auto high = Binary(state, spv::OpISub, TypeU32(state), high0, borrow_u32);
return MakePair(state, low, high); return MakePair(state, low, high);
} }
@@ -73,61 +73,62 @@ uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signe
uint32_t lhs_operand = lhs; uint32_t lhs_operand = lhs;
uint32_t rhs_operand = rhs; uint32_t rhs_operand = rhs;
if (signed_value) { if (signed_value) {
lhs_operand = Unary(state, OpBitcast, TypeI32(state), lhs); lhs_operand = Unary(state, spv::OpBitcast, TypeI32(state), lhs);
rhs_operand = Unary(state, OpBitcast, TypeI32(state), rhs); rhs_operand = Unary(state, spv::OpBitcast, TypeI32(state), rhs);
} }
const auto extended = state.builder.AllocateId(); const auto extended = state.builder.AllocateId();
state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended, state.builder.AddFunction({signed_value ? spv::OpSMulExtended : spv::OpUMulExtended, pair_type,
lhs_operand, rhs_operand}); extended, lhs_operand, rhs_operand});
const auto high = state.builder.AllocateId(); const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1}); state.builder.AddFunction({spv::OpCompositeExtract, operand_type, high, extended, 1});
return signed_value ? Unary(state, OpBitcast, TypeU32(state), high) : high; return signed_value ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high;
} }
uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) { uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
const auto lhs = ExtractPair(state, lhs_value); const auto lhs = ExtractPair(state, lhs_value);
const auto rhs = ExtractPair(state, rhs_value); const auto rhs = ExtractPair(state, rhs_value);
const auto low = Binary(state, OpIMul, TypeU32(state), lhs.low, rhs.low); const auto low = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.low);
const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false); const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
const auto high1 = Binary(state, OpIMul, TypeU32(state), lhs.low, rhs.high); const auto high1 = Binary(state, spv::OpIMul, TypeU32(state), lhs.low, rhs.high);
const auto high2 = Binary(state, OpIMul, TypeU32(state), lhs.high, rhs.low); const auto high2 = Binary(state, spv::OpIMul, TypeU32(state), lhs.high, rhs.low);
return MakePair(state, low, return MakePair(state, low,
Binary(state, OpIAdd, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, OpIAdd, TypeU32(state), high0, high1), high2)); Binary(state, spv::OpIAdd, TypeU32(state), high0, high1), high2));
} }
uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) { uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
const auto pair = ExtractPair(state, value); const auto pair = ExtractPair(state, value);
uint32_t low = 0; uint32_t low = 0;
uint32_t high = 0; uint32_t high = 0;
if (opcode == OpShiftLeftLogical) { if (opcode == spv::OpShiftLeftLogical) {
EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high); EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else if (opcode == OpShiftRightLogical) { } else if (opcode == spv::OpShiftRightLogical) {
EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high); EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
} else { } else {
const auto amount = const auto amount =
Binary(state, OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63)); Binary(state, spv::OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
const auto word_shift = const auto word_shift =
Binary(state, OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31)); Binary(state, spv::OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
const auto at_least_32 = const auto at_least_32 = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), amount,
Binary(state, OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32)); ConstantU32(state, 32));
const auto nonzero = const auto nonzero =
Binary(state, OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0)); Binary(state, spv::OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
const auto high_shifted = const auto high_shifted =
Binary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift); Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
const auto low_shifted = const auto low_shifted =
Binary(state, OpShiftRightLogical, TypeU32(state), pair.low, word_shift); Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
const auto carry_count = const auto carry_count =
Binary(state, OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, OpISub, TypeU32(state), ConstantU32(state, 32), word_shift), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
ConstantU32(state, 31)); ConstantU32(state, 31));
const auto carry = const auto carry =
Select(state, TypeU32(state), nonzero, Select(state, TypeU32(state), nonzero,
Binary(state, OpShiftLeftLogical, TypeU32(state), pair.high, carry_count), Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
ConstantU32(state, 0)); ConstantU32(state, 0));
const auto low_below_32 = Binary(state, OpBitwiseOr, TypeU32(state), low_shifted, carry); const auto low_below_32 =
Binary(state, spv::OpBitwiseOr, TypeU32(state), low_shifted, carry);
const auto negative = const auto negative =
Binary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0)); Binary(state, spv::OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
const auto sign_fill = Select(state, TypeU32(state), negative, const auto sign_fill = Select(state, TypeU32(state), negative,
ConstantU32(state, 0xffffffffu), ConstantU32(state, 0)); ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
return MakePair(state, return MakePair(state,
@@ -143,34 +144,34 @@ uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t valu
return value; return value;
} }
const auto pair = ExtractPair(state, value); const auto pair = ExtractPair(state, value);
if (opcode == OpShiftLeftLogical) { if (opcode == spv::OpShiftLeftLogical) {
if (shift < 32u) { if (shift < 32u) {
const auto low = Binary(state, OpShiftLeftLogical, TypeU32(state), pair.low, const auto low = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.low,
ConstantU32(state, shift)); ConstantU32(state, shift));
const auto high = Binary(state, OpBitwiseOr, TypeU32(state), const auto high = Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
pair.high, ConstantU32(state, shift)), pair.high, ConstantU32(state, shift)),
Binary(state, OpShiftRightLogical, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeU32(state),
pair.low, ConstantU32(state, 32u - shift))); pair.low, ConstantU32(state, 32u - shift)));
return MakePair(state, low, high); return MakePair(state, low, high);
} }
return MakePair(state, ConstantU32(state, 0), return MakePair(state, ConstantU32(state, 0),
shift == 32u ? pair.low shift == 32u ? pair.low
: Binary(state, OpShiftLeftLogical, TypeU32(state), : Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
pair.low, ConstantU32(state, shift - 32u))); pair.low, ConstantU32(state, shift - 32u)));
} }
if (shift < 32u) { if (shift < 32u) {
const auto low = Binary(state, OpBitwiseOr, TypeU32(state), const auto low = Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), pair.low, Binary(state, spv::OpShiftRightLogical, TypeU32(state), pair.low,
ConstantU32(state, shift)), ConstantU32(state, shift)),
Binary(state, OpShiftLeftLogical, TypeU32(state), pair.high, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), pair.high,
ConstantU32(state, 32u - shift))); ConstantU32(state, 32u - shift)));
const auto high = const auto high =
Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift)); Binary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
return MakePair(state, low, high); return MakePair(state, low, high);
} }
const auto high = opcode == OpShiftRightArithmetic const auto high = opcode == spv::OpShiftRightArithmetic
? Binary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, ? Binary(state, spv::OpShiftRightArithmetic, TypeU32(state), pair.high,
ConstantU32(state, 31u)) ConstantU32(state, 31u))
: ConstantU32(state, 0); : ConstantU32(state, 0);
const auto low = shift == 32u ? pair.high const auto low = shift == 32u ? pair.high
@@ -184,15 +185,15 @@ uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
const auto high_i = state.builder.AllocateId(); const auto high_i = state.builder.AllocateId();
const auto low_i = state.builder.AllocateId(); const auto low_i = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeI32(state), high_i, GlslStd450(state), GlslFindUMsb, pair.high}); {spv::OpExtInst, TypeI32(state), high_i, GlslStd450(state), GLSLstd450FindUMsb, pair.high});
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeI32(state), low_i, GlslStd450(state), GlslFindUMsb, pair.low}); {spv::OpExtInst, TypeI32(state), low_i, GlslStd450(state), GLSLstd450FindUMsb, pair.low});
const auto high = Unary(state, OpBitcast, TypeU32(state), high_i); const auto high = Unary(state, spv::OpBitcast, TypeU32(state), high_i);
const auto low = Unary(state, OpBitcast, TypeU32(state), low_i); const auto low = Unary(state, spv::OpBitcast, TypeU32(state), low_i);
const auto high_nonzero = const auto high_nonzero =
Binary(state, OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0)); Binary(state, spv::OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
return Select(state, TypeU32(state), high_nonzero, return Select(state, TypeU32(state), high_nonzero,
Binary(state, OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low); Binary(state, spv::OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
} }
uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value, uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
@@ -206,10 +207,10 @@ uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bo
uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) { uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false); const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true); const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
const auto ab = Binary(state, OpIAdd, TypeU32(state), a, b); const auto ab = Binary(state, spv::OpIAdd, TypeU32(state), a, b);
const auto abc = Binary(state, OpIAdd, TypeU32(state), ab, c); const auto abc = Binary(state, spv::OpIAdd, TypeU32(state), ab, c);
return Binary(state, OpISub, TypeU32(state), return Binary(state, spv::OpISub, TypeU32(state),
Binary(state, OpISub, TypeU32(state), abc, minimum), maximum); Binary(state, spv::OpISub, TypeU32(state), abc, minimum), maximum);
} }
uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) { uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
@@ -222,17 +223,17 @@ uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
const auto max_ab = EmitMinMaxF32Value(state, a, b, true); const auto max_ab = EmitMinMaxF32Value(state, a, b, true);
const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false); const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true); const auto median = EmitMinMaxF32Value(state, min_ab, high_min, true);
const auto nan_ab = Binary(state, OpLogicalOr, TypeBool(state), const auto nan_ab = Binary(state, spv::OpLogicalOr, TypeBool(state),
EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan); EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
const auto any_nan = const auto any_nan =
Binary(state, OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan); Binary(state, spv::OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
return Select(state, TypeF32(state), any_nan, min3, median); return Select(state, TypeF32(state), any_nan, min3, median);
} }
uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode, uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
std::initializer_list<uint32_t> args) { std::initializer_list<uint32_t> args) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
std::vector<uint32_t> words {OpExtInst, type, result, GlslStd450(state), opcode}; std::vector<uint32_t> words {spv::OpExtInst, type, result, GlslStd450(state), opcode};
words.insert(words.end(), args.begin(), args.end()); words.insert(words.end(), args.begin(), args.end());
state.builder.AddFunction(words); state.builder.AddFunction(words);
return result; return result;
@@ -243,16 +244,16 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
const auto converted_raw = state.builder.AllocateId(); const auto converted_raw = state.builder.AllocateId();
if (signed_value) { if (signed_value) {
const auto converted_i = state.builder.AllocateId(); const auto converted_i = state.builder.AllocateId();
state.builder.AddFunction({OpConvertFToS, TypeI32(state), converted_i, trunc}); state.builder.AddFunction({spv::OpConvertFToS, TypeI32(state), converted_i, trunc});
state.builder.AddFunction({OpBitcast, TypeU32(state), converted_raw, converted_i}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), converted_raw, converted_i});
} else { } else {
state.builder.AddFunction({OpConvertFToU, TypeU32(state), converted_raw, trunc}); state.builder.AddFunction({spv::OpConvertFToU, TypeU32(state), converted_raw, trunc});
} }
const auto nan = EmitClassifyF32(state, src).nan; const auto nan = EmitClassifyF32(state, src).nan;
if (signed_value) { if (signed_value) {
const auto below = Binary(state, OpFOrdLessThanEqual, TypeBool(state), src, const auto below = Binary(state, spv::OpFOrdLessThanEqual, TypeBool(state), src,
ConstantF32(state, 0xcf000000u)); ConstantF32(state, 0xcf000000u));
const auto above = Binary(state, OpFOrdGreaterThanEqual, TypeBool(state), src, const auto above = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
ConstantF32(state, 0x4f000000u)); ConstantF32(state, 0x4f000000u));
const auto high = const auto high =
Select(state, TypeU32(state), above, ConstantU32(state, 0x7fffffffu), converted_raw); Select(state, TypeU32(state), above, ConstantU32(state, 0x7fffffffu), converted_raw);
@@ -261,17 +262,17 @@ uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
return Select(state, TypeU32(state), nan, ConstantU32(state, 0), low); return Select(state, TypeU32(state), nan, ConstantU32(state, 0), low);
} }
const auto below = const auto below =
Binary(state, OpFOrdLessThanEqual, TypeBool(state), src, ConstantF32(state, 0)); Binary(state, spv::OpFOrdLessThanEqual, TypeBool(state), src, ConstantF32(state, 0));
const auto above = Binary(state, OpFOrdGreaterThanEqual, TypeBool(state), src, const auto above = Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), src,
ConstantF32(state, 0x4f800000u)); ConstantF32(state, 0x4f800000u));
const auto zero = Binary(state, OpLogicalOr, TypeBool(state), nan, below); const auto zero = Binary(state, spv::OpLogicalOr, TypeBool(state), nan, below);
const auto high = const auto high =
Select(state, TypeU32(state), above, ConstantU32(state, 0xffffffffu), converted_raw); Select(state, TypeU32(state), above, ConstantU32(state, 0xffffffffu), converted_raw);
return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high); return Select(state, TypeU32(state), zero, ConstantU32(state, 0), high);
} }
uint32_t EmitPackHalf(EmitterState& state, uint32_t src) { uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
return EmitExt(state, TypeU32(state), GlslPackHalf2x16, {src}); return EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {src});
} }
} // namespace } // namespace
@@ -296,19 +297,19 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::BitCastF16U16: case IR::ValueOpcode::BitCastF16U16:
case IR::ValueOpcode::ConvertU32U16: case IR::ValueOpcode::ConvertU32U16:
case IR::ValueOpcode::ConvertU32U8: case IR::ValueOpcode::ConvertU32U8:
case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32); case IR::ValueOpcode::BitCastU32F32: return unary(spv::OpBitcast, IR::Type::U32);
case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32); case IR::ValueOpcode::BitCastF32U32: return unary(spv::OpBitcast, IR::Type::F32);
case IR::ValueOpcode::ConvertU16U32: case IR::ValueOpcode::ConvertU16U32:
ctx.Emit(inst, OpBitwiseAnd, IR::Type::U16, ctx.Emit(inst, spv::OpBitwiseAnd, IR::Type::U16,
{ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)}); {ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)});
return true; return true;
case IR::ValueOpcode::ConvertU8U32: case IR::ValueOpcode::ConvertU8U32:
ctx.Emit(inst, OpBitwiseAnd, IR::Type::U8, ctx.Emit(inst, spv::OpBitwiseAnd, IR::Type::U8,
{ctx.Arg(inst, 0), ConstantU32(state, 0xffu)}); {ctx.Arg(inst, 0), ConstantU32(state, 0xffu)});
return true; return true;
case IR::ValueOpcode::ConvertF16F32: { case IR::ValueOpcode::ConvertF16F32: {
const auto pair = state.builder.AllocateId(); const auto pair = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 2), pair, state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 2), pair,
ctx.Arg(inst, 0), ConstantF32(state, 0)}); ctx.Arg(inst, 0), ConstantF32(state, 0)});
ctx.Define(inst, EmitPackHalf(state, pair)); ctx.Define(inst, EmitPackHalf(state, pair));
return true; return true;
@@ -323,53 +324,57 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false)); ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
return true; return true;
case IR::ValueOpcode::ConvertF32S32: { case IR::ValueOpcode::ConvertF32S32: {
const auto signed_value = Unary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 0)); const auto signed_value =
ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value}); Unary(state, spv::OpBitcast, TypeI32(state), ctx.Arg(inst, 0));
ctx.Emit(inst, spv::OpConvertSToF, IR::Type::F32, {signed_value});
return true; return true;
} }
case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32); case IR::ValueOpcode::ConvertF32U32: return unary(spv::OpConvertUToF, IR::Type::F32);
case IR::ValueOpcode::CompositeConstructU64: case IR::ValueOpcode::CompositeConstructU64:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U64, ctx.Emit(inst, spv::OpCompositeConstruct, IR::Type::U64,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true; return true;
case IR::ValueOpcode::CompositeConstructU32x2: case IR::ValueOpcode::CompositeConstructU32x2:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2, ctx.Emit(inst, spv::OpCompositeConstruct, IR::Type::U32x2,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true; return true;
case IR::ValueOpcode::CompositeConstructU32x3: case IR::ValueOpcode::CompositeConstructU32x3:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x3, ctx.Emit(inst, spv::OpCompositeConstruct, IR::Type::U32x3,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true; return true;
case IR::ValueOpcode::CompositeConstructF32x2: case IR::ValueOpcode::CompositeConstructF32x2:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2, ctx.Emit(inst, spv::OpCompositeConstruct, IR::Type::F32x2,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true; return true;
case IR::ValueOpcode::CompositeConstructU32x4: case IR::ValueOpcode::CompositeConstructU32x4:
ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4, ctx.Emit(inst, spv::OpCompositeConstruct, IR::Type::U32x4,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
return true; return true;
case IR::ValueOpcode::CompositeExtractU64: case IR::ValueOpcode::CompositeExtractU64:
case IR::ValueOpcode::CompositeExtractU32x2: case IR::ValueOpcode::CompositeExtractU32x2:
case IR::ValueOpcode::CompositeExtractU32x3: case IR::ValueOpcode::CompositeExtractU32x3:
case IR::ValueOpcode::CompositeExtractU32x4: case IR::ValueOpcode::CompositeExtractU32x4:
ctx.Emit(inst, OpCompositeExtract, IR::Type::U32, ctx.Emit(inst, spv::OpCompositeExtract, IR::Type::U32,
{ctx.Arg(inst, 0), inst.Arg(1).U32()}); {ctx.Arg(inst, 0), inst.Arg(1).U32()});
return true; return true;
case IR::ValueOpcode::PackHalf2x16: case IR::ValueOpcode::PackHalf2x16:
ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackHalf2x16, {ctx.Arg(inst, 0)})); ctx.Define(inst,
EmitExt(state, TypeU32(state), GLSLstd450PackHalf2x16, {ctx.Arg(inst, 0)}));
return true; return true;
case IR::ValueOpcode::PackSnorm2x16: case IR::ValueOpcode::PackSnorm2x16:
ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackSnorm2x16, {ctx.Arg(inst, 0)})); ctx.Define(inst,
EmitExt(state, TypeU32(state), GLSLstd450PackSnorm2x16, {ctx.Arg(inst, 0)}));
return true; return true;
case IR::ValueOpcode::PackUnorm2x16: case IR::ValueOpcode::PackUnorm2x16:
ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackUnorm2x16, {ctx.Arg(inst, 0)})); ctx.Define(inst,
EmitExt(state, TypeU32(state), GLSLstd450PackUnorm2x16, {ctx.Arg(inst, 0)}));
return true; return true;
case IR::ValueOpcode::PackFloat2x16Rtz: { case IR::ValueOpcode::PackFloat2x16Rtz: {
const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0)); const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
const auto high = const auto high =
Binary(state, OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16)); EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
ctx.Define(inst, Binary(state, OpBitwiseOr, TypeU32(state), low, high)); ctx.Define(inst, Binary(state, spv::OpBitwiseOr, TypeU32(state), low, high));
return true; return true;
} }
case IR::ValueOpcode::FPAbs32: case IR::ValueOpcode::FPAbs32:
@@ -379,38 +384,38 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0))); ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
return true; return true;
case IR::ValueOpcode::FPSaturate32: case IR::ValueOpcode::FPSaturate32:
ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFClamp, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450FClamp,
{ctx.Arg(inst, 0), ConstantF32(state, 0), {ctx.Arg(inst, 0), ConstantF32(state, 0),
ConstantF32(state, 0x3f800000u)})); ConstantF32(state, 0x3f800000u)}));
return true; return true;
case IR::ValueOpcode::BitFieldInsert: case IR::ValueOpcode::BitFieldInsert:
ctx.Emit(inst, OpBitFieldInsert, IR::Type::U32, ctx.Emit(inst, spv::OpBitFieldInsert, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
return true; return true;
case IR::ValueOpcode::BitFieldUExtract: case IR::ValueOpcode::BitFieldUExtract:
ctx.Emit(inst, OpBitFieldUExtract, IR::Type::U32, ctx.Emit(inst, spv::OpBitFieldUExtract, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true; return true;
case IR::ValueOpcode::BitFieldSExtract: case IR::ValueOpcode::BitFieldSExtract:
ctx.Emit(inst, OpBitFieldSExtract, IR::Type::U32, ctx.Emit(inst, spv::OpBitFieldSExtract, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true; return true;
case IR::ValueOpcode::SelectU1: case IR::ValueOpcode::SelectU1:
ctx.Emit(inst, OpSelect, IR::Type::U1, ctx.Emit(inst, spv::OpSelect, IR::Type::U1,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true; return true;
case IR::ValueOpcode::SelectU32: case IR::ValueOpcode::SelectU32:
ctx.Emit(inst, OpSelect, IR::Type::U32, ctx.Emit(inst, spv::OpSelect, IR::Type::U32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true; return true;
case IR::ValueOpcode::SelectF32: case IR::ValueOpcode::SelectF32:
ctx.Emit(inst, OpSelect, IR::Type::F32, ctx.Emit(inst, spv::OpSelect, IR::Type::F32,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
return true; return true;
case IR::ValueOpcode::IAdd32: return binary(OpIAdd, IR::Type::U32); case IR::ValueOpcode::IAdd32: return binary(spv::OpIAdd, IR::Type::U32);
case IR::ValueOpcode::ISub32: return binary(OpISub, IR::Type::U32); case IR::ValueOpcode::ISub32: return binary(spv::OpISub, IR::Type::U32);
case IR::ValueOpcode::IMul32: return binary(OpIMul, IR::Type::U32); case IR::ValueOpcode::IMul32: return binary(spv::OpIMul, IR::Type::U32);
case IR::ValueOpcode::UDiv32: return binary(OpUDiv, IR::Type::U32); case IR::ValueOpcode::UDiv32: return binary(spv::OpUDiv, IR::Type::U32);
case IR::ValueOpcode::IAdd64: case IR::ValueOpcode::IAdd64:
ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true; return true;
@@ -421,7 +426,7 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true; return true;
case IR::ValueOpcode::IAddCarry32: case IR::ValueOpcode::IAddCarry32:
ctx.Emit(inst, OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); ctx.Emit(inst, spv::OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true; return true;
case IR::ValueOpcode::SMulHi: case IR::ValueOpcode::SMulHi:
ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true)); ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
@@ -431,63 +436,68 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
return true; return true;
case IR::ValueOpcode::IAbs32: { case IR::ValueOpcode::IAbs32: {
const auto value = ctx.Arg(inst, 0); const auto value = ctx.Arg(inst, 0);
const auto neg = Unary(state, OpSNegate, TypeU32(state), value); const auto neg = Unary(state, spv::OpSNegate, TypeU32(state), value);
const auto negative = const auto negative =
Binary(state, OpSLessThan, TypeBool(state), value, ConstantU32(state, 0)); Binary(state, spv::OpSLessThan, TypeBool(state), value, ConstantU32(state, 0));
ctx.Define(inst, Select(state, TypeU32(state), negative, neg, value)); ctx.Define(inst, Select(state, TypeU32(state), negative, neg, value));
return true; return true;
} }
case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32); case IR::ValueOpcode::ShiftLeftLogical32:
return binary(spv::OpShiftLeftLogical, IR::Type::U32);
case IR::ValueOpcode::ShiftRightLogical32: case IR::ValueOpcode::ShiftRightLogical32:
return binary(OpShiftRightLogical, IR::Type::U32); return binary(spv::OpShiftRightLogical, IR::Type::U32);
case IR::ValueOpcode::ShiftRightArithmetic32: case IR::ValueOpcode::ShiftRightArithmetic32:
return binary(OpShiftRightArithmetic, IR::Type::U32); return binary(spv::OpShiftRightArithmetic, IR::Type::U32);
case IR::ValueOpcode::ShiftLeftLogical64: case IR::ValueOpcode::ShiftLeftLogical64:
ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate() ctx.Define(inst,
? EmitConstantShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), inst.Arg(1).Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftLeftLogical, ctx.Arg(inst, 0),
inst.Arg(1).Resolve().U32()) inst.Arg(1).Resolve().U32())
: EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), : EmitShift64(state, spv::OpShiftLeftLogical, ctx.Arg(inst, 0),
ctx.Arg(inst, 1))); ctx.Arg(inst, 1)));
return true; return true;
case IR::ValueOpcode::ShiftRightLogical64: case IR::ValueOpcode::ShiftRightLogical64:
ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate() ctx.Define(inst,
? EmitConstantShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), inst.Arg(1).Resolve().IsImmediate()
? EmitConstantShift64(state, spv::OpShiftRightLogical, ctx.Arg(inst, 0),
inst.Arg(1).Resolve().U32()) inst.Arg(1).Resolve().U32())
: EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), : EmitShift64(state, spv::OpShiftRightLogical, ctx.Arg(inst, 0),
ctx.Arg(inst, 1))); ctx.Arg(inst, 1)));
return true; return true;
case IR::ValueOpcode::ShiftRightArithmetic64: case IR::ValueOpcode::ShiftRightArithmetic64:
ctx.Define(inst, ctx.Define(inst,
inst.Arg(1).Resolve().IsImmediate() inst.Arg(1).Resolve().IsImmediate()
? EmitConstantShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0), ? EmitConstantShift64(state, spv::OpShiftRightArithmetic,
inst.Arg(1).Resolve().U32()) ctx.Arg(inst, 0), inst.Arg(1).Resolve().U32())
: EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0), : EmitShift64(state, spv::OpShiftRightArithmetic, ctx.Arg(inst, 0),
ctx.Arg(inst, 1))); ctx.Arg(inst, 1)));
return true; return true;
case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32); case IR::ValueOpcode::BitwiseAnd32: return binary(spv::OpBitwiseAnd, IR::Type::U32);
case IR::ValueOpcode::BitwiseOr32: return binary(OpBitwiseOr, IR::Type::U32); case IR::ValueOpcode::BitwiseOr32: return binary(spv::OpBitwiseOr, IR::Type::U32);
case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32); case IR::ValueOpcode::BitwiseXor32: return binary(spv::OpBitwiseXor, IR::Type::U32);
case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32); case IR::ValueOpcode::BitwiseNot32: return unary(spv::OpNot, IR::Type::U32);
case IR::ValueOpcode::BitwiseAnd64: case IR::ValueOpcode::BitwiseAnd64:
ctx.Define(inst, Binary(state, OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0), ctx.Define(inst, Binary(state, spv::OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0),
ctx.Arg(inst, 1))); ctx.Arg(inst, 1)));
return true; return true;
case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32); case IR::ValueOpcode::BitReverse32: return unary(spv::OpBitReverse, IR::Type::U32);
case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32); case IR::ValueOpcode::BitCount32: return unary(spv::OpBitCount, IR::Type::U32);
case IR::ValueOpcode::BitCount64: { case IR::ValueOpcode::BitCount64: {
const auto pair = const auto pair =
ExtractPair(state, Unary(state, OpBitCount, TypeU64(state), ctx.Arg(inst, 0))); ExtractPair(state, Unary(state, spv::OpBitCount, TypeU64(state), ctx.Arg(inst, 0)));
ctx.Define(inst, Binary(state, OpIAdd, TypeU32(state), pair.low, pair.high)); ctx.Define(inst, Binary(state, spv::OpIAdd, TypeU32(state), pair.low, pair.high));
return true; return true;
} }
case IR::ValueOpcode::FindILsb32: { case IR::ValueOpcode::FindILsb32: {
const auto value = EmitExt(state, TypeI32(state), GlslFindILsb, {ctx.Arg(inst, 0)}); const auto value =
ctx.Define(inst, Unary(state, OpBitcast, TypeU32(state), value)); EmitExt(state, TypeI32(state), GLSLstd450FindILsb, {ctx.Arg(inst, 0)});
ctx.Define(inst, Unary(state, spv::OpBitcast, TypeU32(state), value));
return true; return true;
} }
case IR::ValueOpcode::FindUMsb32: { case IR::ValueOpcode::FindUMsb32: {
const auto value = EmitExt(state, TypeI32(state), GlslFindUMsb, {ctx.Arg(inst, 0)}); const auto value =
ctx.Define(inst, Unary(state, OpBitcast, TypeU32(state), value)); EmitExt(state, TypeI32(state), GLSLstd450FindUMsb, {ctx.Arg(inst, 0)});
ctx.Define(inst, Unary(state, spv::OpBitcast, TypeU32(state), value));
return true; return true;
} }
case IR::ValueOpcode::FindUMsb64: case IR::ValueOpcode::FindUMsb64:
@@ -529,16 +539,18 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
ctx.Define( ctx.Define(
inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false)); inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false));
return true; return true;
case IR::ValueOpcode::SLessThan32: return binary(OpSLessThan, IR::Type::U1); case IR::ValueOpcode::SLessThan32: return binary(spv::OpSLessThan, IR::Type::U1);
case IR::ValueOpcode::ULessThan32: return binary(OpULessThan, IR::Type::U1); case IR::ValueOpcode::ULessThan32: return binary(spv::OpULessThan, IR::Type::U1);
case IR::ValueOpcode::IEqual32: return binary(OpIEqual, IR::Type::U1); case IR::ValueOpcode::IEqual32: return binary(spv::OpIEqual, IR::Type::U1);
case IR::ValueOpcode::SLessThanEqual32: return binary(OpSLessThanEqual, IR::Type::U1); case IR::ValueOpcode::SLessThanEqual32: return binary(spv::OpSLessThanEqual, IR::Type::U1);
case IR::ValueOpcode::ULessThanEqual32: return binary(OpULessThanEqual, IR::Type::U1); case IR::ValueOpcode::ULessThanEqual32: return binary(spv::OpULessThanEqual, IR::Type::U1);
case IR::ValueOpcode::SGreaterThan32: return binary(OpSGreaterThan, IR::Type::U1); case IR::ValueOpcode::SGreaterThan32: return binary(spv::OpSGreaterThan, IR::Type::U1);
case IR::ValueOpcode::UGreaterThan32: return binary(OpUGreaterThan, IR::Type::U1); case IR::ValueOpcode::UGreaterThan32: return binary(spv::OpUGreaterThan, IR::Type::U1);
case IR::ValueOpcode::INotEqual32: return binary(OpINotEqual, IR::Type::U1); case IR::ValueOpcode::INotEqual32: return binary(spv::OpINotEqual, IR::Type::U1);
case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1); case IR::ValueOpcode::SGreaterThanEqual32:
case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1); return binary(spv::OpSGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::UGreaterThanEqual32:
return binary(spv::OpUGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::IEqual64: case IR::ValueOpcode::IEqual64:
ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false)); ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
return true; return true;
@@ -547,48 +559,50 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
return true; return true;
case IR::ValueOpcode::ULessThan64: case IR::ValueOpcode::ULessThan64:
ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
OpULessThan, OpULessThan)); spv::OpULessThan, spv::OpULessThan));
return true; return true;
case IR::ValueOpcode::SLessThan64: case IR::ValueOpcode::SLessThan64:
ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
OpSLessThan, OpULessThan)); spv::OpSLessThan, spv::OpULessThan));
return true; return true;
case IR::ValueOpcode::UGreaterThan64: case IR::ValueOpcode::UGreaterThan64:
ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
OpUGreaterThan, OpUGreaterThan)); spv::OpUGreaterThan, spv::OpUGreaterThan));
return true; return true;
case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1); case IR::ValueOpcode::LogicalOr: return binary(spv::OpLogicalOr, IR::Type::U1);
case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1); case IR::ValueOpcode::LogicalAnd: return binary(spv::OpLogicalAnd, IR::Type::U1);
case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1); case IR::ValueOpcode::LogicalXor: return binary(spv::OpLogicalNotEqual, IR::Type::U1);
case IR::ValueOpcode::LogicalNot: return unary(OpLogicalNot, IR::Type::U1); case IR::ValueOpcode::LogicalNot: return unary(spv::OpLogicalNot, IR::Type::U1);
case IR::ValueOpcode::FPOrdEqual32: return binary(OpFOrdEqual, IR::Type::U1); case IR::ValueOpcode::FPOrdEqual32: return binary(spv::OpFOrdEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordEqual32: return binary(OpFUnordEqual, IR::Type::U1); case IR::ValueOpcode::FPUnordEqual32: return binary(spv::OpFUnordEqual, IR::Type::U1);
case IR::ValueOpcode::FPOrdNotEqual32: return binary(OpFOrdNotEqual, IR::Type::U1); case IR::ValueOpcode::FPOrdNotEqual32: return binary(spv::OpFOrdNotEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordNotEqual32: return binary(OpFUnordNotEqual, IR::Type::U1); case IR::ValueOpcode::FPUnordNotEqual32: return binary(spv::OpFUnordNotEqual, IR::Type::U1);
case IR::ValueOpcode::FPOrdLessThan32: return binary(OpFOrdLessThan, IR::Type::U1); case IR::ValueOpcode::FPOrdLessThan32: return binary(spv::OpFOrdLessThan, IR::Type::U1);
case IR::ValueOpcode::FPUnordLessThan32: return binary(OpFUnordLessThan, IR::Type::U1); case IR::ValueOpcode::FPUnordLessThan32: return binary(spv::OpFUnordLessThan, IR::Type::U1);
case IR::ValueOpcode::FPOrdGreaterThan32: return binary(OpFOrdGreaterThan, IR::Type::U1); case IR::ValueOpcode::FPOrdGreaterThan32:
return binary(spv::OpFOrdGreaterThan, IR::Type::U1);
case IR::ValueOpcode::FPUnordGreaterThan32: case IR::ValueOpcode::FPUnordGreaterThan32:
return binary(OpFUnordGreaterThan, IR::Type::U1); return binary(spv::OpFUnordGreaterThan, IR::Type::U1);
case IR::ValueOpcode::FPOrdLessThanEqual32: case IR::ValueOpcode::FPOrdLessThanEqual32:
return binary(OpFOrdLessThanEqual, IR::Type::U1); return binary(spv::OpFOrdLessThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordLessThanEqual32: case IR::ValueOpcode::FPUnordLessThanEqual32:
return binary(OpFUnordLessThanEqual, IR::Type::U1); return binary(spv::OpFUnordLessThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPOrdGreaterThanEqual32: case IR::ValueOpcode::FPOrdGreaterThanEqual32:
return binary(OpFOrdGreaterThanEqual, IR::Type::U1); return binary(spv::OpFOrdGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPUnordGreaterThanEqual32: case IR::ValueOpcode::FPUnordGreaterThanEqual32:
return binary(OpFUnordGreaterThanEqual, IR::Type::U1); return binary(spv::OpFUnordGreaterThanEqual, IR::Type::U1);
case IR::ValueOpcode::FPIsNan32: case IR::ValueOpcode::FPIsNan32:
ctx.Emit(inst, OpFUnordNotEqual, IR::Type::U1, {ctx.Arg(inst, 0), ctx.Arg(inst, 0)}); ctx.Emit(inst, spv::OpFUnordNotEqual, IR::Type::U1,
{ctx.Arg(inst, 0), ctx.Arg(inst, 0)});
return true; return true;
case IR::ValueOpcode::FPCmpClass32: case IR::ValueOpcode::FPCmpClass32:
ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1))); ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
return true; return true;
case IR::ValueOpcode::FPAdd32: return binary(OpFAdd, IR::Type::F32); case IR::ValueOpcode::FPAdd32: return binary(spv::OpFAdd, IR::Type::F32);
case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32); case IR::ValueOpcode::FPSub32: return binary(spv::OpFSub, IR::Type::F32);
case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32); case IR::ValueOpcode::FPMul32: return binary(spv::OpFMul, IR::Type::F32);
case IR::ValueOpcode::FPFma32: case IR::ValueOpcode::FPFma32:
ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFma, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450Fma,
{ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)})); {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
return true; return true;
case IR::ValueOpcode::FPMin32: case IR::ValueOpcode::FPMin32:
@@ -611,51 +625,52 @@ bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
return true; return true;
case IR::ValueOpcode::FPRecip32: { case IR::ValueOpcode::FPRecip32: {
const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0)); const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
ctx.Define(inst, Binary(state, OpFDiv, TypeF32(state), ctx.Define(inst, Binary(state, spv::OpFDiv, TypeF32(state),
ConstantF32(state, 0x3f800000u), source)); ConstantF32(state, 0x3f800000u), source));
return true; return true;
} }
case IR::ValueOpcode::FPRecipIFlag32: case IR::ValueOpcode::FPRecipIFlag32:
// Integer-to-float inputs used by IFLAG cannot be denormal. // Integer-to-float inputs used by IFLAG cannot be denormal.
ctx.Define(inst, Binary(state, OpFDiv, TypeF32(state), ctx.Define(inst, Binary(state, spv::OpFDiv, TypeF32(state),
ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0))); ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
return true; return true;
case IR::ValueOpcode::FPRecipSqrt32: case IR::ValueOpcode::FPRecipSqrt32:
ctx.Define(inst, EmitExt(state, TypeF32(state), GlslInverseSqrt, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450InverseSqrt,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true; return true;
case IR::ValueOpcode::FPSqrt: case IR::ValueOpcode::FPSqrt:
ctx.Define(inst, EmitExt(state, TypeF32(state), GlslSqrt, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450Sqrt,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true; return true;
case IR::ValueOpcode::FPSin: case IR::ValueOpcode::FPSin:
case IR::ValueOpcode::FPCos: { case IR::ValueOpcode::FPCos: {
auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin); auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
source = source =
Binary(state, OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu)); Binary(state, spv::OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
ctx.Define(inst, EmitExt(state, TypeF32(state), ctx.Define(inst, EmitExt(state, TypeF32(state),
op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source})); op == IR::ValueOpcode::FPSin ? GLSLstd450Sin : GLSLstd450Cos,
{source}));
return true; return true;
} }
case IR::ValueOpcode::FPExp2: case IR::ValueOpcode::FPExp2:
ctx.Define(inst, EmitExt(state, TypeF32(state), GlslExp2, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450Exp2,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true; return true;
case IR::ValueOpcode::FPLog2: case IR::ValueOpcode::FPLog2:
ctx.Define(inst, EmitExt(state, TypeF32(state), GlslLog2, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450Log2,
{EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))})); {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
return true; return true;
case IR::ValueOpcode::FPLdexp: { case IR::ValueOpcode::FPLdexp: {
const auto exponent = Unary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 1)); const auto exponent = Unary(state, spv::OpBitcast, TypeI32(state), ctx.Arg(inst, 1));
ctx.Define(inst, ctx.Define(inst, EmitExt(state, TypeF32(state), GLSLstd450Ldexp,
EmitExt(state, TypeF32(state), GlslLdexp, {ctx.Arg(inst, 0), exponent})); {ctx.Arg(inst, 0), exponent}));
return true; return true;
} }
case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven); case IR::ValueOpcode::FPRoundEven32: return ext_unary(GLSLstd450RoundEven);
case IR::ValueOpcode::FPFloor32: return ext_unary(GlslFloor); case IR::ValueOpcode::FPFloor32: return ext_unary(GLSLstd450Floor);
case IR::ValueOpcode::FPCeil32: return ext_unary(GlslCeil); case IR::ValueOpcode::FPCeil32: return ext_unary(GLSLstd450Ceil);
case IR::ValueOpcode::FPTrunc32: return ext_unary(GlslTrunc); case IR::ValueOpcode::FPTrunc32: return ext_unary(GLSLstd450Trunc);
case IR::ValueOpcode::FPFract32: return ext_unary(GlslFract); case IR::ValueOpcode::FPFract32: return ext_unary(GLSLstd450Fract);
default: return false; default: return false;
} }
} }
@@ -5,7 +5,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value, uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
uint32_t false_value) { uint32_t false_value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value}); state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, true_value, false_value});
return ret; return ret;
} }
@@ -26,20 +26,23 @@ void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t h
const auto high_less = state.builder.AllocateId(); const auto high_less = state.builder.AllocateId();
const auto high_ge = state.builder.AllocateId(); const auto high_ge = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)}); {spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction(
{OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift}); {spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)}); {spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
state.builder.AddFunction( state.builder.AddFunction(
{OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)}); {spv::OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry}); state.builder.AddFunction(
state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift}); {spv::OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
state.builder.AddFunction({spv::OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0)); out_low = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
out_high = EmitSelectValueU32(state, less32, high_less, high_ge); out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
} }
@@ -59,40 +62,45 @@ void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t
const auto low_less = state.builder.AllocateId(); const auto low_less = state.builder.AllocateId();
const auto low_ge = state.builder.AllocateId(); const auto low_ge = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)}); {spv::OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction(
{OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift}); {spv::OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)}); {spv::OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
state.builder.AddFunction( state.builder.AddFunction(
{OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)}); {spv::OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry}); state.builder.AddFunction(
state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift}); {spv::OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
state.builder.AddFunction(
{spv::OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
state.builder.AddFunction(
{spv::OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
out_low = EmitSelectValueU32(state, less32, low_less, low_ge); out_low = EmitSelectValueU32(state, less32, low_less, low_ge);
out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0)); out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
} }
uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) { uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)}); state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
return ret; return ret;
} }
uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) { uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)}); {spv::OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
return ret; return ret;
} }
uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs}); state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
return ret; return ret;
} }
@@ -105,13 +113,14 @@ uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t v
uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) { uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value}); state.builder.AddFunction(
{spv::OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
return ret; return ret;
} }
uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) { uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, f32});
const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u); const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
const auto exp = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu); const auto exp = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
@@ -122,17 +131,18 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
const auto normal_mant = EmitShiftRightConstant(state, mant, 13); const auto normal_mant = EmitShiftRightConstant(state, mant, 13);
const auto normal_payload = state.builder.AllocateId(); const auto normal_payload = state.builder.AllocateId();
const auto normal = state.builder.AllocateId(); const auto normal = state.builder.AllocateId();
state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)}); {spv::OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant}); {spv::OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload}); state.builder.AddFunction(
{spv::OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u)); const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
const auto raw_sub_shift = EmitSubConstantMinusU32(state, 126, exp); const auto raw_sub_shift = EmitSubConstantMinusU32(state, 126, exp);
const auto exp_lt_103 = EmitCompareU32Constant(state, OpULessThan, exp, 103); const auto exp_lt_103 = EmitCompareU32Constant(state, spv::OpULessThan, exp, 103);
const auto exp_gt_112 = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112); const auto exp_gt_112 = EmitCompareU32Constant(state, spv::OpUGreaterThan, exp, 112);
const auto sub_shift_low = const auto sub_shift_low =
EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift); EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
const auto sub_shift = const auto sub_shift =
@@ -140,8 +150,8 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
const auto sub_mant = state.builder.AllocateId(); const auto sub_mant = state.builder.AllocateId();
const auto subnormal = state.builder.AllocateId(); const auto subnormal = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift}); {spv::OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant}); state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
const auto nan_payload = const auto nan_payload =
EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u)); EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
@@ -149,12 +159,12 @@ uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload)); EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
const auto inf = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u)); const auto inf = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu)); const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
const auto mant_zero = EmitCompareU32Constant(state, OpIEqual, mant, 0); const auto mant_zero = EmitCompareU32Constant(state, spv::OpIEqual, mant, 0);
const auto special = EmitSelectValueU32(state, mant_zero, inf, nan); const auto special = EmitSelectValueU32(state, mant_zero, inf, nan);
const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112); const auto exp_le_112 = EmitCompareU32Constant(state, spv::OpULessThanEqual, exp, 112);
const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143); const auto exp_ge_143 = EmitCompareU32Constant(state, spv::OpUGreaterThanEqual, exp, 143);
const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255); const auto exp_eq_255 = EmitCompareU32Constant(state, spv::OpIEqual, exp, 255);
const auto finite0 = EmitSelectValueU32(state, exp_le_112, subnormal, normal); const auto finite0 = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
const auto finite1 = EmitSelectValueU32(state, exp_lt_103, sign, finite0); const auto finite1 = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
const auto finite2 = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1); const auto finite2 = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
@@ -165,8 +175,8 @@ uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
const auto cond = state.builder.AllocateId(); const auto cond = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs}); {max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state), cond, lhs, rhs});
state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs}); state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs});
return ret; return ret;
} }
@@ -174,44 +184,44 @@ uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
const auto cond = state.builder.AllocateId(); const auto cond = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs}); {max_value ? spv::OpSGreaterThan : spv::OpSLessThan, TypeBool(state), cond, lhs, rhs});
state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs}); state.builder.AddFunction({spv::OpSelect, TypeU32(state), ret, cond, lhs, rhs});
return ret; return ret;
} }
uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) { uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), ret, value});
return ret; return ret;
} }
uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) { uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value}); state.builder.AddFunction({spv::OpBitcast, TypeF32(state), ret, value});
return ret; return ret;
} }
uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs}); state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
return ret; return ret;
} }
uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs}); state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
return ret; return ret;
} }
uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs}); state.builder.AddFunction({spv::OpLogicalOr, TypeBool(state), ret, lhs, rhs});
return ret; return ret;
} }
uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) { uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value}); state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), ret, value});
return ret; return ret;
} }
@@ -221,10 +231,11 @@ F32Class EmitClassifyF32Bits(EmitterState& state, uint32_t bits) {
const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu); const auto abs_bits = EmitAndConstant(state, cls.bits, 0x7fffffffu);
const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u); const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
const auto exponent_max = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); const auto exponent_max =
const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0); EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
const auto mantissa_nonzero = EmitCompareU32Constant(state, spv::OpINotEqual, mantissa_bits, 0);
cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero); cls.nan = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); cls.zero = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
return cls; return cls;
} }
@@ -235,7 +246,7 @@ F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit, uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
uint32_t class_match) { uint32_t class_match) {
const auto selected = const auto selected =
EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0); EmitCompareU32Constant(state, spv::OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
return EmitLogicalAndBool(state, selected, class_match); return EmitLogicalAndBool(state, selected, class_match);
} }
@@ -247,15 +258,16 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu); const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
const auto quiet_bits = EmitAndConstant(state, mantissa_bits, 0x00400000u); const auto quiet_bits = EmitAndConstant(state, mantissa_bits, 0x00400000u);
const auto sign = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0); const auto sign = EmitCompareU32Constant(state, spv::OpINotEqual, sign_bits, 0);
const auto positive = EmitLogicalNotBool(state, sign); const auto positive = EmitLogicalNotBool(state, sign);
const auto exponent_zero = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0); const auto exponent_zero = EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0);
const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero); const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u); const auto exponent_inf =
EmitCompareU32Constant(state, spv::OpIEqual, exponent_bits, 0x7f800000u);
const auto finite_exponent = EmitLogicalNotBool(state, exponent_inf); const auto finite_exponent = EmitLogicalNotBool(state, exponent_inf);
const auto mantissa_zero = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0); const auto mantissa_zero = EmitCompareU32Constant(state, spv::OpIEqual, mantissa_bits, 0);
const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero); const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
const auto quiet = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0); const auto quiet = EmitCompareU32Constant(state, spv::OpINotEqual, quiet_bits, 0);
const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero); const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
const auto snan = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet)); const auto snan = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
@@ -263,7 +275,7 @@ uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
const auto inf = EmitLogicalAndBool(state, exponent_inf, mantissa_zero); const auto inf = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
const auto normal = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent); const auto normal = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
const auto denorm = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero); const auto denorm = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
const auto zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0); const auto zero = EmitCompareU32Constant(state, spv::OpIEqual, abs_bits, 0);
uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan); uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
match = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan)); match = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
@@ -296,8 +308,8 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
const auto rhs_class = EmitClassifyF32(state, rhs); const auto rhs_class = EmitClassifyF32(state, rhs);
const auto numeric_cond = state.builder.AllocateId(); const auto numeric_cond = state.builder.AllocateId();
state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state), state.builder.AddFunction({max_value ? spv::OpFOrdGreaterThanEqual : spv::OpFOrdLessThan,
numeric_cond, lhs, rhs}); TypeBool(state), numeric_cond, lhs, rhs});
const auto ordered_bits = const auto ordered_bits =
EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits); EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
@@ -315,7 +327,7 @@ uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, boo
uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) { uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value}); {spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450Trunc, value});
return ret; return ret;
} }
@@ -328,18 +340,18 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
const auto flush = state.builder.AllocateId(); const auto flush = state.builder.AllocateId();
const auto selected = state.builder.AllocateId(); const auto selected = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); {spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)}); {spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction(
{OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)}); {spv::OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal}); state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits}); state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected}); state.builder.AddFunction({spv::OpBitcast, TypeF32(state), ret, selected});
return ret; return ret;
} }
@@ -352,45 +364,47 @@ uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signe
const auto large_finite = state.builder.AllocateId(); const auto large_finite = state.builder.AllocateId();
const auto reduced = state.builder.AllocateId(); const auto reduced = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src}); {spv::OpExtInst, TypeF32(state), fract, GlslStd450(state), GLSLstd450Fract, src});
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, src});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)}); {spv::OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
state.builder.AddFunction({spv::OpUGreaterThanEqual, TypeBool(state), large, abs_bits,
ConstantU32(state, 0x4b000000u)});
state.builder.AddFunction( state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)}); {spv::OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), large_finite, large, finite});
state.builder.AddFunction( state.builder.AddFunction(
{OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)}); {spv::OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
state.builder.AddFunction(
{OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
if (!preserve_signed_zero) { if (!preserve_signed_zero) {
return reduced; return reduced;
} }
const auto zero = state.builder.AllocateId(); const auto zero = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)}); state.builder.AddFunction(
state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced}); {spv::OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpSelect, TypeF32(state), ret, zero, src, reduced});
return ret; return ret;
} }
uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) { uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value}); state.builder.AddFunction({spv::OpFNegate, TypeF32(state), ret, value});
return ret; return ret;
} }
uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) { uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value}); state.builder.AddFunction(
{spv::OpExtInst, TypeF32(state), ret, GlslStd450(state), GLSLstd450FAbs, value});
return ret; return ret;
} }
uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) { uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
const auto unpacked = state.builder.AllocateId(); const auto unpacked = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), state.builder.AddFunction({spv::OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
GlslUnpackHalf2x16, bits}); GLSLstd450UnpackHalf2x16, bits});
state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
return ret; return ret;
} }
@@ -14,38 +14,28 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
active_inputs[active_count++] = input.location; active_inputs[active_count++] = input.location;
} }
} }
return state.stage == ShaderType::Pixel return state.program.stage == ShaderType::Pixel
? ShaderPixelParameterLocation(*state.input_info.pixel, ? ShaderPixelParameterLocation(*state.input_info.pixel,
{active_inputs.data(), active_count}, attr) {active_inputs.data(), active_count}, attr)
: attr; : attr;
} }
bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) { bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
return state.stage == ShaderType::Pixel && return state.program.stage == ShaderType::Pixel &&
ShaderPixelParameterIsFlat(*state.input_info.pixel, attr); ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
} }
bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) { bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
return state.stage == ShaderType::Pixel && return state.program.stage == ShaderType::Pixel &&
ShaderPixelParameterIsCustom(*state.input_info.pixel, attr); ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
} }
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
uint32_t index) {
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
return binding.kind == kind && binding.index == index;
});
}
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) { void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
for (const auto& input: program.info.inputs) { for (const auto& input: program.info.inputs) {
state.inputs.push_back({input.kind, input.location, input.component_count, 0, state.inputs.push_back({input.kind, input.location, input.component_count, 0,
input.debug_name, input.per_vertex}); input.debug_name, input.per_vertex});
} }
for (const auto& output: program.info.outputs) { for (const auto& output: program.info.outputs) {
if (HasOutput(state.outputs, output.kind, output.index)) {
continue;
}
state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name}); state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name});
} }
} }
@@ -74,7 +64,7 @@ uint32_t ConstantU32(EmitterState& state, uint32_t value);
const char* reason) { const char* reason) {
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64 EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n", " stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
state.program.shader_hash, static_cast<unsigned>(state.stage), resource, state.program.shader_hash, static_cast<unsigned>(state.program.stage), resource,
static_cast<unsigned>(kind), reason); static_cast<unsigned>(kind), reason);
std::abort(); std::abort();
} }
@@ -102,8 +92,8 @@ uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
ExitDescriptorBindingFailure(state, kind, resource, variable_name); ExitDescriptorBindingFailure(state, kind, resource, variable_name);
} }
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpAccessChain, result_ptr_type, pointer, variable_id,
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)}); ConstantU32(state, array_index)});
return pointer; return pointer;
} }
@@ -129,12 +119,13 @@ uint32_t ImageScalarType(EmitterState& state, Prospero::TextureNumericClass nume
uint32_t ImageVectorType(EmitterState& state, Prospero::TextureNumericClass numeric_class, uint32_t ImageVectorType(EmitterState& state, Prospero::TextureNumericClass numeric_class,
uint32_t components) { uint32_t components) {
return state.builder.Type(OpTypeVector, {ImageScalarType(state, numeric_class), components}); return state.builder.Type(spv::OpTypeVector,
{ImageScalarType(state, numeric_class), components});
} }
uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) { uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
uint32_t sampled = 0; uint32_t sampled = 0;
uint32_t format = ImageFormatUnknown; uint32_t format = spv::ImageFormatUnknown;
if (image.resource_class == IR::ImageResourceClass::Sampled) { if (image.resource_class == IR::ImageResourceClass::Sampled) {
EXIT_IF(image.atomic); EXIT_IF(image.atomic);
sampled = 1; sampled = 1;
@@ -144,16 +135,16 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
sampled = 2; sampled = 2;
if (image.atomic) { if (image.atomic) {
EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint); EXIT_IF(image.numeric_class != Prospero::TextureNumericClass::Uint);
format = ImageFormatR32ui; format = spv::ImageFormatR32ui;
} }
} else { } else {
EXIT("invalid image resource class"); EXIT("invalid image resource class");
} }
const auto& info = ImageDimensionInfoFor(image.dimension); const auto& info = ImageDimensionInfoFor(image.dimension);
return state.builder.Type(OpTypeImage, return state.builder.Type(spv::OpTypeImage,
{ImageScalarType(state, image.numeric_class), info.spirv_dimension, {ImageScalarType(state, image.numeric_class), info.spirv_dimension,
image.depth_compare ? 1u : 0u, image.depth_compare ? 1u : 0u, info.arrayed, info.multisampled,
info.arrayed, info.multisampled, sampled, format}); sampled, format});
} }
uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) { uint32_t ImageViewSizeType(EmitterState& state, ImageDimension dimension) {
@@ -173,26 +164,26 @@ uint32_t LoadSampledImageDescriptor(EmitterState& state, uint32_t resource) {
const auto array_index = ResourceForDescriptor(state, *kind, resource); const auto array_index = ResourceForDescriptor(state, *kind, resource);
const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)]; const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
const auto pointer_type = state.builder.Type( const auto pointer_type = state.builder.Type(
OpTypePointer, {StorageClassUniformConstant, ImageType(state, image_resource)}); spv::OpTypePointer, {spv::StorageClassUniformConstant, ImageType(state, image_resource)});
const auto pointer = const auto pointer =
DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource, DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
"sampled image descriptor array was not emitted"); "sampled image descriptor array was not emitted");
const auto image = state.builder.AllocateId(); const auto image = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, ImageType(state, image_resource), image, pointer}); state.builder.AddFunction({spv::OpLoad, ImageType(state, image_resource), image, pointer});
return image; return image;
} }
uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) { uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler) {
const auto array_index = const auto array_index =
ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler); ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
const auto sampler_type = state.builder.Type(OpTypeSampler); const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
const auto pointer_type = const auto pointer_type =
state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type}); state.builder.Type(spv::OpTypePointer, {spv::StorageClassUniformConstant, sampler_type});
const auto pointer = DescriptorElementPointer( const auto pointer = DescriptorElementPointer(
state, pointer_type, state.sampler_variable, array_index, state, pointer_type, state.sampler_variable, array_index,
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
const auto sampler_id = state.builder.AllocateId(); const auto sampler_id = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer}); state.builder.AddFunction({spv::OpLoad, sampler_type, sampler_id, pointer});
return sampler_id; return sampler_id;
} }
@@ -202,8 +193,9 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
const auto sampler_id = LoadSamplerDescriptor(state, sampler); const auto sampler_id = LoadSamplerDescriptor(state, sampler);
const auto sampled_image = state.builder.AllocateId(); const auto sampled_image = state.builder.AllocateId();
const auto sampled_type = const auto sampled_type =
state.builder.Type(OpTypeSampledImage, {ImageType(state, image_resource)}); state.builder.Type(spv::OpTypeSampledImage, {ImageType(state, image_resource)});
state.builder.AddFunction({OpSampledImage, sampled_type, sampled_image, image, sampler_id}); state.builder.AddFunction(
{spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id});
return sampled_image; return sampled_image;
} }
@@ -213,8 +205,8 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource) {
const auto kind = IR::DescriptorBindingForImage(image); const auto kind = IR::DescriptorBindingForImage(image);
EXIT_IF(!kind.has_value()); EXIT_IF(!kind.has_value());
const auto array_index = ResourceForDescriptor(state, *kind, resource); const auto array_index = ResourceForDescriptor(state, *kind, resource);
const auto pointer_type = const auto pointer_type = state.builder.Type(
state.builder.Type(OpTypePointer, {StorageClassUniformConstant, ImageType(state, image)}); spv::OpTypePointer, {spv::StorageClassUniformConstant, ImageType(state, image)});
const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)]; const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource, return DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
"storage image descriptor array was not emitted"); "storage image descriptor array was not emitted");
@@ -225,25 +217,25 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
const auto& image = state.program.info.images.at(resource); const auto& image = state.program.info.images.at(resource);
EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage); EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
if (!image.atomic) { if (!image.atomic) {
state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat); state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
} }
const auto kind = IR::DescriptorBindingForImage(image); const auto kind = IR::DescriptorBindingForImage(image);
EXIT_IF(!kind.has_value()); EXIT_IF(!kind.has_value());
const auto array_index = ResourceForDescriptor(state, *kind, resource); const auto array_index = ResourceForDescriptor(state, *kind, resource);
const auto image_type = ImageType(state, image); const auto image_type = ImageType(state, image);
const auto pointer_type = const auto pointer_type =
state.builder.Type(OpTypePointer, {StorageClassUniformConstant, image_type}); state.builder.Type(spv::OpTypePointer, {spv::StorageClassUniformConstant, image_type});
const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)]; const auto variable = state.image_variables[IR::ImageBindingIndex(*kind)];
const auto LoadAt = [&](uint32_t array_index) { const auto LoadAt = [&](uint32_t array_index) {
const auto pointer = const auto pointer =
DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource, DescriptorElementPointer(state, pointer_type, variable, array_index, *kind, resource,
"storage image descriptor array was not emitted"); "storage image descriptor array was not emitted");
const auto descriptor = state.builder.AllocateId(); const auto descriptor = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, image_type, descriptor, pointer}); state.builder.AddFunction({spv::OpLoad, image_type, descriptor, pointer});
return descriptor; return descriptor;
}; };
if (image.mip_mode != IR::ImageMipMode::DynamicStorage) { if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
state.builder.AddFunction({OpImageWrite, LoadAt(array_index), coord, texel}); state.builder.AddFunction({spv::OpImageWrite, LoadAt(array_index), coord, texel});
return; return;
} }
if (image.mip_count == 0u) { if (image.mip_count == 0u) {
@@ -253,28 +245,28 @@ void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
std::vector<uint32_t> labels(image.mip_count); std::vector<uint32_t> labels(image.mip_count);
std::vector<uint32_t> words {OpSwitch, mip_lod, merge_label}; std::vector<uint32_t> words {spv::OpSwitch, mip_lod, merge_label};
for (uint32_t mip = 0; mip < image.mip_count; mip++) { for (uint32_t mip = 0; mip < image.mip_count; mip++) {
labels[mip] = state.builder.AllocateId(); labels[mip] = state.builder.AllocateId();
words.push_back(mip); words.push_back(mip);
words.push_back(labels[mip]); words.push_back(labels[mip]);
} }
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
state.builder.AddFunction(words); state.builder.AddFunction(words);
for (uint32_t mip = 0; mip < image.mip_count; mip++) { for (uint32_t mip = 0; mip < image.mip_count; mip++) {
EmitLabel(state, labels[mip]); EmitLabel(state, labels[mip]);
state.builder.AddFunction({OpImageWrite, LoadAt(array_index + mip), coord, texel}); state.builder.AddFunction({spv::OpImageWrite, LoadAt(array_index + mip), coord, texel});
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
} }
EmitLabel(state, merge_label); EmitLabel(state, merge_label);
} }
uint32_t ExecutionModelForStage(ShaderType stage) { spv::ExecutionModel ExecutionModelForStage(ShaderType stage) {
switch (stage) { switch (stage) {
case ShaderType::Vertex: return ExecutionModelVertex; case ShaderType::Vertex: return spv::ExecutionModelVertex;
case ShaderType::Mesh: return 5365u; // MeshEXT case ShaderType::Mesh: return spv::ExecutionModelMeshEXT; // MeshEXT
case ShaderType::Pixel: return ExecutionModelFragment; case ShaderType::Pixel: return spv::ExecutionModelFragment;
default: return ExecutionModelGLCompute; default: return spv::ExecutionModelGLCompute;
} }
} }
@@ -23,16 +23,17 @@ uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
const auto merged_two = state.builder.AllocateId(); const auto merged_two = state.builder.AllocateId();
const auto quad_bits = state.builder.AllocateId(); const auto quad_bits = state.builder.AllocateId();
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value, state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_one, value,
ConstantU64(state, 0x0000000100000001ull)}); ConstantU64(state, 0x0000000100000001ull)});
state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one}); state.builder.AddFunction({spv::OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one, state.builder.AddFunction({spv::OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
ConstantU64(state, 0x0000000200000002ull)}); ConstantU64(state, 0x0000000200000002ull)});
state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull)});
state.builder.AddFunction( state.builder.AddFunction(
{OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)}); {spv::OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
ConstantU64(state, 0x1111111111111111ull)});
state.builder.AddFunction({spv::OpIMul, TypeU64(state), result, quad_bits,
ConstantU64(state, 0x0000000f0000000full)});
return result; return result;
} }
@@ -43,21 +44,21 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
} }
if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId || if (state.lane_count == 2 && (kind == IR::StageInputKind::LocalInvocationId ||
kind == IR::StageInputKind::GlobalInvocationId)) { kind == IR::StageInputKind::GlobalInvocationId)) {
const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info); const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info);
uint32_t divisor = 1; uint32_t divisor = 1;
for (uint32_t axis = 0; axis < component; axis++) { for (uint32_t axis = 0; axis < component; axis++) {
divisor *= std::max(cs->threads_num[axis], 1u); divisor *= std::max(cs->threads_num[axis], 1u);
} }
const auto size = std::max(cs->threads_num[component], 1u); const auto size = std::max(cs->threads_num[component], 1u);
const auto divided = EmitBinaryU32(state, OpUDiv, EmitLocalInvocationIndex(state), const auto divided = EmitBinaryU32(state, spv::OpUDiv, EmitLocalInvocationIndex(state),
ConstantU32(state, divisor)); ConstantU32(state, divisor));
const auto local = EmitBinaryU32(state, OpUMod, divided, ConstantU32(state, size)); const auto local = EmitBinaryU32(state, spv::OpUMod, divided, ConstantU32(state, size));
if (kind == IR::StageInputKind::LocalInvocationId) { if (kind == IR::StageInputKind::LocalInvocationId) {
return local; return local;
} }
const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component); const auto group = EmitInputComponentU32(state, IR::StageInputKind::WorkgroupId, component);
return EmitAddU32(state, local, return EmitAddU32(state, local,
EmitBinaryU32(state, OpIMul, group, ConstantU32(state, size))); EmitBinaryU32(state, spv::OpIMul, group, ConstantU32(state, size)));
} }
const auto variable = InputVariableForKind(state, kind); const auto variable = InputVariableForKind(state, kind);
if (variable == 0) { if (variable == 0) {
@@ -66,28 +67,28 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
if (kind == IR::StageInputKind::FrontFacing) { if (kind == IR::StageInputKind::FrontFacing) {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeBool(state), value, variable}); state.builder.AddFunction({spv::OpLoad, TypeBool(state), value, variable});
state.builder.AddFunction( state.builder.AddFunction({spv::OpSelect, TypeU32(state), bits, value,
{OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)}); ConstantU32(state, 1), ConstantU32(state, 0)});
return bits; return bits;
} }
if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex || if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex ||
kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) { kind == IR::StageInputKind::Layer || kind == IR::StageInputKind::SampleId) {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeI32(state), value, variable}); state.builder.AddFunction({spv::OpLoad, TypeI32(state), value, variable});
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
return bits; return bits;
} }
if (kind == IR::StageInputKind::FragCoord) { if (kind == IR::StageInputKind::FragCoord) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.builder.AddFunction({spv::OpAccessChain,
TypePointer(state, StorageClassInput, TypeF32(state)), pointer, TypePointer(state, spv::StorageClassInput, TypeF32(state)),
variable, ConstantU32(state, component)}); pointer, variable, ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer}); state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
return bits; return bits;
} }
if (kind == IR::StageInputKind::BaryCoordSmooth || if (kind == IR::StageInputKind::BaryCoordSmooth ||
@@ -95,11 +96,11 @@ uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.builder.AddFunction({spv::OpAccessChain,
TypePointer(state, StorageClassInput, TypeF32(state)), pointer, TypePointer(state, spv::StorageClassInput, TypeF32(state)),
variable, ConstantU32(state, component + 1u)}); pointer, variable, ConstantU32(state, component + 1u)});
state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer}); state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
return bits; return bits;
} }
return EmitInputComponentU32(state, kind, component); return EmitInputComponentU32(state, kind, component);
@@ -121,35 +122,36 @@ uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& fl
const auto row_ok = state.builder.AllocateId(); const auto row_ok = state.builder.AllocateId();
const auto masks_ok = state.builder.AllocateId(); const auto masks_ok = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)}); {spv::OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)}); {spv::OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)}); {spv::OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)}); {spv::OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank}); {spv::OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row}); {spv::OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), bank_hit,
ConstantU32(state, flags.bank_mask), bank_bit});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit}); {spv::OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit}); {spv::OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
{OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
uint32_t writable = masks_ok; uint32_t writable = masks_ok;
if (!flags.bound_control) { if (!flags.bound_control) {
const auto target = EmitDppTargetLane(state, flags.control); const auto target = EmitDppTargetLane(state, flags.control);
const auto bounded = state.builder.AllocateId(); const auto bounded = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid}); state.builder.AddFunction(
{spv::OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
writable = bounded; writable = bounded;
} }
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable}); state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), result, exec, writable});
return result; return result;
} }
@@ -159,16 +161,17 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
if (input == nullptr || input->variable_id == 0) { if (input == nullptr || input->variable_id == 0) {
return ConstantU32(state, 0); return ConstantU32(state, 0);
} }
if (state.stage == ShaderType::Vertex) { if (state.program.stage == ShaderType::Vertex) {
return EmitVertexParameterComponentU32(state, *input, chan & 3u); return EmitVertexParameterComponentU32(state, *input, chan & 3u);
} }
const auto load_per_vertex = [&](uint32_t vertex) { const auto load_per_vertex = [&](uint32_t vertex) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpAccessChain,
{OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer, TypePointer(state, spv::StorageClassInput, TypeF32(state)),
input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)}); pointer, input->variable_id, ConstantU32(state, vertex),
state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer}); ConstantU32(state, chan & 3u)});
state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
return value; return value;
}; };
if (input->per_vertex) { if (input->per_vertex) {
@@ -181,30 +184,31 @@ uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto weight = state.builder.AllocateId(); const auto weight = state.builder.AllocateId();
const auto product = state.builder.AllocateId(); const auto product = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, state.builder.AddFunction({spv::OpAccessChain,
TypePointer(state, StorageClassInput, TypeF32(state)), TypePointer(state, spv::StorageClassInput, TypeF32(state)),
pointer, barycentric, ConstantU32(state, vertex)}); pointer, barycentric, ConstantU32(state, vertex)});
state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer}); state.builder.AddFunction({spv::OpLoad, TypeF32(state), weight, pointer});
state.builder.AddFunction( state.builder.AddFunction(
{OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight}); {spv::OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
if (vertex == 0u) { if (vertex == 0u) {
sum = product; sum = product;
} else { } else {
const auto next = state.builder.AllocateId(); const auto next = state.builder.AllocateId();
state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product}); state.builder.AddFunction({spv::OpFAdd, TypeF32(state), next, sum, product});
sum = next; sum = next;
} }
} }
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, sum});
return bits; return bits;
} }
const auto vector = state.builder.AllocateId(); const auto vector = state.builder.AllocateId();
const auto component = state.builder.AllocateId(); const auto component = state.builder.AllocateId();
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id}); state.builder.AddFunction({spv::OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u}); state.builder.AddFunction(
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component}); {spv::OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, component});
return bits; return bits;
} }
@@ -218,10 +222,11 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
const auto load_vertex = [&](uint32_t vertex) { const auto load_vertex = [&](uint32_t vertex) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpAccessChain,
{OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer, TypePointer(state, spv::StorageClassInput, TypeF32(state)),
input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)}); pointer, input->variable_id, ConstantU32(state, vertex),
state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer}); ConstantU32(state, chan & 3u)});
state.builder.AddFunction({spv::OpLoad, TypeF32(state), value, pointer});
return value; return value;
}; };
@@ -229,16 +234,16 @@ uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32
uint32_t value = load_vertex(selected_vertex); uint32_t value = load_vertex(selected_vertex);
if (!PixelParameterIsCustom(state, attr) && mode < 2u) { if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
const auto delta = state.builder.AllocateId(); const auto delta = state.builder.AllocateId();
state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)}); state.builder.AddFunction({spv::OpFSub, TypeF32(state), delta, value, load_vertex(0)});
value = delta; value = delta;
} }
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, value});
return bits; return bits;
} }
uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) { uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt || if (state.program.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
exp.index >= std::size(state.input_info.pixel->target_output_mode)) { exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
return 0; return 0;
} }
@@ -248,7 +253,7 @@ uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) { uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
const auto value = ctx.state.builder.AllocateId(); const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeExtract, TypeU32(ctx.state), value, vector, component}); {spv::OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
return value; return value;
} }
@@ -257,7 +262,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
auto& state = ctx.state; auto& state = ctx.state;
if (exp.compr && !uint_output) { if (exp.compr && !uint_output) {
const auto unpack = const auto unpack =
MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16; MrtOutputMode(state, exp) == 5u ? GLSLstd450UnpackUnorm2x16 : GLSLstd450UnpackHalf2x16;
uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0), uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
ConstantF32(state, 0x3f800000u)}; ConstantF32(state, 0x3f800000u)};
for (uint32_t pair = 0; pair < 2u; pair++) { for (uint32_t pair = 0; pair < 2u; pair++) {
@@ -266,21 +271,22 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
} }
const auto packed = state.builder.AllocateId(); const auto packed = state.builder.AllocateId();
const auto unpacked = state.builder.AllocateId(); const auto unpacked = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed}); {spv::OpCompositeExtract, TypeU32(state), packed, data, pair});
state.builder.AddFunction({spv::OpExtInst, TypeF32Vector(state, 2), unpacked,
GlslStd450(state), unpack, packed});
for (uint32_t lane = 0; lane < 2u; lane++) { for (uint32_t lane = 0; lane < 2u; lane++) {
const auto component = pair * 2u + lane; const auto component = pair * 2u + lane;
if (((exp.en >> component) & 1u) != 0u) { if (((exp.en >> component) & 1u) != 0u) {
f32[component] = state.builder.AllocateId(); f32[component] = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane}); {spv::OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
} }
} }
} }
const auto vector = state.builder.AllocateId(); const auto vector = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector,
f32[1], f32[2], f32[3]}); f32[0], f32[1], f32[2], f32[3]});
return vector; return vector;
} }
uint32_t raw[4] = { uint32_t raw[4] = {
@@ -301,7 +307,7 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
continue; continue;
} }
raw[component] = state.builder.AllocateId(); raw[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component], state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(state), raw[component],
packed, ConstantU32(state, lane * 16u), packed, ConstantU32(state, lane * 16u),
ConstantU32(state, 16)}); ConstantU32(state, 16)});
} }
@@ -315,18 +321,18 @@ uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo
} }
if (uint_output) { if (uint_output) {
const auto vector = state.builder.AllocateId(); const auto vector = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0], state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(state, 4), vector,
raw[1], raw[2], raw[3]}); raw[0], raw[1], raw[2], raw[3]});
return vector; return vector;
} }
uint32_t f32[4] {}; uint32_t f32[4] {};
for (uint32_t component = 0; component < 4u; component++) { for (uint32_t component = 0; component < 4u; component++) {
f32[component] = state.builder.AllocateId(); f32[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]}); state.builder.AddFunction({spv::OpBitcast, TypeF32(state), f32[component], raw[component]});
} }
const auto vector = state.builder.AllocateId(); const auto vector = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
{OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]}); f32[1], f32[2], f32[3]});
return vector; return vector;
} }
@@ -342,19 +348,19 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
const auto raw = ExportRawComponent(ctx, data, component); const auto raw = ExportRawComponent(ctx, data, component);
if (output.layer) { if (output.layer) {
const auto layer = state.builder.AllocateId(); const auto layer = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw, state.builder.AddFunction(
ConstantU32(state, 0x7ffu)}); {spv::OpBitwiseAnd, TypeU32(state), layer, raw, ConstantU32(state, 0x7ffu)});
const auto pointer = state.stage == ShaderType::Mesh const auto pointer = state.program.stage == ShaderType::Mesh
? MeshOutputPointer(state, IR::StageOutputKind::Layer) ? MeshOutputPointer(state, IR::StageOutputKind::Layer)
: state.layer_variable; : state.layer_variable;
state.builder.AddFunction({OpStore, pointer, layer}); state.builder.AddFunction({spv::OpStore, pointer, layer});
} }
if (output.viewport) { if (output.viewport) {
// GFX10 MISC.z packs the viewport index in bits 16..19 alongside the layer. // GFX10 MISC.z packs the viewport index in bits 16..19 alongside the layer.
const auto viewport = state.builder.AllocateId(); const auto viewport = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), viewport, raw, state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(state), viewport, raw,
ConstantU32(state, 16), ConstantU32(state, 4)}); ConstantU32(state, 16), ConstantU32(state, 4)});
state.builder.AddFunction({OpStore, state.viewport_index_variable, viewport}); state.builder.AddFunction({spv::OpStore, state.viewport_index_variable, viewport});
} }
continue; continue;
} }
@@ -365,9 +371,9 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
const auto raw = ExportRawComponent(ctx, data, component); const auto raw = ExportRawComponent(ctx, data, component);
const auto f32 = state.builder.AllocateId(); const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw}); state.builder.AddFunction({spv::OpBitcast, TypeF32(state), f32, raw});
if (output.point_size) { if (output.point_size) {
state.builder.AddFunction({OpStore, state.point_size_variable, f32}); state.builder.AddFunction({spv::OpStore, state.point_size_variable, f32});
continue; continue;
} }
auto StoreDistance = [&](uint32_t variable, uint32_t index) { auto StoreDistance = [&](uint32_t variable, uint32_t index) {
@@ -375,10 +381,10 @@ void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::Expor
return; return;
} }
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpAccessChain,
{OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer, TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
variable, ConstantU32(state, index)}); pointer, variable, ConstantU32(state, index)});
state.builder.AddFunction({OpStore, pointer, f32}); state.builder.AddFunction({spv::OpStore, pointer, f32});
}; };
StoreDistance(state.clip_distance_variable, output.clip_distance); StoreDistance(state.clip_distance_variable, output.clip_distance);
StoreDistance(state.cull_distance_variable, output.cull_distance); StoreDistance(state.cull_distance_variable, output.cull_distance);
@@ -392,13 +398,13 @@ uint32_t ConvertClipCoordinate(EmitterState& state, uint32_t coordinate, float s
const auto divided = state.builder.AllocateId(); const auto divided = state.builder.AllocateId();
const auto ndc = state.builder.AllocateId(); const auto ndc = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpFMul, TypeF32(state), window, coordinate, ConstantF32Value(state, scale)}); {spv::OpFMul, TypeF32(state), window, coordinate, ConstantF32Value(state, scale)});
state.builder.AddFunction( state.builder.AddFunction(
{OpFAdd, TypeF32(state), biased, window, ConstantF32Value(state, offset)}); {spv::OpFAdd, TypeF32(state), biased, window, ConstantF32Value(state, offset)});
state.builder.AddFunction( state.builder.AddFunction(
{OpFDiv, TypeF32(state), divided, biased, ConstantF32Value(state, half_extent)}); {spv::OpFDiv, TypeF32(state), divided, biased, ConstantF32Value(state, half_extent)});
state.builder.AddFunction( state.builder.AddFunction(
{OpFSub, TypeF32(state), ndc, divided, ConstantF32Value(state, 1.0f)}); {spv::OpFSub, TypeF32(state), ndc, divided, ConstantF32Value(state, 1.0f)});
return ndc; return ndc;
} }
@@ -408,14 +414,14 @@ uint32_t ConvertPositionToClipSpace(EmitterState& state, uint32_t position) {
for (uint32_t i = 0; i < 4; i++) { for (uint32_t i = 0; i < 4; i++) {
components[i] = state.builder.AllocateId(); components[i] = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpCompositeExtract, TypeF32(state), components[i], position, i}); {spv::OpCompositeExtract, TypeF32(state), components[i], position, i});
} }
components[0] = ConvertClipCoordinate(state, components[0], transform.scale[0], components[0] = ConvertClipCoordinate(state, components[0], transform.scale[0],
transform.offset[0], transform.half_extent[0]); transform.offset[0], transform.half_extent[0]);
components[1] = ConvertClipCoordinate(state, components[1], transform.scale[1], components[1] = ConvertClipCoordinate(state, components[1], transform.scale[1],
transform.offset[1], transform.half_extent[1]); transform.offset[1], transform.half_extent[1]);
const auto converted = state.builder.AllocateId(); const auto converted = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), converted, state.builder.AddFunction({spv::OpCompositeConstruct, TypeF32Vector(state, 4), converted,
components[0], components[1], components[2], components[3]}); components[0], components[1], components[2], components[3]});
return converted; return converted;
} }
@@ -424,12 +430,12 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state; auto& state = ctx.state;
const auto& exp = ctx.Export(inst); const auto& exp = ctx.Export(inst);
const auto exec = ctx.Arg(inst, 1); const auto exec = ctx.Arg(inst, 1);
if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask && if (state.program.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
state.pixel_valid_mask_variable != 0) { state.pixel_valid_mask_variable != 0) {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpSelect, TypeU32(state), value, exec,
{OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)}); ConstantU32(state, 1), ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value}); state.builder.AddFunction({spv::OpStore, state.pixel_valid_mask_variable, value});
} }
if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) { if (exp.kind == IR::ExportTargetKind::Null || exp.en == 0u) {
return; return;
@@ -437,9 +443,9 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
EmitIfCondition(state, exec, [&]() { EmitIfCondition(state, exec, [&]() {
const auto data = ctx.Arg(inst, 0); const auto data = ctx.Arg(inst, 0);
if (exp.kind == IR::ExportTargetKind::Primitive) { if (exp.kind == IR::ExportTargetKind::Primitive) {
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
state.builder.AddFunction( state.builder.AddFunction(
{OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)}); {spv::OpStore, MeshPrimitivePointer(state), ExportRawComponent(ctx, data, 0)});
} }
return; return;
} }
@@ -451,35 +457,36 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
if ((exp.en & 1u) != 0u && state.depth_variable != 0) { if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
const auto raw = ExportRawComponent(ctx, data, 0); const auto raw = ExportRawComponent(ctx, data, 0);
const auto f32 = state.builder.AllocateId(); const auto f32 = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw}); state.builder.AddFunction({spv::OpBitcast, TypeF32(state), f32, raw});
state.builder.AddFunction({OpStore, state.depth_variable, f32}); state.builder.AddFunction({spv::OpStore, state.depth_variable, f32});
} }
if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) { if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
const auto raw = ExportRawComponent(ctx, data, 2); const auto raw = ExportRawComponent(ctx, data, 2);
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw}); state.builder.AddFunction({spv::OpBitcast, TypeI32(state), value, raw});
state.builder.AddFunction( state.builder.AddFunction(
{OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer, {spv::OpAccessChain,
TypePointer(state, spv::StorageClassOutput, TypeI32(state)), pointer,
state.sample_mask_variable, ConstantU32(state, 0)}); state.sample_mask_variable, ConstantU32(state, 0)});
state.builder.AddFunction({OpStore, pointer, value}); state.builder.AddFunction({spv::OpStore, pointer, value});
} }
return; return;
} }
const auto variable = const auto variable =
state.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp); state.program.stage == ShaderType::Mesh ? 0u : OutputVariableForExport(state, exp);
if (state.stage != ShaderType::Mesh && variable == 0) { if (state.program.stage != ShaderType::Mesh && variable == 0) {
return; return;
} }
const bool uint_output = MrtOutputMode(state, exp) == 7u; const bool uint_output = MrtOutputMode(state, exp) == 7u;
const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4); const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
auto value = ExportVector(ctx, data, exp, uint_output); auto value = ExportVector(ctx, data, exp, uint_output);
if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt && if (state.program.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
exp.index < state.input_info.pixel->target_export_mapping.size()) { exp.index < state.input_info.pixel->target_export_mapping.size()) {
const auto mapping = state.input_info.pixel->target_export_mapping[exp.index]; const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];
if (!mapping.IsIdentity()) { if (!mapping.IsIdentity()) {
const auto mapped = state.builder.AllocateId(); const auto mapped = state.builder.AllocateId();
state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value, state.builder.AddFunction({spv::OpVectorShuffle, vector_type, mapped, value, value,
mapping.Map(0), mapping.Map(1), mapping.Map(2), mapping.Map(0), mapping.Map(1), mapping.Map(2),
mapping.Map(3)}); mapping.Map(3)});
value = mapped; value = mapped;
@@ -489,19 +496,21 @@ void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
state.input_info.vertex->clip_space.enabled) { state.input_info.vertex->clip_space.enabled) {
value = ConvertPositionToClipSpace(state, value); value = ConvertPositionToClipSpace(state, value);
} }
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
const auto kind = exp.kind == IR::ExportTargetKind::Position const auto kind = exp.kind == IR::ExportTargetKind::Position
? IR::StageOutputKind::Position ? IR::StageOutputKind::Position
: IR::StageOutputKind::Parameter; : IR::StageOutputKind::Parameter;
state.builder.AddFunction({OpStore, MeshOutputPointer(state, kind, exp.index), value}); state.builder.AddFunction(
{spv::OpStore, MeshOutputPointer(state, kind, exp.index), value});
} else if (exp.kind == IR::ExportTargetKind::Position) { } else if (exp.kind == IR::ExportTargetKind::Position) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)), {spv::OpAccessChain,
pointer, variable, ConstantU32(state, 0)}); TypePointer(state, spv::StorageClassOutput, TypeF32Vector(state, 4)), pointer,
state.builder.AddFunction({OpStore, pointer, value}); variable, ConstantU32(state, 0)});
state.builder.AddFunction({spv::OpStore, pointer, value});
} else { } else {
state.builder.AddFunction({OpStore, variable, value}); state.builder.AddFunction({spv::OpStore, variable, value});
} }
}); });
} }
@@ -521,23 +530,25 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::TtraceData: case IR::ValueOpcode::TtraceData:
case IR::ValueOpcode::InstPrefetch: return true; case IR::ValueOpcode::InstPrefetch: return true;
case IR::ValueOpcode::Barrier: { case IR::ValueOpcode::Barrier: {
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; const auto semantics =
state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup), spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
ConstantU32(state, ScopeWorkgroup), state.builder.AddFunction(
ConstantU32(state, semantics)}); {spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, semantics)});
return true; return true;
} }
case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true; case IR::ValueOpcode::MeshAllocate: EmitMeshAllocate(ctx, inst); return true;
case IR::ValueOpcode::MeshDrawParameter: { case IR::ValueOpcode::MeshDrawParameter: {
const auto index = inst.Arg(0).U32(); const auto index = inst.Arg(0).U32();
if (state.stage != ShaderType::Mesh || index >= IR::PushData::MeshDrawDwordCount) { if (state.program.stage != ShaderType::Mesh ||
index >= IR::PushData::MeshDrawDwordCount) {
ctx.Fail(inst, "invalid mesh draw parameter"); ctx.Fail(inst, "invalid mesh draw parameter");
} }
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), state.builder.AddFunction({spv::OpAccessChain, TypePushConstantElementPointer(state),
pointer, state.push_constant_variable, ConstantU32(state, 0), pointer, state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, index)}); ConstantU32(state, index)});
state.builder.AddFunction({OpLoad, TypeU32(state), ctx.Result(inst), pointer}); state.builder.AddFunction({spv::OpLoad, TypeU32(state), ctx.Result(inst), pointer});
return true; return true;
} }
case IR::ValueOpcode::GetUserData: { case IR::ValueOpcode::GetUserData: {
@@ -559,7 +570,7 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
case IR::ValueOpcode::UndefU16: case IR::ValueOpcode::UndefU16:
case IR::ValueOpcode::UndefU32: case IR::ValueOpcode::UndefU32:
case IR::ValueOpcode::UndefU64: case IR::ValueOpcode::UndefU64:
state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)}); state.builder.AddFunction({spv::OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
return true; return true;
case IR::ValueOpcode::DppMoveU32: { case IR::ValueOpcode::DppMoveU32: {
const auto flags = inst.Flags<IR::DppMoveFlags>(); const auto flags = inst.Flags<IR::DppMoveFlags>();
@@ -573,14 +584,16 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane); const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
const auto can_fetch = state.builder.AllocateId(); const auto can_fetch = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active}); {spv::OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)}); ctx.Emit(inst, spv::OpSelect, IR::Type::U32,
{can_fetch, shuffled, ConstantU32(state, 0)});
return true; return true;
} }
case IR::ValueOpcode::DppUpdateU32: { case IR::ValueOpcode::DppUpdateU32: {
const auto flags = inst.Flags<IR::DppMoveFlags>(); const auto flags = inst.Flags<IR::DppMoveFlags>();
const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2)); const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)}); ctx.Emit(inst, spv::OpSelect, IR::Type::U32,
{write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
return true; return true;
} }
case IR::ValueOpcode::WqmU64: case IR::ValueOpcode::WqmU64:
@@ -601,9 +614,9 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
return true; return true;
case IR::ValueOpcode::WriteLane: { case IR::ValueOpcode::WriteLane: {
const auto hit = state.builder.AllocateId(); const auto hit = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, TypeBool(state), hit, state.builder.AddFunction({spv::OpIEqual, TypeBool(state), hit,
EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)}); EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)}); ctx.Emit(inst, spv::OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
return true; return true;
} }
case IR::ValueOpcode::Permlane16U32: { case IR::ValueOpcode::Permlane16U32: {
@@ -620,35 +633,35 @@ bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto index = state.builder.AllocateId(); const auto index = state.builder.AllocateId();
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)}); {spv::OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
if (flags.x16) { if (flags.x16) {
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)}); {spv::OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
} else { } else {
state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row}); state.builder.AddFunction({spv::OpCopyObject, TypeU32(state), row_value, row});
} }
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)}); {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)}); {spv::OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)}); {spv::OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
state.builder.AddFunction( state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)}); {spv::OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, upper,
ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
state.builder.AddFunction( state.builder.AddFunction(
{OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)}); {spv::OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), shifted, selected, shift}); {spv::OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
state.builder.AddFunction( state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, row_value, index});
{OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
const auto shuffled = ctx.Shuffle(inst, 0, target); const auto shuffled = ctx.Shuffle(inst, 0, target);
uint32_t result = shuffled; uint32_t result = shuffled;
if (!flags.fetch_inactive) { if (!flags.fetch_inactive) {
const auto source_exec = ctx.Shuffle(inst, 3, target); const auto source_exec = ctx.Shuffle(inst, 3, target);
result = state.builder.AllocateId(); result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled, state.builder.AddFunction({spv::OpSelect, TypeU32(state), result, source_exec,
ConstantU32(state, 0)}); shuffled, ConstantU32(state, 0)});
} }
ctx.Define(inst, result); ctx.Define(inst, result);
return true; return true;
@@ -10,15 +10,16 @@ DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uin
const auto selected = state.builder.AllocateId(); const auto selected = state.builder.AllocateId();
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)}); {spv::OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)}); {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift}); {spv::OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)}); {spv::OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected}); state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, quad_base, selected});
return {target, ConstantBool(state, true)}; return {target, ConstantBool(state, true)};
} }
@@ -30,20 +31,21 @@ DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uin
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
const auto valid = state.builder.AllocateId(); const auto valid = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)}); {spv::OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)}); state.builder.AddFunction(
{spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
if (left) { if (left) {
state.builder.AddFunction( state.builder.AddFunction(
{OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)}); {spv::OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
state.builder.AddFunction( state.builder.AddFunction(
{OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)}); {spv::OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
} else { } else {
state.builder.AddFunction( state.builder.AddFunction(
{OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)}); {spv::OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
state.builder.AddFunction( state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)}); {spv::OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
} }
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted}); state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
return {target, valid}; return {target, valid};
} }
@@ -57,15 +59,17 @@ DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subi
const auto selected = state.builder.AllocateId(); const auto selected = state.builder.AllocateId();
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)}); {spv::OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction( state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)}); {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
state.builder.AddFunction( state.builder.AddFunction(
{OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)}); {spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus}); state.builder.AddFunction(
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected}); {spv::OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
state.builder.AddFunction(
{spv::OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, in_high, minus, plus});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, row, selected});
return {target, ConstantBool(state, true)}; return {target, ConstantBool(state, true)};
} }
@@ -77,12 +81,12 @@ DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool
const auto mirrored = state.builder.AllocateId(); const auto mirrored = state.builder.AllocateId();
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)}); {spv::OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)}); {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
state.builder.AddFunction( state.builder.AddFunction(
{OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane}); {spv::OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored}); state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, base, mirrored});
return {target, ConstantBool(state, true)}; return {target, ConstantBool(state, true)};
} }
@@ -109,7 +113,7 @@ DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
if (control >= 0x160u && control <= 0x16fu) { if (control >= 0x160u && control <= 0x16fu) {
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseXor, TypeU32(state), target, subid, ConstantU32(state, control & 0xfu)}); {spv::OpBitwiseXor, TypeU32(state), target, subid, ConstantU32(state, control & 0xfu)});
return {target, ConstantBool(state, true)}; return {target, ConstantBool(state, true)};
} }
return {subid, ConstantBool(state, true)}; return {subid, ConstantBool(state, true)};
@@ -121,7 +125,7 @@ uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
} }
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable}); {spv::OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32)); return state.lane_half == 0 ? value : EmitAddU32(state, value, ConstantU32(state, 32));
} }
@@ -150,9 +154,10 @@ uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uin
} }
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)), state.builder.AddFunction({spv::OpAccessChain,
pointer, variable, ConstantU32(state, component)}); TypePointer(state, spv::StorageClassInput, TypeU32(state)), pointer,
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); variable, ConstantU32(state, component)});
state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
return value; return value;
} }
@@ -162,9 +167,10 @@ uint32_t EmitLocalInvocationIndex(EmitterState& state) {
return ConstantU32(state, 0); return ConstantU32(state, 0);
} }
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, variable}); state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, variable});
if (state.lane_count == 2) { if (state.lane_count == 2) {
const auto wave_base = EmitBinaryU32(state, OpBitwiseAnd, value, ConstantU32(state, ~31u)); const auto wave_base =
EmitBinaryU32(state, spv::OpBitwiseAnd, value, ConstantU32(state, ~31u));
return EmitAddU32(state, EmitAddU32(state, value, wave_base), return EmitAddU32(state, EmitAddU32(state, value, wave_base),
ConstantU32(state, state.lane_half * 32)); ConstantU32(state, state.lane_half * 32));
} }
@@ -178,13 +184,13 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
const auto scalar_type = VertexParameterScalarType(state, kind); const auto scalar_type = VertexParameterScalarType(state, kind);
uint32_t raw = state.builder.AllocateId(); uint32_t raw = state.builder.AllocateId();
if (count == 1u) { if (count == 1u) {
state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id}); state.builder.AddFunction({spv::OpLoad, scalar_type, raw, input.variable_id});
} else { } else {
const auto pointer_type = TypePointer(state, StorageClassInput, scalar_type); const auto pointer_type = TypePointer(state, spv::StorageClassInput, scalar_type);
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id, state.builder.AddFunction({spv::OpAccessChain, pointer_type, pointer, input.variable_id,
ConstantU32(state, component)}); ConstantU32(state, component)});
state.builder.AddFunction({OpLoad, scalar_type, raw, pointer}); state.builder.AddFunction({spv::OpLoad, scalar_type, raw, pointer});
} }
if (kind == VertexInputScalarKind::Uint) { if (kind == VertexInputScalarKind::Uint) {
@@ -192,47 +198,49 @@ uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding
} }
const auto bits = state.builder.AllocateId(); const auto bits = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw}); state.builder.AddFunction({spv::OpBitcast, TypeU32(state), bits, raw});
return bits; return bits;
} }
uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) { uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
const auto active_ballot = state.builder.AllocateId(); const auto active_ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot, state.builder.AddFunction({spv::OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
ConstantU32(state, ScopeSubgroup), ConstantBool(state, true)}); ConstantU32(state, spv::ScopeSubgroup), ConstantBool(state, true)});
return EmitBallotLaneActiveBool(state, active_ballot, lane); return EmitBallotLaneActiveBool(state, active_ballot, lane);
} }
uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) { uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
uint32_t mask = low; uint32_t mask = low;
if (state.program.wave_size == 64u) { if (state.program.wave_size == 64u) {
const auto high = state.builder.AllocateId(); const auto high = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId(); const auto in_high = state.builder.AllocateId();
const auto selected = state.builder.AllocateId(); const auto selected = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
state.builder.AddFunction( state.builder.AddFunction(
{OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)}); {spv::OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low}); state.builder.AddFunction(
{spv::OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
state.builder.AddFunction({spv::OpSelect, TypeU32(state), selected, in_high, high, low});
mask = selected; mask = selected;
} }
const auto lane_low = state.builder.AllocateId(); const auto lane_low = state.builder.AllocateId();
const auto bit = state.builder.AllocateId(); const auto bit = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)}); {spv::OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low}); {spv::OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
const auto hit = state.builder.AllocateId(); const auto hit = state.builder.AllocateId();
const auto active = state.builder.AllocateId(); const auto active = state.builder.AllocateId();
const auto in_range = state.builder.AllocateId(); const auto in_range = state.builder.AllocateId();
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit}); state.builder.AddFunction({spv::OpBitwiseAnd, TypeU32(state), hit, mask, bit});
state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction(
{OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.program.wave_size)}); {spv::OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range}); state.builder.AddFunction({spv::OpULessThan, TypeBool(state), in_range, lane,
ConstantU32(state, state.program.wave_size)});
state.builder.AddFunction({spv::OpLogicalAnd, TypeBool(state), ret, active, in_range});
return ret; return ret;
} }
@@ -44,10 +44,10 @@ uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
auto value = ctx.Def(address.Arg(packed)); auto value = ctx.Def(address.Arg(packed));
if (layout.bit_width == 16u) { if (layout.bit_width == 16u) {
if ((layout.bit_offset & 31u) != 0u) { if ((layout.bit_offset & 31u) != 0u) {
value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value, value = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 16)); ConstantU32(ctx.state, 16));
} }
value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value, value = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 0xffffu)); ConstantU32(ctx.state, 0xffffu));
} }
return value; return value;
@@ -58,7 +58,7 @@ uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::
const auto value = AddressU32(ctx, mem, address, component); const auto value = AddressU32(ctx, mem, address, component);
return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u return Decoder::ImageAddressComponentLayout(mem.image_sample_flags, component).bit_width == 16u
? EmitF16BitsToF32(ctx.state, value) ? EmitF16BitsToF32(ctx.state, value)
: Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value); : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), value);
} }
ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) { ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageDimension dimension) {
@@ -85,19 +85,19 @@ uint32_t ZeroF32(EmitterState& state) {
} }
uint32_t CubeAxis(EmitterState& state, uint32_t value) { uint32_t CubeAxis(EmitterState& state, uint32_t value) {
return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u)); return Binary(state, spv::OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
} }
uint32_t CubeLayer(EmitterState& state, uint32_t value) { uint32_t CubeLayer(EmitterState& state, uint32_t value) {
const auto guest = state.builder.AllocateId(); const auto guest = state.builder.AllocateId();
state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value}); state.builder.AddFunction({spv::OpConvertFToU, TypeU32(state), guest, value});
const auto padding = const auto padding = Binary(
Binary(state, OpShiftLeftLogical, TypeU32(state), state, spv::OpShiftLeftLogical, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)), Binary(state, spv::OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
ConstantU32(state, 1)); ConstantU32(state, 1));
const auto host = Binary(state, OpISub, TypeU32(state), guest, padding); const auto host = Binary(state, spv::OpISub, TypeU32(state), guest, padding);
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host}); state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), result, host});
return result; return result;
} }
@@ -119,10 +119,10 @@ uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
: ZeroF32(ctx.state); : ZeroF32(ctx.state);
if (cube) z = CubeLayer(ctx.state, z); if (cube) z = CubeLayer(ctx.state, z);
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z}); {spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
} else { } else {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y}); {spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
} }
return result; return result;
} }
@@ -139,10 +139,10 @@ uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::In
const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2) const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
: ConstantU32(ctx.state, 0); : ConstantU32(ctx.state, 0);
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z}); {spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
} else { } else {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y}); {spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
} }
return result; return result;
} }
@@ -156,7 +156,7 @@ uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst
} }
uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) { uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value); return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
} }
uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value, uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
@@ -164,7 +164,7 @@ uint32_t SampledComponentBits(ValueEmitContext& ctx, uint32_t value,
if (numeric_class == Prospero::TextureNumericClass::Uint) { if (numeric_class == Prospero::TextureNumericClass::Uint) {
return value; return value;
} }
return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value); return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), value);
} }
uint32_t SampledComponentZero(EmitterState& state, Prospero::TextureNumericClass numeric_class) { uint32_t SampledComponentZero(EmitterState& state, Prospero::TextureNumericClass numeric_class) {
@@ -193,7 +193,7 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
if (dref) return value; if (dref) return value;
const auto component = gather ? index : DmaskComponent(mem.dmask, index); const auto component = gather ? index : DmaskComponent(mem.dmask, index);
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeExtract, ctx.state.builder.AddFunction({spv::OpCompositeExtract,
ImageScalarType(ctx.state, value_class), result, value, ImageScalarType(ctx.state, value_class), result, value,
component}); component});
return result; return result;
@@ -209,24 +209,26 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
const auto low_bits = SampledComponentBits(ctx, low, value_class); const auto low_bits = SampledComponentBits(ctx, low, value_class);
const auto high_bits = SampledComponentBits(ctx, high, value_class); const auto high_bits = SampledComponentBits(ctx, high, value_class);
const auto mask = ConstantU32(ctx.state, 0xffffu); const auto mask = ConstantU32(ctx.state, 0xffffu);
packed[word] = Binary( packed[word] =
ctx.state, OpBitwiseOr, TypeU32(ctx.state), Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask), Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), low_bits, mask),
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high_bits, mask), Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
high_bits, mask),
ConstantU32(ctx.state, 16u))); ConstantU32(ctx.state, 16u)));
} else { } else {
const auto pair = ctx.state.builder.AllocateId(); const auto pair = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high}); {spv::OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
packed[word] = ctx.state.builder.AllocateId(); packed[word] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word], ctx.state.builder.AddFunction({spv::OpExtInst, TypeU32(ctx.state), packed[word],
GlslStd450(ctx.state), GlslPackHalf2x16, pair}); GlslStd450(ctx.state), GLSLstd450PackHalf2x16,
pair});
} }
} }
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4),
packed[0], packed[1], packed[2], packed[3]}); result, packed[0], packed[1], packed[2], packed[3]});
return result; return result;
} }
uint32_t component[4] {}; uint32_t component[4] {};
@@ -236,12 +238,13 @@ uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value,
continue; continue;
} }
const auto scalar = ctx.state.builder.AllocateId(); const auto scalar = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction({spv::OpCompositeExtract,
{OpCompositeExtract, ImageScalarType(ctx.state, value_class), scalar, value, index}); ImageScalarType(ctx.state, value_class), scalar, value,
index});
component[index] = SampledComponentBits(ctx, scalar, value_class); component[index] = SampledComponentBits(ctx, scalar, value_class);
} }
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
component[0], component[1], component[2], component[3]}); component[0], component[1], component[2], component[3]});
return result; return result;
} }
@@ -254,10 +257,11 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const auto size = ctx.state.builder.AllocateId(); const auto size = ctx.state.builder.AllocateId();
if (info.multisampled != 0u) { if (info.multisampled != 0u) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpImageQuerySize, ImageViewSizeType(ctx.state, dimension), size, image}); {spv::OpImageQuerySize, ImageViewSizeType(ctx.state, dimension), size, image});
} else { } else {
ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, dimension), ctx.state.builder.AddFunction({spv::OpImageQuerySizeLod,
size, image, AddressU32(ctx, mem, address, 0)}); ImageViewSizeType(ctx.state, dimension), size, image,
AddressU32(ctx, mem, address, 0)});
} }
const auto components = info.coordinate_components; const auto components = info.coordinate_components;
uint32_t result[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0), uint32_t result[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
@@ -268,15 +272,16 @@ uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
for (uint32_t index = 0; index < components; index++) { for (uint32_t index = 0; index < components; index++) {
result[index] = ctx.state.builder.AllocateId(); result[index] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeExtract, TypeU32(ctx.state), result[index], size, index}); {spv::OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
} }
} }
if (info.multisampled == 0u) { if (info.multisampled == 0u) {
result[3] = ctx.state.builder.AllocateId(); result[3] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image}); ctx.state.builder.AddFunction(
{spv::OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
} }
const auto vector = ctx.state.builder.AllocateId(); const auto vector = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector, ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
result[0], result[1], result[2], result[3]}); result[0], result[1], result[2], result[3]});
return vector; return vector;
} }
@@ -290,30 +295,30 @@ uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
if (components == 3u) { if (components == 3u) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero}); {spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
} else { } else {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero}); {spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
} }
return result; return result;
} }
const auto packed = Unary(ctx.state, OpBitcast, TypeI32(ctx.state), const auto packed = Unary(ctx.state, spv::OpBitcast, TypeI32(ctx.state),
AddressU32(ctx, mem, address, layout.offset)); AddressU32(ctx, mem, address, layout.offset));
uint32_t values[3] = {zero, zero, zero}; uint32_t values[3] = {zero, zero, zero};
for (uint32_t index = 0; index < components; index++) { for (uint32_t index = 0; index < components; index++) {
values[index] = ctx.state.builder.AllocateId(); values[index] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index], ctx.state.builder.AddFunction({spv::OpBitFieldSExtract, TypeI32(ctx.state), values[index],
packed, ConstantU32(ctx.state, index * 8u), packed, ConstantU32(ctx.state, index * 8u),
ConstantU32(ctx.state, 6)}); ConstantU32(ctx.state, 6)});
} }
if (components == 1u) return values[0]; if (components == 1u) return values[0];
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
if (components == 3u) { if (components == 3u) {
ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 3),
values[0], values[1], values[2]}); result, values[0], values[1], values[2]});
} else { } else {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]}); {spv::OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
} }
return result; return result;
} }
@@ -322,18 +327,18 @@ uint32_t HorizontalOffsets(EmitterState& state, ImageDimension dimension) {
const auto components = ImageDimensionInfoFor(dimension).spatial_components; const auto components = ImageDimensionInfoFor(dimension).spatial_components;
const auto count = ConstantU32(state, 4); const auto count = ConstantU32(state, 4);
const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2); const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
const auto array_type = state.builder.Type(OpTypeArray, {element_type, count}); const auto array_type = state.builder.Type(spv::OpTypeArray, {element_type, count});
uint32_t offsets[4] {}; uint32_t offsets[4] {};
for (uint32_t index = 0; index < 4u; index++) { for (uint32_t index = 0; index < 4u; index++) {
const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1); const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
if (components == 1u) { if (components == 1u) {
offsets[index] = x; offsets[index] = x;
} else { } else {
offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2), offsets[index] = state.builder.Constant(
{x, ConstantI32(state, 0)}); spv::OpConstantComposite, TypeI32Vector(state, 2), {x, ConstantI32(state, 0)});
} }
} }
return state.builder.Constant(OpConstantComposite, array_type, return state.builder.Constant(spv::OpConstantComposite, array_type,
{offsets[0], offsets[1], offsets[2], offsets[3]}); {offsets[0], offsets[1], offsets[2], offsets[3]});
} }
@@ -362,12 +367,12 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
if (info.format == Prospero::BufferFormat::kInvalid) return texel; if (info.format == Prospero::BufferFormat::kInvalid) return texel;
const auto packed = ctx.state.builder.AllocateId(); const auto packed = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u}); ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0), uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)}; ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
for (uint32_t component = 0; component < info.component_count; component++) { for (uint32_t component = 0; component < info.component_count; component++) {
components[component] = ctx.state.builder.AllocateId(); components[component] = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), ctx.state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(ctx.state),
components[component], packed, components[component], packed,
ConstantU32(ctx.state, info.component_bit_offset[component]), ConstantU32(ctx.state, info.component_bit_offset[component]),
ConstantU32(ctx.state, info.component_bits[component])}); ConstantU32(ctx.state, info.component_bits[component])});
@@ -389,7 +394,7 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
} }
} }
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
selected[0], selected[1], selected[2], selected[3]}); selected[0], selected[1], selected[2], selected[3]});
return result; return result;
} }
@@ -403,7 +408,7 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
(ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u; (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
if (selector < 4u) { if (selector < 4u) {
const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u); const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4), return ctx.state.builder.Constant(spv::OpConstantComposite, TypeU32Vector(ctx.state, 4),
{value, value, value, value}); {value, value, value, value});
} }
@@ -411,15 +416,16 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
for (uint32_t lane = 0; lane < 4u; lane++) { for (uint32_t lane = 0; lane < 4u; lane++) {
const auto packed = ctx.state.builder.AllocateId(); const auto packed = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane}); {spv::OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
values[lane] = ctx.state.builder.AllocateId(); values[lane] = ctx.state.builder.AllocateId();
const auto physical = (selector - 4u) % info.component_count; const auto physical = (selector - 4u) % info.component_count;
ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed, ctx.state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(ctx.state), values[lane],
packed,
ConstantU32(ctx.state, info.component_bit_offset[physical]), ConstantU32(ctx.state, info.component_bit_offset[physical]),
ConstantU32(ctx.state, info.component_bits[physical])}); ConstantU32(ctx.state, info.component_bits[physical])});
} }
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, ctx.state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
values[0], values[1], values[2], values[3]}); values[0], values[1], values[2], values[3]});
return result; return result;
} }
@@ -427,17 +433,18 @@ uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uin
uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
uint32_t coord, Prospero::TextureNumericClass numeric_class) { uint32_t coord, Prospero::TextureNumericClass numeric_class) {
auto& state = ctx.state; auto& state = ctx.state;
state.builder.RequireCapability(CapabilityImageQuery); state.builder.RequireCapability(spv::CapabilityImageQuery);
const auto image = LoadSampledImageDescriptor(state, mem.resource); const auto image = LoadSampledImageDescriptor(state, mem.resource);
const auto width = state.builder.AllocateId(); const auto width = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)}); {spv::OpImageQuerySizeLod, TypeU32(state), width, image, ConstantU32(state, 0)});
const auto width_f32 = state.builder.AllocateId(); const auto width_f32 = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, TypeF32(state), width_f32, width}); state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), width_f32, width});
const auto left = state.builder.AllocateId(); const auto left = state.builder.AllocateId();
state.builder.AddFunction({OpExtInst, TypeF32(state), left, GlslStd450(state), GlslFloor, state.builder.AddFunction({spv::OpExtInst, TypeF32(state), left, GlslStd450(state),
Binary(state, OpFSub, TypeF32(state), GLSLstd450Floor,
Binary(state, OpFMul, TypeF32(state), coord, width_f32), Binary(state, spv::OpFSub, TypeF32(state),
Binary(state, spv::OpFMul, TypeF32(state), coord, width_f32),
ConstantF32(state, 0x3f000000u))}); ConstantF32(state, 0x3f000000u))});
const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler); const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler);
@@ -450,20 +457,20 @@ uint32_t EmitOneDimensionalGatherLz(ValueEmitContext& ctx, const IR::MemoryInfo&
uint32_t values[2] {}; uint32_t values[2] {};
for (uint32_t index = 0; index < 2u; index++) { for (uint32_t index = 0; index < 2u; index++) {
const auto sample_coord = const auto sample_coord =
Binary(state, OpFDiv, TypeF32(state), Binary(state, spv::OpFDiv, TypeF32(state),
Binary(state, OpFAdd, TypeF32(state), left, Binary(state, spv::OpFAdd, TypeF32(state), left,
ConstantF32(state, index == 0u ? 0x3f000000u : 0x3fc00000u)), ConstantF32(state, index == 0u ? 0x3f000000u : 0x3fc00000u)),
width_f32); width_f32);
const auto texel = state.builder.AllocateId(); const auto texel = state.builder.AllocateId();
state.builder.AddFunction({OpImageSampleExplicitLod, vector_type, texel, sampled, state.builder.AddFunction({spv::OpImageSampleExplicitLod, vector_type, texel, sampled,
sample_coord, ImageOperandsLodMask, ZeroF32(state)}); sample_coord, spv::ImageOperandsLodMask, ZeroF32(state)});
values[index] = state.builder.AllocateId(); values[index] = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpCompositeExtract, scalar_type, values[index], texel, component}); {spv::OpCompositeExtract, scalar_type, values[index], texel, component});
} }
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpCompositeConstruct, vector_type, result, values[0], values[1],
{OpCompositeConstruct, vector_type, result, values[0], values[1], values[1], values[0]}); values[1], values[0]});
return result; return result;
} }
@@ -475,26 +482,27 @@ uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32
for (uint32_t component = 0; component < info.component_count; component++) { for (uint32_t component = 0; component < info.component_count; component++) {
const auto value = ctx.state.builder.AllocateId(); const auto value = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeExtract, TypeU32(ctx.state), value, texel, component}); {spv::OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
const auto maximum = const auto maximum =
ConstantU32(ctx.state, info.component_bits[component] == 32u ConstantU32(ctx.state, info.component_bits[component] == 32u
? UINT32_MAX ? UINT32_MAX
: (1u << info.component_bits[component]) - 1u); : (1u << info.component_bits[component]) - 1u);
const auto within = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum); const auto within =
Binary(ctx.state, spv::OpULessThan, TypeBool(ctx.state), value, maximum);
const auto clamped = ctx.state.builder.AllocateId(); const auto clamped = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpSelect, TypeU32(ctx.state), clamped, within, value, maximum}); {spv::OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
const auto shifted = const auto shifted =
info.component_bit_offset[component] == 0u info.component_bit_offset[component] == 0u
? clamped ? clamped
: Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped, : Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), clamped,
ConstantU32(ctx.state, info.component_bit_offset[component])); ConstantU32(ctx.state, info.component_bit_offset[component]));
packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted); packed = Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
} }
const auto zero = ConstantU32(ctx.state, 0u); const auto zero = ConstantU32(ctx.state, 0u);
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero}); {spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
return result; return result;
} }
@@ -509,38 +517,39 @@ uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t d
const auto packed_index = DmaskComponentIndex(dmask, source); const auto packed_index = DmaskComponentIndex(dmask, source);
raw = ctx.state.builder.AllocateId(); raw = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeExtract, TypeU32(ctx.state), raw, data, {spv::OpCompositeExtract, TypeU32(ctx.state), raw, data,
mem.data_bits == 16u ? packed_index / 2u : packed_index}); mem.data_bits == 16u ? packed_index / 2u : packed_index});
if (mem.data_bits == 16u) { if (mem.data_bits == 16u) {
if ((packed_index & 1u) != 0u) { if ((packed_index & 1u) != 0u) {
raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw, raw = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), raw,
ConstantU32(ctx.state, 16u)); ConstantU32(ctx.state, 16u));
} }
raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw, raw = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), raw,
ConstantU32(ctx.state, 0xffffu)); ConstantU32(ctx.state, 0xffffu));
} }
} }
values[component] = integer ? raw values[component] = integer ? raw
: mem.data_bits == 16u : mem.data_bits == 16u
? EmitF16BitsToF32(ctx.state, raw) ? EmitF16BitsToF32(ctx.state, raw)
: Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw); : Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), raw);
} }
const auto texel = ctx.state.builder.AllocateId(); const auto texel = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4), {spv::OpCompositeConstruct,
texel, values[0], values[1], values[2], values[3]}); integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4), texel, values[0],
values[1], values[2], values[3]});
return PackImageTexel(ctx, mem, texel); return PackImageTexel(ctx, mem, texel);
} }
uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) { uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
switch (opcode) { switch (opcode) {
case IR::ValueOpcode::ImageAtomicSwap32: return OpAtomicExchange; case IR::ValueOpcode::ImageAtomicSwap32: return spv::OpAtomicExchange;
case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd; case IR::ValueOpcode::ImageAtomicIAdd32: return spv::OpAtomicIAdd;
case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin; case IR::ValueOpcode::ImageAtomicUMin32: return spv::OpAtomicUMin;
case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax; case IR::ValueOpcode::ImageAtomicUMax32: return spv::OpAtomicUMax;
case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd; case IR::ValueOpcode::ImageAtomicAnd32: return spv::OpAtomicAnd;
case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr; case IR::ValueOpcode::ImageAtomicOr32: return spv::OpAtomicOr;
case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor; case IR::ValueOpcode::ImageAtomicXor32: return spv::OpAtomicXor;
default: return 0; default: return 0;
} }
} }
@@ -561,28 +570,29 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1)); const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
if (address == nullptr) return true; if (address == nullptr) return true;
if (op == IR::ValueOpcode::ImageQueryDimensions) { if (op == IR::ValueOpcode::ImageQueryDimensions) {
state.builder.RequireCapability(CapabilityImageQuery); state.builder.RequireCapability(spv::CapabilityImageQuery);
ctx.Define(inst, QueryDimensions(ctx, mem, *address)); ctx.Define(inst, QueryDimensions(ctx, mem, *address));
return true; return true;
} }
if (op == IR::ValueOpcode::ImageQueryLod) { if (op == IR::ValueOpcode::ImageQueryLod) {
state.builder.RequireCapability(CapabilityImageQuery); state.builder.RequireCapability(spv::CapabilityImageQuery);
const auto dimension = image.dimension; const auto dimension = image.dimension;
const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler); const auto sampled = MakeSampledImage(state, mem.resource, mem.sampler);
const auto lod = state.builder.AllocateId(); const auto lod = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled, {spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components)}); CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components)});
uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
ConstantU32(state, 0)}; ConstantU32(state, 0)};
for (uint32_t index = 0; index < 2u; index++) { for (uint32_t index = 0; index < 2u; index++) {
const auto component = state.builder.AllocateId(); const auto component = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index}); state.builder.AddFunction(
{spv::OpCompositeExtract, TypeF32(state), component, lod, index});
values[index] = FloatBits(ctx, component); values[index] = FloatBits(ctx, component);
} }
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0], state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(state, 4), result,
values[1], values[2], values[3]}); values[0], values[1], values[2], values[3]});
ctx.Define(inst, result); ctx.Define(inst, result);
return true; return true;
} }
@@ -601,13 +611,13 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto coord = CoordU32(ctx, mem, *address, dimension); const auto coord = CoordU32(ctx, mem, *address, dimension);
if (dimension_info.multisampled != 0u) { if (dimension_info.multisampled != 0u) {
state.builder.AddFunction( state.builder.AddFunction(
{OpImageFetch, ImageVectorType(state, numeric_class, 4), color, {spv::OpImageFetch, ImageVectorType(state, numeric_class, 4), color,
descriptor, coord, ImageOperandsSampleMask, descriptor, coord, spv::ImageOperandsSampleMask,
AddressU32(ctx, mem, *address, dimension_info.coordinate_components)}); AddressU32(ctx, mem, *address, dimension_info.coordinate_components)});
} else { } else {
state.builder.AddFunction({OpImageFetch, state.builder.AddFunction({spv::OpImageFetch,
ImageVectorType(state, numeric_class, 4), color, ImageVectorType(state, numeric_class, 4), color,
descriptor, coord, ImageOperandsLodMask, descriptor, coord, spv::ImageOperandsLodMask,
LodU32(ctx, mem, *address, dimension)}); LodU32(ctx, mem, *address, dimension)});
} }
return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class, return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), numeric_class,
@@ -667,22 +677,29 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
if (layout.dref != NoImageComponent) { if (layout.dref != NoImageComponent) {
dref_value = AddressF32(ctx, mem, *address, layout.dref); dref_value = AddressF32(ctx, mem, *address, layout.dref);
} }
words = {OpImageDrefGather, TypeF32Vector(state, 4), sample, sampled, coord, words = {spv::OpImageDrefGather,
TypeF32Vector(state, 4),
sample,
sampled,
coord,
dref_value}; dref_value};
} else { } else {
uint32_t component = 0; uint32_t component = 0;
if (ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid) { if (ImageConversionFormat(state, mem).format == Prospero::BufferFormat::kInvalid) {
component = ImageGatherComponent(mem.dmask); component = ImageGatherComponent(mem.dmask);
} }
words = {OpImageGather, ImageVectorType(state, numeric_class, 4), words = {spv::OpImageGather,
sample, sampled, ImageVectorType(state, numeric_class, 4),
coord, ConstantU32(state, component)}; sample,
sampled,
coord,
ConstantU32(state, component)};
} }
if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) { if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
words.push_back(ImageOperandsConstOffsetsMask); words.push_back(spv::ImageOperandsConstOffsetsMask);
words.push_back(HorizontalOffsets(state, dimension)); words.push_back(HorizontalOffsets(state, dimension));
} else if (layout.offset != NoImageComponent) { } else if (layout.offset != NoImageComponent) {
words.push_back(ImageOperandsOffsetMask); words.push_back(spv::ImageOperandsOffsetMask);
words.push_back(PackedOffset(ctx, mem, *address, layout, dimension)); words.push_back(PackedOffset(ctx, mem, *address, layout, dimension));
} }
state.builder.AddFunction(words); state.builder.AddFunction(words);
@@ -697,12 +714,12 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) || const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
HasFlag(mem, Decoder::ImageSampleFlagLod) || HasFlag(mem, Decoder::ImageSampleFlagLod) ||
HasFlag(mem, Decoder::ImageSampleFlagLevelZero) || HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
state.stage != ShaderType::Pixel; state.program.stage != ShaderType::Pixel;
uint32_t opcode = OpImageSampleImplicitLod; uint32_t opcode = spv::OpImageSampleImplicitLod;
if (explicit_lod) { if (explicit_lod) {
opcode = dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod; opcode = dref ? spv::OpImageSampleDrefExplicitLod : spv::OpImageSampleExplicitLod;
} else if (dref) { } else if (dref) {
opcode = OpImageSampleDrefImplicitLod; opcode = spv::OpImageSampleDrefImplicitLod;
} }
uint32_t result_type = ImageVectorType(state, numeric_class, 4); uint32_t result_type = ImageVectorType(state, numeric_class, 4);
uint32_t dref_value = 0; uint32_t dref_value = 0;
@@ -716,20 +733,20 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
uint32_t operand_mask = 0; uint32_t operand_mask = 0;
std::vector<uint32_t> operands; std::vector<uint32_t> operands;
if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) { if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
operand_mask |= ImageOperandsGradMask; operand_mask |= spv::ImageOperandsGradMask;
operands.push_back( operands.push_back(
CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components)); CoordF32(ctx, mem, *address, layout.grad_x, dimension_info.spatial_components));
operands.push_back( operands.push_back(
CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components)); CoordF32(ctx, mem, *address, layout.grad_y, dimension_info.spatial_components));
} else if (explicit_lod) { } else if (explicit_lod) {
operand_mask |= ImageOperandsLodMask; operand_mask |= spv::ImageOperandsLodMask;
auto lod = ZeroF32(state); auto lod = ZeroF32(state);
if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) { if (HasFlag(mem, Decoder::ImageSampleFlagLod) && layout.lod != NoImageComponent) {
lod = AddressF32(ctx, mem, *address, layout.lod); lod = AddressF32(ctx, mem, *address, layout.lod);
} }
operands.push_back(lod); operands.push_back(lod);
} else if (layout.bias != NoImageComponent) { } else if (layout.bias != NoImageComponent) {
operand_mask |= ImageOperandsBiasMask; operand_mask |= spv::ImageOperandsBiasMask;
operands.push_back(AddressF32(ctx, mem, *address, layout.bias)); operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
} }
const auto EmitSample = [&](uint32_t resource) { const auto EmitSample = [&](uint32_t resource) {
@@ -772,11 +789,11 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
} }
const auto LoadMapping = [&](uint32_t index) { const auto LoadMapping = [&](uint32_t index) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state),
pointer, state.flattened_srt_variable, ConstantU32(state, 0), pointer, state.flattened_srt_variable, ConstantU32(state, 0),
index}); index});
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
return value; return value;
}; };
const auto mapping = ConstantU32(state, image.indirect_mapping_offset); const auto mapping = ConstantU32(state, image.indirect_mapping_offset);
@@ -785,61 +802,63 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
auto selected = ConstantU32(state, 0u); auto selected = ConstantU32(state, 0u);
for (uint32_t iteration = 0; iteration < image.indirect_search_iterations; for (uint32_t iteration = 0; iteration < image.indirect_search_iterations;
iteration++) { iteration++) {
const auto active = Binary(state, OpULessThan, TypeBool(state), low, high); const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high);
const auto mid = const auto mid = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state), low, high),
Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u)); ConstantU32(state, 1u));
const auto probe = state.builder.AllocateId(); const auto probe = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)}); {spv::OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
const auto entry = Binary(state, OpIAdd, TypeU32(state), mapping, const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), mapping,
Binary(state, OpIAdd, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
probe, ConstantU32(state, 1u)), probe, ConstantU32(state, 1u)),
ConstantU32(state, 1u))); ConstantU32(state, 1u)));
const auto mapped_key = LoadMapping(entry); const auto mapped_key = LoadMapping(entry);
const auto candidate = const auto candidate = LoadMapping(
LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u))); Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
const auto equal = Binary(state, OpIEqual, TypeBool(state), mapped_key, key); const auto equal = Binary(state, spv::OpIEqual, TypeBool(state), mapped_key, key);
const auto match = Binary(state, OpLogicalAnd, TypeBool(state), active, equal); const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, equal);
const auto next_selected = state.builder.AllocateId(); const auto next_selected = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpSelect, TypeU32(state), next_selected, match, candidate, selected}); {spv::OpSelect, TypeU32(state), next_selected, match, candidate, selected});
selected = next_selected; selected = next_selected;
const auto less = Binary(state, OpULessThan, TypeBool(state), mapped_key, key); const auto less = Binary(state, spv::OpULessThan, TypeBool(state), mapped_key, key);
const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less); const auto take_upper = Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less);
const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active, const auto take_lower = Binary(state, spv::OpLogicalAnd, TypeBool(state), active,
Unary(state, OpLogicalNot, TypeBool(state), less)); Unary(state, spv::OpLogicalNot, TypeBool(state), less));
const auto next_low = state.builder.AllocateId(); const auto next_low = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpSelect, TypeU32(state), next_low, take_upper, {spv::OpSelect, TypeU32(state), next_low, take_upper,
Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low}); Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
low = next_low; low = next_low;
const auto next_high = state.builder.AllocateId(); const auto next_high = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high}); state.builder.AddFunction(
{spv::OpSelect, TypeU32(state), next_high, take_lower, mid, high});
high = next_high; high = next_high;
} }
const auto default_label = state.builder.AllocateId(); const auto default_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
std::vector<uint32_t> labels(image.indirect_resources.size() - 1u); std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
std::vector<uint32_t> switch_words {OpSwitch, selected, default_label}; std::vector<uint32_t> switch_words {spv::OpSwitch, selected, default_label};
for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) { for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
labels[candidate - 1u] = state.builder.AllocateId(); labels[candidate - 1u] = state.builder.AllocateId();
switch_words.push_back(candidate); switch_words.push_back(candidate);
switch_words.push_back(labels[candidate - 1u]); switch_words.push_back(labels[candidate - 1u]);
} }
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); state.builder.AddFunction(
{spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
state.builder.AddFunction(switch_words); state.builder.AddFunction(switch_words);
std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()}; std::vector<uint32_t> phi_words {spv::OpPhi, result_type, state.builder.AllocateId()};
EmitLabel(state, default_label); EmitLabel(state, default_label);
phi_words.push_back(EmitSample(image.indirect_resources[0])); phi_words.push_back(EmitSample(image.indirect_resources[0]));
phi_words.push_back(default_label); phi_words.push_back(default_label);
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) { for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
EmitLabel(state, labels[candidate - 1u]); EmitLabel(state, labels[candidate - 1u]);
phi_words.push_back(EmitSample(image.indirect_resources[candidate])); phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
phi_words.push_back(labels[candidate - 1u]); phi_words.push_back(labels[candidate - 1u]);
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
} }
EmitLabel(state, merge_label); EmitLabel(state, merge_label);
state.builder.AddFunction(phi_words); state.builder.AddFunction(phi_words);
@@ -855,16 +874,16 @@ bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto dimension = image.dimension; const auto dimension = image.dimension;
ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() { ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto pointer_type = const auto pointer_type = state.builder.Type(
state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)}); spv::OpTypePointer, {spv::StorageClassImage, TypeU32(state)});
state.builder.AddFunction( state.builder.AddFunction(
{OpImageTexelPointer, pointer_type, pointer, {spv::OpImageTexelPointer, pointer_type, pointer,
StorageImageDescriptorPointer(state, mem.resource), StorageImageDescriptorPointer(state, mem.resource),
CoordU32(ctx, mem, *address, dimension), ConstantU32(state, 0)}); CoordU32(ctx, mem, *address, dimension), ConstantU32(state, 0)});
const auto old = state.builder.AllocateId(); const auto old = state.builder.AllocateId();
state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer, state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
ConstantU32(state, ScopeDevice), ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, MemorySemanticsNone), ConstantU32(state, spv::MemorySemanticsMaskNone),
ctx.Arg(inst, 2)}); ctx.Arg(inst, 2)});
EmitDeviceAtomicMemoryBarrier(state); EmitDeviceAtomicMemoryBarrier(state);
return old; return old;
@@ -17,6 +17,7 @@
#include <cstring> #include <cstring>
#include <iterator> #include <iterator>
#include <map> #include <map>
#include <spirv/unified1/GLSL.std.450.h>
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include <utility> #include <utility>
@@ -24,291 +25,6 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
enum : uint32_t {
ExecutionModelVertex = 0,
ExecutionModelFragment = 4,
ExecutionModelGLCompute = 5,
ExecutionModeOriginUpperLeft = 7,
ExecutionModeEarlyFragmentTests = 9,
ExecutionModeDepthReplacing = 12,
ExecutionModeLocalSize = 17,
ExecutionModeSignedZeroInfNanPreserve = 4461,
ExecutionModeDerivativeGroupQuadsKHR = 5289,
AddressingModelLogical = 0,
AddressingModelPhysicalStorageBuffer64 = 5348,
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
CapabilityInt64 = 11,
CapabilityInt64Atomics = 12,
CapabilityImageGatherExtended = 25,
CapabilityClipDistance = 32,
CapabilityCullDistance = 33,
CapabilitySampleRateShading = 35,
CapabilitySampled1D = 43,
CapabilityImage1D = 44,
CapabilityImageQuery = 50,
CapabilityStorageImageWriteWithoutFormat = 56,
CapabilityGroupNonUniform = 61,
CapabilityGroupNonUniformBallot = 64,
CapabilityGroupNonUniformShuffle = 65,
CapabilityShaderLayer = 69,
CapabilityShaderViewportIndex = 70,
CapabilitySignedZeroInfNanPreserve = 4466,
CapabilityFragmentBarycentricKHR = 5284,
CapabilityComputeDerivativeGroupQuadsKHR = 5288,
CapabilityPhysicalStorageBufferAddresses = 5347,
StorageClassUniformConstant = 0,
StorageClassInput = 1,
StorageClassOutput = 3,
StorageClassWorkgroup = 4,
StorageClassPrivate = 6,
StorageClassFunction = 7,
StorageClassPushConstant = 9,
StorageClassImage = 11,
StorageClassStorageBuffer = 12,
StorageClassPhysicalStorageBuffer = 5349,
FunctionControlNone = 0,
SelectionControlNone = 0,
LoopControlNone = 0,
};
enum : uint32_t {
DecorationBlock = 2,
DecorationBuiltIn = 11,
DecorationNoPerspective = 13,
DecorationFlat = 14,
DecorationAliased = 20,
DecorationLocation = 30,
DecorationArrayStride = 6,
DecorationBinding = 33,
DecorationDescriptorSet = 34,
DecorationOffset = 35,
DecorationPerVertexKHR = 5285,
};
enum : uint32_t {
BuiltInPosition = 0,
BuiltInPointSize = 1,
BuiltInClipDistance = 3,
BuiltInCullDistance = 4,
BuiltInLayer = 9,
BuiltInViewportIndex = 10,
BuiltInFragCoord = 15,
BuiltInFrontFacing = 17,
BuiltInSampleId = 18,
BuiltInSampleMask = 20,
BuiltInFragDepth = 22,
BuiltInWorkgroupId = 26,
BuiltInLocalInvocationId = 27,
BuiltInGlobalInvocationId = 28,
BuiltInLocalInvocationIndex = 29,
BuiltInSubgroupLocalInvocationId = 41,
BuiltInVertexIndex = 42,
BuiltInInstanceIndex = 43,
BuiltInBaryCoordKHR = 5286,
BuiltInBaryCoordNoPerspKHR = 5287,
};
enum : uint32_t {
Dim1D = 0,
Dim3D = 2,
Dim2D = 1,
ImageFormatUnknown = 0,
ImageFormatRgba32f = 1,
ImageFormatR32ui = 33,
};
enum : uint32_t {
ImageOperandsBiasMask = 0x00000001u,
ImageOperandsLodMask = 0x00000002u,
ImageOperandsGradMask = 0x00000004u,
ImageOperandsOffsetMask = 0x00000010u,
ImageOperandsConstOffsetsMask = 0x00000020u,
ImageOperandsSampleMask = 0x00000040u,
};
enum : uint32_t {
ScopeDevice = 1,
ScopeWorkgroup = 2,
ScopeSubgroup = 3,
MemorySemanticsNone = 0,
MemorySemanticsAcquireRelease = 0x00000008u,
MemorySemanticsUniformMemory = 0x00000040u,
MemorySemanticsWorkgroupMemory = 0x00000100u,
MemorySemanticsImageMemory = 0x00000800u,
};
enum : uint32_t {
OpExtInst = 12,
OpTypeVoid = 19,
OpTypeBool = 20,
OpTypeInt = 21,
OpTypeFloat = 22,
OpTypeVector = 23,
OpTypeImage = 25,
OpTypeSampler = 26,
OpTypeSampledImage = 27,
OpTypeArray = 28,
OpTypeRuntimeArray = 29,
OpTypeStruct = 30,
OpTypePointer = 32,
OpTypeFunction = 33,
OpConstantTrue = 41,
OpConstantFalse = 42,
OpConstant = 43,
OpConstantComposite = 44,
OpUndef = 1,
OpFunction = 54,
OpFunctionParameter = 55,
OpFunctionEnd = 56,
OpFunctionCall = 57,
OpVariable = 59,
OpImageTexelPointer = 60,
OpLoad = 61,
OpStore = 62,
OpAccessChain = 65,
OpArrayLength = 68,
OpDecorate = 71,
OpMemberDecorate = 72,
OpVectorShuffle = 79,
OpCompositeConstruct = 80,
OpCompositeExtract = 81,
OpCopyObject = 83,
OpSampledImage = 86,
OpImageSampleImplicitLod = 87,
OpImageSampleExplicitLod = 88,
OpImageSampleDrefImplicitLod = 89,
OpImageSampleDrefExplicitLod = 90,
OpImageFetch = 95,
OpImageGather = 96,
OpImageDrefGather = 97,
OpImageWrite = 99,
OpImageQuerySizeLod = 103,
OpImageQueryLod = 105,
OpImageQuerySize = 104,
OpImageQueryLevels = 106,
OpConvertFToU = 109,
OpConvertFToS = 110,
OpConvertSToF = 111,
OpConvertUToF = 112,
OpUConvert = 113,
OpConvertUToPtr = 120,
OpBitcast = 124,
OpSNegate = 126,
OpFNegate = 127,
OpIAdd = 128,
OpFAdd = 129,
OpISub = 130,
OpFSub = 131,
OpIMul = 132,
OpFMul = 133,
OpUDiv = 134,
OpUMod = 137,
OpFDiv = 136,
OpIAddCarry = 149,
OpUMulExtended = 151,
OpSMulExtended = 152,
OpAny = 154,
OpAll = 155,
OpLogicalNotEqual = 165,
OpLogicalOr = 166,
OpLogicalAnd = 167,
OpLogicalNot = 168,
OpSelect = 169,
OpIEqual = 170,
OpINotEqual = 171,
OpUGreaterThan = 172,
OpSGreaterThan = 173,
OpUGreaterThanEqual = 174,
OpSGreaterThanEqual = 175,
OpULessThan = 176,
OpSLessThan = 177,
OpULessThanEqual = 178,
OpSLessThanEqual = 179,
OpFOrdEqual = 180,
OpFUnordEqual = 181,
OpFOrdNotEqual = 182,
OpFUnordNotEqual = 183,
OpFOrdLessThan = 184,
OpFUnordLessThan = 185,
OpFOrdGreaterThan = 186,
OpFUnordGreaterThan = 187,
OpFOrdLessThanEqual = 188,
OpFUnordLessThanEqual = 189,
OpFOrdGreaterThanEqual = 190,
OpFUnordGreaterThanEqual = 191,
OpShiftRightLogical = 194,
OpShiftRightArithmetic = 195,
OpShiftLeftLogical = 196,
OpBitwiseOr = 197,
OpBitwiseXor = 198,
OpBitwiseAnd = 199,
OpNot = 200,
OpBitFieldInsert = 201,
OpBitFieldSExtract = 202,
OpBitFieldUExtract = 203,
OpBitReverse = 204,
OpBitCount = 205,
OpControlBarrier = 224,
OpMemoryBarrier = 225,
OpAtomicLoad = 227,
OpAtomicExchange = 229,
OpAtomicCompareExchange = 230,
OpAtomicIAdd = 234,
OpAtomicISub = 235,
OpAtomicSMin = 236,
OpAtomicUMin = 237,
OpAtomicSMax = 238,
OpAtomicUMax = 239,
OpAtomicAnd = 240,
OpAtomicOr = 241,
OpAtomicXor = 242,
OpPhi = 245,
OpLoopMerge = 246,
OpSelectionMerge = 247,
OpLabel = 248,
OpBranch = 249,
OpBranchConditional = 250,
OpSwitch = 251,
OpKill = 252,
OpReturn = 253,
OpReturnValue = 254,
OpGroupNonUniformBallot = 339,
OpGroupNonUniformBallotFindLSB = 343,
OpGroupNonUniformShuffle = 345,
};
enum : uint32_t {
MemoryAccessAlignedMask = 0x2,
};
enum : uint32_t {
GlslRoundEven = 2,
GlslTrunc = 3,
GlslFAbs = 4,
GlslFloor = 8,
GlslCeil = 9,
GlslFract = 10,
GlslSin = 13,
GlslCos = 14,
GlslExp2 = 29,
GlslLog2 = 30,
GlslSqrt = 31,
GlslInverseSqrt = 32,
GlslFMin = 37,
GlslFMax = 40,
GlslFClamp = 43,
GlslLdexp = 53,
GlslFma = 50,
GlslPackSnorm2x16 = 56,
GlslPackUnorm2x16 = 57,
GlslPackHalf2x16 = 58,
GlslUnpackUnorm2x16 = 61,
GlslUnpackHalf2x16 = 62,
GlslFindILsb = 73,
GlslFindUMsb = 75,
};
struct InputBinding { struct InputBinding {
IR::StageInputKind kind = IR::StageInputKind::VertexIndex; IR::StageInputKind kind = IR::StageInputKind::VertexIndex;
uint32_t location = 0; uint32_t location = 0;
@@ -339,27 +55,41 @@ struct ImageDimensionInfo {
}; };
constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{ constexpr std::array<ImageDimensionInfo, 7> ImageDimensions {{
{ImageDimension::Dim1D, Dim1D, 1, 1, 0, 0}, {ImageDimension::Dim1D, spv::Dim1D, 1, 1, 0, 0},
{ImageDimension::Dim1DArray, Dim1D, 2, 1, 1, 0}, {ImageDimension::Dim1DArray, spv::Dim1D, 2, 1, 1, 0},
{ImageDimension::Dim2D, Dim2D, 2, 2, 0, 0}, {ImageDimension::Dim2D, spv::Dim2D, 2, 2, 0, 0},
{ImageDimension::Dim2DArray, Dim2D, 3, 2, 1, 0}, {ImageDimension::Dim2DArray, spv::Dim2D, 3, 2, 1, 0},
{ImageDimension::Dim3D, Dim3D, 3, 3, 0, 0}, {ImageDimension::Dim3D, spv::Dim3D, 3, 3, 0, 0},
{ImageDimension::Dim2DMsaa, Dim2D, 2, 2, 0, 1}, {ImageDimension::Dim2DMsaa, spv::Dim2D, 2, 2, 0, 1},
{ImageDimension::Dim2DMsaaArray, Dim2D, 3, 2, 1, 1}, {ImageDimension::Dim2DMsaaArray, spv::Dim2D, 3, 2, 1, 1},
}}; }};
const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension); const ImageDimensionInfo& ImageDimensionInfoFor(ImageDimension dimension);
struct SpirvRequirements {
bool subgroup_ballot = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
bool compute_derivatives = false;
bool image_gather_extended = false;
bool function_lds = false;
bool function_scratch = false;
bool pixel_valid_mask = false;
bool buffer_int64_atomics = false;
};
SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
struct EmitterState { struct EmitterState {
EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_) EmitterState(const IR::Program& program_, ShaderStageInputInfo input_info_)
: builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u), : builder(program_.stage == ShaderType::Mesh ? 0x00010400u : 0x00010300u),
program(program_), input_info(input_info_), requirements(*program_.spirv_requirements) {} program(program_), input_info(input_info_),
requirements(AnalyzeProgramRequirements(program_)) {}
Builder builder; Builder builder;
const IR::Program& program; const IR::Program& program;
ShaderStageInputInfo input_info; ShaderStageInputInfo input_info;
const IR::SpirvRequirements& requirements; const SpirvRequirements requirements;
ShaderType stage = ShaderType::Unknown;
uint32_t lane_count = 1; uint32_t lane_count = 1;
uint32_t lane_half = 0; uint32_t lane_half = 0;
uint32_t storage_buffer_variable = 0; uint32_t storage_buffer_variable = 0;
@@ -409,7 +139,6 @@ uint32_t TypeBool(EmitterState& state);
uint32_t TypeBoolVector(EmitterState& state, uint32_t components); uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
uint32_t TypeU32(EmitterState& state); uint32_t TypeU32(EmitterState& state);
uint32_t TypeU64(EmitterState& state); uint32_t TypeU64(EmitterState& state);
uint32_t TypeDeviceAddress(EmitterState& state);
uint32_t TypeScalarU64(EmitterState& state); uint32_t TypeScalarU64(EmitterState& state);
uint32_t TypeU32Pair(EmitterState& state); uint32_t TypeU32Pair(EmitterState& state);
uint32_t TypeI32(EmitterState& state); uint32_t TypeI32(EmitterState& state);
@@ -420,21 +149,20 @@ uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
uint32_t TypeU32Composite(EmitterState& state, uint32_t components); uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
uint32_t TypeI32Vector(EmitterState& state, uint32_t components); uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
uint32_t TypeF32Vector(EmitterState& state, uint32_t components); uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee); uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee);
uint32_t TypeFunction(EmitterState& state); uint32_t TypeFunction(EmitterState& state);
uint32_t TypeStorageBufferPointer(EmitterState& state); uint32_t TypeStorageBufferPointer(EmitterState& state);
uint32_t TypeStorageBufferElementPointer(EmitterState& state); uint32_t TypeStorageBufferElementPointer(EmitterState& state);
uint32_t TypeStorageBufferU64Pointer(EmitterState& state); uint32_t TypeStorageBufferU64Pointer(EmitterState& state);
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state); uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state);
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
uint32_t TypePhysicalU32Pointer(EmitterState& state); uint32_t TypePhysicalU32Pointer(EmitterState& state);
uint32_t TypePushConstantElementPointer(EmitterState& state); uint32_t TypePushConstantElementPointer(EmitterState& state);
uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords); uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class, uint32_t dwords);
uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class); uint32_t TypeU32ElementPointer(EmitterState& state, spv::StorageClass storage_class);
inline void EmitLabel(EmitterState& state, uint32_t label) { inline void EmitLabel(EmitterState& state, uint32_t label) {
state.current_label = label; state.current_label = label;
state.builder.AddFunction({OpLabel, label}); state.builder.AddFunction({spv::OpLabel, label});
} }
inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) { inline uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
@@ -453,7 +181,7 @@ inline uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint
inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, inline uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition,
uint32_t true_value, uint32_t false_value) { uint32_t true_value, uint32_t false_value) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value}); state.builder.AddFunction({spv::OpSelect, type, result, condition, true_value, false_value});
return result; return result;
} }
@@ -528,7 +256,6 @@ uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind ki
uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind, uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
uint32_t components); uint32_t components);
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program); void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
@@ -570,7 +297,7 @@ uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource);
void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord, void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
uint32_t texel); uint32_t texel);
uint32_t ExecutionModelForStage(ShaderType stage); spv::ExecutionModel ExecutionModelForStage(ShaderType stage);
uint32_t ConstantU32(EmitterState& state, uint32_t value); uint32_t ConstantU32(EmitterState& state, uint32_t value);
@@ -780,11 +507,11 @@ template <typename Fn>
void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) { void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
const auto then_label = state.builder.AllocateId(); const auto then_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label}); state.builder.AddFunction({spv::OpBranchConditional, condition, then_label, merge_label});
EmitLabel(state, then_label); EmitLabel(state, then_label);
fn(); fn();
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
EmitLabel(state, merge_label); EmitLabel(state, merge_label);
} }
@@ -795,19 +522,19 @@ uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition,
const auto then_exit = state.builder.AllocateId(); const auto then_exit = state.builder.AllocateId();
const auto else_label = state.builder.AllocateId(); const auto else_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label}); state.builder.AddFunction({spv::OpBranchConditional, condition, then_label, else_label});
EmitLabel(state, then_label); EmitLabel(state, then_label);
const auto then_value = fn(); const auto then_value = fn();
state.builder.AddFunction({OpBranch, then_exit}); state.builder.AddFunction({spv::OpBranch, then_exit});
EmitLabel(state, then_exit); EmitLabel(state, then_exit);
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
EmitLabel(state, else_label); EmitLabel(state, else_label);
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
EmitLabel(state, merge_label); EmitLabel(state, merge_label);
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpPhi, type, value, then_value, then_exit, default_value, else_label}); {spv::OpPhi, type, value, then_value, then_exit, default_value, else_label});
return value; return value;
} }
@@ -819,12 +546,12 @@ uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&
template <typename Fn> template <typename Fn>
uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) { uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
const auto scope = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; const auto scope = kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
const auto memory = [&] { const auto memory = [&] {
switch (kind) { switch (kind) {
case IR::ResourceKind::Lds: return MemorySemanticsWorkgroupMemory; case IR::ResourceKind::Lds: return spv::MemorySemanticsWorkgroupMemoryMask;
case IR::ResourceKind::Image: return MemorySemanticsImageMemory; case IR::ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask;
default: return MemorySemanticsUniformMemory; default: return spv::MemorySemanticsUniformMemoryMask;
} }
}(); }();
const auto preheader = state.builder.AllocateId(); const auto preheader = state.builder.AllocateId();
@@ -834,27 +561,30 @@ uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind ki
const auto initial = state.builder.AllocateId(); const auto initial = state.builder.AllocateId();
const auto observed = state.builder.AllocateId(); const auto observed = state.builder.AllocateId();
const auto exchanged = state.builder.AllocateId(); const auto exchanged = state.builder.AllocateId();
state.builder.AddFunction({OpBranch, preheader}); state.builder.AddFunction({spv::OpBranch, preheader});
EmitLabel(state, preheader); EmitLabel(state, preheader);
state.builder.AddFunction({OpAtomicLoad, TypeU32(state), initial, pointer, state.builder.AddFunction({spv::OpAtomicLoad, TypeU32(state), initial, pointer,
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)}); ConstantU32(state, scope),
state.builder.AddFunction({OpBranch, header}); ConstantU32(state, spv::MemorySemanticsMaskNone)});
state.builder.AddFunction({spv::OpBranch, header});
EmitLabel(state, header); EmitLabel(state, header);
state.builder.AddFunction( state.builder.AddFunction(
{OpPhi, TypeU32(state), observed, initial, preheader, exchanged, cont}); {spv::OpPhi, TypeU32(state), observed, initial, preheader, exchanged, cont});
const auto next = desired(observed); const auto next = desired(observed);
state.builder.AddFunction({OpAtomicCompareExchange, TypeU32(state), exchanged, pointer, state.builder.AddFunction({spv::OpAtomicCompareExchange, TypeU32(state), exchanged, pointer,
ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone), ConstantU32(state, scope),
ConstantU32(state, MemorySemanticsNone), next, observed}); ConstantU32(state, spv::MemorySemanticsMaskNone),
ConstantU32(state, spv::MemorySemanticsMaskNone), next, observed});
const auto success = state.builder.AllocateId(); const auto success = state.builder.AllocateId();
state.builder.AddFunction({OpIEqual, TypeBool(state), success, exchanged, observed}); state.builder.AddFunction({spv::OpIEqual, TypeBool(state), success, exchanged, observed});
state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone}); state.builder.AddFunction({spv::OpLoopMerge, merge, cont, spv::LoopControlMaskNone});
state.builder.AddFunction({OpBranchConditional, success, merge, cont}); state.builder.AddFunction({spv::OpBranchConditional, success, merge, cont});
EmitLabel(state, cont); EmitLabel(state, cont);
state.builder.AddFunction({OpBranch, header}); state.builder.AddFunction({spv::OpBranch, header});
EmitLabel(state, merge); EmitLabel(state, merge);
state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope), state.builder.AddFunction(
ConstantU32(state, MemorySemanticsAcquireRelease | memory)}); {spv::OpMemoryBarrier, ConstantU32(state, scope),
ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | memory)});
return observed; return observed;
} }
@@ -8,20 +8,20 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace { namespace {
uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
return Binary(state, OpLogicalAnd, TypeBool(state), lhs, rhs); return Binary(state, spv::OpLogicalAnd, TypeBool(state), lhs, rhs);
} }
uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t target, uint32_t EmitDsMaskedLaneRead(EmitterState& state, uint32_t source, uint32_t target,
uint32_t exec) { uint32_t exec) {
if (state.lane_count == 2) { if (state.lane_count == 2) {
target = Binary(state, OpBitwiseAnd, TypeU32(state), target, ConstantU32(state, 31)); target = Binary(state, spv::OpBitwiseAnd, TypeU32(state), target, ConstantU32(state, 31));
} }
const auto shuffled = state.builder.AllocateId(); const auto shuffled = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled, state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), shuffled,
ConstantU32(state, ScopeSubgroup), source, target}); ConstantU32(state, spv::ScopeSubgroup), source, target});
const auto source_exec = state.builder.AllocateId(); const auto source_exec = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec, state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeBool(state), source_exec,
ConstantU32(state, ScopeSubgroup), exec, target}); ConstantU32(state, spv::ScopeSubgroup), exec, target});
const auto source_active = const auto source_active =
AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target)); AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target));
return Select(state, TypeU32(state), source_active, shuffled, ConstantU32(state, 0)); return Select(state, TypeU32(state), source_active, shuffled, ConstantU32(state, 0));
@@ -34,12 +34,12 @@ uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR
const uint32_t stride = packed & 0x3fffu; const uint32_t stride = packed & 0x3fffu;
const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u; const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u;
if (((packed >> 20u) & 1u) != 0u) { if (((packed >> 20u) & 1u) != 0u) {
const auto lane = Binary(state, OpBitwiseAnd, TypeU32(state), const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63)); EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63));
index = Binary(state, OpIAdd, TypeU32(state), index, lane); index = Binary(state, spv::OpIAdd, TypeU32(state), index, lane);
} }
if (mem.offset != 0u) { if (mem.offset != 0u) {
offset = Binary(state, OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset)); offset = Binary(state, spv::OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset));
} }
uint32_t address = 0; uint32_t address = 0;
@@ -48,32 +48,32 @@ uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR
address = offset; address = offset;
} else { } else {
const auto indexed = stride == 1u ? index const auto indexed = stride == 1u ? index
: Binary(state, OpIMul, TypeU32(state), index, : Binary(state, spv::OpIMul, TypeU32(state), index,
ConstantU32(state, stride)); ConstantU32(state, stride));
address = Binary(state, OpIAdd, TypeU32(state), indexed, offset); address = Binary(state, spv::OpIAdd, TypeU32(state), indexed, offset);
} }
} else { } else {
const uint32_t stride_enum = (packed >> 16u) & 3u; const uint32_t stride_enum = (packed >> 16u) & 3u;
const uint32_t index_stride = 8u << stride_enum; const uint32_t index_stride = 8u << stride_enum;
const auto index_msb = Binary(state, OpShiftRightLogical, TypeU32(state), index, const auto index_msb = Binary(state, spv::OpShiftRightLogical, TypeU32(state), index,
ConstantU32(state, stride_enum + 3u)); ConstantU32(state, stride_enum + 3u));
const auto index_lsb = Binary(state, OpBitwiseAnd, TypeU32(state), index, const auto index_lsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), index,
ConstantU32(state, index_stride - 1u)); ConstantU32(state, index_stride - 1u));
const auto offset_msb = const auto offset_msb =
Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u)); Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u));
const auto offset_lsb = const auto offset_lsb =
Binary(state, OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u)); Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u));
const auto indexed_msb = stride == 1u ? index_msb const auto indexed_msb = stride == 1u ? index_msb
: Binary(state, OpIMul, TypeU32(state), index_msb, : Binary(state, spv::OpIMul, TypeU32(state),
ConstantU32(state, stride)); index_msb, ConstantU32(state, stride));
const auto msb = Binary(state, OpIMul, TypeU32(state), const auto msb = Binary(state, spv::OpIMul, TypeU32(state),
Binary(state, OpIAdd, TypeU32(state), indexed_msb, offset_msb), Binary(state, spv::OpIAdd, TypeU32(state), indexed_msb, offset_msb),
ConstantU32(state, index_stride)); ConstantU32(state, index_stride));
const auto lsb = Binary( const auto lsb = Binary(state, spv::OpIAdd, TypeU32(state),
state, OpIAdd, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index_lsb,
Binary(state, OpShiftLeftLogical, TypeU32(state), index_lsb, ConstantU32(state, 2u)), ConstantU32(state, 2u)),
offset_lsb); offset_lsb);
address = Binary(state, OpIAdd, TypeU32(state), msb, lsb); address = Binary(state, spv::OpIAdd, TypeU32(state), msb, lsb);
} }
const auto soffset_value = inst.Arg(3).Resolve(); const auto soffset_value = inst.Arg(3).Resolve();
@@ -81,15 +81,16 @@ uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR
soffset_value.U32() == 0u) { soffset_value.U32() == 0u) {
return address; return address;
} }
return Binary(state, OpIAdd, TypeU32(state), address, soffset); return Binary(state, spv::OpIAdd, TypeU32(state), address, soffset);
} }
uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low, uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low,
uint32_t add_high, uint32_t& out_high) { uint32_t add_high, uint32_t& out_high) {
const auto result = Binary(state, OpIAdd, TypeU32(state), low, add_low); const auto result = Binary(state, spv::OpIAdd, TypeU32(state), low, add_low);
const auto carry = Binary(state, OpULessThan, TypeBool(state), result, low); const auto carry = Binary(state, spv::OpULessThan, TypeBool(state), result, low);
out_high = out_high =
Binary(state, OpIAdd, TypeU32(state), Binary(state, OpIAdd, TypeU32(state), high, add_high), Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, spv::OpIAdd, TypeU32(state), high, add_high),
Select(state, TypeU32(state), carry, ConstantU32(state, 1), ConstantU32(state, 0))); Select(state, TypeU32(state), carry, ConstantU32(state, 1), ConstantU32(state, 0)));
return result; return result;
} }
@@ -101,30 +102,29 @@ uint32_t ScratchByteAddress(ValueEmitContext& ctx, const IR::MemoryInfo& mem, ui
const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate)); const auto immediate_low = ConstantU32(state, static_cast<uint32_t>(immediate));
const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u); const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
low = AddU64Low(state, low, high, immediate_low, immediate_high, high); low = AddU64Low(state, low, high, immediate_low, immediate_high, high);
const auto valid = const auto valid = Binary(state, spv::OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
Binary(state, OpIEqual, TypeBool(state), high, ConstantU32(state, 0));
return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX)); return Select(state, TypeU32(state), valid, low, ConstantU32(state, UINT32_MAX));
} }
uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) { uint32_t ConstantDeviceAddress(EmitterState& state, uint64_t value) {
return state.builder.Constant(OpConstant, TypeDeviceAddress(state), return state.builder.Constant(
{static_cast<uint32_t>(value), spv::OpConstant, TypeScalarU64(state),
static_cast<uint32_t>(value >> 32u)}); {static_cast<uint32_t>(value), static_cast<uint32_t>(value >> 32u)});
} }
uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) { uint32_t DeviceAddressFromWords(EmitterState& state, uint32_t low, uint32_t high) {
const auto low64 = Unary(state, OpUConvert, TypeDeviceAddress(state), low); const auto low64 = Unary(state, spv::OpUConvert, TypeScalarU64(state), low);
const auto high64 = Binary(state, OpShiftLeftLogical, TypeDeviceAddress(state), const auto high64 = Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state),
Unary(state, OpUConvert, TypeDeviceAddress(state), high), Unary(state, spv::OpUConvert, TypeScalarU64(state), high),
ConstantDeviceAddress(state, 32)); ConstantDeviceAddress(state, 32));
return Binary(state, OpBitwiseOr, TypeDeviceAddress(state), low64, high64); return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64);
} }
uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
auto& state = ctx.state; auto& state = ctx.state;
auto low = ctx.Arg(inst, 1); auto low = ctx.Arg(inst, 1);
if (mem.kind == IR::ResourceKind::ScalarAddress) { if (mem.kind == IR::ResourceKind::ScalarAddress) {
low = Binary(state, OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u)); low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), low, ConstantU32(state, ~3u));
} }
uint32_t address = 0; uint32_t address = 0;
if (mem.address_is_full) { if (mem.address_is_full) {
@@ -137,8 +137,8 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
return ConstantDeviceAddress(state, 0); return ConstantDeviceAddress(state, 0);
} }
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1)); const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
address = Binary(state, OpIAdd, TypeDeviceAddress(state), base, address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
Unary(state, OpUConvert, TypeDeviceAddress(state), low)); Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
} }
auto immediate = static_cast<int32_t>(mem.offset); auto immediate = static_cast<int32_t>(mem.offset);
if (mem.kind == IR::ResourceKind::ScalarAddress) { if (mem.kind == IR::ResourceKind::ScalarAddress) {
@@ -146,51 +146,51 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
} }
return immediate == 0 return immediate == 0
? address ? address
: Binary(state, OpIAdd, TypeDeviceAddress(state), address, : Binary(state, spv::OpIAdd, TypeScalarU64(state), address,
ConstantDeviceAddress(state, ConstantDeviceAddress(
static_cast<uint64_t>(static_cast<int64_t>(immediate)))); state, static_cast<uint64_t>(static_cast<int64_t>(immediate))));
} }
uint32_t FaultElementPointer(EmitterState& state, uint32_t index) { uint32_t FaultElementPointer(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.fault_buffer_variable, ConstantU32(state, 0), index}); state.fault_buffer_variable, ConstantU32(state, 0), index});
return pointer; return pointer;
} }
void RecordBdaFault(EmitterState& state, uint32_t page) { void RecordBdaFault(EmitterState& state, uint32_t page) {
const auto word = Binary(state, OpShiftRightLogical, TypeU32(state), page, const auto word =
ConstantU32(state, 5)); Binary(state, spv::OpShiftRightLogical, TypeU32(state), page, ConstantU32(state, 5));
const auto bit = Binary( const auto bit =
state, OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1), Binary(state, spv::OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1),
Binary(state, OpBitwiseAnd, TypeU32(state), page, ConstantU32(state, 31))); Binary(state, spv::OpBitwiseAnd, TypeU32(state), page, ConstantU32(state, 31)));
const auto pointer = FaultElementPointer(state, word); const auto pointer = FaultElementPointer(state, word);
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
state.builder.AddFunction( state.builder.AddFunction(
{OpStore, pointer, Binary(state, OpBitwiseOr, TypeU32(state), value, bit)}); {spv::OpStore, pointer, Binary(state, spv::OpBitwiseOr, TypeU32(state), value, bit)});
} }
uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) { uint32_t GetBdaPointer(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state; auto& state = ctx.state;
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpFunctionCall, TypeDeviceAddress(state), result, state.bda_pointer_function, address}); {spv::OpFunctionCall, TypeScalarU64(state), result, state.bda_pointer_function, address});
return result; return result;
} }
uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) { uint32_t LoadBdaDword(ValueEmitContext& ctx, uint32_t address) {
auto& state = ctx.state; auto& state = ctx.state;
const auto bda = GetBdaPointer(ctx, address); const auto bda = GetBdaPointer(ctx, address);
const auto present = Binary(state, OpINotEqual, TypeBool(state), bda, const auto present =
ConstantDeviceAddress(state, 0)); Binary(state, spv::OpINotEqual, TypeBool(state), bda, ConstantDeviceAddress(state, 0));
return EmitValueOrZeroIfCondition(state, present, [&]() { return EmitValueOrZeroIfCondition(state, present, [&]() {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, bda}); {spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, bda});
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer,
{OpLoad, TypeU32(state), value, pointer, MemoryAccessAlignedMask, sizeof(uint32_t)}); spv::MemoryAccessAlignedMask, sizeof(uint32_t)});
return value; return value;
}); });
} }
@@ -201,37 +201,34 @@ uint32_t LoadBda(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryIn
const auto address = GuestAddress(ctx, inst, mem); const auto address = GuestAddress(ctx, inst, mem);
const auto active = ctx.Arg(inst, inst.NumArgs() - 1); const auto active = ctx.Arg(inst, inst.NumArgs() - 1);
return EmitValueOrZeroIfCondition(state, active, [&]() { return EmitValueOrZeroIfCondition(state, active, [&]() {
const auto aligned = Binary(state, OpBitwiseAnd, TypeDeviceAddress(state), address, const auto aligned = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address,
ConstantDeviceAddress(state, ~uint64_t {3})); ConstantDeviceAddress(state, ~uint64_t {3}));
const auto first = LoadBdaDword(ctx, aligned); const auto first = LoadBdaDword(ctx, aligned);
const auto byte = Binary(state, OpBitwiseAnd, TypeU32(state), const auto byte =
Unary(state, OpUConvert, TypeU32(state), address), Binary(state, spv::OpBitwiseAnd, TypeU32(state),
ConstantU32(state, 3)); Unary(state, spv::OpUConvert, TypeU32(state), address), ConstantU32(state, 3));
const auto crosses = bits == 8u const auto crosses =
? ConstantBool(state, false) bits == 8u ? ConstantBool(state, false)
: Binary(state, bits == 16u ? OpUGreaterThan : OpINotEqual, : Binary(state, bits == 16u ? spv::OpUGreaterThan : spv::OpINotEqual,
TypeBool(state), byte, TypeBool(state), byte, ConstantU32(state, bits == 16u ? 2u : 0u));
ConstantU32(state, bits == 16u ? 2u : 0u));
const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() { const auto second = EmitValueOrZeroIfCondition(state, crosses, [&]() {
return LoadBdaDword( return LoadBdaDword(ctx, Binary(state, spv::OpIAdd, TypeScalarU64(state), aligned,
ctx, Binary(state, OpIAdd, TypeDeviceAddress(state), aligned,
ConstantDeviceAddress(state, sizeof(uint32_t)))); ConstantDeviceAddress(state, sizeof(uint32_t))));
}); });
const auto shift = Binary(state, OpShiftLeftLogical, TypeU32(state), byte, const auto shift =
ConstantU32(state, 3)); Binary(state, spv::OpShiftLeftLogical, TypeU32(state), byte, ConstantU32(state, 3));
const auto upper_shift = Binary( const auto upper_shift =
state, OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state),
Binary(state, OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state),
Binary(state, OpISub, TypeU32(state), ConstantU32(state, 4), byte), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 4), byte),
ConstantU32(state, 3)), ConstantU32(state, 3)),
ConstantU32(state, 3)); ConstantU32(state, 3));
const auto merged = Binary( const auto merged =
state, OpBitwiseOr, TypeU32(state), Binary(state, spv::OpBitwiseOr, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), first, shift), Binary(state, spv::OpShiftRightLogical, TypeU32(state), first, shift),
Binary(state, OpShiftLeftLogical, TypeU32(state), second, upper_shift)); Binary(state, spv::OpShiftLeftLogical, TypeU32(state), second, upper_shift));
return bits == 32u return bits == 32u ? merged
? merged : Binary(state, spv::OpBitwiseAnd, TypeU32(state), merged,
: Binary(state, OpBitwiseAnd, TypeU32(state), merged,
ConstantU32(state, bits == 8u ? 0xffu : 0xffffu)); ConstantU32(state, bits == 8u ? 0xffu : 0xffffu));
}); });
} }
@@ -245,7 +242,7 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
if (mem.offset == 0u) { if (mem.offset == 0u) {
return ctx.Arg(inst, 0); return ctx.Arg(inst, 0);
} }
return Binary(ctx.state, OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0), return Binary(ctx.state, spv::OpIAdd, TypeU32(ctx.state), ctx.Arg(inst, 0),
ConstantU32(ctx.state, mem.offset)); ConstantU32(ctx.state, mem.offset));
} }
if (mem.kind != IR::ResourceKind::Scratch) { if (mem.kind != IR::ResourceKind::Scratch) {
@@ -255,8 +252,8 @@ uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memo
} }
uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
return Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), ByteAddress(ctx, inst, mem), return Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state),
ConstantU32(ctx.state, 2)); ByteAddress(ctx, inst, mem), ConstantU32(ctx.state, 2));
} }
struct PreparedMemoryElement { struct PreparedMemoryElement {
@@ -296,7 +293,7 @@ uint32_t LoadSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const
const MemoryResourceAccess& resource, uint32_t bits, const MemoryResourceAccess& resource, uint32_t bits,
bool sign_extend) { bool sign_extend) {
const auto address = ByteAddress(ctx, inst, mem); const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2)); ConstantU32(ctx.state, 2));
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index); const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
return EmitValueOrZeroIfCondition( return EmitValueOrZeroIfCondition(
@@ -309,25 +306,25 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
uint32_t index) { uint32_t index) {
const auto value = ctx.state.builder.AllocateId(); const auto value = ctx.state.builder.AllocateId();
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index); const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), value, pointer}); ctx.state.builder.AddFunction({spv::OpLoad, TypeU32(ctx.state), value, pointer});
return value; return value;
} }
uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t LoadSubwordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource,
uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) { uint32_t address, uint32_t index, uint32_t bits, bool sign_extend) {
const auto word = LoadWordInBounds(ctx, resource, index); const auto word = LoadWordInBounds(ctx, resource, index);
const auto byte = const auto byte = Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3)); ConstantU32(ctx.state, 3));
const auto shift = const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), byte,
Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), byte, ConstantU32(ctx.state, 3)); ConstantU32(ctx.state, 3));
const auto value = const auto value =
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state),
Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), word, shift), Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), word, shift),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)); ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
if (!sign_extend) return value; if (!sign_extend) return value;
const auto left = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), value, const auto left = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state), value,
ConstantU32(ctx.state, 32u - bits)); ConstantU32(ctx.state, 32u - bits));
return Binary(ctx.state, OpShiftRightArithmetic, TypeU32(ctx.state), left, return Binary(ctx.state, spv::OpShiftRightArithmetic, TypeU32(ctx.state), left,
ConstantU32(ctx.state, 32u - bits)); ConstantU32(ctx.state, 32u - bits));
} }
@@ -402,14 +399,15 @@ uint32_t LoadFormattedComponent(ValueEmitContext& ctx, const IR::MemoryInfo& mem
const auto type = const auto type =
IsSignedFormatComponent(info.type) ? TypeI32(ctx.state) : TypeU32(ctx.state); IsSignedFormatComponent(info.type) ? TypeI32(ctx.state) : TypeU32(ctx.state);
const auto source_value = const auto source_value =
type == TypeI32(ctx.state) ? Unary(ctx.state, OpBitcast, type, raw) : raw; type == TypeI32(ctx.state) ? Unary(ctx.state, spv::OpBitcast, type, raw) : raw;
const auto extracted = ctx.state.builder.AllocateId(); const auto extracted = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type, {IsSignedFormatComponent(info.type) ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract,
extracted, source_value, ConstantU32(ctx.state, info.component_bit_offset[component]), type, extracted, source_value,
ConstantU32(ctx.state, info.component_bit_offset[component]),
ConstantU32(ctx.state, bits)}); ConstantU32(ctx.state, bits)});
raw = type == TypeI32(ctx.state) raw = type == TypeI32(ctx.state)
? Unary(ctx.state, OpBitcast, TypeU32(ctx.state), extracted) ? Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state), extracted)
: extracted; : extracted;
} else if (bits == 32u) { } else if (bits == 32u) {
raw = load_word(component); raw = load_word(component);
@@ -449,26 +447,26 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t address, uint32_t index, const MemoryResourceAccess& resource, uint32_t address, uint32_t index,
uint32_t bits, uint32_t data) { uint32_t bits, uint32_t data) {
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index); const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
const auto shift = Binary( const auto shift = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), address,
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), address, ConstantU32(ctx.state, 3)), ConstantU32(ctx.state, 3)),
ConstantU32(ctx.state, 3)); ConstantU32(ctx.state, 3));
const auto mask = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), const auto mask = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift); ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
const auto value = Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), const auto value = Binary(ctx.state, spv::OpShiftLeftLogical, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), data, Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), data,
ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)), ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
shift); shift);
const auto merge = [&](uint32_t old) { const auto merge = [&](uint32_t old) {
return Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), return Binary(ctx.state, spv::OpBitwiseOr, TypeU32(ctx.state),
Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), old, Binary(ctx.state, spv::OpBitwiseAnd, TypeU32(ctx.state), old,
Unary(ctx.state, OpNot, TypeU32(ctx.state), mask)), Unary(ctx.state, spv::OpNot, TypeU32(ctx.state), mask)),
value); value);
}; };
if (mem.kind == IR::ResourceKind::Scratch) { if (mem.kind == IR::ResourceKind::Scratch) {
const auto old = ctx.state.builder.AllocateId(); const auto old = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpLoad, TypeU32(ctx.state), old, pointer}); ctx.state.builder.AddFunction({spv::OpLoad, TypeU32(ctx.state), old, pointer});
ctx.state.builder.AddFunction({OpStore, pointer, merge(old)}); ctx.state.builder.AddFunction({spv::OpStore, pointer, merge(old)});
} else { } else {
AtomicUpdate(ctx.state, pointer, mem.kind, merge); AtomicUpdate(ctx.state, pointer, mem.kind, merge);
} }
@@ -477,7 +475,7 @@ void StoreSubwordInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, void StoreSubwordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) { const MemoryResourceAccess& resource, uint32_t bits, uint32_t data) {
const auto address = ByteAddress(ctx, inst, mem); const auto address = ByteAddress(ctx, inst, mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), address, const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state), address,
ConstantU32(ctx.state, 2)); ConstantU32(ctx.state, 2));
const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index); const auto index = EmitMemoryElementIndex(ctx.state, resource, raw_index);
EmitIfCondition( EmitIfCondition(
@@ -496,17 +494,16 @@ void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem, void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
const MemoryResourceAccess& resource, uint32_t data) { const MemoryResourceAccess& resource, uint32_t data) {
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem)); const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
EmitIfCondition( EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpStore, EmitMemoryElementPointer(ctx.state, resource, index), data}); {spv::OpStore, EmitMemoryElementPointer(ctx.state, resource, index), data});
}); });
} }
void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index, void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index,
uint32_t data) { uint32_t data) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpStore, EmitMemoryElementPointer(ctx.state, resource, index), data}); {spv::OpStore, EmitMemoryElementPointer(ctx.state, resource, index), data});
} }
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) { void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
@@ -543,29 +540,29 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
uint32_t SpirvAtomicOpcode(IR::ValueOpcode opcode) { uint32_t SpirvAtomicOpcode(IR::ValueOpcode opcode) {
switch (opcode) { switch (opcode) {
case IR::ValueOpcode::BufferAtomicCmpSwap32: return OpAtomicCompareExchange; case IR::ValueOpcode::BufferAtomicCmpSwap32: return spv::OpAtomicCompareExchange;
case IR::ValueOpcode::BufferAtomicSwap32: case IR::ValueOpcode::BufferAtomicSwap32:
case IR::ValueOpcode::BufferAtomicSwap64: case IR::ValueOpcode::BufferAtomicSwap64:
case IR::ValueOpcode::SharedAtomicSwap32: return OpAtomicExchange; case IR::ValueOpcode::SharedAtomicSwap32: return spv::OpAtomicExchange;
case IR::ValueOpcode::BufferAtomicIAdd32: case IR::ValueOpcode::BufferAtomicIAdd32:
case IR::ValueOpcode::SharedAtomicIAdd32: return OpAtomicIAdd; case IR::ValueOpcode::SharedAtomicIAdd32: return spv::OpAtomicIAdd;
case IR::ValueOpcode::BufferAtomicISub32: case IR::ValueOpcode::BufferAtomicISub32:
case IR::ValueOpcode::SharedAtomicISub32: return OpAtomicISub; case IR::ValueOpcode::SharedAtomicISub32: return spv::OpAtomicISub;
case IR::ValueOpcode::BufferAtomicSMin32: case IR::ValueOpcode::BufferAtomicSMin32:
case IR::ValueOpcode::SharedAtomicSMin32: return OpAtomicSMin; case IR::ValueOpcode::SharedAtomicSMin32: return spv::OpAtomicSMin;
case IR::ValueOpcode::BufferAtomicUMin32: case IR::ValueOpcode::BufferAtomicUMin32:
case IR::ValueOpcode::SharedAtomicUMin32: return OpAtomicUMin; case IR::ValueOpcode::SharedAtomicUMin32: return spv::OpAtomicUMin;
case IR::ValueOpcode::BufferAtomicSMax32: case IR::ValueOpcode::BufferAtomicSMax32:
case IR::ValueOpcode::SharedAtomicSMax32: return OpAtomicSMax; case IR::ValueOpcode::SharedAtomicSMax32: return spv::OpAtomicSMax;
case IR::ValueOpcode::BufferAtomicUMax32: case IR::ValueOpcode::BufferAtomicUMax32:
case IR::ValueOpcode::SharedAtomicUMax32: return OpAtomicUMax; case IR::ValueOpcode::SharedAtomicUMax32: return spv::OpAtomicUMax;
case IR::ValueOpcode::BufferAtomicAnd32: case IR::ValueOpcode::BufferAtomicAnd32:
case IR::ValueOpcode::SharedAtomicAnd32: return OpAtomicAnd; case IR::ValueOpcode::SharedAtomicAnd32: return spv::OpAtomicAnd;
case IR::ValueOpcode::BufferAtomicOr32: case IR::ValueOpcode::BufferAtomicOr32:
case IR::ValueOpcode::BufferAtomicOr64: case IR::ValueOpcode::BufferAtomicOr64:
case IR::ValueOpcode::SharedAtomicOr32: return OpAtomicOr; case IR::ValueOpcode::SharedAtomicOr32: return spv::OpAtomicOr;
case IR::ValueOpcode::BufferAtomicXor32: case IR::ValueOpcode::BufferAtomicXor32:
case IR::ValueOpcode::SharedAtomicXor32: return OpAtomicXor; case IR::ValueOpcode::SharedAtomicXor32: return spv::OpAtomicXor;
default: return 0; default: return 0;
} }
} }
@@ -577,14 +574,15 @@ uint32_t EmitAtomicOperation(ValueEmitContext& ctx, const IR::Inst& inst, uint32
const auto desired = ctx.Arg(inst, inst.NumArgs() - 3); const auto desired = ctx.Arg(inst, inst.NumArgs() - 3);
const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2); const auto comparator = ctx.Arg(inst, inst.NumArgs() - 2);
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer, ConstantU32(ctx.state, scope), {spv::OpAtomicCompareExchange, TypeU32(ctx.state), old, pointer,
ConstantU32(ctx.state, MemorySemanticsNone), ConstantU32(ctx.state, scope), ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
ConstantU32(ctx.state, MemorySemanticsNone), desired, comparator}); ConstantU32(ctx.state, spv::MemorySemanticsMaskNone), desired, comparator});
} else { } else {
const auto value = ctx.Arg(inst, inst.NumArgs() - 2); const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction({SpirvAtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old,
{SpirvAtomicOpcode(inst.GetOpcode()), TypeU32(ctx.state), old, pointer, pointer, ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, scope), ConstantU32(ctx.state, MemorySemanticsNone), value}); ConstantU32(ctx.state, spv::MemorySemanticsMaskNone),
value});
} }
return old; return old;
} }
@@ -603,11 +601,13 @@ uint32_t EmitAtomicAccess(ValueEmitContext& ctx, const IR::Inst& inst,
uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) { uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) { return EmitAtomicAccess(ctx, inst, mem, [&](uint32_t pointer) {
const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice; const auto scope =
mem.kind == IR::ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice;
const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); const auto old = EmitAtomicOperation(ctx, inst, pointer, scope);
if (mem.kind == IR::ResourceKind::Lds) { if (mem.kind == IR::ResourceKind::Lds) {
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory; const auto semantics =
ctx.state.builder.AddFunction({OpMemoryBarrier, ConstantU32(ctx.state, scope), spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask;
ctx.state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(ctx.state, scope),
ConstantU32(ctx.state, semantics)}); ConstantU32(ctx.state, semantics)});
} else { } else {
EmitDeviceAtomicMemoryBarrier(ctx.state); EmitDeviceAtomicMemoryBarrier(ctx.state);
@@ -629,17 +629,17 @@ uint32_t EmitAtomicUpdate(ValueEmitContext& ctx, const IR::Inst& inst,
uint32_t AtomicIncrement(EmitterState& state, uint32_t old, uint32_t limit) { uint32_t AtomicIncrement(EmitterState& state, uint32_t old, uint32_t limit) {
// old >= limit ? 0 : old + 1 (unsigned). // old >= limit ? 0 : old + 1 (unsigned).
const auto wrap = Binary(state, OpUGreaterThanEqual, TypeBool(state), old, limit); const auto wrap = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), old, limit);
const auto next = Binary(state, OpIAdd, TypeU32(state), old, ConstantU32(state, 1)); const auto next = Binary(state, spv::OpIAdd, TypeU32(state), old, ConstantU32(state, 1));
return Select(state, TypeU32(state), wrap, ConstantU32(state, 0), next); return Select(state, TypeU32(state), wrap, ConstantU32(state, 0), next);
} }
uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) { uint32_t AtomicDecrement(EmitterState& state, uint32_t old, uint32_t limit) {
// old == 0 || old > limit ? limit : old - 1 (unsigned). // old == 0 || old > limit ? limit : old - 1 (unsigned).
const auto zero = Binary(state, OpIEqual, TypeBool(state), old, ConstantU32(state, 0)); const auto zero = Binary(state, spv::OpIEqual, TypeBool(state), old, ConstantU32(state, 0));
const auto above = Binary(state, OpUGreaterThan, TypeBool(state), old, limit); const auto above = Binary(state, spv::OpUGreaterThan, TypeBool(state), old, limit);
const auto wrap = Binary(state, OpLogicalOr, TypeBool(state), zero, above); const auto wrap = Binary(state, spv::OpLogicalOr, TypeBool(state), zero, above);
const auto next = Binary(state, OpISub, TypeU32(state), old, ConstantU32(state, 1)); const auto next = Binary(state, spv::OpISub, TypeU32(state), old, ConstantU32(state, 1));
return Select(state, TypeU32(state), wrap, limit, next); return Select(state, TypeU32(state), wrap, limit, next);
} }
@@ -650,24 +650,24 @@ uint32_t EmitBufferAtomic64(ValueEmitContext& ctx, const IR::Inst& inst,
state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() { state, ctx.Arg(inst, inst.NumArgs() - 1), TypeU64(state), ConstantU64(state, 0), [&]() {
const auto resource = PrepareStorageBufferResourceAccess( const auto resource = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state)); state, mem, state.storage_buffer_u64_variable, TypeStorageBufferU64Pointer(state));
const auto byte_address = Binary(state, OpIAdd, TypeU32(state), const auto byte_address = Binary(state, spv::OpIAdd, TypeU32(state),
ByteAddress(ctx, inst, mem), resource.byte_offset); ByteAddress(ctx, inst, mem), resource.byte_offset);
const auto index = Binary(state, OpShiftRightLogical, TypeU32(state), byte_address, const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte_address,
ConstantU32(state, 3u)); ConstantU32(state, 3u));
return EmitValueOrDefaultIfCondition( return EmitValueOrDefaultIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state),
ConstantU64(state, 0), [&]() { ConstantU64(state, 0), [&]() {
const auto value = Unary(state, OpBitcast, TypeScalarU64(state), const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state),
ctx.Arg(inst, inst.NumArgs() - 2)); ctx.Arg(inst, inst.NumArgs() - 2));
const auto old = state.builder.AllocateId(); const auto old = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old, {SpirvAtomicOpcode(inst.GetOpcode()), TypeScalarU64(state), old,
EmitStorageBufferElementPointer( EmitStorageBufferElementPointer(state, resource, index,
state, resource, index, TypeStorageBufferU64ElementPointer(state)), TypeStorageBufferU64ElementPointer(state)),
ConstantU32(state, ScopeDevice), ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, MemorySemanticsNone), value}); ConstantU32(state, spv::MemorySemanticsMaskNone), value});
EmitDeviceAtomicMemoryBarrier(state); EmitDeviceAtomicMemoryBarrier(state);
return Unary(state, OpBitcast, TypeU64(state), old); return Unary(state, spv::OpBitcast, TypeU64(state), old);
}); });
}); });
} }
@@ -687,22 +687,24 @@ uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR
EmitIfCondition( EmitIfCondition(
ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() { ctx.state, EmitMemoryElementInBounds(ctx.state, access.resource, access.index), [&]() {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)}); {spv::OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 1)});
const auto data = ctx.state.builder.AllocateId(); const auto data = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable}); {spv::OpLoad, TypeU32(ctx.state), data, ctx.scratch_u32_variable});
AtomicUpdate( AtomicUpdate(
ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index), ctx.state, EmitMemoryElementPointer(ctx.state, access.resource, access.index),
mem.kind, [&](uint32_t old) { mem.kind, [&](uint32_t old) {
const auto old_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), old); const auto old_f =
Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), old);
const auto compare_f = const auto compare_f =
Unary(ctx.state, OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2)); Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), ctx.Arg(inst, 2));
const auto data_f = Unary(ctx.state, OpBitcast, TypeF32(ctx.state), data); const auto data_f =
const auto compare = Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), data);
Binary(ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, const auto compare = Binary(
ctx.state, max_value ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan,
TypeBool(ctx.state), max_value ? old_f : compare_f, TypeBool(ctx.state), max_value ? old_f : compare_f,
max_value ? compare_f : old_f); max_value ? compare_f : old_f);
return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), return Unary(ctx.state, spv::OpBitcast, TypeU32(ctx.state),
Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f)); Select(ctx.state, TypeF32(ctx.state), compare, data_f, old_f));
}); });
}); });
@@ -717,12 +719,13 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
} }
const auto m0 = ctx.Arg(inst, 0); const auto m0 = ctx.Arg(inst, 0);
const auto base = const auto base =
Binary(state, OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16)); Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
const auto size = Binary(state, OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); const auto size =
const auto address = Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
Binary(state, OpIAdd, TypeU32(state), base, ConstantU32(state, ctx.Memory(inst).offset)); const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, ctx.Memory(inst).offset));
const auto raw_index = const auto raw_index =
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto mem = ctx.Memory(inst); const auto mem = ctx.Memory(inst);
const auto access = PrepareMemoryResourceAccess(state, mem); const auto access = PrepareMemoryResourceAccess(state, mem);
const auto index = EmitMemoryElementIndex(state, access, raw_index); const auto index = EmitMemoryElementIndex(state, access, raw_index);
@@ -730,42 +733,44 @@ uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool append)
const auto ballot = ctx.Ballot(inst.Arg(1)); const auto ballot = ctx.Ballot(inst.Arg(1));
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
const auto high = state.builder.AllocateId(); const auto high = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
const auto count = const auto count = Binary(state, spv::OpIAdd, TypeU32(state),
Binary(state, OpIAdd, TypeU32(state), Unary(state, OpBitCount, TypeU32(state), low), Unary(state, spv::OpBitCount, TypeU32(state), low),
Unary(state, OpBitCount, TypeU32(state), high)); Unary(state, spv::OpBitCount, TypeU32(state), high));
const auto first = ctx.FirstLane(ballot); const auto first = ctx.FirstLane(ballot);
const auto source_lane = state.lane_count == 2 ? Binary(state, OpBitwiseAnd, TypeU32(state), const auto source_lane =
first, ConstantU32(state, 31)) state.lane_count == 2
? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31))
: first; : first;
const auto is_first = const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
Binary(state, OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), source_lane); EmitSubgroupLocalInvocationId(state), source_lane);
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index); const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
const auto m0_bounds = const auto m0_bounds =
mem.kind == IR::ResourceKind::Gds mem.kind == IR::ResourceKind::Gds
? Binary(state, OpINotEqual, TypeBool(state), size, ConstantU32(state, 0)) ? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
: Binary(state, OpULessThan, TypeBool(state), : Binary(state, spv::OpULessThan, TypeBool(state),
ConstantU32(state, ctx.Memory(inst).offset + 3u), size); ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
const auto condition = AndCondition( const auto condition = AndCondition(
state, is_first, state, is_first,
AndCondition(state, AndCondition(state,
state.lane_count == 2 state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
? Binary(state, OpINotEqual, TypeBool(state), count, ConstantU32(state, 0)) ConstantU32(state, 0))
: exec, : exec,
AndCondition(state, storage_bounds, m0_bounds))); AndCondition(state, storage_bounds, m0_bounds)));
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{append ? OpAtomicIAdd : OpAtomicISub, TypeU32(state), value, {append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value,
EmitMemoryElementPointer(state, access, index), EmitMemoryElementPointer(state, access, index),
ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? ScopeDevice : ScopeWorkgroup), ConstantU32(state, mem.kind == IR::ResourceKind::Gds ? spv::ScopeDevice
ConstantU32(state, MemorySemanticsNone), count}); : spv::ScopeWorkgroup),
ConstantU32(state, spv::MemorySemanticsMaskNone), count});
return value; return value;
}); });
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), result, state.builder.AddFunction({spv::OpGroupNonUniformShuffle, TypeU32(state), result,
ConstantU32(state, ScopeSubgroup), atomic, source_lane}); ConstantU32(state, spv::ScopeSubgroup), atomic, source_lane});
return result; return result;
} }
@@ -818,7 +823,7 @@ PreparedFormattedMemory PrepareFormattedMemory(ValueEmitContext& ctx, const IR::
if (reused) continue; if (reused) continue;
const auto component_mem = RebaseFormattedComponent(mem, plan.info, component); const auto component_mem = RebaseFormattedComponent(mem, plan.info, component);
plan.addresses[component] = ByteAddress(ctx, inst, component_mem); plan.addresses[component] = ByteAddress(ctx, inst, component_mem);
const auto raw_index = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), const auto raw_index = Binary(ctx.state, spv::OpShiftRightLogical, TypeU32(ctx.state),
plan.addresses[component], ConstantU32(ctx.state, 2)); plan.addresses[component], ConstantU32(ctx.state, 2));
plan.indices[component] = EmitMemoryElementIndex(ctx.state, resource, raw_index); plan.indices[component] = EmitMemoryElementIndex(ctx.state, resource, raw_index);
const auto component_bound = const auto component_bound =
@@ -850,7 +855,8 @@ uint32_t LoadFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
uint32_t ConstructU32Composite(EmitterState& state, uint32_t components, uint32_t ConstructU32Composite(EmitterState& state, uint32_t components,
const std::array<uint32_t, 4>& values) { const std::array<uint32_t, 4>& values) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
std::vector<uint32_t> words {OpCompositeConstruct, TypeU32Composite(state, components), result}; std::vector<uint32_t> words {spv::OpCompositeConstruct, TypeU32Composite(state, components),
result};
words.insert(words.end(), values.begin(), values.begin() + components); words.insert(words.end(), values.begin(), values.begin() + components);
state.builder.AddFunction(words); state.builder.AddFunction(words);
return result; return result;
@@ -925,7 +931,7 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
for (uint32_t component = 0; component < components; component++) { for (uint32_t component = 0; component < components; component++) {
const auto data = state.builder.AllocateId(); const auto data = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpCompositeExtract, TypeU32(state), data, composite, component}); {spv::OpCompositeExtract, TypeU32(state), data, composite, component});
StoreFormattedInBounds(ctx, mem, plan, component, data); StoreFormattedInBounds(ctx, mem, plan, component, data);
} }
}); });
@@ -934,7 +940,7 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
for (uint32_t component = 0; component < components; component++) { for (uint32_t component = 0; component < components; component++) {
const auto data = state.builder.AllocateId(); const auto data = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpCompositeExtract, TypeU32(state), data, composite, component}); {spv::OpCompositeExtract, TypeU32(state), data, composite, component});
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data); StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data);
} }
}); });
@@ -950,11 +956,12 @@ uint32_t LoadWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t co
const auto base = ByteAddress(ctx, inst, mem); const auto base = ByteAddress(ctx, inst, mem);
std::array<uint32_t, 4> values {}; std::array<uint32_t, 4> values {};
for (uint32_t component = 0; component < components; component++) { for (uint32_t component = 0; component < components; component++) {
const auto address = component == 0u ? base const auto address = component == 0u
: Binary(state, OpIAdd, TypeU32(state), base, ? base
: Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, component * 4u)); ConstantU32(state, component * 4u));
const auto raw_index = Binary(state, OpShiftRightLogical, TypeU32(state), address, const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state),
ConstantU32(state, 2)); address, ConstantU32(state, 2));
const auto index = EmitMemoryElementIndex(state, resource, raw_index); const auto index = EmitMemoryElementIndex(state, resource, raw_index);
values[component] = EmitValueOrZeroIfCondition( values[component] = EmitValueOrZeroIfCondition(
state, EmitMemoryElementInBounds(state, resource, index), state, EmitMemoryElementInBounds(state, resource, index),
@@ -972,10 +979,10 @@ void StoreWideShared(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
const auto base = ByteAddress(ctx, inst, mem); const auto base = ByteAddress(ctx, inst, mem);
for (uint32_t component = 0; component < components; component++) { for (uint32_t component = 0; component < components; component++) {
const auto address = component == 0u ? base const auto address = component == 0u ? base
: Binary(state, OpIAdd, TypeU32(state), base, : Binary(state, spv::OpIAdd, TypeU32(state), base,
ConstantU32(state, component * 4u)); ConstantU32(state, component * 4u));
const auto raw_index = const auto raw_index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address,
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); ConstantU32(state, 2));
const auto index = EmitMemoryElementIndex(state, resource, raw_index); const auto index = EmitMemoryElementIndex(state, resource, raw_index);
EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index),
[&]() { [&]() {
@@ -991,52 +998,50 @@ void DefineGetBdaPointer(EmitterState& state) {
if (!state.program.info.uses_dma) { if (!state.program.info.uses_dma) {
return; return;
} }
const auto type = TypeDeviceAddress(state); const auto type = TypeScalarU64(state);
const auto function_type = state.builder.Type(OpTypeFunction, {type, type}); const auto function_type = state.builder.Type(spv::OpTypeFunction, {type, type});
state.bda_pointer_function = state.builder.AllocateId(); state.bda_pointer_function = state.builder.AllocateId();
const auto address = state.builder.AllocateId(); const auto address = state.builder.AllocateId();
const auto entry_label = state.builder.AllocateId(); const auto entry_label = state.builder.AllocateId();
state.builder.AddName(state.bda_pointer_function, "get_bda_pointer"); state.builder.AddName(state.bda_pointer_function, "get_bda_pointer");
state.builder.AddFunction( state.builder.AddFunction({spv::OpFunction, type, state.bda_pointer_function,
{OpFunction, type, state.bda_pointer_function, FunctionControlNone, function_type}); spv::FunctionControlMaskNone, function_type});
state.builder.AddFunction({OpFunctionParameter, type, address}); state.builder.AddFunction({spv::OpFunctionParameter, type, address});
EmitLabel(state, entry_label); EmitLabel(state, entry_label);
const auto page64 = Binary( const auto page64 = Binary(state, spv::OpShiftRightLogical, type, address,
state, OpShiftRightLogical, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS)); ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
const auto page = Unary(state, OpUConvert, TypeU32(state), page64); const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64);
const auto entry_pointer = state.builder.AllocateId(); const auto entry_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeDeviceAddressStoragePointer(state), entry_pointer, state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferU64ElementPointer(state),
state.bda_pagetable_variable, ConstantU32(state, 0), page}); entry_pointer, state.bda_pagetable_variable, ConstantU32(state, 0),
page});
const auto base = state.builder.AllocateId(); const auto base = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, base, entry_pointer}); state.builder.AddFunction({spv::OpLoad, type, base, entry_pointer});
const auto missing = const auto missing =
Binary(state, OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0)); Binary(state, spv::OpIEqual, TypeBool(state), base, ConstantDeviceAddress(state, 0));
const auto fault_label = state.builder.AllocateId(); const auto fault_label = state.builder.AllocateId();
const auto available_label = state.builder.AllocateId(); const auto available_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
state.builder.AddFunction( state.builder.AddFunction({spv::OpBranchConditional, missing, fault_label, available_label});
{OpBranchConditional, missing, fault_label, available_label});
EmitLabel(state, fault_label); EmitLabel(state, fault_label);
RecordBdaFault(state, page); RecordBdaFault(state, page);
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
EmitLabel(state, available_label); EmitLabel(state, available_label);
const auto offset = Binary( const auto offset = Binary(state, spv::OpBitwiseAnd, type, address,
state, OpBitwiseAnd, type, address,
ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1)); ConstantDeviceAddress(state, BufferCache::CACHING_PAGESIZE - 1));
const auto available = Binary(state, OpIAdd, type, base, offset); const auto available = Binary(state, spv::OpIAdd, type, base, offset);
state.builder.AddFunction({OpBranch, merge_label}); state.builder.AddFunction({spv::OpBranch, merge_label});
EmitLabel(state, merge_label); EmitLabel(state, merge_label);
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpPhi, type, result, ConstantDeviceAddress(state, 0), fault_label, state.builder.AddFunction({spv::OpPhi, type, result, ConstantDeviceAddress(state, 0),
available, available_label}); fault_label, available, available_label});
state.builder.AddFunction({OpReturnValue, result}); state.builder.AddFunction({spv::OpReturnValue, result});
state.builder.AddFunction({OpFunctionEnd}); state.builder.AddFunction({spv::OpFunctionEnd});
} }
bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) { bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
@@ -1073,26 +1078,26 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
return true; return true;
} }
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state),
state.flattened_srt_variable, ConstantU32(state, 0), pointer, state.flattened_srt_variable, ConstantU32(state, 0),
ctx.Arg(inst, 1)}); ctx.Arg(inst, 1)});
ctx.Emit(inst, OpLoad, IR::Type::U32, {pointer}); ctx.Emit(inst, spv::OpLoad, IR::Type::U32, {pointer});
return true; return true;
} }
if (op == IR::ValueOpcode::ReadConstBuffer) { if (op == IR::ValueOpcode::ReadConstBuffer) {
auto mem = ctx.Memory(inst); auto mem = ctx.Memory(inst);
mem.kind = IR::ResourceKind::ScalarBuffer; mem.kind = IR::ResourceKind::ScalarBuffer;
const auto address = const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
Binary(state, OpIAdd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, mem.offset)); ConstantU32(state, mem.offset));
const auto index = const auto index =
Binary(state, OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2)); Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
const auto access = PrepareMemoryResourceAccess(state, mem); const auto access = PrepareMemoryResourceAccess(state, mem);
const auto element = EmitMemoryElementIndex(state, access, index); const auto element = EmitMemoryElementIndex(state, access, index);
const auto condition = EmitMemoryElementInBounds(state, access, element); const auto condition = EmitMemoryElementInBounds(state, access, element);
ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() { ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpLoad, TypeU32(state), value, {spv::OpLoad, TypeU32(state), value,
EmitMemoryElementPointer(state, access, element)}); EmitMemoryElementPointer(state, access, element)});
return value; return value;
})); }));
@@ -1175,21 +1180,21 @@ bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
uint32_t source = ctx.Arg(inst, 0); uint32_t source = ctx.Arg(inst, 0);
uint32_t target = 0; uint32_t target = 0;
if (op == IR::ValueOpcode::SwizzleU32) { if (op == IR::ValueOpcode::SwizzleU32) {
state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, source}); state.builder.AddFunction({spv::OpStore, ctx.scratch_u32_variable, source});
source = state.builder.AllocateId(); source = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeU32(state), source, ctx.scratch_u32_variable}); state.builder.AddFunction(
{spv::OpLoad, TypeU32(state), source, ctx.scratch_u32_variable});
target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state), target = EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0); inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
} else { } else {
const auto index = Binary( const auto index = Binary(state, spv::OpBitwiseAnd, TypeU32(state),
state, OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeU32(state),
Binary(state, OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1), ctx.Arg(inst, 1), ConstantU32(state, 2)),
ConstantU32(state, 2)),
ConstantU32(state, 31)); ConstantU32(state, 31));
const auto base = const auto base =
Binary(state, OpBitwiseAnd, TypeU32(state), EmitSubgroupLocalInvocationId(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state),
ConstantU32(state, ~31u)); EmitSubgroupLocalInvocationId(state), ConstantU32(state, ~31u));
target = Binary(state, OpBitwiseOr, TypeU32(state), base, index); target = Binary(state, spv::OpBitwiseOr, TypeU32(state), base, index);
} }
ctx.Define(inst, EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2))); ctx.Define(inst, EmitDsMaskedLaneRead(state, source, target, ctx.Arg(inst, 2)));
return true; return true;
@@ -7,19 +7,19 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
dword_index += state.program.bindings.push_data_start_dword; dword_index += state.program.bindings.push_data_start_dword;
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer, state.builder.AddFunction({spv::OpAccessChain, TypePushConstantElementPointer(state),
state.push_constant_variable, ConstantU32(state, 0), pointer, state.push_constant_variable, ConstantU32(state, 0),
ConstantU32(state, dword_index)}); ConstantU32(state, dword_index)});
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
return value; return value;
} }
if (state.shader_data_storage_variable != 0) { if (state.shader_data_storage_variable != 0) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.builder.AddFunction({spv::OpAccessChain, TypeStorageBufferElementPointer(state),
state.shader_data_storage_variable, ConstantU32(state, 0), pointer, state.shader_data_storage_variable,
ConstantU32(state, dword_index)}); ConstantU32(state, 0), ConstantU32(state, dword_index)});
state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer}); state.builder.AddFunction({spv::OpLoad, TypeU32(state), value, pointer});
return value; return value;
} }
return ConstantU32(state, 0); return ConstantU32(state, 0);
@@ -27,7 +27,7 @@ uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) { uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpIAdd, TypeU32(state), ret, lhs, rhs}); state.builder.AddFunction({spv::OpIAdd, TypeU32(state), ret, lhs, rhs});
return ret; return ret;
} }
@@ -59,13 +59,13 @@ void EmitMemoryOffsets(EmitterState& state) {
EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u); EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
const auto shift = ConstantU32(state, (i % 4u) * 8u); const auto shift = ConstantU32(state, (i % 4u) * 8u);
state.memory_byte_offsets[i] = EmitBinaryU32( state.memory_byte_offsets[i] = EmitBinaryU32(
state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift), state, spv::OpBitwiseAnd, EmitBinaryU32(state, spv::OpShiftRightLogical, word, shift),
ConstantU32(state, 0xffu)); ConstantU32(state, 0xffu));
} }
} }
uint32_t LdsDwordCount(const EmitterState& state) { uint32_t LdsDwordCount(const EmitterState& state) {
const auto* workgroup = ShaderWorkgroupInput(state.stage, state.input_info); const auto* workgroup = ShaderWorkgroupInput(state.program.stage, state.input_info);
return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u; return workgroup != nullptr ? workgroup->lds_size_dwords : 8192u;
} }
@@ -73,12 +73,12 @@ static void EnsureLdsStorage(EmitterState& state) {
if (state.lds_variable != 0) { if (state.lds_variable != 0) {
return; return;
} }
if (ShaderWorkgroupInput(state.stage, state.input_info) == nullptr) { if (ShaderWorkgroupInput(state.program.stage, state.input_info) == nullptr) {
EXIT("function LDS was not prepared before SPIR-V function emission\n"); EXIT("function LDS was not prepared before SPIR-V function emission\n");
} }
state.lds_variable = state.builder.DefineGlobalVariable( state.lds_variable = state.builder.DefineGlobalVariable(
TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)), TypeU32ArrayPointer(state, spv::StorageClassWorkgroup, LdsDwordCount(state)),
StorageClassWorkgroup); spv::StorageClassWorkgroup);
state.builder.AddName(state.lds_variable, "lds_dwords"); state.builder.AddName(state.lds_variable, "lds_dwords");
} }
@@ -94,12 +94,12 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource); ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
MemoryResourceAccess access {.kind = mem.kind}; MemoryResourceAccess access {.kind = mem.kind};
access.object_pointer = state.builder.AllocateId(); access.object_pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, access.object_pointer, variable, state.builder.AddFunction({spv::OpAccessChain, pointer_type, access.object_pointer, variable,
ConstantU32(state, array_index)}); ConstantU32(state, array_index)});
access.byte_offset = state.memory_byte_offsets[array_index]; access.byte_offset = state.memory_byte_offsets[array_index];
access.length = state.builder.AllocateId(); access.length = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0}); {spv::OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
return access; return access;
} }
@@ -137,8 +137,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
case IR::ResourceKind::Buffer: { case IR::ResourceKind::Buffer: {
access = PrepareStorageBufferResourceAccess( access = PrepareStorageBufferResourceAccess(
state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state)); state, mem, state.storage_buffer_variable, TypeStorageBufferPointer(state));
access.index_offset = access.index_offset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byte_offset,
EmitBinaryU32(state, OpShiftRightLogical, access.byte_offset,
ConstantU32(state, 2u)); ConstantU32(state, 2u));
access.add_index_offset = true; access.add_index_offset = true;
return access; return access;
@@ -147,7 +146,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::
} }
access.length = state.builder.AllocateId(); access.length = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0}); {spv::OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
return access; return access;
} }
@@ -159,7 +158,7 @@ uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess&
uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access, uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
uint32_t index) { uint32_t index) {
const auto in_bounds = state.builder.AllocateId(); const auto in_bounds = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length}); state.builder.AddFunction({spv::OpULessThan, TypeBool(state), in_bounds, index, access.length});
return in_bounds; return in_bounds;
} }
@@ -167,11 +166,12 @@ uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAcces
uint32_t index) { uint32_t index) {
if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) { if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction const auto storage_class =
: ShaderWorkgroupInput(state.stage, state.input_info) != nullptr access.kind == IR::ResourceKind::Scratch ? spv::StorageClassFunction
? StorageClassWorkgroup : ShaderWorkgroupInput(state.program.stage, state.input_info) != nullptr
: StorageClassFunction; ? spv::StorageClassWorkgroup
state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class), : spv::StorageClassFunction;
state.builder.AddFunction({spv::OpAccessChain, TypeU32ElementPointer(state, storage_class),
pointer, access.object_pointer, index}); pointer, access.object_pointer, index});
return pointer; return pointer;
} }
@@ -183,21 +183,22 @@ uint32_t EmitStorageBufferElementPointer(EmitterState& state,
const MemoryResourceAccess& access, uint32_t index, const MemoryResourceAccess& access, uint32_t index,
uint32_t pointer_type) { uint32_t pointer_type) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction({OpAccessChain, pointer_type, pointer, access.object_pointer, state.builder.AddFunction({spv::OpAccessChain, pointer_type, pointer, access.object_pointer,
ConstantU32(state, 0), index}); ConstantU32(state, 0), index});
return pointer; return pointer;
} }
uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) { uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpBitcast, TypeI32(state), ret, value}); state.builder.AddFunction({spv::OpBitcast, TypeI32(state), ret, value});
return ret; return ret;
} }
uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value, uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
uint32_t false_value) { uint32_t false_value) {
const auto ret = state.builder.AllocateId(); const auto ret = state.builder.AllocateId();
state.builder.AddFunction({OpSelect, TypeF32(state), ret, condition, true_value, false_value}); state.builder.AddFunction(
{spv::OpSelect, TypeF32(state), ret, condition, true_value, false_value});
return ret; return ret;
} }
@@ -210,33 +211,34 @@ uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
const auto mantissa_bits = bits == 11u ? 6u : 5u; const auto mantissa_bits = bits == 11u ? 6u : 5u;
const auto mantissa_mask = (1u << mantissa_bits) - 1u; const auto mantissa_mask = (1u << mantissa_bits) - 1u;
const auto mantissa = const auto mantissa =
EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask)); EmitBinaryU32(state, spv::OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
const auto exponent = EmitBinaryU32( const auto exponent = EmitBinaryU32(
state, OpBitwiseAnd, state, spv::OpBitwiseAnd,
EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)), EmitBinaryU32(state, spv::OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
ConstantU32(state, 0x1fu)); ConstantU32(state, 0x1fu));
const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u)); const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
const auto exponent_bits = const auto exponent_bits =
EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23)); EmitBinaryU32(state, spv::OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
const auto mantissa_bits_32 = const auto mantissa_bits_32 = EmitBinaryU32(state, spv::OpShiftLeftLogical, mantissa,
EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits)); ConstantU32(state, 23u - mantissa_bits));
const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32); const auto normal_bits =
EmitBinaryU32(state, spv::OpBitwiseOr, exponent_bits, mantissa_bits_32);
const auto normal = EmitBitcastU32ToF32(state, normal_bits); const auto normal = EmitBitcastU32ToF32(state, normal_bits);
const auto special_bits = const auto special_bits =
EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32); EmitBinaryU32(state, spv::OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
const auto special = EmitBitcastU32ToF32(state, special_bits); const auto special = EmitBitcastU32ToF32(state, special_bits);
const auto mantissa_f32 = state.builder.AllocateId(); const auto mantissa_f32 = state.builder.AllocateId();
const auto subnormal = state.builder.AllocateId(); const auto subnormal = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, TypeF32(state), mantissa_f32, mantissa}); state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
state.builder.AddFunction( state.builder.AddFunction(
{OpFMul, TypeF32(state), subnormal, mantissa_f32, {spv::OpFMul, TypeF32(state), subnormal, mantissa_f32,
ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))}); ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
const auto zero_exp = EmitCompareU32Constant(state, OpIEqual, exponent, 0); const auto zero_exp = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 0);
const auto special_exp = EmitCompareU32Constant(state, OpIEqual, exponent, 31); const auto special_exp = EmitCompareU32Constant(state, spv::OpIEqual, exponent, 31);
const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal); const auto finite = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
const auto result = EmitTBufferSelectF32(state, special_exp, special, finite); const auto result = EmitTBufferSelectF32(state, special_exp, special, finite);
return EmitBitcastF32ToU32(state, result); return EmitBitcastF32ToU32(state, result);
@@ -250,22 +252,22 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
case Format::ComponentType::Sint: return raw; case Format::ComponentType::Sint: return raw;
case Format::ComponentType::Uscaled: { case Format::ComponentType::Uscaled: {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw}); state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), value, raw});
return EmitBitcastF32ToU32(state, value); return EmitBitcastF32ToU32(state, value);
} }
case Format::ComponentType::Sscaled: { case Format::ComponentType::Sscaled: {
const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw); const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw}); state.builder.AddFunction({spv::OpConvertSToF, TypeF32(state), value, signed_raw});
return EmitBitcastF32ToU32(state, value); return EmitBitcastF32ToU32(state, value);
} }
case Format::ComponentType::Unorm: { case Format::ComponentType::Unorm: {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
const auto normalized = state.builder.AllocateId(); const auto normalized = state.builder.AllocateId();
const auto max_value = static_cast<float>((1u << bits) - 1u); const auto max_value = static_cast<float>((1u << bits) - 1u);
state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw}); state.builder.AddFunction({spv::OpConvertUToF, TypeF32(state), value, raw});
state.builder.AddFunction( state.builder.AddFunction({spv::OpFDiv, TypeF32(state), normalized, value,
{OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)}); ConstantF32Value(state, max_value)});
return EmitBitcastF32ToU32(state, normalized); return EmitBitcastF32ToU32(state, normalized);
} }
case Format::ComponentType::Snorm: { case Format::ComponentType::Snorm: {
@@ -274,11 +276,11 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
const auto normalized = state.builder.AllocateId(); const auto normalized = state.builder.AllocateId();
const auto clamped = state.builder.AllocateId(); const auto clamped = state.builder.AllocateId();
const auto max_value = static_cast<float>((1u << (bits - 1u)) - 1u); const auto max_value = static_cast<float>((1u << (bits - 1u)) - 1u);
state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw}); state.builder.AddFunction({spv::OpConvertSToF, TypeF32(state), value, signed_raw});
state.builder.AddFunction( state.builder.AddFunction({spv::OpFDiv, TypeF32(state), normalized, value,
{OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)}); ConstantF32Value(state, max_value)});
state.builder.AddFunction({OpExtInst, TypeF32(state), clamped, GlslStd450(state), state.builder.AddFunction({spv::OpExtInst, TypeF32(state), clamped, GlslStd450(state),
GlslFMax, normalized, ConstantF32Value(state, -1.0f)}); GLSLstd450FMax, normalized, ConstantF32Value(state, -1.0f)});
return EmitBitcastF32ToU32(state, clamped); return EmitBitcastF32ToU32(state, clamped);
} }
case Format::ComponentType::Float: case Format::ComponentType::Float:
@@ -294,9 +296,10 @@ uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferForma
} }
void EmitDeviceAtomicMemoryBarrier(EmitterState& state) { void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory; const auto semantics =
state.builder.AddFunction( spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask;
{OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)}); state.builder.AddFunction({spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeDevice),
ConstantU32(state, semantics)});
} }
uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source, uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
@@ -308,12 +311,12 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
}; };
const auto classify = [&](uint32_t bits) { const auto classify = [&](uint32_t bits) {
const auto cls = EmitClassifyF32Bits(state, bits); const auto cls = EmitClassifyF32Bits(state, bits);
const auto negative = EmitCompareU32Constant( const auto negative = EmitCompareU32Constant(state, spv::OpINotEqual,
state, OpINotEqual, EmitAndConstant(state, bits, 0x80000000u), 0u); EmitAndConstant(state, bits, 0x80000000u), 0u);
const auto negative_key = state.builder.AllocateId(); const auto negative_key = state.builder.AllocateId();
state.builder.AddFunction({OpNot, TypeU32(state), negative_key, bits}); state.builder.AddFunction({spv::OpNot, TypeU32(state), negative_key, bits});
const auto positive_key = EmitBinaryU32( const auto positive_key =
state, OpBitwiseXor, bits, ConstantU32(state, 0x80000000u)); EmitBinaryU32(state, spv::OpBitwiseXor, bits, ConstantU32(state, 0x80000000u));
return OrderedBits { return OrderedBits {
cls.nan, cls.zero, cls.nan, cls.zero,
EmitSelectValueU32(state, negative, negative_key, positive_key)}; EmitSelectValueU32(state, negative, negative_key, positive_key)};
@@ -324,8 +327,8 @@ uint32_t EmitFloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t
state, EmitLogicalOrBool(state, source_class.nan, old_class.nan), state, EmitLogicalOrBool(state, source_class.nan, old_class.nan),
EmitLogicalAndBool(state, source_class.zero, old_class.zero)); EmitLogicalAndBool(state, source_class.zero, old_class.zero));
const auto compare = state.builder.AllocateId(); const auto compare = state.builder.AllocateId();
state.builder.AddFunction({max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), compare, state.builder.AddFunction({max_value ? spv::OpUGreaterThan : spv::OpULessThan, TypeBool(state),
source_class.key, old_class.key}); compare, source_class.key, old_class.key});
return EmitSelectValueU32( return EmitSelectValueU32(
state, EmitLogicalAndBool(state, EmitLogicalNotBool(state, unordered), compare), source, state, EmitLogicalAndBool(state, EmitLogicalNotBool(state, unordered), compare), source,
old); old);
@@ -354,16 +357,16 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
const auto base = state.builder.AllocateId(); const auto base = state.builder.AllocateId();
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)}); {spv::OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)}); {spv::OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
state.builder.AddFunction( state.builder.AddFunction(
{OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift}); {spv::OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)}); {spv::OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)}); {spv::OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, perm}); state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, base, perm});
return target; return target;
} }
@@ -373,16 +376,17 @@ uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t c
const auto xored = state.builder.AllocateId(); const auto xored = state.builder.AllocateId();
const auto base = state.builder.AllocateId(); const auto base = state.builder.AllocateId();
const auto target = state.builder.AllocateId(); const auto target = state.builder.AllocateId();
state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)}); {spv::OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseOr, TypeU32(state), ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)}); {spv::OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), ored, masked,
ConstantU32(state, (control >> 5u) & 0x1fu)});
state.builder.AddFunction({spv::OpBitwiseXor, TypeU32(state), xored, ored,
ConstantU32(state, (control >> 10u) & 0x1fu)});
state.builder.AddFunction( state.builder.AddFunction(
{OpBitwiseXor, TypeU32(state), xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)}); {spv::OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
state.builder.AddFunction( state.builder.AddFunction({spv::OpBitwiseOr, TypeU32(state), target, base, xored});
{OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, xored});
return target; return target;
} }
@@ -3,24 +3,24 @@
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter { namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace { namespace {
uint32_t MeshArray(EmitterState& state, uint32_t storage, uint32_t type, uint32_t count) { uint32_t MeshArray(EmitterState& state, spv::StorageClass storage, uint32_t type, uint32_t count) {
const auto array = state.builder.Type(OpTypeArray, {type, ConstantU32(state, count)}); const auto array = state.builder.Type(spv::OpTypeArray, {type, ConstantU32(state, count)});
return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage); return state.builder.DefineGlobalVariable(TypePointer(state, storage, array), storage);
} }
uint32_t MeshElement(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type, uint32_t MeshElement(EmitterState& state, uint32_t variable, spv::StorageClass storage,
uint32_t index) { uint32_t type, uint32_t index) {
const auto pointer = state.builder.AllocateId(); const auto pointer = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpAccessChain, TypePointer(state, storage, type), pointer, variable, index}); {spv::OpAccessChain, TypePointer(state, storage, type), pointer, variable, index});
return pointer; return pointer;
} }
uint32_t MeshLoad(EmitterState& state, uint32_t variable, uint32_t storage, uint32_t type, uint32_t MeshLoad(EmitterState& state, uint32_t variable, spv::StorageClass storage, uint32_t type,
uint32_t index) { uint32_t index) {
const auto pointer = MeshElement(state, variable, storage, type, index); const auto pointer = MeshElement(state, variable, storage, type, index);
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, type, value, pointer}); state.builder.AddFunction({spv::OpLoad, type, value, pointer});
return value; return value;
} }
@@ -40,40 +40,44 @@ void DefineMeshOutputs(EmitterState& state) {
} }
const auto type = MeshOutputType(state, output.kind); const auto type = MeshOutputType(state, output.kind);
output.variable_id = MeshArray( output.variable_id = MeshArray(
state, StorageClassOutput, type, state, spv::StorageClassOutput, type,
output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices); output.kind == IR::StageOutputKind::Layer ? mesh.max_primitives : mesh.max_vertices);
// Only Layer is read by another invocation, through the primitive's provoking vertex. // Only Layer is read by another invocation, through the primitive's provoking vertex.
const bool shared = output.kind == IR::StageOutputKind::Layer; const bool shared = output.kind == IR::StageOutputKind::Layer;
output.mesh_data_variable = MeshArray( output.mesh_data_variable =
state, shared ? StorageClassWorkgroup : StorageClassPrivate, type, MeshArray(state, shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, type,
shared ? mesh.max_vertices : state.lane_count); shared ? mesh.max_vertices : state.lane_count);
state.interface_variables.push_back(output.variable_id); state.interface_variables.push_back(output.variable_id);
state.builder.AddName(output.variable_id, output.debug_name.c_str()); state.builder.AddName(output.variable_id, output.debug_name.c_str());
if (output.kind == IR::StageOutputKind::Parameter) { if (output.kind == IR::StageOutputKind::Parameter) {
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, output.variable_id, DecorationLocation, output.location}); {spv::OpDecorate, output.variable_id, spv::DecorationLocation, output.location});
} else { } else {
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, output.variable_id, DecorationBuiltIn, {spv::OpDecorate, output.variable_id, spv::DecorationBuiltIn,
output.kind == IR::StageOutputKind::Layer ? BuiltInLayer : BuiltInPosition}); output.kind == IR::StageOutputKind::Layer ? spv::BuiltInLayer
: spv::BuiltInPosition});
} }
if (output.kind == IR::StageOutputKind::Layer) { if (output.kind == IR::StageOutputKind::Layer) {
state.builder.AddAnnotation({OpDecorate, output.variable_id, 5271u}); // PerPrimitiveEXT state.builder.AddAnnotation({spv::OpDecorate, output.variable_id,
spv::DecorationPerPrimitiveEXT}); // PerPrimitiveEXT
} }
} }
state.mesh_allocation = MeshArray(state, StorageClassWorkgroup, TypeU32(state), 2); state.mesh_allocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2);
state.mesh_primitive_data = state.mesh_primitive_data =
MeshArray(state, StorageClassPrivate, TypeU32(state), state.lane_count); MeshArray(state, spv::StorageClassPrivate, TypeU32(state), state.lane_count);
state.mesh_primitives = state.mesh_primitives =
MeshArray(state, StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives); MeshArray(state, spv::StorageClassOutput, TypeU32Vector(state, 3), mesh.max_primitives);
state.mesh_cull = MeshArray(state, StorageClassOutput, TypeBool(state), mesh.max_primitives); state.mesh_cull =
MeshArray(state, spv::StorageClassOutput, TypeBool(state), mesh.max_primitives);
state.interface_variables.push_back(state.mesh_primitives); state.interface_variables.push_back(state.mesh_primitives);
state.interface_variables.push_back(state.mesh_cull); state.interface_variables.push_back(state.mesh_cull);
state.builder.AddAnnotation({OpDecorate, state.mesh_primitives, DecorationBuiltIn,
5296u}); // PrimitiveTriangleIndicesEXT
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, state.mesh_cull, DecorationBuiltIn, 5299u}); // CullPrimitiveEXT {spv::OpDecorate, state.mesh_primitives, spv::DecorationBuiltIn,
state.builder.AddAnnotation({OpDecorate, state.mesh_cull, 5271u}); spv::BuiltInPrimitiveTriangleIndicesEXT}); // PrimitiveTriangleIndicesEXT
state.builder.AddAnnotation({spv::OpDecorate, state.mesh_cull, spv::DecorationBuiltIn,
spv::BuiltInCullPrimitiveEXT}); // CullPrimitiveEXT
state.builder.AddAnnotation({spv::OpDecorate, state.mesh_cull, spv::DecorationPerPrimitiveEXT});
} }
uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) { uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32_t index) {
@@ -85,113 +89,118 @@ uint32_t MeshOutputPointer(EmitterState& state, IR::StageOutputKind kind, uint32
index); index);
} }
const bool shared = kind == IR::StageOutputKind::Layer; const bool shared = kind == IR::StageOutputKind::Layer;
return MeshElement(state, output->mesh_data_variable, return MeshElement(
shared ? StorageClassWorkgroup : StorageClassPrivate, state, output->mesh_data_variable,
MeshOutputType(state, kind), shared ? spv::StorageClassWorkgroup : spv::StorageClassPrivate, MeshOutputType(state, kind),
shared ? EmitLocalInvocationIndex(state) shared ? EmitLocalInvocationIndex(state) : ConstantU32(state, state.lane_half));
: ConstantU32(state, state.lane_half));
} }
uint32_t MeshPrimitivePointer(EmitterState& state) { uint32_t MeshPrimitivePointer(EmitterState& state) {
return MeshElement(state, state.mesh_primitive_data, StorageClassPrivate, TypeU32(state), return MeshElement(state, state.mesh_primitive_data, spv::StorageClassPrivate, TypeU32(state),
ConstantU32(state, state.lane_half)); ConstantU32(state, state.lane_half));
} }
void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) { void EmitMeshAllocate(ValueEmitContext& ctx, const IR::Inst& inst) {
auto& state = ctx.state; auto& state = ctx.state;
const auto first = state.builder.AllocateId(); const auto first = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction({spv::OpIEqual, TypeBool(state), first,
{OpIEqual, TypeBool(state), first, EmitLocalInvocationIndex(state), ConstantU32(state, 0)}); EmitLocalInvocationIndex(state), ConstantU32(state, 0)});
EmitIfCondition(state, first, [&] { EmitIfCondition(state, first, [&] {
const auto allocation = ctx.Arg(inst, 0); const auto allocation = ctx.Arg(inst, 0);
for (uint32_t field = 0; field < 2; field++) { for (uint32_t field = 0; field < 2; field++) {
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), value, allocation, state.builder.AddFunction({spv::OpBitFieldUExtract, TypeU32(state), value, allocation,
ConstantU32(state, field * 12u), ConstantU32(state, field * 12u),
ConstantU32(state, field == 0 ? 10u : 11u)}); ConstantU32(state, field == 0 ? 10u : 11u)});
const auto pointer = MeshElement(state, state.mesh_allocation, StorageClassWorkgroup, const auto pointer =
MeshElement(state, state.mesh_allocation, spv::StorageClassWorkgroup,
TypeU32(state), ConstantU32(state, field)); TypeU32(state), ConstantU32(state, field));
state.builder.AddFunction({OpStore, pointer, value}); state.builder.AddFunction({spv::OpStore, pointer, value});
} }
}); });
} }
void EmitMeshEntryPoint(EmitterState& state) { void EmitMeshEntryPoint(EmitterState& state) {
state.builder.AddFunction( state.builder.AddFunction({spv::OpFunction, TypeVoid(state), state.main_func,
{OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)}); spv::FunctionControlMaskNone, TypeFunction(state)});
EmitLabel(state, state.builder.AllocateId()); EmitLabel(state, state.builder.AllocateId());
state.builder.AddFunction( state.builder.AddFunction(
{OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func}); {spv::OpFunctionCall, TypeVoid(state), state.builder.AllocateId(), state.mesh_guest_func});
// All guest waves finish before the uniform Vulkan allocation and output stores. // All guest waves finish before the uniform Vulkan allocation and output stores.
state.builder.AddFunction( state.builder.AddFunction({spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup),
{OpControlBarrier, ConstantU32(state, ScopeWorkgroup), ConstantU32(state, ScopeWorkgroup), ConstantU32(state, spv::ScopeWorkgroup),
ConstantU32(state, MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory)}); ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask |
const auto vertices = MeshLoad(state, state.mesh_allocation, StorageClassWorkgroup, spv::MemorySemanticsWorkgroupMemoryMask)});
const auto vertices = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
TypeU32(state), ConstantU32(state, 0)); TypeU32(state), ConstantU32(state, 0));
const auto primitives = MeshLoad(state, state.mesh_allocation, StorageClassWorkgroup, const auto primitives = MeshLoad(state, state.mesh_allocation, spv::StorageClassWorkgroup,
TypeU32(state), ConstantU32(state, 1)); TypeU32(state), ConstantU32(state, 1));
state.builder.AddFunction({5295u, vertices, primitives}); // OpSetMeshOutputsEXT state.builder.AddFunction(
{spv::OpSetMeshOutputsEXT, vertices, primitives}); // OpSetMeshOutputsEXT
for (uint32_t half = 0; half < state.lane_count; half++) { for (uint32_t half = 0; half < state.lane_count; half++) {
state.lane_half = half; state.lane_half = half;
const auto index = EmitLocalInvocationIndex(state); const auto index = EmitLocalInvocationIndex(state);
const auto is_vertex = state.builder.AllocateId(); const auto is_vertex = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), is_vertex, index, vertices}); state.builder.AddFunction({spv::OpULessThan, TypeBool(state), is_vertex, index, vertices});
EmitIfCondition(state, is_vertex, [&] { EmitIfCondition(state, is_vertex, [&] {
for (const auto& output: state.outputs) { for (const auto& output: state.outputs) {
if (output.kind == IR::StageOutputKind::Layer) { if (output.kind == IR::StageOutputKind::Layer) {
continue; continue;
} }
const auto type = MeshOutputType(state, output.kind); const auto type = MeshOutputType(state, output.kind);
const auto value = MeshLoad(state, output.mesh_data_variable, StorageClassPrivate, const auto value =
type, ConstantU32(state, half)); MeshLoad(state, output.mesh_data_variable, spv::StorageClassPrivate, type,
ConstantU32(state, half));
const auto pointer = const auto pointer =
MeshElement(state, output.variable_id, StorageClassOutput, type, index); MeshElement(state, output.variable_id, spv::StorageClassOutput, type, index);
state.builder.AddFunction({OpStore, pointer, value}); state.builder.AddFunction({spv::OpStore, pointer, value});
} }
}); });
const auto is_primitive = state.builder.AllocateId(); const auto is_primitive = state.builder.AllocateId();
state.builder.AddFunction({OpULessThan, TypeBool(state), is_primitive, index, primitives}); state.builder.AddFunction(
{spv::OpULessThan, TypeBool(state), is_primitive, index, primitives});
EmitIfCondition(state, is_primitive, [&] { EmitIfCondition(state, is_primitive, [&] {
const auto packed = MeshLoad(state, state.mesh_primitive_data, StorageClassPrivate, const auto packed = MeshLoad(state, state.mesh_primitive_data, spv::StorageClassPrivate,
TypeU32(state), ConstantU32(state, half)); TypeU32(state), ConstantU32(state, half));
uint32_t vertex[3] {}; uint32_t vertex[3] {};
for (uint32_t component = 0; component < 3; component++) { for (uint32_t component = 0; component < 3; component++) {
vertex[component] = state.builder.AllocateId(); vertex[component] = state.builder.AllocateId();
state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), vertex[component], state.builder.AddFunction(
packed, ConstantU32(state, component * 10u), {spv::OpBitFieldUExtract, TypeU32(state), vertex[component], packed,
ConstantU32(state, 10)}); ConstantU32(state, component * 10u), ConstantU32(state, 10)});
} }
const auto triangle = state.builder.AllocateId(); const auto triangle = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 3), triangle, state.builder.AddFunction({spv::OpCompositeConstruct, TypeU32Vector(state, 3), triangle,
vertex[0], vertex[1], vertex[2]}); vertex[0], vertex[1], vertex[2]});
const auto triangle_pointer = MeshElement( const auto triangle_pointer =
state, state.mesh_primitives, StorageClassOutput, TypeU32Vector(state, 3), index); MeshElement(state, state.mesh_primitives, spv::StorageClassOutput,
state.builder.AddFunction({OpStore, triangle_pointer, triangle}); TypeU32Vector(state, 3), index);
state.builder.AddFunction({spv::OpStore, triangle_pointer, triangle});
const auto null_bit = const auto null_bit =
EmitBinaryU32(state, OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u)); EmitBinaryU32(state, spv::OpBitwiseAnd, packed, ConstantU32(state, 0x80000000u));
const auto culled = state.builder.AllocateId(); const auto culled = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), culled, null_bit, ConstantU32(state, 0)});
state.builder.AddFunction( state.builder.AddFunction({spv::OpStore,
{OpStore, MeshElement(state, state.mesh_cull, spv::StorageClassOutput,
MeshElement(state, state.mesh_cull, StorageClassOutput, TypeBool(state), index), TypeBool(state), index),
culled}); culled});
for (const auto& output: state.outputs) { for (const auto& output: state.outputs) {
if (output.kind != IR::StageOutputKind::Layer) { if (output.kind != IR::StageOutputKind::Layer) {
continue; continue;
} }
const auto layer = MeshLoad( const auto layer = MeshLoad(state, output.mesh_data_variable,
state, output.mesh_data_variable, StorageClassWorkgroup, TypeU32(state), spv::StorageClassWorkgroup, TypeU32(state),
vertex[state.input_info.vertex->mesh.provoking_vertex]); vertex[state.input_info.vertex->mesh.provoking_vertex]);
const auto pointer = MeshElement(state, output.variable_id, StorageClassOutput, const auto pointer = MeshElement(state, output.variable_id, spv::StorageClassOutput,
TypeU32(state), index); TypeU32(state), index);
state.builder.AddFunction({OpStore, pointer, layer}); state.builder.AddFunction({spv::OpStore, pointer, layer});
} }
}); });
} }
state.lane_half = 0; state.lane_half = 0;
state.builder.AddFunction({OpReturn}); state.builder.AddFunction({spv::OpReturn});
state.builder.AddFunction({OpFunctionEnd}); state.builder.AddFunction({spv::OpFunctionEnd});
} }
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter } // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
@@ -9,53 +9,49 @@ const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state,
uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind); uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind);
uint32_t TypeVoid(EmitterState& state) { uint32_t TypeVoid(EmitterState& state) {
return state.builder.Type(OpTypeVoid); return state.builder.Type(spv::OpTypeVoid);
} }
uint32_t TypeBool(EmitterState& state) { uint32_t TypeBool(EmitterState& state) {
return state.builder.Type(OpTypeBool); return state.builder.Type(spv::OpTypeBool);
} }
uint32_t TypeBoolVector(EmitterState& state, uint32_t components) { uint32_t TypeBoolVector(EmitterState& state, uint32_t components) {
return state.builder.Type(OpTypeVector, {TypeBool(state), components}); return state.builder.Type(spv::OpTypeVector, {TypeBool(state), components});
} }
uint32_t TypeU32(EmitterState& state) { uint32_t TypeU32(EmitterState& state) {
return state.builder.Type(OpTypeInt, {32, 0}); return state.builder.Type(spv::OpTypeInt, {32, 0});
} }
uint32_t TypeU64(EmitterState& state) { uint32_t TypeU64(EmitterState& state) {
return TypeU32Vector(state, 2); return TypeU32Vector(state, 2);
} }
uint32_t TypeDeviceAddress(EmitterState& state) {
return TypeScalarU64(state);
}
uint32_t TypeScalarU64(EmitterState& state) { uint32_t TypeScalarU64(EmitterState& state) {
return state.builder.Type(OpTypeInt, {64, 0}); return state.builder.Type(spv::OpTypeInt, {64, 0});
} }
uint32_t TypeU32Pair(EmitterState& state) { uint32_t TypeU32Pair(EmitterState& state) {
const auto element = TypeU32(state); const auto element = TypeU32(state);
return state.builder.Type(OpTypeStruct, {element, element}); return state.builder.Type(spv::OpTypeStruct, {element, element});
} }
uint32_t TypeI32(EmitterState& state) { uint32_t TypeI32(EmitterState& state) {
return state.builder.Type(OpTypeInt, {32, 1}); return state.builder.Type(spv::OpTypeInt, {32, 1});
} }
uint32_t TypeI32Pair(EmitterState& state) { uint32_t TypeI32Pair(EmitterState& state) {
const auto element = TypeI32(state); const auto element = TypeI32(state);
return state.builder.Type(OpTypeStruct, {element, element}); return state.builder.Type(spv::OpTypeStruct, {element, element});
} }
uint32_t TypeF32(EmitterState& state) { uint32_t TypeF32(EmitterState& state) {
return state.builder.Type(OpTypeFloat, {32}); return state.builder.Type(spv::OpTypeFloat, {32});
} }
uint32_t TypeU32Vector(EmitterState& state, uint32_t components) { uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
return state.builder.Type(OpTypeVector, {TypeU32(state), components}); return state.builder.Type(spv::OpTypeVector, {TypeU32(state), components});
} }
uint32_t TypeU32Composite(EmitterState& state, uint32_t components) { uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
@@ -64,78 +60,77 @@ uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
} }
uint32_t TypeI32Vector(EmitterState& state, uint32_t components) { uint32_t TypeI32Vector(EmitterState& state, uint32_t components) {
return state.builder.Type(OpTypeVector, {TypeI32(state), components}); return state.builder.Type(spv::OpTypeVector, {TypeI32(state), components});
} }
uint32_t TypeF32Vector(EmitterState& state, uint32_t components) { uint32_t TypeF32Vector(EmitterState& state, uint32_t components) {
return state.builder.Type(OpTypeVector, {TypeF32(state), components}); return state.builder.Type(spv::OpTypeVector, {TypeF32(state), components});
} }
uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee) { uint32_t TypePointer(EmitterState& state, spv::StorageClass storage_class, uint32_t pointee) {
return state.builder.Type(OpTypePointer, {storage_class, pointee}); return state.builder.Type(spv::OpTypePointer, {storage_class, pointee});
} }
uint32_t TypeFunction(EmitterState& state) { uint32_t TypeFunction(EmitterState& state) {
return state.builder.Type(OpTypeFunction, {TypeVoid(state)}); return state.builder.Type(spv::OpTypeFunction, {TypeVoid(state)});
} }
uint32_t StorageRuntimeArrayType(EmitterState& state) { uint32_t StorageRuntimeArrayType(EmitterState& state) {
return state.builder.DecoratedType(OpTypeRuntimeArray, {TypeU32(state)}, return state.builder.DecoratedType(
{{OpDecorate, {DecorationArrayStride, sizeof(uint32_t)}}}); spv::OpTypeRuntimeArray, {TypeU32(state)},
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}});
} }
uint32_t StorageBufferType(EmitterState& state) { uint32_t StorageBufferType(EmitterState& state) {
return state.builder.DecoratedType( return state.builder.DecoratedType(spv::OpTypeStruct, {StorageRuntimeArrayType(state)},
OpTypeStruct, {StorageRuntimeArrayType(state)}, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}}); {spv::OpDecorate, {spv::DecorationBlock}}});
} }
uint32_t TypeStorageBufferPointer(EmitterState& state) { uint32_t TypeStorageBufferPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, StorageBufferType(state)); return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferType(state));
} }
uint32_t TypeStorageBufferElementPointer(EmitterState& state) { uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeU32(state)); return TypePointer(state, spv::StorageClassStorageBuffer, TypeU32(state));
} }
uint32_t StorageU64RuntimeArrayType(EmitterState& state) { uint32_t StorageU64RuntimeArrayType(EmitterState& state) {
return state.builder.DecoratedType(OpTypeRuntimeArray, {TypeScalarU64(state)}, return state.builder.DecoratedType(
{{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}}); spv::OpTypeRuntimeArray, {TypeScalarU64(state)},
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}});
} }
uint32_t StorageBufferU64Type(EmitterState& state) { uint32_t StorageBufferU64Type(EmitterState& state) {
return state.builder.DecoratedType( return state.builder.DecoratedType(spv::OpTypeStruct, {StorageU64RuntimeArrayType(state)},
OpTypeStruct, {StorageU64RuntimeArrayType(state)}, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}}); {spv::OpDecorate, {spv::DecorationBlock}}});
} }
uint32_t TypeStorageBufferU64Pointer(EmitterState& state) { uint32_t TypeStorageBufferU64Pointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, StorageBufferU64Type(state)); return TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64Type(state));
} }
uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) { uint32_t TypeStorageBufferU64ElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeScalarU64(state)); return TypePointer(state, spv::StorageClassStorageBuffer, TypeScalarU64(state));
}
uint32_t TypeDeviceAddressStoragePointer(EmitterState& state) {
return TypePointer(state, StorageClassStorageBuffer, TypeDeviceAddress(state));
} }
uint32_t TypePhysicalU32Pointer(EmitterState& state) { uint32_t TypePhysicalU32Pointer(EmitterState& state) {
return TypePointer(state, StorageClassPhysicalStorageBuffer, TypeU32(state)); return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state));
} }
uint32_t TypePushConstantElementPointer(EmitterState& state) { uint32_t TypePushConstantElementPointer(EmitterState& state) {
return TypePointer(state, StorageClassPushConstant, TypeU32(state)); return TypePointer(state, spv::StorageClassPushConstant, TypeU32(state));
} }
uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords) { uint32_t TypeU32ArrayPointer(EmitterState& state, spv::StorageClass storage_class,
uint32_t dwords) {
const auto count = ConstantU32(state, std::max(dwords, 1u)); const auto count = ConstantU32(state, std::max(dwords, 1u));
const auto array = state.builder.Type(OpTypeArray, {TypeU32(state), count}); const auto array = state.builder.Type(spv::OpTypeArray, {TypeU32(state), count});
return TypePointer(state, storage_class, array); return TypePointer(state, storage_class, array);
} }
uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class) { uint32_t TypeU32ElementPointer(EmitterState& state, spv::StorageClass storage_class) {
return TypePointer(state, storage_class, TypeU32(state)); return TypePointer(state, storage_class, TypeU32(state));
} }
@@ -143,78 +138,80 @@ namespace {
uint32_t PushConstantArrayType(EmitterState& state) { uint32_t PushConstantArrayType(EmitterState& state) {
const auto count = ConstantU32(state, IR::PushData::DwordCount); const auto count = ConstantU32(state, IR::PushData::DwordCount);
return state.builder.DecoratedType(OpTypeArray, {TypeU32(state), count}, return state.builder.DecoratedType(
{{OpDecorate, {DecorationArrayStride, sizeof(uint32_t)}}}); spv::OpTypeArray, {TypeU32(state), count},
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}});
} }
uint32_t PushConstantBlockType(EmitterState& state) { uint32_t PushConstantBlockType(EmitterState& state) {
return state.builder.DecoratedType( return state.builder.DecoratedType(spv::OpTypeStruct, {PushConstantArrayType(state)},
OpTypeStruct, {PushConstantArrayType(state)}, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {spv::OpDecorate, {spv::DecorationBlock}}});
{OpDecorate, {DecorationBlock}}});
} }
uint32_t PerVertexType(EmitterState& state) { uint32_t PerVertexType(EmitterState& state) {
return state.builder.DecoratedType(OpTypeStruct, {TypeF32Vector(state, 4)}, return state.builder.DecoratedType(
{{OpMemberDecorate, {0, DecorationBuiltIn, BuiltInPosition}}, spv::OpTypeStruct, {TypeF32Vector(state, 4)},
{OpDecorate, {DecorationBlock}}}); {{spv::OpMemberDecorate, {0, spv::DecorationBuiltIn, spv::BuiltInPosition}},
{spv::OpDecorate, {spv::DecorationBlock}}});
} }
uint32_t SampleMaskArrayType(EmitterState& state) { uint32_t SampleMaskArrayType(EmitterState& state) {
return state.builder.Type(OpTypeArray, {TypeI32(state), ConstantU32(state, 1)}); return state.builder.Type(spv::OpTypeArray, {TypeI32(state), ConstantU32(state, 1)});
} }
uint32_t BdaPagetableType(EmitterState& state) { uint32_t BdaPagetableType(EmitterState& state) {
const auto array = state.builder.DecoratedType( const auto array = state.builder.DecoratedType(
OpTypeRuntimeArray, {TypeDeviceAddress(state)}, spv::OpTypeRuntimeArray, {TypeScalarU64(state)},
{{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}}); {{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint64_t)}}});
return state.builder.DecoratedType( return state.builder.DecoratedType(spv::OpTypeStruct, {array},
OpTypeStruct, {array}, {{spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}},
{{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}}); {spv::OpDecorate, {spv::DecorationBlock}}});
} }
uint32_t F32ArrayType(EmitterState& state, uint32_t count) { uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
return state.builder.Type(OpTypeArray, {TypeF32(state), ConstantU32(state, count)}); return state.builder.Type(spv::OpTypeArray, {TypeF32(state), ConstantU32(state, count)});
} }
void DefineDescriptorVariables(EmitterState& state) { void DefineDescriptorVariables(EmitterState& state) {
if (DescriptorBinding(state, IR::DescriptorBindingKind::Buffers) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::Buffers) != nullptr) {
const auto count = const auto count =
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Buffers)); ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Buffers));
const auto array_type = state.builder.Type(OpTypeArray, {StorageBufferType(state), count}); const auto array_type =
const auto pointer_type = TypePointer(state, StorageClassStorageBuffer, array_type); state.builder.Type(spv::OpTypeArray, {StorageBufferType(state), count});
const auto pointer_type = TypePointer(state, spv::StorageClassStorageBuffer, array_type);
state.storage_buffer_variable = state.storage_buffer_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer); state.builder.DefineGlobalVariable(pointer_type, spv::StorageClassStorageBuffer);
if (state.requirements.buffer_int64_atomics) { if (state.requirements.buffer_int64_atomics) {
const auto u64_array_type = const auto u64_array_type =
state.builder.Type(OpTypeArray, {StorageBufferU64Type(state), count}); state.builder.Type(spv::OpTypeArray, {StorageBufferU64Type(state), count});
state.storage_buffer_u64_variable = state.builder.DefineGlobalVariable( state.storage_buffer_u64_variable = state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassStorageBuffer, u64_array_type), TypePointer(state, spv::StorageClassStorageBuffer, u64_array_type),
StorageClassStorageBuffer); spv::StorageClassStorageBuffer);
} }
} }
if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) {
state.bda_pagetable_variable = state.builder.DefineGlobalVariable( state.bda_pagetable_variable = state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassStorageBuffer, BdaPagetableType(state)), TypePointer(state, spv::StorageClassStorageBuffer, BdaPagetableType(state)),
StorageClassStorageBuffer); spv::StorageClassStorageBuffer);
} }
if (DescriptorBinding(state, IR::DescriptorBindingKind::FaultBuffer) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::FaultBuffer) != nullptr) {
state.fault_buffer_variable = state.builder.DefineGlobalVariable( state.fault_buffer_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer); TypeStorageBufferPointer(state), spv::StorageClassStorageBuffer);
} }
if (state.program.bindings.UsesPushData() || state.stage == ShaderType::Mesh) { if (state.program.bindings.UsesPushData() || state.program.stage == ShaderType::Mesh) {
const auto pointer_type = const auto pointer_type =
TypePointer(state, StorageClassPushConstant, PushConstantBlockType(state)); TypePointer(state, spv::StorageClassPushConstant, PushConstantBlockType(state));
state.push_constant_variable = state.push_constant_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassPushConstant); state.builder.DefineGlobalVariable(pointer_type, spv::StorageClassPushConstant);
} }
if (DescriptorBinding(state, IR::DescriptorBindingKind::ShaderData) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::ShaderData) != nullptr) {
state.shader_data_storage_variable = state.builder.DefineGlobalVariable( state.shader_data_storage_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer); TypeStorageBufferPointer(state), spv::StorageClassStorageBuffer);
} }
if (DescriptorBinding(state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
state.flattened_srt_variable = state.builder.DefineGlobalVariable( state.flattened_srt_variable = state.builder.DefineGlobalVariable(
TypeStorageBufferPointer(state), StorageClassStorageBuffer); TypeStorageBufferPointer(state), spv::StorageClassStorageBuffer);
} }
for (const auto& binding: state.program.bindings.descriptors) { for (const auto& binding: state.program.bindings.descriptors) {
if (IR::ImageBindingResourceClass(binding.kind) == IR::ImageResourceClass::None) { if (IR::ImageBindingResourceClass(binding.kind) == IR::ImageResourceClass::None) {
@@ -223,30 +220,30 @@ void DefineDescriptorVariables(EmitterState& state) {
const auto& image = state.program.info.images.at(binding.resources.front()); const auto& image = state.program.info.images.at(binding.resources.front());
const auto count = ConstantU32(state, DescriptorCount(state, binding.kind)); const auto count = ConstantU32(state, DescriptorCount(state, binding.kind));
const auto image_type = ImageType(state, image); const auto image_type = ImageType(state, image);
const auto array_type = state.builder.Type(OpTypeArray, {image_type, count}); const auto array_type = state.builder.Type(spv::OpTypeArray, {image_type, count});
const auto pointer_type = TypePointer(state, StorageClassUniformConstant, array_type); const auto pointer_type = TypePointer(state, spv::StorageClassUniformConstant, array_type);
state.image_variables[IR::ImageBindingIndex(binding.kind)] = state.image_variables[IR::ImageBindingIndex(binding.kind)] =
state.builder.DefineGlobalVariable(pointer_type, StorageClassUniformConstant); state.builder.DefineGlobalVariable(pointer_type, spv::StorageClassUniformConstant);
if (image.dimension == ImageDimension::Dim1D || if (image.dimension == ImageDimension::Dim1D ||
image.dimension == ImageDimension::Dim1DArray) { image.dimension == ImageDimension::Dim1DArray) {
const auto capability = image.resource_class == IR::ImageResourceClass::Sampled const auto capability = image.resource_class == IR::ImageResourceClass::Sampled
? CapabilitySampled1D ? spv::CapabilitySampled1D
: CapabilityImage1D; : spv::CapabilityImage1D;
state.builder.RequireCapability(capability); state.builder.RequireCapability(capability);
} }
} }
if (DescriptorBinding(state, IR::DescriptorBindingKind::Samplers) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::Samplers) != nullptr) {
const auto sampler_type = state.builder.Type(OpTypeSampler); const auto sampler_type = state.builder.Type(spv::OpTypeSampler);
const auto count = const auto count =
ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Samplers)); ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Samplers));
const auto array_type = state.builder.Type(OpTypeArray, {sampler_type, count}); const auto array_type = state.builder.Type(spv::OpTypeArray, {sampler_type, count});
const auto pointer_type = TypePointer(state, StorageClassUniformConstant, array_type); const auto pointer_type = TypePointer(state, spv::StorageClassUniformConstant, array_type);
state.sampler_variable = state.sampler_variable =
state.builder.DefineGlobalVariable(pointer_type, StorageClassUniformConstant); state.builder.DefineGlobalVariable(pointer_type, spv::StorageClassUniformConstant);
} }
if (DescriptorBinding(state, IR::DescriptorBindingKind::Gds) != nullptr) { if (DescriptorBinding(state, IR::DescriptorBindingKind::Gds) != nullptr) {
state.gds_variable = state.builder.DefineGlobalVariable(TypeStorageBufferPointer(state), state.gds_variable = state.builder.DefineGlobalVariable(TypeStorageBufferPointer(state),
StorageClassStorageBuffer); spv::StorageClassStorageBuffer);
} }
} }
@@ -263,15 +260,15 @@ uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind ki
} }
uint32_t ConstantU32(EmitterState& state, uint32_t value) { uint32_t ConstantU32(EmitterState& state, uint32_t value) {
return state.builder.Constant(OpConstant, TypeU32(state), {value}); return state.builder.Constant(spv::OpConstant, TypeU32(state), {value});
} }
uint32_t ConstantI32(EmitterState& state, int32_t value) { uint32_t ConstantI32(EmitterState& state, int32_t value) {
return state.builder.Constant(OpConstant, TypeI32(state), {static_cast<uint32_t>(value)}); return state.builder.Constant(spv::OpConstant, TypeI32(state), {static_cast<uint32_t>(value)});
} }
uint32_t ConstantF32(EmitterState& state, uint32_t bits) { uint32_t ConstantF32(EmitterState& state, uint32_t bits) {
return state.builder.Constant(OpConstant, TypeF32(state), {bits}); return state.builder.Constant(spv::OpConstant, TypeF32(state), {bits});
} }
uint32_t FloatBits(float value) { uint32_t FloatBits(float value) {
@@ -285,11 +282,12 @@ uint32_t ConstantF32Value(EmitterState& state, float value) {
} }
uint32_t ConstantBool(EmitterState& state, bool value) { uint32_t ConstantBool(EmitterState& state, bool value) {
return state.builder.Constant(value ? OpConstantTrue : OpConstantFalse, TypeBool(state)); return state.builder.Constant(value ? spv::OpConstantTrue : spv::OpConstantFalse,
TypeBool(state));
} }
uint32_t ConstantU64(EmitterState& state, uint64_t value) { uint32_t ConstantU64(EmitterState& state, uint64_t value) {
return state.builder.Constant(OpConstantComposite, TypeU64(state), return state.builder.Constant(spv::OpConstantComposite, TypeU64(state),
{ConstantU32(state, static_cast<uint32_t>(value)), {ConstantU32(state, static_cast<uint32_t>(value)),
ConstantU32(state, static_cast<uint32_t>(value >> 32u))}); ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
} }
@@ -298,7 +296,8 @@ uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
EXIT_IF(components < 2u || components > 4u); EXIT_IF(components < 2u || components > 4u);
const auto zero = ConstantU32(state, 0); const auto zero = ConstantU32(state, 0);
std::vector<uint32_t> values(components, zero); std::vector<uint32_t> values(components, zero);
return state.builder.Constant(OpConstantComposite, TypeU32Composite(state, components), values); return state.builder.Constant(spv::OpConstantComposite, TypeU32Composite(state, components),
values);
} }
uint32_t GlslStd450(EmitterState& state) { uint32_t GlslStd450(EmitterState& state) {
@@ -306,7 +305,7 @@ uint32_t GlslStd450(EmitterState& state) {
} }
VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) { VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
if (state.stage != ShaderType::Vertex || location >= ShaderVertexInputInfo::RES_MAX || if (state.program.stage != ShaderType::Vertex || location >= ShaderVertexInputInfo::RES_MAX ||
location >= static_cast<uint32_t>(state.input_info.vertex->resources_num)) { location >= static_cast<uint32_t>(state.input_info.vertex->resources_num)) {
return VertexInputScalarKind::Float; return VertexInputScalarKind::Float;
} }
@@ -354,31 +353,31 @@ uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarK
switch (kind) { switch (kind) {
case VertexInputScalarKind::Sint: case VertexInputScalarKind::Sint:
switch (components) { switch (components) {
case 1: return TypePointer(state, StorageClassInput, TypeI32(state)); case 1: return TypePointer(state, spv::StorageClassInput, TypeI32(state));
case 2: return TypePointer(state, StorageClassInput, TypeI32Vector(state, 2)); case 2: return TypePointer(state, spv::StorageClassInput, TypeI32Vector(state, 2));
case 3: return TypePointer(state, StorageClassInput, TypeI32Vector(state, 3)); case 3: return TypePointer(state, spv::StorageClassInput, TypeI32Vector(state, 3));
default: return TypePointer(state, StorageClassInput, TypeI32Vector(state, 4)); default: return TypePointer(state, spv::StorageClassInput, TypeI32Vector(state, 4));
} }
case VertexInputScalarKind::Uint: case VertexInputScalarKind::Uint:
switch (components) { switch (components) {
case 1: return TypePointer(state, StorageClassInput, TypeU32(state)); case 1: return TypePointer(state, spv::StorageClassInput, TypeU32(state));
case 2: return TypePointer(state, StorageClassInput, TypeU32Vector(state, 2)); case 2: return TypePointer(state, spv::StorageClassInput, TypeU32Vector(state, 2));
case 3: return TypePointer(state, StorageClassInput, TypeU32Vector(state, 3)); case 3: return TypePointer(state, spv::StorageClassInput, TypeU32Vector(state, 3));
default: return TypePointer(state, StorageClassInput, TypeU32Vector(state, 4)); default: return TypePointer(state, spv::StorageClassInput, TypeU32Vector(state, 4));
} }
case VertexInputScalarKind::Float: case VertexInputScalarKind::Float:
default: default:
switch (components) { switch (components) {
case 1: return TypePointer(state, StorageClassInput, TypeF32(state)); case 1: return TypePointer(state, spv::StorageClassInput, TypeF32(state));
case 2: return TypePointer(state, StorageClassInput, TypeF32Vector(state, 2)); case 2: return TypePointer(state, spv::StorageClassInput, TypeF32Vector(state, 2));
case 3: return TypePointer(state, StorageClassInput, TypeF32Vector(state, 3)); case 3: return TypePointer(state, spv::StorageClassInput, TypeF32Vector(state, 3));
default: return TypePointer(state, StorageClassInput, TypeF32Vector(state, 4)); default: return TypePointer(state, spv::StorageClassInput, TypeF32Vector(state, 4));
} }
} }
} }
static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) { static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) {
return state.stage == ShaderType::Pixel && return state.program.stage == ShaderType::Pixel &&
index < std::size(state.input_info.pixel->target_output_mode) && index < std::size(state.input_info.pixel->target_output_mode) &&
state.input_info.pixel->target_output_mode[index] == 7u; state.input_info.pixel->target_output_mode[index] == 7u;
} }
@@ -424,7 +423,7 @@ static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& vari
} }
void AllocateOutputVariables(EmitterState& state) { void AllocateOutputVariables(EmitterState& state) {
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
DefineMeshOutputs(state); DefineMeshOutputs(state);
return; return;
} }
@@ -472,18 +471,18 @@ void AllocateOutputVariables(EmitterState& state) {
uint32_t BuiltInForInput(IR::StageInputKind kind) { uint32_t BuiltInForInput(IR::StageInputKind kind) {
switch (kind) { switch (kind) {
case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex; case IR::StageInputKind::VertexIndex: return spv::BuiltInVertexIndex;
case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex; case IR::StageInputKind::InstanceIndex: return spv::BuiltInInstanceIndex;
case IR::StageInputKind::FragCoord: return BuiltInFragCoord; case IR::StageInputKind::FragCoord: return spv::BuiltInFragCoord;
case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing; case IR::StageInputKind::FrontFacing: return spv::BuiltInFrontFacing;
case IR::StageInputKind::Layer: return BuiltInLayer; case IR::StageInputKind::Layer: return spv::BuiltInLayer;
case IR::StageInputKind::SampleId: return BuiltInSampleId; case IR::StageInputKind::SampleId: return spv::BuiltInSampleId;
case IR::StageInputKind::BaryCoordSmooth: return BuiltInBaryCoordKHR; case IR::StageInputKind::BaryCoordSmooth: return spv::BuiltInBaryCoordKHR;
case IR::StageInputKind::BaryCoordNoPerspective: return BuiltInBaryCoordNoPerspKHR; case IR::StageInputKind::BaryCoordNoPerspective: return spv::BuiltInBaryCoordNoPerspKHR;
case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId; case IR::StageInputKind::WorkgroupId: return spv::BuiltInWorkgroupId;
case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId; case IR::StageInputKind::LocalInvocationId: return spv::BuiltInLocalInvocationId;
case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex; case IR::StageInputKind::LocalInvocationIndex: return spv::BuiltInLocalInvocationIndex;
case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId; case IR::StageInputKind::GlobalInvocationId: return spv::BuiltInGlobalInvocationId;
default: return UINT32_MAX; default: return UINT32_MAX;
} }
} }
@@ -492,46 +491,49 @@ void AddInputAnnotationsAndNames(EmitterState& state) {
if (state.subgroup_local_invocation_id_variable != 0) { if (state.subgroup_local_invocation_id_variable != 0) {
state.builder.AddName(state.subgroup_local_invocation_id_variable, state.builder.AddName(state.subgroup_local_invocation_id_variable,
"gl_SubgroupInvocationID"); "gl_SubgroupInvocationID");
state.builder.AddAnnotation({OpDecorate, state.subgroup_local_invocation_id_variable, state.builder.AddAnnotation({spv::OpDecorate, state.subgroup_local_invocation_id_variable,
DecorationBuiltIn, BuiltInSubgroupLocalInvocationId}); spv::DecorationBuiltIn,
if (state.stage == ShaderType::Pixel) { spv::BuiltInSubgroupLocalInvocationId});
state.builder.AddAnnotation( if (state.program.stage == ShaderType::Pixel) {
{OpDecorate, state.subgroup_local_invocation_id_variable, DecorationFlat}); state.builder.AddAnnotation({spv::OpDecorate,
state.subgroup_local_invocation_id_variable,
spv::DecorationFlat});
} }
} }
for (const auto& input: state.inputs) { for (const auto& input: state.inputs) {
state.builder.AddName(input.variable_id, input.debug_name.c_str()); state.builder.AddName(input.variable_id, input.debug_name.c_str());
if (input.kind == IR::StageInputKind::Layer || input.kind == IR::StageInputKind::SampleId) { if (input.kind == IR::StageInputKind::Layer || input.kind == IR::StageInputKind::SampleId) {
state.builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat}); state.builder.AddAnnotation({spv::OpDecorate, input.variable_id, spv::DecorationFlat});
} }
if (input.kind == IR::StageInputKind::Parameter) { if (input.kind == IR::StageInputKind::Parameter) {
const auto flat = PixelParameterIsFlat(state, input.location); const auto flat = PixelParameterIsFlat(state, input.location);
if (input.per_vertex) { if (input.per_vertex) {
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationPerVertexKHR}); {spv::OpDecorate, input.variable_id, spv::DecorationPerVertexKHR});
} else if (flat) { } else if (flat) {
state.builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
}
if (state.stage == ShaderType::Pixel && state.input_info.pixel->ps_no_perspective &&
!flat && !input.per_vertex) {
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationNoPerspective}); {spv::OpDecorate, input.variable_id, spv::DecorationFlat});
}
if (state.program.stage == ShaderType::Pixel &&
state.input_info.pixel->ps_no_perspective && !flat && !input.per_vertex) {
state.builder.AddAnnotation(
{spv::OpDecorate, input.variable_id, spv::DecorationNoPerspective});
} }
const auto location = PixelParameterLocation(state, input.location); const auto location = PixelParameterLocation(state, input.location);
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationLocation, location}); {spv::OpDecorate, input.variable_id, spv::DecorationLocation, location});
continue; continue;
} }
const auto builtin = BuiltInForInput(input.kind); const auto builtin = BuiltInForInput(input.kind);
if (builtin != UINT32_MAX) { if (builtin != UINT32_MAX) {
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationBuiltIn, builtin}); {spv::OpDecorate, input.variable_id, spv::DecorationBuiltIn, builtin});
} }
} }
} }
void AddOutputAnnotationsAndNames(EmitterState& state) { void AddOutputAnnotationsAndNames(EmitterState& state) {
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
return; return;
} }
if (state.per_vertex_variable != 0) { if (state.per_vertex_variable != 0) {
@@ -541,30 +543,31 @@ void AddOutputAnnotationsAndNames(EmitterState& state) {
auto AddBuiltIn = [&](uint32_t variable, const char* name, uint32_t builtin) { auto AddBuiltIn = [&](uint32_t variable, const char* name, uint32_t builtin) {
if (variable != 0) { if (variable != 0) {
state.builder.AddName(variable, name); state.builder.AddName(variable, name);
state.builder.AddAnnotation({OpDecorate, variable, DecorationBuiltIn, builtin}); state.builder.AddAnnotation(
{spv::OpDecorate, variable, spv::DecorationBuiltIn, builtin});
} }
}; };
AddBuiltIn(state.point_size_variable, "gl_PointSize", BuiltInPointSize); AddBuiltIn(state.point_size_variable, "gl_PointSize", spv::BuiltInPointSize);
AddBuiltIn(state.clip_distance_variable, "gl_ClipDistance", BuiltInClipDistance); AddBuiltIn(state.clip_distance_variable, "gl_ClipDistance", spv::BuiltInClipDistance);
AddBuiltIn(state.cull_distance_variable, "gl_CullDistance", BuiltInCullDistance); AddBuiltIn(state.cull_distance_variable, "gl_CullDistance", spv::BuiltInCullDistance);
AddBuiltIn(state.layer_variable, "gl_Layer", BuiltInLayer); AddBuiltIn(state.layer_variable, "gl_Layer", spv::BuiltInLayer);
AddBuiltIn(state.viewport_index_variable, "gl_ViewportIndex", BuiltInViewportIndex); AddBuiltIn(state.viewport_index_variable, "gl_ViewportIndex", spv::BuiltInViewportIndex);
if (state.depth_variable != 0) { if (state.depth_variable != 0) {
state.builder.AddName(state.depth_variable, "gl_FragDepth"); state.builder.AddName(state.depth_variable, "gl_FragDepth");
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, state.depth_variable, DecorationBuiltIn, BuiltInFragDepth}); {spv::OpDecorate, state.depth_variable, spv::DecorationBuiltIn, spv::BuiltInFragDepth});
} }
if (state.sample_mask_variable != 0) { if (state.sample_mask_variable != 0) {
state.builder.AddName(state.sample_mask_variable, "gl_SampleMask"); state.builder.AddName(state.sample_mask_variable, "gl_SampleMask");
state.builder.AddAnnotation( state.builder.AddAnnotation({spv::OpDecorate, state.sample_mask_variable,
{OpDecorate, state.sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask}); spv::DecorationBuiltIn, spv::BuiltInSampleMask});
} }
for (const auto& binding: state.outputs) { for (const auto& binding: state.outputs) {
if (binding.kind == IR::StageOutputKind::Parameter || if (binding.kind == IR::StageOutputKind::Parameter ||
binding.kind == IR::StageOutputKind::Mrt) { binding.kind == IR::StageOutputKind::Mrt) {
state.builder.AddName(binding.variable_id, binding.debug_name.c_str()); state.builder.AddName(binding.variable_id, binding.debug_name.c_str());
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, binding.variable_id, DecorationLocation, binding.location}); {spv::OpDecorate, binding.variable_id, spv::DecorationLocation, binding.location});
} }
} }
} }
@@ -575,9 +578,9 @@ void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name
return; return;
} }
state.builder.AddName(variable, name); state.builder.AddName(variable, name);
state.builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, 0}); state.builder.AddAnnotation({spv::OpDecorate, variable, spv::DecorationDescriptorSet, 0});
state.builder.AddAnnotation( state.builder.AddAnnotation({spv::OpDecorate, variable, spv::DecorationBinding,
{OpDecorate, variable, DecorationBinding, IR::NativeBinding(state.program.stage, kind)}); IR::NativeBinding(state.program.stage, kind)});
} }
void AddDescriptorAnnotationsAndNames(EmitterState& state) { void AddDescriptorAnnotationsAndNames(EmitterState& state) {
@@ -591,9 +594,9 @@ void AddDescriptorAnnotationsAndNames(EmitterState& state) {
Decorate(state.storage_buffer_u64_variable, "buffers_u64", Decorate(state.storage_buffer_u64_variable, "buffers_u64",
IR::DescriptorBindingKind::Buffers); IR::DescriptorBindingKind::Buffers);
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, state.storage_buffer_variable, DecorationAliased}); {spv::OpDecorate, state.storage_buffer_variable, spv::DecorationAliased});
state.builder.AddAnnotation( state.builder.AddAnnotation(
{OpDecorate, state.storage_buffer_u64_variable, DecorationAliased}); {spv::OpDecorate, state.storage_buffer_u64_variable, spv::DecorationAliased});
} }
if (state.bda_pagetable_variable != 0) { if (state.bda_pagetable_variable != 0) {
Decorate(state.bda_pagetable_variable, "bda_pagetable", Decorate(state.bda_pagetable_variable, "bda_pagetable",
@@ -645,82 +648,84 @@ void DefineModule(EmitterState& state) {
} }
} }
state.main_func = state.builder.AllocateId(); state.main_func = state.builder.AllocateId();
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
state.mesh_guest_func = state.builder.AllocateId(); state.mesh_guest_func = state.builder.AllocateId();
state.builder.RequireCapability(5283u); // MeshShadingEXT state.builder.RequireCapability(spv::CapabilityMeshShadingEXT); // MeshShadingEXT
state.builder.RequireExtension("SPV_EXT_mesh_shader"); state.builder.RequireExtension("SPV_EXT_mesh_shader");
state.builder.AddExecutionMode({state.main_func, 5298u}); // OutputTrianglesEXT
state.builder.AddExecutionMode( state.builder.AddExecutionMode(
{state.main_func, 26u, state.input_info.vertex->mesh.max_vertices}); {state.main_func, spv::ExecutionModeOutputTrianglesEXT}); // OutputTrianglesEXT
state.builder.AddExecutionMode( state.builder.AddExecutionMode({state.main_func, spv::ExecutionModeOutputVertices,
{state.main_func, 5270u, state.input_info.vertex->mesh.max_primitives}); state.input_info.vertex->mesh.max_vertices});
state.builder.AddExecutionMode({state.main_func, spv::ExecutionModeOutputPrimitivesEXT,
state.input_info.vertex->mesh.max_primitives});
} }
state.entry_label = state.builder.AllocateId(); state.entry_label = state.builder.AllocateId();
state.builder.RequireCapability(CapabilityShader); state.builder.RequireCapability(spv::CapabilityShader);
state.builder.RequireCapability(CapabilitySignedZeroInfNanPreserve); state.builder.RequireCapability(spv::CapabilitySignedZeroInfNanPreserve);
if (state.program.info.uses_dma) { if (state.program.info.uses_dma) {
state.builder.RequireCapability(CapabilityInt64); state.builder.RequireCapability(spv::CapabilityInt64);
state.builder.RequireCapability(CapabilityPhysicalStorageBufferAddresses); state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer"); state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
} }
if (state.requirements.buffer_int64_atomics) { if (state.requirements.buffer_int64_atomics) {
state.builder.RequireCapability(CapabilityInt64); state.builder.RequireCapability(spv::CapabilityInt64);
state.builder.RequireCapability(CapabilityInt64Atomics); state.builder.RequireCapability(spv::CapabilityInt64Atomics);
} }
if (state.clip_distance_variable != 0) { if (state.clip_distance_variable != 0) {
state.builder.RequireCapability(CapabilityClipDistance); state.builder.RequireCapability(spv::CapabilityClipDistance);
} }
if (state.cull_distance_variable != 0) { if (state.cull_distance_variable != 0) {
state.builder.RequireCapability(CapabilityCullDistance); state.builder.RequireCapability(spv::CapabilityCullDistance);
} }
if (state.layer_variable != 0 || InputVariableForKind(state, IR::StageInputKind::Layer) != 0) { if (state.layer_variable != 0 || InputVariableForKind(state, IR::StageInputKind::Layer) != 0) {
state.builder.RequireVersion(0x00010500u); state.builder.RequireVersion(0x00010500u);
state.builder.RequireCapability(CapabilityShaderLayer); state.builder.RequireCapability(spv::CapabilityShaderLayer);
} }
if (state.viewport_index_variable != 0) { if (state.viewport_index_variable != 0) {
state.builder.RequireVersion(0x00010500u); state.builder.RequireVersion(0x00010500u);
state.builder.RequireCapability(CapabilityShaderViewportIndex); state.builder.RequireCapability(spv::CapabilityShaderViewportIndex);
} }
if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) { if (InputVariableForKind(state, IR::StageInputKind::SampleId) != 0) {
state.builder.RequireCapability(CapabilitySampleRateShading); state.builder.RequireCapability(spv::CapabilitySampleRateShading);
} }
if (state.requirements.image_gather_extended) { if (state.requirements.image_gather_extended) {
state.builder.RequireCapability(CapabilityImageGatherExtended); state.builder.RequireCapability(spv::CapabilityImageGatherExtended);
} }
if (state.lane_count == 2 || state.requirements.subgroup_ballot || if (state.lane_count == 2 || state.requirements.subgroup_ballot ||
state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) { state.requirements.subgroup_shuffle || state.requirements.subgroup_local_invocation_id) {
state.builder.RequireCapability(CapabilityGroupNonUniform); state.builder.RequireCapability(spv::CapabilityGroupNonUniform);
} }
if (state.lane_count == 2 || state.requirements.subgroup_ballot) { if (state.lane_count == 2 || state.requirements.subgroup_ballot) {
state.builder.RequireCapability(CapabilityGroupNonUniformBallot); state.builder.RequireCapability(spv::CapabilityGroupNonUniformBallot);
} }
if (state.requirements.subgroup_shuffle) { if (state.requirements.subgroup_shuffle) {
state.builder.RequireCapability(CapabilityGroupNonUniformShuffle); state.builder.RequireCapability(spv::CapabilityGroupNonUniformShuffle);
} }
if (state.requirements.compute_derivatives && state.stage == ShaderType::Compute) { if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) {
state.builder.RequireCapability(CapabilityComputeDerivativeGroupQuadsKHR); state.builder.RequireCapability(spv::CapabilityComputeDerivativeGroupQuadsKHR);
state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives"); state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives");
} }
const bool fragment_barycentric = const bool fragment_barycentric =
state.stage == ShaderType::Pixel && state.program.stage == ShaderType::Pixel &&
std::any_of(state.inputs.begin(), state.inputs.end(), [](const InputBinding& input) { std::any_of(state.inputs.begin(), state.inputs.end(), [](const InputBinding& input) {
return input.per_vertex || input.kind == IR::StageInputKind::BaryCoordSmooth || return input.per_vertex || input.kind == IR::StageInputKind::BaryCoordSmooth ||
input.kind == IR::StageInputKind::BaryCoordNoPerspective; input.kind == IR::StageInputKind::BaryCoordNoPerspective;
}); });
if (fragment_barycentric) { if (fragment_barycentric) {
state.builder.RequireCapability(CapabilityFragmentBarycentricKHR); state.builder.RequireCapability(spv::CapabilityFragmentBarycentricKHR);
state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric"); state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
} }
state.builder.RequireExtension("SPV_KHR_float_controls"); state.builder.RequireExtension("SPV_KHR_float_controls");
state.builder.AddMemoryModel( state.builder.AddMemoryModel({state.program.info.uses_dma
{state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64 ? spv::AddressingModelPhysicalStorageBuffer64
: AddressingModelLogical, : spv::AddressingModelLogical,
MemoryModelGLSL450}); spv::MemoryModelGLSL450});
// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that // GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
// contract prevents host compilers from treating synthesized IEEE values as finite. // contract prevents host compilers from treating synthesized IEEE values as finite.
state.builder.AddExecutionMode({state.main_func, ExecutionModeSignedZeroInfNanPreserve, 32u}); state.builder.AddExecutionMode(
if (const auto* cs = ShaderWorkgroupInput(state.stage, state.input_info)) { {state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve, 32u});
if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) {
uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u; uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u; uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u;
uint32_t local_z = 1u; uint32_t local_z = 1u;
@@ -736,21 +741,22 @@ void DefineModule(EmitterState& state) {
} }
} }
state.builder.AddExecutionMode( state.builder.AddExecutionMode(
{state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z}); {state.main_func, spv::ExecutionModeLocalSize, local_x, local_y, local_z});
} }
if (state.stage == ShaderType::Pixel) { if (state.program.stage == ShaderType::Pixel) {
state.builder.AddExecutionMode({state.main_func, ExecutionModeOriginUpperLeft}); state.builder.AddExecutionMode({state.main_func, spv::ExecutionModeOriginUpperLeft});
if (state.depth_variable != 0) { if (state.depth_variable != 0) {
state.builder.AddExecutionMode({state.main_func, ExecutionModeDepthReplacing}); state.builder.AddExecutionMode({state.main_func, spv::ExecutionModeDepthReplacing});
} }
if (state.input_info.pixel->ps_early_z && !state.input_info.pixel->ps_pixel_kill_enable && if (state.input_info.pixel->ps_early_z && !state.input_info.pixel->ps_pixel_kill_enable &&
!state.input_info.pixel->ps_depth_export_enable && !state.input_info.pixel->ps_depth_export_enable &&
!state.input_info.pixel->ps_sample_mask_export_enable) { !state.input_info.pixel->ps_sample_mask_export_enable) {
state.builder.AddExecutionMode({state.main_func, ExecutionModeEarlyFragmentTests}); state.builder.AddExecutionMode({state.main_func, spv::ExecutionModeEarlyFragmentTests});
} }
} }
if (state.requirements.compute_derivatives && state.stage == ShaderType::Compute) { if (state.requirements.compute_derivatives && state.program.stage == ShaderType::Compute) {
state.builder.AddExecutionMode({state.main_func, ExecutionModeDerivativeGroupQuadsKHR}); state.builder.AddExecutionMode(
{state.main_func, spv::ExecutionModeDerivativeGroupQuadsKHR});
} }
state.builder.AddName(state.main_func, "main"); state.builder.AddName(state.main_func, "main");
if (state.requirements.function_lds) { if (state.requirements.function_lds) {
@@ -767,108 +773,110 @@ void DefineModule(EmitterState& state) {
AddVsharpAnnotationsAndNames(state); AddVsharpAnnotationsAndNames(state);
if (state.subgroup_local_invocation_id_variable != 0) { if (state.subgroup_local_invocation_id_variable != 0) {
state.builder.DefineGlobalVariable(state.subgroup_local_invocation_id_variable, state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassInput, TypeU32(state)), state.subgroup_local_invocation_id_variable,
StorageClassInput); TypePointer(state, spv::StorageClassInput, TypeU32(state)), spv::StorageClassInput);
} }
for (const auto& input: state.inputs) { for (const auto& input: state.inputs) {
uint32_t ptr_type = TypePointer(state, StorageClassInput, TypeU32(state)); uint32_t ptr_type = TypePointer(state, spv::StorageClassInput, TypeU32(state));
switch (input.kind) { switch (input.kind) {
case IR::StageInputKind::VertexIndex: case IR::StageInputKind::VertexIndex:
case IR::StageInputKind::InstanceIndex: case IR::StageInputKind::InstanceIndex:
case IR::StageInputKind::Layer: case IR::StageInputKind::Layer:
case IR::StageInputKind::SampleId: case IR::StageInputKind::SampleId:
ptr_type = TypePointer(state, StorageClassInput, TypeI32(state)); ptr_type = TypePointer(state, spv::StorageClassInput, TypeI32(state));
break; break;
case IR::StageInputKind::WorkgroupId: case IR::StageInputKind::WorkgroupId:
case IR::StageInputKind::LocalInvocationId: case IR::StageInputKind::LocalInvocationId:
case IR::StageInputKind::GlobalInvocationId: case IR::StageInputKind::GlobalInvocationId:
ptr_type = TypePointer(state, StorageClassInput, TypeU32Vector(state, 3)); ptr_type = TypePointer(state, spv::StorageClassInput, TypeU32Vector(state, 3));
break; break;
case IR::StageInputKind::FragCoord: case IR::StageInputKind::FragCoord:
ptr_type = TypePointer(state, StorageClassInput, TypeF32Vector(state, 4)); ptr_type = TypePointer(state, spv::StorageClassInput, TypeF32Vector(state, 4));
break; break;
case IR::StageInputKind::BaryCoordSmooth: case IR::StageInputKind::BaryCoordSmooth:
case IR::StageInputKind::BaryCoordNoPerspective: case IR::StageInputKind::BaryCoordNoPerspective:
ptr_type = TypePointer(state, StorageClassInput, TypeF32Vector(state, 3)); ptr_type = TypePointer(state, spv::StorageClassInput, TypeF32Vector(state, 3));
break; break;
case IR::StageInputKind::FrontFacing: case IR::StageInputKind::FrontFacing:
ptr_type = TypePointer(state, StorageClassInput, TypeBool(state)); ptr_type = TypePointer(state, spv::StorageClassInput, TypeBool(state));
break; break;
case IR::StageInputKind::Parameter: case IR::StageInputKind::Parameter:
if (state.stage == ShaderType::Vertex) { if (state.program.stage == ShaderType::Vertex) {
const auto kind = VertexParameterScalarKind(state, input.location); const auto kind = VertexParameterScalarKind(state, input.location);
const auto components = VertexParameterComponentCount(input); const auto components = VertexParameterComponentCount(input);
ptr_type = VertexParameterInputPointerType(state, kind, components); ptr_type = VertexParameterInputPointerType(state, kind, components);
} else if (input.per_vertex) { } else if (input.per_vertex) {
const auto array_type = state.builder.Type( const auto array_type = state.builder.Type(
OpTypeArray, {TypeF32Vector(state, 4), ConstantU32(state, 3)}); spv::OpTypeArray, {TypeF32Vector(state, 4), ConstantU32(state, 3)});
ptr_type = TypePointer(state, StorageClassInput, array_type); ptr_type = TypePointer(state, spv::StorageClassInput, array_type);
} else { } else {
ptr_type = TypePointer(state, StorageClassInput, TypeF32Vector(state, 4)); ptr_type = TypePointer(state, spv::StorageClassInput, TypeF32Vector(state, 4));
} }
break; break;
default: break; default: break;
} }
state.builder.DefineGlobalVariable(input.variable_id, ptr_type, StorageClassInput); state.builder.DefineGlobalVariable(input.variable_id, ptr_type, spv::StorageClassInput);
} }
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
return; return;
} }
if (state.per_vertex_variable != 0) { if (state.per_vertex_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.per_vertex_variable, TypePointer(state, StorageClassOutput, PerVertexType(state)), state.per_vertex_variable,
StorageClassOutput); TypePointer(state, spv::StorageClassOutput, PerVertexType(state)),
spv::StorageClassOutput);
} }
if (state.point_size_variable != 0) { if (state.point_size_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.point_size_variable, TypePointer(state, StorageClassOutput, TypeF32(state)), state.point_size_variable, TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
StorageClassOutput); spv::StorageClassOutput);
} }
if (state.clip_distance_variable != 0) { if (state.clip_distance_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.clip_distance_variable, state.clip_distance_variable,
TypePointer(state, StorageClassOutput, TypePointer(state, spv::StorageClassOutput,
F32ArrayType(state, state.clip_distance_count)), F32ArrayType(state, state.clip_distance_count)),
StorageClassOutput); spv::StorageClassOutput);
} }
if (state.cull_distance_variable != 0) { if (state.cull_distance_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.cull_distance_variable, state.cull_distance_variable,
TypePointer(state, StorageClassOutput, TypePointer(state, spv::StorageClassOutput,
F32ArrayType(state, state.cull_distance_count)), F32ArrayType(state, state.cull_distance_count)),
StorageClassOutput); spv::StorageClassOutput);
} }
if (state.layer_variable != 0) { if (state.layer_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.layer_variable, TypePointer(state, StorageClassOutput, TypeU32(state)), state.layer_variable, TypePointer(state, spv::StorageClassOutput, TypeU32(state)),
StorageClassOutput); spv::StorageClassOutput);
} }
if (state.viewport_index_variable != 0) { if (state.viewport_index_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.viewport_index_variable, TypePointer(state, StorageClassOutput, TypeU32(state)), state.viewport_index_variable,
StorageClassOutput); TypePointer(state, spv::StorageClassOutput, TypeU32(state)), spv::StorageClassOutput);
} }
for (const auto& binding: state.outputs) { for (const auto& binding: state.outputs) {
if (binding.kind == IR::StageOutputKind::Parameter || if (binding.kind == IR::StageOutputKind::Parameter ||
binding.kind == IR::StageOutputKind::Mrt) { binding.kind == IR::StageOutputKind::Mrt) {
const auto pointer_type = const auto pointer_type =
binding.kind == IR::StageOutputKind::Mrt && MrtUsesUintOutput(state, binding.index) binding.kind == IR::StageOutputKind::Mrt && MrtUsesUintOutput(state, binding.index)
? TypePointer(state, StorageClassOutput, TypeU32Vector(state, 4)) ? TypePointer(state, spv::StorageClassOutput, TypeU32Vector(state, 4))
: TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)); : TypePointer(state, spv::StorageClassOutput, TypeF32Vector(state, 4));
state.builder.DefineGlobalVariable(binding.variable_id, pointer_type, state.builder.DefineGlobalVariable(binding.variable_id, pointer_type,
StorageClassOutput); spv::StorageClassOutput);
} }
} }
if (state.depth_variable != 0) { if (state.depth_variable != 0) {
state.builder.DefineGlobalVariable(state.depth_variable, state.builder.DefineGlobalVariable(
TypePointer(state, StorageClassOutput, TypeF32(state)), state.depth_variable, TypePointer(state, spv::StorageClassOutput, TypeF32(state)),
StorageClassOutput); spv::StorageClassOutput);
} }
if (state.sample_mask_variable != 0) { if (state.sample_mask_variable != 0) {
state.builder.DefineGlobalVariable( state.builder.DefineGlobalVariable(
state.sample_mask_variable, state.sample_mask_variable,
TypePointer(state, StorageClassOutput, SampleMaskArrayType(state)), StorageClassOutput); TypePointer(state, spv::StorageClassOutput, SampleMaskArrayType(state)),
spv::StorageClassOutput);
} }
} }
@@ -15,11 +15,11 @@ void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
const auto kill_label = state.builder.AllocateId(); const auto kill_label = state.builder.AllocateId();
const auto merge_label = state.builder.AllocateId(); const auto merge_label = state.builder.AllocateId();
const auto inactive = state.builder.AllocateId(); const auto inactive = state.builder.AllocateId();
state.builder.AddFunction({OpLogicalNot, TypeBool(state), inactive, active}); state.builder.AddFunction({spv::OpLogicalNot, TypeBool(state), inactive, active});
state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone}); state.builder.AddFunction({spv::OpSelectionMerge, merge_label, spv::SelectionControlMaskNone});
state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label}); state.builder.AddFunction({spv::OpBranchConditional, inactive, kill_label, merge_label});
EmitLabel(state, kill_label); EmitLabel(state, kill_label);
state.builder.AddFunction({OpKill}); state.builder.AddFunction({spv::OpKill});
EmitLabel(state, merge_label); EmitLabel(state, merge_label);
} }
@@ -31,15 +31,15 @@ void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
const auto mask_value = state.builder.AllocateId(); const auto mask_value = state.builder.AllocateId();
const auto active = state.builder.AllocateId(); const auto active = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable}); {spv::OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
EmitKillIfBoolFalse(state, active); EmitKillIfBoolFalse(state, active);
} }
uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) { uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
const auto value_type = ctx.TypeId(type); const auto value_type = ctx.TypeId(type);
return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type); return value_type == 0 ? 0 : TypePointer(ctx.state, spv::StorageClassFunction, value_type);
} }
struct DeferredPhiPatch { struct DeferredPhiPatch {
@@ -74,7 +74,7 @@ void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState
for (size_t index = 0; index < phi.NumArgs(); index++) { for (size_t index = 0; index < phi.NumArgs(); index++) {
if (phi.PhiBlock(index) == from) { if (phi.PhiBlock(index) == from) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))}); {spv::OpStore, dispatcher.spills[ctx.half].at(&phi), ctx.Def(phi.Arg(index))});
break; break;
} }
} }
@@ -92,7 +92,7 @@ const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
void EmitReturn(ValueEmitContext& ctx) { void EmitReturn(ValueEmitContext& ctx) {
EmitKillIfPixelValidMaskInactive(ctx.state); EmitKillIfPixelValidMaskInactive(ctx.state);
ctx.state.builder.AddFunction({OpReturn}); ctx.state.builder.AddFunction({spv::OpReturn});
} }
uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) { uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
@@ -106,15 +106,16 @@ uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
const auto low = ctx.state.builder.AllocateId(); const auto low = ctx.state.builder.AllocateId();
const auto high = ctx.state.builder.AllocateId(); const auto high = ctx.state.builder.AllocateId();
const auto result = ctx.state.builder.AllocateId(); const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0}); ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0});
ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1}); ctx.state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1});
const auto kind = info.terminator.condition; const auto kind = info.terminator.condition;
const bool zero = kind == CFG::BranchCondition::ExecZero || const bool zero = kind == CFG::BranchCondition::ExecZero ||
kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::VccZero ||
kind == CFG::BranchCondition::SccZero; kind == CFG::BranchCondition::SccZero;
const auto combined = EmitBinaryU32(ctx.state, zero ? OpBitwiseAnd : OpBitwiseOr, low, high); const auto combined =
ctx.state.builder.AddFunction({zero ? OpIEqual : OpINotEqual, TypeBool(ctx.state), result, EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
combined, ConstantU32(ctx.state, zero ? ~0u : 0u)}); ctx.state.builder.AddFunction({zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u)});
return result; return result;
} }
@@ -127,14 +128,14 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
const auto* merge = TargetBlock(program, term.merge_block); const auto* merge = TargetBlock(program, term.merge_block);
const auto* cont = TargetBlock(program, term.continue_block); const auto* cont = TargetBlock(program, term.continue_block);
if (merge != nullptr && cont != nullptr) { if (merge != nullptr && cont != nullptr) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction({spv::OpLoopMerge, ctx.Label(merge), ctx.Label(cont),
{OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone}); spv::LoopControlMaskNone});
} }
} else if (term.kind == CFG::TerminatorKind::ConditionalBranch && } else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
term.merge_block != UINT32_MAX) { term.merge_block != UINT32_MAX) {
if (const auto* merge = TargetBlock(program, term.merge_block); merge != nullptr) { if (const auto* merge = TargetBlock(program, term.merge_block); merge != nullptr) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpSelectionMerge, ctx.Label(merge), SelectionControlNone}); {spv::OpSelectionMerge, ctx.Label(merge), spv::SelectionControlMaskNone});
} }
} }
}; };
@@ -147,7 +148,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
return; return;
} }
emit_merge(); emit_merge();
ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)}); ctx.state.builder.AddFunction({spv::OpBranch, ctx.Label(target)});
return; return;
} }
case CFG::TerminatorKind::ConditionalBranch: { case CFG::TerminatorKind::ConditionalBranch: {
@@ -159,8 +160,8 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
} }
const auto condition = BranchCondition(ctx, info); const auto condition = BranchCondition(ctx, info);
emit_merge(); emit_merge();
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction({spv::OpBranchConditional, condition,
{OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)}); ctx.Label(true_block), ctx.Label(false_block)});
return; return;
} }
default: EmitReturn(ctx); return; default: EmitReturn(ctx); return;
@@ -189,7 +190,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
EmitDispatcherTarget(ctx, dispatcher, block, term.true_block); EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
EmitDispatcherTarget(ctx, dispatcher, block, term.false_block); EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
const auto selected = ctx.state.builder.AllocateId(); const auto selected = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected, ctx.state.builder.AddFunction({spv::OpSelect, TypeU32(ctx.state), selected,
BranchCondition(ctx, info), BranchCondition(ctx, info),
ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.true_block),
ConstantU32(ctx.state, term.false_block)}); ConstantU32(ctx.state, term.false_block)});
@@ -211,9 +212,10 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) { for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
const auto match = ctx.state.builder.AllocateId(); const auto match = ctx.state.builder.AllocateId();
const auto next = ctx.state.builder.AllocateId(); const auto next = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction({OpIEqual, TypeBool(ctx.state), match, selector, ctx.state.builder.AddFunction({spv::OpIEqual, TypeBool(ctx.state), match,
selector,
ConstantU32(ctx.state, values[index])}); ConstantU32(ctx.state, values[index])});
ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), next, match, ctx.state.builder.AddFunction({spv::OpSelect, TypeU32(ctx.state), next, match,
ConstantU32(ctx.state, targets[index]), ConstantU32(ctx.state, targets[index]),
selected}); selected});
selected = next; selected = next;
@@ -262,14 +264,14 @@ void EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionSt
ctx.Fail(inst, "cannot be loaded by the dispatcher"); ctx.Fail(inst, "cannot be loaded by the dispatcher");
} }
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)}); {spv::OpLoad, type, ctx.Result(inst), dispatcher.spills[ctx.half].at(&inst)});
return; return;
} }
EmitDirectInstruction(ctx, inst); EmitDirectInstruction(ctx, inst);
if (const auto found = dispatcher.spills[ctx.half].find(&inst); if (const auto found = dispatcher.spills[ctx.half].find(&inst);
found != dispatcher.spills[ctx.half].end()) { found != dispatcher.spills[ctx.half].end()) {
ctx.state.builder.AddFunction( ctx.state.builder.AddFunction(
{OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))}); {spv::OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
} }
} }
@@ -316,7 +318,7 @@ void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structu
void EmitStructuredFunction(ValueEmitContext& ctx) { void EmitStructuredFunction(ValueEmitContext& ctx) {
const auto& program = ctx.state.program; const auto& program = ctx.state.program;
StructuredFunctionState structured; StructuredFunctionState structured;
ctx.state.builder.AddFunction({OpBranch, ctx.Label(program.blocks.front())}); ctx.state.builder.AddFunction({spv::OpBranch, ctx.Label(program.blocks.front())});
for (size_t index = 0; index < program.blocks.size(); index++) { for (size_t index = 0; index < program.blocks.size(); index++) {
const auto* block = program.blocks[index]; const auto* block = program.blocks[index];
EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) { EmitBlock(ctx, block, [&](ValueEmitContext& lane, const IR::Inst& inst) {
@@ -331,38 +333,38 @@ void EmitStructuredFunction(ValueEmitContext& ctx) {
void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) { void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
auto& state = ctx.state; auto& state = ctx.state;
const auto* entry = state.program.blocks.front(); const auto* entry = state.program.blocks.front();
state.builder.AddFunction({OpBranch, ctx.Label(entry)}); state.builder.AddFunction({spv::OpBranch, ctx.Label(entry)});
EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) { EmitBlock(ctx, entry, [&](ValueEmitContext& lane, const IR::Inst& inst) {
EmitDispatcherInstruction(lane, dispatcher, inst); EmitDispatcherInstruction(lane, dispatcher, inst);
}); });
const auto initial_pc = const auto initial_pc =
EmitDispatcherNextPc(ctx, dispatcher, entry, state.program.block_info.front()); EmitDispatcherNextPc(ctx, dispatcher, entry, state.program.block_info.front());
const auto initial_parent = state.current_label; const auto initial_parent = state.current_label;
state.builder.AddFunction({OpBranch, dispatcher.header_label}); state.builder.AddFunction({spv::OpBranch, dispatcher.header_label});
EmitLabel(state, dispatcher.header_label); EmitLabel(state, dispatcher.header_label);
const auto pc = state.builder.AllocateId(); const auto pc = state.builder.AllocateId();
const auto next_pc = state.builder.AllocateId(); const auto next_pc = state.builder.AllocateId();
state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc, state.builder.AddFunction({spv::OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
dispatcher.continue_label}); dispatcher.continue_label});
const auto done = state.builder.AllocateId(); const auto done = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)}); {spv::OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
state.builder.AddFunction({spv::OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label,
spv::LoopControlMaskNone});
state.builder.AddFunction( state.builder.AddFunction(
{OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label, LoopControlNone}); {spv::OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
state.builder.AddFunction(
{OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
EmitLabel(state, dispatcher.select_label); EmitLabel(state, dispatcher.select_label);
state.builder.AddFunction( state.builder.AddFunction(
{OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone}); {spv::OpSelectionMerge, dispatcher.after_switch_label, spv::SelectionControlMaskNone});
std::vector<uint32_t> words {OpSwitch, pc, dispatcher.after_switch_label}; std::vector<uint32_t> words {spv::OpSwitch, pc, dispatcher.after_switch_label};
for (size_t index = 1; index < state.program.blocks.size(); index++) { for (size_t index = 1; index < state.program.blocks.size(); index++) {
words.push_back(state.program.block_info[index].id); words.push_back(state.program.block_info[index].id);
words.push_back(ctx.Label(state.program.blocks[index])); words.push_back(ctx.Label(state.program.blocks[index]));
} }
state.builder.AddFunction(words); state.builder.AddFunction(words);
std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc, std::vector<uint32_t> next_pc_words {spv::OpPhi, TypeU32(state), next_pc,
ConstantU32(state, UINT32_MAX), dispatcher.select_label}; ConstantU32(state, UINT32_MAX), dispatcher.select_label};
for (size_t index = 1; index < state.program.blocks.size(); index++) { for (size_t index = 1; index < state.program.blocks.size(); index++) {
@@ -374,13 +376,13 @@ void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState
state.program.block_info[index]); state.program.block_info[index]);
next_pc_words.push_back(selected); next_pc_words.push_back(selected);
next_pc_words.push_back(state.current_label); next_pc_words.push_back(state.current_label);
state.builder.AddFunction({OpBranch, dispatcher.after_switch_label}); state.builder.AddFunction({spv::OpBranch, dispatcher.after_switch_label});
} }
EmitLabel(state, dispatcher.after_switch_label); EmitLabel(state, dispatcher.after_switch_label);
state.builder.AddFunction(next_pc_words); state.builder.AddFunction(next_pc_words);
state.builder.AddFunction({OpBranch, dispatcher.continue_label}); state.builder.AddFunction({spv::OpBranch, dispatcher.continue_label});
EmitLabel(state, dispatcher.continue_label); EmitLabel(state, dispatcher.continue_label);
state.builder.AddFunction({OpBranch, dispatcher.header_label}); state.builder.AddFunction({spv::OpBranch, dispatcher.header_label});
EmitLabel(state, dispatcher.merge_label); EmitLabel(state, dispatcher.merge_label);
EmitReturn(ctx); EmitReturn(ctx);
} }
@@ -432,7 +434,7 @@ uint32_t ValueEmitContext::Def(IR::Value value) {
return loaded->second.second; return loaded->second.second;
} }
const auto id = state.builder.AllocateId(); const auto id = state.builder.AllocateId();
state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second}); state.builder.AddFunction({spv::OpLoad, TypeId(inst->GetType()), id, found->second});
dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id}); dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
return id; return id;
} }
@@ -450,10 +452,10 @@ uint32_t ValueEmitContext::HalfArg(const IR::Inst& inst, size_t index, uint32_t
uint32_t ValueEmitContext::Ballot(IR::Value predicate) { uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
const auto ballot_type = TypeU32Vector(state, 4); const auto ballot_type = TypeU32Vector(state, 4);
const auto scope = ConstantU32(state, ScopeSubgroup); const auto scope = ConstantU32(state, spv::ScopeSubgroup);
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpGroupNonUniformBallot, ballot_type, low, scope, {spv::OpGroupNonUniformBallot, ballot_type, low, scope,
other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)}); other_half == nullptr || half == 0 ? Def(predicate) : other_half->Def(predicate)});
if (other_half == nullptr) { if (other_half == nullptr) {
return low; return low;
@@ -462,11 +464,11 @@ uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
const auto low_word = state.builder.AllocateId(); const auto low_word = state.builder.AllocateId();
const auto high_word = state.builder.AllocateId(); const auto high_word = state.builder.AllocateId();
const auto ballot = state.builder.AllocateId(); const auto ballot = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallot, ballot_type, high, scope, state.builder.AddFunction({spv::OpGroupNonUniformBallot, ballot_type, high, scope,
half == 1 ? Def(predicate) : other_half->Def(predicate)}); half == 1 ? Def(predicate) : other_half->Def(predicate)});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low_word, low, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low_word, low, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high_word, high, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high_word, high, 0});
state.builder.AddFunction({OpCompositeConstruct, ballot_type, ballot, low_word, high_word, state.builder.AddFunction({spv::OpCompositeConstruct, ballot_type, ballot, low_word, high_word,
ConstantU32(state, 0), ConstantU32(state, 0)}); ConstantU32(state, 0), ConstantU32(state, 0)});
return ballot; return ballot;
} }
@@ -474,8 +476,8 @@ uint32_t ValueEmitContext::Ballot(IR::Value predicate) {
uint32_t ValueEmitContext::FirstLane(uint32_t ballot) { uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
if (other_half == nullptr) { if (other_half == nullptr) {
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), result, state.builder.AddFunction({spv::OpGroupNonUniformBallotFindLSB, TypeU32(state), result,
ConstantU32(state, ScopeSubgroup), ballot}); ConstantU32(state, spv::ScopeSubgroup), ballot});
return result; return result;
} }
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
@@ -484,40 +486,42 @@ uint32_t ValueEmitContext::FirstLane(uint32_t ballot) {
const auto high_first = state.builder.AllocateId(); const auto high_first = state.builder.AllocateId();
const auto low_active = state.builder.AllocateId(); const auto low_active = state.builder.AllocateId();
const auto result = state.builder.AllocateId(); const auto result = state.builder.AllocateId();
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), low, ballot, 0});
state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1}); state.builder.AddFunction({spv::OpCompositeExtract, TypeU32(state), high, ballot, 1});
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeU32(state), low_first, GlslStd450(state), GlslFindILsb, low}); {spv::OpExtInst, TypeU32(state), low_first, GlslStd450(state), GLSLstd450FindILsb, low});
state.builder.AddFunction( state.builder.AddFunction(
{OpExtInst, TypeU32(state), high_first, GlslStd450(state), GlslFindILsb, high}); {spv::OpExtInst, TypeU32(state), high_first, GlslStd450(state), GLSLstd450FindILsb, high});
state.builder.AddFunction( state.builder.AddFunction(
{OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)}); {spv::OpINotEqual, TypeBool(state), low_active, low, ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, TypeU32(state), result, low_active, low_first, state.builder.AddFunction({spv::OpSelect, TypeU32(state), result, low_active, low_first,
EmitAddU32(state, high_first, ConstantU32(state, 32))}); EmitAddU32(state, high_first, ConstantU32(state, 32))});
return result; return result;
} }
uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) { uint32_t ValueEmitContext::Shuffle(const IR::Inst& inst, size_t index, uint32_t lane) {
const auto type = TypeId(inst.Arg(index).GetType()); const auto type = TypeId(inst.Arg(index).GetType());
const auto scope = ConstantU32(state, ScopeSubgroup); const auto scope = ConstantU32(state, spv::ScopeSubgroup);
const auto low = state.builder.AllocateId(); const auto low = state.builder.AllocateId();
if (other_half == nullptr) { if (other_half == nullptr) {
state.builder.AddFunction( state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane}); {spv::OpGroupNonUniformShuffle, type, low, scope, Arg(inst, index), lane});
return low; return low;
} }
const auto physical_lane = EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 31)); const auto physical_lane =
EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 31));
const auto high = state.builder.AllocateId(); const auto high = state.builder.AllocateId();
const auto in_high = state.builder.AllocateId(); const auto in_high = state.builder.AllocateId();
const auto value = state.builder.AllocateId(); const auto value = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane}); {spv::OpGroupNonUniformShuffle, type, low, scope, HalfArg(inst, index, 0), physical_lane});
state.builder.AddFunction( state.builder.AddFunction(
{OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane}); {spv::OpGroupNonUniformShuffle, type, high, scope, HalfArg(inst, index, 1), physical_lane});
state.builder.AddFunction({OpINotEqual, TypeBool(state), in_high, state.builder.AddFunction(
EmitBinaryU32(state, OpBitwiseAnd, lane, ConstantU32(state, 32)), {spv::OpINotEqual, TypeBool(state), in_high,
EmitBinaryU32(state, spv::OpBitwiseAnd, lane, ConstantU32(state, 32)),
ConstantU32(state, 0)}); ConstantU32(state, 0)});
state.builder.AddFunction({OpSelect, type, value, in_high, high, low}); state.builder.AddFunction({spv::OpSelect, type, value, in_high, high, low});
return value; return value;
} }
@@ -543,7 +547,8 @@ uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type
uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) { uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
if (const auto found = definitions.find(&inst); found != definitions.end()) { if (const auto found = definitions.find(&inst); found != definitions.end()) {
if (found->second != value) { if (found->second != value) {
state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value}); state.builder.AddFunction(
{spv::OpCopyObject, TypeId(inst.GetType()), found->second, value});
} }
return found->second; return found->second;
} }
@@ -602,7 +607,7 @@ void EmitProgram(EmitterState& state) {
high.half = 1; high.half = 1;
} }
std::optional<DispatcherFunctionState> dispatcher; std::optional<DispatcherFunctionState> dispatcher;
if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) { if (state.program.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
state.pixel_valid_mask_variable = state.builder.AllocateId(); state.pixel_valid_mask_variable = state.builder.AllocateId();
state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active"); state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
} }
@@ -679,27 +684,28 @@ void EmitProgram(EmitterState& state) {
break; break;
} }
} }
state.builder.AddFunction({OpFunction, TypeVoid(state), state.builder.AddFunction({spv::OpFunction, TypeVoid(state),
state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func, state.mesh_guest_func != 0 ? state.mesh_guest_func : state.main_func,
FunctionControlNone, TypeFunction(state)}); spv::FunctionControlMaskNone, TypeFunction(state)});
EmitLabel(state, state.entry_label); EmitLabel(state, state.entry_label);
if (state.requirements.function_lds) { if (state.requirements.function_lds) {
state.builder.AddFunction( state.builder.AddFunction(
{OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)), {spv::OpVariable,
state.lds_variable, StorageClassFunction}); TypeU32ArrayPointer(state, spv::StorageClassFunction, LdsDwordCount(state)),
state.lds_variable, spv::StorageClassFunction});
} }
if (state.requirements.function_scratch) { if (state.requirements.function_scratch) {
for (uint32_t half = 0; half < state.lane_count; half++) { for (uint32_t half = 0; half < state.lane_count; half++) {
state.builder.AddFunction( state.builder.AddFunction({spv::OpVariable,
{OpVariable, TypeU32ArrayPointer(state, spv::StorageClassFunction,
TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords), state.program.scratch_dwords),
state.scratch_variable[half], StorageClassFunction}); state.scratch_variable[half], spv::StorageClassFunction});
} }
} }
if (state.pixel_valid_mask_variable != 0) { if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction({OpVariable, state.builder.AddFunction({spv::OpVariable,
TypePointer(state, StorageClassFunction, TypeU32(state)), TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
state.pixel_valid_mask_variable, StorageClassFunction}); state.pixel_valid_mask_variable, spv::StorageClassFunction});
} }
for (uint32_t half = 0; half < state.lane_count; half++) { for (uint32_t half = 0; half < state.lane_count; half++) {
auto& lane = half == 0 ? ctx : high; auto& lane = half == 0 ? ctx : high;
@@ -708,27 +714,27 @@ void EmitProgram(EmitterState& state) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (const auto found = dispatcher->spills[half].find(&inst); if (const auto found = dispatcher->spills[half].find(&inst);
found != dispatcher->spills[half].end()) { found != dispatcher->spills[half].end()) {
state.builder.AddFunction({OpVariable, state.builder.AddFunction({spv::OpVariable,
SpillPointerType(lane, inst.GetType()), SpillPointerType(lane, inst.GetType()),
found->second, StorageClassFunction}); found->second, spv::StorageClassFunction});
} }
} }
} }
} }
if (lane.scratch_u32_variable != 0) { if (lane.scratch_u32_variable != 0) {
state.builder.AddFunction({OpVariable, state.builder.AddFunction(
TypePointer(state, StorageClassFunction, TypeU32(state)), {spv::OpVariable, TypePointer(state, spv::StorageClassFunction, TypeU32(state)),
lane.scratch_u32_variable, StorageClassFunction}); lane.scratch_u32_variable, spv::StorageClassFunction});
} }
} }
if (state.gds_variable != 0) { if (state.gds_variable != 0) {
state.gds_length = state.builder.AllocateId(); state.gds_length = state.builder.AllocateId();
state.builder.AddFunction( state.builder.AddFunction(
{OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0}); {spv::OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
} }
if (state.pixel_valid_mask_variable != 0) { if (state.pixel_valid_mask_variable != 0) {
state.builder.AddFunction( state.builder.AddFunction(
{OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)}); {spv::OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
} }
EmitMemoryOffsets(state); EmitMemoryOffsets(state);
if (program.blocks.empty()) { if (program.blocks.empty()) {
@@ -738,8 +744,8 @@ void EmitProgram(EmitterState& state) {
} else { } else {
EmitStructuredFunction(ctx); EmitStructuredFunction(ctx);
} }
state.builder.AddFunction({OpFunctionEnd}); state.builder.AddFunction({spv::OpFunctionEnd});
if (state.stage == ShaderType::Mesh) { if (state.program.stage == ShaderType::Mesh) {
EmitMeshEntryPoint(state); EmitMeshEntryPoint(state);
} }
} }
@@ -898,20 +898,23 @@ void ValidateTranslateOptions(const TranslateOptions& options) {
if (options.wave_size != 32u && options.wave_size != 64u) { if (options.wave_size != 32u && options.wave_size != 64u) {
EXIT("shader translation requires wave32 or wave64, got %u", options.wave_size); EXIT("shader translation requires wave32 or wave64, got %u", options.wave_size);
} }
if (options.embedded_fetch != nullptr && options.stage != ShaderType::Vertex) {
EXIT("embedded fetch requires the vertex shader stage");
}
switch (options.stage) { switch (options.stage) {
case ShaderType::Vertex: case ShaderType::Vertex:
case ShaderType::Mesh: case ShaderType::Mesh:
if (options.vertex == nullptr) { if (options.input_info.vertex == nullptr) {
EXIT("vertex shader translation has no vertex input metadata"); EXIT("vertex shader translation has no vertex input metadata");
} }
return; return;
case ShaderType::Pixel: case ShaderType::Pixel:
if (options.pixel == nullptr) { if (options.input_info.pixel == nullptr) {
EXIT("pixel shader translation has no pixel input metadata"); EXIT("pixel shader translation has no pixel input metadata");
} }
return; return;
case ShaderType::Compute: case ShaderType::Compute:
if (options.compute == nullptr) { if (options.input_info.compute == nullptr) {
EXIT("compute shader translation has no compute input metadata"); EXIT("compute shader translation has no compute input metadata");
} }
return; return;
@@ -936,7 +939,21 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
result.shader_hash = options.shader_hash; result.shader_hash = options.shader_hash;
result.user_data_base = options.user_data_base; result.user_data_base = options.user_data_base;
result.user_data_count = options.user_data_count; result.user_data_count = options.user_data_count;
result.scratch_dwords = options.scratch_dwords; switch (options.stage) {
case ShaderType::Vertex:
result.scratch_dwords = options.input_info.vertex->scratch_size_dwords;
break;
case ShaderType::Mesh:
result.scratch_dwords = options.input_info.vertex->mesh.scratch_size_dwords;
break;
case ShaderType::Pixel:
result.scratch_dwords = options.input_info.pixel->scratch_size_dwords;
break;
case ShaderType::Compute:
result.scratch_dwords = options.input_info.compute->scratch_size_dwords;
break;
default: break; // ValidateTranslateOptions rejects unsupported stages.
}
result.dispatcher_fallback = options.dispatcher_fallback; result.dispatcher_fallback = options.dispatcher_fallback;
result.cfg_failure_kind = options.cfg_failure_kind; result.cfg_failure_kind = options.cfg_failure_kind;
result.fallback_reason = options.fallback_reason; result.fallback_reason = options.fallback_reason;
@@ -1017,13 +1034,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
} }
auto initial_exec = IR::U1(IR::Value(true)); auto initial_exec = IR::U1(IR::Value(true));
uint32_t total_threads = 0; uint32_t total_threads = 0;
ShaderStageInputInfo stage_input {}; const auto* workgroup = ShaderWorkgroupInput(options.stage, options.input_info);
if (options.stage == ShaderType::Compute) {
stage_input.compute = options.compute;
} else {
stage_input.vertex = options.vertex;
}
const auto* workgroup = ShaderWorkgroupInput(options.stage, stage_input);
if (workgroup != nullptr) { if (workgroup != nullptr) {
total_threads = std::max(workgroup->threads_num[0], 1u) * total_threads = std::max(workgroup->threads_num[0], 1u) *
std::max(workgroup->threads_num[1], 1u) * std::max(workgroup->threads_num[1], 1u) *
@@ -1040,7 +1051,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
entry_ir.SetExecHi(options.wave_size == 64u ? entry_ir.CompositeExtract(initial_mask, 1) entry_ir.SetExecHi(options.wave_size == 64u ? entry_ir.CompositeExtract(initial_mask, 1)
: IR::U32(IR::Value(0u))); : IR::U32(IR::Value(0u)));
if (options.stage == ShaderType::Compute) { if (options.stage == ShaderType::Compute) {
const auto* cs = options.compute; const auto* cs = options.input_info.compute;
const auto thread_ids = const auto thread_ids =
cs->thread_ids_num > 0 ? std::min<uint32_t>(cs->thread_ids_num, 3u) : 0u; cs->thread_ids_num > 0 ? std::min<uint32_t>(cs->thread_ids_num, 3u) : 0u;
for (uint32_t index = 0; index < thread_ids; index++) { for (uint32_t index = 0; index < thread_ids; index++) {
@@ -1071,7 +1082,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
first_bit)); first_bit));
} }
} else if (options.stage == ShaderType::Mesh) { } else if (options.stage == ShaderType::Mesh) {
const auto& mesh = options.vertex->mesh; const auto& mesh = options.input_info.vertex->mesh;
EXIT_NOT_IMPLEMENTED(options.wave_size != 64u || mesh.primitives_per_group == 0u || EXIT_NOT_IMPLEMENTED(options.wave_size != 64u || mesh.primitives_per_group == 0u ||
mesh.vertices_per_group > 64u || total_threads > 15u * 64u); mesh.vertices_per_group > 64u || total_threads > 15u * 64u);
const auto u32 = [](uint32_t value) { return IR::U32(IR::Value(value)); }; const auto u32 = [](uint32_t value) { return IR::U32(IR::Value(value)); };
@@ -1154,7 +1165,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
static_cast<IR::VectorReg>(8), static_cast<IR::VectorReg>(8),
entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1))); entry_ir.IAdd(draw(2), builtin(IR::StageInputKind::WorkgroupId, 1)));
} else if (options.stage == ShaderType::Pixel) { } else if (options.stage == ShaderType::Pixel) {
const auto* ps = options.pixel; const auto* ps = options.input_info.pixel;
if (ps->ps_perspective_center_vgpr != UINT32_MAX) { if (ps->ps_perspective_center_vgpr != UINT32_MAX) {
entry_ir.SetVectorReg(static_cast<IR::VectorReg>(ps->ps_perspective_center_vgpr), entry_ir.SetVectorReg(static_cast<IR::VectorReg>(ps->ps_perspective_center_vgpr),
builtin(IR::StageInputKind::BaryCoordSmooth, 0)); builtin(IR::StageInputKind::BaryCoordSmooth, 0));
@@ -1212,17 +1223,15 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
if (embedded != nullptr && IsBufferDwordLoad(instruction.opcode) && if (embedded != nullptr && IsBufferDwordLoad(instruction.opcode) &&
instruction.data_dwords == embedded->components && instruction.data_dwords == embedded->components &&
instruction.dst.kind == Decoder::OperandKind::Vgpr) { instruction.dst.kind == Decoder::OperandKind::Vgpr) {
if (options.vertex == nullptr) { const auto resource =
EXIT("embedded vertex fetch plan has no vertex input metadata"); ResolveEmbeddedFetchResource(*options.input_info.vertex, *embedded);
} if (resource < 0 || resource >= options.input_info.vertex->resources_num) {
const auto resource = ResolveEmbeddedFetchResource(*options.vertex, *embedded);
if (resource < 0 || resource >= options.vertex->resources_num) {
EXIT("embedded vertex fetch at 0x%08x has no resource for attribute %d", EXIT("embedded vertex fetch at 0x%08x has no resource for attribute %d",
instruction.pc, embedded->attrib_id); instruction.pc, embedded->attrib_id);
} }
translator.TranslateEmbeddedFetch(instruction, static_cast<uint32_t>(resource), translator.TranslateEmbeddedFetch(instruction, static_cast<uint32_t>(resource),
embedded->components, embedded->components,
options.vertex->resources[resource]); options.input_info.vertex->resources[resource]);
continue; continue;
} }
translator.TranslateInstruction(instruction); translator.TranslateInstruction(instruction);
@@ -29,13 +29,10 @@ struct TranslateOptions {
uint64_t shader_hash = 0; uint64_t shader_hash = 0;
uint32_t user_data_base = 0; uint32_t user_data_base = 0;
uint32_t user_data_count = 64; uint32_t user_data_count = 64;
uint32_t scratch_dwords = 0;
bool dispatcher_fallback = false; bool dispatcher_fallback = false;
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None; CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
std::string_view fallback_reason; std::string_view fallback_reason;
const ShaderVertexInputInfo* vertex = nullptr; ShaderStageInputInfo input_info;
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
const EmbeddedFetchPlan* embedded_fetch = nullptr; const EmbeddedFetchPlan* embedded_fetch = nullptr;
}; };
@@ -434,18 +434,6 @@ struct ShaderInfo {
bool operator==(const ShaderInfo& other) const = default; bool operator==(const ShaderInfo& other) const = default;
}; };
struct SpirvRequirements {
bool subgroup_ballot = false;
bool subgroup_shuffle = false;
bool subgroup_local_invocation_id = false;
bool compute_derivatives = false;
bool image_gather_extended = false;
bool function_lds = false;
bool function_scratch = false;
bool pixel_valid_mask = false;
bool buffer_int64_atomics = false;
};
struct BlockInfo { struct BlockInfo {
uint32_t id = 0; uint32_t id = 0;
uint32_t start_pc = 0; uint32_t start_pc = 0;
@@ -561,7 +549,6 @@ struct Program: ResourcePlan {
BindingLayout bindings; BindingLayout bindings;
bool binding_layout_complete = false; bool binding_layout_complete = false;
std::optional<SpirvRequirements> spirv_requirements;
}; };
std::string ProgramToString(const Program& program); std::string ProgramToString(const Program& program);
@@ -40,31 +40,31 @@ void AddOutput(ShaderInfo& info, StageOutputKind kind, uint32_t index, uint32_t
} }
} }
void ValidateOptions(const Program& program, const ShaderInfoOptions& options) { void ValidateOptions(const Program& program, ShaderStageInputInfo input_info) {
switch (program.stage) { switch (program.stage) {
case ShaderType::Vertex: case ShaderType::Vertex:
case ShaderType::Mesh: case ShaderType::Mesh:
if (options.vertex == nullptr) { if (input_info.vertex == nullptr) {
return Fail("vertex shader has no input metadata"); return Fail("vertex shader has no input metadata");
} }
if (options.vertex->resources_num < 0 || if (input_info.vertex->resources_num < 0 ||
options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) { input_info.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) {
return Fail("vertex resource count is out of range"); return Fail("vertex resource count is out of range");
} }
return; return;
case ShaderType::Pixel: case ShaderType::Pixel:
if (options.pixel == nullptr) { if (input_info.pixel == nullptr) {
return Fail("pixel shader has no input metadata"); return Fail("pixel shader has no input metadata");
} }
if (options.pixel->input_num > std::size(options.pixel->interpolator_settings)) { if (input_info.pixel->input_num > std::size(input_info.pixel->interpolator_settings)) {
return Fail("pixel input count is out of range"); return Fail("pixel input count is out of range");
} }
return; return;
case ShaderType::Compute: case ShaderType::Compute:
if (options.compute == nullptr) { if (input_info.compute == nullptr) {
return Fail("compute shader has no input metadata"); return Fail("compute shader has no input metadata");
} }
if (options.compute->thread_ids_num < 0 || options.compute->thread_ids_num > 3) { if (input_info.compute->thread_ids_num < 0 || input_info.compute->thread_ids_num > 3) {
return Fail("compute thread ID count is out of range"); return Fail("compute thread ID count is out of range");
} }
return; return;
@@ -72,7 +72,7 @@ void ValidateOptions(const Program& program, const ShaderInfoOptions& options) {
} }
} }
void ValidateValueReferences(const Program& program, const ShaderInfoOptions& options) { void ValidateValueReferences(const Program& program, ShaderStageInputInfo input_info) {
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
switch (inst.GetOpcode()) { switch (inst.GetOpcode()) {
@@ -84,12 +84,12 @@ void ValidateValueReferences(const Program& program, const ShaderInfoOptions& op
if (program.stage == ShaderType::Vertex && if (program.stage == ShaderType::Vertex &&
(inst.Arg(1).U32() >= 4u || (inst.Arg(1).U32() >= 4u ||
inst.Arg(0).U32() >= inst.Arg(0).U32() >=
static_cast<uint32_t>(options.vertex->resources_num))) { static_cast<uint32_t>(input_info.vertex->resources_num))) {
return Fail("vertex input reference is out of range"); return Fail("vertex input reference is out of range");
} }
if (program.stage == ShaderType::Pixel && if (program.stage == ShaderType::Pixel &&
(inst.Arg(1).U32() >= 4u || (inst.Arg(1).U32() >= 4u ||
inst.Arg(0).U32() >= options.pixel->input_num)) { inst.Arg(0).U32() >= input_info.pixel->input_num)) {
return Fail("pixel input reference is out of range"); return Fail("pixel input reference is out of range");
} }
break; break;
@@ -101,8 +101,8 @@ void ValidateValueReferences(const Program& program, const ShaderInfoOptions& op
inst.Arg(2).GetType() != Type::U32) { inst.Arg(2).GetType() != Type::U32) {
return Fail("interpolation parameter reference is invalid"); return Fail("interpolation parameter reference is invalid");
} }
if (inst.Arg(0).U32() >= options.pixel->input_num || inst.Arg(1).U32() >= 4u || if (inst.Arg(0).U32() >= input_info.pixel->input_num ||
inst.Arg(2).U32() >= 3u) { inst.Arg(1).U32() >= 4u || inst.Arg(2).U32() >= 3u) {
return Fail("interpolation parameter reference is out of range"); return Fail("interpolation parameter reference is out of range");
} }
break; break;
@@ -154,11 +154,18 @@ void ValidateValueReferences(const Program& program, const ShaderInfoOptions& op
} }
break; break;
} }
case ValueOpcode::SetAttribute: case ValueOpcode::SetAttribute: {
if (inst.Flags<ExportFlags>().index >= program.export_info.size()) { const auto index = inst.Flags<ExportFlags>().index;
if (index >= program.export_info.size()) {
return Fail("typed export metadata index is out of range"); return Fail("typed export metadata index is out of range");
} }
const auto& exp = program.export_info[index];
if (exp.kind == ExportTargetKind::Position && exp.index != 0 && exp.en != 0 &&
program.stage != ShaderType::Vertex && program.stage != ShaderType::Mesh) {
return Fail("auxiliary position export requires the vertex or mesh stage");
}
break; break;
}
default: break; default: break;
} }
} }
@@ -179,9 +186,7 @@ void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* ve
} }
} }
} }
for (uint32_t attr = 0; attr < static_cast<uint32_t>(vertex->resources_num) && for (uint32_t attr = 0; attr < static_cast<uint32_t>(vertex->resources_num); attr++) {
attr < ShaderVertexInputInfo::RES_MAX;
attr++) {
if (used_components[attr] != 0) { if (used_components[attr] != 0) {
AddInput(info, StageInputKind::Parameter, attr, used_components[attr], AddInput(info, StageInputKind::Parameter, attr, used_components[attr],
fmt::format("in_attr_{}", attr)); fmt::format("in_attr_{}", attr));
@@ -286,8 +291,7 @@ void CollectBuiltinInputs(const Program& program, ShaderInfo& info) {
} }
} }
void CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex, void CollectOutputs(const Program& program, ShaderStageInputInfo input_info, ShaderInfo& info) {
const ShaderPixelInputInfo* pixel, ShaderInfo& info) {
for (const auto* block: program.blocks) { for (const auto* block: program.blocks) {
for (const auto& inst: *block) { for (const auto& inst: *block) {
if (inst.GetOpcode() != ValueOpcode::SetAttribute) { if (inst.GetOpcode() != ValueOpcode::SetAttribute) {
@@ -296,11 +300,11 @@ void CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex,
const auto& export_info = program.export_info[inst.Flags<ExportFlags>().index]; const auto& export_info = program.export_info[inst.Flags<ExportFlags>().index];
if (export_info.kind == ExportTargetKind::MrtZ) { if (export_info.kind == ExportTargetKind::MrtZ) {
if (program.stage == ShaderType::Pixel && (export_info.en & 0x1u) != 0 && if (program.stage == ShaderType::Pixel && (export_info.en & 0x1u) != 0 &&
pixel->ps_depth_export_enable) { input_info.pixel->ps_depth_export_enable) {
AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth"); AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth");
} }
if (program.stage == ShaderType::Pixel && (export_info.en & 0x4u) != 0 && if (program.stage == ShaderType::Pixel && (export_info.en & 0x4u) != 0 &&
pixel->ps_sample_mask_export_enable) { input_info.pixel->ps_sample_mask_export_enable) {
AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask"); AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask");
} }
continue; continue;
@@ -322,7 +326,7 @@ void CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex,
continue; continue;
} }
const auto output = DecodePositionExportComponent( const auto output = DecodePositionExportComponent(
vertex->pa_cl_vs_out_cntl, export_info.index, component); input_info.vertex->pa_cl_vs_out_cntl, export_info.index, component);
if (output.viewport) { if (output.viewport) {
AddOutput(info, StageOutputKind::ViewportIndex, 0, 0, "gl_ViewportIndex"); AddOutput(info, StageOutputKind::ViewportIndex, 0, 0, "gl_ViewportIndex");
} }
@@ -358,13 +362,13 @@ void CollectOutputs(const Program& program, const ShaderVertexInputInfo* vertex,
} // namespace } // namespace
void CollectShaderInfo(Program& program, const ShaderInfoOptions& options) { void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
if (!program.resource_tracking_complete || program.shader_info_complete) { if (!program.resource_tracking_complete || program.shader_info_complete) {
return Fail(!program.resource_tracking_complete ? "shader resources were not tracked" return Fail(!program.resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected"); : "shader info already collected");
} }
ValidateOptions(program, options); ValidateOptions(program, input_info);
ValidateValueReferences(program, options); ValidateValueReferences(program, input_info);
auto next = program.info; auto next = program.info;
next.inputs.clear(); next.inputs.clear();
@@ -376,14 +380,14 @@ void CollectShaderInfo(Program& program, const ShaderInfoOptions& options) {
}); });
}); });
switch (program.stage) { switch (program.stage) {
case ShaderType::Vertex: CollectVertexInputs(program, options.vertex, next); break; case ShaderType::Vertex: CollectVertexInputs(program, input_info.vertex, next); break;
case ShaderType::Mesh: break; case ShaderType::Mesh: break;
case ShaderType::Pixel: CollectPixelInputs(program, options.pixel, next); break; case ShaderType::Pixel: CollectPixelInputs(program, input_info.pixel, next); break;
case ShaderType::Compute: CollectComputeInputs(options.compute, next); break; case ShaderType::Compute: CollectComputeInputs(input_info.compute, next); break;
default: return Fail("unsupported shader stage for info collection"); default: return Fail("unsupported shader stage for info collection");
} }
CollectBuiltinInputs(program, next); CollectBuiltinInputs(program, next);
CollectOutputs(program, options.vertex, options.pixel, next); CollectOutputs(program, input_info, next);
program.info = std::move(next); program.info = std::move(next);
program.shader_info_complete = true; program.shader_info_complete = true;
} }
@@ -5,15 +5,9 @@
namespace Libs::Graphics::ShaderRecompiler::IR { namespace Libs::Graphics::ShaderRecompiler::IR {
struct ShaderInfoOptions {
const ShaderVertexInputInfo* vertex = nullptr;
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
};
// Completes the immutable shader interface after resource tracking. On failure Program::info and // Completes the immutable shader interface after resource tracking. On failure Program::info and
// all completion state remain unchanged. // all completion state remain unchanged.
void CollectShaderInfo(Program& program, const ShaderInfoOptions& options); void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info);
} // namespace Libs::Graphics::ShaderRecompiler::IR } // namespace Libs::Graphics::ShaderRecompiler::IR
+98 -112
View File
@@ -4,7 +4,6 @@
#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h" #include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
#include "graphics/shader/recompiler/ir/ShaderIR.h" #include "graphics/shader/recompiler/ir/ShaderIR.h"
#include "graphics/shader/shader.h" #include "graphics/shader/shader.h"
#include "spirv/unified1/spirv.hpp11"
#include <array> #include <array>
#include <bit> #include <bit>
@@ -19,11 +18,6 @@ using ShaderRecompiler::Spirv::Builder;
constexpr uint32_t SpirvVersion15 = 0x00010500u; constexpr uint32_t SpirvVersion15 = 0x00010500u;
template <typename T>
constexpr uint32_t Word(T value) {
return static_cast<uint32_t>(value);
}
struct Parameter { struct Parameter {
uint32_t input_location = 0; uint32_t input_location = 0;
uint32_t output_location = 0; uint32_t output_location = 0;
@@ -61,37 +55,35 @@ class RectListEmitter {
public: public:
RectListEmitter(const std::vector<Parameter>& parameters_, spv::ExecutionModel model) RectListEmitter(const std::vector<Parameter>& parameters_, spv::ExecutionModel model)
: parameters(parameters_) { : parameters(parameters_) {
builder.AddMemoryModel( builder.AddMemoryModel({spv::AddressingModelLogical, spv::MemoryModelGLSL450});
{Word(spv::AddressingModel::Logical), Word(spv::MemoryModel::GLSL450)});
void_type = Type(spv::Op::OpTypeVoid); void_type = Type(spv::OpTypeVoid);
uint_type = Type(spv::Op::OpTypeInt, 32u, 0u); uint_type = Type(spv::OpTypeInt, 32u, 0u);
int_type = Type(spv::Op::OpTypeInt, 32u, 1u); int_type = Type(spv::OpTypeInt, 32u, 1u);
float_type = Type(spv::Op::OpTypeFloat, 32u); float_type = Type(spv::OpTypeFloat, 32u);
vec4_float_type = Type(spv::Op::OpTypeVector, float_type, 4u); vec4_float_type = Type(spv::OpTypeVector, float_type, 4u);
function_type = Type(spv::Op::OpTypeFunction, void_type); function_type = Type(spv::OpTypeFunction, void_type);
per_vertex_type = builder.DecoratedType( per_vertex_type = builder.DecoratedType(
Word(spv::Op::OpTypeStruct), {vec4_float_type}, spv::OpTypeStruct, {vec4_float_type},
{{Word(spv::Op::OpMemberDecorate), {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}},
{0u, Word(spv::Decoration::BuiltIn), Word(spv::BuiltIn::Position)}}, {spv::OpDecorate, {spv::DecorationBlock}}});
{Word(spv::Op::OpDecorate), {Word(spv::Decoration::Block)}}});
ptr_input_vec4_float = Pointer(spv::StorageClass::Input, vec4_float_type); ptr_input_vec4_float = Pointer(spv::StorageClassInput, vec4_float_type);
ptr_output_vec4_float = Pointer(spv::StorageClass::Output, vec4_float_type); ptr_output_vec4_float = Pointer(spv::StorageClassOutput, vec4_float_type);
if (model == spv::ExecutionModel::TessellationControl) { if (model == spv::ExecutionModelTessellationControl) {
bool_type = Type(spv::Op::OpTypeBool); bool_type = Type(spv::OpTypeBool);
vec2_bool_type = Type(spv::Op::OpTypeVector, bool_type, 2u); vec2_bool_type = Type(spv::OpTypeVector, bool_type, 2u);
vec2_float_type = Type(spv::Op::OpTypeVector, float_type, 2u); vec2_float_type = Type(spv::OpTypeVector, float_type, 2u);
ptr_output_float = Pointer(spv::StorageClass::Output, float_type); ptr_output_float = Pointer(spv::StorageClassOutput, float_type);
} else { } else {
vec3_float_type = Type(spv::Op::OpTypeVector, float_type, 3u); vec3_float_type = Type(spv::OpTypeVector, float_type, 3u);
ptr_input_float = Pointer(spv::StorageClass::Input, float_type); ptr_input_float = Pointer(spv::StorageClassInput, float_type);
} }
} }
std::vector<uint32_t> EmitControl() { std::vector<uint32_t> EmitControl() {
DefineEntry(spv::ExecutionModel::TessellationControl); DefineEntry(spv::ExecutionModelTessellationControl);
const auto float_one = Constant(float_type, std::bit_cast<uint32_t>(1.0f)); const auto float_one = Constant(float_type, std::bit_cast<uint32_t>(1.0f));
for (uint32_t i = 0; i < 4; i++) { for (uint32_t i = 0; i < 4; i++) {
@@ -109,11 +101,11 @@ public:
std::array<uint32_t, 3> coordinate_equal {}; std::array<uint32_t, 3> coordinate_equal {};
for (uint32_t i = 0; i < coordinate_equal.size(); i++) { for (uint32_t i = 0; i < coordinate_equal.size(); i++) {
const auto left = Result(spv::Op::OpVectorShuffle, vec2_float_type, positions[i], const auto left =
positions[i], 0u, 1u); Result(spv::OpVectorShuffle, vec2_float_type, positions[i], positions[i], 0u, 1u);
const auto right = Result(spv::Op::OpVectorShuffle, vec2_float_type, const auto right = Result(spv::OpVectorShuffle, vec2_float_type,
positions[(i + 1u) % 3u], positions[(i + 1u) % 3u], 0u, 1u); positions[(i + 1u) % 3u], positions[(i + 1u) % 3u], 0u, 1u);
coordinate_equal[i] = Result(spv::Op::OpFOrdEqual, vec2_bool_type, left, right); coordinate_equal[i] = Result(spv::OpFOrdEqual, vec2_bool_type, left, right);
} }
std::array<uint32_t, 3> barycentric {}; std::array<uint32_t, 3> barycentric {};
@@ -121,30 +113,29 @@ public:
const auto float_minus_one = Constant(float_type, std::bit_cast<uint32_t>(-1.0f)); const auto float_minus_one = Constant(float_type, std::bit_cast<uint32_t>(-1.0f));
for (uint32_t i = 0; i < edge_vertex.size(); i++) { for (uint32_t i = 0; i < edge_vertex.size(); i++) {
const auto previous = (i + 2u) % 3u; const auto previous = (i + 2u) % 3u;
const auto xy = Result( const auto xy =
spv::Op::OpLogicalAnd, bool_type, Result(spv::OpLogicalAnd, bool_type,
Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[i], 0u), Result(spv::OpCompositeExtract, bool_type, coordinate_equal[i], 0u),
Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[previous], 1u)); Result(spv::OpCompositeExtract, bool_type, coordinate_equal[previous], 1u));
const auto yx = Result( const auto yx =
spv::Op::OpLogicalAnd, bool_type, Result(spv::OpLogicalAnd, bool_type,
Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[i], 1u), Result(spv::OpCompositeExtract, bool_type, coordinate_equal[i], 1u),
Result(spv::Op::OpCompositeExtract, bool_type, coordinate_equal[previous], 0u)); Result(spv::OpCompositeExtract, bool_type, coordinate_equal[previous], 0u));
edge_vertex[i] = Result(spv::Op::OpLogicalOr, bool_type, xy, yx); edge_vertex[i] = Result(spv::OpLogicalOr, bool_type, xy, yx);
barycentric[i] = barycentric[i] =
Result(spv::Op::OpSelect, float_type, edge_vertex[i], float_minus_one, float_one); Result(spv::OpSelect, float_type, edge_vertex[i], float_minus_one, float_one);
} }
auto vertex_index = Result(spv::Op::OpSelect, int_type, edge_vertex[2], Int(2), Int(0)); auto vertex_index = Result(spv::OpSelect, int_type, edge_vertex[2], Int(2), Int(0));
vertex_index = Result(spv::Op::OpSelect, int_type, edge_vertex[1], Int(1), vertex_index); vertex_index = Result(spv::OpSelect, int_type, edge_vertex[1], Int(1), vertex_index);
const auto invocation = Load(int_type, invocation_id); const auto invocation = Load(int_type, invocation_id);
const auto is_fourth = Result(spv::Op::OpIEqual, bool_type, invocation, Int(3)); const auto is_fourth = Result(spv::OpIEqual, bool_type, invocation, Int(3));
const auto index = const auto index = Result(spv::OpSMod, int_type,
Result(spv::Op::OpSMod, int_type, Result(spv::OpIAdd, int_type, vertex_index, invocation), Int(3));
Result(spv::Op::OpIAdd, int_type, vertex_index, invocation), Int(3));
const auto position3 = Interpolate(positions[0], positions[1], positions[2], barycentric); const auto position3 = Interpolate(positions[0], positions[1], positions[2], barycentric);
const auto position = const auto position =
Result(spv::Op::OpSelect, vec4_float_type, is_fourth, position3, Result(spv::OpSelect, vec4_float_type, is_fourth, position3,
Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, index, Int(0)))); Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, index, Int(0))));
Store(Access(ptr_output_vec4_float, gl_out, invocation, Int(0)), position); Store(Access(ptr_output_vec4_float, gl_out, invocation, Int(0)), position);
@@ -161,25 +152,25 @@ public:
Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], Int(2))); Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], Int(2)));
const auto input3 = Interpolate(input0, input1, input2, barycentric); const auto input3 = Interpolate(input0, input1, input2, barycentric);
const auto value = const auto value =
Result(spv::Op::OpSelect, vec4_float_type, is_fourth, input3, Result(spv::OpSelect, vec4_float_type, is_fourth, input3,
Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], index))); Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], index)));
Store(Access(ptr_output_vec4_float, outputs[i], invocation), value); Store(Access(ptr_output_vec4_float, outputs[i], invocation), value);
} }
Emit(spv::Op::OpReturn); Emit(spv::OpReturn);
Emit(spv::Op::OpFunctionEnd); Emit(spv::OpFunctionEnd);
return builder.Build(); return builder.Build();
} }
std::vector<uint32_t> EmitEvaluation() { std::vector<uint32_t> EmitEvaluation() {
DefineEntry(spv::ExecutionModel::TessellationEvaluation); DefineEntry(spv::ExecutionModelTessellationEvaluation);
const auto x = Load(float_type, Access(ptr_input_float, tess_coord, Int(0))); const auto x = Load(float_type, Access(ptr_input_float, tess_coord, Int(0)));
const auto y = Load(float_type, Access(ptr_input_float, tess_coord, Int(1))); const auto y = Load(float_type, Access(ptr_input_float, tess_coord, Int(1)));
const auto index = Result( const auto index =
spv::Op::OpIAdd, int_type, Result(spv::OpIAdd, int_type,
Result(spv::Op::OpIMul, int_type, Result(spv::Op::OpConvertFToS, int_type, y), Int(2)), Result(spv::OpIMul, int_type, Result(spv::OpConvertFToS, int_type, y), Int(2)),
Result(spv::Op::OpConvertFToS, int_type, x)); Result(spv::OpConvertFToS, int_type, x));
const auto position = const auto position =
Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, index, Int(0))); Load(vec4_float_type, Access(ptr_input_vec4_float, gl_in, index, Int(0)));
@@ -189,102 +180,99 @@ public:
Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], index))); Load(vec4_float_type, Access(ptr_input_vec4_float, inputs[i], index)));
} }
Emit(spv::Op::OpReturn); Emit(spv::OpReturn);
Emit(spv::Op::OpFunctionEnd); Emit(spv::OpFunctionEnd);
return builder.Build(); return builder.Build();
} }
private: private:
template <typename... Args> template <typename... Args>
uint32_t Type(spv::Op opcode, Args... operands) { uint32_t Type(spv::Op opcode, Args... operands) {
return builder.Type(Word(opcode), {Word(operands)...}); return builder.Type(opcode, {operands...});
} }
uint32_t Constant(uint32_t type, uint32_t value) { uint32_t Constant(uint32_t type, uint32_t value) {
return builder.Constant(Word(spv::Op::OpConstant), type, {value}); return builder.Constant(spv::OpConstant, type, {value});
} }
uint32_t Pointer(spv::StorageClass storage, uint32_t type) { uint32_t Pointer(spv::StorageClass storage, uint32_t type) {
return Type(spv::Op::OpTypePointer, storage, type); return Type(spv::OpTypePointer, storage, type);
} }
uint32_t Array(uint32_t type, uint32_t size) { uint32_t Array(uint32_t type, uint32_t size) {
return Type(spv::Op::OpTypeArray, type, Uint(size)); return Type(spv::OpTypeArray, type, Uint(size));
} }
template <typename... Args> template <typename... Args>
uint32_t Result(spv::Op opcode, uint32_t type, Args... operands) { uint32_t Result(spv::Op opcode, uint32_t type, Args... operands) {
const auto id = builder.AllocateId(); const auto id = builder.AllocateId();
builder.AddFunction({Word(opcode), type, id, Word(operands)...}); builder.AddFunction({opcode, type, id, operands...});
return id; return id;
} }
template <typename... Args> template <typename... Args>
uint32_t ResultWithoutType(spv::Op opcode, Args... operands) { uint32_t ResultWithoutType(spv::Op opcode, Args... operands) {
const auto id = builder.AllocateId(); const auto id = builder.AllocateId();
builder.AddFunction({Word(opcode), id, Word(operands)...}); builder.AddFunction({opcode, id, operands...});
return id; return id;
} }
template <typename... Args> template <typename... Args>
void Emit(spv::Op opcode, Args... operands) { void Emit(spv::Op opcode, Args... operands) {
builder.AddFunction({Word(opcode), Word(operands)...}); builder.AddFunction({opcode, operands...});
} }
template <typename... Args> template <typename... Args>
uint32_t Access(uint32_t pointer_type, uint32_t base, Args... indices) { uint32_t Access(uint32_t pointer_type, uint32_t base, Args... indices) {
return Result(spv::Op::OpAccessChain, pointer_type, base, Word(indices)...); return Result(spv::OpAccessChain, pointer_type, base, indices...);
} }
uint32_t Load(uint32_t type, uint32_t pointer) { uint32_t Load(uint32_t type, uint32_t pointer) { return Result(spv::OpLoad, type, pointer); }
return Result(spv::Op::OpLoad, type, pointer);
}
void Store(uint32_t pointer, uint32_t value) { Emit(spv::Op::OpStore, pointer, value); } void Store(uint32_t pointer, uint32_t value) { Emit(spv::OpStore, pointer, value); }
uint32_t Int(uint32_t value) { return Constant(int_type, value); } uint32_t Int(uint32_t value) { return Constant(int_type, value); }
uint32_t Uint(uint32_t value) { return Constant(uint_type, value); } uint32_t Uint(uint32_t value) { return Constant(uint_type, value); }
uint32_t AddInterface(spv::StorageClass storage, uint32_t type) { uint32_t AddInterface(spv::StorageClass storage, uint32_t type) {
const auto variable = builder.DefineGlobalVariable(Pointer(storage, type), Word(storage)); const auto variable = builder.DefineGlobalVariable(Pointer(storage, type), storage);
interfaces.push_back(variable); interfaces.push_back(variable);
return variable; return variable;
} }
void Decorate(uint32_t target, spv::Decoration decoration, uint32_t value) { void Decorate(uint32_t target, spv::Decoration decoration, uint32_t value) {
builder.AddAnnotation({Word(spv::Op::OpDecorate), target, Word(decoration), value}); builder.AddAnnotation({spv::OpDecorate, target, decoration, value});
} }
void DefineEntry(spv::ExecutionModel model) { void DefineEntry(spv::ExecutionModel model) {
builder.RequireCapability(Word(spv::Capability::Shader)); builder.RequireCapability(spv::CapabilityShader);
builder.RequireCapability(Word(spv::Capability::Tessellation)); builder.RequireCapability(spv::CapabilityTessellation);
main = Result(spv::Op::OpFunction, void_type, spv::FunctionControlMask::MaskNone, main = Result(spv::OpFunction, void_type, spv::FunctionControlMaskNone, function_type);
function_type); if (model == spv::ExecutionModelTessellationControl) {
if (model == spv::ExecutionModel::TessellationControl) { builder.AddExecutionMode({main, spv::ExecutionModeOutputVertices, 4u});
builder.AddExecutionMode({main, Word(spv::ExecutionMode::OutputVertices), 4u});
} else { } else {
builder.AddExecutionMode({main, Word(spv::ExecutionMode::Quads)}); builder.AddExecutionMode({main, spv::ExecutionModeQuads});
builder.AddExecutionMode({main, Word(spv::ExecutionMode::SpacingEqual)}); builder.AddExecutionMode({main, spv::ExecutionModeSpacingEqual});
builder.AddExecutionMode({main, Word(spv::ExecutionMode::VertexOrderCw)}); builder.AddExecutionMode({main, spv::ExecutionModeVertexOrderCw});
} }
DefineInputs(model); DefineInputs(model);
DefineOutputs(model); DefineOutputs(model);
builder.AddEntryPoint(Word(model), main, "main", interfaces); builder.AddEntryPoint(model, main, "main", interfaces);
ResultWithoutType(spv::Op::OpLabel); ResultWithoutType(spv::OpLabel);
} }
void DefineInputs(spv::ExecutionModel model) { void DefineInputs(spv::ExecutionModel model) {
const auto tess_control = model == spv::ExecutionModel::TessellationControl; const auto tess_control = model == spv::ExecutionModelTessellationControl;
if (tess_control) { if (tess_control) {
invocation_id = AddInterface(spv::StorageClass::Input, int_type); invocation_id = AddInterface(spv::StorageClassInput, int_type);
Decorate(invocation_id, spv::Decoration::BuiltIn, Word(spv::BuiltIn::InvocationId)); Decorate(invocation_id, spv::DecorationBuiltIn, spv::BuiltInInvocationId);
} else { } else {
tess_coord = AddInterface(spv::StorageClass::Input, vec3_float_type); tess_coord = AddInterface(spv::StorageClassInput, vec3_float_type);
Decorate(tess_coord, spv::Decoration::BuiltIn, Word(spv::BuiltIn::TessCoord)); Decorate(tess_coord, spv::DecorationBuiltIn, spv::BuiltInTessCoord);
} }
gl_in = gl_in =
AddInterface(spv::StorageClass::Input, Array(per_vertex_type, tess_control ? 3u : 4u)); AddInterface(spv::StorageClassInput, Array(per_vertex_type, tess_control ? 3u : 4u));
inputs.resize(parameters.size()); inputs.resize(parameters.size());
std::array<uint32_t, ShaderVertexInputInfo::RES_MAX> locations {}; std::array<uint32_t, ShaderVertexInputInfo::RES_MAX> locations {};
@@ -295,44 +283,42 @@ private:
inputs[i] = locations[location]; inputs[i] = locations[location];
continue; continue;
} }
inputs[i] = AddInterface(spv::StorageClass::Input, inputs[i] = AddInterface(spv::StorageClassInput,
Array(vec4_float_type, tess_control ? 3u : 4u)); Array(vec4_float_type, tess_control ? 3u : 4u));
Decorate(inputs[i], spv::Decoration::Location, location); Decorate(inputs[i], spv::DecorationLocation, location);
locations[location] = inputs[i]; locations[location] = inputs[i];
} }
} }
void DefineOutputs(spv::ExecutionModel model) { void DefineOutputs(spv::ExecutionModel model) {
const auto tess_control = model == spv::ExecutionModel::TessellationControl; const auto tess_control = model == spv::ExecutionModelTessellationControl;
if (tess_control) { if (tess_control) {
gl_out = AddInterface(spv::StorageClass::Output, Array(per_vertex_type, 4u)); gl_out = AddInterface(spv::StorageClassOutput, Array(per_vertex_type, 4u));
tess_inner = AddInterface(spv::StorageClass::Output, Array(float_type, 2u)); tess_inner = AddInterface(spv::StorageClassOutput, Array(float_type, 2u));
Decorate(tess_inner, spv::Decoration::BuiltIn, Word(spv::BuiltIn::TessLevelInner)); Decorate(tess_inner, spv::DecorationBuiltIn, spv::BuiltInTessLevelInner);
builder.AddAnnotation( builder.AddAnnotation({spv::OpDecorate, tess_inner, spv::DecorationPatch});
{Word(spv::Op::OpDecorate), tess_inner, Word(spv::Decoration::Patch)}); tess_outer = AddInterface(spv::StorageClassOutput, Array(float_type, 4u));
tess_outer = AddInterface(spv::StorageClass::Output, Array(float_type, 4u)); Decorate(tess_outer, spv::DecorationBuiltIn, spv::BuiltInTessLevelOuter);
Decorate(tess_outer, spv::Decoration::BuiltIn, Word(spv::BuiltIn::TessLevelOuter)); builder.AddAnnotation({spv::OpDecorate, tess_outer, spv::DecorationPatch});
builder.AddAnnotation(
{Word(spv::Op::OpDecorate), tess_outer, Word(spv::Decoration::Patch)});
} else { } else {
gl_out = AddInterface(spv::StorageClass::Output, per_vertex_type); gl_out = AddInterface(spv::StorageClassOutput, per_vertex_type);
} }
outputs.resize(parameters.size()); outputs.resize(parameters.size());
for (uint32_t i = 0; i < parameters.size(); i++) { for (uint32_t i = 0; i < parameters.size(); i++) {
outputs[i] = AddInterface(spv::StorageClass::Output, outputs[i] = AddInterface(spv::StorageClassOutput,
tess_control ? Array(vec4_float_type, 4u) : vec4_float_type); tess_control ? Array(vec4_float_type, 4u) : vec4_float_type);
Decorate(outputs[i], spv::Decoration::Location, parameters[i].output_location); Decorate(outputs[i], spv::DecorationLocation, parameters[i].output_location);
} }
} }
uint32_t Interpolate(uint32_t v0, uint32_t v1, uint32_t v2, uint32_t Interpolate(uint32_t v0, uint32_t v1, uint32_t v2,
const std::array<uint32_t, 3>& barycentric) { const std::array<uint32_t, 3>& barycentric) {
const auto p0 = Result(spv::Op::OpVectorTimesScalar, vec4_float_type, v0, barycentric[0]); const auto p0 = Result(spv::OpVectorTimesScalar, vec4_float_type, v0, barycentric[0]);
const auto p1 = Result(spv::Op::OpVectorTimesScalar, vec4_float_type, v1, barycentric[1]); const auto p1 = Result(spv::OpVectorTimesScalar, vec4_float_type, v1, barycentric[1]);
const auto p2 = Result(spv::Op::OpVectorTimesScalar, vec4_float_type, v2, barycentric[2]); const auto p2 = Result(spv::OpVectorTimesScalar, vec4_float_type, v2, barycentric[2]);
return Result(spv::Op::OpFAdd, vec4_float_type, p0, return Result(spv::OpFAdd, vec4_float_type, p0,
Result(spv::Op::OpFAdd, vec4_float_type, p1, p2)); Result(spv::OpFAdd, vec4_float_type, p1, p2));
} }
Builder builder {SpirvVersion15}; Builder builder {SpirvVersion15};
@@ -369,8 +355,8 @@ private:
RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info, RectListShaders BuildRectListShaders(const ShaderVertexInputInfo& vertex_info,
const ShaderPixelInputInfo* pixel_info) { const ShaderPixelInputInfo* pixel_info) {
const auto parameters = GetParameters(vertex_info, pixel_info); const auto parameters = GetParameters(vertex_info, pixel_info);
RectListEmitter control(parameters, spv::ExecutionModel::TessellationControl); RectListEmitter control(parameters, spv::ExecutionModelTessellationControl);
RectListEmitter evaluation(parameters, spv::ExecutionModel::TessellationEvaluation); RectListEmitter evaluation(parameters, spv::ExecutionModelTessellationEvaluation);
return {control.EmitControl(), evaluation.EmitEvaluation()}; return {control.EmitControl(), evaluation.EmitEvaluation()};
} }
+57 -84
View File
@@ -1392,7 +1392,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
compute_info.host_subgroup_size = host_subgroup_size; compute_info.host_subgroup_size = host_subgroup_size;
options.input_info.compute = &compute_info; options.input_info.compute = &compute_info;
options.user_data = user_data; options.user_data = user_data;
options.scratch_dwords = test.compute_info.scratch_size_dwords;
if (test.has_compute_info) { if (test.has_compute_info) {
options.wave_size = test.compute_info.wave_size; options.wave_size = test.compute_info.wave_size;
} }
@@ -1598,86 +1598,59 @@ std::array<u32, 64> MakeSampledTextureData(Prospero::BufferFormat format) {
namespace TestSpv { namespace TestSpv {
enum : u32 {
ExecutionModelVertex = 0,
AddressingModelLogical = 0,
MemoryModelGLSL450 = 1,
CapabilityShader = 1,
StorageClassInput = 1,
StorageClassOutput = 3,
FunctionControlNone = 0,
DecorationBlock = 2,
DecorationBuiltIn = 11,
DecorationLocation = 30,
BuiltInPosition = 0,
OpTypeVoid = 19,
OpTypeInt = 21,
OpTypeFloat = 22,
OpTypeVector = 23,
OpTypeStruct = 30,
OpTypePointer = 32,
OpTypeFunction = 33,
OpConstant = 43,
OpFunction = 54,
OpFunctionEnd = 56,
OpVariable = 59,
OpLoad = 61,
OpStore = 62,
OpAccessChain = 65,
OpDecorate = 71,
OpMemberDecorate = 72,
OpCompositeConstruct = 80,
OpCompositeExtract = 81,
OpLabel = 248,
OpReturn = 253,
};
std::vector<u32> MakePassthroughVertexSpirv(bool layered, float clip_w = 1.0f) { std::vector<u32> MakePassthroughVertexSpirv(bool layered, float clip_w = 1.0f) {
using ShaderRecompiler::Spirv::Builder; using ShaderRecompiler::Spirv::Builder;
Builder b; Builder b;
const auto void_type = b.Type(OpTypeVoid); const auto void_type = b.Type(spv::OpTypeVoid);
const auto uint_type = b.Type(OpTypeInt, {32, 0}); const auto uint_type = b.Type(spv::OpTypeInt, {32, 0});
const auto int_type = b.Type(OpTypeInt, {32, 1}); const auto int_type = b.Type(spv::OpTypeInt, {32, 1});
const auto float_type = b.Type(OpTypeFloat, {32}); const auto float_type = b.Type(spv::OpTypeFloat, {32});
const auto vec2_type = b.Type(OpTypeVector, {float_type, 2}); const auto vec2_type = b.Type(spv::OpTypeVector, {float_type, 2});
const auto vec4_type = b.Type(OpTypeVector, {float_type, 4}); const auto vec4_type = b.Type(spv::OpTypeVector, {float_type, 4});
const auto per_vertex_type = b.DecoratedType( const auto per_vertex_type =
OpTypeStruct, {vec4_type}, b.DecoratedType(spv::OpTypeStruct, {vec4_type},
{{OpDecorate, {DecorationBlock}}, {{spv::OpDecorate, {spv::DecorationBlock}},
{OpMemberDecorate, {0, DecorationBuiltIn, BuiltInPosition}}}); {spv::OpMemberDecorate,
{0, spv::DecorationBuiltIn, spv::BuiltInPosition}}});
const auto ptr_input_vec2 = const auto ptr_input_vec2 =
b.Type(OpTypePointer, {StorageClassInput, vec2_type}); b.Type(spv::OpTypePointer, {spv::StorageClassInput, vec2_type});
const auto ptr_input_vec4 = const auto ptr_input_vec4 =
b.Type(OpTypePointer, {StorageClassInput, vec4_type}); b.Type(spv::OpTypePointer, {spv::StorageClassInput, vec4_type});
const auto ptr_output_vec4 = const auto ptr_output_vec4 =
b.Type(OpTypePointer, {StorageClassOutput, vec4_type}); b.Type(spv::OpTypePointer, {spv::StorageClassOutput, vec4_type});
const auto ptr_output_per_vertex = const auto ptr_output_per_vertex =
b.Type(OpTypePointer, {StorageClassOutput, per_vertex_type}); b.Type(spv::OpTypePointer, {spv::StorageClassOutput, per_vertex_type});
const auto func_type = b.Type(OpTypeFunction, {void_type}); const auto func_type = b.Type(spv::OpTypeFunction, {void_type});
const auto const_u32_0 = b.Constant(OpConstant, uint_type, {0}); const auto const_u32_0 = b.Constant(spv::OpConstant, uint_type, {0});
const auto const_f32_0 = b.Constant(OpConstant, float_type, {0x00000000u}); const auto const_f32_0 =
b.Constant(spv::OpConstant, float_type, {0x00000000u});
const auto const_f32_w = const auto const_f32_w =
b.Constant(OpConstant, float_type, {std::bit_cast<u32>(clip_w)}); b.Constant(spv::OpConstant, float_type, {std::bit_cast<u32>(clip_w)});
const auto in_pos = b.DefineGlobalVariable(ptr_input_vec2, StorageClassInput); const auto in_pos =
b.DefineGlobalVariable(ptr_input_vec2, spv::StorageClassInput);
const auto in_color = const auto in_color =
b.DefineGlobalVariable(ptr_input_vec4, StorageClassInput); b.DefineGlobalVariable(ptr_input_vec4, spv::StorageClassInput);
const auto out_color = const auto out_color =
b.DefineGlobalVariable(ptr_output_vec4, StorageClassOutput); b.DefineGlobalVariable(ptr_output_vec4, spv::StorageClassOutput);
const auto per_vertex = const auto per_vertex =
b.DefineGlobalVariable(ptr_output_per_vertex, StorageClassOutput); b.DefineGlobalVariable(ptr_output_per_vertex, spv::StorageClassOutput);
u32 instance = 0; u32 instance = 0;
u32 layer = 0; u32 layer = 0;
std::vector<u32> interfaces = {in_pos, in_color, per_vertex, out_color}; std::vector<u32> interfaces = {in_pos, in_color, per_vertex, out_color};
if (layered) { if (layered) {
instance = b.DefineGlobalVariable( instance = b.DefineGlobalVariable(
b.Type(OpTypePointer, {StorageClassInput, int_type}), StorageClassInput); b.Type(spv::OpTypePointer, {spv::StorageClassInput, int_type}),
spv::StorageClassInput);
layer = b.DefineGlobalVariable( layer = b.DefineGlobalVariable(
b.Type(OpTypePointer, {StorageClassOutput, int_type}), StorageClassOutput); b.Type(spv::OpTypePointer, {spv::StorageClassOutput, int_type}),
b.AddAnnotation({OpDecorate, instance, DecorationBuiltIn, 43}); // InstanceIndex spv::StorageClassOutput);
b.AddAnnotation({OpDecorate, layer, DecorationBuiltIn, 9}); // Layer b.AddAnnotation({spv::OpDecorate, instance, spv::DecorationBuiltIn,
spv::BuiltInInstanceIndex});
b.AddAnnotation(
{spv::OpDecorate, layer, spv::DecorationBuiltIn, spv::BuiltInLayer});
b.RequireVersion(0x00010500u); b.RequireVersion(0x00010500u);
b.RequireCapability(69); // ShaderLayer b.RequireCapability(spv::CapabilityShaderLayer);
interfaces.insert(interfaces.end(), {instance, layer}); interfaces.insert(interfaces.end(), {instance, layer});
} }
const auto main = b.AllocateId(); const auto main = b.AllocateId();
@@ -1689,32 +1662,33 @@ std::vector<u32> MakePassthroughVertexSpirv(bool layered, float clip_w = 1.0f) {
const auto position = b.AllocateId(); const auto position = b.AllocateId();
const auto position_ptr = b.AllocateId(); const auto position_ptr = b.AllocateId();
b.RequireCapability(CapabilityShader); b.RequireCapability(spv::CapabilityShader);
b.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450}); b.AddMemoryModel({spv::AddressingModelLogical, spv::MemoryModelGLSL450});
b.AddEntryPoint(ExecutionModelVertex, main, "main", interfaces); b.AddEntryPoint(spv::ExecutionModelVertex, main, "main", interfaces);
b.AddAnnotation({OpDecorate, in_pos, DecorationLocation, 0}); b.AddAnnotation({spv::OpDecorate, in_pos, spv::DecorationLocation, 0});
b.AddAnnotation({OpDecorate, in_color, DecorationLocation, 1}); b.AddAnnotation({spv::OpDecorate, in_color, spv::DecorationLocation, 1});
b.AddAnnotation({OpDecorate, out_color, DecorationLocation, 0}); b.AddAnnotation({spv::OpDecorate, out_color, spv::DecorationLocation, 0});
b.AddFunction({OpFunction, void_type, main, FunctionControlNone, func_type}); b.AddFunction({spv::OpFunction, void_type, main, spv::FunctionControlMaskNone,
b.AddFunction({OpLabel, label}); func_type});
b.AddFunction({OpLoad, vec2_type, pos2, in_pos}); b.AddFunction({spv::OpLabel, label});
b.AddFunction({OpLoad, vec4_type, color4, in_color}); b.AddFunction({spv::OpLoad, vec2_type, pos2, in_pos});
b.AddFunction({OpCompositeExtract, float_type, pos_x, pos2, 0}); b.AddFunction({spv::OpLoad, vec4_type, color4, in_color});
b.AddFunction({OpCompositeExtract, float_type, pos_y, pos2, 1}); b.AddFunction({spv::OpCompositeExtract, float_type, pos_x, pos2, 0});
b.AddFunction({OpCompositeConstruct, vec4_type, position, pos_x, pos_y, b.AddFunction({spv::OpCompositeExtract, float_type, pos_y, pos2, 1});
b.AddFunction({spv::OpCompositeConstruct, vec4_type, position, pos_x, pos_y,
const_f32_0, const_f32_w}); const_f32_0, const_f32_w});
b.AddFunction( b.AddFunction({spv::OpAccessChain, ptr_output_vec4, position_ptr, per_vertex,
{OpAccessChain, ptr_output_vec4, position_ptr, per_vertex, const_u32_0}); const_u32_0});
b.AddFunction({OpStore, position_ptr, position}); b.AddFunction({spv::OpStore, position_ptr, position});
b.AddFunction({OpStore, out_color, color4}); b.AddFunction({spv::OpStore, out_color, color4});
if (layered) { if (layered) {
const auto index = b.AllocateId(); const auto index = b.AllocateId();
b.AddFunction({OpLoad, int_type, index, instance}); b.AddFunction({spv::OpLoad, int_type, index, instance});
b.AddFunction({OpStore, layer, index}); b.AddFunction({spv::OpStore, layer, index});
} }
b.AddFunction({OpReturn}); b.AddFunction({spv::OpReturn});
b.AddFunction({OpFunctionEnd}); b.AddFunction({spv::OpFunctionEnd});
return b.Build(); return b.Build();
} }
@@ -23866,7 +23840,6 @@ void CheckIndirectImageKeySwitch() {
AllocateBindings(program); AllocateBindings(program);
ShaderComputeInputInfo compute{}; ShaderComputeInputInfo compute{};
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program);
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, auto spirv = ShaderRecompiler::Spirv::EmitProgram(program,
{.compute = &compute}); {.compute = &compute});
ValidateSpirv(name, spirv); ValidateSpirv(name, spirv);
+234 -50
View File
@@ -154,7 +154,7 @@ void CompilePixelRuntime(const ShaderParams &params,
auto options = MakeCompileOptions(ShaderType::Pixel); auto options = MakeCompileOptions(ShaderType::Pixel);
options.shader_hash = params.hash; options.shader_hash = params.hash;
options.user_data = params.user_data; options.user_data = params.user_data;
options.scratch_dwords = input_info.scratch_size_dwords;
options.input_info.pixel = &input_info; options.input_info.pixel = &input_info;
auto result = RecompileForTest(params.code, options); auto result = RecompileForTest(params.code, options);
static std::deque<ShaderRecompiler::IR::CompiledShaderInfo> programs; static std::deque<ShaderRecompiler::IR::CompiledShaderInfo> programs;
@@ -1260,8 +1260,8 @@ void TestSpirvRequirementsAnalysis() {
block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)}); block->AppendNewInst(ValueOpcode::SetAttribute, {Value(0u), Value(true)});
export_value.SetFlags(ExportFlags{.index = 0}); export_value.SetFlags(ExportFlags{.index = 0});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); const auto requirements =
const auto requirements = *program.spirv_requirements; ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program);
Check(requirements.subgroup_ballot && requirements.subgroup_shuffle && Check(requirements.subgroup_ballot && requirements.subgroup_shuffle &&
requirements.subgroup_local_invocation_id && requirements.subgroup_local_invocation_id &&
requirements.compute_derivatives && requirements.compute_derivatives &&
@@ -1270,8 +1270,8 @@ void TestSpirvRequirementsAnalysis() {
"consolidated SPIR-V requirements missed an IR dependency"); "consolidated SPIR-V requirements missed an IR dependency");
program.stage = ShaderType::Compute; program.stage = ShaderType::Compute;
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); const auto compute_requirements =
const auto compute_requirements = *program.spirv_requirements; ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program);
Check(!compute_requirements.function_lds && Check(!compute_requirements.function_lds &&
!compute_requirements.pixel_valid_mask, !compute_requirements.pixel_valid_mask,
"stage-specific SPIR-V requirements leaked into compute"); "stage-specific SPIR-V requirements leaked into compute");
@@ -1281,14 +1281,14 @@ void TestSpirvRequirementsAnalysis() {
[&] { [&] {
program.stage = ShaderType::Pixel; program.stage = ShaderType::Pixel;
shared.SetFlags(MemoryFlags{.index = 1}); shared.SetFlags(MemoryFlags{.index = 1});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program);
}, },
"invalid shared-memory requirement metadata did not terminate analysis"); "invalid shared-memory requirement metadata did not terminate analysis");
ExpectFatal( ExpectFatal(
[&] { [&] {
program.stage = ShaderType::Pixel; program.stage = ShaderType::Pixel;
export_value.SetFlags(ExportFlags{.index = 1}); export_value.SetFlags(ExportFlags{.index = 1});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program); ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(program);
}, },
"invalid export requirement metadata did not terminate analysis"); "invalid export requirement metadata did not terminate analysis");
#endif #endif
@@ -1309,8 +1309,8 @@ void TestSpirvRequirementsAnalysis() {
ValueOpcode::LoadBufferU32, ValueOpcode::LoadBufferU32,
{Value(&buffer), Value(0u), Value(0u), Value(0u), Value(true)}); {Value(&buffer), Value(0u), Value(0u), Value(0u), Value(true)});
load.SetFlags(MemoryFlags{.index = 0}); load.SetFlags(MemoryFlags{.index = 0});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid); const auto add_tid_requirements =
const auto add_tid_requirements = *add_tid.spirv_requirements; ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(add_tid);
Check(add_tid_requirements.subgroup_local_invocation_id && Check(add_tid_requirements.subgroup_local_invocation_id &&
!add_tid_requirements.subgroup_ballot && !add_tid_requirements.subgroup_ballot &&
!add_tid_requirements.subgroup_shuffle, !add_tid_requirements.subgroup_shuffle,
@@ -1320,20 +1320,20 @@ void TestSpirvRequirementsAnalysis() {
ExpectFatal( ExpectFatal(
[&] { [&] {
add_tid.memory_info[0].resource = 1; add_tid.memory_info[0].resource = 1;
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid); ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(add_tid);
}, },
"invalid buffer requirement metadata did not terminate analysis"); "invalid buffer requirement metadata did not terminate analysis");
ExpectFatal( ExpectFatal(
[&] { [&] {
add_tid.stage = ShaderType::Vertex; add_tid.stage = ShaderType::Vertex;
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(add_tid); ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(add_tid);
}, },
"graphics buffer ADD_TID did not terminate requirements analysis"); "graphics buffer ADD_TID did not terminate requirements analysis");
#endif #endif
Program empty; Program empty;
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(empty); const auto empty_requirements =
const auto empty_requirements = *empty.spirv_requirements; ShaderRecompiler::Spirv::Emitter::AnalyzeProgramRequirements(empty);
Check(!empty_requirements.subgroup_ballot && Check(!empty_requirements.subgroup_ballot &&
!empty_requirements.subgroup_shuffle && !empty_requirements.subgroup_shuffle &&
!empty_requirements.subgroup_local_invocation_id && !empty_requirements.subgroup_local_invocation_id &&
@@ -4383,7 +4383,7 @@ void TestNewShaderRecompilerCapturedVopcSdwaCmpxClass() {
Frontend::TranslateOptions translate_options{}; Frontend::TranslateOptions translate_options{};
translate_options.stage = ShaderType::Pixel; translate_options.stage = ShaderType::Pixel;
translate_options.wave_size = 64u; translate_options.wave_size = 64u;
translate_options.pixel = &pixel; translate_options.input_info.pixel = &pixel;
ShaderRecompiler::Decoder::DecodeProgram(shader, program); ShaderRecompiler::Decoder::DecodeProgram(shader, program);
graph = ShaderRecompiler::CFG::BuildGraph(program); graph = ShaderRecompiler::CFG::BuildGraph(program);
ir = ShaderRecompiler::Frontend::TranslateProgram(program, graph, ir = ShaderRecompiler::Frontend::TranslateProgram(program, graph,
@@ -4431,7 +4431,7 @@ void TestNewShaderRecompilerIrLookupMissFailsExplicitly() {
Frontend::TranslateOptions options{}; Frontend::TranslateOptions options{};
options.stage = ShaderType::Compute; options.stage = ShaderType::Compute;
options.wave_size = 64u; options.wave_size = 64u;
options.compute = &compute; options.input_info.compute = &compute;
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
store.pc = 0u; store.pc = 0u;
@@ -4807,7 +4807,7 @@ void TestImageAddressOperands() {
ShaderComputeInputInfo compute = RegressionComputeInputInfo(); ShaderComputeInputInfo compute = RegressionComputeInputInfo();
Frontend::TranslateOptions options{}; Frontend::TranslateOptions options{};
options.stage = ShaderType::Compute; options.stage = ShaderType::Compute;
options.compute = &compute; options.input_info.compute = &compute;
auto program = Frontend::TranslateProgram(decoded, CFG::BuildGraph(decoded), options); auto program = Frontend::TranslateProgram(decoded, CFG::BuildGraph(decoded), options);
IR::RewriteToSsa(program.blocks); IR::RewriteToSsa(program.blocks);
IR::ConstantPropagationPass(program.blocks); IR::ConstantPropagationPass(program.blocks);
@@ -6211,7 +6211,7 @@ void TestNewShaderRecompilerNativeWideScalarMemoryIr() {
ShaderRecompiler::Frontend::TranslateOptions translate_options{}; ShaderRecompiler::Frontend::TranslateOptions translate_options{};
translate_options.stage = ShaderType::Compute; translate_options.stage = ShaderType::Compute;
translate_options.wave_size = 64u; translate_options.wave_size = 64u;
translate_options.compute = &compute; translate_options.input_info.compute = &compute;
ShaderRecompiler::Decoder::DecodeProgram(shader, decoded); ShaderRecompiler::Decoder::DecodeProgram(shader, decoded);
graph = ShaderRecompiler::CFG::BuildGraph(decoded); graph = ShaderRecompiler::CFG::BuildGraph(decoded);
ir = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph, ir = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph,
@@ -6909,7 +6909,7 @@ void TestNewShaderRecompilerDsReadWrite2Translation() {
ShaderRecompiler::Frontend::TranslateOptions translate_options{}; ShaderRecompiler::Frontend::TranslateOptions translate_options{};
translate_options.stage = ShaderType::Compute; translate_options.stage = ShaderType::Compute;
translate_options.wave_size = 64u; translate_options.wave_size = 64u;
translate_options.compute = &compute; translate_options.input_info.compute = &compute;
ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded);
graph = ShaderRecompiler::CFG::BuildGraph(decoded); graph = ShaderRecompiler::CFG::BuildGraph(decoded);
typed = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph, typed = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph,
@@ -7057,7 +7057,7 @@ void TestNewShaderRecompilerDsWideAndAtomicTranslation() {
ShaderRecompiler::Frontend::TranslateOptions translate_options{}; ShaderRecompiler::Frontend::TranslateOptions translate_options{};
translate_options.stage = ShaderType::Compute; translate_options.stage = ShaderType::Compute;
translate_options.wave_size = 64u; translate_options.wave_size = 64u;
translate_options.compute = &compute; translate_options.input_info.compute = &compute;
ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded); ShaderRecompiler::Decoder::DecodeProgram(std::span{shader}, decoded);
graph = ShaderRecompiler::CFG::BuildGraph(decoded); graph = ShaderRecompiler::CFG::BuildGraph(decoded);
typed = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph, typed = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph,
@@ -7348,7 +7348,6 @@ void TestNewShaderRecompilerStructuredU64Phi() {
IR::IREmitter use(join); IR::IREmitter use(join);
use.Emit(IR::ValueOpcode::CompositeExtractU64, use.Emit(IR::ValueOpcode::CompositeExtractU64,
{IR::Value(&phi), IR::Value(1u)}); {IR::Value(&phi), IR::Value(1u)});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program);
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info);
CheckSpirvBinaryValidates(spirv); CheckSpirvBinaryValidates(spirv);
@@ -8673,7 +8672,7 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
{IR::Value(1u), IR::Value(2u), IR::Value(3u)}); {IR::Value(1u), IR::Value(2u), IR::Value(3u)});
IR::IREmitter use(prologue_program.blocks[1]); IR::IREmitter use(prologue_program.blocks[1]);
use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(prologue_program);
auto spirv = ShaderRecompiler::Spirv::EmitProgram(prologue_program, auto spirv = ShaderRecompiler::Spirv::EmitProgram(prologue_program,
options.input_info); options.input_info);
CheckSpirvBinaryValidates(spirv); CheckSpirvBinaryValidates(spirv);
@@ -8692,7 +8691,6 @@ void TestNewShaderRecompilerDispatcherSpillsU32x3() {
IR::IREmitter use(program.blocks[2]); IR::IREmitter use(program.blocks[2]);
use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)}); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(2u)});
use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(1u)}); use.Emit(IR::ValueOpcode::CompositeExtractU32x3, {vector, IR::Value(1u)});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program);
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, auto spirv = ShaderRecompiler::Spirv::EmitProgram(program,
options.input_info); options.input_info);
@@ -8834,7 +8832,6 @@ void TestNewShaderRecompilerU64PairTranslation() {
ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {base, IR::Value(count)}); ir.Emit(IR::ValueOpcode::ShiftRightLogical64, {base, IR::Value(count)});
ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {base, IR::Value(count)}); ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {base, IR::Value(count)});
} }
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program);
auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); auto spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info);
CheckSpirvBinaryValidates(spirv); CheckSpirvBinaryValidates(spirv);
@@ -8868,7 +8865,7 @@ void TestNewShaderRecompilerU64PairTranslation() {
{IR::Value(1u), IR::Value(2u)}); {IR::Value(1u), IR::Value(2u)});
IR::IREmitter use(program.blocks[2]); IR::IREmitter use(program.blocks[2]);
use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)}); use.Emit(IR::ValueOpcode::CompositeExtractU64, {vector, IR::Value(1u)});
ShaderRecompiler::Spirv::AnalyzeProgramRequirements(program);
spirv.clear(); spirv.clear();
spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info); spirv = ShaderRecompiler::Spirv::EmitProgram(program, options.input_info);
CheckSpirvBinaryValidates(spirv); CheckSpirvBinaryValidates(spirv);
@@ -9429,7 +9426,7 @@ void TestEmbeddedVertexFormatSwizzle() {
options.stage = ShaderType::Vertex; options.stage = ShaderType::Vertex;
options.wave_size = 32; options.wave_size = 32;
options.user_data_count = 0; options.user_data_count = 0;
options.vertex = &input; options.input_info.vertex = &input;
options.embedded_fetch = &fetch; options.embedded_fetch = &fetch;
auto program = Frontend::TranslateProgram(decoded, CFG::BuildGraph(decoded), options); auto program = Frontend::TranslateProgram(decoded, CFG::BuildGraph(decoded), options);
Check(program.info.vertex_fetch_components[0] == test.source_width, Check(program.info.vertex_fetch_components[0] == test.source_width,
@@ -9528,7 +9525,7 @@ void TestMeshInputAssembly() {
options.stage = ShaderType::Mesh; options.stage = ShaderType::Mesh;
options.wave_size = 64; options.wave_size = 64;
options.user_data_count = 0; options.user_data_count = 0;
options.vertex = &input; options.input_info.vertex = &input;
auto program = Frontend::TranslateProgram(decoded, graph, options); auto program = Frontend::TranslateProgram(decoded, graph, options);
const uint32_t draw[] = {test.count, test.base_vertex, 7, test.width, const uint32_t draw[] = {test.count, test.base_vertex, 7, test.width,
test.address_low, 0x12}; test.address_low, 0x12};
@@ -9660,7 +9657,7 @@ void TestNewShaderRecompilerPrunesUnreachableSetpcMetadata() {
ShaderRecompiler::Frontend::TranslateOptions translate_options{}; ShaderRecompiler::Frontend::TranslateOptions translate_options{};
translate_options.stage = ShaderType::Compute; translate_options.stage = ShaderType::Compute;
translate_options.wave_size = 64u; translate_options.wave_size = 64u;
translate_options.compute = &compute; translate_options.input_info.compute = &compute;
ir = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph, ir = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph,
translate_options); translate_options);
size_t scalar_loads = 0; size_t scalar_loads = 0;
@@ -9946,43 +9943,104 @@ void TestNewShaderRecompilerAuxPositionExports() {
"PA_CL_VS_OUT_CNTL is absent from the vertex shader cache key"); "PA_CL_VS_OUT_CNTL is absent from the vertex shader cache key");
} }
void TestDeferredSpirvPhiPatching() {
ShaderRecompiler::Spirv::Builder builder;
const auto type = builder.Type(spv::OpTypeInt, {32u, 0u});
const auto zero = builder.Constant(spv::OpConstant, type, {0u});
const auto one = builder.Constant(spv::OpConstant, type, {1u});
const auto left = builder.AllocateId();
const auto right = builder.AllocateId();
const auto result = builder.AllocateId();
const auto phi = builder.AddDeferredPhi(type, result, 2);
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal([&] { (void)builder.Build(); }, "unpatched phi was serialized");
ExpectFatal([&] { builder.PatchDeferredPhi(phi, 2, one, right); },
"phi patch accepted an incoming outside its encoded length");
#endif
builder.PatchDeferredPhi(phi, 1, one, right);
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
ExpectFatal([&] { builder.PatchDeferredPhi(phi, 1, zero, left); },
"phi incoming was patched twice");
ExpectFatal([&] { (void)builder.Build(); },
"partially patched phi was serialized");
#endif
// Appending another instruction must not change the first phi's patch
// location.
const auto second_result = builder.AllocateId();
const auto second_phi = builder.AddDeferredPhi(type, second_result, 1);
builder.PatchDeferredPhi(phi, 0, zero, left);
builder.PatchDeferredPhi(second_phi, 0, one, right);
const auto binary = builder.Build();
uint32_t phis = 0;
for (size_t offset = 5; offset < binary.size();) {
const auto words = binary[offset] >> spv::WordCountShift;
if ((binary[offset] & spv::OpCodeMask) == spv::OpPhi) {
if (phis++ == 0) {
Check(words == 7 && binary[offset + 2] == result &&
binary[offset + 3] == zero && binary[offset + 4] == left &&
binary[offset + 5] == one && binary[offset + 6] == right,
"out-of-order phi patches changed incoming values or parents");
} else {
Check(words == 5 && binary[offset + 2] == second_result &&
binary[offset + 3] == one && binary[offset + 4] == right,
"patching one phi corrupted its neighbor");
}
}
offset += words;
}
Check(phis == 2, "deferred phi instructions were lost");
}
void TestDemandDrivenSpirvDeclarations() { void TestDemandDrivenSpirvDeclarations() {
using ShaderRecompiler::Spirv::Builder; using ShaderRecompiler::Spirv::Builder;
Builder declarations; Builder declarations;
const auto uint_type = declarations.Type(21u, {32u, 0u}); const auto uint_type = declarations.Type(spv::OpTypeInt, {32u, 0u});
Check(uint_type == declarations.Type(21u, {32u, 0u}), Check(uint_type == declarations.Type(spv::OpTypeInt, {32u, 0u}),
"structural type interning returned different IDs"); "structural type interning returned different IDs");
const auto zero = declarations.Constant(43u, uint_type, {0u}); const auto zero = declarations.Constant(spv::OpConstant, uint_type, {0u});
Check(zero == declarations.Constant(43u, uint_type, {0u}), Check(zero == declarations.Constant(spv::OpConstant, uint_type, {0u}),
"constant interning returned different IDs"); "constant interning returned different IDs");
Check(declarations.Import("GLSL.std.450") == Check(declarations.Import("GLSL.std.450") ==
declarations.Import("GLSL.std.450"), declarations.Import("GLSL.std.450"),
"extended-instruction import was duplicated"); "extended-instruction import was duplicated");
declarations.RequireCapability(1u); declarations.RequireCapability(spv::CapabilityShader);
declarations.RequireCapability(1u); declarations.RequireCapability(spv::CapabilityShader);
const auto plain_array = declarations.Type(28u, {uint_type, zero}); const auto plain_array =
declarations.Type(spv::OpTypeArray, {uint_type, zero});
const auto decorated_array = declarations.DecoratedType( const auto decorated_array = declarations.DecoratedType(
28u, {uint_type, zero}, {{71u, {6u, sizeof(uint32_t)}}}); spv::OpTypeArray, {uint_type, zero},
{{spv::OpDecorate, {spv::DecorationArrayStride, sizeof(uint32_t)}}});
Check(decorated_array == Check(decorated_array ==
declarations.DecoratedType(28u, {uint_type, zero}, declarations.DecoratedType(
{{71u, {6u, sizeof(uint32_t)}}}), spv::OpTypeArray, {uint_type, zero},
{{spv::OpDecorate,
{spv::DecorationArrayStride, sizeof(uint32_t)}}}),
"decorated type interning returned different IDs"); "decorated type interning returned different IDs");
Check(plain_array != decorated_array, Check(plain_array != decorated_array,
"decorated and undecorated aggregate types were aliased"); "decorated and undecorated aggregate types were aliased");
Check(declarations.DecoratedType(28u, {uint_type, zero}, {}) == plain_array, Check(declarations.DecoratedType(spv::OpTypeArray, {uint_type, zero}, {}) ==
plain_array,
"empty decorated type request bypassed structural interning"); "empty decorated type request bypassed structural interning");
const auto ptr = declarations.Type(32u, {7u, uint_type}); const auto ptr = declarations.Type(spv::OpTypePointer,
Check(declarations.DefineGlobalVariable(ptr, 7u) != {spv::StorageClassFunction, uint_type});
declarations.DefineGlobalVariable(ptr, 7u), Check(declarations.DefineGlobalVariable(ptr, spv::StorageClassFunction) !=
declarations.DefineGlobalVariable(ptr, spv::StorageClassFunction),
"distinct variables were structurally interned"); "distinct variables were structurally interned");
const auto declaration_binary = declarations.Build(); const auto declaration_binary = declarations.Build();
Check(SpirvInstructionOpcodeCount(declaration_binary, 17u) == 1u && Check(SpirvInstructionOpcodeCount(declaration_binary, 17u) == 1u &&
SpirvInstructionOpcodeCount(declaration_binary, 11u) == 1u && SpirvInstructionOpcodeCount(declaration_binary, 11u) == 1u &&
SpirvInstructionOpcodeCount(declaration_binary, 21u) == 1u && SpirvInstructionOpcodeCount(declaration_binary, spv::OpTypeInt) ==
SpirvInstructionOpcodeCount(declaration_binary, 28u) == 2u && 1u &&
SpirvInstructionOpcodeCount(declaration_binary, 43u) == 1u && SpirvInstructionOpcodeCount(declaration_binary, spv::OpTypeArray) ==
SpirvInstructionOpcodeCount(declaration_binary, 71u) == 1u && 2u &&
SpirvInstructionOpcodeCount(declaration_binary, spv::OpConstant) ==
1u &&
SpirvInstructionOpcodeCount(declaration_binary, spv::OpDecorate) ==
1u &&
SpirvInstructionOpcodeCount(declaration_binary, 59u) == 2u, SpirvInstructionOpcodeCount(declaration_binary, 59u) == 2u,
"canonical builder emitted duplicate declarations"); "canonical builder emitted duplicate declarations");
@@ -10074,7 +10132,7 @@ void TestTypedEntryStateIsMinimal() {
translate_options.stage = ShaderType::Compute; translate_options.stage = ShaderType::Compute;
translate_options.wave_size = wave_size; translate_options.wave_size = wave_size;
translate_options.user_data_count = 0; translate_options.user_data_count = 0;
translate_options.compute = &compute; translate_options.input_info.compute = &compute;
values = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph, values = ShaderRecompiler::Frontend::TranslateProgram(decoded, graph,
translate_options); translate_options);
@@ -11416,7 +11474,7 @@ void BuildTypedPlan(const uint32_t *code, uint32_t words,
ShaderRecompiler::Frontend::TranslateOptions options{}; ShaderRecompiler::Frontend::TranslateOptions options{};
options.stage = ShaderType::Compute; options.stage = ShaderType::Compute;
options.wave_size = 64u; options.wave_size = 64u;
options.compute = &compute; options.input_info.compute = &compute;
ir = ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options); ir = ShaderRecompiler::Frontend::TranslateProgram(decoded, cfg, options);
ShaderRecompiler::IR::RewriteToSsa(ir.blocks); ShaderRecompiler::IR::RewriteToSsa(ir.blocks);
ShaderRecompiler::IR::ConstantPropagationPass(ir.blocks); ShaderRecompiler::IR::ConstantPropagationPass(ir.blocks);
@@ -12256,7 +12314,7 @@ void TestComputeLdsAllocationIdentity() {
options.user_data = params.user_data; options.user_data = params.user_data;
options.input_info.compute = &input_info; options.input_info.compute = &input_info;
options.wave_size = input_info.wave_size; options.wave_size = input_info.wave_size;
options.scratch_dwords = input_info.scratch_size_dwords;
auto result = RecompileForTest(shader, options); auto result = RecompileForTest(shader, options);
Check(MeasureSpirv(result.spirv).workgroup_variables == 1u, Check(MeasureSpirv(result.spirv).workgroup_variables == 1u,
"LDS shader did not declare exactly one Workgroup variable"); "LDS shader did not declare exactly one Workgroup variable");
@@ -12322,7 +12380,7 @@ void TestComputeLdsAllocationIdentity() {
scratch_options.user_data = scratch_params.user_data; scratch_options.user_data = scratch_params.user_data;
scratch_options.input_info.compute = &scratch_info; scratch_options.input_info.compute = &scratch_info;
scratch_options.wave_size = scratch_info.wave_size; scratch_options.wave_size = scratch_info.wave_size;
scratch_options.scratch_dwords = scratch_info.scratch_size_dwords;
auto scratch_result = auto scratch_result =
RecompileForTest(scratch_params.code, scratch_options); RecompileForTest(scratch_params.code, scratch_options);
Check(scratch_result.program.scratch_dwords == 7, Check(scratch_result.program.scratch_dwords == 7,
@@ -12482,7 +12540,10 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() {
auto options = MakeCompileOptions(ShaderType::Compute); auto options = MakeCompileOptions(ShaderType::Compute);
options.user_data = user_data; options.user_data = user_data;
options.scratch_dwords = 1; ShaderComputeInputInfo compute{};
compute.scratch_size_dwords = 1;
options.input_info.compute = &compute;
auto result = RecompileForTest(segmented_shader, options); auto result = RecompileForTest(segmented_shader, options);
Check(result.program.info.uses_dma, Check(result.program.info.uses_dma,
"GLOBAL null-SADDR did not enable DMA"); "GLOBAL null-SADDR did not enable DMA");
@@ -12491,6 +12552,125 @@ void TestNewShaderRecompilerFlatAddressDomainsUseDma() {
"SCRATCH incorrectly entered guest address tracking"); "SCRATCH incorrectly entered guest address tracking");
} }
void TestCompilerStageInputOwnership() {
using namespace ShaderRecompiler;
const uint32_t shader[] = {EncodeSopp(0x01)};
ShaderVertexInputInfo vertex{};
vertex.scratch_size_dwords = 3;
vertex.mesh.scratch_size_dwords = 11;
vertex.mesh.threads_num[0] = 192;
vertex.mesh.threads_num[1] = vertex.mesh.threads_num[2] = 1;
vertex.mesh.primitives_per_group = 62;
vertex.mesh.vertices_per_group = 64;
ShaderPixelInputInfo pixel{};
pixel.scratch_size_dwords = 5;
ShaderComputeInputInfo compute{};
compute.scratch_size_dwords = 7;
for (const auto stage : {ShaderType::Vertex, ShaderType::Pixel,
ShaderType::Compute, ShaderType::Mesh}) {
CompileOptions options{};
options.stage = stage;
options.dump_ir = false;
uint32_t expected_scratch = 0;
switch (stage) {
case ShaderType::Vertex:
options.input_info.vertex = &vertex;
expected_scratch = 3;
break;
case ShaderType::Pixel:
options.input_info.pixel = &pixel;
expected_scratch = 5;
break;
case ShaderType::Compute:
options.input_info.compute = &compute;
expected_scratch = 7;
break;
case ShaderType::Mesh:
options.input_info.vertex = &vertex;
expected_scratch = 11;
break;
default:
std::abort();
}
const auto translated = TranslateProgram(shader, options);
Check(
translated.program.scratch_dwords == expected_scratch,
"translation did not use its stage's authoritative scratch allocation");
}
auto options = MakeCompileOptions(ShaderType::Compute);
options.shader_hash = 0x1234;
auto translated = TranslateProgram(shader, options);
options.stage = ShaderType::Unknown;
options.shader_hash = 0;
auto compiled = CompileProgram(std::move(translated), options, {});
Check(compiled.program.stage == ShaderType::Compute &&
compiled.program.shader_hash == 0x1234,
"compilation reinterpreted the translated program's identity");
CheckSpirvBinaryValidates(compiled.spirv);
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
options.stage = ShaderType::Vertex;
options.input_info.vertex = nullptr;
ExpectFatal([&] { (void)TranslateProgram(shader, options); },
"missing stage metadata bypassed translation validation");
const uint32_t auxiliary_position[] = {
EncodeExp0(0x0d, 0x1), EncodeExp1(0, 0, 0, 0), EncodeSopp(0x01)};
for (const auto stage : {ShaderType::Pixel, ShaderType::Compute}) {
ExpectFatal(
[&] {
(void)RecompileForTest(auxiliary_position, MakeCompileOptions(stage));
},
"auxiliary position export read another stage's input metadata");
}
Decoder::Program decoded;
Decoder::DecodeProgram(shader, decoded);
const auto cfg = CFG::BuildGraph(decoded);
Frontend::EmbeddedFetchPlan fetch;
Frontend::TranslateOptions invalid_fetch{.stage = ShaderType::Compute,
.input_info = {.compute = &compute},
.embedded_fetch = &fetch};
ExpectFatal(
[&] { (void)Frontend::TranslateProgram(decoded, cfg, invalid_fetch); },
"embedded vertex fetch accepted another stage's metadata");
#endif
}
void TestSpirvEmissionOwnsRequirements() {
using namespace ShaderRecompiler;
const uint32_t shader[] = {EncodeSopp(0x01)};
const auto options = MakeCompileOptions(ShaderType::Compute);
auto compiled = RecompileForTest(shader, options);
IR::IREmitter emitter(compiled.program.blocks.front());
emitter.Emit(IR::ValueOpcode::LaneId, {});
const auto binary = Spirv::EmitProgram(compiled.program, options.input_info);
CheckSpirvBinaryValidates(binary);
Check(Common::ContainsStr(DisassembleSpirvBinary(binary),
"BuiltIn SubgroupLocalInvocationId"),
"emission reused requirements from an earlier IR version");
}
void TestRepeatedExportsHaveOneInterface() {
const uint32_t shader[] = {EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 2, 3),
EncodeExp0(0x0c, 0xf), EncodeExp1(0, 1, 2, 3),
EncodeSopp(0x01)};
const auto result =
RecompileForTest(shader, MakeCompileOptions(ShaderType::Vertex));
Check(result.program.info.outputs.size() == 1,
"repeated exports duplicated the collected shader interface");
CheckSpirvBinaryValidates(result.spirv);
uint32_t outputs = 0;
for (size_t offset = 5; offset < result.spirv.size();) {
if ((result.spirv[offset] & spv::OpCodeMask) == spv::OpVariable &&
result.spirv[offset + 3] == spv::StorageClassOutput) {
outputs++;
}
offset += result.spirv[offset] >> spv::WordCountShift;
}
Check(outputs == 1, "repeated exports produced duplicate output variables");
}
void TestNewShaderRecompilerSpirvSizeBaselines() { void TestNewShaderRecompilerSpirvSizeBaselines() {
const auto compile = [](const char *name, std::span<const uint32_t> shader, const auto compile = [](const char *name, std::span<const uint32_t> shader,
const SpirvMetrics &budget, const SpirvMetrics &budget,
@@ -12758,6 +12938,10 @@ int main() {
TestNativeShaderResourceDependencies(); TestNativeShaderResourceDependencies();
TestNormalizedImageContracts(); TestNormalizedImageContracts();
TestSpirvRequirementsAnalysis(); TestSpirvRequirementsAnalysis();
TestDeferredSpirvPhiPatching();
TestCompilerStageInputOwnership();
TestSpirvEmissionOwnsRequirements();
TestRepeatedExportsHaveOneInterface();
TestNewShaderRecompilerSpirvSizeBaselines(); TestNewShaderRecompilerSpirvSizeBaselines();
TestDemandDrivenSpirvDeclarations(); TestDemandDrivenSpirvDeclarations();
TestNewShaderRecompilerSMovB32(); TestNewShaderRecompilerSMovB32();