gremling/KytyPS5
1
0 Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-27 16:02:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
nmzik committed 2026-07-16 10:21:41 +02:00
commit 147188b143
427 files changed
+205923
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/graphics/shader/recompiler/BindingLayout.cpp
+293
View File
@@ -0,0 +1,293 @@
#include "graphics/shader/recompiler/BindingLayout.h"
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm>
#include <array>
#include <set>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint32_t MaxPushConstantBytes = 128;
constexpr std::array ImageBindingKinds = {
DescriptorBindingKind::Sampled2D, DescriptorBindingKind::Sampled2DArray,
DescriptorBindingKind::Sampled3D, DescriptorBindingKind::SampledUint2D,
DescriptorBindingKind::SampledUint2DArray, DescriptorBindingKind::SampledUint3D,
DescriptorBindingKind::Storage2D, DescriptorBindingKind::Storage2DArray,
DescriptorBindingKind::Storage3D, DescriptorBindingKind::StorageUint2D,
DescriptorBindingKind::StorageUint2DArray, DescriptorBindingKind::StorageUint3D,
};
bool ImageBinding(const ImageResource& image, DescriptorBindingKind& result) {
using Dimension = Decoder::ImageDimension;
using Kind = DescriptorBindingKind;
switch (image.kind) {
case ResourceKind::Image:
switch (image.dimension) {
case Dimension::Dim2D: result = Kind::Sampled2D; return true;
case Dimension::Dim2DArray: result = Kind::Sampled2DArray; return true;
case Dimension::Dim3D: result = Kind::Sampled3D; return true;
default: return false;
}
case ResourceKind::ImageUint:
switch (image.dimension) {
case Dimension::Dim2D: result = Kind::SampledUint2D; return true;
case Dimension::Dim2DArray: result = Kind::SampledUint2DArray; return true;
case Dimension::Dim3D: result = Kind::SampledUint3D; return true;
default: return false;
}
case ResourceKind::StorageImage:
switch (image.dimension) {
case Dimension::Dim2D: result = Kind::Storage2D; return true;
case Dimension::Dim2DArray: result = Kind::Storage2DArray; return true;
case Dimension::Dim3D: result = Kind::Storage3D; return true;
default: return false;
}
case ResourceKind::StorageImageUint:
switch (image.dimension) {
case Dimension::Dim2D: result = Kind::StorageUint2D; return true;
case Dimension::Dim2DArray: result = Kind::StorageUint2DArray; return true;
case Dimension::Dim3D: result = Kind::StorageUint3D; return true;
default: return false;
}
default: return false;
}
}
bool CollectValue(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>* visited,
std::set<uint32_t>* registers) {
if (id <= ScalarProvenance::Unknown) {
return true;
}
if (id >= provenance.values.size()) {
return false;
}
if ((*visited)[id] != 0) {
return true;
}
(*visited)[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::UserData) {
registers->insert(value.imm);
return true;
}
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (!CollectValue(provenance, arg, visited, registers)) {
return false;
}
}
return true;
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (!CollectValue(provenance, value.args[i], visited, registers)) {
return false;
}
}
return true;
}
bool CollectSource(const Program& program, uint32_t source, bool allow_unknown,
std::vector<uint8_t>* visited, std::set<uint32_t>* registers) {
if (allow_unknown && source == ScalarProvenance::Unknown) {
return true;
}
if (source < 2u || source - 2u >= program.provenance.descriptors.size()) {
return false;
}
const auto& descriptor = program.provenance.descriptors[source - 2u];
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (!CollectValue(program.provenance, descriptor.dwords[i], visited, registers)) {
return false;
}
}
return true;
}
bool CollectUserData(const Program& program, std::vector<uint32_t>& result) {
std::set<uint32_t> registers;
std::vector<uint8_t> visited(program.provenance.values.size());
for (const auto& buffer: program.info.buffers) {
if (!CollectSource(program, buffer.source, false, &visited, &registers)) {
return false;
}
}
for (const auto& image: program.info.images) {
if (!CollectSource(program, image.source, false, &visited, &registers)) {
return false;
}
}
for (const auto& sampler: program.info.samplers) {
if (!CollectSource(program, sampler.source, false, &visited, &registers)) {
return false;
}
}
for (const auto& address: program.info.addresses) {
if (!CollectSource(program, address.source, true, &visited, &registers)) {
return false;
}
}
for (const auto& read: program.srt.reads) {
if (!CollectValue(program.provenance, read.value, &visited, &registers)) {
return false;
}
}
for (const auto read: program.srt.dynamic_reads) {
if (!CollectValue(program.provenance, read, &visited, &registers)) {
return false;
}
}
for (const auto source: program.srt.dynamic_sources) {
if (!CollectSource(program, source, true, &visited, &registers)) {
return false;
}
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::SLoadDword && inst.memory.kind == ResourceKind::ScalarBuffer &&
!CollectValue(program.provenance, inst.scalar_value, &visited, &registers)) {
return false;
}
for (uint32_t i = 0; i < inst.src_count; i++) {
if (!CollectValue(program.provenance, inst.scalar_sources[i], &visited,
&registers)) {
return false;
}
}
}
}
result.assign(registers.begin(), registers.end());
return true;
}
void AddBinding(BindingLayout* layout, DescriptorBindingKind kind,
std::vector<uint32_t> resources = {}) {
layout->descriptors.push_back(
{kind, static_cast<uint32_t>(layout->descriptors.size()), std::move(resources)});
}
bool UsesGds(const Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.memory.kind == ResourceKind::Gds) {
return true;
}
}
}
return false;
}
} // namespace
bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std::string* error) {
if (program == nullptr || !program->shader_info_complete || program->binding_layout_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid binding-layout program"
: !program->shader_info_complete ? "shader info is not ready"
: "binding layout already allocated";
}
return false;
}
if (options.push_constant_offset > MaxPushConstantBytes) {
if (error != nullptr) {
*error = "push-constant offset exceeds the Vulkan minimum guarantee";
}
return false;
}
if (options.push_constant_offset % 4u != 0) {
if (error != nullptr) {
*error = "push-constant offset is not dword aligned";
}
return false;
}
BindingLayout next;
next.descriptor_set = options.descriptor_set;
next.push_constant_offset = options.push_constant_offset;
if (!CollectUserData(*program, next.user_data_registers)) {
if (error != nullptr) {
*error = "shader plan contains an invalid scalar provenance reference";
}
return false;
}
if (!program->info.buffers.empty()) {
std::vector<uint32_t> resources(program->info.buffers.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(&next, DescriptorBindingKind::Buffers, std::move(resources));
}
std::array<std::vector<uint32_t>, ImageBindingKinds.size()> image_groups;
for (uint32_t i = 0; i < program->info.images.size(); i++) {
DescriptorBindingKind kind;
if (!ImageBinding(program->info.images[i], kind)) {
if (error != nullptr) {
*error = "shader info contains an invalid image binding class";
}
return false;
}
const auto group = std::find(ImageBindingKinds.begin(), ImageBindingKinds.end(), kind);
if (group == ImageBindingKinds.end()) {
if (error != nullptr) {
*error = "shader info contains an unmapped image binding class";
}
return false;
}
image_groups[static_cast<size_t>(group - ImageBindingKinds.begin())].push_back(i);
}
for (uint32_t i = 0; i < image_groups.size(); i++) {
if (!image_groups[i].empty()) {
AddBinding(&next, ImageBindingKinds[i], std::move(image_groups[i]));
}
}
if (!program->info.samplers.empty()) {
std::vector<uint32_t> resources(program->info.samplers.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(&next, DescriptorBindingKind::Samplers, std::move(resources));
}
if (UsesGds(*program)) {
AddBinding(&next, DescriptorBindingKind::Gds);
}
if (!program->info.addresses.empty()) {
std::vector<uint32_t> resources(program->info.addresses.size());
for (uint32_t i = 0; i < resources.size(); i++) {
resources[i] = i;
}
AddBinding(&next, DescriptorBindingKind::AddressMemory, std::move(resources));
}
if (!program->srt.reads.empty()) {
AddBinding(&next, DescriptorBindingKind::FlattenedSrt);
}
const auto available_push_dwords = (MaxPushConstantBytes - options.push_constant_offset) / 4u;
const auto push_limit = std::min(options.max_push_dwords, available_push_dwords);
if (next.user_data_registers.size() <= push_limit) {
next.push_constant_size = static_cast<uint32_t>(next.user_data_registers.size() * 4u);
} else {
AddBinding(&next, DescriptorBindingKind::UserData);
}
program->bindings = std::move(next);
program->binding_layout_complete = true;
return true;
}
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind) {
for (const auto& binding: layout.descriptors) {
if (binding.kind == kind) {
return &binding;
}
}
return nullptr;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/BindingLayout.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_
#include "graphics/shader/recompiler/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct BindingLayoutOptions {
uint32_t descriptor_set = 0;
uint32_t push_constant_offset = 0;
uint32_t max_push_dwords = 32;
};
bool AllocateBindings(Program* program, const BindingLayoutOptions& options, std::string* error);
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BINDINGLAYOUT_H_ */
src/graphics/shader/recompiler/BufferFormat.h
+256
View File
@@ -0,0 +1,256 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_
#include "graphics/guest_gpu/gpu_defs.h"
namespace Libs::Graphics::ShaderRecompiler::Format {
enum class ComponentType {
Unknown,
Uint,
Sint,
Unorm,
Snorm,
Uscaled,
Sscaled,
Float,
};
struct BufferFormatInfo {
Prospero::BufferFormat format = Prospero::BufferFormat::kInvalid;
ComponentType type = ComponentType::Unknown;
uint32_t component_count = 0;
uint32_t byte_size = 0;
uint32_t component_bits[4] = {};
uint32_t component_bit_offset[4] = {};
bool packed_bitfield = false;
};
constexpr Prospero::BufferFormat DecodeTBufferFormat(uint32_t data_format, uint32_t number_format) {
return static_cast<Prospero::BufferFormat>(((number_format & 0x7u) << 4u) | (data_format & 0xfu));
}
constexpr ComponentType GetFormatComponentType(Prospero::BufferFormat format) {
switch (format) {
case Prospero::BufferFormat::k8UInt:
case Prospero::BufferFormat::k16UInt:
case Prospero::BufferFormat::k8_8UInt:
case Prospero::BufferFormat::k32UInt:
case Prospero::BufferFormat::k16_16UInt:
case Prospero::BufferFormat::k11_11_10UInt:
case Prospero::BufferFormat::k10_11_11UInt:
case Prospero::BufferFormat::k2_10_10_10UInt:
case Prospero::BufferFormat::k10_10_10_2UInt:
case Prospero::BufferFormat::k8_8_8_8UInt:
case Prospero::BufferFormat::k32_32UInt:
case Prospero::BufferFormat::k16_16_16_16UInt:
case Prospero::BufferFormat::k32_32_32UInt:
case Prospero::BufferFormat::k32_32_32_32UInt: return ComponentType::Uint;
case Prospero::BufferFormat::k8SInt:
case Prospero::BufferFormat::k16SInt:
case Prospero::BufferFormat::k8_8SInt:
case Prospero::BufferFormat::k32SInt:
case Prospero::BufferFormat::k16_16SInt:
case Prospero::BufferFormat::k11_11_10SInt:
case Prospero::BufferFormat::k10_11_11SInt:
case Prospero::BufferFormat::k2_10_10_10SInt:
case Prospero::BufferFormat::k10_10_10_2SInt:
case Prospero::BufferFormat::k8_8_8_8SInt:
case Prospero::BufferFormat::k32_32SInt:
case Prospero::BufferFormat::k16_16_16_16SInt:
case Prospero::BufferFormat::k32_32_32SInt:
case Prospero::BufferFormat::k32_32_32_32SInt: return ComponentType::Sint;
case Prospero::BufferFormat::k8UNorm:
case Prospero::BufferFormat::k16UNorm:
case Prospero::BufferFormat::k8_8UNorm:
case Prospero::BufferFormat::k16_16UNorm:
case Prospero::BufferFormat::k11_11_10UNorm:
case Prospero::BufferFormat::k10_11_11UNorm:
case Prospero::BufferFormat::k2_10_10_10UNorm:
case Prospero::BufferFormat::k10_10_10_2UNorm:
case Prospero::BufferFormat::k8_8_8_8UNorm:
case Prospero::BufferFormat::k16_16_16_16UNorm: return ComponentType::Unorm;
case Prospero::BufferFormat::k8SNorm:
case Prospero::BufferFormat::k16SNorm:
case Prospero::BufferFormat::k8_8SNorm:
case Prospero::BufferFormat::k16_16SNorm:
case Prospero::BufferFormat::k11_11_10SNorm:
case Prospero::BufferFormat::k10_11_11SNorm:
case Prospero::BufferFormat::k2_10_10_10SNorm:
case Prospero::BufferFormat::k10_10_10_2SNorm:
case Prospero::BufferFormat::k8_8_8_8SNorm:
case Prospero::BufferFormat::k16_16_16_16SNorm: return ComponentType::Snorm;
case Prospero::BufferFormat::k8UScaled:
case Prospero::BufferFormat::k16UScaled:
case Prospero::BufferFormat::k8_8UScaled:
case Prospero::BufferFormat::k16_16UScaled:
case Prospero::BufferFormat::k11_11_10UScaled:
case Prospero::BufferFormat::k10_11_11UScaled:
case Prospero::BufferFormat::k2_10_10_10UScaled:
case Prospero::BufferFormat::k10_10_10_2UScaled:
case Prospero::BufferFormat::k8_8_8_8UScaled:
case Prospero::BufferFormat::k16_16_16_16UScaled: return ComponentType::Uscaled;
case Prospero::BufferFormat::k8SScaled:
case Prospero::BufferFormat::k16SScaled:
case Prospero::BufferFormat::k8_8SScaled:
case Prospero::BufferFormat::k16_16SScaled:
case Prospero::BufferFormat::k11_11_10SScaled:
case Prospero::BufferFormat::k10_11_11SScaled:
case Prospero::BufferFormat::k2_10_10_10SScaled:
case Prospero::BufferFormat::k10_10_10_2SScaled:
case Prospero::BufferFormat::k8_8_8_8SScaled:
case Prospero::BufferFormat::k16_16_16_16SScaled: return ComponentType::Sscaled;
case Prospero::BufferFormat::k16Float:
case Prospero::BufferFormat::k32Float:
case Prospero::BufferFormat::k16_16Float:
case Prospero::BufferFormat::k11_11_10Float:
case Prospero::BufferFormat::k10_11_11Float:
case Prospero::BufferFormat::k32_32Float:
case Prospero::BufferFormat::k16_16_16_16Float:
case Prospero::BufferFormat::k32_32_32Float:
case Prospero::BufferFormat::k32_32_32_32Float: return ComponentType::Float;
default: return ComponentType::Unknown;
}
}
constexpr BufferFormatInfo GetFormatInfo(Prospero::BufferFormat format) {
const auto type = GetFormatComponentType(format);
if (type == ComponentType::Unknown) {
return {};
}
switch (format) {
case Prospero::BufferFormat::k8UNorm:
case Prospero::BufferFormat::k8SNorm:
case Prospero::BufferFormat::k8UScaled:
case Prospero::BufferFormat::k8SScaled:
case Prospero::BufferFormat::k8UInt:
case Prospero::BufferFormat::k8SInt:
return {format, type, 1, 1, {8, 0, 0, 0}, {0, 0, 0, 0}, false};
case Prospero::BufferFormat::k16UNorm:
case Prospero::BufferFormat::k16SNorm:
case Prospero::BufferFormat::k16UScaled:
case Prospero::BufferFormat::k16SScaled:
case Prospero::BufferFormat::k16UInt:
case Prospero::BufferFormat::k16SInt:
case Prospero::BufferFormat::k16Float:
return {format, type, 1, 2, {16, 0, 0, 0}, {0, 0, 0, 0}, false};
case Prospero::BufferFormat::k8_8UNorm:
case Prospero::BufferFormat::k8_8SNorm:
case Prospero::BufferFormat::k8_8UScaled:
case Prospero::BufferFormat::k8_8SScaled:
case Prospero::BufferFormat::k8_8UInt:
case Prospero::BufferFormat::k8_8SInt:
return {format, type, 2, 2, {8, 8, 0, 0}, {0, 8, 0, 0}, false};
case Prospero::BufferFormat::k32UInt:
case Prospero::BufferFormat::k32SInt:
case Prospero::BufferFormat::k32Float:
return {format, type, 1, 4, {32, 0, 0, 0}, {0, 0, 0, 0}, false};
case Prospero::BufferFormat::k16_16UNorm:
case Prospero::BufferFormat::k16_16SNorm:
case Prospero::BufferFormat::k16_16UScaled:
case Prospero::BufferFormat::k16_16SScaled:
case Prospero::BufferFormat::k16_16UInt:
case Prospero::BufferFormat::k16_16SInt:
case Prospero::BufferFormat::k16_16Float:
return {format, type, 2, 4, {16, 16, 0, 0}, {0, 16, 0, 0}, false};
case Prospero::BufferFormat::k11_11_10UNorm:
case Prospero::BufferFormat::k11_11_10SNorm:
case Prospero::BufferFormat::k11_11_10UScaled:
case Prospero::BufferFormat::k11_11_10SScaled:
case Prospero::BufferFormat::k11_11_10UInt:
case Prospero::BufferFormat::k11_11_10SInt:
case Prospero::BufferFormat::k11_11_10Float:
return {format, type, 3, 4, {11, 11, 10, 0}, {0, 11, 22, 0}, true};
case Prospero::BufferFormat::k10_11_11UNorm:
case Prospero::BufferFormat::k10_11_11SNorm:
case Prospero::BufferFormat::k10_11_11UScaled:
case Prospero::BufferFormat::k10_11_11SScaled:
case Prospero::BufferFormat::k10_11_11UInt:
case Prospero::BufferFormat::k10_11_11SInt:
case Prospero::BufferFormat::k10_11_11Float:
return {format, type, 3, 4, {10, 11, 11, 0}, {0, 10, 21, 0}, true};
case Prospero::BufferFormat::k2_10_10_10UNorm:
case Prospero::BufferFormat::k2_10_10_10SNorm:
case Prospero::BufferFormat::k2_10_10_10UScaled:
case Prospero::BufferFormat::k2_10_10_10SScaled:
case Prospero::BufferFormat::k2_10_10_10UInt:
case Prospero::BufferFormat::k2_10_10_10SInt:
return {format, type, 4, 4, {2, 10, 10, 10}, {0, 2, 12, 22}, true};
case Prospero::BufferFormat::k10_10_10_2UNorm:
case Prospero::BufferFormat::k10_10_10_2SNorm:
case Prospero::BufferFormat::k10_10_10_2UScaled:
case Prospero::BufferFormat::k10_10_10_2SScaled:
case Prospero::BufferFormat::k10_10_10_2UInt:
case Prospero::BufferFormat::k10_10_10_2SInt:
return {format, type, 4, 4, {10, 10, 10, 2}, {0, 10, 20, 30}, true};
case Prospero::BufferFormat::k8_8_8_8UNorm:
case Prospero::BufferFormat::k8_8_8_8SNorm:
case Prospero::BufferFormat::k8_8_8_8UScaled:
case Prospero::BufferFormat::k8_8_8_8SScaled:
case Prospero::BufferFormat::k8_8_8_8UInt:
case Prospero::BufferFormat::k8_8_8_8SInt:
return {format, type, 4, 4, {8, 8, 8, 8}, {0, 8, 16, 24}, false};
case Prospero::BufferFormat::k32_32UInt:
case Prospero::BufferFormat::k32_32SInt:
case Prospero::BufferFormat::k32_32Float:
return {format, type, 2, 8, {32, 32, 0, 0}, {0, 32, 0, 0}, false};
case Prospero::BufferFormat::k16_16_16_16UNorm:
case Prospero::BufferFormat::k16_16_16_16SNorm:
case Prospero::BufferFormat::k16_16_16_16UScaled:
case Prospero::BufferFormat::k16_16_16_16SScaled:
case Prospero::BufferFormat::k16_16_16_16UInt:
case Prospero::BufferFormat::k16_16_16_16SInt:
case Prospero::BufferFormat::k16_16_16_16Float:
return {format, type, 4, 8, {16, 16, 16, 16}, {0, 16, 32, 48}, false};
case Prospero::BufferFormat::k32_32_32UInt:
case Prospero::BufferFormat::k32_32_32SInt:
case Prospero::BufferFormat::k32_32_32Float:
return {format, type, 3, 12, {32, 32, 32, 0}, {0, 32, 64, 0}, false};
case Prospero::BufferFormat::k32_32_32_32UInt:
case Prospero::BufferFormat::k32_32_32_32SInt:
case Prospero::BufferFormat::k32_32_32_32Float:
return {format, type, 4, 16, {32, 32, 32, 32}, {0, 32, 64, 96}, false};
default: return {};
}
}
constexpr bool IsKnownFormat(Prospero::BufferFormat format) {
return GetFormatInfo(format).type != ComponentType::Unknown;
}
constexpr uint32_t GetFormatComponentCount(Prospero::BufferFormat format) {
return GetFormatInfo(format).component_count;
}
constexpr uint32_t GetFormatByteSize(Prospero::BufferFormat format) {
return GetFormatInfo(format).byte_size;
}
constexpr uint32_t GetFormatComponentBits(Prospero::BufferFormat format, uint32_t component) {
const auto info = GetFormatInfo(format);
return component < info.component_count ? info.component_bits[component] : 0u;
}
constexpr uint32_t GetFormatComponentBitOffset(Prospero::BufferFormat format, uint32_t component) {
const auto info = GetFormatInfo(format);
return component < info.component_count ? info.component_bit_offset[component] : 0u;
}
constexpr uint32_t GetFormatComponentByteOffset(Prospero::BufferFormat format, uint32_t component) {
const auto info = GetFormatInfo(format);
if (component >= info.component_count) {
return 0;
}
return info.packed_bitfield ? 0u : info.component_bit_offset[component] / 8u;
}
constexpr bool CanUseTypedBufferLoad(Prospero::BufferFormat format) {
const auto info = GetFormatInfo(format);
return info.type != ComponentType::Unknown && !info.packed_bitfield &&
info.component_bits[0] == 32u;
}
} // namespace Libs::Graphics::ShaderRecompiler::Format
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_BUFFERFORMAT_H_ */
src/graphics/shader/recompiler/ExecMask.cpp
+51
View File
@@ -0,0 +1,51 @@
#include "graphics/shader/recompiler/ExecMask.h"
namespace Libs::Graphics::ShaderRecompiler::Exec {
Mask FullMask(uint32_t wave_size) {
return {0xffffffffu, wave_size == 64u ? 0xffffffffu : 0u};
}
Mask ApplyWaveSize(Mask mask, uint32_t wave_size) {
const auto full = FullMask(wave_size);
mask.low &= full.low;
mask.high &= full.high;
return mask;
}
Mask ReadExec(const State& state) {
return ApplyWaveSize(state.exec, state.wave_size);
}
void WriteExec(State& state, Mask mask) {
state.exec = ApplyWaveSize(mask, state.wave_size);
}
void AndExec(State& state, Mask mask) {
WriteExec(state, {state.exec.low & mask.low, state.exec.high & mask.high});
}
void OrExec(State& state, Mask mask) {
WriteExec(state, {state.exec.low | mask.low, state.exec.high | mask.high});
}
void XorExec(State& state, Mask mask) {
WriteExec(state, {state.exec.low ^ mask.low, state.exec.high ^ mask.high});
}
void InvertExec(State& state) {
const auto full = FullMask(state.wave_size);
WriteExec(state, {~state.exec.low & full.low, ~state.exec.high & full.high});
}
bool IsExecZero(const State& state) {
const auto exec = ReadExec(state);
return exec.low == 0u && exec.high == 0u;
}
bool IsVccZero(const State& state) {
const auto vcc = ApplyWaveSize(state.vcc, state.wave_size);
return vcc.low == 0u && vcc.high == 0u;
}
} // namespace Libs::Graphics::ShaderRecompiler::Exec
src/graphics/shader/recompiler/ExecMask.h
+34
View File
@@ -0,0 +1,34 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_
#include "common/common.h"
namespace Libs::Graphics::ShaderRecompiler::Exec {
struct Mask {
uint32_t low = 0;
uint32_t high = 0;
};
struct State {
uint32_t wave_size = 64;
Mask exec;
Mask vcc;
bool scc = false;
};
Mask FullMask(uint32_t wave_size);
Mask ApplyWaveSize(Mask mask, uint32_t wave_size);
Mask ReadExec(const State& state);
void WriteExec(State& state, Mask mask);
void AndExec(State& state, Mask mask);
void OrExec(State& state, Mask mask);
void XorExec(State& state, Mask mask);
void InvertExec(State& state);
bool IsExecZero(const State& state);
bool IsVccZero(const State& state);
} // namespace Libs::Graphics::ShaderRecompiler::Exec
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_ */
src/graphics/shader/recompiler/ExportOps.cpp
+53
View File
@@ -0,0 +1,53 @@
#include "graphics/shader/recompiler/ExportOps.h"
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated EXP instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t target = (word0 >> 4u) & 0x3fu;
const uint32_t en = word0 & 0xfu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->family = Family::EXP;
inst->opcode_id = target;
inst->opcode = Opcode::Exp;
inst->exp.target = target;
inst->exp.en = en;
inst->exp.compr = ((word0 >> 10u) & 1u) != 0;
inst->exp.done = ((word0 >> 11u) & 1u) != 0;
inst->exp.vm = ((word0 >> 12u) & 1u) != 0;
SetRawWords(inst, code, word_index, 2);
if (!DecodeVectorGpr(word1 & 0xffu, &inst->src0, error) ||
!DecodeVectorGpr((word1 >> 8u) & 0xffu, &inst->src1, error) ||
!DecodeVectorGpr((word1 >> 16u) & 0xffu, &inst->src2, error) ||
!DecodeVectorGpr((word1 >> 24u) & 0xffu, &inst->src3, error)) {
return false;
}
if (target == 0x14u && inst->exp.done && en == 0x1u) {
inst->src_count = 1u;
} else {
inst->src_count = inst->exp.compr ? 2u : 4u;
}
if (en == 0u) {
inst->src_count = 0;
}
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/ExportOps.h
+13
View File
@@ -0,0 +1,13 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_ */
src/graphics/shader/recompiler/ImageOps.cpp
+331
View File
@@ -0,0 +1,331 @@
#include "graphics/shader/recompiler/ImageOps.h"
#include <algorithm>
#include <fmt/format.h>
#include <iterator>
namespace Libs::Graphics::ShaderRecompiler::Decoder {
namespace {
struct MimgSampleInfo {
uint32_t opcode = 0;
const char* name = nullptr;
uint32_t flags = 0;
uint32_t address_components = 3;
};
struct MimgGatherInfo {
uint32_t opcode = 0;
const char* name = nullptr;
Opcode decoded = Opcode::Unsupported;
uint32_t flags = 0;
uint32_t address_components = 3;
};
struct MimgAtomicInfo {
uint32_t opcode = 0;
const char* name = nullptr;
Opcode decoded = Opcode::Unsupported;
};
constexpr ImageDimension DecodeImageDimension(uint32_t dim) {
switch (dim) {
case 1u: return ImageDimension::Dim2D;
case 2u: return ImageDimension::Dim3D;
case 3u: return ImageDimension::Dim2DArray;
case 5u:
case 7u: return ImageDimension::Dim2DArray;
case 6u: return ImageDimension::Dim2D;
default: return ImageDimension::Unknown;
}
}
constexpr uint32_t ImageCoordComponents(ImageDimension dimension) {
switch (dimension) {
case ImageDimension::Dim3D:
case ImageDimension::Dim2DArray: return 3u;
default: return 2u;
}
}
constexpr uint32_t ImageSampleAddressComponents(uint32_t flags, ImageDimension dimension) {
const auto coord_components = ImageCoordComponents(dimension);
uint32_t components = coord_components;
if ((flags & ImageSampleFlagOffset) != 0) {
components++;
}
if ((flags & ImageSampleFlagCompare) != 0) {
components++;
}
if ((flags & ImageSampleFlagBias) != 0) {
components++;
}
if ((flags & ImageSampleFlagLod) != 0) {
components++;
}
if ((flags & ImageSampleFlagDerivative) != 0) {
components += 4u;
}
return components;
}
constexpr MimgSampleInfo SampleInfo(uint32_t opcode, const char* name, uint32_t flags) {
return {opcode, name, flags, ImageSampleAddressComponents(flags, ImageDimension::Dim2D)};
}
constexpr MimgSampleInfo MIMG_SAMPLE_OPS[] = {
SampleInfo(0x20u, "image_sample", 0),
SampleInfo(0x21u, "image_sample_cl", ImageSampleFlagLodClamp),
SampleInfo(0x22u, "image_sample_d", ImageSampleFlagDerivative),
SampleInfo(0x23u, "image_sample_d_cl", ImageSampleFlagDerivative | ImageSampleFlagLodClamp),
SampleInfo(0x24u, "image_sample_l", ImageSampleFlagLod),
SampleInfo(0x25u, "image_sample_b", ImageSampleFlagBias),
SampleInfo(0x26u, "image_sample_b_cl", ImageSampleFlagBias | ImageSampleFlagLodClamp),
SampleInfo(0x27u, "image_sample_lz", ImageSampleFlagLevelZero),
SampleInfo(0x28u, "image_sample_c", ImageSampleFlagCompare),
SampleInfo(0x29u, "image_sample_c_cl", ImageSampleFlagCompare | ImageSampleFlagLodClamp),
SampleInfo(0x2au, "image_sample_c_d", ImageSampleFlagCompare | ImageSampleFlagDerivative),
SampleInfo(0x2bu, "image_sample_c_d_cl",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagLodClamp),
SampleInfo(0x2cu, "image_sample_c_l", ImageSampleFlagCompare | ImageSampleFlagLod),
SampleInfo(0x2du, "image_sample_c_b", ImageSampleFlagCompare | ImageSampleFlagBias),
SampleInfo(0x2eu, "image_sample_c_b_cl",
ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp),
SampleInfo(0x2fu, "image_sample_c_lz", ImageSampleFlagCompare | ImageSampleFlagLevelZero),
SampleInfo(0x30u, "image_sample_o", ImageSampleFlagOffset),
SampleInfo(0x31u, "image_sample_cl_o", ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0x32u, "image_sample_d_o", ImageSampleFlagDerivative | ImageSampleFlagOffset),
SampleInfo(0x33u, "image_sample_d_cl_o",
ImageSampleFlagDerivative | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0x34u, "image_sample_l_o", ImageSampleFlagLod | ImageSampleFlagOffset),
SampleInfo(0x35u, "image_sample_b_o", ImageSampleFlagBias | ImageSampleFlagOffset),
SampleInfo(0x36u, "image_sample_b_cl_o",
ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0x37u, "image_sample_lz_o", ImageSampleFlagLevelZero | ImageSampleFlagOffset),
SampleInfo(0x38u, "image_sample_c_o", ImageSampleFlagCompare | ImageSampleFlagOffset),
SampleInfo(0x39u, "image_sample_c_cl_o",
ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0x3au, "image_sample_c_d_o",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagOffset),
SampleInfo(0x3bu, "image_sample_c_d_cl_o",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagLodClamp |
ImageSampleFlagOffset),
SampleInfo(0x3cu, "image_sample_c_l_o",
ImageSampleFlagCompare | ImageSampleFlagLod | ImageSampleFlagOffset),
SampleInfo(0x3du, "image_sample_c_b_o",
ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset),
SampleInfo(0x3eu, "image_sample_c_b_cl_o",
ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
ImageSampleFlagOffset),
SampleInfo(0x3fu, "image_sample_c_lz_o",
ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset),
SampleInfo(0x68u, "image_sample_cd", ImageSampleFlagDerivative | ImageSampleFlagCd),
SampleInfo(0x69u, "image_sample_cd_cl",
ImageSampleFlagDerivative | ImageSampleFlagCd | ImageSampleFlagLodClamp),
SampleInfo(0x6au, "image_sample_c_cd",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd),
SampleInfo(0x6bu, "image_sample_c_cd_cl",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd |
ImageSampleFlagLodClamp),
SampleInfo(0x6cu, "image_sample_cd_o",
ImageSampleFlagDerivative | ImageSampleFlagCd | ImageSampleFlagOffset),
SampleInfo(0x6du, "image_sample_cd_cl_o",
ImageSampleFlagDerivative | ImageSampleFlagCd | ImageSampleFlagLodClamp |
ImageSampleFlagOffset),
SampleInfo(0x6eu, "image_sample_c_cd_o",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd |
ImageSampleFlagOffset),
SampleInfo(0x6fu, "image_sample_c_cd_cl_o",
ImageSampleFlagCompare | ImageSampleFlagDerivative | ImageSampleFlagCd |
ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0xa0u, "image_sample_a", 0),
SampleInfo(0xa1u, "image_sample_cl_a", ImageSampleFlagLodClamp),
SampleInfo(0xa5u, "image_sample_b_a", ImageSampleFlagBias),
SampleInfo(0xa6u, "image_sample_b_cl_a", ImageSampleFlagBias | ImageSampleFlagLodClamp),
SampleInfo(0xa8u, "image_sample_c_a", ImageSampleFlagCompare),
SampleInfo(0xa9u, "image_sample_c_cl_a", ImageSampleFlagCompare | ImageSampleFlagLodClamp),
SampleInfo(0xadu, "image_sample_c_b_a", ImageSampleFlagCompare | ImageSampleFlagBias),
SampleInfo(0xaeu, "image_sample_c_b_cl_a",
ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp),
SampleInfo(0xb0u, "image_sample_o_a", ImageSampleFlagOffset),
SampleInfo(0xb1u, "image_sample_cl_o_a", ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0xb5u, "image_sample_b_o_a", ImageSampleFlagBias | ImageSampleFlagOffset),
SampleInfo(0xb6u, "image_sample_b_cl_o_a",
ImageSampleFlagBias | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0xb8u, "image_sample_c_o_a", ImageSampleFlagCompare | ImageSampleFlagOffset),
SampleInfo(0xb9u, "image_sample_c_cl_o_a",
ImageSampleFlagCompare | ImageSampleFlagLodClamp | ImageSampleFlagOffset),
SampleInfo(0xbdu, "image_sample_c_b_o_a",
ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagOffset),
SampleInfo(0xbeu, "image_sample_c_b_cl_o_a",
ImageSampleFlagCompare | ImageSampleFlagBias | ImageSampleFlagLodClamp |
ImageSampleFlagOffset),
};
constexpr MimgGatherInfo MIMG_GATHER_OPS[] = {
{0x47u, "image_gather4_lz", Opcode::ImageGather4Lz, ImageSampleFlagLevelZero, 2u},
{0x48u, "image_gather4_c", Opcode::ImageGather4C, ImageSampleFlagCompare, 3u},
{0x4fu, "image_gather4_c_lz", Opcode::ImageGather4CLz,
ImageSampleFlagCompare | ImageSampleFlagLevelZero, 3u},
{0x57u, "image_gather4_lz_o", Opcode::ImageGather4LzO,
ImageSampleFlagLevelZero | ImageSampleFlagOffset, 3u},
{0x58u, "image_gather4_c_o", Opcode::ImageGather4CO,
ImageSampleFlagCompare | ImageSampleFlagOffset, 4u},
{0x5fu, "image_gather4_c_lz_o", Opcode::ImageGather4CLzO,
ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset, 4u},
{0x61u, "image_gather4h", Opcode::ImageGather4H, ImageSampleFlagGatherHorizontal, 2u},
};
constexpr MimgAtomicInfo MIMG_ATOMIC_OPS[] = {
{0x11u, "image_atomic_add", Opcode::ImageAtomicAdd},
{0x15u, "image_atomic_umin", Opcode::ImageAtomicUMin},
{0x17u, "image_atomic_umax", Opcode::ImageAtomicUMax},
{0x18u, "image_atomic_and", Opcode::ImageAtomicAnd},
{0x19u, "image_atomic_or", Opcode::ImageAtomicOr},
{0x1au, "image_atomic_xor", Opcode::ImageAtomicXor},
};
const MimgSampleInfo* LookupSample(uint32_t opcode) {
for (const auto& info: MIMG_SAMPLE_OPS) {
if (info.opcode == opcode) {
return &info;
}
}
return nullptr;
}
const MimgGatherInfo* LookupGather(uint32_t opcode) {
for (const auto& info: MIMG_GATHER_OPS) {
if (info.opcode == opcode) {
return &info;
}
}
return nullptr;
}
const MimgAtomicInfo* LookupAtomic(uint32_t opcode) {
for (const auto& info: MIMG_ATOMIC_OPS) {
if (info.opcode == opcode) {
return &info;
}
}
return nullptr;
}
uint32_t CountDmaskComponents(uint32_t dmask) {
uint32_t count = 0;
for (uint32_t i = 0; i < 4u; i++) {
count += (dmask >> i) & 1u;
}
return count != 0 ? count : 1u;
}
bool IsSingleDmaskBit(uint32_t dmask) {
return dmask != 0u && (dmask & (dmask - 1u)) == 0u;
}
} // namespace
bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated MIMG instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = ((word0 >> 18u) & 0x7fu) | ((word0 & 1u) << 7u);
const uint32_t nsa_dwords = (word0 >> 1u) & 0x3u;
const auto dimension = DecodeImageDimension((word0 >> 3u) & 0x7u);
const uint32_t word_count = 2u + nsa_dwords;
if (word_index + word_count > code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated MIMG NSA instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t vdata = (word1 >> 8u) & 0xffu;
const uint32_t vaddr = word1 & 0xffu;
const uint32_t srsrc = (word1 >> 16u) & 0x1fu;
const uint32_t ssamp = (word1 >> 21u) & 0x1fu;
const bool a16 = ((word1 >> 30u) & 0x1u) != 0u;
const auto* sample = LookupSample(opcode);
const auto* gather = LookupGather(opcode);
const auto* atomic = LookupAtomic(opcode);
inst->pc = pc;
inst->word = word0;
inst->word_count = word_count;
inst->family = Family::MIMG;
inst->opcode_id = opcode;
inst->opcode = (sample != nullptr ? Opcode::ImageSample
: gather != nullptr ? gather->decoded
: atomic != nullptr ? atomic->decoded
: opcode == 0x00u ? Opcode::ImageLoad
: opcode == 0x01u ? Opcode::ImageLoadMip
: opcode == 0x08u ? Opcode::ImageStore
: opcode == 0x09u ? Opcode::ImageStoreMip
: opcode == 0x0eu ? Opcode::ImageGetResinfo
: opcode == 0x60u ? Opcode::ImageGetLod
: Opcode::Unsupported);
inst->dmask = (word0 >> 8u) & 0xfu;
inst->data_dwords = gather != nullptr ? 4u : CountDmaskComponents(inst->dmask);
inst->glc = ((word0 >> 13u) & 1u) != 0;
inst->slc = ((word0 >> 25u) & 1u) != 0;
inst->image_sample_flags = (sample != nullptr ? sample->flags
: gather != nullptr ? gather->flags
: 0u) |
(a16 ? ImageSampleFlagA16 : 0u);
inst->image_dimension = dimension;
inst->image_nsa_dwords = nsa_dwords;
for (uint32_t i = 0; i < nsa_dwords * 4u && i < MaxImageNsaAddressComponents; i++) {
inst->image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
}
inst->image_address_components =
sample != nullptr ? ImageSampleAddressComponents(sample->flags, dimension)
: gather != nullptr ? ImageSampleAddressComponents(gather->flags, dimension)
: atomic != nullptr ? 3u
: opcode == 0x60u ? ImageCoordComponents(dimension)
: opcode == 0x01u ? ImageCoordComponents(dimension) + 1u
: opcode == 0x00u ? ImageCoordComponents(dimension)
: (opcode == 0x09u ? ImageCoordComponents(dimension) + 1u
: opcode == 0x08u ? ImageCoordComponents(dimension)
: 0u);
SetRawWords(inst, code, word_index, word_count);
if (inst->opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode is not implemented");
}
if (gather != nullptr && !IsSingleDmaskBit(inst->dmask)) {
SetUnsupported(inst, Family::MIMG, opcode,
"MIMG image gather requires exactly one dmask bit");
}
DecodeVectorGpr(vdata, &inst->dst, nullptr);
DecodeVectorGpr(vaddr, &inst->src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr);
DecodeScalarSource(ssamp * 4u, pc, &inst->src2, nullptr);
inst->src_count = 3;
return true;
}
const char* MimgSampleOpcodeName(uint32_t opcode) {
const auto* sample = LookupSample(opcode);
return sample != nullptr ? sample->name : nullptr;
}
uint32_t MimgSampleOpcodeFlags(uint32_t opcode) {
const auto* sample = LookupSample(opcode);
return sample != nullptr ? sample->flags : 0u;
}
uint32_t MimgSampleAddressComponents(uint32_t opcode) {
const auto* sample = LookupSample(opcode);
return sample != nullptr ? sample->address_components : 0u;
}
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/ImageOps.h
+17
View File
@@ -0,0 +1,17 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
const char* MimgSampleOpcodeName(uint32_t opcode);
uint32_t MimgSampleOpcodeFlags(uint32_t opcode);
uint32_t MimgSampleAddressComponents(uint32_t opcode);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_ */
src/graphics/shader/recompiler/MemoryOps.cpp
+480
View File
@@ -0,0 +1,480 @@
#include "graphics/shader/recompiler/MemoryOps.h"
#include <fmt/format.h>
#include <iterator>
namespace Libs::Graphics::ShaderRecompiler::Decoder {
namespace {
struct MemoryOpcodeInfo {
uint32_t opcode = 0;
Opcode decoded = Opcode::Unsupported;
uint32_t data_dwords = 1;
uint32_t data_bits = 32;
bool data_signed = false;
bool typed = false;
bool formatted = false;
};
constexpr MemoryOpcodeInfo SMEM_OPS[] = {
{0x00u, Opcode::SLoadDword, 1, 32}, {0x01u, Opcode::SLoadDwordx2, 2, 32},
{0x02u, Opcode::SLoadDwordx4, 4, 32}, {0x03u, Opcode::SLoadDwordx8, 8, 32},
{0x04u, Opcode::SLoadDwordx16, 16, 32}, {0x08u, Opcode::SBufferLoadDword, 1, 32},
{0x09u, Opcode::SBufferLoadDwordx2, 2, 32}, {0x0au, Opcode::SBufferLoadDwordx4, 4, 32},
{0x0bu, Opcode::SBufferLoadDwordx8, 8, 32}, {0x0cu, Opcode::SBufferLoadDwordx16, 16, 32},
};
constexpr MemoryOpcodeInfo MUBUF_OPS[] = {
{0x00u, Opcode::BufferLoadFormatX, 1, 32, false, false, true},
{0x01u, Opcode::BufferLoadFormatXy, 2, 32, false, false, true},
{0x02u, Opcode::BufferLoadFormatXyz, 3, 32, false, false, true},
{0x03u, Opcode::BufferLoadFormatXyzw, 4, 32, false, false, true},
{0x04u, Opcode::BufferStoreFormatX, 1, 32, false, false, true},
{0x05u, Opcode::BufferStoreFormatXy, 2, 32, false, false, true},
{0x06u, Opcode::BufferStoreFormatXyz, 3, 32, false, false, true},
{0x07u, Opcode::BufferStoreFormatXyzw, 4, 32, false, false, true},
{0x08u, Opcode::BufferLoadUbyte, 1, 8},
{0x09u, Opcode::BufferLoadSbyte, 1, 8, true},
{0x0au, Opcode::BufferLoadUshort, 1, 16},
{0x0bu, Opcode::BufferLoadSshort, 1, 16, true},
{0x0cu, Opcode::BufferLoadDword, 1, 32},
{0x0du, Opcode::BufferLoadDwordx2, 2, 32},
{0x0eu, Opcode::BufferLoadDwordx4, 4, 32},
{0x0fu, Opcode::BufferLoadDwordx3, 3, 32},
{0x18u, Opcode::BufferStoreByte, 1, 8},
{0x1au, Opcode::BufferStoreShort, 1, 16},
{0x1cu, Opcode::BufferStoreDword, 1, 32},
{0x1du, Opcode::BufferStoreDwordx2, 2, 32},
{0x1eu, Opcode::BufferStoreDwordx4, 4, 32},
{0x1fu, Opcode::BufferStoreDwordx3, 3, 32},
{0x30u, Opcode::BufferAtomicSwap, 1, 32},
{0x32u, Opcode::BufferAtomicAdd, 1, 32},
{0x33u, Opcode::BufferAtomicSub, 1, 32},
{0x35u, Opcode::BufferAtomicSMin, 1, 32},
{0x36u, Opcode::BufferAtomicUMin, 1, 32},
{0x37u, Opcode::BufferAtomicSMax, 1, 32},
{0x38u, Opcode::BufferAtomicUMax, 1, 32},
{0x39u, Opcode::BufferAtomicAnd, 1, 32},
{0x3au, Opcode::BufferAtomicOr, 1, 32},
{0x3bu, Opcode::BufferAtomicXor, 1, 32},
};
constexpr MemoryOpcodeInfo MTBUF_OPS[] = {
{0x00u, Opcode::TBufferLoadFormatX, 1, 32, false, true, true},
{0x01u, Opcode::TBufferLoadFormatXy, 2, 32, false, true, true},
{0x02u, Opcode::TBufferLoadFormatXyz, 3, 32, false, true, true},
{0x03u, Opcode::TBufferLoadFormatXyzw, 4, 32, false, true, true},
{0x04u, Opcode::TBufferStoreFormatX, 1, 32, false, true, true},
{0x05u, Opcode::TBufferStoreFormatXy, 2, 32, false, true, true},
{0x06u, Opcode::TBufferStoreFormatXyz, 3, 32, false, true, true},
{0x07u, Opcode::TBufferStoreFormatXyzw, 4, 32, false, true, true},
};
constexpr MemoryOpcodeInfo FLAT_OPS[] = {
{0x08u, Opcode::FlatLoadUbyte, 1, 8}, {0x09u, Opcode::FlatLoadSbyte, 1, 8, true},
{0x0au, Opcode::FlatLoadUshort, 1, 16}, {0x0bu, Opcode::FlatLoadSshort, 1, 16, true},
{0x0cu, Opcode::FlatLoadDword, 1, 32}, {0x0du, Opcode::FlatLoadDwordx2, 2, 32},
{0x0eu, Opcode::FlatLoadDwordx4, 4, 32}, {0x0fu, Opcode::FlatLoadDwordx3, 3, 32},
{0x18u, Opcode::FlatStoreByte, 1, 8}, {0x1au, Opcode::FlatStoreShort, 1, 16},
{0x1cu, Opcode::FlatStoreDword, 1, 32}, {0x1du, Opcode::FlatStoreDwordx2, 2, 32},
{0x1eu, Opcode::FlatStoreDwordx4, 4, 32}, {0x1fu, Opcode::FlatStoreDwordx3, 3, 32},
};
constexpr MemoryOpcodeInfo DS_OPS[] = {
{0x00u, Opcode::DsAddU32, 1, 32}, {0x01u, Opcode::DsSubU32, 1, 32},
{0x05u, Opcode::DsMinI32, 1, 32}, {0x06u, Opcode::DsMaxI32, 1, 32},
{0x07u, Opcode::DsMinU32, 1, 32}, {0x08u, Opcode::DsMaxU32, 1, 32},
{0x09u, Opcode::DsAndB32, 1, 32}, {0x0au, Opcode::DsOrB32, 1, 32},
{0x0bu, Opcode::DsXorB32, 1, 32}, {0x0du, Opcode::DsWriteB32, 1, 32},
{0x0eu, Opcode::DsWrite2B32, 2, 32}, {0x0fu, Opcode::DsWrite2St64B32, 2, 32},
{0x12u, Opcode::DsMinF32, 1, 32}, {0x13u, Opcode::DsMaxF32, 1, 32},
{0x1eu, Opcode::DsWriteByte, 1, 8}, {0x1fu, Opcode::DsWriteShort, 1, 16},
{0x20u, Opcode::DsAddRtnU32, 1, 32}, {0x21u, Opcode::DsSubRtnU32, 1, 32},
{0x25u, Opcode::DsMinRtnI32, 1, 32}, {0x26u, Opcode::DsMaxRtnI32, 1, 32},
{0x27u, Opcode::DsMinRtnU32, 1, 32}, {0x28u, Opcode::DsMaxRtnU32, 1, 32},
{0x29u, Opcode::DsAndRtnB32, 1, 32}, {0x2au, Opcode::DsOrRtnB32, 1, 32},
{0x2bu, Opcode::DsXorRtnB32, 1, 32}, {0x2du, Opcode::DsWrxchgRtnB32, 1, 32},
{0x35u, Opcode::DsSwizzleB32, 1, 32}, {0x36u, Opcode::DsReadB32, 1, 32},
{0x37u, Opcode::DsRead2B32, 2, 32}, {0x38u, Opcode::DsRead2B32, 2, 32},
{0x39u, Opcode::DsReadSbyte, 1, 8, true}, {0x3au, Opcode::DsReadUbyte, 1, 8},
{0x3bu, Opcode::DsReadSshort, 1, 16, true}, {0x3cu, Opcode::DsReadUshort, 1, 16},
{0x3du, Opcode::DsConsume, 1, 32}, {0x3eu, Opcode::DsAppend, 1, 32},
{0x4du, Opcode::DsWriteB64, 2, 32}, {0x4eu, Opcode::DsWrite2B64, 4, 32},
{0x4fu, Opcode::DsWrite2St64B64, 4, 32}, {0x76u, Opcode::DsReadB64, 2, 32},
{0x77u, Opcode::DsRead2B64, 4, 32}, {0x78u, Opcode::DsRead2St64B64, 4, 32},
{0xb0u, Opcode::DsWriteAddtidB32, 1, 32}, {0xb1u, Opcode::DsReadAddtidB32, 1, 32},
{0xdeu, Opcode::DsWriteB96, 3, 32}, {0xdfu, Opcode::DsWriteB128, 4, 32},
{0xfeu, Opcode::DsReadB96, 3, 32}, {0xffu, Opcode::DsReadB128, 4, 32},
};
const MemoryOpcodeInfo* LookupMemoryOpcode(const MemoryOpcodeInfo* ops, uint32_t count,
uint32_t opcode) {
for (uint32_t i = 0; i < count; i++) {
if (ops[i].opcode == opcode) {
return &ops[i];
}
}
return nullptr;
}
uint32_t SignExtendU32(uint32_t value, uint32_t bits) {
if (bits == 0u || bits >= 32u) {
return value;
}
const uint32_t sign = 1u << (bits - 1u);
return (value ^ sign) - sign;
}
void MarkMemoryUnsupported(Instruction* inst, Family family, uint32_t opcode, const char* reason) {
inst->family = family;
inst->opcode_id = opcode;
inst->opcode = Opcode::Unsupported;
SetUnsupported(inst, family, opcode, reason);
}
void ApplyMemoryInfo(Instruction* inst, const MemoryOpcodeInfo* info) {
if (info == nullptr) {
inst->opcode = Opcode::Unsupported;
return;
}
inst->opcode = info->decoded;
inst->data_dwords = info->data_dwords;
inst->data_bits = info->data_bits;
inst->data_signed = info->data_signed;
inst->typed = info->typed;
inst->formatted = info->formatted;
}
bool IsDsWriteOpcode(Opcode opcode) {
switch (opcode) {
case Opcode::DsWriteByte:
case Opcode::DsWriteShort:
case Opcode::DsWrite2B32:
case Opcode::DsWrite2St64B32:
case Opcode::DsWrite2B64:
case Opcode::DsWrite2St64B64:
case Opcode::DsWriteB32:
case Opcode::DsWriteB64:
case Opcode::DsWriteB96:
case Opcode::DsWriteB128: return true;
default: return false;
}
}
bool IsDsAtomicOpcode(Opcode opcode) {
switch (opcode) {
case Opcode::DsAddU32:
case Opcode::DsAddRtnU32:
case Opcode::DsSubU32:
case Opcode::DsSubRtnU32:
case Opcode::DsMinI32:
case Opcode::DsMinRtnI32:
case Opcode::DsMaxI32:
case Opcode::DsMaxRtnI32:
case Opcode::DsMinU32:
case Opcode::DsMinRtnU32:
case Opcode::DsMaxU32:
case Opcode::DsMaxRtnU32:
case Opcode::DsAndB32:
case Opcode::DsAndRtnB32:
case Opcode::DsOrB32:
case Opcode::DsOrRtnB32:
case Opcode::DsXorB32:
case Opcode::DsXorRtnB32:
case Opcode::DsWrxchgRtnB32: return true;
default: return false;
}
}
uint32_t DsSourceCount(Opcode opcode) {
switch (opcode) {
case Opcode::DsWrite2B32:
case Opcode::DsWrite2St64B32:
case Opcode::DsWrite2B64:
case Opcode::DsWrite2St64B64:
case Opcode::DsMinF32:
case Opcode::DsMaxF32: return 3u;
case Opcode::DsReadAddtidB32:
case Opcode::DsConsume:
case Opcode::DsAppend: return 0u;
default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u;
}
}
bool IsFlatStoreOpcode(Opcode opcode) {
switch (opcode) {
case Opcode::FlatStoreByte:
case Opcode::FlatStoreShort:
case Opcode::FlatStoreDword:
case Opcode::FlatStoreDwordx2:
case Opcode::FlatStoreDwordx3:
case Opcode::FlatStoreDwordx4: return true;
default: return false;
}
}
} // namespace
bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated SMEM instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = (word0 >> 18u) & 0xffu;
const uint32_t sdst = (word0 >> 6u) & 0x7fu;
const uint32_t sbase = word0 & 0x3fu;
const uint32_t soffset = (word1 >> 25u) & 0x7fu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = SignExtendU32(word1 & 0x1fffffu, 21u);
inst->glc = ((word0 >> 16u) & 1u) != 0;
inst->family = Family::SMEM;
inst->opcode_id = opcode;
const auto* info =
LookupMemoryOpcode(SMEM_OPS, static_cast<uint32_t>(std::size(SMEM_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
}
DecodeScalarDestination(sdst, pc, &inst->dst, nullptr);
// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
// pair index; their descriptor operand consumes four SGPRs from that base.
DecodeScalarSource(sbase * 2u, pc, &inst->src0, nullptr);
DecodeScalarSource(soffset, pc, &inst->src1, nullptr);
inst->src_count = 2;
return true;
}
bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated MUBUF instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = ((word0 >> 18u) & 0x7fu) | (((word0 >> 25u) & 1u) << 7u);
const uint32_t vdata = (word1 >> 8u) & 0xffu;
const uint32_t vaddr = word1 & 0xffu;
const uint32_t srsrc = (word1 >> 16u) & 0x1fu;
const uint32_t soffset = (word1 >> 24u) & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = word0 & 0xfffu;
inst->idxen = ((word0 >> 13u) & 1u) != 0;
inst->offen = ((word0 >> 12u) & 1u) != 0;
inst->glc = ((word0 >> 14u) & 1u) != 0;
inst->slc = ((word1 >> 22u) & 1u) != 0;
inst->family = Family::MUBUF;
inst->opcode_id = opcode;
const auto* info =
LookupMemoryOpcode(MUBUF_OPS, static_cast<uint32_t>(std::size(MUBUF_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented");
}
DecodeVectorGpr(vdata, &inst->dst, nullptr);
DecodeVectorGpr(vaddr, &inst->src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr);
DecodeScalarSource(soffset, pc, &inst->src2, nullptr);
inst->src_count = 3;
return true;
}
bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated MTBUF instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = ((word0 >> 16u) & 0x7u) | (((word1 >> 21u) & 1u) << 3u);
const uint32_t dfmt = (word0 >> 19u) & 0xfu;
const uint32_t nfmt = (word0 >> 23u) & 0x7u;
const uint32_t vdata = (word1 >> 8u) & 0xffu;
const uint32_t vaddr = word1 & 0xffu;
const uint32_t srsrc = (word1 >> 16u) & 0x1fu;
const uint32_t soffset = (word1 >> 24u) & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = word0 & 0xfffu;
inst->idxen = ((word0 >> 13u) & 1u) != 0;
inst->offen = ((word0 >> 12u) & 1u) != 0;
inst->glc = ((word0 >> 14u) & 1u) != 0;
inst->slc = ((word1 >> 22u) & 1u) != 0;
inst->family = Family::MTBUF;
inst->opcode_id = opcode;
inst->data_format = dfmt;
inst->number_format = nfmt;
const auto* info =
LookupMemoryOpcode(MTBUF_OPS, static_cast<uint32_t>(std::size(MTBUF_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::MTBUF, opcode, "MTBUF opcode is not implemented");
}
DecodeVectorGpr(vdata, &inst->dst, nullptr);
DecodeVectorGpr(vaddr, &inst->src0, nullptr);
DecodeScalarSource(srsrc * 4u, pc, &inst->src1, nullptr);
DecodeScalarSource(soffset, pc, &inst->src2, nullptr);
inst->src_count = 3;
return true;
}
bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated FLAT instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t offset = word0 & 0xfffu;
const uint32_t dlc = (word0 >> 12u) & 1u;
const uint32_t lds = (word0 >> 13u) & 1u;
const uint32_t seg = (word0 >> 14u) & 0x3u;
const uint32_t opcode = (word0 >> 18u) & 0x7fu;
const uint32_t vdst = (word1 >> 24u) & 0xffu;
const uint32_t saddr = (word1 >> 16u) & 0x7fu;
const uint32_t data = (word1 >> 8u) & 0xffu;
const uint32_t addr = word1 & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
inst->glc = ((word0 >> 16u) & 1u) != 0;
inst->slc = ((word0 >> 17u) & 1u) != 0;
inst->family = Family::FLAT;
inst->opcode_id = opcode;
inst->memory_segment = seg;
const auto* info =
LookupMemoryOpcode(FLAT_OPS, static_cast<uint32_t>(std::size(FLAT_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (dlc != 0 || lds != 0 || inst->glc || inst->slc || seg == 3u) {
SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
return true;
}
if (inst->opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::FLAT, opcode, "FLAT opcode is not implemented");
return true;
}
DecodeVectorGpr(IsFlatStoreOpcode(inst->opcode) ? data : vdst, &inst->dst, nullptr);
DecodeVectorGpr(addr, &inst->src0, nullptr);
inst->src_count = 1;
if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
DecodeVectorGpr(addr + 1u, &inst->src1, nullptr);
inst->src_count = 2;
} else {
DecodeScalarSource(saddr, pc, &inst->src1, nullptr);
inst->src_count = 2;
}
return true;
}
bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
if (word_index + 1u >= code.size()) {
if (error != nullptr) {
*error = fmt::format("truncated DS instruction at pc 0x{:08x}", pc);
}
return false;
}
const uint32_t word0 = code[word_index];
const uint32_t word1 = code[word_index + 1u];
const uint32_t opcode = (word0 >> 18u) & 0xffu;
const uint32_t offset0 = word0 & 0xffu;
const uint32_t offset1 = (word0 >> 8u) & 0xffu;
const uint32_t vdst = (word1 >> 24u) & 0xffu;
const uint32_t data1 = (word1 >> 16u) & 0xffu;
const uint32_t data0 = (word1 >> 8u) & 0xffu;
const uint32_t addr = word1 & 0xffu;
inst->pc = pc;
inst->word = word0;
inst->word_count = 2;
inst->offset = offset0 | (offset1 << 8u);
inst->gds = ((word0 >> 17u) & 1u) != 0u;
inst->family = Family::DS;
inst->opcode_id = opcode;
const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
ApplyMemoryInfo(inst, info);
SetRawWords(inst, code, word_index, 2);
if (inst->opcode == Opcode::Unsupported) {
MarkMemoryUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
}
if (inst->opcode == Opcode::DsSwizzleB32 && inst->offset >= 0xe000u) {
SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
}
if (inst->gds &&
(inst->opcode == Opcode::DsSwizzleB32 || inst->opcode == Opcode::DsWriteAddtidB32 ||
inst->opcode == Opcode::DsReadAddtidB32)) {
SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
}
if (inst->gds && (inst->opcode == Opcode::DsAppend || inst->opcode == Opcode::DsConsume) &&
inst->offset != 0u) {
SetUnsupported(inst, Family::DS, opcode,
"GDS append/consume requires a zero instruction offset");
}
if (inst->opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
SetUnsupported(inst, Family::DS, opcode,
"DS write addtid data1 operand is not implemented");
}
if (inst->opcode == Opcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) {
SetUnsupported(inst, Family::DS, opcode,
"DS read addtid data operands are not implemented");
}
if (inst->opcode == Opcode::DsWrite2B32 || opcode == 0x37u) {
inst->offset = offset0 * 4u;
inst->secondary_offset = offset1 * 4u;
} else if (inst->opcode == Opcode::DsWrite2St64B32 || opcode == 0x38u) {
inst->offset = offset0 * 256u;
inst->secondary_offset = offset1 * 256u;
} else if (inst->opcode == Opcode::DsWrite2B64 || inst->opcode == Opcode::DsRead2B64) {
inst->offset = offset0 * 8u;
inst->secondary_offset = offset1 * 8u;
} else if (inst->opcode == Opcode::DsWrite2St64B64 || inst->opcode == Opcode::DsRead2St64B64) {
inst->offset = offset0 * 512u;
inst->secondary_offset = offset1 * 512u;
}
DecodeVectorGpr(vdst, &inst->dst, nullptr);
DecodeVectorGpr(addr, &inst->src0, nullptr);
DecodeVectorGpr(data0, &inst->src1, nullptr);
DecodeVectorGpr(data1, &inst->src2, nullptr);
inst->src_count = DsSourceCount(inst->opcode);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/MemoryOps.h
+21
View File
@@ -0,0 +1,21 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ */
src/graphics/shader/recompiler/ResourceMaterialization.cpp
+354
View File
@@ -0,0 +1,354 @@
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/guest_gpu/gpu_format.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
Decoder::ImageDimension DescriptorDimension(const DescriptorValue& descriptor) {
switch (static_cast<Prospero::ImageType>((descriptor.dwords[3] >> 28u) & 0xfu)) {
case Prospero::ImageType::kColor3D: return Decoder::ImageDimension::Dim3D;
case Prospero::ImageType::kCube:
case Prospero::ImageType::kColor1DArray:
case Prospero::ImageType::kColor2DArray:
case Prospero::ImageType::kColor2DMsaaArray: return Decoder::ImageDimension::Dim2DArray;
case Prospero::ImageType::kColor1D:
case Prospero::ImageType::kColor2D:
case Prospero::ImageType::kColor2DMsaa: return Decoder::ImageDimension::Dim2D;
default: return Decoder::ImageDimension::Unknown;
}
}
bool NullImageDescriptor(const DescriptorValue& descriptor) {
return descriptor.dwords[0] == 0 && (descriptor.dwords[1] & 0xffu) == 0;
}
uint32_t DescriptorImageSwizzle(const DescriptorValue& descriptor) {
return descriptor.dwords[3] & 0xfffu;
}
uint32_t DescriptorPackedStride(const DescriptorValue& descriptor) {
const auto stride = (descriptor.dwords[1] >> 16u) & 0x3fffu;
const auto swizzle = (descriptor.dwords[1] >> 31u) & 1u;
const auto index_stride = (descriptor.dwords[3] >> 21u) & 3u;
const auto add_tid = (descriptor.dwords[3] >> 23u) & 1u;
return stride | (swizzle << 14u) | (index_stride << 16u) | (add_tid << 20u);
}
uint32_t DescriptorBufferFormat(const DescriptorValue& descriptor) {
return (descriptor.dwords[3] >> 12u) & 0x7fu;
}
uint64_t AddressSpecialization(const AddressResource& resource,
const ResourceSnapshot::Address& snapshot) {
return resource.kind == ResourceKind::Flat || resource.source == ScalarProvenance::Unknown
? snapshot.binding_base
: snapshot.guest_base - snapshot.binding_base;
}
} // namespace
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot,
std::string* error) {
if (!program.resource_tracking_complete) {
if (error != nullptr) {
*error = "shader resources were not tracked";
}
return false;
}
if (snapshot.buffers.size() != program.info.buffers.size() ||
snapshot.images.size() != program.info.images.size() ||
snapshot.samplers.size() != program.info.samplers.size() ||
snapshot.addresses.size() != program.info.addresses.size()) {
if (error != nullptr) {
*error = "resource snapshot does not match dense shader topology";
}
return false;
}
if (snapshot.flattened_srt.size() != program.srt.reads.size()) {
if (error != nullptr) {
*error = "flattened SRT snapshot does not match the shader plan";
}
return false;
}
if (program.binding_layout_complete) {
for (const auto reg: program.bindings.user_data_registers) {
if (reg < program.user_data_base ||
reg - program.user_data_base >= snapshot.user_data.size()) {
if (error != nullptr) {
*error = fmt::format("runtime snapshot is missing user SGPR {}", reg);
}
return false;
}
}
}
const auto CheckWidth = [&](const auto& values, uint32_t width, const char* kind) {
for (uint32_t i = 0; i < values.size(); i++) {
if (values[i].dword_count != width) {
if (error != nullptr) {
*error = fmt::format("{} descriptor {} has {} dwords", kind, i,
values[i].dword_count);
}
return false;
}
}
return true;
};
for (uint32_t i = 0; i < snapshot.addresses.size(); i++) {
if (snapshot.addresses[i].binding_base > snapshot.addresses[i].guest_base) {
if (error != nullptr) {
*error = fmt::format("address resource {} binds above its guest base", i);
}
return false;
}
}
return CheckWidth(snapshot.buffers, 4, "buffer") && CheckWidth(snapshot.images, 8, "image") &&
CheckWidth(snapshot.samplers, 4, "sampler");
}
bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot,
std::string* error) {
if (!ValidateResourceSnapshot(program, snapshot, error)) {
return false;
}
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
const auto& buffer = program.info.buffers[i];
const auto& descriptor = snapshot.buffers[i];
if (buffer.packed_stride != DescriptorPackedStride(descriptor) ||
buffer.descriptor_format != DescriptorBufferFormat(descriptor)) {
if (error != nullptr) {
*error = fmt::format("buffer descriptor {} no longer matches specialization", i);
}
return false;
}
}
for (uint32_t i = 0; i < program.info.images.size(); i++) {
const auto& image = program.info.images[i];
const auto& descriptor = snapshot.images[i];
if (NullImageDescriptor(descriptor)) {
continue;
}
const auto dimension = DescriptorDimension(descriptor);
if (dimension == Decoder::ImageDimension::Unknown || dimension != image.dimension) {
if (error != nullptr) {
*error =
fmt::format("image descriptor {} no longer matches specialized dimension", i);
}
return false;
}
if (image.kind == ResourceKind::Image || image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint) {
const bool storage = image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint;
if (storage && image.storage_swizzle != DescriptorImageSwizzle(descriptor)) {
if (error != nullptr) {
*error = fmt::format("storage image descriptor {} changed swizzle", i);
}
return false;
}
const auto uint_descriptor =
Prospero::IsUintTextureFormat((descriptor.dwords[1] >> 20u) & 0x1ffu);
const auto uint_program = image.kind == ResourceKind::ImageUint ||
image.kind == ResourceKind::StorageImageUint;
if (uint_descriptor != uint_program && !(image.atomic && uint_program)) {
if (error != nullptr) {
*error =
fmt::format("image descriptor {} no longer matches specialized format", i);
}
return false;
}
if (image.depth_compare && uint_program) {
if (error != nullptr) {
*error = fmt::format("integer image descriptor {} uses depth comparison", i);
}
return false;
}
}
}
for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
if (program.info.addresses[i].specialized_base !=
AddressSpecialization(program.info.addresses[i], snapshot.addresses[i])) {
if (error != nullptr) {
*error = fmt::format("address resource {} no longer matches specialization", i);
}
return false;
}
}
return true;
}
bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot* snapshot, std::string* error) {
if (snapshot == nullptr || !program.resource_tracking_complete) {
if (error != nullptr) {
*error = snapshot == nullptr ? "invalid resource snapshot"
: "shader resources were not tracked";
}
return false;
}
std::vector<DescriptorSourceRequest> requests;
requests.reserve(program.info.buffers.size() + program.info.images.size() +
program.info.samplers.size() + program.info.addresses.size());
for (const auto& buffer: program.info.buffers) {
requests.push_back({buffer.source, buffer.first_use_pc});
}
for (const auto& image: program.info.images) {
requests.push_back({image.source, image.first_use_pc});
}
for (const auto& sampler: program.info.samplers) {
requests.push_back({sampler.source, sampler.first_use_pc});
}
for (const auto& address: program.info.addresses) {
if (address.source != ScalarProvenance::Unknown) {
requests.push_back({address.source, address.first_use_pc});
}
}
std::vector<DescriptorValue> values;
std::vector<uint32_t> flattened_srt;
if (!EvaluateRuntimeSources(program, requests, runtime, &values, &flattened_srt, error)) {
return false;
}
ResourceSnapshot next;
auto cursor = values.begin();
next.buffers.assign(cursor, cursor + program.info.buffers.size());
cursor += program.info.buffers.size();
next.images.assign(cursor, cursor + program.info.images.size());
cursor += program.info.images.size();
next.samplers.assign(cursor, cursor + program.info.samplers.size());
cursor += program.info.samplers.size();
for (const auto& address: program.info.addresses) {
if (address.source != ScalarProvenance::Unknown) {
const auto value = *cursor++;
auto base = (static_cast<uint64_t>(value.dwords[0]) |
static_cast<uint64_t>(value.dwords[1]) << 32u) &
AddressMask;
if (address.kind == ResourceKind::ScalarBuffer) {
base &= ~uint64_t {3};
}
const auto before = static_cast<uint64_t>(-static_cast<int64_t>(address.min_offset));
const auto binding_base = address.kind == ResourceKind::Flat
? base & ~(FlatAddressWindowSize - 1u)
: base >= before ? base - before
: 0;
next.addresses.push_back({base, binding_base});
} else {
if (!runtime.flat_memory_base.has_value()) {
if (error != nullptr) {
*error =
fmt::format("unbased {} address at pc 0x{:08x} requires runtime "
"guest-address translation",
address.kind == ResourceKind::Flat ? "FLAT" : "global/scratch",
address.first_use_pc);
}
return false;
}
next.addresses.push_back({*runtime.flat_memory_base, *runtime.flat_memory_base});
}
}
next.flattened_srt = std::move(flattened_srt);
next.user_data.assign(runtime.user_data.begin(), runtime.user_data.end());
if (!ValidateResourceSnapshot(program, next, error)) {
return false;
}
*snapshot = std::move(next);
return true;
}
bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std::string* error) {
if (program == nullptr || !program->resource_tracking_complete ||
program->shader_info_complete || program->binding_layout_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid resource specialization program"
: !program->resource_tracking_complete ? "shader resources were not tracked"
: "resource specialization is too late";
}
return false;
}
if (!ValidateResourceSnapshot(*program, snapshot, error)) {
return false;
}
auto next = program->info;
for (uint32_t i = 0; i < next.buffers.size(); i++) {
next.buffers[i].packed_stride = DescriptorPackedStride(snapshot.buffers[i]);
next.buffers[i].descriptor_format = DescriptorBufferFormat(snapshot.buffers[i]);
}
for (uint32_t i = 0; i < next.addresses.size(); i++) {
next.addresses[i].specialized_base =
AddressSpecialization(next.addresses[i], snapshot.addresses[i]);
}
for (uint32_t i = 0; i < next.images.size(); i++) {
const auto& descriptor = snapshot.images[i];
auto& image = next.images[i];
if (NullImageDescriptor(descriptor)) {
continue;
}
const auto descriptor_dimension = DescriptorDimension(descriptor);
if (descriptor_dimension == Decoder::ImageDimension::Unknown) {
if (error != nullptr) {
*error = fmt::format(
"image descriptor {} has unsupported type {}: {:08x},{:08x},{:08x},{:08x},"
"{:08x},{:08x},{:08x},{:08x}",
i, (descriptor.dwords[3] >> 28u) & 0xfu, descriptor.dwords[0],
descriptor.dwords[1], descriptor.dwords[2], descriptor.dwords[3],
descriptor.dwords[4], descriptor.dwords[5], descriptor.dwords[6],
descriptor.dwords[7]);
}
return false;
}
image.dimension = descriptor_dimension;
if (image.kind == ResourceKind::StorageImage ||
image.kind == ResourceKind::StorageImageUint) {
image.storage_swizzle = DescriptorImageSwizzle(descriptor);
}
if (Prospero::IsUintTextureFormat((descriptor.dwords[1] >> 20u) & 0x1ffu)) {
switch (image.kind) {
case ResourceKind::Image: image.kind = ResourceKind::ImageUint; break;
case ResourceKind::StorageImage: image.kind = ResourceKind::StorageImageUint; break;
default: break;
}
}
}
struct ImagePatch {
Instruction* inst;
ResourceKind kind;
Decoder::ImageDimension dimension;
};
std::vector<ImagePatch> patches;
for (auto& block: program->blocks) {
for (auto& inst: block.instructions) {
if (inst.memory.kind != ResourceKind::Image &&
inst.memory.kind != ResourceKind::ImageUint &&
inst.memory.kind != ResourceKind::StorageImage &&
inst.memory.kind != ResourceKind::StorageImageUint) {
continue;
}
if (inst.memory.resource >= next.images.size()) {
if (error != nullptr) {
*error = fmt::format("image instruction at pc 0x{:08x} has invalid resource {}",
inst.pc, inst.memory.resource);
}
return false;
}
const auto& image = next.images[inst.memory.resource];
patches.push_back({&inst, image.kind, image.dimension});
}
}
program->info = std::move(next);
for (const auto& patch: patches) {
patch.inst->memory.kind = patch.kind;
patch.inst->memory.image_dimension = patch.dimension;
}
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ResourceMaterialization.h
+42
View File
@@ -0,0 +1,42 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_
#include "graphics/shader/recompiler/SrtWalker.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
constexpr uint64_t FlatAddressWindowSize = 0x10000;
struct ResourceSnapshot {
struct Address {
uint64_t guest_base = 0;
uint64_t binding_base = 0;
bool operator==(const Address& other) const = default;
};
std::vector<DescriptorValue> buffers;
std::vector<DescriptorValue> images;
std::vector<DescriptorValue> samplers;
std::vector<Address> addresses;
std::vector<uint32_t> flattened_srt;
std::vector<uint32_t> user_data;
};
bool ValidateResourceSnapshot(const Program& program, const ResourceSnapshot& snapshot,
std::string* error);
bool ValidateResourceSpecialization(const Program& program, const ResourceSnapshot& snapshot,
std::string* error);
// Resolves the immutable dense resource topology against one runtime user-data/SRT snapshot.
// On failure the destination is unchanged.
bool MaterializeResources(const Program& program, const SrtRuntime& runtime,
ResourceSnapshot* snapshot, std::string* error);
// Applies runtime descriptor shape/format facts to a copied dense topology before layout and
// emission. On failure the program is unchanged.
bool SpecializeResources(Program* program, const ResourceSnapshot& snapshot, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */
src/graphics/shader/recompiler/ResourceTracking.cpp
+449
View File
@@ -0,0 +1,449 @@
#include "graphics/shader/recompiler/ResourceTracking.h"
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
bool IsAtomic(Opcode op) {
switch (op) {
case Opcode::AtomicSwapU32:
case Opcode::AtomicAddU32:
case Opcode::AtomicSubU32:
case Opcode::AtomicSMinI32:
case Opcode::AtomicUMinU32:
case Opcode::AtomicSMaxI32:
case Opcode::AtomicUMaxU32:
case Opcode::AtomicAndU32:
case Opcode::AtomicOrU32:
case Opcode::AtomicXorU32: return true;
default: return false;
}
}
bool IsWrite(Opcode op) {
switch (op) {
case Opcode::BufferStoreByte:
case Opcode::BufferStoreShort:
case Opcode::BufferStoreDword:
case Opcode::ImageStore: return true;
default: return IsAtomic(op);
}
}
bool NeedsSampler(Opcode op) {
return op == Opcode::ImageSample || op == Opcode::ImageGather4 || op == Opcode::ImageGetLod;
}
bool IsDepthCompare(const Instruction& inst) {
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0;
}
ImageMipMode MipMode(const Instruction& inst) {
const bool storage = inst.memory.kind == ResourceKind::StorageImage ||
inst.memory.kind == ResourceKind::StorageImageUint;
return storage && inst.memory.image_has_mip ? ImageMipMode::DynamicStorage : ImageMipMode::None;
}
bool IsBuffer(const Instruction& inst) {
return inst.memory.kind == ResourceKind::Buffer ||
(inst.memory.kind == ResourceKind::ScalarBuffer && inst.op == Opcode::SBufferLoadDword);
}
bool IsImage(const Instruction& inst) {
return inst.memory.kind == ResourceKind::Image || inst.memory.kind == ResourceKind::ImageUint ||
inst.memory.kind == ResourceKind::StorageImage ||
inst.memory.kind == ResourceKind::StorageImageUint;
}
bool IsAddress(const Instruction& inst) {
return inst.op == Opcode::SLoadDword || inst.memory.kind == ResourceKind::Flat ||
inst.memory.kind == ResourceKind::Global || inst.memory.kind == ResourceKind::Scratch;
}
uint32_t ByteExtent(const Instruction& inst) {
const auto bytes = std::max((inst.memory.data_bits + 7u) / 8u, 1u);
const auto count = std::max(inst.memory.data_dwords, 1u);
const auto end =
static_cast<uint64_t>(inst.memory.offset) + static_cast<uint64_t>(bytes) * count;
return end > UINT32_MAX ? UINT32_MAX : static_cast<uint32_t>(end);
}
bool ContainsUnknown(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>* visited,
std::vector<uint32_t>* path) {
path->push_back(id);
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return true;
}
if ((*visited)[id] != 0) {
path->pop_back();
return false;
}
(*visited)[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (ContainsUnknown(provenance, arg, visited, path)) {
return true;
}
}
path->pop_back();
return false;
}
const auto args = ScalarValueArgCount(value.op);
for (uint32_t i = 0; i < args; i++) {
if (ContainsUnknown(provenance, value.args[i], visited, path)) {
return true;
}
}
path->pop_back();
return false;
}
bool IsLoopInvariantValue(const ScalarProvenance& provenance, uint32_t id,
std::vector<uint8_t>* visiting) {
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return false;
}
if ((*visiting)[id] != 0) {
return true;
}
(*visiting)[id] = 1;
const auto& value = provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
uint32_t invariant = ScalarProvenance::Undefined;
for (const auto arg: value.phi_args) {
if (arg == id) {
continue;
}
if (invariant == ScalarProvenance::Undefined) {
invariant = arg;
} else if (arg != invariant) {
return false;
}
}
return invariant != ScalarProvenance::Undefined &&
IsLoopInvariantValue(provenance, invariant, visiting);
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (!IsLoopInvariantValue(provenance, value.args[i], visiting)) {
return false;
}
}
return true;
}
bool IsLoopInvariantDescriptor(const ScalarProvenance& provenance,
const DescriptorValue& descriptor) {
std::vector<uint8_t> visiting(provenance.values.size());
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (!IsLoopInvariantValue(provenance, descriptor.dwords[i], &visiting)) {
return false;
}
}
return true;
}
class Tracker {
public:
explicit Tracker(Program& program): m_program(program) {}
bool Run(std::string* error) {
if (m_program.resource_tracking_complete) {
return Fail(0, error, "resources already tracked");
}
if (!m_program.srt_plan_complete) {
return Fail(0, error, "SRT plan is not ready");
}
if (!m_program.srt_patching_complete) {
return Fail(0, error, "SRT reads were not patched");
}
for (auto& block: m_program.blocks) {
for (auto& inst: block.instructions) {
if (!Collect(&inst, error)) {
return false;
}
}
}
m_program.info = std::move(m_info);
for (const auto& patch: m_patches) {
patch.inst->memory.resource = patch.resource;
patch.inst->memory.sampler = patch.sampler;
patch.inst->memory.resource_source = ScalarProvenance::Undefined;
patch.inst->memory.sampler_source = ScalarProvenance::Undefined;
}
m_program.resource_tracking_complete = true;
return true;
}
private:
struct Patch {
Instruction* inst = nullptr;
uint32_t resource = 0;
uint32_t sampler = 0;
};
bool Fail(uint32_t pc, std::string* error, const std::string& reason) const {
if (error != nullptr) {
*error = fmt::format("shader resource tracking: hash=0x{:016x} stage={} pc=0x{:08x} {}",
m_program.shader_hash, StageName(m_program.stage), pc, reason);
}
return false;
}
bool ValidateSource(uint32_t source, uint32_t dwords, uint32_t pc, std::string* error) const {
const auto* descriptor = GetDescriptorSource(m_program, source);
if (descriptor == nullptr || descriptor->dword_count != dwords) {
return Fail(pc, error,
fmt::format("descriptor source {} is missing or has wrong width", source));
}
std::vector<uint8_t> visited(m_program.provenance.values.size());
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
std::vector<uint32_t> path;
if (ContainsUnknown(m_program.provenance, descriptor->dwords[i], &visited, &path)) {
const auto value = descriptor->dwords[i];
std::string chain;
for (const auto id: path) {
const auto op = id < m_program.provenance.values.size()
? static_cast<uint32_t>(m_program.provenance.values[id].op)
: UINT32_MAX;
chain += fmt::format("{}{}:{}({})", chain.empty() ? "" : " -> ", id, op,
ScalarValueToString(m_program.provenance, id));
}
return Fail(
pc, error,
fmt::format(
"descriptor source {} dword {} contains an unknown value {} ({}) path {}",
source, i, value, ScalarValueToString(m_program.provenance, value), chain));
}
}
const auto dynamic =
std::find(m_program.srt.dynamic_sources.begin(), m_program.srt.dynamic_sources.end(),
source) != m_program.srt.dynamic_sources.end();
if (dynamic && !IsLoopInvariantDescriptor(m_program.provenance, *descriptor)) {
std::string detail;
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
const auto id = descriptor->dwords[i];
detail +=
fmt::format(" d{}={}({})", i, id,
id < m_program.provenance.values.size()
? static_cast<uint32_t>(m_program.provenance.values[id].op)
: UINT32_MAX);
if (id < m_program.provenance.values.size()) {
for (const auto arg: m_program.provenance.values[id].phi_args) {
detail += fmt::format(
"/{}({})", arg,
arg < m_program.provenance.values.size()
? static_cast<uint32_t>(m_program.provenance.values[arg].op)
: UINT32_MAX);
}
}
}
return Fail(pc, error,
fmt::format("descriptor source {} requires unsupported GPU selection{}",
source, detail));
}
if (!DescriptorSourceResolved(m_program, source) && !dynamic) {
return Fail(pc, error, fmt::format("descriptor source {} is unresolved", source));
}
return true;
}
uint32_t AddBuffer(const Instruction& inst) {
for (uint32_t i = 0; i < m_info.buffers.size(); i++) {
if (m_info.buffers[i].source == inst.memory.resource_source) {
Merge(&m_info.buffers[i], inst);
return i;
}
}
if (m_info.buffers.size() >= ShaderInfo::MaxBuffers) {
return UINT32_MAX;
}
BufferResource resource;
resource.source = inst.memory.resource_source;
resource.first_use_pc = inst.pc;
Merge(&resource, inst);
m_info.buffers.push_back(resource);
return static_cast<uint32_t>(m_info.buffers.size() - 1);
}
void Merge(BufferResource* resource, const Instruction& inst) const {
resource->first_use_pc = std::min(resource->first_use_pc, inst.pc);
resource->max_byte_extent = std::max(resource->max_byte_extent, ByteExtent(inst));
resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op);
resource->written = resource->written || IsWrite(inst.op);
resource->atomic = resource->atomic || IsAtomic(inst.op);
resource->formatted = resource->formatted || inst.memory.formatted;
resource->scalar = resource->scalar || inst.memory.kind == ResourceKind::ScalarBuffer;
}
uint32_t AddImage(const Instruction& inst) {
for (uint32_t i = 0; i < m_info.images.size(); i++) {
auto& image = m_info.images[i];
if (image.source == inst.memory.resource_source && image.kind == inst.memory.kind &&
image.dimension == inst.memory.image_dimension && image.mip_mode == MipMode(inst) &&
image.depth_compare == IsDepthCompare(inst)) {
Merge(&image, inst);
return i;
}
}
if (m_info.images.size() >= ShaderInfo::MaxImages) {
return UINT32_MAX;
}
ImageResource image;
image.source = inst.memory.resource_source;
image.first_use_pc = inst.pc;
image.kind = inst.memory.kind;
image.dimension = inst.memory.image_dimension;
image.mip_mode = MipMode(inst);
image.depth_compare = IsDepthCompare(inst);
Merge(&image, inst);
m_info.images.push_back(image);
return static_cast<uint32_t>(m_info.images.size() - 1);
}
uint32_t AddAddress(const Instruction& inst) {
auto immediate = static_cast<int32_t>(inst.memory.offset);
if (inst.memory.kind == ResourceKind::ScalarBuffer) {
immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
}
const auto min_offset =
inst.memory.resource_source == ScalarProvenance::Unknown ? 0 : std::min(immediate, 0);
for (uint32_t i = 0; i < m_info.addresses.size(); i++) {
auto& address = m_info.addresses[i];
if (address.source == inst.memory.resource_source && address.kind == inst.memory.kind) {
address.first_use_pc = std::min(address.first_use_pc, inst.pc);
address.min_offset = std::min(address.min_offset, min_offset);
address.read = address.read || !IsWrite(inst.op) || IsAtomic(inst.op);
address.written = address.written || IsWrite(inst.op);
address.atomic = address.atomic || IsAtomic(inst.op);
return i;
}
}
if (m_info.addresses.size() >= ShaderInfo::MaxAddresses) {
return UINT32_MAX;
}
AddressResource address {inst.memory.resource_source, inst.pc, inst.memory.kind,
min_offset};
address.read = !IsWrite(inst.op) || IsAtomic(inst.op);
address.written = IsWrite(inst.op);
address.atomic = IsAtomic(inst.op);
m_info.addresses.push_back(address);
return static_cast<uint32_t>(m_info.addresses.size() - 1);
}
void Merge(ImageResource* resource, const Instruction& inst) const {
resource->first_use_pc = std::min(resource->first_use_pc, inst.pc);
resource->read = resource->read || !IsWrite(inst.op) || IsAtomic(inst.op);
resource->written = resource->written || IsWrite(inst.op);
resource->atomic = resource->atomic || IsAtomic(inst.op);
}
uint32_t AddSampler(const Instruction& inst) {
for (uint32_t i = 0; i < m_info.samplers.size(); i++) {
if (m_info.samplers[i].source == inst.memory.sampler_source) {
m_info.samplers[i].first_use_pc =
std::min(m_info.samplers[i].first_use_pc, inst.pc);
return i;
}
}
if (m_info.samplers.size() >= ShaderInfo::MaxSamplers) {
return UINT32_MAX;
}
m_info.samplers.push_back({inst.memory.sampler_source, inst.pc});
return static_cast<uint32_t>(m_info.samplers.size() - 1);
}
bool AddSampledPair(uint32_t image, uint32_t sampler, uint32_t pc, std::string* error) {
for (auto& pair: m_info.sampled_pairs) {
if (pair.image == image && pair.sampler == sampler) {
pair.first_use_pc = std::min(pair.first_use_pc, pc);
return true;
}
}
if (m_info.sampled_pairs.size() >= ShaderInfo::MaxSampledPairs) {
return Fail(pc, error, "sampled image/sampler pair limit exceeded");
}
m_info.sampled_pairs.push_back({image, sampler, pc});
return true;
}
bool Collect(Instruction* inst, std::string* error) {
if (IsAddress(*inst)) {
const bool unbased = inst->memory.resource_source == ScalarProvenance::Unknown;
if ((!unbased && !ValidateSource(inst->memory.resource_source, 2, inst->pc, error)) ||
(unbased && inst->memory.kind != ResourceKind::Flat &&
inst->memory.kind != ResourceKind::Global &&
inst->memory.kind != ResourceKind::Scratch)) {
return unbased ? Fail(inst->pc, error, "scalar memory base is unresolved") : false;
}
const auto resource = AddAddress(*inst);
if (resource == UINT32_MAX) {
return Fail(inst->pc, error, "address resource limit exceeded");
}
m_patches.push_back({inst, resource, 0});
return true;
}
if (!IsBuffer(*inst) && !IsImage(*inst)) {
return true;
}
if (!ValidateSource(inst->memory.resource_source, IsBuffer(*inst) ? 4u : 8u, inst->pc,
error)) {
return false;
}
Patch patch;
patch.inst = inst;
patch.resource = IsBuffer(*inst) ? AddBuffer(*inst) : AddImage(*inst);
if (patch.resource == UINT32_MAX) {
return Fail(inst->pc, error,
IsBuffer(*inst) ? "buffer resource limit exceeded"
: "image resource limit exceeded");
}
if (NeedsSampler(inst->op)) {
if (!ValidateSource(inst->memory.sampler_source, 4, inst->pc, error)) {
return false;
}
patch.sampler = AddSampler(*inst);
if (patch.sampler == UINT32_MAX) {
return Fail(inst->pc, error, "sampler resource limit exceeded");
}
if (!AddSampledPair(patch.resource, patch.sampler, inst->pc, error)) {
return false;
}
}
m_patches.push_back(patch);
return true;
}
Program& m_program;
ShaderInfo m_info;
std::vector<Patch> m_patches;
};
} // namespace
bool TrackResources(Program* program, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error = "shader resource tracking: hash=0x0000000000000000 stage=unknown "
"pc=0x00000000 invalid program";
}
return false;
}
return Tracker(*program).Run(error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ResourceTracking.h
+14
View File
@@ -0,0 +1,14 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_
#include "graphics/shader/recompiler/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
// Collects immutable resource topology, then replaces descriptor operands with dense indices.
// On failure the program is left unpatched.
bool TrackResources(Program* program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCETRACKING_H_ */
src/graphics/shader/recompiler/ScalarAluOps.cpp
+283
View File
@@ -0,0 +1,283 @@
#include "graphics/shader/recompiler/ScalarAluOps.h"
#include <iterator>
namespace Libs::Graphics::ShaderRecompiler::Decoder {
namespace {
struct OpcodeMap {
uint32_t opcode = 0;
Opcode ir = Opcode::Unknown;
};
constexpr OpcodeMap SOP2_OPS[] = {
{0x00u, Opcode::SAddU32}, {0x01u, Opcode::SSubU32}, {0x02u, Opcode::SAddI32},
{0x03u, Opcode::SSubI32}, {0x04u, Opcode::SAddcU32}, {0x05u, Opcode::SSubbU32},
{0x06u, Opcode::SMinI32}, {0x07u, Opcode::SMinU32}, {0x08u, Opcode::SMaxI32},
{0x09u, Opcode::SMaxU32}, {0x0au, Opcode::SCselectB32}, {0x0bu, Opcode::SCselectB64},
{0x0eu, Opcode::SAndB32}, {0x0fu, Opcode::SAndB64}, {0x10u, Opcode::SOrB32},
{0x11u, Opcode::SOrB64}, {0x12u, Opcode::SXorB32}, {0x13u, Opcode::SXorB64},
{0x14u, Opcode::SAndn2B32}, {0x15u, Opcode::SAndn2B64}, {0x16u, Opcode::SOrn2B32},
{0x17u, Opcode::SOrn2B64}, {0x18u, Opcode::SNandB32}, {0x19u, Opcode::SNandB64},
{0x1au, Opcode::SNorB32}, {0x1bu, Opcode::SNorB64}, {0x1cu, Opcode::SXnorB32},
{0x1du, Opcode::SXnorB64}, {0x1eu, Opcode::SLshlB32}, {0x1fu, Opcode::SLshlB64},
{0x20u, Opcode::SLshrB32}, {0x21u, Opcode::SLshrB64}, {0x22u, Opcode::SAshrI32},
{0x24u, Opcode::SBfmB32}, {0x25u, Opcode::SBfmB64}, {0x26u, Opcode::SMulI32},
{0x27u, Opcode::SBfeU32}, {0x29u, Opcode::SBfeU64}, {0x2eu, Opcode::SLshl1AddU32},
{0x2fu, Opcode::SLshl2AddU32}, {0x30u, Opcode::SLshl3AddU32}, {0x31u, Opcode::SLshl4AddU32},
{0x32u, Opcode::SPackLlB32B16}, {0x33u, Opcode::SPackLhB32B16}, {0x34u, Opcode::SPackHhB32B16},
{0x35u, Opcode::SMulHiU32},
};
constexpr OpcodeMap SOP1_OPS[] = {
{0x03u, Opcode::SMovB32},
{0x04u, Opcode::SMovB64},
{0x07u, Opcode::SNotB32},
{0x08u, Opcode::SNotB64},
{0x0au, Opcode::SWqmB64},
{0x0bu, Opcode::SBrevB32},
{0x0fu, Opcode::SBcnt1I32B32},
{0x10u, Opcode::SBcnt1I32B64},
{0x13u, Opcode::SFf1I32B32},
{0x16u, Opcode::SFlbitI32B64},
{0x1bu, Opcode::SBitset0B32},
{0x1du, Opcode::SBitset1B32},
{0x1fu, Opcode::SGetpcB64},
{0x20u, Opcode::SSetpcB64},
{0x24u, Opcode::SAndSaveexecB64},
{0x28u, Opcode::SOrn2SaveexecB64},
{0x34u, Opcode::SAbsI32},
{0x37u, Opcode::SAndn1SaveexecB64},
{0x3bu, Opcode::SBitreplicateB64B32},
{0x3cu, Opcode::SAndSaveexecB32},
{0x44u, Opcode::SAndn1SaveexecB32},
};
constexpr OpcodeMap SOPC_OPS[] = {
{0x00u, Opcode::SCmpEqI32}, {0x01u, Opcode::SCmpLgI32}, {0x02u, Opcode::SCmpGtI32},
{0x03u, Opcode::SCmpGeI32}, {0x04u, Opcode::SCmpLtI32}, {0x05u, Opcode::SCmpLeI32},
{0x06u, Opcode::SCmpEqU32}, {0x07u, Opcode::SCmpLgU32}, {0x08u, Opcode::SCmpGtU32},
{0x09u, Opcode::SCmpGeU32}, {0x0au, Opcode::SCmpLtU32}, {0x0bu, Opcode::SCmpLeU32},
{0x0cu, Opcode::SBitcmp0B32}, {0x0du, Opcode::SBitcmp1B32}, {0x13u, Opcode::SCmpLgU64},
};
constexpr OpcodeMap SOPK_OPS[] = {
{0x00u, Opcode::SMovkI32}, {0x03u, Opcode::SCmpEqI32}, {0x04u, Opcode::SCmpLgI32},
{0x05u, Opcode::SCmpGtI32}, {0x06u, Opcode::SCmpGeI32}, {0x07u, Opcode::SCmpLtI32},
{0x08u, Opcode::SCmpLeI32}, {0x09u, Opcode::SCmpEqU32}, {0x0au, Opcode::SCmpLgU32},
{0x0bu, Opcode::SCmpGtU32}, {0x0cu, Opcode::SCmpGeU32}, {0x0du, Opcode::SCmpLtU32},
{0x0eu, Opcode::SCmpLeU32}, {0x0fu, Opcode::SAddI32}, {0x10u, Opcode::SMulkI32},
{0x13u, Opcode::SSetregB32}, {0x17u, Opcode::SWaitcnt}, {0x18u, Opcode::SWaitcnt},
{0x19u, Opcode::SWaitcnt}, {0x1au, Opcode::SWaitcnt},
};
constexpr OpcodeMap SOPP_OPS[] = {
{0x00u, Opcode::SNop}, {0x01u, Opcode::SEndpgm}, {0x02u, Opcode::SBranch},
{0x04u, Opcode::SCbranchScc0}, {0x05u, Opcode::SCbranchScc1}, {0x06u, Opcode::SCbranchVccz},
{0x07u, Opcode::SCbranchVccnz}, {0x08u, Opcode::SCbranchExecz}, {0x09u, Opcode::SCbranchExecnz},
{0x0au, Opcode::SBarrier}, {0x0cu, Opcode::SWaitcnt}, {0x0eu, Opcode::SSleep},
{0x10u, Opcode::SSendmsg}, {0x16u, Opcode::STtraceData}, {0x20u, Opcode::SInstPrefetch},
};
Opcode Lookup(const OpcodeMap* ops, uint32_t count, uint32_t opcode) {
for (uint32_t i = 0; i < count; i++) {
if (ops[i].opcode == opcode) {
return ops[i].ir;
}
}
return Opcode::Unsupported;
}
bool DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) {
if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error) ||
!DecodeScalarSource(ssrc1, pc, &inst->src1, error)) {
return false;
}
inst->src_count = 2;
return ReadLiteralOperands(code, word_index, inst, error);
}
} // namespace
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu;
const uint32_t sdst = (word >> 16u) & 0x7fu;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOP1;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOP1_OPS, static_cast<uint32_t>(std::size(SOP1_OPS)), opcode);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented");
return true;
}
switch (inst->opcode) {
case Opcode::SGetpcB64:
inst->src_count = 0;
return DecodeScalarDestination(sdst, pc, &inst->dst, error);
case Opcode::SSetpcB64:
inst->src_count = 1;
inst->dst.kind = OperandKind::Null;
if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error)) {
return false;
}
return ReadLiteralOperands(code, word_index, inst, error);
default: break;
}
if (!DecodeScalarSource(ssrc0, pc, &inst->src0, error) ||
!DecodeScalarDestination(sdst, pc, &inst->dst, error)) {
return false;
}
inst->src_count = 1;
return ReadLiteralOperands(code, word_index, inst, error);
}
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 23u) & 0x7fu;
const uint32_t ssrc1 = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu;
const uint32_t sdst = (word >> 16u) & 0x7fu;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOP2;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOP2_OPS, static_cast<uint32_t>(std::size(SOP2_OPS)), opcode);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented");
return true;
}
if (!DecodeScalarDestination(sdst, pc, &inst->dst, error)) {
return false;
}
return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
}
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 23u) & 0x1fu;
const uint32_t sdst = (word >> 16u) & 0x7fu;
const auto imm = static_cast<int16_t>(word & 0xffffu);
(void)error;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOPK;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOPK_OPS, static_cast<uint32_t>(std::size(SOPK_OPS)), opcode);
inst->src0.kind = OperandKind::IntegerInlineConstant;
inst->src0.signed_val = imm;
inst->src0.value = static_cast<uint32_t>(imm);
inst->src_count = 1;
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented");
return true;
}
switch (inst->opcode) {
case Opcode::SMovkI32: return DecodeScalarDestination(sdst, pc, &inst->dst, error);
case Opcode::SWaitcnt: {
const uint32_t waitcnt = word & 0xffffu;
inst->dst.kind = OperandKind::Null;
inst->src0.signed_val = static_cast<int32_t>(waitcnt);
inst->src0.value = waitcnt;
inst->src_count = 1;
return true;
}
case Opcode::SSetregB32:
inst->dst.kind = OperandKind::Null;
inst->src1.kind = OperandKind::LiteralConstant;
inst->src1.value = word & 0xffffu;
inst->src1.signed_val = static_cast<int32_t>(imm);
inst->src_count = 2;
return DecodeScalarSource(sdst, pc, &inst->src0, error);
default: break;
}
inst->src1 = inst->src0;
if (!DecodeScalarSource(sdst, pc, &inst->src0, error)) {
return false;
}
if (inst->opcode == Opcode::SAddI32 || inst->opcode == Opcode::SMulkI32) {
inst->src_count = 2;
return DecodeScalarDestination(sdst, pc, &inst->dst, error);
}
inst->dst.kind = OperandKind::Scc;
inst->src_count = 2;
return true;
}
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t ssrc1 = (word >> 8u) & 0xffu;
const uint32_t ssrc0 = word & 0xffu;
const uint32_t opcode = (word >> 16u) & 0x7fu;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOPC;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOPC_OPS, static_cast<uint32_t>(std::size(SOPC_OPS)), opcode);
inst->dst.kind = OperandKind::Scc;
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented");
return true;
}
return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
}
bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error) {
const uint32_t word = code[word_index];
const uint32_t opcode = (word >> 16u) & 0x7fu;
const uint32_t simm = word & 0xffffu;
(void)error;
inst->pc = pc;
inst->word = word;
inst->family = Family::SOPP;
inst->opcode_id = opcode;
inst->opcode = Lookup(SOPP_OPS, static_cast<uint32_t>(std::size(SOPP_OPS)), opcode);
inst->src0.kind = OperandKind::LiteralConstant;
inst->src0.value = simm;
inst->src0.signed_val = static_cast<int16_t>(simm);
inst->src_count = (inst->opcode == Opcode::SNop || inst->opcode == Opcode::SWaitcnt ||
inst->opcode == Opcode::SSleep || inst->opcode == Opcode::SSendmsg ||
inst->opcode == Opcode::STtraceData || inst->opcode == Opcode::SInstPrefetch)
? 1
: 0;
inst->branch_offset = static_cast<int32_t>(static_cast<int16_t>(simm)) * 4;
inst->branch_target = pc + 4u + static_cast<uint32_t>(inst->branch_offset);
SetRawWords(inst, code, word_index, 1);
if (inst->opcode == Opcode::Unsupported) {
SetUnsupported(inst, Family::SOPP, opcode, "SOPP control-flow opcode is not implemented");
}
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
src/graphics/shader/recompiler/ScalarAluOps.h
+21
View File
@@ -0,0 +1,21 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_ */
src/graphics/shader/recompiler/ScalarProvenance.cpp
+946
View File
@@ -0,0 +1,946 @@
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm>
#include <array>
#include <compare>
#include <deque>
#include <fmt/format.h>
#include <map>
namespace Libs::Graphics::ShaderRecompiler::IR {
uint32_t ScalarValueArgCount(ScalarValueOp op) {
switch (op) {
case ScalarValueOp::Not: return 1;
case ScalarValueOp::Add:
case ScalarValueOp::Sub:
case ScalarValueOp::Mul:
case ScalarValueOp::And:
case ScalarValueOp::AndNot:
case ScalarValueOp::Or:
case ScalarValueOp::OrNot:
case ScalarValueOp::Xor:
case ScalarValueOp::ShiftLeft:
case ScalarValueOp::ShiftRight:
case ScalarValueOp::ShiftRightArithmetic:
case ScalarValueOp::BitFieldMaskU32:
case ScalarValueOp::BitFieldMaskU64Low:
case ScalarValueOp::BitFieldMaskU64High: return 2;
case ScalarValueOp::AddCarry:
case ScalarValueOp::Carry:
case ScalarValueOp::SubBorrow:
case ScalarValueOp::Borrow:
case ScalarValueOp::Add3:
case ScalarValueOp::ShiftLeftAdd:
case ScalarValueOp::ShiftLeftAddCarry:
case ScalarValueOp::AddShiftLeft:
case ScalarValueOp::XorAdd:
case ScalarValueOp::ShiftLeftOr:
case ScalarValueOp::ReadConst: return 3;
case ScalarValueOp::ReadConstBuffer: return 5;
default: return 0;
}
}
namespace {
constexpr uint32_t ScalarRegisters = 128;
constexpr uint32_t VectorRegisters = 256;
struct ScalarState {
std::array<uint32_t, ScalarRegisters> regs = {};
std::array<uint32_t, VectorRegisters> address_bases = {};
uint32_t scc = ScalarProvenance::Undefined;
uint32_t m0 = ScalarProvenance::Undefined;
std::map<uint64_t, uint32_t> vector_lanes;
void Fill(uint32_t value) {
regs.fill(value);
address_bases.fill(value);
scc = value;
m0 = value;
vector_lanes.clear();
}
bool operator==(const ScalarState&) const = default;
};
struct ValueKey {
ScalarValueOp op = ScalarValueOp::Undefined;
uint32_t imm = 0;
std::array<uint32_t, 6> args = {};
auto operator<=>(const ValueKey&) const = default;
};
struct DescriptorKey {
std::array<uint32_t, 8> dwords = {};
uint32_t dword_count = 0;
auto operator<=>(const DescriptorKey&) const = default;
};
bool ScalarRegister(const Operand& operand, uint32_t& reg) {
if (operand.kind != OperandKind::Register) {
return false;
}
if (operand.reg.file == RegisterFile::Scalar && operand.reg.index < ScalarRegisters) {
reg = operand.reg.index;
return true;
}
if (operand.reg.file == RegisterFile::Vcc && operand.reg.index < 2) {
reg = 106u + operand.reg.index;
return true;
}
return false;
}
bool VectorRegister(const Operand& operand, uint32_t& reg) {
if (operand.kind != OperandKind::Register || operand.reg.file != RegisterFile::Vector ||
operand.reg.index >= VectorRegisters) {
return false;
}
reg = operand.reg.index;
return true;
}
bool FlatStore(Opcode op) {
switch (op) {
case Opcode::FlatStoreByte:
case Opcode::FlatStoreShort:
case Opcode::FlatStoreDword: return true;
default: return false;
}
}
bool PairDwordOpcode(Opcode op) {
switch (op) {
case Opcode::MoveU64:
case Opcode::WqmB64:
case Opcode::SaveexecB64:
case Opcode::BitwiseAndU64:
case Opcode::BitwiseAndNotU64:
case Opcode::BitwiseOrU64:
case Opcode::BitwiseOrNotU64:
case Opcode::BitwiseXorU64:
case Opcode::BitwiseNandU64:
case Opcode::BitwiseNorU64:
case Opcode::BitwiseXnorU64:
case Opcode::BitwiseNotU64:
case Opcode::BitFieldMaskU64:
case Opcode::BitFieldExtractU64:
case Opcode::BitReplicateB64B32:
case Opcode::ShiftLeftLogicalU64:
case Opcode::ShiftRightLogicalU64:
case Opcode::SelectU64: return true;
default: return false;
}
}
bool ScalarMemoryLoad(Opcode op) {
return op == Opcode::SLoadDword || op == Opcode::SBufferLoadDword;
}
bool ValueResolved(const ScalarProvenance& provenance, uint32_t id, std::vector<uint8_t>* visited) {
if (id <= ScalarProvenance::Unknown || id >= provenance.values.size()) {
return false;
}
if ((*visited)[id] == 1) {
return false;
}
if ((*visited)[id] == 2) {
return true;
}
if ((*visited)[id] == 3) {
return false;
}
(*visited)[id] = 1;
const auto& value = provenance.values[id];
bool resolved = true;
if (value.op == ScalarValueOp::Phi) {
resolved = !value.phi_args.empty() &&
std::all_of(value.phi_args.begin(), value.phi_args.end(), [&](uint32_t arg) {
return ValueResolved(provenance, arg, visited);
});
} else {
const auto args = ScalarValueArgCount(value.op);
for (uint32_t i = 0; i < args && resolved; i++) {
resolved = ValueResolved(provenance, value.args[i], visited);
}
}
(*visited)[id] = resolved ? 2 : 3;
return resolved;
}
class Builder {
public:
explicit Builder(Program* program): m_program(program), m_graph(program->provenance) {}
bool Run(std::string* error) {
m_graph = {};
m_graph.values.resize(2);
m_graph.values[ScalarProvenance::Undefined].op = ScalarValueOp::Undefined;
m_graph.values[ScalarProvenance::Unknown].op = ScalarValueOp::Unknown;
const auto block_count = m_program->blocks.size();
m_entry.resize(block_count);
m_exit.resize(block_count);
m_phi.resize(block_count);
m_exit_ready.assign(block_count, false);
m_queued.assign(block_count, false);
for (auto& state: m_entry) {
state.Fill(ScalarProvenance::Undefined);
}
for (auto& state: m_exit) {
state.Fill(ScalarProvenance::Undefined);
}
for (auto& phis: m_phi) {
phis.Fill(ScalarProvenance::Undefined);
}
m_user_data.Fill(ScalarProvenance::Undefined);
for (uint32_t reg = 0; reg < ScalarRegisters; reg++) {
m_user_data.regs[reg] =
reg >= m_program->user_data_base &&
reg - m_program->user_data_base < m_program->user_data_count
? InternValue({ScalarValueOp::UserData, 0, reg})
: ScalarProvenance::Unknown;
}
m_user_data.scc = ScalarProvenance::Unknown;
m_user_data.m0 = ScalarProvenance::Unknown;
if (block_count == 0) {
return true;
}
for (const auto& block: m_program->blocks) {
for (const auto predecessor: block.predecessors) {
if (predecessor >= block_count) {
return Fail(error, fmt::format("scalar provenance has invalid predecessor {}",
predecessor));
}
}
for (const auto successor: block.successors) {
if (successor >= block_count) {
return Fail(error, fmt::format("scalar provenance has invalid successor {}",
successor));
}
}
if (!ValidateScalarMemoryGroups(block, error)) {
return false;
}
}
Queue(0);
while (!m_work.empty()) {
const auto block_index = m_work.front();
m_work.pop_front();
m_queued[block_index] = false;
auto next_entry = MergeEntry(block_index);
const bool entry_changed = next_entry != m_entry[block_index];
m_entry[block_index] = next_entry;
auto next_exit = Execute(m_program->blocks[block_index], next_entry, false);
const bool was_ready = m_exit_ready[block_index];
if (was_ready && !entry_changed && next_exit == m_exit[block_index]) {
continue;
}
m_exit[block_index] = next_exit;
m_exit_ready[block_index] = true;
for (const auto successor: m_program->blocks[block_index].successors) {
Queue(successor);
}
}
for (size_t block = 0; block < block_count; block++) {
Execute(m_program->blocks[block], m_entry[block], true);
}
return true;
}
private:
bool Fail(std::string* error, std::string text) const {
if (error != nullptr) {
*error = std::move(text);
}
return false;
}
uint32_t AddValue(ScalarValue value) {
m_graph.values.push_back(std::move(value));
return static_cast<uint32_t>(m_graph.values.size() - 1);
}
bool ValidateScalarMemoryGroups(const BasicBlock& block, std::string* error) const {
for (size_t index = 0; index < block.instructions.size();) {
const auto& first = block.instructions[index];
if (!ScalarMemoryLoad(first.op) || first.memory.component_count == 1) {
index++;
continue;
}
const auto count = first.memory.component_count;
uint32_t first_dst = 0;
if (count == 0 || first.memory.component_index != 0 ||
index + count > block.instructions.size() ||
!ScalarRegister(first.dst, first_dst)) {
return Fail(error,
fmt::format("malformed scalar-memory group at pc=0x{:x}", first.pc));
}
for (uint32_t component = 0; component < count; component++) {
const auto& inst = block.instructions[index + component];
uint32_t dst = 0;
if (inst.op != first.op || inst.pc != first.pc ||
inst.memory.component_count != count ||
inst.memory.component_index != component ||
inst.memory.resource != first.memory.resource ||
inst.memory.offset != first.memory.offset + component * 4u ||
inst.src_count != first.src_count ||
!std::equal(std::begin(inst.src), std::end(inst.src), std::begin(first.src)) ||
!ScalarRegister(inst.dst, dst) || dst != first_dst + component) {
return Fail(error,
fmt::format("malformed scalar-memory component {} at pc=0x{:x}",
component, first.pc));
}
}
index += count;
}
return true;
}
uint32_t InternValue(ScalarValue value) {
const ValueKey key {value.op, value.imm, value.args};
if (const auto found = m_values.find(key); found != m_values.end()) {
return found->second;
}
const auto id = AddValue(std::move(value));
m_values.emplace(key, id);
return id;
}
uint32_t Constant(uint32_t value) { return InternValue({ScalarValueOp::Constant, 0, value}); }
static uint64_t VectorLaneKey(uint32_t reg, uint32_t lane) {
return (static_cast<uint64_t>(reg) << 32u) | lane;
}
uint32_t OperandValue(const Operand& operand, const ScalarState& state) {
uint32_t reg = 0;
if (ScalarRegister(operand, reg)) {
return state.regs[reg];
}
if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::Scc) {
return state.scc;
}
if (operand.kind == OperandKind::Register && operand.reg.file == RegisterFile::M0) {
return state.m0;
}
if (operand.kind == OperandKind::ImmediateU32 ||
operand.kind == OperandKind::PcRelativeU32) {
if (operand.kind == OperandKind::PcRelativeU32) {
return InternValue({ScalarValueOp::PcRelativeLow, 0, operand.imm});
}
return Constant(operand.imm);
}
return ScalarProvenance::Unknown;
}
uint32_t Define(const Instruction& inst, ScalarValueOp op, const ScalarState& state,
uint32_t imm = 0) {
ScalarValue node;
node.op = op;
node.pc = inst.pc;
node.imm = imm;
for (uint32_t i = 0; i < inst.src_count && i < node.args.size(); i++) {
node.args[i] = OperandValue(inst.src[i], state);
}
return InternValue(std::move(node));
}
uint32_t DefineBorrow(const Instruction& inst, const ScalarState& state) {
ScalarValue node;
node.op = ScalarValueOp::Borrow;
node.pc = inst.pc;
node.args[0] =
inst.src_count > 0 ? OperandValue(inst.src[0], state) : ScalarProvenance::Unknown;
node.args[1] =
inst.src_count > 1 ? OperandValue(inst.src[1], state) : ScalarProvenance::Unknown;
node.args[2] = inst.src_count > 2 ? OperandValue(inst.src[2], state) : Constant(0);
return InternValue(std::move(node));
}
bool ConstantOperand(const Operand& operand, const ScalarState& state, uint32_t* value) {
if (value == nullptr) {
return false;
}
const auto id = OperandValue(operand, state);
if (id >= m_graph.values.size() || m_graph.values[id].op != ScalarValueOp::Constant) {
return false;
}
*value = m_graph.values[id].imm;
return true;
}
uint32_t ReadVectorLane(const Instruction& inst, const ScalarState& state) {
if (inst.src_count < 2 || inst.src[0].kind != OperandKind::Register ||
inst.src[0].reg.file != RegisterFile::Vector ||
(m_program->wave_size != 32 && m_program->wave_size != 64)) {
return ScalarProvenance::Unknown;
}
uint32_t lane = 0;
if (!ConstantOperand(inst.src[1], state, &lane)) {
return ScalarProvenance::Unknown;
}
const auto found = state.vector_lanes.find(
VectorLaneKey(inst.src[0].reg.index, lane % m_program->wave_size));
return found != state.vector_lanes.end() ? found->second : ScalarProvenance::Unknown;
}
void ClearVectorLanes(uint32_t reg, ScalarState* state) {
const auto first = state->vector_lanes.lower_bound(VectorLaneKey(reg, 0));
const auto last = state->vector_lanes.lower_bound((static_cast<uint64_t>(reg) + 1u) << 32u);
state->vector_lanes.erase(first, last);
}
void WriteVectorDestination(const Instruction& inst, ScalarState* state) {
if (inst.dst.kind != OperandKind::Register || inst.dst.reg.file != RegisterFile::Vector) {
return;
}
const uint32_t reg = inst.dst.reg.index;
uint32_t address_base = ScalarProvenance::Unknown;
if (inst.op == Opcode::MoveU32 || inst.op == Opcode::IAddU32 ||
inst.op == Opcode::IAddCarryU32 || inst.op == Opcode::ISubU32) {
for (uint32_t i = 0; i < inst.src_count; i++) {
uint32_t src_reg = 0;
if (VectorRegister(inst.src[i], src_reg) &&
state->address_bases[src_reg] > ScalarProvenance::Unknown) {
address_base = state->address_bases[src_reg];
break;
}
}
if (address_base == ScalarProvenance::Unknown) {
for (uint32_t i = 0; i < inst.src_count; i++) {
uint32_t src_reg = 0;
if (ScalarRegister(inst.src[i], src_reg) &&
state->regs[src_reg] > ScalarProvenance::Unknown) {
address_base = state->regs[src_reg];
break;
}
}
}
}
state->address_bases[reg] = ScalarProvenance::Unknown;
if (inst.op == Opcode::MoveRelDestU32) {
state->vector_lanes.clear();
return;
}
if (inst.op != Opcode::WriteLaneU32) {
uint32_t dwords = std::max(inst.memory.data_dwords, 1u);
if (PairDwordOpcode(inst.op) || inst.op == Opcode::UMadU64U32) {
dwords = std::max(dwords, 2u);
}
for (uint32_t i = 0; i < dwords && reg <= UINT32_MAX - i; i++) {
ClearVectorLanes(reg + i, state);
if (reg + i < state->address_bases.size()) {
state->address_bases[reg + i] = ScalarProvenance::Unknown;
}
}
state->address_bases[reg] = address_base;
return;
}
if (inst.src_count < 2 || (m_program->wave_size != 32 && m_program->wave_size != 64)) {
ClearVectorLanes(reg, state);
return;
}
uint32_t lane = 0;
if (!ConstantOperand(inst.src[1], *state, &lane)) {
ClearVectorLanes(reg, state);
return;
}
state->vector_lanes[VectorLaneKey(reg, lane % m_program->wave_size)] =
OperandValue(inst.src[0], *state);
}
uint32_t ReadConst(const Instruction& inst, const ScalarState& state, bool buffer) {
ScalarValue node;
node.op = buffer ? ScalarValueOp::ReadConstBuffer : ScalarValueOp::ReadConst;
node.pc = inst.pc;
const auto base = buffer ? inst.memory.resource * 4u : inst.memory.resource;
if (base + 1u >= ScalarRegisters) {
return ScalarProvenance::Unknown;
}
node.imm = inst.memory.offset;
if (buffer) {
if (base + 3u >= ScalarRegisters) {
return ScalarProvenance::Unknown;
}
for (uint32_t i = 0; i < 4; i++) {
node.args[i] = state.regs[base + i];
}
node.args[4] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0);
} else {
node.args[0] = state.regs[base];
node.args[1] = state.regs[base + 1u];
node.args[2] = inst.src_count != 0 ? OperandValue(inst.src[0], state) : Constant(0);
}
return InternValue(std::move(node));
}
ScalarValueOp Operation(Opcode op) const {
switch (op) {
case Opcode::IAddU32:
case Opcode::ScalarSignedAddOverflowI32: return ScalarValueOp::Add;
case Opcode::IAddCarryU32:
case Opcode::ScalarAddCarryU32: return ScalarValueOp::AddCarry;
case Opcode::ISubU32:
case Opcode::ScalarSubBorrowU32:
case Opcode::ScalarSignedSubOverflowI32: return ScalarValueOp::Sub;
case Opcode::ScalarSubBorrowCarryU32: return ScalarValueOp::SubBorrow;
case Opcode::IMulU32: return ScalarValueOp::Mul;
case Opcode::BitwiseAndU32: return ScalarValueOp::And;
case Opcode::BitwiseAndNotU32: return ScalarValueOp::AndNot;
case Opcode::BitwiseOrU32: return ScalarValueOp::Or;
case Opcode::BitwiseOrNotU32: return ScalarValueOp::OrNot;
case Opcode::BitwiseXorU32: return ScalarValueOp::Xor;
case Opcode::BitwiseNotU32: return ScalarValueOp::Not;
case Opcode::ShiftLeftLogicalU32: return ScalarValueOp::ShiftLeft;
case Opcode::ShiftRightLogicalU32: return ScalarValueOp::ShiftRight;
case Opcode::ShiftRightArithmeticI32: return ScalarValueOp::ShiftRightArithmetic;
case Opcode::BitFieldMaskU32: return ScalarValueOp::BitFieldMaskU32;
case Opcode::IAdd3U32: return ScalarValueOp::Add3;
case Opcode::ScalarShiftLeftAddCarryU32:
case Opcode::ShiftLeftAddU32: return ScalarValueOp::ShiftLeftAdd;
case Opcode::AddShiftLeftU32: return ScalarValueOp::AddShiftLeft;
case Opcode::XorAddU32: return ScalarValueOp::XorAdd;
case Opcode::ShiftLeftOrU32: return ScalarValueOp::ShiftLeftOr;
default: return ScalarValueOp::Unknown;
}
}
void UpdateScc(const Instruction& inst, const ScalarState& before, ScalarState* state) {
switch (inst.op) {
case Opcode::ScalarAddCarryU32:
state->scc = Define(inst, ScalarValueOp::Carry, before);
break;
case Opcode::ScalarSubBorrowU32:
case Opcode::ScalarSubBorrowCarryU32: state->scc = DefineBorrow(inst, before); break;
case Opcode::ScalarShiftLeftAddCarryU32:
state->scc = Define(inst, ScalarValueOp::ShiftLeftAddCarry, before);
break;
default: break;
}
}
void WriteDestination(const Instruction& inst, ScalarState* state) {
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::Scc) {
state->scc = inst.op == Opcode::MoveU32 && inst.src_count != 0
? OperandValue(inst.src[0], *state)
: ScalarProvenance::Unknown;
return;
}
if (inst.dst.kind == OperandKind::Register && inst.dst.reg.file == RegisterFile::M0) {
const auto before = *state;
const auto op = Operation(inst.op);
state->m0 = inst.op == Opcode::MoveU32 && inst.src_count != 0
? OperandValue(inst.src[0], before)
: op != ScalarValueOp::Unknown ? Define(inst, op, before)
: ScalarProvenance::Unknown;
UpdateScc(inst, before, state);
return;
}
uint32_t dst = 0;
if (!ScalarRegister(inst.dst, dst)) {
return;
}
const auto before = *state;
state->regs[dst] = ScalarProvenance::Unknown;
if (PairDwordOpcode(inst.op) && dst + 1 < ScalarRegisters) {
state->regs[dst + 1] = ScalarProvenance::Unknown;
}
uint32_t value = ScalarProvenance::Unknown;
switch (inst.op) {
case Opcode::MoveU32:
case Opcode::MoveF32Bits:
value = inst.src_count == 0 ? ScalarProvenance::Unknown
: inst.src[0].kind == OperandKind::PcRelativeU32
? InternValue({ScalarValueOp::PcRelativeLow, inst.pc, inst.src[0].imm})
: OperandValue(inst.src[0], before);
if (inst.op == Opcode::MoveU32 && inst.src_count != 0 &&
inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].imm == 0 &&
dst != 0) {
const auto low = before.regs[dst - 1];
if (low < m_graph.values.size() &&
m_graph.values[low].op == ScalarValueOp::PcRelativeLow &&
m_graph.values[low].pc == inst.pc) {
value = InternValue(
{ScalarValueOp::PcRelativeHigh, inst.pc, m_graph.values[low].imm});
}
}
break;
case Opcode::MoveU64: {
uint32_t src = 0;
if (inst.src_count != 0 && dst + 1 < ScalarRegisters) {
if (ScalarRegister(inst.src[0], src) && src + 1 < ScalarRegisters) {
value = before.regs[src];
state->regs[dst + 1] = before.regs[src + 1];
} else {
value = OperandValue(inst.src[0], before);
state->regs[dst + 1] = Constant(
inst.src[0].kind == OperandKind::ImmediateU32 && inst.src[0].sext_64
? UINT32_MAX
: 0u);
}
}
break;
}
case Opcode::BitFieldMaskU64:
value = Define(inst, ScalarValueOp::BitFieldMaskU64Low, before);
if (dst + 1 < ScalarRegisters) {
state->regs[dst + 1] = Define(inst, ScalarValueOp::BitFieldMaskU64High, before);
}
break;
case Opcode::SLoadDword: value = ReadConst(inst, before, false); break;
case Opcode::SBufferLoadDword: value = ReadConst(inst, before, true); break;
case Opcode::ReadLaneU32: value = ReadVectorLane(inst, before); break;
default: {
const auto op = Operation(inst.op);
if (op != ScalarValueOp::Unknown) {
value = Define(inst, op, before);
}
break;
}
}
state->regs[dst] = value;
UpdateScc(inst, before, state);
uint32_t dst2 = 0;
if (ScalarRegister(inst.dst2, dst2)) {
state->regs[dst2] =
inst.src_count > 1 ? OperandValue(inst.src[1], before) : ScalarProvenance::Unknown;
}
}
uint32_t AddDescriptor(DescriptorValue descriptor) {
const DescriptorKey key {descriptor.dwords, descriptor.dword_count};
if (const auto found = m_descriptors.find(key); found != m_descriptors.end()) {
return found->second;
}
m_graph.descriptors.push_back(descriptor);
const auto id = static_cast<uint32_t>(m_graph.descriptors.size() - 1) + 2u;
m_descriptors.emplace(key, id);
return id;
}
uint32_t AddDescriptor(const ScalarState& state, uint32_t base, uint32_t dwords) {
if (base >= ScalarRegisters || dwords == 0 || dwords > 8 ||
base + dwords > ScalarRegisters) {
return ScalarProvenance::Unknown;
}
DescriptorValue descriptor;
descriptor.dword_count = dwords;
for (uint32_t i = 0; i < dwords; i++) {
descriptor.dwords[i] = state.regs[base + i];
}
return AddDescriptor(descriptor);
}
uint32_t AddFlatAddressDescriptor(const Instruction& inst, const ScalarState& state) {
const uint32_t first = FlatStore(inst.op) ? 1u : 0u;
if (inst.src_count < first + 2u) {
return ScalarProvenance::Unknown;
}
uint32_t low = 0;
uint32_t high = 0;
if (!VectorRegister(inst.src[first], low) || !VectorRegister(inst.src[first + 1u], high) ||
state.address_bases[low] <= ScalarProvenance::Unknown ||
state.address_bases[high] <= ScalarProvenance::Unknown) {
return ScalarProvenance::Unknown;
}
DescriptorValue descriptor;
descriptor.dword_count = 2;
descriptor.dwords[0] = state.address_bases[low];
descriptor.dwords[1] = state.address_bases[high];
return AddDescriptor(descriptor);
}
void AttachSources(Instruction* inst, const ScalarState& state) {
inst->memory.resource_source = 0;
inst->memory.sampler_source = 0;
if (inst->op == Opcode::SLoadDword) {
inst->memory.resource_source = AddDescriptor(state, inst->memory.resource, 2);
return;
}
if (inst->memory.kind == ResourceKind::Flat || inst->memory.kind == ResourceKind::Global ||
inst->memory.kind == ResourceKind::Scratch) {
inst->memory.resource_source = ScalarProvenance::Unknown;
if (inst->memory.kind == ResourceKind::Flat) {
inst->memory.resource_source = AddFlatAddressDescriptor(*inst, state);
}
for (uint32_t i = 0; i < inst->src_count; i++) {
const auto& source = inst->src[i];
if (inst->memory.resource_source == ScalarProvenance::Unknown &&
source.kind == OperandKind::Register &&
source.reg.file == RegisterFile::Scalar) {
inst->memory.resource_source = AddDescriptor(state, source.reg.index, 2);
break;
}
}
return;
}
if (inst->memory.kind == ResourceKind::Buffer ||
(inst->memory.kind == ResourceKind::ScalarBuffer &&
inst->op == Opcode::SBufferLoadDword)) {
const auto base = inst->memory.resource * 4u;
inst->memory.resource_source = AddDescriptor(state, base, 4);
return;
}
if (inst->memory.kind == ResourceKind::Image ||
inst->memory.kind == ResourceKind::ImageUint ||
inst->memory.kind == ResourceKind::StorageImage ||
inst->memory.kind == ResourceKind::StorageImageUint) {
inst->memory.resource_source = AddDescriptor(state, inst->memory.resource * 4u, 8);
if (inst->op == Opcode::ImageSample || inst->op == Opcode::ImageGather4 ||
inst->op == Opcode::ImageGetLod) {
inst->memory.sampler_source = AddDescriptor(state, inst->memory.sampler * 4u, 4);
}
}
}
void AttachScalarSources(Instruction* inst, const ScalarState& state) {
std::fill(std::begin(inst->scalar_sources), std::end(inst->scalar_sources),
ScalarProvenance::Undefined);
for (uint32_t i = 0; i < inst->src_count; i++) {
const auto& source = inst->src[i];
if (source.kind == OperandKind::Register && source.reg.file != RegisterFile::Vector &&
source.reg.file != RegisterFile::Exec) {
inst->scalar_sources[i] = OperandValue(source, state);
}
}
}
ScalarState Execute(BasicBlock& block, ScalarState state, bool attach_sources) {
for (size_t index = 0; index < block.instructions.size();) {
auto& inst = block.instructions[index];
uint32_t first_dst = 0;
if (ScalarMemoryLoad(inst.op) && inst.memory.component_count > 1 &&
ScalarRegister(inst.dst, first_dst)) {
const auto before = state;
for (uint32_t component = 0; component < inst.memory.component_count; component++) {
auto& component_inst = block.instructions[index + component];
if (attach_sources) {
component_inst.scalar_value = ScalarProvenance::Undefined;
AttachScalarSources(&component_inst, before);
AttachSources(&component_inst, before);
}
auto component_state = before;
WriteDestination(component_inst, &component_state);
uint32_t dst = 0;
ScalarRegister(component_inst.dst, dst);
state.regs[dst] = component_state.regs[dst];
if (attach_sources) {
component_inst.scalar_value = component_state.regs[dst];
}
}
index += inst.memory.component_count;
continue;
}
if (attach_sources) {
inst.scalar_value = ScalarProvenance::Undefined;
AttachScalarSources(&inst, state);
AttachSources(&inst, state);
}
WriteVectorDestination(inst, &state);
WriteDestination(inst, &state);
if (attach_sources) {
uint32_t dst = 0;
if (ScalarRegister(inst.dst, dst)) {
inst.scalar_value = state.regs[dst];
}
}
index++;
}
return state;
}
ScalarState MergeEntry(size_t block_index) {
ScalarState result;
result.Fill(ScalarProvenance::Undefined);
const auto& block = m_program->blocks[block_index];
const auto merge = [&](uint32_t initial, auto predecessor_value, uint32_t* phi) {
std::vector<uint32_t> incoming;
if (initial != ScalarProvenance::Undefined) {
incoming.push_back(initial);
}
for (const auto predecessor: block.predecessors) {
const auto value = predecessor_value(predecessor, m_exit[predecessor]);
if (value != ScalarProvenance::Undefined &&
std::find(incoming.begin(), incoming.end(), value) == incoming.end()) {
incoming.push_back(value);
}
}
if (incoming.empty()) {
return ScalarProvenance::Undefined;
}
if (incoming.size() == 1) {
return incoming[0];
}
if (*phi == ScalarProvenance::Undefined) {
*phi = AddValue({ScalarValueOp::Phi, block.start_pc});
}
m_graph.values[*phi].phi_args = std::move(incoming);
return *phi;
};
for (uint32_t reg = 0; reg < ScalarRegisters; reg++) {
const auto initial =
block_index == 0 ? m_user_data.regs[reg] : ScalarProvenance::Undefined;
result.regs[reg] = merge(
initial, [reg](uint32_t, const ScalarState& state) { return state.regs[reg]; },
&m_phi[block_index].regs[reg]);
}
for (uint32_t reg = 0; reg < VectorRegisters; reg++) {
result.address_bases[reg] = merge(
block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined,
[reg](uint32_t, const ScalarState& state) { return state.address_bases[reg]; },
&m_phi[block_index].address_bases[reg]);
}
result.scc = merge(
block_index == 0 ? m_user_data.scc : ScalarProvenance::Undefined,
[](uint32_t, const ScalarState& state) { return state.scc; }, &m_phi[block_index].scc);
result.m0 = merge(
block_index == 0 ? m_user_data.m0 : ScalarProvenance::Undefined,
[](uint32_t, const ScalarState& state) { return state.m0; }, &m_phi[block_index].m0);
std::vector<uint64_t> vector_lanes;
for (const auto predecessor: block.predecessors) {
for (const auto& [lane, value]: m_exit[predecessor].vector_lanes) {
(void)value;
vector_lanes.push_back(lane);
}
}
std::sort(vector_lanes.begin(), vector_lanes.end());
vector_lanes.erase(std::unique(vector_lanes.begin(), vector_lanes.end()),
vector_lanes.end());
for (const auto lane: vector_lanes) {
auto& phi = m_phi[block_index].vector_lanes[lane];
result.vector_lanes[lane] = merge(
block_index == 0 ? ScalarProvenance::Unknown : ScalarProvenance::Undefined,
[this, lane](uint32_t predecessor, const ScalarState& state) {
if (!m_exit_ready[predecessor]) {
return ScalarProvenance::Undefined;
}
const auto found = state.vector_lanes.find(lane);
return found != state.vector_lanes.end() ? found->second
: ScalarProvenance::Unknown;
},
&phi);
}
return result;
}
void Queue(uint32_t block) {
if (!m_queued[block]) {
m_queued[block] = true;
m_work.push_back(block);
}
}
Program* m_program;
ScalarProvenance& m_graph;
ScalarState m_user_data;
std::vector<ScalarState> m_entry;
std::vector<ScalarState> m_exit;
std::vector<ScalarState> m_phi;
std::vector<bool> m_exit_ready;
std::vector<bool> m_queued;
std::deque<uint32_t> m_work;
std::map<ValueKey, uint32_t> m_values;
std::map<DescriptorKey, uint32_t> m_descriptors;
};
} // namespace
bool BuildScalarProvenance(Program* program, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error = "invalid scalar provenance program";
}
return false;
}
if (program->resource_tracking_complete) {
if (error != nullptr) {
*error = "cannot rebuild scalar provenance after resource tracking";
}
return false;
}
if (program->user_data_count > 64) {
if (error != nullptr) {
*error = "scalar provenance user-data count exceeds 64 SGPRs";
}
return false;
}
if (program->user_data_base > ScalarRegisters ||
program->user_data_count > ScalarRegisters - program->user_data_base) {
if (error != nullptr) {
*error = "scalar provenance user-data window is out of range";
}
return false;
}
if (program->srt_patching_complete) {
if (error != nullptr) {
*error = "cannot rebuild scalar provenance after SRT patching";
}
return false;
}
program->srt = {};
program->srt_plan_complete = false;
return Builder(program).Run(error);
}
const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source) {
if (source < 2 || source - 2 >= program.provenance.descriptors.size()) {
return nullptr;
}
return &program.provenance.descriptors[source - 2];
}
bool DescriptorSourceResolved(const Program& program, uint32_t source) {
const auto* descriptor = GetDescriptorSource(program, source);
if (descriptor == nullptr) {
return false;
}
std::vector<uint8_t> visited(program.provenance.values.size());
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!ValueResolved(program.provenance, descriptor->dwords[i], &visited)) {
return false;
}
}
return true;
}
std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value) {
if (value >= provenance.values.size()) {
return "invalid";
}
const auto& node = provenance.values[value];
switch (node.op) {
case ScalarValueOp::Undefined: return "undefined";
case ScalarValueOp::Unknown: return "unknown";
case ScalarValueOp::UserData: return fmt::format("ud{}", node.imm);
case ScalarValueOp::Constant: return fmt::format("0x{:08x}", node.imm);
case ScalarValueOp::PcRelativeLow: return fmt::format("pc_lo+0x{:x}", node.imm);
case ScalarValueOp::PcRelativeHigh: return fmt::format("pc_hi+0x{:x}", node.imm);
case ScalarValueOp::ReadConst:
return fmt::format("read_const({}, {}, {}, +{})", node.args[0], node.args[1],
node.args[2], node.imm);
case ScalarValueOp::ReadConstBuffer:
return fmt::format("read_const_buffer({}, {}, {}, {}, {}, +{})", node.args[0],
node.args[1], node.args[2], node.args[3], node.args[4], node.imm);
case ScalarValueOp::Phi: return fmt::format("phi{}", value);
case ScalarValueOp::BitFieldMaskU32:
return fmt::format("bfm_u32({}, {})", node.args[0], node.args[1]);
default: return fmt::format("value{}", value);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ScalarProvenance.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_
#include "graphics/shader/recompiler/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
// Builds scalar reaching definitions and attaches descriptor sources to individual memory uses.
bool BuildScalarProvenance(Program* program, std::string* error);
uint32_t ScalarValueArgCount(ScalarValueOp op);
const DescriptorValue* GetDescriptorSource(const Program& program, uint32_t source);
bool DescriptorSourceResolved(const Program& program, uint32_t source);
std::string ScalarValueToString(const ScalarProvenance& provenance, uint32_t value);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARPROVENANCE_H_ */
src/graphics/shader/recompiler/ShaderCFG.cpp
+1539
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/ShaderCFG.h
+118
View File
@@ -0,0 +1,118 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERCFG_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERCFG_H_
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::CFG {
enum class BranchCondition {
Always,
SccZero,
SccNonZero,
VccZero,
VccNonZero,
ExecZero,
ExecNonZero,
Unknown
};
enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported };
enum class FailureKind {
None,
InvalidInput,
UnsupportedInstruction,
InvalidBranchTarget,
MissingFallthrough,
InvalidLabel,
IrreducibleControlFlow,
StructuredControlFlow
};
struct Terminator {
TerminatorKind kind = TerminatorKind::Return;
BranchCondition condition = BranchCondition::Always;
uint32_t true_block = UINT32_MAX;
uint32_t false_block = UINT32_MAX;
uint32_t merge_block = UINT32_MAX;
uint32_t continue_block = UINT32_MAX;
uint32_t indirect_pc_sgpr = UINT32_MAX;
uint32_t indirect_selector_code = UINT32_MAX;
std::vector<uint32_t> indirect_target_pcs;
std::vector<uint32_t> indirect_targets;
std::vector<uint32_t> indirect_selector_values;
std::vector<uint32_t> indirect_selector_targets;
bool loop_header = false;
};
struct BasicBlock {
uint32_t id = 0;
uint32_t start_pc = 0;
uint32_t end_pc = 0;
uint32_t inst_begin = 0;
uint32_t inst_end = 0;
std::vector<uint32_t> predecessors;
std::vector<uint32_t> successors;
std::vector<uint32_t> dominators;
std::vector<uint32_t> post_dominators;
Terminator terminator;
};
struct BackEdge {
uint32_t from = UINT32_MAX;
uint32_t to = UINT32_MAX;
bool natural = false;
};
struct NaturalLoop {
uint32_t header = UINT32_MAX;
uint32_t latch = UINT32_MAX;
uint32_t merge = UINT32_MAX;
uint32_t continue_block = UINT32_MAX;
std::vector<uint32_t> body_blocks;
std::vector<uint32_t> exit_blocks;
};
struct StronglyConnectedComponent {
std::vector<uint32_t> blocks;
std::vector<uint32_t> entry_blocks;
bool irreducible = false;
};
struct Graph {
std::vector<BasicBlock> blocks;
std::vector<BackEdge> back_edges;
std::vector<NaturalLoop> natural_loops;
std::vector<StronglyConnectedComponent> components;
std::vector<uint32_t> code_table_load_pcs;
uint32_t entry_block = UINT32_MAX;
bool irreducible = false;
bool unsupported = false;
FailureKind failure_kind = FailureKind::None;
uint32_t failure_block = UINT32_MAX;
std::string unsupported_reason;
const BasicBlock* FindBlock(uint32_t id) const;
BasicBlock* FindBlock(uint32_t id);
const BasicBlock* FindBlockByPc(uint32_t pc) const;
BasicBlock* FindBlockByPc(uint32_t pc);
bool Dominates(uint32_t dominator, uint32_t block) const;
bool PostDominates(uint32_t post_dominator, uint32_t block) const;
uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
};
bool BuildGraph(const Decoder::Program& program, Graph* graph, std::string* error);
bool Structurize(Graph* graph, std::string* error);
std::string BranchConditionToString(BranchCondition condition);
std::string FailureKindToString(FailureKind kind);
std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage,
const std::string& reason);
std::string GraphToString(const Graph& graph);
} // namespace Libs::Graphics::ShaderRecompiler::CFG
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERCFG_H_ */
src/graphics/shader/recompiler/ShaderDecoder.cpp
+1216
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/ShaderDecoder.h
+677
View File
@@ -0,0 +1,677 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_
#include "common/common.h"
#include "common/stringUtils.h"
#include <span>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::Decoder {
enum class Family {
Unknown,
SOP1,
SOP2,
SOPK,
SOPC,
SOPP,
VOP1,
VOP2,
VOP3,
VOP3P,
VOPC,
VINTRP,
SMEM,
MUBUF,
MTBUF,
FLAT,
DS,
MIMG,
EXP
};
enum class Opcode {
Unknown,
Unsupported,
SMovB32,
SMovB64,
SMovkI32,
SAbsI32,
SBrevB32,
SBcnt1I32B32,
SBcnt1I32B64,
SFf1I32B32,
SFlbitI32B64,
SBitreplicateB64B32,
SGetpcB64,
SSetpcB64,
SAndSaveexecB32,
SAndn1SaveexecB32,
SAndSaveexecB64,
SOrn2SaveexecB64,
SAndn1SaveexecB64,
SNotB32,
SNotB64,
SWqmB64,
SAddU32,
SAddcU32,
SSubbU32,
SSubU32,
SAddI32,
SSubI32,
SBitcmp0B32,
SBitcmp1B32,
SBitset0B32,
SBitset1B32,
SMinI32,
SMaxI32,
SMinU32,
SMaxU32,
SAndB32,
SAndB64,
SAndn2B32,
SAndn2B64,
SOrB32,
SOrB64,
SOrn2B32,
SOrn2B64,
SXorB32,
SXorB64,
SNandB32,
SNandB64,
SNorB32,
SNorB64,
SXnorB32,
SXnorB64,
SLshlB32,
SLshlB64,
SLshl1AddU32,
SLshl2AddU32,
SLshl3AddU32,
SLshl4AddU32,
SLshrB32,
SLshrB64,
SAshrI32,
SMulI32,
SMulHiU32,
SMulkI32,
SBfeU32,
SBfeU64,
SBfmB32,
SBfmB64,
SCselectB32,
SCselectB64,
SPackLlB32B16,
SPackLhB32B16,
SPackHhB32B16,
SCmpEqI32,
SCmpLgI32,
SCmpGtI32,
SCmpGeI32,
SCmpLtI32,
SCmpLeI32,
SCmpEqU32,
SCmpLgU32,
SCmpGtU32,
SCmpGeU32,
SCmpLtU32,
SCmpLeU32,
SCmpLgU64,
VNop,
VMovB32,
VMovreldB32,
VMovrelsB32,
VReadfirstlaneB32,
VReadlaneB32,
VWritelaneB32,
VPermlane16B32,
VPermlanex16B32,
VCubeidF32,
VCubescF32,
VCubetcF32,
VCubemaF32,
VCndmaskB32,
VDot2cF32F16,
VCvtF32I32,
VCvtF32U32,
VCvtU32F32,
VCvtI32F32,
VCvtF16F32,
VCvtF32F16,
VCvtF16U16,
VCvtU16F16,
VCvtF16I16,
VCvtI16F16,
VCvtRpiI32F32,
VCvtFlrI32F32,
VCvtOffF32I4,
VCvtF32Ubyte0,
VCvtF32Ubyte1,
VCvtF32Ubyte2,
VCvtF32Ubyte3,
VRcpF32,
VFractF32,
VTruncF32,
VCeilF32,
VRndneF32,
VFloorF32,
VExpF32,
VLogF32,
VRsqF32,
VSqrtF32,
VRcpF16,
VSqrtF16,
VRsqF16,
VLogF16,
VExpF16,
VFloorF16,
VCeilF16,
VTruncF16,
VRndneF16,
VSinF32,
VCosF32,
VNotB32,
VBfrevB32,
VFfbhU32,
VFfblB32,
VAddF32,
VSubF32,
VSubrevF32,
VMulF32,
VMacF32,
VMadmkF32,
VMadakF32,
VMinF32,
VMaxF32,
VAddF16,
VSubF16,
VSubrevF16,
VMulF16,
VFmacF16,
VFmamkF16,
VFmaakF16,
VMinI32,
VMaxI32,
VMulI32I24,
VMulU32U24,
VMulLoU32,
VMulHiU32,
VMulLoI32,
VMulHiI32,
VCvtPkrtzF16F32,
VCvtPkU8F32,
VMadF32,
VMadI32I24,
VMadU32U24,
VMadU64U32,
VFmaF32,
VFmaF16,
VPackB32F16,
VBfeU32,
VBfeI32,
VBfiB32,
VAlignbitB32,
VMin3F32,
VMin3I32,
VMin3U32,
VMin3F16,
VMax3F32,
VMax3I32,
VMax3U32,
VMax3F16,
VMed3F32,
VMed3I32,
VMed3U32,
VMed3F16,
VSadU32,
VAdd3U32,
VLshlAddU32,
VAddLshlU32,
VXadU32,
VLshlOrB32,
VAndOrB32,
VOr3B32,
VXor3B32,
VAddI32,
VSubI32,
VSubrevI32,
VBfmB32,
VLdexpF32,
VCvtPknormI16F32,
VCvtPknormU16F32,
VCvtPkU16U32,
VPkMadI16,
VPkMulLoU16,
VPkAddI16,
VPkSubI16,
VPkLshlrevB16,
VPkLshrrevB16,
VPkAshrrevI16,
VPkMaxI16,
VPkMinI16,
VPkMadU16,
VPkAddF16,
VPkMulF16,
VPkMinF16,
VPkMaxF16,
VPkFmaF16,
VPkFmacF16,
VPkAddU16,
VPkSubU16,
VPkMaxU16,
VPkMinU16,
VMadMixloF16,
VMadMixhiF16,
VAddNcU32,
VAddcU32,
VSubNcU32,
VSubrevNcU32,
VAddNcU16,
VSubNcU16,
VMaxU16,
VMaxI16,
VMinU16,
VMinI16,
VAddNcI16,
VSubNcI16,
VAndB32,
VOrB32,
VXorB32,
VXnorB32,
VLshlB32,
VLshlrevB32,
VLshrB32,
VLshrrevB32,
VAshrI32,
VAshrrevI32,
VLshlrevB16,
VLshrrevB16,
VAshrrevI16,
VBcntU32B32,
VMbcntLoU32B32,
VMbcntHiU32B32,
VMinU32,
VMaxU32,
VMaxF16,
VMinF16,
VCmpFF32,
VCmpLtF32,
VCmpEqF32,
VCmpLeF32,
VCmpGtF32,
VCmpLgF32,
VCmpGeF32,
VCmpOF32,
VCmpUF32,
VCmpNgeF32,
VCmpNlgF32,
VCmpNgtF32,
VCmpNleF32,
VCmpNeqF32,
VCmpNltF32,
VCmpTruF32,
VCmpxLtF32,
VCmpxEqF32,
VCmpxLeF32,
VCmpxGtF32,
VCmpxLgF32,
VCmpxGeF32,
VCmpxNgeF32,
VCmpxNlgF32,
VCmpxNgtF32,
VCmpxNleF32,
VCmpxNeqF32,
VCmpxNltF32,
VCmpFI32,
VCmpLtI32,
VCmpEqI32,
VCmpLeI32,
VCmpGtI32,
VCmpNeI32,
VCmpGeI32,
VCmpTI32,
VCmpClassF32,
VCmpLtI16,
VCmpEqI16,
VCmpLeI16,
VCmpGtI16,
VCmpNeI16,
VCmpGeI16,
VCmpLtF16,
VCmpEqF16,
VCmpLeF16,
VCmpGtF16,
VCmpLgF16,
VCmpGeF16,
VCmpNeqF16,
VCmpxLtF16,
VCmpxEqF16,
VCmpxLeF16,
VCmpxGtF16,
VCmpxGeF16,
VCmpxNeqF16,
VCmpxNltF16,
VCmpxLtI32,
VCmpxEqI32,
VCmpxLeI32,
VCmpxGtI32,
VCmpxNeI32,
VCmpxGeI32,
VCmpLtU16,
VCmpEqU16,
VCmpLeU16,
VCmpGtU16,
VCmpNeU16,
VCmpGeU16,
VCmpFU32,
VCmpLtU32,
VCmpEqU32,
VCmpLeU32,
VCmpGtU32,
VCmpNeU32,
VCmpGeU32,
VCmpTU32,
VCmpNeU64,
VCmpxLtU32,
VCmpxEqU32,
VCmpxLeU32,
VCmpxGtU32,
VCmpxNeU32,
VCmpxGeU32,
SLoadDword,
SLoadDwordx2,
SLoadDwordx4,
SLoadDwordx8,
SLoadDwordx16,
SBufferLoadDword,
SBufferLoadDwordx2,
SBufferLoadDwordx4,
SBufferLoadDwordx8,
SBufferLoadDwordx16,
BufferLoadFormatX,
BufferLoadFormatXy,
BufferLoadFormatXyz,
BufferLoadFormatXyzw,
BufferStoreFormatX,
BufferStoreFormatXy,
BufferStoreFormatXyz,
BufferStoreFormatXyzw,
BufferLoadUbyte,
BufferLoadSbyte,
BufferLoadUshort,
BufferLoadSshort,
BufferLoadDword,
BufferLoadDwordx2,
BufferLoadDwordx3,
BufferLoadDwordx4,
BufferStoreByte,
BufferStoreShort,
BufferStoreDword,
BufferStoreDwordx2,
BufferStoreDwordx3,
BufferStoreDwordx4,
TBufferLoadFormatX,
TBufferLoadFormatXy,
TBufferLoadFormatXyz,
TBufferLoadFormatXyzw,
TBufferStoreFormatX,
TBufferStoreFormatXy,
TBufferStoreFormatXyz,
TBufferStoreFormatXyzw,
BufferAtomicSwap,
BufferAtomicAdd,
BufferAtomicSub,
BufferAtomicSMin,
BufferAtomicUMin,
BufferAtomicSMax,
BufferAtomicUMax,
BufferAtomicAnd,
BufferAtomicOr,
BufferAtomicXor,
FlatLoadUbyte,
FlatLoadSbyte,
FlatLoadUshort,
FlatLoadSshort,
FlatLoadDword,
FlatLoadDwordx2,
FlatLoadDwordx3,
FlatLoadDwordx4,
FlatStoreByte,
FlatStoreShort,
FlatStoreDword,
FlatStoreDwordx2,
FlatStoreDwordx3,
FlatStoreDwordx4,
DsAddU32,
DsAddRtnU32,
DsSubU32,
DsSubRtnU32,
DsMinI32,
DsMinRtnI32,
DsMaxI32,
DsMaxRtnI32,
DsMinU32,
DsMinRtnU32,
DsMaxU32,
DsMaxRtnU32,
DsAndB32,
DsAndRtnB32,
DsOrB32,
DsOrRtnB32,
DsXorB32,
DsXorRtnB32,
DsWrxchgRtnB32,
DsMinF32,
DsMaxF32,
DsSwizzleB32,
DsConsume,
DsAppend,
DsReadSbyte,
DsReadUbyte,
DsReadSshort,
DsReadUshort,
DsRead2B32,
DsReadB32,
DsReadB64,
DsRead2B64,
DsRead2St64B64,
DsReadB96,
DsReadB128,
DsWriteByte,
DsWriteShort,
DsWrite2B32,
DsWrite2St64B32,
DsWrite2B64,
DsWrite2St64B64,
DsWriteB32,
DsWriteB64,
DsWriteB96,
DsWriteB128,
DsWriteAddtidB32,
DsReadAddtidB32,
ImageGetResinfo,
ImageGetLod,
ImageLoad,
ImageLoadMip,
ImageStore,
ImageStoreMip,
ImageAtomicAdd,
ImageAtomicUMin,
ImageAtomicUMax,
ImageAtomicAnd,
ImageAtomicOr,
ImageAtomicXor,
ImageSample,
ImageGather4Lz,
ImageGather4C,
ImageGather4CLz,
ImageGather4LzO,
ImageGather4CO,
ImageGather4CLzO,
ImageGather4H,
VInterpP1F32,
VInterpP2F32,
VInterpMovF32,
SNop,
SWaitcnt,
SBarrier,
SBranch,
SCbranchScc0,
SCbranchScc1,
SCbranchVccz,
SCbranchVccnz,
SCbranchExecz,
SCbranchExecnz,
SSendmsg,
SSetregB32,
SSleep,
STtraceData,
SInstPrefetch,
SEndpgm,
Exp
};
enum class OperandKind {
Unknown,
LiteralConstant,
IntegerInlineConstant,
FloatInlineConstant,
Sgpr,
VccLo,
VccHi,
VccZ,
ExecLo,
ExecHi,
ExecZ,
Scc,
M0,
PopsExitingWaveId,
Null,
Vgpr,
};
enum ImageSampleFlag : uint32_t {
ImageSampleFlagLod = 1u << 0u,
ImageSampleFlagBias = 1u << 1u,
ImageSampleFlagDerivative = 1u << 2u,
ImageSampleFlagCompare = 1u << 3u,
ImageSampleFlagOffset = 1u << 4u,
ImageSampleFlagLevelZero = 1u << 5u,
ImageSampleFlagLodClamp = 1u << 6u,
ImageSampleFlagA16 = 1u << 7u,
ImageSampleFlagCd = 1u << 8u,
ImageSampleFlagGatherHorizontal = 1u << 9u,
};
enum class ImageDimension : uint32_t {
Unknown,
Dim2D,
Dim3D,
Dim2DArray,
};
constexpr uint32_t MaxInstructionRawWords = 5u;
constexpr uint32_t MaxImageNsaAddressComponents = 12u;
struct Operand {
OperandKind kind = OperandKind::Unknown;
uint32_t value = 0;
int32_t signed_val = 0;
float float_val = 0.0f;
uint32_t reg = 0;
uint32_t sdwa_sel = 6;
uint32_t sdwa_dst_unused = 0;
uint32_t omod = 0;
uint32_t dpp_ctrl = 0;
uint32_t dpp_row_mask = 0xf;
uint32_t dpp_bank_mask = 0xf;
bool sdwa_sext = false;
bool dpp_fetch_inactive = false;
bool dpp_bound_ctrl = false;
bool op_sel = false;
bool op_sel_hi = false;
bool negate = false;
bool negate_hi = false;
bool absolute = false;
bool clamp = false;
bool dpp = false;
};
struct Instruction {
uint32_t pc = 0;
uint32_t word = 0;
uint32_t word_count = 1;
uint32_t raw[MaxInstructionRawWords] = {};
uint32_t raw_count = 1;
Family family = Family::Unknown;
uint32_t opcode_id = 0;
Opcode opcode = Opcode::Unknown;
Operand dst;
Operand dst2;
Operand src0;
Operand src1;
Operand src2;
Operand src3;
uint32_t src_count = 0;
uint32_t offset = 0;
uint32_t secondary_offset = 0;
uint32_t dmask = 0;
uint32_t data_dwords = 1;
uint32_t data_bits = 32;
uint32_t data_format = 0;
uint32_t number_format = 0;
uint32_t image_sample_flags = 0;
ImageDimension image_dimension = ImageDimension::Unknown;
uint32_t image_address_components = 0;
uint32_t image_nsa_dwords = 0;
uint32_t image_nsa_addr[MaxImageNsaAddressComponents] = {};
uint32_t memory_segment = 0;
bool data_signed = false;
bool typed = false;
bool formatted = false;
bool gds = false;
bool glc = false;
bool slc = false;
bool idxen = false;
bool offen = false;
int32_t branch_offset = 0;
uint32_t branch_target = 0;
struct {
uint32_t target = 0;
uint32_t en = 0;
bool done = false;
bool compr = false;
bool vm = false;
} exp;
std::string unsupported_reason;
};
struct Program {
std::span<const uint32_t> code;
std::vector<Instruction> instructions;
};
bool DecodeProgram(std::span<const uint32_t> code, Program* program, std::string* error);
bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand* operand, std::string* error);
bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand* operand, std::string* error);
bool DecodeVectorGpr(uint32_t reg, Operand* operand, std::string* error);
bool ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
void SetRawWords(Instruction* inst, std::span<const uint32_t> code, uint32_t word_index,
uint32_t word_count);
void SetUnsupported(Instruction* inst, Family family, uint32_t opcode_id, const char* reason);
std::string FamilyToString(Family family);
std::string OpcodeToString(Opcode opcode);
std::string OperandToString(const Operand& operand);
std::string InstructionToString(const Instruction& inst);
std::string ProgramToString(const Program& program);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_ */
src/graphics/shader/recompiler/ShaderIR.h
+763
View File
@@ -0,0 +1,763 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ShaderCFG.h"
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/shader.h"
#include <array>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
enum class Opcode {
ControlNop,
Waitcnt,
Barrier,
Sendmsg,
TtraceData,
InstPrefetch,
MoveU32,
MoveF32Bits,
MoveRelDestU32,
MoveRelSourceU32,
MoveU64,
WqmB64,
SaveexecB32,
SaveexecB64,
ReadFirstLaneU32,
ReadLaneU32,
WriteLaneU32,
Permlane16B32,
Permlanex16B32,
AbsI32,
IAddU32,
IAddCarryU32,
ISubBorrowU32,
ScalarAddCarryU32,
ScalarSubBorrowU32,
ScalarSubBorrowCarryU32,
ScalarSignedAddOverflowI32,
ScalarSignedSubOverflowI32,
ScalarShiftLeftAddCarryU32,
ISubU32,
IMulU32,
UMulHighU32,
SMulHighI32,
IMadI24U32,
UMadU24U32,
UMadU64U32,
SadU32,
IAdd3U32,
IMulI24U32,
UMulU24U32,
IMinI32,
IMaxI32,
IMin3I32,
IMax3I32,
IMed3I32,
UMinU32,
UMaxU32,
UMin3U32,
UMax3U32,
UMed3U32,
BitwiseAndU32,
BitwiseAndU64,
BitwiseAndNotU32,
BitwiseAndNotU64,
BitwiseOrU32,
BitwiseOrU64,
BitwiseOrNotU32,
BitwiseOrNotU64,
BitwiseAndOrU32,
BitwiseOr3U32,
BitwiseXorU32,
BitwiseXorU64,
BitwiseXor3U32,
BitwiseNandU32,
BitwiseNandU64,
BitwiseNorU32,
BitwiseNorU64,
BitwiseXnorU32,
BitwiseXnorU64,
BitwiseNotU32,
BitwiseNotU64,
BitClearU32,
BitSetU32,
BitReverseU32,
BitCountU32,
BitCountU64,
BitReplicateB64B32,
BitCountAddU32,
MaskedBitCountLowU32,
MaskedBitCountHighU32,
FindLsbU32,
FindMsbFromHighU32,
FindMsbFromHighU64,
BitFieldMaskU32,
BitFieldMaskU64,
BitFieldExtractU32,
BitFieldExtractU64,
BitFieldExtract3U32,
BitFieldExtract3I32,
BitFieldInsertSelectU32,
BitCompare0B32,
BitCompare1B32,
AlignBitU32,
ShiftLeftAddU32,
AddShiftLeftU32,
XorAddU32,
ShiftLeftOrU32,
ShiftLeftLogicalU32,
ShiftLeftLogicalU64,
ShiftLeftLogicalU16,
ShiftRightLogicalU32,
ShiftRightLogicalU64,
ShiftRightLogicalU16,
ShiftRightArithmeticI32,
ShiftRightArithmeticI16,
SelectU32,
SelectMaskU32,
SelectF32Bits,
SelectMaskF32Bits,
SelectU64,
PackLowLowU16,
PackLowHighU16,
PackHighHighU16,
CompareFalse,
CompareTrue,
CompareEqU32,
CompareNeU32,
CompareGtU32,
CompareGeU32,
CompareLtU32,
CompareLeU32,
CompareNeU64,
CompareMaskEqU32,
CompareMaskNeU32,
CompareMaskGtU32,
CompareMaskGeU32,
CompareMaskLtU32,
CompareMaskLeU32,
CompareEqI32,
CompareNeI32,
CompareGtI32,
CompareGeI32,
CompareLtI32,
CompareLeI32,
CompareEqI16,
CompareNeI16,
CompareGtI16,
CompareGeI16,
CompareLtI16,
CompareLeI16,
CompareMaskEqI32,
CompareMaskNeI32,
CompareMaskGtI32,
CompareMaskGeI32,
CompareMaskLtI32,
CompareMaskLeI32,
CompareEqU16,
CompareNeU16,
CompareGtU16,
CompareGeU16,
CompareLtU16,
CompareLeU16,
CompareEqF32,
CompareNeF32,
CompareGtF32,
CompareGeF32,
CompareLtF32,
CompareLeF32,
CompareOrderedF32,
CompareUnorderedF32,
CompareUnordEqF32,
CompareUnordNeF32,
CompareUnordGtF32,
CompareUnordGeF32,
CompareUnordLtF32,
CompareUnordLeF32,
CompareClassF32,
CompareEqF16,
CompareNeF16,
CompareGtF16,
CompareGeF16,
CompareLtF16,
CompareLeF16,
CompareUnordNeF16,
CompareMaskEqF16,
CompareMaskNeF16,
CompareMaskGtF16,
CompareMaskGeF16,
CompareMaskLtF16,
CompareMaskLeF16,
CompareMaskUnordNeF16,
CompareMaskUnordGeF16,
CompareMaskEqF32,
CompareMaskNeF32,
CompareMaskGtF32,
CompareMaskGeF32,
CompareMaskLtF32,
CompareMaskLeF32,
CompareMaskUnordEqF32,
CompareMaskUnordNeF32,
CompareMaskUnordGtF32,
CompareMaskUnordGeF32,
CompareMaskUnordLtF32,
CompareMaskUnordLeF32,
ConvertByteU32ToF32,
ConvertU32ToF32,
ConvertI32ToF32,
ConvertF32ToU32,
ConvertF32ToI32,
ConvertF32ToF16,
ConvertF16ToF32,
ConvertU16ToF16,
ConvertF16ToU16,
ConvertI16ToF16,
ConvertF16ToI16,
ConvertRoundPlusInfF32ToI32,
ConvertFloorF32ToI32,
ConvertI4ToOffsetF32,
LdexpF32,
PackF32ToF16Rtz,
PackSnorm2x16F32,
PackUnorm2x16F32,
PackU16U32,
PackU8F32,
PackB32F16,
PackedMadI16,
PackedMulLoU16,
PackedAddI16,
PackedSubI16,
PackedLshlrevB16,
PackedLshrrevB16,
PackedAshrrevI16,
PackedMaxI16,
PackedMinI16,
PackedMadU16,
PackedAddU16,
PackedSubU16,
PackedMaxU16,
PackedMinU16,
PackedAddF16,
PackedMulF16,
PackedMinF16,
PackedMaxF16,
PackedFmaF16,
AddF16,
SubF16,
MulF16,
MinF16,
MaxF16,
FmaF16,
MadMixF16,
IAddU16,
ISubI16,
IMinI16,
IMaxI16,
UMinU16,
UMaxU16,
RcpF32,
FractF32,
TruncF32,
CeilF32,
RoundEvenF32,
FloorF32,
Exp2F32,
Log2F32,
InverseSqrtF32,
SqrtF32,
RcpF16,
SqrtF16,
InverseSqrtF16,
Log2F16,
Exp2F16,
FloorF16,
CeilF16,
TruncF16,
RoundEvenF16,
SinF32,
CosF32,
CubeIdF32,
CubeScF32,
CubeTcF32,
CubeMaF32,
FAddF32,
FSubF32,
FMulF32,
FMinF32,
FMaxF32,
FMadF32,
Dot2AccF32F16,
FMin3F32,
FMax3F32,
FMed3F32,
Min3F16,
Max3F16,
Med3F16,
LoadSrtDword,
SLoadDword,
SBufferLoadDword,
BufferLoadUbyte,
BufferLoadSbyte,
BufferLoadUshort,
BufferLoadSshort,
BufferLoadDword,
BufferStoreByte,
BufferStoreShort,
BufferStoreDword,
AtomicSwapU32,
AtomicAddU32,
AtomicSubU32,
AtomicSMinI32,
AtomicUMinU32,
AtomicSMaxI32,
AtomicUMaxU32,
AtomicAndU32,
AtomicOrU32,
AtomicXorU32,
FlatLoadUbyte,
FlatLoadSbyte,
FlatLoadUshort,
FlatLoadSshort,
FlatLoadDword,
FlatStoreByte,
FlatStoreShort,
FlatStoreDword,
DsReadUbyte,
DsReadSbyte,
DsReadUshort,
DsReadSshort,
DsReadB32,
DsWriteByte,
DsWriteShort,
DsWriteB32,
DsMinF32,
DsMaxF32,
DsSwizzleB32,
DsConsume,
DsAppend,
DsWriteAddtidB32,
DsReadAddtidB32,
ImageGetResinfo,
ImageGetLod,
ImageLoad,
ImageStore,
ImageSample,
ImageGather4,
LoadInputF32,
Export,
};
enum class RegisterFile { Scalar, Vector, Vcc, Exec, Scc, M0 };
struct Register {
RegisterFile file = RegisterFile::Scalar;
uint32_t index = 0;
bool operator==(const Register& other) const {
return file == other.file && index == other.index;
}
};
enum class OperandKind { Register, ImmediateU32, PcRelativeU32, Null };
struct Operand {
OperandKind kind = OperandKind::Null;
Register reg;
uint32_t imm = 0;
bool sext_64 = false;
uint32_t sdwa_sel = 6;
uint32_t omod = 0;
bool sdwa_sext = false;
bool op_sel = false;
bool op_sel_hi = false;
bool negate = false;
bool negate_hi = false;
bool absolute = false;
bool clamp = false;
uint32_t dpp_ctrl = 0;
uint32_t dpp_row_mask = 0xf;
uint32_t dpp_bank_mask = 0xf;
bool dpp_fetch_inactive = false;
bool dpp_bound_ctrl = false;
bool dpp = false;
bool operator==(const Operand& other) const = default;
};
enum class ResourceKind {
None,
ScalarBuffer,
Buffer,
Flat,
Global,
Scratch,
Lds,
Gds,
Image,
ImageUint,
StorageImage,
StorageImageUint,
Sampler
};
struct MemoryInfo {
ResourceKind kind = ResourceKind::None;
uint32_t resource = 0;
uint32_t sampler = 0;
uint32_t offset = 0;
uint32_t secondary_offset = 0;
uint32_t dmask = 0;
uint32_t data_dwords = 1;
uint32_t data_bits = 32;
uint32_t component_index = 0;
uint32_t component_count = 1;
uint32_t data_format = 0;
uint32_t number_format = 0;
uint32_t image_sample_flags = 0;
Decoder::ImageDimension image_dimension = Decoder::ImageDimension::Unknown;
uint32_t image_address_components = 0;
uint32_t image_nsa_dwords = 0;
uint32_t image_nsa_addr[Decoder::MaxImageNsaAddressComponents] = {};
uint32_t memory_segment = 0;
// Dense resource indices are assigned by resource tracking. These source IDs refer to the
// exact scalar definitions reaching this instruction before that patching step.
uint32_t resource_source = 0;
uint32_t sampler_source = 0;
bool data_signed = false;
bool typed = false;
bool formatted = false;
bool image_has_mip = false;
bool glc = false;
bool slc = false;
bool idxen = false;
bool offen = false;
bool operator==(const MemoryInfo& other) const = default;
};
enum class ExportTargetKind { Unknown, Null, Position, Primitive, Parameter, Mrt, MrtZ };
struct ExportInfo {
ExportTargetKind kind = ExportTargetKind::Unknown;
uint32_t target = 0;
uint32_t index = 0;
uint32_t en = 0;
bool done = false;
bool compr = false;
bool vm = false;
bool operator==(const ExportInfo& other) const = default;
};
struct InputInfo {
uint32_t attr = 0;
uint32_t chan = 0;
bool operator==(const InputInfo& other) const = default;
};
enum class SaveexecMode { And, Orn2, Andn1 };
struct Instruction {
uint32_t pc = 0;
Opcode op = Opcode::MoveU32;
Operand dst;
Operand dst2;
Operand src[4];
uint32_t src_count = 0;
uint32_t scalar_sources[4] = {};
uint32_t scalar_value = 0;
MemoryInfo memory;
ExportInfo export_info;
InputInfo input_info;
SaveexecMode saveexec_mode = SaveexecMode::And;
bool operator==(const Instruction& other) const = default;
};
struct BasicBlock {
uint32_t id = 0;
uint32_t start_pc = 0;
uint32_t end_pc = 0;
uint32_t inst_begin = 0;
uint32_t inst_end = 0;
std::vector<uint32_t> predecessors;
std::vector<uint32_t> successors;
CFG::Terminator terminator;
std::vector<Instruction> instructions;
};
enum class ScalarValueOp {
Undefined,
Unknown,
UserData,
Constant,
PcRelativeLow,
PcRelativeHigh,
Add,
AddCarry,
Carry,
Sub,
SubBorrow,
Borrow,
Mul,
And,
AndNot,
Or,
OrNot,
Xor,
Not,
ShiftLeft,
ShiftRight,
ShiftRightArithmetic,
BitFieldMaskU32,
BitFieldMaskU64Low,
BitFieldMaskU64High,
Add3,
ShiftLeftAdd,
ShiftLeftAddCarry,
AddShiftLeft,
XorAdd,
ShiftLeftOr,
ReadConst,
ReadConstBuffer,
Phi,
};
struct ScalarValue {
ScalarValueOp op = ScalarValueOp::Undefined;
uint32_t pc = 0;
uint32_t imm = 0;
std::array<uint32_t, 6> args = {};
std::vector<uint32_t> phi_args;
bool operator==(const ScalarValue& other) const = default;
};
struct DescriptorValue {
std::array<uint32_t, 8> dwords = {};
uint32_t dword_count = 0;
bool operator==(const DescriptorValue& other) const {
return dword_count == other.dword_count && dwords == other.dwords;
}
};
struct ScalarProvenance {
static constexpr uint32_t Undefined = 0;
static constexpr uint32_t Unknown = 1;
std::vector<ScalarValue> values;
std::vector<DescriptorValue> descriptors;
bool operator==(const ScalarProvenance& other) const = default;
};
struct SrtRead {
uint32_t value = 0;
uint32_t flat_offset = 0;
uint32_t use_pc = 0;
bool operator==(const SrtRead& other) const = default;
};
struct SrtPlan {
std::vector<SrtRead> reads;
std::vector<uint32_t> dynamic_reads;
std::vector<uint32_t> dynamic_sources;
bool operator==(const SrtPlan& other) const = default;
};
struct BufferResource {
uint32_t source = 0;
uint32_t first_use_pc = 0;
uint32_t max_byte_extent = 0;
uint32_t packed_stride = 0;
uint32_t descriptor_format = 0;
bool read = false;
bool written = false;
bool atomic = false;
bool formatted = false;
bool scalar = false;
bool operator==(const BufferResource& other) const = default;
};
enum class ImageMipMode { None, DynamicStorage };
constexpr uint32_t StorageImageIdentitySwizzle = 0x00000facu;
struct ImageResource {
uint32_t source = 0;
uint32_t first_use_pc = 0;
ResourceKind kind = ResourceKind::None;
Decoder::ImageDimension dimension = Decoder::ImageDimension::Unknown;
ImageMipMode mip_mode = ImageMipMode::None;
uint32_t storage_swizzle = StorageImageIdentitySwizzle;
bool read = false;
bool written = false;
bool atomic = false;
bool depth_compare = false;
bool operator==(const ImageResource& other) const = default;
};
struct SamplerResource {
uint32_t source = 0;
uint32_t first_use_pc = 0;
bool operator==(const SamplerResource& other) const = default;
};
struct SampledResourcePair {
uint32_t image = 0;
uint32_t sampler = 0;
uint32_t first_use_pc = 0;
bool operator==(const SampledResourcePair& other) const = default;
};
struct AddressResource {
uint32_t source = 0;
uint32_t first_use_pc = 0;
ResourceKind kind = ResourceKind::Flat;
int32_t min_offset = 0;
uint64_t specialized_base = 0;
bool read = false;
bool written = false;
bool atomic = false;
bool operator==(const AddressResource& other) const = default;
};
enum class StageInputKind {
VertexIndex,
InstanceIndex,
FragCoord,
FrontFacing,
WorkgroupId,
LocalInvocationId,
LocalInvocationIndex,
GlobalInvocationId,
Parameter,
};
enum class StageOutputKind { Position, Parameter, Mrt, Depth, SampleMask };
struct StageInput {
StageInputKind kind = StageInputKind::VertexIndex;
uint32_t location = 0;
uint32_t component_count = 1;
std::string debug_name;
bool operator==(const StageInput& other) const = default;
};
struct StageOutput {
StageOutputKind kind = StageOutputKind::Parameter;
uint32_t index = 0;
uint32_t location = 0;
std::string debug_name;
bool operator==(const StageOutput& other) const = default;
};
enum class DescriptorBindingKind {
Buffers,
Sampled2D,
Sampled2DArray,
Sampled3D,
SampledUint2D,
SampledUint2DArray,
SampledUint3D,
Storage2D,
Storage2DArray,
Storage3D,
StorageUint2D,
StorageUint2DArray,
StorageUint3D,
Samplers,
Gds,
AddressMemory,
FlattenedSrt,
UserData,
Count,
};
struct DescriptorBinding {
DescriptorBindingKind kind = DescriptorBindingKind::Buffers;
uint32_t binding = 0;
std::vector<uint32_t> resources;
bool operator==(const DescriptorBinding& other) const = default;
};
struct BindingLayout {
uint32_t descriptor_set = 0;
uint32_t push_constant_offset = 0;
uint32_t push_constant_size = 0;
std::vector<uint32_t> user_data_registers;
std::vector<DescriptorBinding> descriptors;
bool operator==(const BindingLayout& other) const = default;
};
struct ShaderInfo {
static constexpr uint32_t MaxBuffers = 32;
static constexpr uint32_t MaxAddresses = 32;
static constexpr uint32_t MaxImages = 32;
static constexpr uint32_t MaxSamplers = 32;
static constexpr uint32_t MaxSampledPairs = 64;
std::vector<BufferResource> buffers;
std::vector<AddressResource> addresses;
std::vector<ImageResource> images;
std::vector<SamplerResource> samplers;
std::vector<SampledResourcePair> sampled_pairs;
std::vector<StageInput> inputs;
std::vector<StageOutput> outputs;
int32_t vertex_offset_sgpr = -1;
bool operator==(const ShaderInfo& other) const = default;
};
struct Program {
ShaderType stage = ShaderType::Unknown;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint64_t shader_hash = 0;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
bool dispatcher_fallback = false;
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
std::string fallback_reason;
std::vector<BasicBlock> blocks;
ScalarProvenance provenance;
SrtPlan srt;
bool srt_plan_complete = false;
bool srt_patching_complete = false;
ShaderInfo info;
bool resource_tracking_complete = false;
bool shader_info_complete = false;
BindingLayout bindings;
bool binding_layout_complete = false;
};
bool LowerProgram(const Decoder::Program& decoded, const CFG::Graph& cfg, ShaderType stage,
uint32_t wave_size, Program* program, std::string* error);
std::string RegisterToString(Register reg);
std::string OperandToString(const Operand& operand);
std::string ExportTargetKindToString(ExportTargetKind kind);
std::string InstructionToString(const Instruction& inst);
std::string ProgramToString(const Program& program);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERIR_H_ */
src/graphics/shader/recompiler/ShaderInfoCollection.cpp
+225
View File
@@ -0,0 +1,225 @@
#include "graphics/shader/recompiler/ShaderInfoCollection.h"
#include <algorithm>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
bool HasInput(const ShaderInfo& info, StageInputKind kind, uint32_t location) {
return std::any_of(info.inputs.begin(), info.inputs.end(), [=](const auto& input) {
return input.kind == kind && input.location == location;
});
}
void AddInput(ShaderInfo* info, StageInputKind kind, uint32_t location, uint32_t components,
std::string name) {
if (!HasInput(*info, kind, location)) {
info->inputs.push_back({kind, location, components, std::move(name)});
}
}
bool HasOutput(const ShaderInfo& info, StageOutputKind kind, uint32_t index) {
return std::any_of(info.outputs.begin(), info.outputs.end(), [=](const auto& output) {
return output.kind == kind && output.index == index;
});
}
void AddOutput(ShaderInfo* info, StageOutputKind kind, uint32_t index, uint32_t location,
std::string name) {
if (!HasOutput(*info, kind, index)) {
info->outputs.push_back({kind, index, location, std::move(name)});
}
}
bool NeedsLocalInvocationIndex(const Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::DsWriteAddtidB32 || inst.op == Opcode::DsReadAddtidB32) {
return true;
}
}
}
return false;
}
bool ValidateOptions(const Program& program, const ShaderInfoOptions& options, std::string* error) {
auto Fail = [&](const char* message) {
if (error != nullptr) {
*error = message;
}
return false;
};
switch (program.stage) {
case ShaderType::Vertex:
if (options.vertex == nullptr) {
return Fail("vertex shader info requires vertex metadata");
}
if (options.vertex->resources_num < 0 ||
options.vertex->resources_num > ShaderVertexInputInfo::RES_MAX) {
return Fail("vertex resource count is out of range");
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::LoadInputF32 &&
(inst.input_info.chan >= 4 ||
inst.input_info.attr >=
static_cast<uint32_t>(options.vertex->resources_num))) {
return Fail("vertex input reference is out of range");
}
}
}
return true;
case ShaderType::Pixel:
if (options.pixel == nullptr) {
return Fail("pixel shader info requires pixel metadata");
}
return options.pixel->input_num <= std::size(options.pixel->interpolator_settings) ||
Fail("pixel input count is out of range");
case ShaderType::Compute:
if (options.compute == nullptr) {
return Fail("compute shader info requires compute metadata");
}
return (options.compute->thread_ids_num >= 0 && options.compute->thread_ids_num <= 3) ||
Fail("compute thread ID count is out of range");
default: return Fail("unsupported shader stage for info collection");
}
}
void CollectVertexInputs(const Program& program, const ShaderVertexInputInfo* vertex,
ShaderInfo* info) {
AddInput(info, StageInputKind::VertexIndex, 0, 1, "gl_VertexIndex");
AddInput(info, StageInputKind::InstanceIndex, 0, 1, "gl_InstanceIndex");
if (vertex == nullptr) {
return;
}
uint32_t used_components[ShaderVertexInputInfo::RES_MAX] = {};
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::LoadInputF32 &&
inst.input_info.attr < static_cast<uint32_t>(vertex->resources_num) &&
inst.input_info.attr < ShaderVertexInputInfo::RES_MAX) {
used_components[inst.input_info.attr] =
std::max(used_components[inst.input_info.attr], inst.input_info.chan + 1u);
}
}
}
for (uint32_t attr = 0; attr < static_cast<uint32_t>(vertex->resources_num) &&
attr < ShaderVertexInputInfo::RES_MAX;
attr++) {
if (used_components[attr] != 0) {
AddInput(info, StageInputKind::Parameter, attr, used_components[attr],
fmt::format("in_attr_{}", attr));
}
}
}
void CollectPixelInputs(const ShaderPixelInputInfo* pixel, ShaderInfo* info) {
if (pixel == nullptr) {
return;
}
if (pixel->HasPositionInput()) {
AddInput(info, StageInputKind::FragCoord, 0, 4, "gl_FragCoord");
}
if (pixel->ps_front_face) {
AddInput(info, StageInputKind::FrontFacing, 0, 1, "gl_FrontFacing");
}
for (uint32_t input = 0; input < pixel->input_num; input++) {
AddInput(info, StageInputKind::Parameter, input, 4, fmt::format("in_param_{}", input));
}
}
void CollectComputeInputs(const Program& program, const ShaderComputeInputInfo* compute,
ShaderInfo* info) {
if (compute != nullptr) {
if (compute->group_id[0] || compute->group_id[1] || compute->group_id[2]) {
AddInput(info, StageInputKind::WorkgroupId, 0, 3, "gl_WorkGroupID");
}
if (compute->thread_ids_num > 0) {
AddInput(info, StageInputKind::LocalInvocationId, 0, 3, "gl_LocalInvocationID");
}
if (compute->thread_ids_num > 0 || compute->tg_size_en) {
AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex");
}
if (compute->dispatch_thread_dimensions) {
AddInput(info, StageInputKind::GlobalInvocationId, 0, 3, "gl_GlobalInvocationID");
}
}
if (NeedsLocalInvocationIndex(program)) {
AddInput(info, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex");
}
}
void CollectOutputs(const Program& program, const ShaderPixelInputInfo* pixel, ShaderInfo* info) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != Opcode::Export) {
continue;
}
if (inst.export_info.kind == ExportTargetKind::MrtZ) {
if (pixel != nullptr && (inst.export_info.en & 0x1u) != 0 &&
pixel->ps_depth_export_enable) {
AddOutput(info, StageOutputKind::Depth, 0, 0, "gl_FragDepth");
}
if (pixel != nullptr && (inst.export_info.en & 0x4u) != 0 &&
pixel->ps_sample_mask_export_enable) {
AddOutput(info, StageOutputKind::SampleMask, 0, 0, "gl_SampleMask");
}
continue;
}
if (inst.export_info.en == 0) {
continue;
}
switch (inst.export_info.kind) {
case ExportTargetKind::Position:
AddOutput(info, StageOutputKind::Position, inst.export_info.index, 0,
"out_position");
break;
case ExportTargetKind::Parameter:
AddOutput(info, StageOutputKind::Parameter, inst.export_info.index,
inst.export_info.index,
fmt::format("out_param_{}", inst.export_info.index));
break;
case ExportTargetKind::Mrt:
AddOutput(info, StageOutputKind::Mrt, inst.export_info.index,
inst.export_info.index,
fmt::format("out_mrt_{}", inst.export_info.index));
break;
default: break;
}
}
}
}
} // namespace
bool CollectShaderInfo(Program* program, const ShaderInfoOptions& options, std::string* error) {
if (program == nullptr || !program->resource_tracking_complete ||
program->shader_info_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid shader info program"
: !program->resource_tracking_complete ? "shader resources were not tracked"
: "shader info already collected";
}
return false;
}
if (!ValidateOptions(*program, options, error)) {
return false;
}
auto next = program->info;
next.inputs.clear();
next.outputs.clear();
switch (program->stage) {
case ShaderType::Vertex: CollectVertexInputs(*program, options.vertex, &next); break;
case ShaderType::Pixel: CollectPixelInputs(options.pixel, &next); break;
case ShaderType::Compute: CollectComputeInputs(*program, options.compute, &next); break;
default: return false;
}
CollectOutputs(*program, options.pixel, &next);
program->info = std::move(next);
program->shader_info_complete = true;
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/ShaderInfoCollection.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_
#include "graphics/shader/recompiler/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
struct ShaderInfoOptions {
const ShaderVertexInputInfo* vertex = nullptr;
const ShaderPixelInputInfo* pixel = nullptr;
const ShaderComputeInputInfo* compute = nullptr;
};
// Completes the immutable shader interface after resource tracking. On failure Program::info and
// all completion state remain unchanged.
bool CollectShaderInfo(Program* program, const ShaderInfoOptions& options, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERINFOCOLLECTION_H_ */
src/graphics/shader/recompiler/ShaderRecompiler.cpp
+872
View File
@@ -0,0 +1,872 @@
#include "graphics/shader/recompiler/ShaderRecompiler.h"
#include "common/assert.h"
#include "common/logging/log.h"
#include "graphics/shader/recompiler/BindingLayout.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/recompiler/ResourceTracking.h"
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include "graphics/shader/recompiler/ShaderCFG.h"
#include "graphics/shader/recompiler/ShaderDecoder.h"
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/ShaderInfoCollection.h"
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/SrtPatcher.h"
#include "graphics/shader/recompiler/SrtWalker.h"
#include <algorithm>
#include <array>
#include <chrono>
#include <fmt/format.h>
#include <map>
#include <span>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler {
namespace {
bool ReadZeroMemory(void*, uint64_t, uint32_t* value) {
if (value == nullptr) {
return false;
}
*value = 0;
return true;
}
const char* GetDumpLabel(const CompileOptions& options) {
return options.dump_label != nullptr ? options.dump_label : "ShaderRecompiler";
}
std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
std::string dump = "CFG:\n";
dump += CFG::GraphToString(cfg);
dump += "\nIR:\n";
dump += IR::ProgramToString(ir);
return dump;
}
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Compute: return "CS";
case ShaderType::Vertex: return "VS";
case ShaderType::Pixel: return "PS";
default: return "unknown";
}
}
std::string FormatCfgFailure(const CFG::Graph& cfg, const CompileOptions& options,
const std::string& reason) {
const auto block_id = cfg.failure_block != UINT32_MAX ? cfg.failure_block : cfg.entry_block;
return CFG::FormatBlockDiagnostic(cfg, block_id, StageName(options.stage), reason);
}
bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::SLoadDword:
case IR::Opcode::SBufferLoadDword:
case IR::Opcode::BufferLoadUbyte:
case IR::Opcode::BufferLoadSbyte:
case IR::Opcode::BufferLoadUshort:
case IR::Opcode::BufferLoadSshort:
case IR::Opcode::BufferLoadDword:
case IR::Opcode::BufferStoreByte:
case IR::Opcode::BufferStoreShort:
case IR::Opcode::BufferStoreDword:
case IR::Opcode::AtomicSwapU32:
case IR::Opcode::AtomicAddU32:
case IR::Opcode::AtomicSubU32:
case IR::Opcode::AtomicSMinI32:
case IR::Opcode::AtomicUMinU32:
case IR::Opcode::AtomicSMaxI32:
case IR::Opcode::AtomicUMaxU32:
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32:
case IR::Opcode::FlatLoadUbyte:
case IR::Opcode::FlatLoadSbyte:
case IR::Opcode::FlatLoadUshort:
case IR::Opcode::FlatLoadSshort:
case IR::Opcode::FlatLoadDword:
case IR::Opcode::FlatStoreByte:
case IR::Opcode::FlatStoreShort:
case IR::Opcode::FlatStoreDword:
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32:
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32:
case IR::Opcode::DsWriteAddtidB32:
case IR::Opcode::DsAppend:
case IR::Opcode::DsConsume:
case IR::Opcode::ImageStore:
case IR::Opcode::Export: return true;
default: return false;
}
}
bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* reason) {
for (const auto& block: ir.blocks) {
if (!block.terminator.loop_header) {
continue;
}
for (const auto& inst: block.instructions) {
if (!InstructionMaySplitSpirvBlock(inst)) {
continue;
}
if (reason != nullptr) {
*reason = fmt::format("loop header block {} contains an instruction whose SPIR-V "
"lowering emits internal control flow: {}",
block.id, IR::InstructionToString(inst).c_str());
}
return true;
}
}
return false;
}
enum class EmbeddedFetchValueType {
Unknown,
Constant,
AttribTable,
Attrib,
BufferTable,
Buffer,
Index
};
struct EmbeddedFetchSgprInfo {
EmbeddedFetchValueType type = EmbeddedFetchValueType::Unknown;
int attrib_id = 0;
uint32_t value = 0;
std::vector<uint32_t> prolog_loads;
};
struct EmbeddedFetchVgprInfo {
EmbeddedFetchValueType type = EmbeddedFetchValueType::Unknown;
};
using EmbeddedFetchVectorLanes = std::map<uint64_t, EmbeddedFetchSgprInfo>;
uint64_t EmbeddedFetchVectorLaneKey(uint32_t reg, uint32_t lane) {
return (static_cast<uint64_t>(reg) << 32u) | lane;
}
uint32_t EmbeddedFetchLane(uint32_t lane, uint32_t wave_size) {
return wave_size == 32 || wave_size == 64 ? lane % wave_size : lane;
}
void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg) {
const auto first = lanes->lower_bound(EmbeddedFetchVectorLaneKey(reg, 0));
const auto last = lanes->lower_bound(EmbeddedFetchVectorLaneKey(reg + 1u, 0));
lanes->erase(first, last);
}
struct EmbeddedFetchLoad {
uint32_t pc = 0;
int attrib_id = -1;
uint32_t components = 0;
std::vector<uint32_t> prolog_loads;
};
struct EmbeddedFetchData {
std::vector<EmbeddedFetchLoad> loads;
int32_t vertex_offset_sgpr = -1;
};
bool IsDecodedSgpr(const Decoder::Operand& op) {
return op.kind == Decoder::OperandKind::Sgpr || op.kind == Decoder::OperandKind::VccLo ||
op.kind == Decoder::OperandKind::VccHi;
}
uint32_t DecodedSgprReg(const Decoder::Operand& op) {
return op.kind == Decoder::OperandKind::VccLo
? 106u
: (op.kind == Decoder::OperandKind::VccHi ? 107u : op.reg);
}
bool IsDecodedVgpr(const Decoder::Operand& op) {
return op.kind == Decoder::OperandKind::Vgpr;
}
uint32_t DecodedDstSize(const Decoder::Instruction& inst) {
return std::max(inst.data_dwords, 1u);
}
uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) {
return inst.opcode == Decoder::Opcode::VMadU64U32 ? 2u : DecodedDstSize(inst);
}
bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::SSetpcB64:
case Decoder::Opcode::SBranch:
case Decoder::Opcode::SCbranchScc0:
case Decoder::Opcode::SCbranchScc1:
case Decoder::Opcode::SCbranchVccz:
case Decoder::Opcode::SCbranchVccnz:
case Decoder::Opcode::SCbranchExecz:
case Decoder::Opcode::SCbranchExecnz: return true;
default: return false;
}
}
void ClearEmbeddedFetchSgprs(std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
const Decoder::Operand& dst, uint32_t size) {
if (!IsDecodedSgpr(dst)) {
return;
}
const auto register_id = DecodedSgprReg(dst);
for (uint32_t i = 0; i < size && register_id + i < sgprs.size(); i++) {
sgprs[register_id + i] = {};
}
}
bool TryDecodedOperandConstant(const std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
const Decoder::Operand& op, uint32_t& value) {
switch (op.kind) {
case Decoder::OperandKind::LiteralConstant:
case Decoder::OperandKind::IntegerInlineConstant:
case Decoder::OperandKind::FloatInlineConstant: value = op.value; return true;
case Decoder::OperandKind::Null: value = 0; return true;
default: break;
}
if (IsDecodedSgpr(op) && DecodedSgprReg(op) < sgprs.size() &&
sgprs[DecodedSgprReg(op)].type == EmbeddedFetchValueType::Constant) {
value = sgprs[DecodedSgprReg(op)].value;
return true;
}
return false;
}
bool TryDecodedSmemOffset(const std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
const Decoder::Instruction& inst, uint32_t& raw_offset) {
uint32_t base = 0;
if (!TryDecodedOperandConstant(sgprs, inst.src1, base)) {
return false;
}
const auto value = static_cast<uint64_t>(base) + inst.offset;
if (value > 0xffffffffull) {
return false;
}
raw_offset = static_cast<uint32_t>(value);
return true;
}
bool IsEmbeddedFetchSLoad(const Decoder::Instruction& inst) {
switch (inst.opcode) {
case Decoder::Opcode::SLoadDword:
case Decoder::Opcode::SLoadDwordx2:
case Decoder::Opcode::SLoadDwordx4:
case Decoder::Opcode::SLoadDwordx8:
case Decoder::Opcode::SLoadDwordx16: return true;
default: return false;
}
}
bool IsEmbeddedFetchBufferLoad(const Decoder::Instruction& inst) {
switch (inst.opcode) {
case Decoder::Opcode::BufferLoadFormatX:
case Decoder::Opcode::BufferLoadFormatXy:
case Decoder::Opcode::BufferLoadFormatXyz:
case Decoder::Opcode::BufferLoadFormatXyzw: return true;
default: return false;
}
}
bool IsEmbeddedFetchAttribPropagationAlu(const Decoder::Instruction& inst) {
switch (inst.opcode) {
case Decoder::Opcode::SBfeU32:
case Decoder::Opcode::SAndB32:
case Decoder::Opcode::SAddI32:
case Decoder::Opcode::SAddU32:
case Decoder::Opcode::SLshlB32: return true;
default: return false;
}
}
int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) {
return static_cast<int>((raw_offset + static_cast<uint32_t>(dword) * 4u) / 16u);
}
void AppendUniquePcs(std::vector<uint32_t>* dst, const std::vector<uint32_t>& src) {
EXIT_IF(dst == nullptr);
dst->reserve(dst->size() + src.size());
for (auto pc: src) {
if (std::find(dst->begin(), dst->end(), pc) == dst->end()) {
dst->push_back(pc);
}
}
}
EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program& decoded,
const ShaderVertexInputInfo* input_info,
uint32_t user_data_base, uint32_t user_data_count,
uint32_t wave_size) {
EmbeddedFetchData data;
if (input_info == nullptr || !input_info->fetch_embedded) {
return data;
}
data.loads.reserve(input_info->resources_num);
int32_t offset_candidate = -1;
bool offset_conflict = false;
const int shift_regs = 8;
const int attrib_reg = input_info->fetch_attrib_reg + shift_regs;
const int buffer_reg = input_info->fetch_buffer_reg + shift_regs;
std::array<EmbeddedFetchSgprInfo, 108> sgprs {};
std::array<EmbeddedFetchVgprInfo, 256> vgprs {};
EmbeddedFetchVectorLanes vector_lanes;
const bool track_vector_lanes =
std::none_of(decoded.instructions.begin(), decoded.instructions.end(),
[](const auto& inst) { return EmbeddedFetchHasBranch(inst.opcode); });
if (attrib_reg >= 0 && attrib_reg < static_cast<int>(sgprs.size())) {
sgprs[attrib_reg].type = EmbeddedFetchValueType::AttribTable;
}
if (attrib_reg + 1 >= 0 && attrib_reg + 1 < static_cast<int>(sgprs.size())) {
sgprs[attrib_reg + 1].type = EmbeddedFetchValueType::AttribTable;
}
if (buffer_reg >= 0 && buffer_reg < static_cast<int>(sgprs.size())) {
sgprs[buffer_reg].type = EmbeddedFetchValueType::BufferTable;
}
if (buffer_reg + 1 >= 0 && buffer_reg + 1 < static_cast<int>(sgprs.size())) {
sgprs[buffer_reg + 1].type = EmbeddedFetchValueType::BufferTable;
}
for (const auto& inst: decoded.instructions) {
if (data.loads.empty() && inst.opcode == Decoder::Opcode::VAddI32 &&
IsDecodedVgpr(inst.dst) && inst.dst.reg == 0 && IsDecodedSgpr(inst.src0) &&
IsDecodedVgpr(inst.src1) && inst.src1.reg == 0) {
const auto reg = DecodedSgprReg(inst.src0);
if (reg >= user_data_base && reg - user_data_base < user_data_count) {
if (offset_candidate >= 0 && offset_candidate != static_cast<int32_t>(reg)) {
offset_conflict = true;
} else {
offset_candidate = static_cast<int32_t>(reg);
}
}
}
switch (inst.opcode) {
case Decoder::Opcode::VWritelaneB32: {
uint32_t lane = 0;
if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
vgprs[inst.dst.reg] = {};
}
if (track_vector_lanes && IsDecodedVgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
DecodedSgprReg(inst.src0) < sgprs.size() &&
TryDecodedOperandConstant(sgprs, inst.src1, lane)) {
vector_lanes[EmbeddedFetchVectorLaneKey(inst.dst.reg,
EmbeddedFetchLane(lane, wave_size))] =
sgprs[DecodedSgprReg(inst.src0)];
} else if (IsDecodedVgpr(inst.dst)) {
ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
}
break;
}
case Decoder::Opcode::VReadlaneB32: {
uint32_t lane = 0;
if (track_vector_lanes && IsDecodedSgpr(inst.dst) &&
DecodedSgprReg(inst.dst) < sgprs.size() && IsDecodedVgpr(inst.src0) &&
TryDecodedOperandConstant(sgprs, inst.src1, lane)) {
const auto found = vector_lanes.find(EmbeddedFetchVectorLaneKey(
inst.src0.reg, EmbeddedFetchLane(lane, wave_size)));
sgprs[DecodedSgprReg(inst.dst)] =
found != vector_lanes.end() ? found->second : EmbeddedFetchSgprInfo {};
} else if (IsDecodedSgpr(inst.dst)) {
ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1);
}
break;
}
case Decoder::Opcode::SMovB32:
if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
DecodedSgprReg(inst.src0) < sgprs.size()) {
sgprs[DecodedSgprReg(inst.dst)] = sgprs[DecodedSgprReg(inst.src0)];
} else if (IsDecodedSgpr(inst.dst)) {
uint32_t value = 0;
if (TryDecodedOperandConstant(sgprs, inst.src0, value)) {
auto& dst = sgprs[DecodedSgprReg(inst.dst)];
dst.type = EmbeddedFetchValueType::Constant;
dst.value = value;
dst.prolog_loads.clear();
} else {
ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1);
}
}
break;
case Decoder::Opcode::SMovkI32:
if (IsDecodedSgpr(inst.dst)) {
auto& dst = sgprs[DecodedSgprReg(inst.dst)];
dst.type = EmbeddedFetchValueType::Constant;
dst.value = inst.src0.value;
dst.prolog_loads.clear();
}
break;
default:
if (IsEmbeddedFetchSLoad(inst)) {
if (IsDecodedSgpr(inst.src0) && DecodedSgprReg(inst.src0) < sgprs.size() &&
sgprs[DecodedSgprReg(inst.src0)].type ==
EmbeddedFetchValueType::AttribTable) {
uint32_t raw_offset = 0;
if (TryDecodedSmemOffset(sgprs, inst, raw_offset)) {
const auto register_id = DecodedSgprReg(inst.dst);
const int index = static_cast<int>(raw_offset / 4u);
for (uint32_t i = 0;
i < DecodedDstSize(inst) && register_id + i < sgprs.size(); i++) {
auto& dst = sgprs[register_id + i];
dst.type = EmbeddedFetchValueType::Attrib;
dst.attrib_id = index + static_cast<int>(i);
dst.prolog_loads = {inst.pc};
}
} else {
ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
}
} else if (IsDecodedSgpr(inst.src0) &&
DecodedSgprReg(inst.src0) < sgprs.size() &&
sgprs[DecodedSgprReg(inst.src0)].type ==
EmbeddedFetchValueType::BufferTable) {
const auto register_id = DecodedSgprReg(inst.dst);
uint32_t raw_offset = 0;
if (TryDecodedSmemOffset(sgprs, inst, raw_offset)) {
for (uint32_t i = 0;
i < DecodedDstSize(inst) && register_id + i < sgprs.size(); i++) {
auto& dst = sgprs[register_id + i];
dst.type = EmbeddedFetchValueType::Buffer;
dst.attrib_id =
BufferTableAttribFromOffset(raw_offset, static_cast<int>(i));
dst.prolog_loads = {inst.pc};
}
} else if (IsDecodedSgpr(inst.src1) &&
DecodedSgprReg(inst.src1) < sgprs.size() &&
sgprs[DecodedSgprReg(inst.src1)].type ==
EmbeddedFetchValueType::Attrib &&
(inst.offset & 0x3u) == 0) {
for (uint32_t i = 0;
i < DecodedDstSize(inst) && register_id + i < sgprs.size(); i++) {
auto& dst = sgprs[register_id + i];
dst.type = EmbeddedFetchValueType::Buffer;
dst.attrib_id = sgprs[DecodedSgprReg(inst.src1)].attrib_id;
dst.prolog_loads = sgprs[DecodedSgprReg(inst.src1)].prolog_loads;
dst.prolog_loads.push_back(inst.pc);
}
} else {
ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
}
} else {
ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
}
} else if (inst.opcode == Decoder::Opcode::VCndmaskB32) {
if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
}
if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size() &&
IsDecodedVgpr(inst.src0) && inst.src0.reg == 8 &&
IsDecodedVgpr(inst.src1) && inst.src1.reg == 5) {
vgprs[inst.dst.reg].type = EmbeddedFetchValueType::Index;
}
} else if (IsEmbeddedFetchAttribPropagationAlu(inst)) {
if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
DecodedSgprReg(inst.src0) < sgprs.size() &&
sgprs[DecodedSgprReg(inst.src0)].type == EmbeddedFetchValueType::Attrib) {
sgprs[DecodedSgprReg(inst.dst)] = sgprs[DecodedSgprReg(inst.src0)];
} else if (IsDecodedSgpr(inst.dst)) {
uint32_t src0 = 0;
uint32_t src1 = 0;
if (TryDecodedOperandConstant(sgprs, inst.src0, src0) &&
TryDecodedOperandConstant(sgprs, inst.src1, src1)) {
auto& dst = sgprs[DecodedSgprReg(inst.dst)];
dst.type = EmbeddedFetchValueType::Constant;
switch (inst.opcode) {
case Decoder::Opcode::SAndB32: dst.value = src0 & src1; break;
case Decoder::Opcode::SLshlB32:
dst.value = src0 << (src1 & 31u);
break;
case Decoder::Opcode::SBfeU32:
dst.value = src0 >> (src1 & 31u);
break;
default: dst.value = src0 + src1; break;
}
dst.prolog_loads.clear();
} else {
ClearEmbeddedFetchSgprs(sgprs, inst.dst, 1);
}
}
} else if (IsEmbeddedFetchBufferLoad(inst)) {
if (IsDecodedVgpr(inst.src0) && inst.src0.reg < vgprs.size() &&
vgprs[inst.src0.reg].type == EmbeddedFetchValueType::Index &&
IsDecodedSgpr(inst.src1) && DecodedSgprReg(inst.src1) < sgprs.size() &&
sgprs[DecodedSgprReg(inst.src1)].type == EmbeddedFetchValueType::Buffer) {
const auto& buffer = sgprs[DecodedSgprReg(inst.src1)];
EmbeddedFetchLoad load;
load.pc = inst.pc;
load.attrib_id = buffer.attrib_id;
load.components = DecodedDstSize(inst);
load.prolog_loads = buffer.prolog_loads;
if (data.loads.empty() && !offset_conflict) {
data.vertex_offset_sgpr = offset_candidate;
}
data.loads.push_back(load);
}
}
break;
}
if (inst.opcode == Decoder::Opcode::VMovreldB32) {
vector_lanes.clear();
} else if (inst.opcode != Decoder::Opcode::VWritelaneB32 && IsDecodedVgpr(inst.dst)) {
for (uint32_t i = 0; i < EmbeddedFetchDstSize(inst) && inst.dst.reg + i < vgprs.size();
i++) {
ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg + i);
}
}
}
return data;
}
const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const std::vector<EmbeddedFetchLoad>& loads,
uint32_t pc) {
for (const auto& load: loads) {
if (load.pc == pc) {
return &load;
}
}
return nullptr;
}
bool EmbeddedFetchPcInList(const std::vector<uint32_t>& pcs, uint32_t pc) {
return std::find(pcs.begin(), pcs.end(), pc) != pcs.end();
}
bool IsIrFetchPrologLoad(const IR::Instruction& inst) {
return inst.op == IR::Opcode::SLoadDword || inst.op == IR::Opcode::SBufferLoadDword;
}
int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
const EmbeddedFetchLoad& load) {
if (input_info == nullptr) {
return -1;
}
if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num &&
input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) {
return load.attrib_id;
}
for (int i = 0; i < input_info->resources_num; i++) {
const auto& dst = input_info->resources_dst[i];
if (dst.attr_id == load.attrib_id &&
load.components <= static_cast<uint32_t>(std::max(dst.registers_num, 1))) {
return i;
}
}
for (int i = 0; i < input_info->resources_num; i++) {
if (input_info->resources_dst[i].attr_id == load.attrib_id) {
return i;
}
}
return -1;
}
uint32_t RewriteEmbeddedVertexFetches(IR::Program* ir, const ShaderVertexInputInfo* input_info,
const std::vector<EmbeddedFetchLoad>& loads) {
if (ir == nullptr || input_info == nullptr || loads.empty()) {
return 0;
}
std::vector<uint32_t> prolog_pcs;
prolog_pcs.reserve(loads.size());
for (const auto& load: loads) {
AppendUniquePcs(&prolog_pcs, load.prolog_loads);
}
auto* mutable_input_info = const_cast<ShaderVertexInputInfo*>(input_info);
uint32_t rewritten = 0;
for (auto& block: ir->blocks) {
for (auto& inst: block.instructions) {
if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) {
auto pc = inst.pc;
inst = {};
inst.pc = pc;
inst.op = IR::Opcode::ControlNop;
continue;
}
if (inst.op != IR::Opcode::BufferLoadDword) {
continue;
}
const auto* load = FindEmbeddedFetchLoad(loads, inst.pc);
if (load == nullptr || inst.memory.component_index >= load->components) {
continue;
}
const auto resource_id = ResolveEmbeddedFetchResource(input_info, *load);
if (resource_id < 0 || resource_id >= input_info->resources_num) {
LOGF("ShaderRecompiler VS embedded fetch remap failed: pc=0x%08" PRIx32
" attrib=%d resources=%d\n",
inst.pc, load->attrib_id, input_info->resources_num);
continue;
}
inst.op = IR::Opcode::LoadInputF32;
inst.input_info.attr = static_cast<uint32_t>(resource_id);
inst.input_info.chan = inst.memory.component_index;
inst.memory = {};
inst.src_count = 0;
mutable_input_info->resource_fetch_components[resource_id] =
std::max(mutable_input_info->resource_fetch_components[resource_id],
static_cast<int>(inst.input_info.chan + 1u));
rewritten++;
}
}
return rewritten;
}
} // namespace
bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
CompileResult* result, std::string* error) {
if (code.empty() || result == nullptr) {
if (error != nullptr) {
*error = "invalid shader recompiler input";
}
return false;
}
if (options.stage != ShaderType::Compute && options.stage != ShaderType::Vertex &&
options.stage != ShaderType::Pixel) {
if (error != nullptr) {
*error = "shader recompiler supports compute, vertex, and pixel stages";
}
return false;
}
const auto compile_begin = std::chrono::steady_clock::now();
const auto phase_ms = [&compile_begin]() {
return static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - compile_begin)
.count());
};
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " code_words=%" PRIu64 " decode\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(code.size()));
Decoder::Program decoded;
if (!Decoder::DecodeProgram(code, &decoded, error)) {
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " decode instructions=%" PRIu64
" elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
std::string decoded_dump;
if (options.dump_ir) {
decoded_dump = Decoder::ProgramToString(decoded);
if (options.early_dump) {
LOGF("%s decoded RDNA2 (early):\n%s", GetDumpLabel(options), decoded_dump.c_str());
}
}
CFG::Graph cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash);
if (!CFG::BuildGraph(decoded, &cfg, error)) {
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
" loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
bool dispatcher_fallback = false;
std::string dispatcher_reason;
if (cfg.irreducible) {
dispatcher_fallback = true;
dispatcher_reason = cfg.unsupported_reason;
const auto diagnostic = FormatCfgFailure(cfg, options, cfg.unsupported_reason);
LOGF("%s irreducible CFG detected: %s\n", GetDumpLabel(options), diagnostic.c_str());
} else {
std::string structure_error;
const auto unstructured_cfg = cfg;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!CFG::Structurize(&cfg, &structure_error)) {
const auto diagnostic = FormatCfgFailure(cfg, options, structure_error);
LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str());
dispatcher_fallback = true;
dispatcher_reason = structure_error;
const auto failure_kind = cfg.failure_kind;
const auto failure_block = cfg.failure_block;
cfg = unstructured_cfg;
cfg.unsupported = true;
cfg.failure_kind = failure_kind;
cfg.failure_block = failure_block;
cfg.unsupported_reason = structure_error;
} else {
LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
static_cast<uint64_t>(cfg.blocks.size()));
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
" loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(cfg.blocks.size()),
static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
}
IR::Program ir;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
StageName(options.stage), options.shader_hash);
if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, &ir, error)) {
return false;
}
ir.lane_mask_mode = options.lane_mask_mode;
ir.shader_hash = options.shader_hash;
ir.user_data_base = options.user_data_base;
ir.user_data_count = options.user_data_count;
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
" elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(ir.blocks.size()), phase_ms());
EmbeddedFetchData embedded_fetch;
if (options.stage == ShaderType::Vertex && options.vertex_input_info != nullptr &&
options.vertex_input_info->fetch_embedded) {
embedded_fetch =
DetectEmbeddedVertexFetch(decoded, options.vertex_input_info, ir.user_data_base,
ir.user_data_count, options.wave_size);
auto rewritten =
RewriteEmbeddedVertexFetches(&ir, options.vertex_input_info, embedded_fetch.loads);
if (rewritten > 0 || !embedded_fetch.loads.empty()) {
LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n",
GetDumpLabel(options), static_cast<uint64_t>(embedded_fetch.loads.size()),
rewritten);
}
}
if (!IR::BuildScalarProvenance(&ir, error)) {
return false;
}
std::string srt_error;
if (!IR::BuildSrtPlan(&ir, &srt_error)) {
LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str());
if (error != nullptr) {
*error = std::move(srt_error);
}
return false;
}
ir.dispatcher_fallback = dispatcher_fallback;
if (!dispatcher_reason.empty()) {
ir.fallback_reason = dispatcher_reason;
}
if (!IR::PatchSrtReads(&ir, error) || !IR::TrackResources(&ir, error)) {
return false;
}
if (options.stage == ShaderType::Vertex) {
ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
}
if (!dispatcher_fallback) {
std::string emitter_reason;
if (NeedsDispatcherForStructuredLoopHeader(ir, &emitter_reason)) {
dispatcher_fallback = true;
dispatcher_reason = emitter_reason;
ir.dispatcher_fallback = true;
ir.cfg_failure_kind = CFG::FailureKind::StructuredControlFlow;
ir.fallback_reason = emitter_reason;
LOGF("%s structured CFG emitter fallback: %s\n", GetDumpLabel(options),
emitter_reason.c_str());
}
}
IR::ResourceSnapshot resources;
if (options.resource_snapshot != nullptr) {
resources = *options.resource_snapshot;
} else {
IR::SrtRuntime runtime;
std::array<uint32_t, 64> zero_user_data {};
if (options.user_data == nullptr) {
for (uint32_t i = 2; i < zero_user_data.size(); i += 4) {
zero_user_data[i] = UINT32_MAX;
}
}
const auto* user_data =
options.user_data != nullptr ? options.user_data : zero_user_data.data();
runtime.user_data = std::span<const uint32_t>(user_data, options.user_data_count);
runtime.shader_base = options.shader_base != 0 ? options.shader_base
: reinterpret_cast<uint64_t>(code.data());
runtime.read_memory = options.read_memory != nullptr ? options.read_memory
: options.user_data == nullptr ? ReadZeroMemory
: nullptr;
runtime.userdata = options.read_memory_data;
runtime.flat_memory_base = options.flat_memory_base;
if (!IR::MaterializeResources(ir, runtime, &resources, error)) {
return false;
}
}
if (!IR::SpecializeResources(&ir, resources, error)) {
return false;
}
ShaderVertexInputInfo default_vertex {};
ShaderPixelInputInfo default_pixel {};
ShaderComputeInputInfo default_compute {};
IR::ShaderInfoOptions info_options;
info_options.vertex =
options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex;
info_options.pixel =
options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel;
info_options.compute =
options.compute_input_info != nullptr ? options.compute_input_info : &default_compute;
if (!IR::CollectShaderInfo(&ir, info_options, error)) {
return false;
}
IR::BindingLayoutOptions layout_options;
layout_options.descriptor_set = options.descriptor_set;
layout_options.push_constant_offset = options.push_constant_offset;
if (!IR::AllocateBindings(&ir, layout_options, error)) {
return false;
}
std::string ir_dump;
if (options.dump_ir) {
ir_dump = MakeIrDump(cfg, ir);
if (options.early_dump) {
LOGF("%s native IR and bindings (early):\n%s", GetDumpLabel(options), ir_dump.c_str());
}
}
std::vector<uint32_t> spirv;
std::string emit_error;
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info,
options.compute_input_info, &spirv, &emit_error)) {
if (dispatcher_fallback && error != nullptr) {
*error = fmt::format("dispatcher fallback failed after {}: {}",
dispatcher_reason.c_str(), emit_error.c_str());
LOGF("%s dispatcher fallback emit failed: %s\n", GetDumpLabel(options), error->c_str());
} else if (error != nullptr) {
*error = emit_error;
}
return false;
}
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram words=%" PRIu64
" elapsed_ms=%" PRIu64 "\n",
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
static_cast<uint64_t>(spirv.size()), phase_ms());
if (dispatcher_fallback) {
LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str());
}
result->spirv = std::move(spirv);
result->program = std::move(ir);
result->resources = std::move(resources);
if (options.dump_ir) {
result->decoded_dump = std::move(decoded_dump);
result->ir_dump = std::move(ir_dump);
} else {
result->decoded_dump.clear();
result->ir_dump.clear();
}
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler
src/graphics/shader/recompiler/ShaderRecompiler.h
+51
View File
@@ -0,0 +1,51 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERRECOMPILER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERRECOMPILER_H_
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include "graphics/shader/shader.h"
#include <optional>
#include <span>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler {
struct CompileOptions {
ShaderType stage = ShaderType::Compute;
ShaderLaneMaskMode lane_mask_mode = ShaderLaneMaskMode::NativeWave;
uint32_t wave_size = 64;
uint32_t user_data_base = 0;
uint32_t user_data_count = 64;
uint64_t shader_hash = 0;
uint64_t shader_base = 0;
std::optional<uint64_t> flat_memory_base;
uint32_t descriptor_set = 0;
uint32_t push_constant_offset = 0;
bool dump_ir = true;
bool early_dump = false;
const char* dump_label = nullptr;
const uint32_t* user_data = nullptr;
IR::SrtMemoryReader read_memory = nullptr;
void* read_memory_data = nullptr;
const IR::ResourceSnapshot* resource_snapshot = nullptr;
const ShaderVertexInputInfo* vertex_input_info = nullptr;
const ShaderPixelInputInfo* pixel_input_info = nullptr;
const ShaderComputeInputInfo* compute_input_info = nullptr;
};
struct CompileResult {
std::vector<uint32_t> spirv;
std::string decoded_dump;
std::string ir_dump;
IR::Program program;
IR::ResourceSnapshot resources;
};
bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
CompileResult* result, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERRECOMPILER_H_ */
src/graphics/shader/recompiler/SpirvBuilder.cpp
+160
View File
@@ -0,0 +1,160 @@
#include "graphics/shader/recompiler/SpirvBuilder.h"
#include <cstdio>
#include <cstring>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
static constexpr size_t InitialSpirvSectionReserve = 4096;
static constexpr size_t InitialSpirvFunctionSectionReserve = 450000;
static void AppendInstructionWords(std::vector<uint32_t>* section, const uint32_t* words,
size_t words_num) {
if (words_num == 0) {
return;
}
const auto opcode = words[0];
const auto word_count = static_cast<uint32_t>(words_num);
section->push_back((word_count << 16u) | opcode);
section->insert(section->end(), words + 1, words + words_num);
}
Builder::Builder() {
m_debug.reserve(InitialSpirvSectionReserve);
m_annotations.reserve(InitialSpirvSectionReserve);
m_types.reserve(InitialSpirvSectionReserve);
m_functions.reserve(InitialSpirvFunctionSectionReserve);
}
uint32_t Builder::AllocateId() {
return m_next_id++;
}
void Builder::AppendString(std::vector<uint32_t>* words, const char* text) {
const auto len = text != nullptr ? std::strlen(text) : 0;
const auto word_count = (len + 1u + 3u) / 4u;
for (size_t i = 0; i < word_count; i++) {
uint32_t word = 0;
for (size_t byte = 0; byte < 4; byte++) {
const auto index = i * 4u + byte;
if (index < len) {
word |= static_cast<uint32_t>(static_cast<unsigned char>(text[index]))
<< (byte * 8u);
}
}
words->push_back(word);
}
}
void Builder::AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
const std::vector<uint32_t>& operands) {
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
section->push_back((word_count << 16u) | opcode);
section->insert(section->end(), operands.begin(), operands.end());
}
void Builder::AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
std::initializer_list<uint32_t> operands) {
const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
section->push_back((word_count << 16u) | opcode);
section->insert(section->end(), operands.begin(), operands.end());
}
void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
AppendInstruction(&m_capabilities, 17u, operands);
}
void Builder::AddExtension(const char* name) {
std::vector<uint32_t> operands;
AppendString(&operands, name);
AppendInstruction(&m_extensions, 10u, operands);
}
void Builder::AddExtInstImport(uint32_t id, const char* name) {
std::vector<uint32_t> operands = {id};
AppendString(&operands, name);
AppendInstruction(&m_ext_inst_imports, 11u, operands);
}
void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
AppendInstruction(&m_memory_model, 14u, operands);
}
void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
const std::vector<uint32_t>& interfaces) {
std::vector<uint32_t> operands = {execution_model, entry_point};
AppendString(&operands, name);
operands.insert(operands.end(), interfaces.begin(), interfaces.end());
AppendInstruction(&m_entry_points, 15u, operands);
}
void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
AppendInstruction(&m_execution_modes, 16u, operands);
}
void Builder::AddName(uint32_t target, const char* name) {
std::vector<uint32_t> operands = {target};
AppendString(&operands, name);
AppendInstruction(&m_debug, 5u, operands);
}
void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
AppendInstructionWords(&m_annotations, words.begin(), words.size());
}
void Builder::AddType(std::initializer_list<uint32_t> words) {
AppendInstructionWords(&m_types, words.begin(), words.size());
}
void Builder::AddFunction(std::initializer_list<uint32_t> words) {
AppendInstructionWords(&m_functions, words.begin(), words.size());
}
void Builder::AddFunction(const std::vector<uint32_t>& words) {
AppendInstructionWords(&m_functions, words.data(), words.size());
}
std::vector<uint32_t> Builder::Build() const {
if (m_debug.size() > InitialSpirvSectionReserve) {
std::printf("SPIR-V builder reserve exceeded: section=debug size=%zu reserve=%zu\n",
m_debug.size(), InitialSpirvSectionReserve);
}
if (m_annotations.size() > InitialSpirvSectionReserve) {
std::printf("SPIR-V builder reserve exceeded: section=annotations size=%zu reserve=%zu\n",
m_annotations.size(), InitialSpirvSectionReserve);
}
if (m_types.size() > InitialSpirvSectionReserve) {
std::printf("SPIR-V builder reserve exceeded: section=types size=%zu reserve=%zu\n",
m_types.size(), InitialSpirvSectionReserve);
}
if (m_functions.size() > InitialSpirvFunctionSectionReserve) {
std::printf("SPIR-V builder reserve exceeded: section=functions size=%zu reserve=%zu\n",
m_functions.size(), InitialSpirvFunctionSectionReserve);
}
std::vector<uint32_t> module;
module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() +
m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() +
m_debug.size() + m_annotations.size() + m_types.size() + m_functions.size());
module.push_back(0x07230203u);
module.push_back(0x00010300u);
module.push_back(0u);
module.push_back(m_next_id);
module.push_back(0u);
module.insert(module.end(), m_capabilities.begin(), m_capabilities.end());
module.insert(module.end(), m_extensions.begin(), m_extensions.end());
module.insert(module.end(), m_ext_inst_imports.begin(), m_ext_inst_imports.end());
module.insert(module.end(), m_memory_model.begin(), m_memory_model.end());
module.insert(module.end(), m_entry_points.begin(), m_entry_points.end());
module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end());
module.insert(module.end(), m_debug.begin(), m_debug.end());
module.insert(module.end(), m_annotations.begin(), m_annotations.end());
module.insert(module.end(), m_types.begin(), m_types.end());
module.insert(module.end(), m_functions.begin(), m_functions.end());
return module;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/SpirvBuilder.h
+56
View File
@@ -0,0 +1,56 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
#include "common/common.h"
#include <initializer_list>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
class Builder {
public:
Builder();
~Builder() = default;
KYTY_CLASS_DEFAULT_COPY(Builder);
uint32_t AllocateId();
void AddCapability(std::initializer_list<uint32_t> operands);
void AddExtension(const char* name);
void AddExtInstImport(uint32_t id, const char* name);
void AddMemoryModel(std::initializer_list<uint32_t> operands);
void AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
const std::vector<uint32_t>& interfaces);
void AddExecutionMode(std::initializer_list<uint32_t> operands);
void AddName(uint32_t target, const char* name);
void AddAnnotation(std::initializer_list<uint32_t> words);
void AddType(std::initializer_list<uint32_t> words);
void AddFunction(std::initializer_list<uint32_t> words);
void AddFunction(const std::vector<uint32_t>& words);
[[nodiscard]] std::vector<uint32_t> Build() const;
private:
static void AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
const std::vector<uint32_t>& operands);
static void AppendInstruction(std::vector<uint32_t>* section, uint32_t opcode,
std::initializer_list<uint32_t> operands);
static void AppendString(std::vector<uint32_t>* words, const char* text);
uint32_t m_next_id = 1;
std::vector<uint32_t> m_capabilities;
std::vector<uint32_t> m_extensions;
std::vector<uint32_t> m_ext_inst_imports;
std::vector<uint32_t> m_memory_model;
std::vector<uint32_t> m_entry_points;
std::vector<uint32_t> m_execution_modes;
std::vector<uint32_t> m_debug;
std::vector<uint32_t> m_annotations;
std::vector<uint32_t> m_types;
std::vector<uint32_t> m_functions;
};
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ */
src/graphics/shader/recompiler/SpirvEmitter.cpp
+491
View File
@@ -0,0 +1,491 @@
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/SrtWalker.h"
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
#include <algorithm>
#include <array>
#include <cinttypes>
#include <cstdio>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
namespace {
bool Fail(std::string* error, const std::string& message) {
if (error != nullptr) {
*error = message;
}
return false;
}
bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
using Kind = IR::DescriptorBindingKind;
using Dim = Decoder::ImageDimension;
if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
const bool integer = image.kind == IR::ResourceKind::ImageUint;
switch (image.dimension) {
case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
case Dim::Dim2DArray:
kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
return true;
case Dim::Unknown: return false;
}
}
const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
return false;
}
switch (image.dimension) {
case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
case Dim::Dim2DArray:
kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
return true;
case Dim::Unknown: return false;
}
return false;
}
bool NeedsSampler(IR::Opcode op) {
return op == IR::Opcode::ImageSample || op == IR::Opcode::ImageGather4 ||
op == IR::Opcode::ImageGetLod;
}
bool IsBufferLoad(IR::Opcode op) {
switch (op) {
case IR::Opcode::BufferLoadUbyte:
case IR::Opcode::BufferLoadSbyte:
case IR::Opcode::BufferLoadUshort:
case IR::Opcode::BufferLoadSshort:
case IR::Opcode::BufferLoadDword: return true;
default: return false;
}
}
bool IsBufferStore(IR::Opcode op) {
return op == IR::Opcode::BufferStoreByte || op == IR::Opcode::BufferStoreShort ||
op == IR::Opcode::BufferStoreDword;
}
bool IsFlatLoad(IR::Opcode op) {
return op == IR::Opcode::FlatLoadUbyte || op == IR::Opcode::FlatLoadSbyte ||
op == IR::Opcode::FlatLoadUshort || op == IR::Opcode::FlatLoadSshort ||
op == IR::Opcode::FlatLoadDword;
}
bool IsFlatStore(IR::Opcode op) {
return op == IR::Opcode::FlatStoreByte || op == IR::Opcode::FlatStoreShort ||
op == IR::Opcode::FlatStoreDword;
}
bool IsDsKind(IR::ResourceKind kind) {
return kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds;
}
bool IsDsRead(IR::Opcode op) {
switch (op) {
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32: return true;
default: return false;
}
}
bool IsDsWrite(IR::Opcode op) {
switch (op) {
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32: return true;
default: return false;
}
}
bool IsAtomic(IR::Opcode op) {
switch (op) {
case IR::Opcode::AtomicSwapU32:
case IR::Opcode::AtomicAddU32:
case IR::Opcode::AtomicSubU32:
case IR::Opcode::AtomicSMinI32:
case IR::Opcode::AtomicUMinU32:
case IR::Opcode::AtomicSMaxI32:
case IR::Opcode::AtomicUMaxU32:
case IR::Opcode::AtomicAndU32:
case IR::Opcode::AtomicOrU32:
case IR::Opcode::AtomicXorU32: return true;
default: return false;
}
}
uint32_t BufferAddressOperandCount(const IR::Instruction& inst) {
return 1u + (inst.memory.idxen ? 1u : 0u) + (inst.memory.offen ? 1u : 0u);
}
bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error) {
if (inst.src_count > std::size(inst.src)) {
return Fail(error, "instruction source count exceeds fixed IR storage");
}
const auto kind = inst.memory.kind;
const auto flat_kind = kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global ||
kind == IR::ResourceKind::Scratch;
const auto buffer_address_count = BufferAddressOperandCount(inst);
if ((IsBufferLoad(inst.op) &&
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count)) ||
(IsBufferStore(inst.op) &&
(kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) ||
(inst.op == IR::Opcode::SLoadDword &&
(kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
(inst.op == IR::Opcode::SBufferLoadDword &&
(kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
(IsFlatLoad(inst.op) && (!flat_kind || inst.src_count != 2)) ||
(IsFlatStore(inst.op) && (!flat_kind || inst.src_count != 3))) {
return Fail(error, "memory instruction opcode/kind/operand contract is invalid");
}
const auto ds_kind = IsDsKind(kind);
const auto ds_resource = !ds_kind || inst.memory.resource == 0;
if ((IsDsRead(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register)) ||
(IsDsWrite(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 2 ||
inst.dst.kind != IR::OperandKind::Null)) ||
((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
(!ds_kind || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register ||
(kind == IR::ResourceKind::Gds && inst.memory.offset != 0))) ||
((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
(!ds_kind || !ds_resource || inst.src_count != 3 ||
inst.dst.kind != IR::OperandKind::Null)) ||
(inst.op == IR::Opcode::DsWriteAddtidB32 &&
(kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 2 ||
inst.dst.kind != IR::OperandKind::Null)) ||
(inst.op == IR::Opcode::DsReadAddtidB32 &&
(kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 1 ||
inst.dst.kind != IR::OperandKind::Register)) ||
(inst.op == IR::Opcode::DsSwizzleB32 &&
(kind != IR::ResourceKind::None || inst.src_count != 2 ||
inst.dst.kind != IR::OperandKind::Register))) {
return Fail(error, "DS instruction has an invalid resource/operand contract");
}
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4 ||
inst.op == IR::Opcode::ImageGetLod || inst.op == IR::Opcode::ImageLoad ||
inst.op == IR::Opcode::ImageGetResinfo) &&
((kind != IR::ResourceKind::Image && kind != IR::ResourceKind::ImageUint) ||
inst.src_count != 1)) {
return Fail(error, "sampled image instruction has an invalid resource class");
}
if (inst.op == IR::Opcode::ImageStore &&
((kind != IR::ResourceKind::StorageImage && kind != IR::ResourceKind::StorageImageUint) ||
inst.src_count != 2)) {
return Fail(error, "storage image instruction has an invalid resource class");
}
if (IsAtomic(inst.op) &&
!((kind == IR::ResourceKind::Buffer && inst.src_count == buffer_address_count + 1u) ||
((kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds) &&
inst.memory.resource == 0 && inst.src_count == 2 &&
(inst.dst.kind == IR::OperandKind::Null ||
inst.dst.kind == IR::OperandKind::Register)) ||
(kind == IR::ResourceKind::StorageImageUint && inst.src_count == 2))) {
return Fail(error, "atomic instruction has an invalid resource/operand contract");
}
return true;
}
bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
using Kind = IR::DescriptorBindingKind;
constexpr auto KindCount = static_cast<size_t>(Kind::Count);
std::array<std::vector<uint32_t>, KindCount> expected;
std::array<bool, KindCount> present {};
const auto Dense = [](size_t size) {
std::vector<uint32_t> values(size);
for (uint32_t i = 0; i < values.size(); i++) {
values[i] = i;
}
return values;
};
auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
const auto index = static_cast<size_t>(kind);
present[index] = true;
expected[index] = std::move(resources);
};
if (!program.info.buffers.empty()) {
Expect(Kind::Buffers, Dense(program.info.buffers.size()));
}
for (uint32_t i = 0; i < program.info.images.size(); i++) {
Kind kind;
if (!ImageBinding(program.info.images[i], kind)) {
return Fail(error, "native shader plan has an invalid image class");
}
present[static_cast<size_t>(kind)] = true;
expected[static_cast<size_t>(kind)].push_back(i);
}
if (!program.info.samplers.empty()) {
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
}
const auto uses_gds =
std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) {
return std::any_of(
block.instructions.begin(), block.instructions.end(),
[](const auto& inst) { return inst.memory.kind == IR::ResourceKind::Gds; });
});
if (uses_gds) {
Expect(Kind::Gds);
}
if (!program.info.addresses.empty()) {
Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
}
if (!program.srt.reads.empty()) {
Expect(Kind::FlattenedSrt);
}
if (!program.bindings.user_data_registers.empty() && program.bindings.push_constant_size == 0) {
Expect(Kind::UserData);
}
std::array<bool, KindCount> seen {};
for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
const auto& binding = program.bindings.descriptors[i];
const auto kind = static_cast<size_t>(binding.kind);
if (kind >= KindCount || seen[kind] || !present[kind] ||
binding.resources != expected[kind]) {
return Fail(error, "native descriptor groups do not match shader topology");
}
for (uint32_t j = 0; j < i; j++) {
if (program.bindings.descriptors[j].binding == binding.binding) {
return Fail(error, "native descriptor binding numbers are not unique");
}
}
seen[kind] = true;
}
for (size_t i = 0; i < KindCount; i++) {
if (present[i] != seen[i]) {
return Fail(error, "native shader plan is missing a required descriptor group");
}
}
const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
if ((!has_user_storage &&
program.bindings.push_constant_size != program.bindings.user_data_registers.size() * 4u) ||
(has_user_storage && program.bindings.push_constant_size != 0) ||
!std::is_sorted(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) ||
std::adjacent_find(program.bindings.user_data_registers.begin(),
program.bindings.user_data_registers.end()) !=
program.bindings.user_data_registers.end()) {
return Fail(error, "native user-data layout is inconsistent");
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (!ValidateInstructionContract(inst, error)) {
return false;
}
const auto memory = inst.memory.kind;
if ((memory == IR::ResourceKind::Buffer || (memory == IR::ResourceKind::ScalarBuffer &&
inst.op == IR::Opcode::SBufferLoadDword)) &&
inst.memory.resource >= program.info.buffers.size()) {
return Fail(error, "buffer instruction has an invalid dense resource");
}
if ((memory == IR::ResourceKind::Image || memory == IR::ResourceKind::ImageUint ||
memory == IR::ResourceKind::StorageImage ||
memory == IR::ResourceKind::StorageImageUint) &&
(inst.memory.resource >= program.info.images.size() ||
program.info.images[inst.memory.resource].kind != memory ||
program.info.images[inst.memory.resource].dimension !=
inst.memory.image_dimension)) {
return Fail(error, "image instruction has an invalid dense resource");
}
const bool address =
inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat ||
memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch;
if (address && (inst.memory.resource >= program.info.addresses.size() ||
program.info.addresses[inst.memory.resource].kind != memory)) {
return Fail(error, "address instruction has an invalid dense resource");
}
if (NeedsSampler(inst.op) && inst.memory.sampler >= program.info.samplers.size()) {
return Fail(error, "sample instruction has an invalid dense sampler");
}
if (inst.op == IR::Opcode::LoadSrtDword &&
(inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32 ||
inst.src[0].imm >= program.srt.reads.size())) {
return Fail(error, "flattened SRT instruction has an invalid dense slot");
}
}
}
return true;
}
} // namespace
static constexpr size_t InitialEmitterVectorReserve = 4096;
static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
size_t size) {
if (size <= InitialEmitterVectorReserve) {
return;
}
std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
" stage=%u vector=%s size=%zu reserve=%zu\n",
program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
InitialEmitterVectorReserve);
}
static bool IsExecDestination(const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Exec;
}
static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruction& inst) {
if (inst.op != IR::Opcode::BitFieldMaskU64 || !IsExecDestination(inst.dst) ||
inst.src_count < 1u) {
return false;
}
uint32_t count = 0;
return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], &count) &&
(count & 63u) == 0u;
}
static bool ProducesLaneMask(const IR::Instruction& inst) {
switch (inst.op) {
case IR::Opcode::IAddCarryU32:
case IR::Opcode::ISubBorrowU32:
case IR::Opcode::UMadU64U32: return inst.dst2.kind != IR::OperandKind::Null;
default: return false;
}
}
bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
for (const auto& block: program.blocks) {
if (block.terminator.kind == CFG::TerminatorKind::ConditionalBranch) {
switch (block.terminator.condition) {
case CFG::BranchCondition::VccZero:
case CFG::BranchCondition::VccNonZero:
case CFG::BranchCondition::ExecZero:
case CFG::BranchCondition::ExecNonZero: return true;
default: break;
}
}
for (const auto& inst: block.instructions) {
if (Emitter::InstructionHasDppSource(inst)) {
return true;
}
if (Emitter::IsCompareOpcode(inst.op) && !Emitter::IsSccOperand(inst.dst)) {
return true;
}
if ((IsExecDestination(inst.dst) || IsExecDestination(inst.dst2)) &&
!IsUniformZeroExecWrite(program, inst)) {
return true;
}
if (ProducesLaneMask(inst)) {
return true;
}
if (inst.op == IR::Opcode::SelectMaskU32 || inst.op == IR::Opcode::SelectMaskF32Bits) {
return true;
}
if (inst.op == IR::Opcode::SaveexecB32 || inst.op == IR::Opcode::SaveexecB64) {
return true;
}
switch (inst.op) {
case IR::Opcode::WqmB64:
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file != IR::RegisterFile::Scalar) {
return true;
}
break;
case IR::Opcode::MaskedBitCountLowU32:
case IR::Opcode::MaskedBitCountHighU32:
case IR::Opcode::ReadFirstLaneU32:
case IR::Opcode::ReadLaneU32:
case IR::Opcode::WriteLaneU32:
case IR::Opcode::Permlane16B32:
case IR::Opcode::Permlanex16B32:
case IR::Opcode::DsSwizzleB32:
case IR::Opcode::DsConsume:
case IR::Opcode::DsAppend: return true;
default: break;
}
}
}
return false;
}
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>* spirv,
std::string* error) {
using namespace Emitter;
if (spirv == nullptr) {
SetError(error, "invalid SPIR-V output");
return false;
}
if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
program.stage != ShaderType::Pixel) {
SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
return false;
}
if (!program.srt_patching_complete || !program.resource_tracking_complete ||
!program.shader_info_complete || !program.binding_layout_complete) {
SetError(error, "SPIR-V emitter requires a fully planned native shader program");
return false;
}
if (!IR::ValidateResourceSnapshot(program, resources, error)) {
return false;
}
if (!IR::ValidateResourceSpecialization(program, resources, error)) {
return false;
}
if (!ValidateNativeProgram(program, error)) {
return false;
}
EmitterState state;
state.program = &program;
state.resources = &resources;
state.vertex_input_info = vertex_input_info;
state.pixel_input_info = pixel_input_info;
state.compute_input_info = compute_input_info;
state.stage = program.stage;
state.wave_size = program.wave_size;
state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
state.exact_subgroup_operations =
!state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
state.registers.reserve(InitialEmitterVectorReserve);
state.inputs.reserve(InitialEmitterVectorReserve);
state.outputs.reserve(InitialEmitterVectorReserve);
state.interface_variables.reserve(InitialEmitterVectorReserve);
state.reachable_blocks.reserve(InitialEmitterVectorReserve);
CollectRegisters(program, &state.registers);
CopyProgramInputsAndOutputs(&state, program);
state.needs_subgroup_ballot = ProgramNeedsSubgroupBallot(program);
state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
state.needs_compute_derivatives = ProgramNeedsComputeDerivatives(program);
state.needs_image_gather_extended = ProgramNeedsImageGatherExtended(program);
state.needs_function_lds = ProgramNeedsFunctionLds(program);
state.needs_pixel_valid_mask = ProgramNeedsPixelValidMask(program);
ComputeReachableBlocks(&state, program);
AllocateInputVariables(&state);
AllocateOutputVariables(&state);
AllocateDescriptorVariables(&state);
EmitHeaderAndTypes(&state);
AllocateRegisterVariables(&state);
AllocateBlockLabels(&state, program);
AllocateDispatcherState(&state, program);
EmitFunction(&state, program);
ReportReserveExceeded(program, "registers", state.registers.size());
ReportReserveExceeded(program, "inputs", state.inputs.size());
ReportReserveExceeded(program, "outputs", state.outputs.size());
ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
auto binary = state.builder.Build();
if (binary.empty()) {
SetError(error, "SPIR-V builder returned an empty module");
return false;
}
*spirv = std::move(binary);
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
src/graphics/shader/recompiler/SpirvEmitter.h
+22
View File
@@ -0,0 +1,22 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_
#include "common/common.h"
#include "common/stringUtils.h"
#include "graphics/shader/recompiler/ResourceMaterialization.h"
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::Spirv {
bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
const ShaderVertexInputInfo* vertex_input_info,
const ShaderPixelInputInfo* pixel_input_info,
const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>* spirv,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_H_ */
src/graphics/shader/recompiler/SrtPatcher.cpp
+133
View File
@@ -0,0 +1,133 @@
#include "graphics/shader/recompiler/SrtPatcher.h"
#include <fmt/format.h>
#include <map>
#include <set>
#include <vector>
namespace Libs::Graphics::ShaderRecompiler::IR {
bool PatchSrtReads(Program* program, std::string* error) {
if (program == nullptr || !program->srt_plan_complete || program->srt_patching_complete ||
program->resource_tracking_complete) {
if (error != nullptr) {
*error = program == nullptr ? "invalid SRT patch program"
: !program->srt_plan_complete ? "SRT plan is not ready"
: program->srt_patching_complete ? "SRT reads already patched"
: "resources already tracked";
}
return false;
}
std::map<uint32_t, uint32_t> offsets;
std::map<uint32_t, uint32_t> values;
for (const auto& read: program->srt.reads) {
if (read.value >= program->provenance.values.size() ||
read.flat_offset >= program->srt.reads.size() ||
(program->provenance.values[read.value].op != ScalarValueOp::ReadConst &&
program->provenance.values[read.value].op != ScalarValueOp::ReadConstBuffer)) {
if (error != nullptr) {
*error = fmt::format("invalid SRT read value={} flat_offset={}", read.value,
read.flat_offset);
}
return false;
}
const auto [it, inserted] = offsets.emplace(read.value, read.flat_offset);
if (!inserted && it->second != read.flat_offset) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has conflicting flat offsets", read.value);
}
return false;
}
const auto [value, unique] = values.emplace(read.flat_offset, read.value);
if (!unique && value->second != read.value) {
if (error != nullptr) {
*error = fmt::format("SRT flat offset {} has conflicting values", read.flat_offset);
}
return false;
}
}
if (offsets.size() != program->srt.reads.size() || values.size() != program->srt.reads.size()) {
if (error != nullptr) {
*error = "SRT reads do not form a dense value-to-offset bijection";
}
return false;
}
std::set<uint32_t> dynamic_reads;
for (const auto value: program->srt.dynamic_reads) {
if (value >= program->provenance.values.size() || offsets.contains(value) ||
(program->provenance.values[value].op != ScalarValueOp::ReadConst &&
program->provenance.values[value].op != ScalarValueOp::ReadConstBuffer) ||
!dynamic_reads.insert(value).second) {
if (error != nullptr) {
*error = fmt::format("invalid dynamic SRT read value={}", value);
}
return false;
}
}
std::set<uint32_t> dynamic_sources;
for (const auto source: program->srt.dynamic_sources) {
if (source == ScalarProvenance::Undefined ||
(source > ScalarProvenance::Unknown &&
source - 2u >= program->provenance.descriptors.size()) ||
!dynamic_sources.insert(source).second) {
if (error != nullptr) {
*error = fmt::format("invalid dynamic descriptor source={}", source);
}
return false;
}
}
struct Patch {
Instruction* inst = nullptr;
uint32_t flat_offset = 0;
};
std::vector<Patch> patches;
std::map<uint32_t, bool> matched;
for (const auto& [value, offset]: offsets) {
matched.emplace(value, false);
}
for (auto& block: program->blocks) {
for (auto& inst: block.instructions) {
if (inst.op != Opcode::SLoadDword && inst.op != Opcode::SBufferLoadDword) {
continue;
}
if (const auto found = offsets.find(inst.scalar_value); found != offsets.end()) {
const auto value_op = program->provenance.values[found->first].op;
if ((inst.op == Opcode::SLoadDword) != (value_op == ScalarValueOp::ReadConst)) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has the wrong scalar-load producer",
found->first);
}
return false;
}
patches.push_back({&inst, found->second});
matched[found->first] = true;
}
}
}
for (const auto& [value, found]: matched) {
if (!found) {
if (error != nullptr) {
*error = fmt::format("SRT read value {} has no scalar-load producer", value);
}
return false;
}
}
for (const auto& patch: patches) {
patch.inst->op = Opcode::LoadSrtDword;
patch.inst->src_count = 1;
for (auto& src: patch.inst->src) {
src = {};
}
patch.inst->src[0].kind = OperandKind::ImmediateU32;
patch.inst->src[0].imm = patch.flat_offset;
patch.inst->memory = {};
}
program->srt_patching_complete = true;
return true;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/SrtPatcher.h
+14
View File
@@ -0,0 +1,14 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_
#include "graphics/shader/recompiler/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
// Replaces each immediate ReadConst producer selected by the SRT plan with a dense flat-buffer
// dword load. Dynamic-offset reads remain explicit. On failure no instruction is changed.
bool PatchSrtReads(Program* program, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTPATCHER_H_ */
src/graphics/shader/recompiler/SrtWalker.cpp
+655
View File
@@ -0,0 +1,655 @@
#include "graphics/shader/recompiler/SrtWalker.h"
#include "graphics/host_gpu/hostMemory.h"
#include "graphics/shader/recompiler/ScalarProvenance.h"
#include <algorithm>
#include <fmt/format.h>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
const char* StageName(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return "vertex";
case ShaderType::Pixel: return "pixel";
case ShaderType::Fetch: return "fetch";
case ShaderType::Compute: return "compute";
default: return "unknown";
}
}
std::string Diagnostic(const Program& program, uint32_t pc, const std::string& message) {
return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
StageName(program.stage), pc, message);
}
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t* result) {
if (result == nullptr || base > AddressMask) {
return false;
}
if (offset < 0) {
const auto magnitude = uint64_t {0} - static_cast<uint64_t>(offset);
if (magnitude > base) {
return false;
}
*result = base - magnitude;
return true;
}
const auto magnitude = static_cast<uint64_t>(offset);
if (magnitude > AddressMask - base) {
return false;
}
*result = base + magnitude;
return true;
}
bool ApplyOperation(ScalarValueOp op, const uint32_t args[3], uint32_t* result) {
if (result == nullptr) {
return false;
}
const auto shift = args[1] & 31u;
switch (op) {
case ScalarValueOp::Add: *result = args[0] + args[1]; break;
case ScalarValueOp::AddCarry: *result = args[0] + args[1] + (args[2] & 1u); break;
case ScalarValueOp::Carry:
*result = static_cast<uint32_t>(
(static_cast<uint64_t>(args[0]) + args[1] + (args[2] & 1u)) >> 32u);
break;
case ScalarValueOp::Sub: *result = args[0] - args[1]; break;
case ScalarValueOp::SubBorrow: *result = args[0] - args[1] - (args[2] & 1u); break;
case ScalarValueOp::Borrow:
*result = static_cast<uint64_t>(args[1]) + (args[2] & 1u) > args[0] ? 1u : 0u;
break;
case ScalarValueOp::Mul: *result = args[0] * args[1]; break;
case ScalarValueOp::And: *result = args[0] & args[1]; break;
case ScalarValueOp::AndNot: *result = args[0] & ~args[1]; break;
case ScalarValueOp::Or: *result = args[0] | args[1]; break;
case ScalarValueOp::OrNot: *result = args[0] | ~args[1]; break;
case ScalarValueOp::Xor: *result = args[0] ^ args[1]; break;
case ScalarValueOp::Not: *result = ~args[0]; break;
case ScalarValueOp::ShiftLeft: *result = args[0] << shift; break;
case ScalarValueOp::ShiftRight: *result = args[0] >> shift; break;
case ScalarValueOp::ShiftRightArithmetic:
*result = static_cast<uint32_t>(static_cast<int32_t>(args[0]) >> shift);
break;
case ScalarValueOp::BitFieldMaskU32: {
const auto count = args[0] & 31u;
const auto offset = args[1] & 31u;
*result = ((uint32_t {1} << count) - 1u) << offset;
break;
}
case ScalarValueOp::BitFieldMaskU64Low:
case ScalarValueOp::BitFieldMaskU64High: {
const auto count = args[0] & 63u;
const auto value = ((uint64_t {1} << count) - 1u) << (args[1] & 63u);
*result = op == ScalarValueOp::BitFieldMaskU64Low ? static_cast<uint32_t>(value)
: static_cast<uint32_t>(value >> 32u);
break;
}
case ScalarValueOp::Add3: *result = args[0] + args[1] + args[2]; break;
case ScalarValueOp::ShiftLeftAdd: *result = (args[0] << shift) + args[2]; break;
case ScalarValueOp::ShiftLeftAddCarry:
*result =
static_cast<uint32_t>(((static_cast<uint64_t>(args[0]) << shift) + args[2]) >> 32u);
break;
case ScalarValueOp::AddShiftLeft: *result = (args[0] + args[1]) << (args[2] & 31u); break;
case ScalarValueOp::XorAdd: *result = (args[0] ^ args[1]) + args[2]; break;
case ScalarValueOp::ShiftLeftOr: *result = (args[0] << shift) | args[2]; break;
default: return false;
}
return true;
}
class ConstantFolder {
public:
explicit ConstantFolder(const ScalarProvenance& provenance)
: m_provenance(provenance), m_values(provenance.values.size()),
m_state(provenance.values.size()) {}
bool Fold(uint32_t id, uint32_t& result) {
if (id >= m_provenance.values.size() || m_state[id] == 1 || m_state[id] == 3) {
return false;
}
if (m_state[id] == 2) {
result = m_values[id];
return true;
}
m_state[id] = 1;
const auto& value = m_provenance.values[id];
uint32_t out = 0;
switch (value.op) {
case ScalarValueOp::Constant: out = value.imm; break;
case ScalarValueOp::Phi:
if (value.phi_args.empty() || !Fold(value.phi_args[0], out)) {
return Dynamic(id);
}
for (size_t i = 1; i < value.phi_args.size(); i++) {
uint32_t other = 0;
if (!Fold(value.phi_args[i], other) || other != out) {
return Dynamic(id);
}
}
break;
default: {
const auto count = ScalarValueArgCount(value.op);
if (count == 0 || count > 3 || value.op == ScalarValueOp::ReadConst) {
return Dynamic(id);
}
uint32_t args[3] = {};
for (uint32_t i = 0; i < count; i++) {
if (!Fold(value.args[i], args[i])) {
return Dynamic(id);
}
}
if (!ApplyOperation(value.op, args, &out)) {
return Dynamic(id);
}
break;
}
}
m_state[id] = 2;
m_values[id] = out;
result = out;
return true;
}
private:
bool Dynamic(uint32_t id) {
m_state[id] = 3;
return false;
}
const ScalarProvenance& m_provenance;
std::vector<uint32_t> m_values;
std::vector<uint8_t> m_state;
};
class PlanBuilder {
public:
explicit PlanBuilder(Program* program): m_program(program), m_folder(program->provenance) {
m_state.resize(program->provenance.values.size());
}
bool Run(std::string* error) {
m_program->srt = {};
for (const auto& block: m_program->blocks) {
for (const auto& inst: block.instructions) {
if (!CollectDescriptor(inst.memory.resource_source, inst.pc, error) ||
!CollectDescriptor(inst.memory.sampler_source, inst.pc, error)) {
return false;
}
}
}
for (const auto& block: m_program->blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == Opcode::SLoadDword && inst.scalar_value < m_state.size() &&
m_state[inst.scalar_value] == 0) {
const auto& value = m_program->provenance.values[inst.scalar_value];
uint32_t ignored = 0;
if (value.op == ScalarValueOp::ReadConst &&
m_folder.Fold(value.args[2], ignored) &&
!CollectValue(inst.scalar_value, inst.pc, error)) {
return false;
}
}
}
}
return true;
}
private:
bool Fail(uint32_t pc, std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(*m_program, pc, message);
}
return false;
}
bool CollectDescriptor(uint32_t source, uint32_t use_pc, std::string* error) {
const auto* descriptor = GetDescriptorSource(*m_program, source);
if (descriptor == nullptr) {
if (source <= ScalarProvenance::Unknown) {
return source == ScalarProvenance::Undefined || AddDynamicSource(source);
}
return Fail(use_pc, error, fmt::format("invalid descriptor source {}", source));
}
if (!DescriptorSourceResolved(*m_program, source) ||
DescriptorNeedsControlFlow(*descriptor)) {
MarkDescriptor(*descriptor);
return AddDynamicSource(source);
}
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!CollectValue(descriptor->dwords[i], use_pc, error)) {
return false;
}
}
return true;
}
void MarkDescriptor(const DescriptorValue& descriptor) {
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
MarkValue(descriptor.dwords[i]);
}
}
void MarkValue(uint32_t id) {
if (id >= m_state.size() || m_state[id] != 0) {
return;
}
m_state[id] = 2;
const auto& value = m_program->provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
MarkValue(arg);
}
return;
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
MarkValue(value.args[i]);
}
}
bool DescriptorNeedsControlFlow(const DescriptorValue& descriptor) {
std::vector<uint8_t> visited(m_program->provenance.values.size());
for (uint32_t i = 0; i < descriptor.dword_count; i++) {
if (ValueNeedsControlFlow(descriptor.dwords[i], &visited)) {
return true;
}
}
return false;
}
bool ValueNeedsControlFlow(uint32_t id, std::vector<uint8_t>* visited) {
if (id >= m_program->provenance.values.size() || (*visited)[id] != 0) {
return false;
}
(*visited)[id] = 1;
const auto& value = m_program->provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
uint32_t ignored = 0;
return !m_folder.Fold(id, ignored);
}
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (ValueNeedsControlFlow(value.args[i], visited)) {
return true;
}
}
return false;
}
bool AddDynamicSource(uint32_t source) {
if (std::find(m_program->srt.dynamic_sources.begin(), m_program->srt.dynamic_sources.end(),
source) == m_program->srt.dynamic_sources.end()) {
m_program->srt.dynamic_sources.push_back(source);
}
return true;
}
bool CollectValue(uint32_t id, uint32_t use_pc, std::string* error) {
if (id >= m_program->provenance.values.size()) {
return Fail(use_pc, error, fmt::format("invalid scalar value {}", id));
}
if (m_state[id] == 2) {
return true;
}
if (m_state[id] == 1) {
return Fail(use_pc, error, fmt::format("cyclic scalar value {}", id));
}
m_state[id] = 1;
const auto& value = m_program->provenance.values[id];
if (value.op == ScalarValueOp::Phi) {
for (const auto arg: value.phi_args) {
if (!CollectValue(arg, use_pc, error)) {
return false;
}
}
} else {
for (uint32_t i = 0; i < ScalarValueArgCount(value.op); i++) {
if (!CollectValue(value.args[i], use_pc, error)) {
return false;
}
}
}
if (value.op == ScalarValueOp::ReadConst || value.op == ScalarValueOp::ReadConstBuffer) {
const auto offset_arg = value.op == ScalarValueOp::ReadConst ? 2u : 4u;
uint32_t ignored = 0;
if (m_folder.Fold(value.args[offset_arg], ignored)) {
m_program->srt.reads.push_back(
{id, static_cast<uint32_t>(m_program->srt.reads.size()), use_pc});
} else {
m_program->srt.dynamic_reads.push_back(id);
}
}
m_state[id] = 2;
return true;
}
Program* m_program;
ConstantFolder m_folder;
std::vector<uint8_t> m_state;
};
class Evaluator {
public:
Evaluator(const Program& program, const SrtRuntime& runtime, uint32_t use_pc)
: m_program(program), m_runtime(runtime), m_use_pc(use_pc),
m_values(program.provenance.values.size()), m_state(program.provenance.values.size()) {}
void SetUsePc(uint32_t use_pc) { m_use_pc = use_pc; }
bool Evaluate(uint32_t id, uint32_t& result, std::string* error) {
if (id >= m_program.provenance.values.size()) {
return Fail(error, fmt::format("invalid scalar value {}", id));
}
if (m_state[id] == 2) {
result = m_values[id];
return true;
}
if (m_state[id] == 1) {
return Fail(error, fmt::format("cyclic scalar value {}", id));
}
m_state[id] = 1;
const auto& value = m_program.provenance.values[id];
uint32_t out = 0;
switch (value.op) {
case ScalarValueOp::UserData:
if (value.imm < m_program.user_data_base ||
value.imm - m_program.user_data_base >= m_runtime.user_data.size()) {
return Fail(error, fmt::format("user SGPR {} is unavailable", value.imm));
}
out = m_runtime.user_data[value.imm - m_program.user_data_base];
break;
case ScalarValueOp::Constant: out = value.imm; break;
case ScalarValueOp::PcRelativeLow:
out = static_cast<uint32_t>(m_runtime.shader_base + value.imm);
break;
case ScalarValueOp::PcRelativeHigh:
out = static_cast<uint32_t>((m_runtime.shader_base + value.imm) >> 32u);
break;
case ScalarValueOp::Phi: {
const auto first = std::find_if(value.phi_args.begin(), value.phi_args.end(),
[id](uint32_t arg) { return arg != id; });
if (first == value.phi_args.end()) {
return Fail(error, fmt::format("empty scalar phi {}", id));
}
if (!Evaluate(*first, out, error)) {
return false;
}
for (const auto arg: value.phi_args) {
if (arg == id || arg == *first) {
continue;
}
uint32_t other = 0;
if (!Evaluate(arg, other, error)) {
return false;
}
if (other != out) {
return Fail(error,
fmt::format("scalar phi {} has runtime-dependent values", id));
}
}
break;
}
case ScalarValueOp::ReadConst:
case ScalarValueOp::ReadConstBuffer:
if (!Read(value, &out, error)) {
return false;
}
break;
case ScalarValueOp::Undefined:
case ScalarValueOp::Unknown:
return Fail(error, fmt::format("scalar value {} is unresolved", id));
default:
if (!EvaluateOperation(value, &out, error)) {
return false;
}
break;
}
m_state[id] = 2;
m_values[id] = out;
result = out;
return true;
}
private:
bool Fail(std::string* error, const std::string& message) const {
if (error != nullptr) {
*error = Diagnostic(m_program, m_use_pc, message);
}
return false;
}
bool EvaluateOperation(const ScalarValue& value, uint32_t* result, std::string* error) {
const auto count = ScalarValueArgCount(value.op);
if (count == 0 || count > 3) {
return Fail(error, fmt::format("unsupported scalar operation {}",
static_cast<uint32_t>(value.op)));
}
uint32_t args[3] = {};
for (uint32_t i = 0; i < count; i++) {
if (!Evaluate(value.args[i], args[i], error)) {
return false;
}
}
return ApplyOperation(value.op, args, result) ||
Fail(error, fmt::format("unsupported scalar operation {}",
static_cast<uint32_t>(value.op)));
}
bool Read(const ScalarValue& value, uint32_t* result, std::string* error) {
const bool buffer = value.op == ScalarValueOp::ReadConstBuffer;
uint32_t lo = 0;
uint32_t hi = 0;
uint32_t offset = 0;
if (!Evaluate(value.args[0], lo, error) || !Evaluate(value.args[1], hi, error) ||
!Evaluate(value.args[buffer ? 4u : 2u], offset, error)) {
return false;
}
const auto base = (static_cast<uint64_t>(hi) << 32u | lo) & AddressMask;
const auto immediate = static_cast<int64_t>(static_cast<int32_t>(value.imm));
uint64_t address = 0;
if (buffer) {
uint32_t num_records = 0;
uint32_t ignored = 0;
if (!Evaluate(value.args[2], num_records, error) ||
!Evaluate(value.args[3], ignored, error)) {
return false;
}
if (immediate < 0) {
return Fail(error,
fmt::format("ReadConstBuffer pc=0x{:08x} has negative immediate {}",
value.pc, immediate));
}
const auto byte_offset = static_cast<uint64_t>(immediate) + offset;
const auto aligned_offset = byte_offset & ~uint64_t {3};
const auto stride = (hi >> 16u) & 0x3fffu;
const auto size = stride == 0 ? static_cast<uint64_t>(num_records)
: static_cast<uint64_t>(stride) * num_records;
if (aligned_offset > size || size - aligned_offset < sizeof(uint32_t)) {
return Fail(error,
fmt::format("ReadConstBuffer pc=0x{:08x} offset={} exceeds size={}",
value.pc, aligned_offset, size));
}
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
} else {
const auto base_aligned = base & ~uint64_t {3};
const auto immediate_aligned = immediate & ~int64_t {3};
const auto relative = immediate_aligned + static_cast<int64_t>(offset & ~uint32_t {3});
if (!AddSignedAddress(base_aligned, relative, &address)) {
return Fail(error,
fmt::format("ReadConst pc=0x{:08x} address base=0x{:016x} offset={} "
"is outside the 48-bit address space",
value.pc, base_aligned, relative));
}
}
const auto read_memory =
m_runtime.read_memory != nullptr ? m_runtime.read_memory : HostMemoryReadDword;
if (!read_memory(m_runtime.userdata, address, result)) {
return Fail(
error, fmt::format("ReadConst pc=0x{:08x} failed at 0x{:016x}", value.pc, address));
}
return true;
}
const Program& m_program;
const SrtRuntime& m_runtime;
uint32_t m_use_pc;
std::vector<uint32_t> m_values;
std::vector<uint8_t> m_state;
};
} // namespace
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t* result) {
if (result == nullptr) {
return false;
}
ConstantFolder folder(provenance);
return folder.Fold(value, *result);
}
bool BuildSrtPlan(Program* program, std::string* error) {
if (program == nullptr) {
if (error != nullptr) {
*error = "invalid SRT program";
}
return false;
}
if (program->resource_tracking_complete) {
if (error != nullptr) {
*error = "cannot rebuild SRT plan after resource tracking";
}
return false;
}
if (program->srt_patching_complete) {
if (error != nullptr) {
*error = "cannot rebuild SRT plan after SRT patching";
}
return false;
}
program->srt_plan_complete = false;
if (!PlanBuilder(program).Run(error)) {
return false;
}
program->srt_plan_complete = true;
return true;
}
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue* result,
std::string* error) {
if (result == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, use_pc, "invalid descriptor result");
}
return false;
}
const DescriptorSourceRequest request {source, use_pc};
std::vector<DescriptorValue> results;
if (!EvaluateDescriptorSources(program, std::span {&request, 1}, runtime, &results, error)) {
return false;
}
*result = results[0];
return true;
}
static bool EvaluateRuntimeSourcesImpl(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime,
std::vector<DescriptorValue>* results,
std::vector<uint32_t>* flat, bool evaluate_flat,
std::string* error) {
if (results == nullptr || flat == nullptr || !program.srt_plan_complete) {
if (error != nullptr) {
*error = Diagnostic(program, 0,
results == nullptr || flat == nullptr
? "invalid runtime evaluation output"
: "SRT plan is not ready");
}
return false;
}
std::vector<DescriptorValue> evaluated;
evaluated.reserve(requests.size());
std::vector<uint32_t> flattened(evaluate_flat ? program.srt.reads.size() : 0u);
Evaluator evaluator(program, runtime, 0);
for (const auto& request: requests) {
const auto* descriptor = GetDescriptorSource(program, request.source);
const auto dynamic =
std::find(program.srt.dynamic_sources.begin(), program.srt.dynamic_sources.end(),
request.source) != program.srt.dynamic_sources.end();
if (descriptor == nullptr || dynamic ||
!DescriptorSourceResolved(program, request.source)) {
if (error != nullptr) {
*error = Diagnostic(program, request.use_pc,
fmt::format("descriptor source {} is {}", request.source,
dynamic ? "GPU-dynamic" : "unresolved"));
}
return false;
}
evaluator.SetUsePc(request.use_pc);
auto value = *descriptor;
for (uint32_t i = 0; i < descriptor->dword_count; i++) {
if (!evaluator.Evaluate(descriptor->dwords[i], value.dwords[i], error)) {
return false;
}
}
evaluated.push_back(value);
}
if (evaluate_flat) {
for (const auto& read: program.srt.reads) {
evaluator.SetUsePc(read.use_pc);
if (read.flat_offset >= flattened.size()) {
if (error != nullptr) {
*error = Diagnostic(program, read.use_pc, "invalid flat SRT offset");
}
return false;
}
if (!evaluator.Evaluate(read.value, flattened[read.flat_offset], error)) {
return false;
}
}
}
*results = std::move(evaluated);
if (evaluate_flat) {
*flat = std::move(flattened);
}
return true;
}
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::string* error) {
if (results == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "invalid descriptor results");
}
return false;
}
std::vector<uint32_t> ignored;
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, &ignored, false, error);
}
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::vector<uint32_t>* flat, std::string* error) {
return EvaluateRuntimeSourcesImpl(program, requests, runtime, results, flat, true, error);
}
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>* flat,
std::string* error) {
if (flat == nullptr) {
if (error != nullptr) {
*error = Diagnostic(program, 0, "invalid SRT walker output");
}
return false;
}
std::vector<DescriptorValue> ignored;
return EvaluateRuntimeSources(program, {}, runtime, &ignored, flat, error);
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/SrtWalker.h
+55
View File
@@ -0,0 +1,55 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_
#include "graphics/shader/recompiler/ShaderIR.h"
#include <optional>
#include <span>
namespace Libs::Graphics::ShaderRecompiler::IR {
using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, uint32_t* value);
struct SrtRuntime {
std::span<const uint32_t> user_data;
uint64_t shader_base = 0;
SrtMemoryReader read_memory = nullptr;
void* userdata = nullptr;
std::optional<uint64_t> flat_memory_base;
};
struct DescriptorSourceRequest {
uint32_t source = 0;
uint32_t use_pc = 0;
};
bool FoldScalarConstant(const ScalarProvenance& provenance, uint32_t value, uint32_t* result);
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
// dynamic offsets remain explicit and are never assigned a fake slot.
bool BuildSrtPlan(Program* program, std::string* error);
bool EvaluateDescriptorSource(const Program& program, uint32_t source, uint32_t use_pc,
const SrtRuntime& runtime, DescriptorValue* result,
std::string* error);
// Evaluates one runtime snapshot transactionally. Scalar values and ReadConst results shared by
// several descriptors are memoized once across the batch.
bool EvaluateDescriptorSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::string* error);
// Evaluates descriptor sources and the flattened immediate SRT with one memoized scalar walk.
// On failure neither destination is changed.
bool EvaluateRuntimeSources(const Program& program,
std::span<const DescriptorSourceRequest> requests,
const SrtRuntime& runtime, std::vector<DescriptorValue>* results,
std::vector<uint32_t>* flat, std::string* error);
bool WalkSrt(const Program& program, const SrtRuntime& runtime, std::vector<uint32_t>* flat,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SRTWALKER_H_ */
src/graphics/shader/recompiler/VectorAluOps.cpp
+1833
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/VectorAluOps.h
+23
View File
@@ -0,0 +1,23 @@
#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
#include "graphics/shader/recompiler/ShaderDecoder.h"
namespace Libs::Graphics::ShaderRecompiler::Decoder {
bool DecodeVop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeVop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeVopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction* inst,
std::string* error);
bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
bool DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
Instruction* inst, std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::Decoder
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_ */
src/graphics/shader/recompiler/shaderIR/ShaderIR.cpp
+1227
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/shaderIR/ShaderIRInternal.h
+57
View File
@@ -0,0 +1,57 @@
#ifndef EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_
#define EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_
#include "graphics/shader/recompiler/ShaderIR.h"
namespace Libs::Graphics::ShaderRecompiler::IR {
void SetError(std::string* error, const char* message);
Opcode LookupIrOpcode(Decoder::Opcode opcode);
bool IsImplemented(Decoder::Opcode opcode);
bool IsReversedBinary(Decoder::Opcode opcode);
bool IsVectorCarryOutOpcode(Decoder::Opcode opcode);
bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode);
bool ScalarResultIs64Bit(Decoder::Opcode opcode);
Operand MakeSccOperand();
Operand MakeImmediateU32(uint32_t value);
Operand MakePcRelativeU32(uint32_t value);
bool AppendScalarResultSccNonZero(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error);
uint32_t VectorByteConvertIndex(Decoder::Opcode opcode);
void CopyOperandModifiers(const Decoder::Operand& decoded, Operand& operand);
bool LowerRegisterOperand(const Decoder::Operand& decoded, Operand* operand,
std::string* error);
bool LowerSourceOperand(const Decoder::Operand& decoded, Operand* operand, std::string* error);
void ApplyDppDestinationMask(const Decoder::Instruction& decoded, Operand& dst);
uint32_t ResourceIndexFromOperand(const Decoder::Operand& operand);
MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded, ResourceKind kind);
uint32_t RawScalarLoadBase(const Decoder::Operand& operand);
void ClearRegisterOffsetModifiers(Decoder::Operand& operand);
bool TryGetScalarDestinationCode(const Decoder::Operand& operand, uint32_t& code);
bool TryOffsetScalarDestination(const Decoder::Operand& operand, uint32_t index,
Decoder::Operand* result);
Decoder::Operand OffsetDecodedRegister(const Decoder::Operand& operand, uint32_t index);
MemoryInfo OffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t dword_index);
uint32_t TypedBufferComponentOffsetBytes(const Decoder::Instruction& decoded,
uint32_t component_index);
uint32_t TypedBufferFormatComponentCount(const Decoder::Instruction& decoded);
bool LowerMoveImmediateU32(uint32_t pc, const Decoder::Operand& dst, uint32_t value,
BasicBlock* block, std::string* error);
MemoryInfo OffsetBufferMemoryInfo(const Decoder::Instruction& decoded,
uint32_t component_index);
MemoryInfo ByteOffsetMemoryInfo(const Decoder::Instruction& decoded, ResourceKind kind,
uint32_t byte_offset);
ResourceKind FlatSegmentResourceKind(uint32_t segment);
const Decoder::Operand& DecodedSourceAt(const Decoder::Instruction& decoded, uint32_t index);
Decoder::Operand M0Operand();
bool IsMemoryOpcode(Decoder::Opcode opcode);
bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif // EMULATOR_SRC_GRAPHICS_SHADER_RECOMPILER_SHADERIR_SHADERIRINTERNAL_H_
src/graphics/shader/recompiler/shaderIR/ShaderIRLog.cpp
+559
View File
@@ -0,0 +1,559 @@
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include <algorithm>
#include <fmt/format.h>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
std::string RegisterToString(Register reg) {
switch (reg.file) {
case RegisterFile::Scalar: return fmt::format("s{}", reg.index);
case RegisterFile::Vector: return fmt::format("v{}", reg.index);
case RegisterFile::Vcc:
return reg.index == 0 ? std::string("vcc_lo") : std::string("vcc_hi");
case RegisterFile::Exec:
return reg.index == 0 ? std::string("exec_lo") : std::string("exec_hi");
case RegisterFile::Scc:
if (reg.index == static_cast<uint32_t>(Decoder::OperandKind::Scc)) {
return "scc";
}
if (reg.index == static_cast<uint32_t>(Decoder::OperandKind::VccZ)) {
return "vccz";
}
if (reg.index == static_cast<uint32_t>(Decoder::OperandKind::ExecZ)) {
return "execz";
}
return fmt::format("scc_like_{}", reg.index);
case RegisterFile::M0: return "m0";
default: return "unknown_reg";
}
}
std::string OperandToString(const Operand& operand) {
std::string text;
switch (operand.kind) {
case OperandKind::Register: text = RegisterToString(operand.reg); break;
case OperandKind::ImmediateU32: text = fmt::format("0x{:08x}", operand.imm); break;
case OperandKind::PcRelativeU32: text = fmt::format("pc_rel(0x{:08x})", operand.imm); break;
case OperandKind::Null: text = "null"; break;
default: text = "unknown"; break;
}
if (operand.sdwa_sel != 6 || operand.sdwa_sext) {
text += fmt::format(".sdwa(sel={},sext={})", operand.sdwa_sel, operand.sdwa_sext ? 1u : 0u);
}
if (operand.absolute) {
text += ".abs";
}
if (operand.negate) {
text += ".neg";
}
if (operand.op_sel || operand.op_sel_hi || operand.negate_hi) {
text += fmt::format(".opsel(lo={},hi={},neghi={})", operand.op_sel ? 1u : 0u,
operand.op_sel_hi ? 1u : 0u, operand.negate_hi ? 1u : 0u);
}
if (operand.omod != 0) {
text += fmt::format(".omod({})", operand.omod);
}
if (operand.clamp) {
text += ".clamp";
}
if (operand.dpp) {
text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl,
operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u);
}
return text;
}
std::string ResourceKindToString(ResourceKind kind) {
switch (kind) {
case ResourceKind::ScalarBuffer: return "scalar_buffer";
case ResourceKind::Buffer: return "buffer";
case ResourceKind::Flat: return "flat";
case ResourceKind::Global: return "global";
case ResourceKind::Scratch: return "scratch";
case ResourceKind::Lds: return "lds";
case ResourceKind::Gds: return "gds";
case ResourceKind::Image: return "image";
case ResourceKind::ImageUint: return "image_uint";
case ResourceKind::StorageImage: return "storage_image";
case ResourceKind::StorageImageUint: return "storage_image_uint";
case ResourceKind::Sampler: return "sampler";
default: return "none";
}
}
const char* ImageDimensionToString(Decoder::ImageDimension dimension) {
switch (dimension) {
case Decoder::ImageDimension::Dim2D: return "2d";
case Decoder::ImageDimension::Dim3D: return "3d";
case Decoder::ImageDimension::Dim2DArray: return "2d_array";
default: return "unknown";
}
}
std::string MemoryInfoToString(const MemoryInfo& mem) {
if (mem.kind == ResourceKind::None) {
return "";
}
std::string text = fmt::format(
" ; {} resource={} sampler={} offset={} offset2={} dmask=0x{:x} data_dwords={} "
"data_bits={} component={}/{} dfmt={} nfmt={} signed={} typed={} formatted={} segment={} "
"glc={} slc={} idxen={} offen={} image_flags=0x{:x} image_dim={} image_addr={} "
"image_mip={}",
ResourceKindToString(mem.kind).c_str(), mem.resource, mem.sampler, mem.offset,
mem.secondary_offset, mem.dmask, mem.data_dwords, mem.data_bits, mem.component_index,
mem.component_count, mem.data_format, mem.number_format, mem.data_signed ? 1u : 0u,
mem.typed ? 1u : 0u, mem.formatted ? 1u : 0u, mem.memory_segment, mem.glc ? 1u : 0u,
mem.slc ? 1u : 0u, mem.idxen ? 1u : 0u, mem.offen ? 1u : 0u, mem.image_sample_flags,
ImageDimensionToString(mem.image_dimension), mem.image_address_components,
mem.image_has_mip ? 1u : 0u);
if (mem.image_nsa_dwords != 0) {
text += fmt::format(" image_nsa_dwords={} image_nsa_addr=", mem.image_nsa_dwords);
const auto count =
std::min<uint32_t>(mem.image_nsa_dwords * 4u, Decoder::MaxImageNsaAddressComponents);
for (uint32_t i = 0; i < count; i++) {
if (i != 0) {
text += ",";
}
text += fmt::format("{}", mem.image_nsa_addr[i]);
}
}
return text;
}
std::string ExportInfoToString(const ExportInfo& exp) {
return fmt::format(" ; target={} index={} raw=0x{:02x} en=0x{:x} done={} compr={} vm={}",
ExportTargetKindToString(exp.kind).c_str(), exp.index, exp.target, exp.en,
exp.done ? 1u : 0u, exp.compr ? 1u : 0u, exp.vm ? 1u : 0u);
}
std::string InputInfoToString(Opcode op, const InputInfo& input) {
if (op != Opcode::LoadInputF32) {
return "";
}
return fmt::format(" ; input_attr={} input_chan={}", input.attr, input.chan);
}
std::string TerminatorToString(const CFG::Terminator& term) {
return fmt::format(
"term={} cond={} true={} false={} merge={} continue={} loop={} indirect_sgpr={} "
"indirect_selector={} indirect_targets={} selector_values={}",
static_cast<uint32_t>(term.kind), CFG::BranchConditionToString(term.condition).c_str(),
term.true_block, term.false_block, term.merge_block, term.continue_block,
term.loop_header ? 1u : 0u, term.indirect_pc_sgpr, term.indirect_selector_code,
static_cast<uint32_t>(term.indirect_targets.size()),
static_cast<uint32_t>(term.indirect_selector_values.size()));
}
std::string ExportSourcesToString(const Instruction& inst) {
std::string text;
for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
text += i == 0 ? " " : ", ";
text += OperandToString(inst.src[i]);
}
return text;
}
std::string InstructionToString(const Instruction& inst) {
std::string text = fmt::format("0x{:08x}: ", inst.pc);
if (inst.op == Opcode::Export) {
text += "Export";
text += ExportSourcesToString(inst);
text += ExportInfoToString(inst.export_info);
return text;
}
switch (inst.op) {
case Opcode::ControlNop: text += "ControlNop"; break;
case Opcode::Waitcnt: text += "Waitcnt"; break;
case Opcode::Barrier: text += "Barrier"; break;
case Opcode::Sendmsg: text += "Sendmsg"; break;
case Opcode::TtraceData: text += "TtraceData"; break;
case Opcode::InstPrefetch: text += "InstPrefetch"; break;
case Opcode::MoveU32: text += "MoveU32"; break;
case Opcode::MoveF32Bits: text += "MoveF32Bits"; break;
case Opcode::MoveRelDestU32: text += "MoveRelDestU32"; break;
case Opcode::MoveRelSourceU32: text += "MoveRelSourceU32"; break;
case Opcode::MoveU64: text += "MoveU64"; break;
case Opcode::WqmB64: text += "WqmB64"; break;
case Opcode::SaveexecB32: text += "SaveexecB32"; break;
case Opcode::SaveexecB64: text += "SaveexecB64"; break;
case Opcode::ReadFirstLaneU32: text += "ReadFirstLaneU32"; break;
case Opcode::ReadLaneU32: text += "ReadLaneU32"; break;
case Opcode::WriteLaneU32: text += "WriteLaneU32"; break;
case Opcode::Permlane16B32: text += "Permlane16B32"; break;
case Opcode::Permlanex16B32: text += "Permlanex16B32"; break;
case Opcode::AbsI32: text += "AbsI32"; break;
case Opcode::IAddU32: text += "IAddU32"; break;
case Opcode::IAddCarryU32: text += "IAddCarryU32"; break;
case Opcode::ISubBorrowU32: text += "ISubBorrowU32"; break;
case Opcode::ScalarAddCarryU32: text += "ScalarAddCarryU32"; break;
case Opcode::ScalarSubBorrowU32: text += "ScalarSubBorrowU32"; break;
case Opcode::ScalarSubBorrowCarryU32: text += "ScalarSubBorrowCarryU32"; break;
case Opcode::ScalarSignedAddOverflowI32: text += "ScalarSignedAddOverflowI32"; break;
case Opcode::ScalarSignedSubOverflowI32: text += "ScalarSignedSubOverflowI32"; break;
case Opcode::ScalarShiftLeftAddCarryU32: text += "ScalarShiftLeftAddCarryU32"; break;
case Opcode::ISubU32: text += "ISubU32"; break;
case Opcode::IMulU32: text += "IMulU32"; break;
case Opcode::UMulHighU32: text += "UMulHighU32"; break;
case Opcode::SMulHighI32: text += "SMulHighI32"; break;
case Opcode::IMadI24U32: text += "IMadI24U32"; break;
case Opcode::UMadU24U32: text += "UMadU24U32"; break;
case Opcode::UMadU64U32: text += "UMadU64U32"; break;
case Opcode::SadU32: text += "SadU32"; break;
case Opcode::IAdd3U32: text += "IAdd3U32"; break;
case Opcode::IMulI24U32: text += "IMulI24U32"; break;
case Opcode::UMulU24U32: text += "UMulU24U32"; break;
case Opcode::IMinI32: text += "IMinI32"; break;
case Opcode::IMaxI32: text += "IMaxI32"; break;
case Opcode::IMin3I32: text += "IMin3I32"; break;
case Opcode::IMax3I32: text += "IMax3I32"; break;
case Opcode::IMed3I32: text += "IMed3I32"; break;
case Opcode::UMinU32: text += "UMinU32"; break;
case Opcode::UMaxU32: text += "UMaxU32"; break;
case Opcode::UMin3U32: text += "UMin3U32"; break;
case Opcode::UMax3U32: text += "UMax3U32"; break;
case Opcode::UMed3U32: text += "UMed3U32"; break;
case Opcode::BitwiseAndU32: text += "BitwiseAndU32"; break;
case Opcode::BitwiseAndU64: text += "BitwiseAndU64"; break;
case Opcode::BitwiseAndNotU32: text += "BitwiseAndNotU32"; break;
case Opcode::BitwiseAndNotU64: text += "BitwiseAndNotU64"; break;
case Opcode::BitwiseOrU32: text += "BitwiseOrU32"; break;
case Opcode::BitwiseOrU64: text += "BitwiseOrU64"; break;
case Opcode::BitwiseOrNotU32: text += "BitwiseOrNotU32"; break;
case Opcode::BitwiseOrNotU64: text += "BitwiseOrNotU64"; break;
case Opcode::BitwiseAndOrU32: text += "BitwiseAndOrU32"; break;
case Opcode::BitwiseOr3U32: text += "BitwiseOr3U32"; break;
case Opcode::BitwiseXorU32: text += "BitwiseXorU32"; break;
case Opcode::BitwiseXorU64: text += "BitwiseXorU64"; break;
case Opcode::BitwiseXor3U32: text += "BitwiseXor3U32"; break;
case Opcode::BitwiseNandU32: text += "BitwiseNandU32"; break;
case Opcode::BitwiseNandU64: text += "BitwiseNandU64"; break;
case Opcode::BitwiseNorU32: text += "BitwiseNorU32"; break;
case Opcode::BitwiseNorU64: text += "BitwiseNorU64"; break;
case Opcode::BitwiseXnorU32: text += "BitwiseXnorU32"; break;
case Opcode::BitwiseXnorU64: text += "BitwiseXnorU64"; break;
case Opcode::BitwiseNotU32: text += "BitwiseNotU32"; break;
case Opcode::BitwiseNotU64: text += "BitwiseNotU64"; break;
case Opcode::BitClearU32: text += "BitClearU32"; break;
case Opcode::BitSetU32: text += "BitSetU32"; break;
case Opcode::BitReverseU32: text += "BitReverseU32"; break;
case Opcode::BitCountU32: text += "BitCountU32"; break;
case Opcode::BitCountU64: text += "BitCountU64"; break;
case Opcode::BitReplicateB64B32: text += "BitReplicateB64B32"; break;
case Opcode::BitCountAddU32: text += "BitCountAddU32"; break;
case Opcode::MaskedBitCountLowU32: text += "MaskedBitCountLowU32"; break;
case Opcode::MaskedBitCountHighU32: text += "MaskedBitCountHighU32"; break;
case Opcode::FindLsbU32: text += "FindLsbU32"; break;
case Opcode::FindMsbFromHighU32: text += "FindMsbFromHighU32"; break;
case Opcode::FindMsbFromHighU64: text += "FindMsbFromHighU64"; break;
case Opcode::BitFieldMaskU32: text += "BitFieldMaskU32"; break;
case Opcode::BitFieldMaskU64: text += "BitFieldMaskU64"; break;
case Opcode::BitFieldExtractU32: text += "BitFieldExtractU32"; break;
case Opcode::BitFieldExtractU64: text += "BitFieldExtractU64"; break;
case Opcode::BitFieldExtract3U32: text += "BitFieldExtract3U32"; break;
case Opcode::BitFieldExtract3I32: text += "BitFieldExtract3I32"; break;
case Opcode::BitFieldInsertSelectU32: text += "BitFieldInsertSelectU32"; break;
case Opcode::BitCompare0B32: text += "BitCompare0B32"; break;
case Opcode::BitCompare1B32: text += "BitCompare1B32"; break;
case Opcode::AlignBitU32: text += "AlignBitU32"; break;
case Opcode::ShiftLeftAddU32: text += "ShiftLeftAddU32"; break;
case Opcode::AddShiftLeftU32: text += "AddShiftLeftU32"; break;
case Opcode::XorAddU32: text += "XorAddU32"; break;
case Opcode::ShiftLeftOrU32: text += "ShiftLeftOrU32"; break;
case Opcode::ShiftLeftLogicalU32: text += "ShiftLeftLogicalU32"; break;
case Opcode::ShiftLeftLogicalU64: text += "ShiftLeftLogicalU64"; break;
case Opcode::ShiftLeftLogicalU16: text += "ShiftLeftLogicalU16"; break;
case Opcode::ShiftRightLogicalU32: text += "ShiftRightLogicalU32"; break;
case Opcode::ShiftRightLogicalU64: text += "ShiftRightLogicalU64"; break;
case Opcode::ShiftRightLogicalU16: text += "ShiftRightLogicalU16"; break;
case Opcode::ShiftRightArithmeticI32: text += "ShiftRightArithmeticI32"; break;
case Opcode::ShiftRightArithmeticI16: text += "ShiftRightArithmeticI16"; break;
case Opcode::SelectU32: text += "SelectU32"; break;
case Opcode::SelectMaskU32: text += "SelectMaskU32"; break;
case Opcode::SelectF32Bits: text += "SelectF32Bits"; break;
case Opcode::SelectMaskF32Bits: text += "SelectMaskF32Bits"; break;
case Opcode::SelectU64: text += "SelectU64"; break;
case Opcode::PackLowLowU16: text += "PackLowLowU16"; break;
case Opcode::PackLowHighU16: text += "PackLowHighU16"; break;
case Opcode::PackHighHighU16: text += "PackHighHighU16"; break;
case Opcode::CompareFalse: text += "CompareFalse"; break;
case Opcode::CompareTrue: text += "CompareTrue"; break;
case Opcode::CompareEqU32: text += "CompareEqU32"; break;
case Opcode::CompareNeU32: text += "CompareNeU32"; break;
case Opcode::CompareGtU32: text += "CompareGtU32"; break;
case Opcode::CompareGeU32: text += "CompareGeU32"; break;
case Opcode::CompareLtU32: text += "CompareLtU32"; break;
case Opcode::CompareLeU32: text += "CompareLeU32"; break;
case Opcode::CompareNeU64: text += "CompareNeU64"; break;
case Opcode::CompareMaskEqU32: text += "CompareMaskEqU32"; break;
case Opcode::CompareMaskNeU32: text += "CompareMaskNeU32"; break;
case Opcode::CompareMaskGtU32: text += "CompareMaskGtU32"; break;
case Opcode::CompareMaskGeU32: text += "CompareMaskGeU32"; break;
case Opcode::CompareMaskLtU32: text += "CompareMaskLtU32"; break;
case Opcode::CompareMaskLeU32: text += "CompareMaskLeU32"; break;
case Opcode::CompareEqI32: text += "CompareEqI32"; break;
case Opcode::CompareNeI32: text += "CompareNeI32"; break;
case Opcode::CompareGtI32: text += "CompareGtI32"; break;
case Opcode::CompareGeI32: text += "CompareGeI32"; break;
case Opcode::CompareLtI32: text += "CompareLtI32"; break;
case Opcode::CompareLeI32: text += "CompareLeI32"; break;
case Opcode::CompareEqI16: text += "CompareEqI16"; break;
case Opcode::CompareNeI16: text += "CompareNeI16"; break;
case Opcode::CompareGtI16: text += "CompareGtI16"; break;
case Opcode::CompareGeI16: text += "CompareGeI16"; break;
case Opcode::CompareLtI16: text += "CompareLtI16"; break;
case Opcode::CompareLeI16: text += "CompareLeI16"; break;
case Opcode::CompareMaskEqI32: text += "CompareMaskEqI32"; break;
case Opcode::CompareMaskNeI32: text += "CompareMaskNeI32"; break;
case Opcode::CompareMaskGtI32: text += "CompareMaskGtI32"; break;
case Opcode::CompareMaskGeI32: text += "CompareMaskGeI32"; break;
case Opcode::CompareMaskLtI32: text += "CompareMaskLtI32"; break;
case Opcode::CompareMaskLeI32: text += "CompareMaskLeI32"; break;
case Opcode::CompareEqU16: text += "CompareEqU16"; break;
case Opcode::CompareNeU16: text += "CompareNeU16"; break;
case Opcode::CompareGtU16: text += "CompareGtU16"; break;
case Opcode::CompareGeU16: text += "CompareGeU16"; break;
case Opcode::CompareLtU16: text += "CompareLtU16"; break;
case Opcode::CompareLeU16: text += "CompareLeU16"; break;
case Opcode::CompareEqF32: text += "CompareEqF32"; break;
case Opcode::CompareNeF32: text += "CompareNeF32"; break;
case Opcode::CompareGtF32: text += "CompareGtF32"; break;
case Opcode::CompareGeF32: text += "CompareGeF32"; break;
case Opcode::CompareLtF32: text += "CompareLtF32"; break;
case Opcode::CompareLeF32: text += "CompareLeF32"; break;
case Opcode::CompareOrderedF32: text += "CompareOrderedF32"; break;
case Opcode::CompareUnorderedF32: text += "CompareUnorderedF32"; break;
case Opcode::CompareUnordEqF32: text += "CompareUnordEqF32"; break;
case Opcode::CompareUnordNeF32: text += "CompareUnordNeF32"; break;
case Opcode::CompareUnordGtF32: text += "CompareUnordGtF32"; break;
case Opcode::CompareUnordGeF32: text += "CompareUnordGeF32"; break;
case Opcode::CompareUnordLtF32: text += "CompareUnordLtF32"; break;
case Opcode::CompareUnordLeF32: text += "CompareUnordLeF32"; break;
case Opcode::CompareClassF32: text += "CompareClassF32"; break;
case Opcode::CompareEqF16: text += "CompareEqF16"; break;
case Opcode::CompareNeF16: text += "CompareNeF16"; break;
case Opcode::CompareGtF16: text += "CompareGtF16"; break;
case Opcode::CompareGeF16: text += "CompareGeF16"; break;
case Opcode::CompareLtF16: text += "CompareLtF16"; break;
case Opcode::CompareLeF16: text += "CompareLeF16"; break;
case Opcode::CompareUnordNeF16: text += "CompareUnordNeF16"; break;
case Opcode::CompareMaskEqF16: text += "CompareMaskEqF16"; break;
case Opcode::CompareMaskNeF16: text += "CompareMaskNeF16"; break;
case Opcode::CompareMaskGtF16: text += "CompareMaskGtF16"; break;
case Opcode::CompareMaskGeF16: text += "CompareMaskGeF16"; break;
case Opcode::CompareMaskLtF16: text += "CompareMaskLtF16"; break;
case Opcode::CompareMaskLeF16: text += "CompareMaskLeF16"; break;
case Opcode::CompareMaskUnordNeF16: text += "CompareMaskUnordNeF16"; break;
case Opcode::CompareMaskUnordGeF16: text += "CompareMaskUnordGeF16"; break;
case Opcode::CompareMaskEqF32: text += "CompareMaskEqF32"; break;
case Opcode::CompareMaskNeF32: text += "CompareMaskNeF32"; break;
case Opcode::CompareMaskGtF32: text += "CompareMaskGtF32"; break;
case Opcode::CompareMaskGeF32: text += "CompareMaskGeF32"; break;
case Opcode::CompareMaskLtF32: text += "CompareMaskLtF32"; break;
case Opcode::CompareMaskLeF32: text += "CompareMaskLeF32"; break;
case Opcode::CompareMaskUnordEqF32: text += "CompareMaskUnordEqF32"; break;
case Opcode::CompareMaskUnordNeF32: text += "CompareMaskUnordNeF32"; break;
case Opcode::CompareMaskUnordGtF32: text += "CompareMaskUnordGtF32"; break;
case Opcode::CompareMaskUnordGeF32: text += "CompareMaskUnordGeF32"; break;
case Opcode::CompareMaskUnordLtF32: text += "CompareMaskUnordLtF32"; break;
case Opcode::CompareMaskUnordLeF32: text += "CompareMaskUnordLeF32"; break;
case Opcode::ConvertByteU32ToF32: text += "ConvertByteU32ToF32"; break;
case Opcode::ConvertU32ToF32: text += "ConvertU32ToF32"; break;
case Opcode::ConvertI32ToF32: text += "ConvertI32ToF32"; break;
case Opcode::ConvertF32ToU32: text += "ConvertF32ToU32"; break;
case Opcode::ConvertF32ToI32: text += "ConvertF32ToI32"; break;
case Opcode::ConvertF32ToF16: text += "ConvertF32ToF16"; break;
case Opcode::ConvertF16ToF32: text += "ConvertF16ToF32"; break;
case Opcode::ConvertU16ToF16: text += "ConvertU16ToF16"; break;
case Opcode::ConvertF16ToU16: text += "ConvertF16ToU16"; break;
case Opcode::ConvertI16ToF16: text += "ConvertI16ToF16"; break;
case Opcode::ConvertF16ToI16: text += "ConvertF16ToI16"; break;
case Opcode::ConvertRoundPlusInfF32ToI32: text += "ConvertRoundPlusInfF32ToI32"; break;
case Opcode::ConvertFloorF32ToI32: text += "ConvertFloorF32ToI32"; break;
case Opcode::ConvertI4ToOffsetF32: text += "ConvertI4ToOffsetF32"; break;
case Opcode::LdexpF32: text += "LdexpF32"; break;
case Opcode::PackF32ToF16Rtz: text += "PackF32ToF16Rtz"; break;
case Opcode::PackSnorm2x16F32: text += "PackSnorm2x16F32"; break;
case Opcode::PackUnorm2x16F32: text += "PackUnorm2x16F32"; break;
case Opcode::PackU16U32: text += "PackU16U32"; break;
case Opcode::PackU8F32: text += "PackU8F32"; break;
case Opcode::PackB32F16: text += "PackB32F16"; break;
case Opcode::PackedMadI16: text += "PackedMadI16"; break;
case Opcode::PackedMulLoU16: text += "PackedMulLoU16"; break;
case Opcode::PackedAddI16: text += "PackedAddI16"; break;
case Opcode::PackedSubI16: text += "PackedSubI16"; break;
case Opcode::PackedLshlrevB16: text += "PackedLshlrevB16"; break;
case Opcode::PackedLshrrevB16: text += "PackedLshrrevB16"; break;
case Opcode::PackedAshrrevI16: text += "PackedAshrrevI16"; break;
case Opcode::PackedMaxI16: text += "PackedMaxI16"; break;
case Opcode::PackedMinI16: text += "PackedMinI16"; break;
case Opcode::PackedMadU16: text += "PackedMadU16"; break;
case Opcode::PackedAddU16: text += "PackedAddU16"; break;
case Opcode::PackedSubU16: text += "PackedSubU16"; break;
case Opcode::PackedMaxU16: text += "PackedMaxU16"; break;
case Opcode::PackedMinU16: text += "PackedMinU16"; break;
case Opcode::PackedAddF16: text += "PackedAddF16"; break;
case Opcode::PackedMulF16: text += "PackedMulF16"; break;
case Opcode::PackedMinF16: text += "PackedMinF16"; break;
case Opcode::PackedMaxF16: text += "PackedMaxF16"; break;
case Opcode::PackedFmaF16: text += "PackedFmaF16"; break;
case Opcode::AddF16: text += "AddF16"; break;
case Opcode::SubF16: text += "SubF16"; break;
case Opcode::MulF16: text += "MulF16"; break;
case Opcode::MinF16: text += "MinF16"; break;
case Opcode::MaxF16: text += "MaxF16"; break;
case Opcode::FmaF16: text += "FmaF16"; break;
case Opcode::MadMixF16: text += "MadMixF16"; break;
case Opcode::IAddU16: text += "IAddU16"; break;
case Opcode::ISubI16: text += "ISubI16"; break;
case Opcode::IMinI16: text += "IMinI16"; break;
case Opcode::IMaxI16: text += "IMaxI16"; break;
case Opcode::UMinU16: text += "UMinU16"; break;
case Opcode::UMaxU16: text += "UMaxU16"; break;
case Opcode::RcpF32: text += "RcpF32"; break;
case Opcode::FractF32: text += "FractF32"; break;
case Opcode::TruncF32: text += "TruncF32"; break;
case Opcode::CeilF32: text += "CeilF32"; break;
case Opcode::RoundEvenF32: text += "RoundEvenF32"; break;
case Opcode::FloorF32: text += "FloorF32"; break;
case Opcode::Exp2F32: text += "Exp2F32"; break;
case Opcode::Log2F32: text += "Log2F32"; break;
case Opcode::InverseSqrtF32: text += "InverseSqrtF32"; break;
case Opcode::SqrtF32: text += "SqrtF32"; break;
case Opcode::RcpF16: text += "RcpF16"; break;
case Opcode::SqrtF16: text += "SqrtF16"; break;
case Opcode::InverseSqrtF16: text += "InverseSqrtF16"; break;
case Opcode::Log2F16: text += "Log2F16"; break;
case Opcode::Exp2F16: text += "Exp2F16"; break;
case Opcode::FloorF16: text += "FloorF16"; break;
case Opcode::CeilF16: text += "CeilF16"; break;
case Opcode::TruncF16: text += "TruncF16"; break;
case Opcode::RoundEvenF16: text += "RoundEvenF16"; break;
case Opcode::SinF32: text += "SinF32"; break;
case Opcode::CosF32: text += "CosF32"; break;
case Opcode::CubeIdF32: text += "CubeIdF32"; break;
case Opcode::CubeScF32: text += "CubeScF32"; break;
case Opcode::CubeTcF32: text += "CubeTcF32"; break;
case Opcode::CubeMaF32: text += "CubeMaF32"; break;
case Opcode::FAddF32: text += "FAddF32"; break;
case Opcode::FSubF32: text += "FSubF32"; break;
case Opcode::FMulF32: text += "FMulF32"; break;
case Opcode::FMinF32: text += "FMinF32"; break;
case Opcode::FMaxF32: text += "FMaxF32"; break;
case Opcode::FMadF32: text += "FMadF32"; break;
case Opcode::Dot2AccF32F16: text += "Dot2AccF32F16"; break;
case Opcode::FMin3F32: text += "FMin3F32"; break;
case Opcode::FMax3F32: text += "FMax3F32"; break;
case Opcode::FMed3F32: text += "FMed3F32"; break;
case Opcode::Min3F16: text += "Min3F16"; break;
case Opcode::Max3F16: text += "Max3F16"; break;
case Opcode::Med3F16: text += "Med3F16"; break;
case Opcode::LoadSrtDword: text += "LoadSrtDword"; break;
case Opcode::SLoadDword: text += "SLoadDword"; break;
case Opcode::SBufferLoadDword: text += "SBufferLoadDword"; break;
case Opcode::BufferLoadUbyte: text += "BufferLoadUbyte"; break;
case Opcode::BufferLoadSbyte: text += "BufferLoadSbyte"; break;
case Opcode::BufferLoadUshort: text += "BufferLoadUshort"; break;
case Opcode::BufferLoadSshort: text += "BufferLoadSshort"; break;
case Opcode::BufferLoadDword: text += "BufferLoadDword"; break;
case Opcode::BufferStoreByte: text += "BufferStoreByte"; break;
case Opcode::BufferStoreShort: text += "BufferStoreShort"; break;
case Opcode::BufferStoreDword: text += "BufferStoreDword"; break;
case Opcode::AtomicSwapU32: text += "AtomicSwapU32"; break;
case Opcode::AtomicAddU32: text += "AtomicAddU32"; break;
case Opcode::AtomicSubU32: text += "AtomicSubU32"; break;
case Opcode::AtomicSMinI32: text += "AtomicSMinI32"; break;
case Opcode::AtomicUMinU32: text += "AtomicUMinU32"; break;
case Opcode::AtomicSMaxI32: text += "AtomicSMaxI32"; break;
case Opcode::AtomicUMaxU32: text += "AtomicUMaxU32"; break;
case Opcode::AtomicAndU32: text += "AtomicAndU32"; break;
case Opcode::AtomicOrU32: text += "AtomicOrU32"; break;
case Opcode::AtomicXorU32: text += "AtomicXorU32"; break;
case Opcode::FlatLoadUbyte: text += "FlatLoadUbyte"; break;
case Opcode::FlatLoadSbyte: text += "FlatLoadSbyte"; break;
case Opcode::FlatLoadUshort: text += "FlatLoadUshort"; break;
case Opcode::FlatLoadSshort: text += "FlatLoadSshort"; break;
case Opcode::FlatLoadDword: text += "FlatLoadDword"; break;
case Opcode::FlatStoreByte: text += "FlatStoreByte"; break;
case Opcode::FlatStoreShort: text += "FlatStoreShort"; break;
case Opcode::FlatStoreDword: text += "FlatStoreDword"; break;
case Opcode::DsReadUbyte: text += "DsReadUbyte"; break;
case Opcode::DsReadSbyte: text += "DsReadSbyte"; break;
case Opcode::DsReadUshort: text += "DsReadUshort"; break;
case Opcode::DsReadSshort: text += "DsReadSshort"; break;
case Opcode::DsReadB32: text += "DsReadB32"; break;
case Opcode::DsWriteByte: text += "DsWriteByte"; break;
case Opcode::DsWriteShort: text += "DsWriteShort"; break;
case Opcode::DsWriteB32: text += "DsWriteB32"; break;
case Opcode::DsMinF32: text += "DsMinF32"; break;
case Opcode::DsMaxF32: text += "DsMaxF32"; break;
case Opcode::DsSwizzleB32: text += "DsSwizzleB32"; break;
case Opcode::DsConsume: text += "DsConsume"; break;
case Opcode::DsAppend: text += "DsAppend"; break;
case Opcode::DsWriteAddtidB32: text += "DsWriteAddtidB32"; break;
case Opcode::DsReadAddtidB32: text += "DsReadAddtidB32"; break;
case Opcode::ImageGetResinfo: text += "ImageGetResinfo"; break;
case Opcode::ImageGetLod: text += "ImageGetLod"; break;
case Opcode::ImageLoad: text += "ImageLoad"; break;
case Opcode::ImageStore: text += "ImageStore"; break;
case Opcode::ImageSample: text += "ImageSample"; break;
case Opcode::ImageGather4: text += "ImageGather4"; break;
case Opcode::LoadInputF32: text += "LoadInputF32"; break;
case Opcode::Export: text += "Export"; break;
}
text += " ";
text += OperandToString(inst.dst);
if (inst.dst2.kind != OperandKind::Null) {
text += ", ";
text += OperandToString(inst.dst2);
}
for (uint32_t i = 0; i < inst.src_count; i++) {
text += i == 0 ? ", " : ", ";
text += OperandToString(inst.src[i]);
}
text += MemoryInfoToString(inst.memory);
text += InputInfoToString(inst.op, inst.input_info);
return text;
}
std::string ExportTargetKindToString(ExportTargetKind kind) {
switch (kind) {
case ExportTargetKind::Null: return "null";
case ExportTargetKind::Position: return "position";
case ExportTargetKind::Primitive: return "primitive";
case ExportTargetKind::Parameter: return "parameter";
case ExportTargetKind::Mrt: return "mrt";
case ExportTargetKind::MrtZ: return "mrtz";
default: return "unknown";
}
}
std::string ProgramToString(const Program& program) {
std::string text;
text += fmt::format("mode={} masks={} fallback={} reason={}\n",
program.dispatcher_fallback ? "dispatcher" : "structured",
program.lane_mask_mode == ShaderLaneMaskMode::NativeWave ? "native-wave"
: "per-invocation",
CFG::FailureKindToString(program.cfg_failure_kind).c_str(),
program.fallback_reason.c_str());
for (const auto& block: program.blocks) {
text += fmt::format("block_{} pc=0x{:08x} end=0x{:08x} preds={} succs={} {}\n", block.id,
block.start_pc, block.end_pc,
static_cast<uint32_t>(block.predecessors.size()),
static_cast<uint32_t>(block.successors.size()),
TerminatorToString(block.terminator).c_str());
for (const auto& inst: block.instructions) {
text += " ";
text += InstructionToString(inst);
text += "\n";
}
}
return text;
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/shaderIR/ShaderIRMemory.cpp
+788
View File
@@ -0,0 +1,788 @@
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include <algorithm>
#include <fmt/format.h>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
bool LowerScalarBufferLoadDword(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
std::string* error) {
for (uint32_t i = 0; i < decoded.data_dwords; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.src_count = 1;
inst.memory = OffsetMemoryInfo(decoded, ResourceKind::ScalarBuffer, i);
// Raw s_load uses a two-SGPR pointer base; s_buffer_load uses a four-SGPR
// descriptor resource index.
inst.memory.resource = op == Opcode::SLoadDword ? RawScalarLoadBase(decoded.src0)
: ResourceIndexFromOperand(decoded.src0);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) ||
!LowerSourceOperand(decoded.src1, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerBufferAddressSources(const Decoder::Instruction& decoded, Instruction* inst,
uint32_t first_src, std::string* error) {
uint32_t src = first_src;
if (decoded.idxen) {
if (!LowerSourceOperand(decoded.src0, &inst->src[src++], error)) {
return false;
}
}
if (decoded.offen) {
const auto offset_source =
decoded.idxen ? OffsetDecodedRegister(decoded.src0, 1) : decoded.src0;
if (!LowerSourceOperand(offset_source, &inst->src[src++], error)) {
return false;
}
}
if (!LowerSourceOperand(decoded.src2, &inst->src[src++], error)) {
return false;
}
inst->src_count = src;
return true;
}
bool LowerBufferLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
const auto ir_op =
decoded.data_bits == 8u
? (decoded.data_signed ? Opcode::BufferLoadSbyte : Opcode::BufferLoadUbyte)
: decoded.data_bits == 16u
? (decoded.data_signed ? Opcode::BufferLoadSshort : Opcode::BufferLoadUshort)
: Opcode::BufferLoadDword;
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
for (uint32_t i = 0; i < count; i++) {
if (i >= typed_format_components) {
if (!LowerMoveImmediateU32(decoded.pc, OffsetDecodedRegister(decoded.dst, i), 0, block,
error)) {
return false;
}
continue;
}
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) ||
!LowerBufferAddressSources(decoded, &inst, 0, error)) {
return false;
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerBufferStore(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
const auto ir_op = decoded.data_bits == 8u ? Opcode::BufferStoreByte
: decoded.data_bits == 16u ? Opcode::BufferStoreShort
: Opcode::BufferStoreDword;
auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
if (decoded.data_bits == 32u) {
const auto typed_format_components = TypedBufferFormatComponentCount(decoded);
if (typed_format_components < count) {
count = typed_format_components;
}
}
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.memory = OffsetBufferMemoryInfo(decoded, i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, i), &inst.src[0], error) ||
!LowerBufferAddressSources(decoded, &inst, 1, error)) {
return false;
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerBufferAtomicDword(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Buffer);
inst.dst.kind = OperandKind::Null;
if ((decoded.glc && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) ||
!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerBufferAddressSources(decoded, &inst, 1, error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
ResourceKind DsMemoryKind(const Decoder::Instruction& decoded) {
return decoded.gds ? ResourceKind::Gds : ResourceKind::Lds;
}
bool LowerDsRead(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
const auto ir_op = decoded.data_bits == 8u
? (decoded.data_signed ? Opcode::DsReadSbyte : Opcode::DsReadUbyte)
: decoded.data_bits == 16u
? (decoded.data_signed ? Opcode::DsReadSshort : Opcode::DsReadUshort)
: Opcode::DsReadB32;
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = 1;
inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, i), &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerDsRead2(const Decoder::Instruction& decoded, BasicBlock* block, uint32_t dwords_per_read,
std::string* error) {
const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
for (uint32_t read = 0; read < 2u; read++) {
for (uint32_t dword = 0; dword < dwords_per_read; dword++) {
const auto index = read * dwords_per_read + dword;
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsReadB32;
inst.src_count = 1;
inst.memory =
ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[read] + dword * 4u);
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, index), &inst.dst,
error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
}
}
return true;
}
bool LowerDsAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
bool return_old, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.dst.kind = return_old ? OperandKind::Register : OperandKind::Null;
if ((return_old && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) ||
!LowerSourceOperand(decoded.src1, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
bool LowerDsSwizzleB32(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsSwizzleB32;
inst.src_count = 2;
inst.src[1].kind = OperandKind::ImmediateU32;
inst.src[1].imm = decoded.offset & 0xffffu;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
bool LowerDsFloatMinMaxF32(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.resource = 0;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.src1, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error) ||
!LowerSourceOperand(decoded.src2, &inst.src[2], error)) {
return false;
}
inst.src_count = 3;
block->instructions.push_back(inst);
return true;
}
bool LowerDsWriteAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsWriteAddtidB32;
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.resource = 0;
inst.dst.kind = OperandKind::Null;
const auto m0 = M0Operand();
if (!LowerSourceOperand(decoded.src1, &inst.src[0], error) ||
!LowerSourceOperand(m0, &inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
bool LowerDsReadAddtidB32(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsReadAddtidB32;
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
inst.memory.resource = 0;
const auto m0 = M0Operand();
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(m0, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
bool LowerDsAppendConsume(const Decoder::Instruction& decoded, BasicBlock* block, Opcode op,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = op;
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, DsMemoryKind(decoded));
const auto m0 = M0Operand();
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(m0, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
Opcode ImageAtomicIrOpcode(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::ImageAtomicAdd: return Opcode::AtomicAddU32;
case Decoder::Opcode::ImageAtomicUMin: return Opcode::AtomicUMinU32;
case Decoder::Opcode::ImageAtomicUMax: return Opcode::AtomicUMaxU32;
case Decoder::Opcode::ImageAtomicAnd: return Opcode::AtomicAndU32;
case Decoder::Opcode::ImageAtomicOr: return Opcode::AtomicOrU32;
case Decoder::Opcode::ImageAtomicXor: return Opcode::AtomicXorU32;
default: return Opcode::AtomicAddU32;
}
}
bool LowerImageAtomicU32(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ImageAtomicIrOpcode(decoded.opcode);
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImageUint);
inst.memory.sampler = 0;
inst.dst.kind = OperandKind::Null;
if ((decoded.glc && !LowerRegisterOperand(decoded.dst, &inst.dst, error)) ||
!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
bool LowerFlatLoad(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
const auto kind = FlatSegmentResourceKind(decoded.memory_segment);
const auto ir_op = decoded.data_bits == 8u
? (decoded.data_signed ? Opcode::FlatLoadSbyte : Opcode::FlatLoadUbyte)
: decoded.data_bits == 16u
? (decoded.data_signed ? Opcode::FlatLoadSshort : Opcode::FlatLoadUshort)
: Opcode::FlatLoadDword;
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
if (decoded.data_bits == 32u && count > 1u) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = decoded.src_count;
inst.memory = MemoryInfoFromDecoded(decoded, kind);
inst.memory.resource = 0;
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error)) {
return false;
}
for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i], error)) {
return false;
}
}
block->instructions.push_back(inst);
return true;
}
for (uint32_t dword = 0; dword < count; dword++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = decoded.src_count;
inst.memory = OffsetMemoryInfo(decoded, kind, dword);
inst.memory.resource = 0;
if (!LowerRegisterOperand(OffsetDecodedRegister(decoded.dst, dword), &inst.dst, error)) {
return false;
}
for (uint32_t i = 0; i < decoded.src_count && i < 3u; i++) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i], error)) {
return false;
}
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerFlatStore(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
const auto kind = FlatSegmentResourceKind(decoded.memory_segment);
const auto ir_op = decoded.data_bits == 8u ? Opcode::FlatStoreByte
: decoded.data_bits == 16u ? Opcode::FlatStoreShort
: Opcode::FlatStoreDword;
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
for (uint32_t dword = 0; dword < count; dword++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = decoded.src_count + 1u;
inst.memory = OffsetMemoryInfo(decoded, kind, dword);
inst.memory.resource = 0;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.dst, dword), &inst.src[0], error)) {
return false;
}
for (uint32_t i = 0; i < decoded.src_count && i + 1u < 3u; i++) {
if (!LowerSourceOperand(DecodedSourceAt(decoded, i), &inst.src[i + 1u], error)) {
return false;
}
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerDsWrite(const Decoder::Instruction& decoded, BasicBlock* block, std::string* error) {
const auto ir_op = decoded.data_bits == 8u ? Opcode::DsWriteByte
: decoded.data_bits == 16u ? Opcode::DsWriteShort
: Opcode::DsWriteB32;
const auto count = decoded.data_bits == 32u ? decoded.data_dwords : 1u;
for (uint32_t i = 0; i < count; i++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = ir_op;
inst.src_count = 2;
inst.memory = OffsetMemoryInfo(decoded, DsMemoryKind(decoded), i);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(decoded.src1, i), &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
}
return true;
}
bool LowerDsWrite2(const Decoder::Instruction& decoded, BasicBlock* block,
uint32_t dwords_per_write, std::string* error) {
const Decoder::Operand* data[] = {&decoded.src1, &decoded.src2};
const uint32_t offsets[] = {decoded.offset, decoded.secondary_offset};
for (uint32_t write = 0; write < 2u; write++) {
for (uint32_t dword = 0; dword < dwords_per_write; dword++) {
Instruction inst;
inst.pc = decoded.pc;
inst.op = Opcode::DsWriteB32;
inst.src_count = 2;
inst.memory =
ByteOffsetMemoryInfo(decoded, DsMemoryKind(decoded), offsets[write] + dword * 4u);
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(OffsetDecodedRegister(*data[write], dword), &inst.src[0],
error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
}
}
return true;
}
bool LowerImageOperation(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error) {
Instruction inst;
inst.pc = decoded.pc;
if (decoded.opcode == Decoder::Opcode::ImageStore ||
decoded.opcode == Decoder::Opcode::ImageStoreMip) {
inst.op = Opcode::ImageStore;
inst.src_count = 2;
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::StorageImage);
inst.memory.sampler = 0;
inst.dst.kind = OperandKind::Null;
if (!LowerSourceOperand(decoded.dst, &inst.src[0], error) ||
!LowerSourceOperand(decoded.src0, &inst.src[1], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
inst.op = decoded.opcode == Decoder::Opcode::ImageGetResinfo ? Opcode::ImageGetResinfo
: decoded.opcode == Decoder::Opcode::ImageGetLod ? Opcode::ImageGetLod
: decoded.opcode == Decoder::Opcode::ImageLoad ||
decoded.opcode == Decoder::Opcode::ImageLoadMip
? Opcode::ImageLoad
: decoded.opcode == Decoder::Opcode::ImageGather4Lz ||
decoded.opcode == Decoder::Opcode::ImageGather4C ||
decoded.opcode == Decoder::Opcode::ImageGather4CLz ||
decoded.opcode == Decoder::Opcode::ImageGather4LzO ||
decoded.opcode == Decoder::Opcode::ImageGather4CO ||
decoded.opcode == Decoder::Opcode::ImageGather4CLzO ||
decoded.opcode == Decoder::Opcode::ImageGather4H
? Opcode::ImageGather4
: Opcode::ImageSample;
inst.src_count = 1;
inst.memory = MemoryInfoFromDecoded(decoded, ResourceKind::Image);
if (!LowerRegisterOperand(decoded.dst, &inst.dst, error) ||
!LowerSourceOperand(decoded.src0, &inst.src[0], error)) {
return false;
}
block->instructions.push_back(inst);
return true;
}
} // namespace
bool LowerMemoryInstruction(const Decoder::Instruction& decoded, BasicBlock* block,
std::string* error) {
switch (decoded.opcode) {
case Decoder::Opcode::SLoadDword:
case Decoder::Opcode::SLoadDwordx2:
case Decoder::Opcode::SLoadDwordx4:
case Decoder::Opcode::SLoadDwordx8:
case Decoder::Opcode::SLoadDwordx16:
return LowerScalarBufferLoadDword(decoded, block, Opcode::SLoadDword, error);
case Decoder::Opcode::SBufferLoadDword:
case Decoder::Opcode::SBufferLoadDwordx2:
case Decoder::Opcode::SBufferLoadDwordx4:
case Decoder::Opcode::SBufferLoadDwordx8:
case Decoder::Opcode::SBufferLoadDwordx16:
return LowerScalarBufferLoadDword(decoded, block, Opcode::SBufferLoadDword, error);
case Decoder::Opcode::BufferLoadUbyte:
case Decoder::Opcode::BufferLoadSbyte:
case Decoder::Opcode::BufferLoadUshort:
case Decoder::Opcode::BufferLoadSshort:
case Decoder::Opcode::BufferLoadDword:
case Decoder::Opcode::BufferLoadDwordx2:
case Decoder::Opcode::BufferLoadDwordx3:
case Decoder::Opcode::BufferLoadDwordx4:
case Decoder::Opcode::BufferLoadFormatX:
case Decoder::Opcode::BufferLoadFormatXy:
case Decoder::Opcode::BufferLoadFormatXyz:
case Decoder::Opcode::BufferLoadFormatXyzw:
case Decoder::Opcode::TBufferLoadFormatX:
case Decoder::Opcode::TBufferLoadFormatXy:
case Decoder::Opcode::TBufferLoadFormatXyz:
case Decoder::Opcode::TBufferLoadFormatXyzw: return LowerBufferLoad(decoded, block, error);
case Decoder::Opcode::BufferStoreDword:
case Decoder::Opcode::BufferStoreDwordx2:
case Decoder::Opcode::BufferStoreDwordx3:
case Decoder::Opcode::BufferStoreDwordx4:
case Decoder::Opcode::BufferStoreByte:
case Decoder::Opcode::BufferStoreShort:
case Decoder::Opcode::BufferStoreFormatX:
case Decoder::Opcode::BufferStoreFormatXy:
case Decoder::Opcode::BufferStoreFormatXyz:
case Decoder::Opcode::BufferStoreFormatXyzw:
case Decoder::Opcode::TBufferStoreFormatX:
case Decoder::Opcode::TBufferStoreFormatXy:
case Decoder::Opcode::TBufferStoreFormatXyz:
case Decoder::Opcode::TBufferStoreFormatXyzw:
return LowerBufferStore(decoded, block, error);
case Decoder::Opcode::BufferAtomicSwap:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSwapU32, error);
case Decoder::Opcode::BufferAtomicAdd:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAddU32, error);
case Decoder::Opcode::BufferAtomicSub:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSubU32, error);
case Decoder::Opcode::BufferAtomicSMin:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMinI32, error);
case Decoder::Opcode::BufferAtomicUMin:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMinU32, error);
case Decoder::Opcode::BufferAtomicSMax:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicSMaxI32, error);
case Decoder::Opcode::BufferAtomicUMax:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicUMaxU32, error);
case Decoder::Opcode::BufferAtomicAnd:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicAndU32, error);
case Decoder::Opcode::BufferAtomicOr:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicOrU32, error);
case Decoder::Opcode::BufferAtomicXor:
return LowerBufferAtomicDword(decoded, block, Opcode::AtomicXorU32, error);
case Decoder::Opcode::FlatLoadUbyte:
case Decoder::Opcode::FlatLoadSbyte:
case Decoder::Opcode::FlatLoadUshort:
case Decoder::Opcode::FlatLoadSshort:
case Decoder::Opcode::FlatLoadDword:
case Decoder::Opcode::FlatLoadDwordx2:
case Decoder::Opcode::FlatLoadDwordx3:
case Decoder::Opcode::FlatLoadDwordx4: return LowerFlatLoad(decoded, block, error);
case Decoder::Opcode::FlatStoreByte:
case Decoder::Opcode::FlatStoreShort:
case Decoder::Opcode::FlatStoreDword:
case Decoder::Opcode::FlatStoreDwordx2:
case Decoder::Opcode::FlatStoreDwordx3:
case Decoder::Opcode::FlatStoreDwordx4: return LowerFlatStore(decoded, block, error);
case Decoder::Opcode::DsAddU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, false, error);
case Decoder::Opcode::DsAddRtnU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicAddU32, true, error);
case Decoder::Opcode::DsSubU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, false, error);
case Decoder::Opcode::DsSubRtnU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSubU32, true, error);
case Decoder::Opcode::DsMinI32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, false, error);
case Decoder::Opcode::DsMinRtnI32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMinI32, true, error);
case Decoder::Opcode::DsMaxI32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, false, error);
case Decoder::Opcode::DsMaxRtnI32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSMaxI32, true, error);
case Decoder::Opcode::DsMinU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, false, error);
case Decoder::Opcode::DsMinRtnU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMinU32, true, error);
case Decoder::Opcode::DsMaxU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, false, error);
case Decoder::Opcode::DsMaxRtnU32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicUMaxU32, true, error);
case Decoder::Opcode::DsAndB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, false, error);
case Decoder::Opcode::DsAndRtnB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicAndU32, true, error);
case Decoder::Opcode::DsOrB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, false, error);
case Decoder::Opcode::DsOrRtnB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicOrU32, true, error);
case Decoder::Opcode::DsXorB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, false, error);
case Decoder::Opcode::DsXorRtnB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicXorU32, true, error);
case Decoder::Opcode::DsWrxchgRtnB32:
return LowerDsAtomicU32(decoded, block, Opcode::AtomicSwapU32, true, error);
case Decoder::Opcode::DsMinF32:
return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMinF32, error);
case Decoder::Opcode::DsMaxF32:
return LowerDsFloatMinMaxF32(decoded, block, Opcode::DsMaxF32, error);
case Decoder::Opcode::DsSwizzleB32: return LowerDsSwizzleB32(decoded, block, error);
case Decoder::Opcode::DsConsume:
return LowerDsAppendConsume(decoded, block, Opcode::DsConsume, error);
case Decoder::Opcode::DsAppend:
return LowerDsAppendConsume(decoded, block, Opcode::DsAppend, error);
case Decoder::Opcode::DsWriteAddtidB32: return LowerDsWriteAddtidB32(decoded, block, error);
case Decoder::Opcode::DsReadAddtidB32: return LowerDsReadAddtidB32(decoded, block, error);
case Decoder::Opcode::DsRead2B32: return LowerDsRead2(decoded, block, 1, error);
case Decoder::Opcode::DsRead2B64:
case Decoder::Opcode::DsRead2St64B64: return LowerDsRead2(decoded, block, 2, error);
case Decoder::Opcode::DsReadSbyte:
case Decoder::Opcode::DsReadUbyte:
case Decoder::Opcode::DsReadSshort:
case Decoder::Opcode::DsReadUshort:
case Decoder::Opcode::DsReadB32:
case Decoder::Opcode::DsReadB64:
case Decoder::Opcode::DsReadB96:
case Decoder::Opcode::DsReadB128: return LowerDsRead(decoded, block, error);
case Decoder::Opcode::DsWrite2B32:
case Decoder::Opcode::DsWrite2St64B32: return LowerDsWrite2(decoded, block, 1, error);
case Decoder::Opcode::DsWrite2B64:
case Decoder::Opcode::DsWrite2St64B64: return LowerDsWrite2(decoded, block, 2, error);
case Decoder::Opcode::DsWriteByte:
case Decoder::Opcode::DsWriteShort:
case Decoder::Opcode::DsWriteB32:
case Decoder::Opcode::DsWriteB64:
case Decoder::Opcode::DsWriteB96:
case Decoder::Opcode::DsWriteB128: return LowerDsWrite(decoded, block, error);
case Decoder::Opcode::ImageGetResinfo:
case Decoder::Opcode::ImageGetLod:
case Decoder::Opcode::ImageLoad:
case Decoder::Opcode::ImageLoadMip:
case Decoder::Opcode::ImageStore:
case Decoder::Opcode::ImageStoreMip:
case Decoder::Opcode::ImageGather4Lz:
case Decoder::Opcode::ImageGather4C:
case Decoder::Opcode::ImageGather4CLz:
case Decoder::Opcode::ImageGather4LzO:
case Decoder::Opcode::ImageGather4CO:
case Decoder::Opcode::ImageGather4CLzO:
case Decoder::Opcode::ImageGather4H:
case Decoder::Opcode::ImageSample: return LowerImageOperation(decoded, block, error);
case Decoder::Opcode::ImageAtomicAdd:
case Decoder::Opcode::ImageAtomicUMin:
case Decoder::Opcode::ImageAtomicUMax:
case Decoder::Opcode::ImageAtomicAnd:
case Decoder::Opcode::ImageAtomicOr:
case Decoder::Opcode::ImageAtomicXor: return LowerImageAtomicU32(decoded, block, error);
default: return false;
}
}
bool IsMemoryOpcode(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::SLoadDword:
case Decoder::Opcode::SLoadDwordx2:
case Decoder::Opcode::SLoadDwordx4:
case Decoder::Opcode::SLoadDwordx8:
case Decoder::Opcode::SLoadDwordx16:
case Decoder::Opcode::SBufferLoadDword:
case Decoder::Opcode::SBufferLoadDwordx2:
case Decoder::Opcode::SBufferLoadDwordx4:
case Decoder::Opcode::SBufferLoadDwordx8:
case Decoder::Opcode::SBufferLoadDwordx16:
case Decoder::Opcode::BufferLoadUbyte:
case Decoder::Opcode::BufferLoadSbyte:
case Decoder::Opcode::BufferLoadUshort:
case Decoder::Opcode::BufferLoadSshort:
case Decoder::Opcode::BufferLoadDword:
case Decoder::Opcode::BufferLoadDwordx2:
case Decoder::Opcode::BufferLoadDwordx3:
case Decoder::Opcode::BufferLoadDwordx4:
case Decoder::Opcode::BufferLoadFormatX:
case Decoder::Opcode::BufferLoadFormatXy:
case Decoder::Opcode::BufferLoadFormatXyz:
case Decoder::Opcode::BufferLoadFormatXyzw:
case Decoder::Opcode::TBufferLoadFormatX:
case Decoder::Opcode::TBufferLoadFormatXy:
case Decoder::Opcode::TBufferLoadFormatXyz:
case Decoder::Opcode::TBufferLoadFormatXyzw:
case Decoder::Opcode::BufferStoreDword:
case Decoder::Opcode::BufferStoreDwordx2:
case Decoder::Opcode::BufferStoreDwordx3:
case Decoder::Opcode::BufferStoreDwordx4:
case Decoder::Opcode::BufferStoreByte:
case Decoder::Opcode::BufferStoreShort:
case Decoder::Opcode::BufferStoreFormatX:
case Decoder::Opcode::BufferStoreFormatXy:
case Decoder::Opcode::BufferStoreFormatXyz:
case Decoder::Opcode::BufferStoreFormatXyzw:
case Decoder::Opcode::TBufferStoreFormatX:
case Decoder::Opcode::TBufferStoreFormatXy:
case Decoder::Opcode::TBufferStoreFormatXyz:
case Decoder::Opcode::TBufferStoreFormatXyzw:
case Decoder::Opcode::BufferAtomicSwap:
case Decoder::Opcode::BufferAtomicAdd:
case Decoder::Opcode::BufferAtomicSub:
case Decoder::Opcode::BufferAtomicSMin:
case Decoder::Opcode::BufferAtomicUMin:
case Decoder::Opcode::BufferAtomicSMax:
case Decoder::Opcode::BufferAtomicUMax:
case Decoder::Opcode::BufferAtomicAnd:
case Decoder::Opcode::BufferAtomicOr:
case Decoder::Opcode::BufferAtomicXor:
case Decoder::Opcode::FlatLoadUbyte:
case Decoder::Opcode::FlatLoadSbyte:
case Decoder::Opcode::FlatLoadUshort:
case Decoder::Opcode::FlatLoadSshort:
case Decoder::Opcode::FlatLoadDword:
case Decoder::Opcode::FlatLoadDwordx2:
case Decoder::Opcode::FlatLoadDwordx3:
case Decoder::Opcode::FlatLoadDwordx4:
case Decoder::Opcode::FlatStoreByte:
case Decoder::Opcode::FlatStoreShort:
case Decoder::Opcode::FlatStoreDword:
case Decoder::Opcode::FlatStoreDwordx2:
case Decoder::Opcode::FlatStoreDwordx3:
case Decoder::Opcode::FlatStoreDwordx4:
case Decoder::Opcode::DsAddU32:
case Decoder::Opcode::DsAddRtnU32:
case Decoder::Opcode::DsSubU32:
case Decoder::Opcode::DsSubRtnU32:
case Decoder::Opcode::DsMinI32:
case Decoder::Opcode::DsMinRtnI32:
case Decoder::Opcode::DsMaxI32:
case Decoder::Opcode::DsMaxRtnI32:
case Decoder::Opcode::DsMinU32:
case Decoder::Opcode::DsMinRtnU32:
case Decoder::Opcode::DsMaxU32:
case Decoder::Opcode::DsMaxRtnU32:
case Decoder::Opcode::DsAndB32:
case Decoder::Opcode::DsAndRtnB32:
case Decoder::Opcode::DsOrB32:
case Decoder::Opcode::DsOrRtnB32:
case Decoder::Opcode::DsXorB32:
case Decoder::Opcode::DsXorRtnB32:
case Decoder::Opcode::DsWrxchgRtnB32:
case Decoder::Opcode::DsMinF32:
case Decoder::Opcode::DsMaxF32:
case Decoder::Opcode::DsSwizzleB32:
case Decoder::Opcode::DsConsume:
case Decoder::Opcode::DsAppend:
case Decoder::Opcode::DsReadSbyte:
case Decoder::Opcode::DsReadUbyte:
case Decoder::Opcode::DsReadSshort:
case Decoder::Opcode::DsReadUshort:
case Decoder::Opcode::DsRead2B32:
case Decoder::Opcode::DsReadB32:
case Decoder::Opcode::DsReadB64:
case Decoder::Opcode::DsRead2B64:
case Decoder::Opcode::DsRead2St64B64:
case Decoder::Opcode::DsReadB96:
case Decoder::Opcode::DsReadB128:
case Decoder::Opcode::DsWriteByte:
case Decoder::Opcode::DsWriteShort:
case Decoder::Opcode::DsWrite2B32:
case Decoder::Opcode::DsWrite2St64B32:
case Decoder::Opcode::DsWrite2B64:
case Decoder::Opcode::DsWrite2St64B64:
case Decoder::Opcode::DsWriteB32:
case Decoder::Opcode::DsWriteB64:
case Decoder::Opcode::DsWriteB96:
case Decoder::Opcode::DsWriteB128:
case Decoder::Opcode::DsWriteAddtidB32:
case Decoder::Opcode::DsReadAddtidB32:
case Decoder::Opcode::ImageGetResinfo:
case Decoder::Opcode::ImageGetLod:
case Decoder::Opcode::ImageLoad:
case Decoder::Opcode::ImageLoadMip:
case Decoder::Opcode::ImageStore:
case Decoder::Opcode::ImageStoreMip:
case Decoder::Opcode::ImageAtomicAdd:
case Decoder::Opcode::ImageAtomicUMin:
case Decoder::Opcode::ImageAtomicUMax:
case Decoder::Opcode::ImageAtomicAnd:
case Decoder::Opcode::ImageAtomicOr:
case Decoder::Opcode::ImageAtomicXor:
case Decoder::Opcode::ImageGather4Lz:
case Decoder::Opcode::ImageGather4C:
case Decoder::Opcode::ImageGather4CLz:
case Decoder::Opcode::ImageGather4LzO:
case Decoder::Opcode::ImageGather4CO:
case Decoder::Opcode::ImageGather4CLzO:
case Decoder::Opcode::ImageGather4H:
case Decoder::Opcode::ImageSample: return true;
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/shaderIR/ShaderIROpcodes.cpp
+534
View File
@@ -0,0 +1,534 @@
#include "graphics/shader/recompiler/ShaderIR.h"
#include "graphics/shader/recompiler/shaderIR/ShaderIRInternal.h"
#include "common/assert.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include <algorithm>
#include <fmt/format.h>
#include <utility>
namespace Libs::Graphics::ShaderRecompiler::IR {
namespace {
struct LowerMap {
Decoder::Opcode decoded = Decoder::Opcode::Unknown;
Opcode ir = Opcode::MoveU32;
};
constexpr LowerMap LOWER_OPS[] = {
{Decoder::Opcode::SMovB32, Opcode::MoveU32},
{Decoder::Opcode::SMovB64, Opcode::MoveU64},
{Decoder::Opcode::SWqmB64, Opcode::WqmB64},
{Decoder::Opcode::SMovkI32, Opcode::MoveU32},
{Decoder::Opcode::SAbsI32, Opcode::AbsI32},
{Decoder::Opcode::SBrevB32, Opcode::BitReverseU32},
{Decoder::Opcode::SBcnt1I32B32, Opcode::BitCountU32},
{Decoder::Opcode::SBcnt1I32B64, Opcode::BitCountU64},
{Decoder::Opcode::SFf1I32B32, Opcode::FindLsbU32},
{Decoder::Opcode::SFlbitI32B64, Opcode::FindMsbFromHighU64},
{Decoder::Opcode::SBitreplicateB64B32, Opcode::BitReplicateB64B32},
{Decoder::Opcode::SNotB32, Opcode::BitwiseNotU32},
{Decoder::Opcode::SNotB64, Opcode::BitwiseNotU64},
{Decoder::Opcode::SAddU32, Opcode::IAddU32},
{Decoder::Opcode::SSubU32, Opcode::ISubU32},
{Decoder::Opcode::SAddI32, Opcode::IAddU32},
{Decoder::Opcode::SSubI32, Opcode::ISubU32},
{Decoder::Opcode::SMinI32, Opcode::IMinI32},
{Decoder::Opcode::SMaxI32, Opcode::IMaxI32},
{Decoder::Opcode::SMinU32, Opcode::UMinU32},
{Decoder::Opcode::SMaxU32, Opcode::UMaxU32},
{Decoder::Opcode::SAndB32, Opcode::BitwiseAndU32},
{Decoder::Opcode::SAndB64, Opcode::BitwiseAndU64},
{Decoder::Opcode::SAndn2B32, Opcode::BitwiseAndNotU32},
{Decoder::Opcode::SAndn2B64, Opcode::BitwiseAndNotU64},
{Decoder::Opcode::SOrB32, Opcode::BitwiseOrU32},
{Decoder::Opcode::SOrB64, Opcode::BitwiseOrU64},
{Decoder::Opcode::SOrn2B32, Opcode::BitwiseOrNotU32},
{Decoder::Opcode::SOrn2B64, Opcode::BitwiseOrNotU64},
{Decoder::Opcode::SXorB32, Opcode::BitwiseXorU32},
{Decoder::Opcode::SXorB64, Opcode::BitwiseXorU64},
{Decoder::Opcode::SNandB32, Opcode::BitwiseNandU32},
{Decoder::Opcode::SNandB64, Opcode::BitwiseNandU64},
{Decoder::Opcode::SNorB32, Opcode::BitwiseNorU32},
{Decoder::Opcode::SNorB64, Opcode::BitwiseNorU64},
{Decoder::Opcode::SXnorB32, Opcode::BitwiseXnorU32},
{Decoder::Opcode::SXnorB64, Opcode::BitwiseXnorU64},
{Decoder::Opcode::SLshlB32, Opcode::ShiftLeftLogicalU32},
{Decoder::Opcode::SLshlB64, Opcode::ShiftLeftLogicalU64},
{Decoder::Opcode::SLshrB32, Opcode::ShiftRightLogicalU32},
{Decoder::Opcode::SLshrB64, Opcode::ShiftRightLogicalU64},
{Decoder::Opcode::SAshrI32, Opcode::ShiftRightArithmeticI32},
{Decoder::Opcode::SMulI32, Opcode::IMulU32},
{Decoder::Opcode::SMulHiU32, Opcode::UMulHighU32},
{Decoder::Opcode::SMulkI32, Opcode::IMulU32},
{Decoder::Opcode::SBfeU32, Opcode::BitFieldExtractU32},
{Decoder::Opcode::SBfeU64, Opcode::BitFieldExtractU64},
{Decoder::Opcode::SBfmB32, Opcode::BitFieldMaskU32},
{Decoder::Opcode::SBfmB64, Opcode::BitFieldMaskU64},
{Decoder::Opcode::SBitcmp0B32, Opcode::BitCompare0B32},
{Decoder::Opcode::SBitcmp1B32, Opcode::BitCompare1B32},
{Decoder::Opcode::SPackLlB32B16, Opcode::PackLowLowU16},
{Decoder::Opcode::SPackLhB32B16, Opcode::PackLowHighU16},
{Decoder::Opcode::SPackHhB32B16, Opcode::PackHighHighU16},
{Decoder::Opcode::SCmpEqI32, Opcode::CompareEqI32},
{Decoder::Opcode::SCmpLgI32, Opcode::CompareNeI32},
{Decoder::Opcode::SCmpGtI32, Opcode::CompareGtI32},
{Decoder::Opcode::SCmpGeI32, Opcode::CompareGeI32},
{Decoder::Opcode::SCmpLtI32, Opcode::CompareLtI32},
{Decoder::Opcode::SCmpLeI32, Opcode::CompareLeI32},
{Decoder::Opcode::SCmpEqU32, Opcode::CompareEqU32},
{Decoder::Opcode::SCmpLgU32, Opcode::CompareNeU32},
{Decoder::Opcode::SCmpGtU32, Opcode::CompareGtU32},
{Decoder::Opcode::SCmpGeU32, Opcode::CompareGeU32},
{Decoder::Opcode::SCmpLtU32, Opcode::CompareLtU32},
{Decoder::Opcode::SCmpLeU32, Opcode::CompareLeU32},
{Decoder::Opcode::SCmpLgU64, Opcode::CompareNeU64},
{Decoder::Opcode::VReadfirstlaneB32, Opcode::ReadFirstLaneU32},
{Decoder::Opcode::VReadlaneB32, Opcode::ReadLaneU32},
{Decoder::Opcode::VWritelaneB32, Opcode::WriteLaneU32},
{Decoder::Opcode::VPermlane16B32, Opcode::Permlane16B32},
{Decoder::Opcode::VPermlanex16B32, Opcode::Permlanex16B32},
{Decoder::Opcode::VCvtF32I32, Opcode::ConvertI32ToF32},
{Decoder::Opcode::VCvtF32U32, Opcode::ConvertU32ToF32},
{Decoder::Opcode::VCvtU32F32, Opcode::ConvertF32ToU32},
{Decoder::Opcode::VCvtI32F32, Opcode::ConvertF32ToI32},
{Decoder::Opcode::VCvtF16F32, Opcode::ConvertF32ToF16},
{Decoder::Opcode::VCvtF32F16, Opcode::ConvertF16ToF32},
{Decoder::Opcode::VCvtF16U16, Opcode::ConvertU16ToF16},
{Decoder::Opcode::VCvtU16F16, Opcode::ConvertF16ToU16},
{Decoder::Opcode::VCvtF16I16, Opcode::ConvertI16ToF16},
{Decoder::Opcode::VCvtI16F16, Opcode::ConvertF16ToI16},
{Decoder::Opcode::VCvtRpiI32F32, Opcode::ConvertRoundPlusInfF32ToI32},
{Decoder::Opcode::VCvtFlrI32F32, Opcode::ConvertFloorF32ToI32},
{Decoder::Opcode::VCvtOffF32I4, Opcode::ConvertI4ToOffsetF32},
{Decoder::Opcode::VRcpF32, Opcode::RcpF32},
{Decoder::Opcode::VFractF32, Opcode::FractF32},
{Decoder::Opcode::VTruncF32, Opcode::TruncF32},
{Decoder::Opcode::VCeilF32, Opcode::CeilF32},
{Decoder::Opcode::VRndneF32, Opcode::RoundEvenF32},
{Decoder::Opcode::VFloorF32, Opcode::FloorF32},
{Decoder::Opcode::VExpF32, Opcode::Exp2F32},
{Decoder::Opcode::VLogF32, Opcode::Log2F32},
{Decoder::Opcode::VRsqF32, Opcode::InverseSqrtF32},
{Decoder::Opcode::VSqrtF32, Opcode::SqrtF32},
{Decoder::Opcode::VRcpF16, Opcode::RcpF16},
{Decoder::Opcode::VSqrtF16, Opcode::SqrtF16},
{Decoder::Opcode::VRsqF16, Opcode::InverseSqrtF16},
{Decoder::Opcode::VLogF16, Opcode::Log2F16},
{Decoder::Opcode::VExpF16, Opcode::Exp2F16},
{Decoder::Opcode::VFloorF16, Opcode::FloorF16},
{Decoder::Opcode::VCeilF16, Opcode::CeilF16},
{Decoder::Opcode::VTruncF16, Opcode::TruncF16},
{Decoder::Opcode::VRndneF16, Opcode::RoundEvenF16},
{Decoder::Opcode::VSinF32, Opcode::SinF32},
{Decoder::Opcode::VCosF32, Opcode::CosF32},
{Decoder::Opcode::VNotB32, Opcode::BitwiseNotU32},
{Decoder::Opcode::VBfrevB32, Opcode::BitReverseU32},
{Decoder::Opcode::VFfblB32, Opcode::FindLsbU32},
{Decoder::Opcode::VFfbhU32, Opcode::FindMsbFromHighU32},
{Decoder::Opcode::VAddF32, Opcode::FAddF32},
{Decoder::Opcode::VSubF32, Opcode::FSubF32},
{Decoder::Opcode::VSubrevF32, Opcode::FSubF32},
{Decoder::Opcode::VMulF32, Opcode::FMulF32},
{Decoder::Opcode::VMacF32, Opcode::FMadF32},
{Decoder::Opcode::VMadmkF32, Opcode::FMadF32},
{Decoder::Opcode::VMadakF32, Opcode::FMadF32},
{Decoder::Opcode::VMinF32, Opcode::FMinF32},
{Decoder::Opcode::VMaxF32, Opcode::FMaxF32},
{Decoder::Opcode::VMinI32, Opcode::IMinI32},
{Decoder::Opcode::VMaxI32, Opcode::IMaxI32},
{Decoder::Opcode::VMadF32, Opcode::FMadF32},
{Decoder::Opcode::VMadI32I24, Opcode::IMadI24U32},
{Decoder::Opcode::VMadU32U24, Opcode::UMadU24U32},
{Decoder::Opcode::VCubeidF32, Opcode::CubeIdF32},
{Decoder::Opcode::VCubescF32, Opcode::CubeScF32},
{Decoder::Opcode::VCubetcF32, Opcode::CubeTcF32},
{Decoder::Opcode::VCubemaF32, Opcode::CubeMaF32},
{Decoder::Opcode::VFmaF32, Opcode::FMadF32},
{Decoder::Opcode::VBfeU32, Opcode::BitFieldExtract3U32},
{Decoder::Opcode::VBfeI32, Opcode::BitFieldExtract3I32},
{Decoder::Opcode::VBfiB32, Opcode::BitFieldInsertSelectU32},
{Decoder::Opcode::VAlignbitB32, Opcode::AlignBitU32},
{Decoder::Opcode::VMin3F32, Opcode::FMin3F32},
{Decoder::Opcode::VMin3I32, Opcode::IMin3I32},
{Decoder::Opcode::VMin3U32, Opcode::UMin3U32},
{Decoder::Opcode::VMin3F16, Opcode::Min3F16},
{Decoder::Opcode::VMax3F32, Opcode::FMax3F32},
{Decoder::Opcode::VMax3I32, Opcode::IMax3I32},
{Decoder::Opcode::VMax3U32, Opcode::UMax3U32},
{Decoder::Opcode::VMax3F16, Opcode::Max3F16},
{Decoder::Opcode::VMed3F32, Opcode::FMed3F32},
{Decoder::Opcode::VMed3I32, Opcode::IMed3I32},
{Decoder::Opcode::VMed3U32, Opcode::UMed3U32},
{Decoder::Opcode::VMed3F16, Opcode::Med3F16},
{Decoder::Opcode::VSadU32, Opcode::SadU32},
{Decoder::Opcode::VAdd3U32, Opcode::IAdd3U32},
{Decoder::Opcode::VLshlAddU32, Opcode::ShiftLeftAddU32},
{Decoder::Opcode::VAddLshlU32, Opcode::AddShiftLeftU32},
{Decoder::Opcode::VXadU32, Opcode::XorAddU32},
{Decoder::Opcode::VLshlOrB32, Opcode::ShiftLeftOrU32},
{Decoder::Opcode::VAndOrB32, Opcode::BitwiseAndOrU32},
{Decoder::Opcode::VOr3B32, Opcode::BitwiseOr3U32},
{Decoder::Opcode::VXor3B32, Opcode::BitwiseXor3U32},
{Decoder::Opcode::VAddNcU32, Opcode::IAddU32},
{Decoder::Opcode::VSubNcU32, Opcode::ISubU32},
{Decoder::Opcode::VSubrevNcU32, Opcode::ISubU32},
{Decoder::Opcode::VAddNcU16, Opcode::IAddU16},
{Decoder::Opcode::VSubNcU16, Opcode::ISubI16},
{Decoder::Opcode::VMaxU16, Opcode::UMaxU16},
{Decoder::Opcode::VMaxI16, Opcode::IMaxI16},
{Decoder::Opcode::VMinU16, Opcode::UMinU16},
{Decoder::Opcode::VMinI16, Opcode::IMinI16},
{Decoder::Opcode::VAddNcI16, Opcode::IAddU16},
{Decoder::Opcode::VSubNcI16, Opcode::ISubI16},
{Decoder::Opcode::VMulI32I24, Opcode::IMulI24U32},
{Decoder::Opcode::VMulU32U24, Opcode::UMulU24U32},
{Decoder::Opcode::VMulLoU32, Opcode::IMulU32},
{Decoder::Opcode::VMulHiU32, Opcode::UMulHighU32},
{Decoder::Opcode::VMulLoI32, Opcode::IMulU32},
{Decoder::Opcode::VMulHiI32, Opcode::SMulHighI32},
{Decoder::Opcode::VAddI32, Opcode::IAddU32},
{Decoder::Opcode::VSubI32, Opcode::ISubU32},
{Decoder::Opcode::VSubrevI32, Opcode::ISubU32},
{Decoder::Opcode::VBfmB32, Opcode::BitFieldMaskU32},
{Decoder::Opcode::VCvtPkrtzF16F32, Opcode::PackF32ToF16Rtz},
{Decoder::Opcode::VLdexpF32, Opcode::LdexpF32},
{Decoder::Opcode::VCvtPknormI16F32, Opcode::PackSnorm2x16F32},
{Decoder::Opcode::VCvtPknormU16F32, Opcode::PackUnorm2x16F32},
{Decoder::Opcode::VCvtPkU16U32, Opcode::PackU16U32},
{Decoder::Opcode::VCvtPkU8F32, Opcode::PackU8F32},
{Decoder::Opcode::VPackB32F16, Opcode::PackB32F16},
{Decoder::Opcode::VPkMadI16, Opcode::PackedMadI16},
{Decoder::Opcode::VPkMulLoU16, Opcode::PackedMulLoU16},
{Decoder::Opcode::VPkAddI16, Opcode::PackedAddI16},
{Decoder::Opcode::VPkSubI16, Opcode::PackedSubI16},
{Decoder::Opcode::VPkLshlrevB16, Opcode::PackedLshlrevB16},
{Decoder::Opcode::VPkLshrrevB16, Opcode::PackedLshrrevB16},
{Decoder::Opcode::VPkAshrrevI16, Opcode::PackedAshrrevI16},
{Decoder::Opcode::VPkMaxI16, Opcode::PackedMaxI16},
{Decoder::Opcode::VPkMinI16, Opcode::PackedMinI16},
{Decoder::Opcode::VPkMadU16, Opcode::PackedMadU16},
{Decoder::Opcode::VPkAddU16, Opcode::PackedAddU16},
{Decoder::Opcode::VPkSubU16, Opcode::PackedSubU16},
{Decoder::Opcode::VPkMaxU16, Opcode::PackedMaxU16},
{Decoder::Opcode::VPkMinU16, Opcode::PackedMinU16},
{Decoder::Opcode::VPkAddF16, Opcode::PackedAddF16},
{Decoder::Opcode::VPkMulF16, Opcode::PackedMulF16},
{Decoder::Opcode::VPkMinF16, Opcode::PackedMinF16},
{Decoder::Opcode::VPkMaxF16, Opcode::PackedMaxF16},
{Decoder::Opcode::VPkFmaF16, Opcode::PackedFmaF16},
{Decoder::Opcode::VPkFmacF16, Opcode::PackedFmaF16},
{Decoder::Opcode::VAddF16, Opcode::AddF16},
{Decoder::Opcode::VSubF16, Opcode::SubF16},
{Decoder::Opcode::VSubrevF16, Opcode::SubF16},
{Decoder::Opcode::VMulF16, Opcode::MulF16},
{Decoder::Opcode::VFmacF16, Opcode::FmaF16},
{Decoder::Opcode::VFmamkF16, Opcode::FmaF16},
{Decoder::Opcode::VFmaakF16, Opcode::FmaF16},
{Decoder::Opcode::VFmaF16, Opcode::FmaF16},
{Decoder::Opcode::VMadMixloF16, Opcode::MadMixF16},
{Decoder::Opcode::VMadMixhiF16, Opcode::MadMixF16},
{Decoder::Opcode::VAndB32, Opcode::BitwiseAndU32},
{Decoder::Opcode::VOrB32, Opcode::BitwiseOrU32},
{Decoder::Opcode::VXorB32, Opcode::BitwiseXorU32},
{Decoder::Opcode::VXnorB32, Opcode::BitwiseXnorU32},
{Decoder::Opcode::VBcntU32B32, Opcode::BitCountAddU32},
{Decoder::Opcode::VMbcntLoU32B32, Opcode::MaskedBitCountLowU32},
{Decoder::Opcode::VMbcntHiU32B32, Opcode::MaskedBitCountHighU32},
{Decoder::Opcode::VLshlB32, Opcode::ShiftLeftLogicalU32},
{Decoder::Opcode::VLshlrevB32, Opcode::ShiftLeftLogicalU32},
{Decoder::Opcode::VLshlrevB16, Opcode::ShiftLeftLogicalU16},
{Decoder::Opcode::VLshrB32, Opcode::ShiftRightLogicalU32},
{Decoder::Opcode::VLshrrevB32, Opcode::ShiftRightLogicalU32},
{Decoder::Opcode::VLshrrevB16, Opcode::ShiftRightLogicalU16},
{Decoder::Opcode::VAshrI32, Opcode::ShiftRightArithmeticI32},
{Decoder::Opcode::VAshrrevI32, Opcode::ShiftRightArithmeticI32},
{Decoder::Opcode::VAshrrevI16, Opcode::ShiftRightArithmeticI16},
{Decoder::Opcode::VMinU32, Opcode::UMinU32},
{Decoder::Opcode::VMaxU32, Opcode::UMaxU32},
{Decoder::Opcode::VMaxF16, Opcode::MaxF16},
{Decoder::Opcode::VMinF16, Opcode::MinF16},
{Decoder::Opcode::VCmpFF32, Opcode::CompareFalse},
{Decoder::Opcode::VCmpLtF32, Opcode::CompareLtF32},
{Decoder::Opcode::VCmpEqF32, Opcode::CompareEqF32},
{Decoder::Opcode::VCmpLeF32, Opcode::CompareLeF32},
{Decoder::Opcode::VCmpGtF32, Opcode::CompareGtF32},
{Decoder::Opcode::VCmpLgF32, Opcode::CompareNeF32},
{Decoder::Opcode::VCmpGeF32, Opcode::CompareGeF32},
{Decoder::Opcode::VCmpOF32, Opcode::CompareOrderedF32},
{Decoder::Opcode::VCmpUF32, Opcode::CompareUnorderedF32},
{Decoder::Opcode::VCmpNgeF32, Opcode::CompareUnordLtF32},
{Decoder::Opcode::VCmpNlgF32, Opcode::CompareUnordEqF32},
{Decoder::Opcode::VCmpNgtF32, Opcode::CompareUnordLeF32},
{Decoder::Opcode::VCmpNleF32, Opcode::CompareUnordGtF32},
{Decoder::Opcode::VCmpNeqF32, Opcode::CompareUnordNeF32},
{Decoder::Opcode::VCmpNltF32, Opcode::CompareUnordGeF32},
{Decoder::Opcode::VCmpTruF32, Opcode::CompareTrue},
{Decoder::Opcode::VCmpClassF32, Opcode::CompareClassF32},
{Decoder::Opcode::VCmpLtF16, Opcode::CompareLtF16},
{Decoder::Opcode::VCmpEqF16, Opcode::CompareEqF16},
{Decoder::Opcode::VCmpLeF16, Opcode::CompareLeF16},
{Decoder::Opcode::VCmpGtF16, Opcode::CompareGtF16},
{Decoder::Opcode::VCmpLgF16, Opcode::CompareNeF16},
{Decoder::Opcode::VCmpGeF16, Opcode::CompareGeF16},
{Decoder::Opcode::VCmpNeqF16, Opcode::CompareUnordNeF16},
{Decoder::Opcode::VCmpxLtF16, Opcode::CompareMaskLtF16},
{Decoder::Opcode::VCmpxEqF16, Opcode::CompareMaskEqF16},
{Decoder::Opcode::VCmpxLeF16, Opcode::CompareMaskLeF16},
{Decoder::Opcode::VCmpxGtF16, Opcode::CompareMaskGtF16},
{Decoder::Opcode::VCmpxGeF16, Opcode::CompareMaskGeF16},
{Decoder::Opcode::VCmpxNeqF16, Opcode::CompareMaskUnordNeF16},
{Decoder::Opcode::VCmpxNltF16, Opcode::CompareMaskUnordGeF16},
{Decoder::Opcode::VCmpxLtF32, Opcode::CompareMaskLtF32},
{Decoder::Opcode::VCmpxEqF32, Opcode::CompareMaskEqF32},
{Decoder::Opcode::VCmpxLeF32, Opcode::CompareMaskLeF32},
{Decoder::Opcode::VCmpxGtF32, Opcode::CompareMaskGtF32},
{Decoder::Opcode::VCmpxLgF32, Opcode::CompareMaskNeF32},
{Decoder::Opcode::VCmpxGeF32, Opcode::CompareMaskGeF32},
{Decoder::Opcode::VCmpxNgeF32, Opcode::CompareMaskUnordLtF32},
{Decoder::Opcode::VCmpxNlgF32, Opcode::CompareMaskUnordEqF32},
{Decoder::Opcode::VCmpxNgtF32, Opcode::CompareMaskUnordLeF32},
{Decoder::Opcode::VCmpxNleF32, Opcode::CompareMaskUnordGtF32},
{Decoder::Opcode::VCmpxNeqF32, Opcode::CompareMaskUnordNeF32},
{Decoder::Opcode::VCmpxNltF32, Opcode::CompareMaskUnordGeF32},
{Decoder::Opcode::VCmpFI32, Opcode::CompareFalse},
{Decoder::Opcode::VCmpLtI32, Opcode::CompareLtI32},
{Decoder::Opcode::VCmpEqI32, Opcode::CompareEqI32},
{Decoder::Opcode::VCmpLeI32, Opcode::CompareLeI32},
{Decoder::Opcode::VCmpGtI32, Opcode::CompareGtI32},
{Decoder::Opcode::VCmpNeI32, Opcode::CompareNeI32},
{Decoder::Opcode::VCmpGeI32, Opcode::CompareGeI32},
{Decoder::Opcode::VCmpTI32, Opcode::CompareTrue},
{Decoder::Opcode::VCmpLtI16, Opcode::CompareLtI16},
{Decoder::Opcode::VCmpEqI16, Opcode::CompareEqI16},
{Decoder::Opcode::VCmpLeI16, Opcode::CompareLeI16},
{Decoder::Opcode::VCmpGtI16, Opcode::CompareGtI16},
{Decoder::Opcode::VCmpNeI16, Opcode::CompareNeI16},
{Decoder::Opcode::VCmpGeI16, Opcode::CompareGeI16},
{Decoder::Opcode::VCmpxLtI32, Opcode::CompareMaskLtI32},
{Decoder::Opcode::VCmpxEqI32, Opcode::CompareMaskEqI32},
{Decoder::Opcode::VCmpxLeI32, Opcode::CompareMaskLeI32},
{Decoder::Opcode::VCmpxGtI32, Opcode::CompareMaskGtI32},
{Decoder::Opcode::VCmpxNeI32, Opcode::CompareMaskNeI32},
{Decoder::Opcode::VCmpxGeI32, Opcode::CompareMaskGeI32},
{Decoder::Opcode::VCmpLtU16, Opcode::CompareLtU16},
{Decoder::Opcode::VCmpEqU16, Opcode::CompareEqU16},
{Decoder::Opcode::VCmpLeU16, Opcode::CompareLeU16},
{Decoder::Opcode::VCmpGtU16, Opcode::CompareGtU16},
{Decoder::Opcode::VCmpNeU16, Opcode::CompareNeU16},
{Decoder::Opcode::VCmpGeU16, Opcode::CompareGeU16},
{Decoder::Opcode::VCmpFU32, Opcode::CompareFalse},
{Decoder::Opcode::VCmpLtU32, Opcode::CompareLtU32},
{Decoder::Opcode::VCmpEqU32, Opcode::CompareEqU32},
{Decoder::Opcode::VCmpLeU32, Opcode::CompareLeU32},
{Decoder::Opcode::VCmpGtU32, Opcode::CompareGtU32},
{Decoder::Opcode::VCmpNeU32, Opcode::CompareNeU32},
{Decoder::Opcode::VCmpGeU32, Opcode::CompareGeU32},
{Decoder::Opcode::VCmpTU32, Opcode::CompareTrue},
{Decoder::Opcode::VCmpNeU64, Opcode::CompareNeU64},
{Decoder::Opcode::VCmpxLtU32, Opcode::CompareMaskLtU32},
{Decoder::Opcode::VCmpxEqU32, Opcode::CompareMaskEqU32},
{Decoder::Opcode::VCmpxLeU32, Opcode::CompareMaskLeU32},
{Decoder::Opcode::VCmpxGtU32, Opcode::CompareMaskGtU32},
{Decoder::Opcode::VCmpxNeU32, Opcode::CompareMaskNeU32},
{Decoder::Opcode::VCmpxGeU32, Opcode::CompareMaskGeU32},
{Decoder::Opcode::SLoadDword, Opcode::SLoadDword},
{Decoder::Opcode::SBufferLoadDword, Opcode::SBufferLoadDword},
{Decoder::Opcode::BufferLoadDword, Opcode::BufferLoadDword},
{Decoder::Opcode::BufferLoadSbyte, Opcode::BufferLoadSbyte},
{Decoder::Opcode::BufferLoadSshort, Opcode::BufferLoadSshort},
{Decoder::Opcode::BufferStoreByte, Opcode::BufferStoreByte},
{Decoder::Opcode::BufferStoreShort, Opcode::BufferStoreShort},
{Decoder::Opcode::BufferStoreDword, Opcode::BufferStoreDword},
{Decoder::Opcode::BufferAtomicSwap, Opcode::AtomicSwapU32},
{Decoder::Opcode::BufferAtomicAdd, Opcode::AtomicAddU32},
{Decoder::Opcode::BufferAtomicSub, Opcode::AtomicSubU32},
{Decoder::Opcode::BufferAtomicSMin, Opcode::AtomicSMinI32},
{Decoder::Opcode::BufferAtomicUMin, Opcode::AtomicUMinU32},
{Decoder::Opcode::BufferAtomicSMax, Opcode::AtomicSMaxI32},
{Decoder::Opcode::BufferAtomicUMax, Opcode::AtomicUMaxU32},
{Decoder::Opcode::BufferAtomicAnd, Opcode::AtomicAndU32},
{Decoder::Opcode::BufferAtomicOr, Opcode::AtomicOrU32},
{Decoder::Opcode::BufferAtomicXor, Opcode::AtomicXorU32},
{Decoder::Opcode::FlatLoadUbyte, Opcode::FlatLoadUbyte},
{Decoder::Opcode::FlatLoadSbyte, Opcode::FlatLoadSbyte},
{Decoder::Opcode::FlatLoadSshort, Opcode::FlatLoadSshort},
{Decoder::Opcode::FlatStoreByte, Opcode::FlatStoreByte},
{Decoder::Opcode::FlatStoreShort, Opcode::FlatStoreShort},
{Decoder::Opcode::FlatStoreDword, Opcode::FlatStoreDword},
{Decoder::Opcode::DsAddU32, Opcode::AtomicAddU32},
{Decoder::Opcode::DsAddRtnU32, Opcode::AtomicAddU32},
{Decoder::Opcode::DsSubU32, Opcode::AtomicSubU32},
{Decoder::Opcode::DsSubRtnU32, Opcode::AtomicSubU32},
{Decoder::Opcode::DsMinI32, Opcode::AtomicSMinI32},
{Decoder::Opcode::DsMinRtnI32, Opcode::AtomicSMinI32},
{Decoder::Opcode::DsMaxI32, Opcode::AtomicSMaxI32},
{Decoder::Opcode::DsMaxRtnI32, Opcode::AtomicSMaxI32},
{Decoder::Opcode::DsMinU32, Opcode::AtomicUMinU32},
{Decoder::Opcode::DsMinRtnU32, Opcode::AtomicUMinU32},
{Decoder::Opcode::DsMaxU32, Opcode::AtomicUMaxU32},
{Decoder::Opcode::DsMaxRtnU32, Opcode::AtomicUMaxU32},
{Decoder::Opcode::DsAndB32, Opcode::AtomicAndU32},
{Decoder::Opcode::DsAndRtnB32, Opcode::AtomicAndU32},
{Decoder::Opcode::DsOrB32, Opcode::AtomicOrU32},
{Decoder::Opcode::DsOrRtnB32, Opcode::AtomicOrU32},
{Decoder::Opcode::DsXorB32, Opcode::AtomicXorU32},
{Decoder::Opcode::DsXorRtnB32, Opcode::AtomicXorU32},
{Decoder::Opcode::DsWrxchgRtnB32, Opcode::AtomicSwapU32},
{Decoder::Opcode::DsReadSbyte, Opcode::DsReadSbyte},
{Decoder::Opcode::DsReadUbyte, Opcode::DsReadUbyte},
{Decoder::Opcode::DsReadSshort, Opcode::DsReadSshort},
{Decoder::Opcode::DsReadUshort, Opcode::DsReadUshort},
{Decoder::Opcode::DsRead2B32, Opcode::DsReadB32},
{Decoder::Opcode::DsReadB32, Opcode::DsReadB32},
{Decoder::Opcode::DsReadB64, Opcode::DsReadB32},
{Decoder::Opcode::DsRead2B64, Opcode::DsReadB32},
{Decoder::Opcode::DsRead2St64B64, Opcode::DsReadB32},
{Decoder::Opcode::DsReadB96, Opcode::DsReadB32},
{Decoder::Opcode::DsReadB128, Opcode::DsReadB32},
{Decoder::Opcode::DsWriteByte, Opcode::DsWriteByte},
{Decoder::Opcode::DsWriteShort, Opcode::DsWriteShort},
{Decoder::Opcode::DsWrite2B32, Opcode::DsWriteB32},
{Decoder::Opcode::DsWrite2St64B32, Opcode::DsWriteB32},
{Decoder::Opcode::DsWrite2B64, Opcode::DsWriteB32},
{Decoder::Opcode::DsWrite2St64B64, Opcode::DsWriteB32},
{Decoder::Opcode::DsWriteB32, Opcode::DsWriteB32},
{Decoder::Opcode::DsWriteB64, Opcode::DsWriteB32},
{Decoder::Opcode::DsWriteB96, Opcode::DsWriteB32},
{Decoder::Opcode::DsWriteB128, Opcode::DsWriteB32},
{Decoder::Opcode::DsMinF32, Opcode::DsMinF32},
{Decoder::Opcode::DsMaxF32, Opcode::DsMaxF32},
{Decoder::Opcode::DsConsume, Opcode::DsConsume},
{Decoder::Opcode::DsAppend, Opcode::DsAppend},
{Decoder::Opcode::DsWriteAddtidB32, Opcode::DsWriteAddtidB32},
{Decoder::Opcode::DsReadAddtidB32, Opcode::DsReadAddtidB32},
{Decoder::Opcode::ImageGetResinfo, Opcode::ImageGetResinfo},
{Decoder::Opcode::ImageGetLod, Opcode::ImageGetLod},
{Decoder::Opcode::ImageLoad, Opcode::ImageLoad},
{Decoder::Opcode::ImageLoadMip, Opcode::ImageLoad},
{Decoder::Opcode::ImageStore, Opcode::ImageStore},
{Decoder::Opcode::ImageStoreMip, Opcode::ImageStore},
{Decoder::Opcode::ImageAtomicAdd, Opcode::AtomicAddU32},
{Decoder::Opcode::ImageAtomicUMin, Opcode::AtomicUMinU32},
{Decoder::Opcode::ImageAtomicUMax, Opcode::AtomicUMaxU32},
{Decoder::Opcode::ImageAtomicAnd, Opcode::AtomicAndU32},
{Decoder::Opcode::ImageAtomicOr, Opcode::AtomicOrU32},
{Decoder::Opcode::ImageAtomicXor, Opcode::AtomicXorU32},
{Decoder::Opcode::ImageSample, Opcode::ImageSample},
{Decoder::Opcode::ImageGather4Lz, Opcode::ImageGather4},
{Decoder::Opcode::ImageGather4C, Opcode::ImageGather4},
{Decoder::Opcode::ImageGather4CLz, Opcode::ImageGather4},
{Decoder::Opcode::ImageGather4LzO, Opcode::ImageGather4},
{Decoder::Opcode::ImageGather4CO, Opcode::ImageGather4},
{Decoder::Opcode::ImageGather4CLzO, Opcode::ImageGather4},
{Decoder::Opcode::ImageGather4H, Opcode::ImageGather4},
};
} // namespace
void SetError(std::string* error, const char* message) {
if (error != nullptr) {
*error = message;
}
}
Opcode LookupIrOpcode(Decoder::Opcode opcode) {
for (const auto& op: LOWER_OPS) {
if (op.decoded == opcode) {
return op.ir;
}
}
return Opcode::MoveU32;
}
bool IsImplemented(Decoder::Opcode opcode) {
for (const auto& op: LOWER_OPS) {
if (op.decoded == opcode) {
return true;
}
}
return false;
}
bool IsReversedBinary(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::VSubrevF32:
case Decoder::Opcode::VSubrevF16:
case Decoder::Opcode::VSubrevNcU32:
case Decoder::Opcode::VLshlrevB32:
case Decoder::Opcode::VLshrrevB32:
case Decoder::Opcode::VAshrrevI32:
case Decoder::Opcode::VLshlrevB16:
case Decoder::Opcode::VLshrrevB16:
case Decoder::Opcode::VAshrrevI16:
case Decoder::Opcode::VSubrevI32: return true;
default: return false;
}
}
bool IsVectorCarryOutOpcode(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::VAddI32:
case Decoder::Opcode::VSubI32:
case Decoder::Opcode::VSubrevI32: return true;
default: return false;
}
}
bool ScalarResultWritesSccNonZero(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::SAbsI32:
case Decoder::Opcode::SBcnt1I32B32:
case Decoder::Opcode::SBcnt1I32B64:
case Decoder::Opcode::SAndB32:
case Decoder::Opcode::SAndn2B32:
case Decoder::Opcode::SOrB32:
case Decoder::Opcode::SOrn2B32:
case Decoder::Opcode::SXorB32:
case Decoder::Opcode::SNandB32:
case Decoder::Opcode::SNorB32:
case Decoder::Opcode::SXnorB32:
case Decoder::Opcode::SNotB32:
case Decoder::Opcode::SLshlB32:
case Decoder::Opcode::SLshrB32:
case Decoder::Opcode::SAshrI32:
case Decoder::Opcode::SBfeU32:
case Decoder::Opcode::SAndB64:
case Decoder::Opcode::SAndn2B64:
case Decoder::Opcode::SNotB64:
case Decoder::Opcode::SOrB64:
case Decoder::Opcode::SOrn2B64:
case Decoder::Opcode::SXorB64:
case Decoder::Opcode::SNandB64:
case Decoder::Opcode::SNorB64:
case Decoder::Opcode::SXnorB64:
case Decoder::Opcode::SLshlB64:
case Decoder::Opcode::SLshrB64:
case Decoder::Opcode::SBfeU64:
case Decoder::Opcode::SWqmB64: return true;
default: return false;
}
}
bool ScalarResultIs64Bit(Decoder::Opcode opcode) {
switch (opcode) {
case Decoder::Opcode::SAndB64:
case Decoder::Opcode::SAndn2B64:
case Decoder::Opcode::SNotB64:
case Decoder::Opcode::SOrB64:
case Decoder::Opcode::SOrn2B64:
case Decoder::Opcode::SXorB64:
case Decoder::Opcode::SNandB64:
case Decoder::Opcode::SNorB64:
case Decoder::Opcode::SXnorB64:
case Decoder::Opcode::SLshlB64:
case Decoder::Opcode::SLshrB64:
case Decoder::Opcode::SBfeU64:
case Decoder::Opcode::SWqmB64: return true;
default: return false;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::IR
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAluOps.cpp
+2549
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterAnalysis.cpp
+752
View File
@@ -0,0 +1,752 @@
#include "common/assert.h"
#include "graphics/guest_gpu/gpu_defs.h"
#include "graphics/shader/recompiler/SpirvEmitter.h"
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
const auto* ps = state.pixel_input_info;
if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
return attr;
}
// VINTRP ATTR selects the PS input slot. SPI_PS_INPUT_CNTL maps that slot to a
// VS parameter export, which is the SPIR-V location we must link against.
return ps->interpolator_settings[attr] & PsInputOffsetMask;
}
uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
bool used_locations[32] = {};
for (const auto& input: state.inputs) {
if (input.kind != IR::StageInputKind::Parameter) {
continue;
}
auto location = PixelParameterMappedLocation(state, input.location);
if (location < std::size(used_locations) && used_locations[location]) {
auto fallback_location = input.location;
while (fallback_location < std::size(used_locations) &&
used_locations[fallback_location]) {
fallback_location++;
}
EXIT_NOT_IMPLEMENTED(fallback_location >= std::size(used_locations));
location = fallback_location;
}
if (input.location == attr) {
return location;
}
if (location < std::size(used_locations)) {
used_locations[location] = true;
}
}
return PixelParameterMappedLocation(state, attr);
}
bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
const auto* ps = state.pixel_input_info;
if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
return false;
}
return (ps->interpolator_settings[attr] & PsInputFlatShade) != 0;
}
void SetError(std::string* error, const char* message) {
if (error != nullptr) {
*error = message;
}
}
void CollectRegister(std::vector<RegisterBinding>* registers, IR::Register reg) {
if (std::any_of(registers->begin(), registers->end(),
[reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
return;
}
registers->push_back({reg, 0});
}
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
IR::Operand operand;
operand.kind = IR::OperandKind::Register;
operand.reg = {file, index};
return operand;
}
IR::Register SccRegister() {
return {IR::RegisterFile::Scc, static_cast<uint32_t>(Decoder::OperandKind::Scc)};
}
IR::Operand SccOperand() {
return MakeRegisterOperand(IR::RegisterFile::Scc,
static_cast<uint32_t>(Decoder::OperandKind::Scc));
}
bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
}
bool IsMaskRegisterFile(IR::RegisterFile file) {
return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc;
}
bool IsSccOperand(const IR::Operand& operand) {
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
}
bool IsCompareOpcode(IR::Opcode op);
void CollectMaskStateRegisters(std::vector<RegisterBinding>* registers) {
CollectRegister(registers, {IR::RegisterFile::Exec, 0});
CollectRegister(registers, {IR::RegisterFile::Exec, 1});
CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
CollectRegister(registers, {IR::RegisterFile::Vcc, 1});
CollectRegister(registers, SccRegister());
}
void CollectSequentialRegisters(std::vector<RegisterBinding>* registers, const IR::Operand& base,
uint32_t count) {
if (base.kind != IR::OperandKind::Register) {
return;
}
for (uint32_t i = 0; i < count; i++) {
auto reg = base.reg;
reg.index += i;
CollectRegister(registers, reg);
}
}
uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& registers) {
uint32_t max_end = 0;
for (const auto& binding: registers) {
if (binding.reg.file == IR::RegisterFile::Vector) {
max_end = std::max(max_end, binding.reg.index + 1u);
}
}
return std::min(max_end, 256u);
}
void CollectMoveRelSourceRegisters(const IR::Program& program,
std::vector<RegisterBinding>* registers) {
const auto max_vector_end = MaxCollectedVectorRegisterEnd(*registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op != IR::Opcode::MoveRelSourceU32 ||
inst.src[0].kind != IR::OperandKind::Register ||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
continue;
}
const auto base_index = inst.src[0].reg.index;
const auto limit = std::min(std::max(max_vector_end, base_index + 1u), 256u);
CollectSequentialRegisters(registers, inst.src[0], limit - base_index);
}
}
}
bool IsPairDwordOpcode(IR::Opcode op) {
switch (op) {
case IR::Opcode::MoveU64:
case IR::Opcode::WqmB64:
case IR::Opcode::SaveexecB64:
case IR::Opcode::BitwiseAndU64:
case IR::Opcode::BitwiseAndNotU64:
case IR::Opcode::BitwiseOrU64:
case IR::Opcode::BitwiseOrNotU64:
case IR::Opcode::BitwiseXorU64:
case IR::Opcode::BitwiseNandU64:
case IR::Opcode::BitwiseNorU64:
case IR::Opcode::BitwiseXnorU64:
case IR::Opcode::BitwiseNotU64:
case IR::Opcode::BitFieldMaskU64:
case IR::Opcode::BitFieldExtractU64:
case IR::Opcode::BitReplicateB64B32:
case IR::Opcode::ShiftLeftLogicalU64:
case IR::Opcode::ShiftRightLogicalU64:
case IR::Opcode::SelectU64: return true;
default: return false;
}
}
uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
switch (op) {
case IR::Opcode::BitFieldMaskU64: return 0;
case IR::Opcode::BitReplicateB64B32: return 0;
case IR::Opcode::BitFieldExtractU64: return 1;
case IR::Opcode::ShiftLeftLogicalU64:
case IR::Opcode::ShiftRightLogicalU64:
case IR::Opcode::BitwiseNotU64:
case IR::Opcode::SaveexecB64:
case IR::Opcode::WqmB64:
case IR::Opcode::MoveU64: return std::min<uint32_t>(src_count, 1u);
default: return src_count;
}
}
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>* registers) {
CollectMaskStateRegisters(registers);
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.dst.kind == IR::OperandKind::Register) {
CollectRegister(registers, inst.dst.reg);
}
if (inst.dst2.kind == IR::OperandKind::Register) {
CollectRegister(registers, inst.dst2.reg);
}
for (uint32_t i = 0; i < inst.src_count; i++) {
if (inst.src[i].kind == IR::OperandKind::Register) {
CollectRegister(registers, inst.src[i].reg);
}
}
if (IsPairDwordOpcode(inst.op)) {
CollectSequentialRegisters(registers, inst.dst, 2);
const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u;
const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count);
for (uint32_t i = first_pair_src; i < pair_src_count; i++) {
CollectSequentialRegisters(registers, inst.src[i], 2);
}
}
if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) {
CollectSequentialRegisters(registers, inst.src[0], 2);
}
if (inst.op == IR::Opcode::CompareNeU64) {
CollectSequentialRegisters(registers, inst.src[0], 2);
CollectSequentialRegisters(registers, inst.src[1], 2);
}
if (IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file != IR::RegisterFile::Scc) {
CollectSequentialRegisters(registers, inst.dst, 2);
}
if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) {
CollectSequentialRegisters(registers, inst.dst2, 2);
}
if (inst.op == IR::Opcode::UMadU64U32) {
CollectSequentialRegisters(registers, inst.dst, 2);
CollectSequentialRegisters(registers, inst.dst2, 2);
CollectSequentialRegisters(registers, inst.src[2], 2);
}
if (inst.op == IR::Opcode::SBufferLoadDword &&
inst.memory.kind == IR::ResourceKind::ScalarBuffer) {
CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u});
}
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) &&
inst.memory.image_address_components > 1u) {
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[0],
inst.memory.image_address_components);
}
if (inst.op == IR::Opcode::ImageGetLod) {
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[0],
inst.memory.image_address_components);
}
if (inst.op == IR::Opcode::ImageLoad) {
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[0],
inst.memory.image_address_components);
}
if (inst.op == IR::Opcode::ImageStore) {
CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords);
CollectSequentialRegisters(registers, inst.src[1],
inst.memory.image_address_components);
}
if (inst.memory.kind == IR::ResourceKind::StorageImageUint) {
CollectSequentialRegisters(registers, inst.src[1],
inst.memory.image_address_components);
}
}
}
CollectMoveRelSourceRegisters(program, registers);
}
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
uint32_t index) {
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
return binding.kind == kind && binding.index == index;
});
}
void CopyProgramInputsAndOutputs(EmitterState* state, const IR::Program& program) {
for (const auto& input: program.info.inputs) {
state->inputs.push_back(
{input.kind, input.location, input.component_count, 0, input.debug_name});
}
for (const auto& output: program.info.outputs) {
if (HasOutput(state->outputs, output.kind, output.index)) {
continue;
}
state->outputs.push_back(
{output.kind, output.index, output.location, 0, output.debug_name});
}
}
uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
if (exp.kind == IR::ExportTargetKind::Position) {
return state.per_vertex_variable;
}
if (exp.kind == IR::ExportTargetKind::MrtZ) {
return state.depth_variable;
}
for (const auto& binding: state.outputs) {
const auto expected_kind = exp.kind == IR::ExportTargetKind::Mrt
? IR::StageOutputKind::Mrt
: IR::StageOutputKind::Parameter;
if (binding.kind == expected_kind && binding.index == exp.index) {
return binding.variable_id;
}
}
return 0;
}
bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::ImageGetLod) {
return true;
}
}
}
return false;
}
bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::ImageGather4) {
return true;
}
}
}
return false;
}
bool IsLdsOpcode(IR::Opcode op) {
switch (op) {
case IR::Opcode::DsReadUbyte:
case IR::Opcode::DsReadSbyte:
case IR::Opcode::DsReadUshort:
case IR::Opcode::DsReadSshort:
case IR::Opcode::DsReadB32:
case IR::Opcode::DsWriteByte:
case IR::Opcode::DsWriteShort:
case IR::Opcode::DsWriteB32:
case IR::Opcode::DsMinF32:
case IR::Opcode::DsMaxF32:
case IR::Opcode::DsSwizzleB32:
case IR::Opcode::DsWriteAddtidB32:
case IR::Opcode::DsReadAddtidB32: return true;
default: return false;
}
}
bool ProgramNeedsFunctionLds(const IR::Program& program) {
if (program.stage == ShaderType::Compute) {
return false;
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (IsLdsOpcode(inst.op)) {
return true;
}
}
}
return false;
}
bool ProgramNeedsPixelValidMask(const IR::Program& program) {
if (program.stage != ShaderType::Pixel) {
return false;
}
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::Export && inst.export_info.vm) {
return true;
}
}
}
return false;
}
bool InstructionHasDppSource(const IR::Instruction& inst) {
for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) {
if (inst.src[i].dpp) {
return true;
}
}
return false;
}
bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::WqmB64) {
return true;
}
}
}
}
return ProgramRequiresExactSubgroupSize(program);
}
bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 ||
inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 ||
inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume ||
inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) {
return true;
}
}
}
return false;
}
bool IsCompareOpcode(IR::Opcode op) {
switch (op) {
case IR::Opcode::CompareFalse:
case IR::Opcode::CompareTrue:
case IR::Opcode::CompareEqU32:
case IR::Opcode::CompareNeU32:
case IR::Opcode::CompareGtU32:
case IR::Opcode::CompareGeU32:
case IR::Opcode::CompareLtU32:
case IR::Opcode::CompareLeU32:
case IR::Opcode::CompareNeU64:
case IR::Opcode::CompareMaskEqU32:
case IR::Opcode::CompareMaskNeU32:
case IR::Opcode::CompareMaskGtU32:
case IR::Opcode::CompareMaskGeU32:
case IR::Opcode::CompareMaskLtU32:
case IR::Opcode::CompareMaskLeU32:
case IR::Opcode::CompareEqI32:
case IR::Opcode::CompareNeI32:
case IR::Opcode::CompareGtI32:
case IR::Opcode::CompareGeI32:
case IR::Opcode::CompareLtI32:
case IR::Opcode::CompareLeI32:
case IR::Opcode::CompareEqI16:
case IR::Opcode::CompareNeI16:
case IR::Opcode::CompareGtI16:
case IR::Opcode::CompareGeI16:
case IR::Opcode::CompareLtI16:
case IR::Opcode::CompareLeI16:
case IR::Opcode::CompareMaskEqI32:
case IR::Opcode::CompareMaskNeI32:
case IR::Opcode::CompareMaskGtI32:
case IR::Opcode::CompareMaskGeI32:
case IR::Opcode::CompareMaskLtI32:
case IR::Opcode::CompareMaskLeI32:
case IR::Opcode::CompareEqU16:
case IR::Opcode::CompareNeU16:
case IR::Opcode::CompareGtU16:
case IR::Opcode::CompareGeU16:
case IR::Opcode::CompareLtU16:
case IR::Opcode::CompareLeU16:
case IR::Opcode::CompareEqF32:
case IR::Opcode::CompareNeF32:
case IR::Opcode::CompareGtF32:
case IR::Opcode::CompareGeF32:
case IR::Opcode::CompareLtF32:
case IR::Opcode::CompareLeF32:
case IR::Opcode::CompareOrderedF32:
case IR::Opcode::CompareUnorderedF32:
case IR::Opcode::CompareUnordEqF32:
case IR::Opcode::CompareUnordNeF32:
case IR::Opcode::CompareUnordGtF32:
case IR::Opcode::CompareUnordGeF32:
case IR::Opcode::CompareUnordLtF32:
case IR::Opcode::CompareUnordLeF32:
case IR::Opcode::CompareClassF32:
case IR::Opcode::CompareEqF16:
case IR::Opcode::CompareNeF16:
case IR::Opcode::CompareGtF16:
case IR::Opcode::CompareGeF16:
case IR::Opcode::CompareLtF16:
case IR::Opcode::CompareLeF16:
case IR::Opcode::CompareUnordNeF16:
case IR::Opcode::CompareMaskEqF16:
case IR::Opcode::CompareMaskNeF16:
case IR::Opcode::CompareMaskGtF16:
case IR::Opcode::CompareMaskGeF16:
case IR::Opcode::CompareMaskLtF16:
case IR::Opcode::CompareMaskLeF16:
case IR::Opcode::CompareMaskUnordNeF16:
case IR::Opcode::CompareMaskUnordGeF16:
case IR::Opcode::CompareMaskEqF32:
case IR::Opcode::CompareMaskNeF32:
case IR::Opcode::CompareMaskGtF32:
case IR::Opcode::CompareMaskGeF32:
case IR::Opcode::CompareMaskLtF32:
case IR::Opcode::CompareMaskLeF32:
case IR::Opcode::CompareMaskUnordEqF32:
case IR::Opcode::CompareMaskUnordNeF32:
case IR::Opcode::CompareMaskUnordGtF32:
case IR::Opcode::CompareMaskUnordGeF32:
case IR::Opcode::CompareMaskUnordLtF32:
case IR::Opcode::CompareMaskUnordLeF32: return true;
default: return false;
}
}
bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
for (const auto& block: program.blocks) {
for (const auto& inst: block.instructions) {
if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 ||
inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend ||
inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 ||
inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export ||
InstructionHasDppSource(inst)) {
return true;
}
if (IsCompareOpcode(inst.op)) {
return true;
}
if (inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Vector) {
return true;
}
if (inst.dst2.kind == IR::OperandKind::Register &&
inst.dst2.reg.file == IR::RegisterFile::Vector) {
return true;
}
}
}
return false;
}
uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
for (const auto& binding: state.registers) {
if (binding.reg == reg) {
return binding.pointer_id;
}
}
return 0;
}
uint32_t ConstantU32(EmitterState* state, uint32_t value);
void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value);
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state);
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane);
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* reason) {
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
state.program->shader_hash, static_cast<unsigned>(state.stage), resource,
static_cast<unsigned>(kind), reason);
std::abort();
}
DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
IR::DescriptorBindingKind kind, uint32_t resource) {
const auto* descriptor = IR::FindBinding(state.program->bindings, kind);
if (descriptor == nullptr) {
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
}
const auto found =
std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
if (found == descriptor->resources.end()) {
ExitDescriptorBindingFailure(state, kind, resource,
"resource is absent from descriptor group");
}
return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
}
uint32_t DescriptorElementPointer(EmitterState* state, uint32_t result_ptr_type,
uint32_t variable_id, uint32_t array_index,
IR::DescriptorBindingKind kind, uint32_t resource,
const char* variable_name) {
if (variable_id == 0) {
ExitDescriptorBindingFailure(*state, kind, resource, variable_name);
}
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction(
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
return pointer;
}
ImageViewKind ImageViewKindFromDimension(Decoder::ImageDimension dimension) {
switch (dimension) {
case Decoder::ImageDimension::Dim2DArray: return ImageViewKind::Dim2DArray;
case Decoder::ImageDimension::Dim3D: return ImageViewKind::Dim3D;
default: return ImageViewKind::Dim2D;
}
}
ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
uint32_t use_pc) {
(void)state;
(void)use_pc;
return ImageViewKindFromDimension(mem.image_dimension);
}
ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
bool uint_image, uint32_t use_pc) {
(void)state;
(void)uint_image;
(void)use_pc;
return ImageViewKindFromDimension(mem.image_dimension);
}
uint32_t ImageViewCoordinateComponents(ImageViewKind view) {
switch (view) {
case ImageViewKind::Dim2DArray:
case ImageViewKind::Dim3D: return 3u;
default: return 2u;
}
}
uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) {
return state.sampled_images[SampledImageIndex(integer, view)].image_type;
}
uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) {
return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type;
}
uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) {
return ImageViewCoordinateComponents(view) == 3u ? state.vec3_uint_type : state.vec2_uint_type;
}
uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) {
if (uint_image) {
switch (view) {
case ImageViewKind::Dim2DArray: return state.storage_image_uint_2d_array_type;
case ImageViewKind::Dim3D: return state.storage_image_uint_3d_type;
default: return state.storage_image_uint_type;
}
}
switch (view) {
case ImageViewKind::Dim2DArray: return state.storage_image_2d_array_type;
case ImageViewKind::Dim3D: return state.storage_image_3d_type;
default: return state.storage_image_type;
}
}
uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) {
if (uint_image) {
switch (view) {
case ImageViewKind::Dim2DArray: return state.ptr_uniform_storage_image_uint_2d_array;
case ImageViewKind::Dim3D: return state.ptr_uniform_storage_image_uint_3d;
default: return state.ptr_uniform_storage_image_uint;
}
}
switch (view) {
case ImageViewKind::Dim2DArray: return state.ptr_uniform_storage_image_2d_array;
case ImageViewKind::Dim3D: return state.ptr_uniform_storage_image_3d;
default: return state.ptr_uniform_storage_image;
}
}
uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) {
if (uint_image) {
switch (view) {
case ImageViewKind::Dim2DArray: return state.storage_image_uint_2d_array_variable;
case ImageViewKind::Dim3D: return state.storage_image_uint_3d_variable;
default: return state.storage_image_uint_variable;
}
}
switch (view) {
case ImageViewKind::Dim2DArray: return state.storage_image_2d_array_variable;
case ImageViewKind::Dim3D: return state.storage_image_3d_variable;
default: return state.storage_image_variable;
}
}
IR::DescriptorBindingKind StorageBindingKind(bool uint_image, ImageViewKind view) {
if (uint_image) {
switch (view) {
case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::StorageUint2DArray;
case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
default: return IR::DescriptorBindingKind::StorageUint2D;
}
}
switch (view) {
case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
default: return IR::DescriptorBindingKind::Storage2D;
}
}
uint32_t LoadSampledImageDescriptor(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
ImageViewKind view) {
(void)use_pc;
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
const auto kind = SampledBindingKind(integer, view);
const auto binding = ResourceForDescriptor(*state, kind, mem.resource);
const auto& descriptors = state->sampled_images[SampledImageIndex(integer, view)];
const auto pointer = DescriptorElementPointer(
state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
mem.resource, "sampled image descriptor array was not emitted");
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, ImageViewImageType(*state, view, integer), image, pointer});
return image;
}
uint32_t LoadSamplerDescriptor(EmitterState* state, uint32_t sampler, uint32_t use_pc) {
(void)use_pc;
const auto binding =
ResourceForDescriptor(*state, IR::DescriptorBindingKind::Samplers, sampler);
const auto pointer = DescriptorElementPointer(
state, state->ptr_uniform_sampler, state->sampler_variable, binding.array_index,
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
const auto sampler_id = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->sampler_type, sampler_id, pointer});
return sampler_id;
}
uint32_t MakeSampledImage(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
ImageViewKind view) {
const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view);
const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
if (image == 0 || sampler == 0) {
ExitDescriptorBindingFailure(
*state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
"sampled image or sampler descriptor load failed");
}
const auto sampled_image = state->builder.AllocateId();
state->builder.AddFunction(
{OpSampledImage,
ImageViewSampledImageType(*state, view, mem.kind == IR::ResourceKind::ImageUint),
sampled_image, image, sampler});
return sampled_image;
}
uint32_t StorageImageDescriptorPointer(EmitterState* state, uint32_t resource, bool uint_image,
uint32_t use_pc, ImageViewKind view) {
(void)use_pc;
const auto kind = StorageBindingKind(uint_image, view);
const auto binding = ResourceForDescriptor(*state, kind, resource);
const auto ptr_type = StorageImagePointerType(*state, uint_image, view);
const auto variable = StorageImageVariable(*state, uint_image, view);
return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
"storage image descriptor array was not emitted");
}
uint32_t LoadStorageImageDescriptor(EmitterState* state, uint32_t resource, bool uint_image,
uint32_t use_pc, ImageViewKind view) {
const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
if (pointer == 0) {
ExitDescriptorBindingFailure(*state, StorageBindingKind(uint_image, view), resource,
"storage image descriptor pointer creation failed");
}
const auto type = StorageImageType(*state, uint_image, view);
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, type, image, pointer});
return image;
}
uint32_t ExecutionModelForStage(ShaderType stage) {
switch (stage) {
case ShaderType::Vertex: return ExecutionModelVertex;
case ShaderType::Pixel: return ExecutionModelFragment;
default: return ExecutionModelGLCompute;
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterControlFlow.cpp
+1095
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterExport.cpp
+154
View File
@@ -0,0 +1,154 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst,
uint32_t component) {
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
if (!enabled || component >= inst.src_count || component >= 4u) {
return ConstantF32(state, component == 3u ? 0x3f800000u : 0u);
}
return EmitFloatLoad(state, inst.src[component]);
}
uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
if (inst.export_info.compr) {
uint32_t components[4] = {
ConstantF32(state, 0u),
ConstantF32(state, 0u),
ConstantF32(state, 0u),
ConstantF32(state, 0x3f800000u),
};
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
pair_index++) {
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
const auto unpacked = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked,
state->glsl_std450, GlslUnpackHalf2x16, raw});
for (uint32_t lane = 0; lane < 2u; lane++) {
const auto component = pair_index * 2u + lane;
if (((inst.export_info.en >> component) & 1u) == 0) {
continue;
}
components[component] = state->builder.AllocateId();
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, components[component], unpacked, lane});
}
}
const auto vec = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec,
components[0], components[1], components[2], components[3]});
return vec;
}
const auto x = EmitExportComponentF32(state, inst, 0);
const auto y = EmitExportComponentF32(state, inst, 1);
const auto z = EmitExportComponentF32(state, inst, 2);
const auto w = EmitExportComponentF32(state, inst, 3);
const auto vec = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec, x, y, z, w});
return vec;
}
bool ExportWritesData(const IR::Instruction& inst) {
switch (inst.export_info.kind) {
case IR::ExportTargetKind::Null:
case IR::ExportTargetKind::Primitive: return false;
case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0;
default: return inst.export_info.en != 0;
}
}
void EmitMrtZExport(EmitterState* state, const IR::Instruction& inst) {
if ((inst.export_info.en & 0x1u) != 0 && state->depth_variable != 0) {
state->builder.AddFunction(
{OpStore, state->depth_variable, EmitExportComponentF32(state, inst, 0)});
}
if ((inst.export_info.en & 0x4u) != 0 && state->sample_mask_variable != 0) {
const auto raw =
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
const auto mask = state->builder.AllocateId();
const auto ptr = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, mask, raw});
state->builder.AddFunction({OpAccessChain, state->ptr_output_int, ptr,
state->sample_mask_variable, ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, ptr, mask});
}
}
void EmitExport(EmitterState* state, const IR::Instruction& inst) {
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
return;
}
if (!ExportWritesData(inst)) {
return;
}
if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) {
EmitMrtZExport(state, inst);
return;
}
const auto variable = OutputVariableForExport(*state, inst.export_info);
if (variable == 0) {
return;
}
const auto value = EmitExportVec4F32(state, inst);
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, state->ptr_output_vec4_float, pointer, variable,
ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, pointer, value});
return;
}
state->builder.AddFunction({OpStore, variable, value});
}
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
}
void EmitKillIfBoolFalse(EmitterState* state, uint32_t active) {
const auto kill_label = state->builder.AllocateId();
const auto merge_label = state->builder.AllocateId();
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
state->builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
state->builder.AddFunction({OpLabel, kill_label});
state->builder.AddFunction({OpKill});
state->builder.AddFunction({OpLabel, merge_label});
}
void EmitUpdatePixelValidMask(EmitterState* state) {
if (state->pixel_valid_mask_variable == 0) {
return;
}
const auto active = EmitExecActiveBool(state);
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
state->builder.AddFunction({OpStore, state->pixel_valid_mask_variable, value});
}
void EmitKillIfPixelValidMaskInactive(EmitterState* state) {
if (state->pixel_valid_mask_variable == 0) {
return;
}
const auto mask_value = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpLoad, state->uint_type, mask_value, state->pixel_valid_mask_variable});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, mask_value, ConstantU32(state, 0)});
EmitKillIfBoolFalse(state, active);
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterFlowOps.cpp
+381
View File
@@ -0,0 +1,381 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
void EmitMoveU32(EmitterState* state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
EmitStoreU32(state, inst.dst, value);
}
void EmitMoveF32Bits(EmitterState* state, const IR::Instruction& inst) {
const auto value = EmitFloatLoad(state, inst.src[0]);
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, value});
EmitStoreU32(state, inst.dst, bits);
}
uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
uint32_t limit = base.index + 1u;
for (const auto& binding: state.registers) {
if (binding.reg.file == IR::RegisterFile::Vector) {
limit = std::max(limit, binding.reg.index + 1u);
}
}
return std::min(limit, 256u);
}
void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) {
if (inst.src[0].kind != IR::OperandKind::Register ||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
EmitMoveU32(state, inst);
return;
}
const auto base_reg = inst.src[0].reg;
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
auto selected = EmitRegisterLoad(state, base_reg);
const auto limit = MoveRelRegisterLimit(*state, base_reg);
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
const auto match = state->builder.AllocateId();
const auto next = state->builder.AllocateId();
state->builder.AddFunction(
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state->builder.AddFunction({OpSelect, state->uint_type, next, match, candidate, selected});
selected = next;
}
EmitStoreU32(state, inst.dst, selected);
}
void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) {
if (inst.dst.kind != IR::OperandKind::Register ||
inst.dst.reg.file != IR::RegisterFile::Vector) {
EmitMoveU32(state, inst);
return;
}
const auto base_reg = inst.dst.reg;
const auto value = EmitValueLoad(state, inst.src[0]);
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
const auto exec_active = EmitExecActiveBool(state);
const auto limit = MoveRelRegisterLimit(*state, base_reg);
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, reg});
if (pointer == 0) {
continue;
}
const auto old_value = state->builder.AllocateId();
const auto match = state->builder.AllocateId();
const auto write = state->builder.AllocateId();
const auto merged = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer});
state->builder.AddFunction(
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
state->builder.AddFunction({OpLogicalAnd, state->bool_type, write, exec_active, match});
state->builder.AddFunction({OpSelect, state->uint_type, merged, write, value, old_value});
state->builder.AddFunction({OpStore, pointer, merged});
}
}
void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
IsMaskRegisterFile(inst.dst.reg.file)) {
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
return;
}
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
EmitStoreU32(state, inst.dst, low);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
}
uint32_t EmitWqmLaneU32(EmitterState* state, uint32_t src) {
uint32_t ret = ConstantU32(state, 0);
for (uint32_t i = 0; i < 8u; i++) {
const auto mask = 0x0fu << (i * 4u);
const auto masked = state->builder.AllocateId();
const auto non_zero = state->builder.AllocateId();
const auto expanded = state->builder.AllocateId();
const auto combined = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, masked, src, ConstantU32(state, mask)});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, non_zero, masked, ConstantU32(state, 0)});
state->builder.AddFunction({OpSelect, state->uint_type, expanded, non_zero,
ConstantU32(state, mask), ConstantU32(state, 0)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, ret, expanded});
ret = combined;
}
return ret;
}
void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) {
if (!state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
inst.dst.reg.file == IR::RegisterFile::Scalar) {
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
const auto ret_low = EmitWqmLaneU32(state, low);
const auto ret_high = EmitWqmLaneU32(state, high);
EmitStoreU32(state, inst.dst, ret_low);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
return;
}
const auto ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup),
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
const auto low = state->builder.AllocateId();
const auto high = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, ballot, 0});
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, ballot, 1});
const auto wqm_low = EmitWqmLaneU32(state, low);
const auto wqm_high = EmitWqmLaneU32(state, high);
const auto lane = EmitSubgroupLocalInvocationId(state);
const auto high_half = state->builder.AllocateId();
const auto mask = state->builder.AllocateId();
const auto bit_index = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
const auto hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, high_half, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, mask, high_half, wqm_high, wqm_low});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, bit_index, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), bit_index});
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
if (state->per_invocation_masks) {
EmitPerInvocationMask(state, inst.dst, active);
} else {
const auto result = state->builder.AllocateId();
state->builder.AddFunction({OpSelect, state->uint_type, result, active,
ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, inst.dst, result);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
}
}
void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst) {
const auto old_exec = EmitExecActiveBool(state);
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
EmitPerInvocationMask(state, inst.dst, old_exec);
uint32_t lhs = old_exec;
uint32_t rhs = src;
bool use_or = false;
switch (inst.saveexec_mode) {
case IR::SaveexecMode::And: break;
case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break;
case IR::SaveexecMode::Orn2:
lhs = EmitLogicalNotBool(state, old_exec);
use_or = true;
break;
}
const auto result =
use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs);
EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result);
EmitStoreSccBool(state, result);
}
void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks) {
EmitSaveexecPerInvocation(state, inst);
return;
}
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
const auto src_low = EmitValueLoad(state, inst.src[0]);
EmitStoreU32(state, inst.dst, old_low);
uint32_t rhs_low = src_low;
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
rhs_low = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
}
const auto new_low = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, new_low, rhs_low, old_low});
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
if (state->wave_size == 32u) {
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
}
const auto cond = state->builder.AllocateId();
const auto scc = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, cond, new_low, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, SccOperand(), scc);
}
void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) {
if (state->per_invocation_masks) {
EmitSaveexecPerInvocation(state, inst);
return;
}
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0);
const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1);
EmitStoreU32(state, inst.dst, old_low);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high);
uint32_t lhs_low = old_low;
uint32_t lhs_high = old_high;
uint32_t rhs_low = src_low;
uint32_t rhs_high = src_high;
uint32_t opcode = OpBitwiseAnd;
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
rhs_low = state->builder.AllocateId();
rhs_high = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
state->builder.AddFunction({OpNot, state->uint_type, rhs_high, src_high});
}
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
lhs_low = state->builder.AllocateId();
lhs_high = state->builder.AllocateId();
state->builder.AddFunction({OpNot, state->uint_type, lhs_low, old_low});
state->builder.AddFunction({OpNot, state->uint_type, lhs_high, old_high});
opcode = OpBitwiseOr;
}
const auto new_low = state->builder.AllocateId();
const auto new_high = state->builder.AllocateId();
state->builder.AddFunction({opcode, state->uint_type, new_low, lhs_low, rhs_low});
state->builder.AddFunction({opcode, state->uint_type, new_high, lhs_high, rhs_high});
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
const auto active_new_high = state->wave_size == 32u ? ConstantU32(state, 0) : new_high;
const auto mask = state->builder.AllocateId();
const auto cond = state->builder.AllocateId();
const auto scc = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, mask, new_low, active_new_high});
state->builder.AddFunction({OpINotEqual, state->bool_type, cond, mask, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
EmitStoreU32(state, SccOperand(), scc);
}
void EmitReadFirstLaneU32(EmitterState* state, const IR::Instruction& inst) {
const auto src = EmitValueLoad(state, inst.src[0]);
const auto active = EmitExecActiveBool(state);
const auto ballot = state->builder.AllocateId();
const auto first_lane = state->builder.AllocateId();
const auto first_value = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), active});
state->builder.AddFunction({OpGroupNonUniformBallotFindLSB, state->uint_type, first_lane,
ConstantU32(state, ScopeSubgroup), ballot});
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, first_value,
ConstantU32(state, ScopeSubgroup), src, first_lane});
EmitStoreU32(state, inst.dst, first_value);
}
uint32_t EmitLaneIndex(EmitterState* state, const IR::Operand& operand) {
const auto lane = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, lane, EmitValueLoad(state, operand),
ConstantU32(state, 63)});
return lane;
}
void EmitReadLaneU32(EmitterState* state, const IR::Instruction& inst) {
const auto src = EmitValueLoad(state, inst.src[0]);
const auto lane = EmitLaneIndex(state, inst.src[1]);
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, value,
ConstantU32(state, ScopeSubgroup), src, lane});
EmitStoreU32(state, inst.dst, value);
}
void EmitWriteLaneU32(EmitterState* state, const IR::Instruction& inst) {
const auto value = EmitValueLoad(state, inst.src[0]);
const auto lane = EmitLaneIndex(state, inst.src[1]);
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto hit = state->builder.AllocateId();
const auto pointer =
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(*state, inst.dst.reg) : 0;
const auto old = pointer != 0 ? state->builder.AllocateId() : ConstantU32(state, 0);
const auto merged = state->builder.AllocateId();
if (pointer != 0) {
state->builder.AddFunction({OpLoad, state->uint_type, old, pointer});
}
state->builder.AddFunction({OpIEqual, state->bool_type, hit, subid, lane});
state->builder.AddFunction({OpSelect, state->uint_type, merged, hit, value, old});
if (pointer != 0) {
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
state->builder.AddFunction({OpStore, pointer, merged});
}
}
void EmitPermlaneB32(EmitterState* state, const IR::Instruction& inst, bool x16) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto lane8 = state->builder.AllocateId();
const auto shift = state->builder.AllocateId();
const auto upper = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto index0 = state->builder.AllocateId();
const auto index1 = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
const auto shuffled = state->builder.AllocateId();
const auto value = EmitValueLoad(state, inst.src[0]);
const auto src1 = EmitValueLoad(state, inst.src[1]);
const auto src2 = EmitValueLoad(state, inst.src[2]);
uint32_t row_value = row;
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
if (x16) {
row_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseXor, state->uint_type, row_value, row, ConstantU32(state, 16)});
}
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane8, lane, ConstantU32(state, 7)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, shift, lane8, ConstantU32(state, 2)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, upper, lane, ConstantU32(state, 8)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, upper, src2, src1});
state->builder.AddFunction({OpShiftRightLogical, state->uint_type, index0, selected, shift});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, index1, index0, ConstantU32(state, 15)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row_value, index1});
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), value, target});
uint32_t ret = shuffled;
if (!inst.dst.op_sel) {
const auto source_active = EmitLaneIndexActiveBool(state, target);
ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
}
EmitStoreU32(state, inst.dst, ret);
}
void EmitControlNop(EmitterState* state, const IR::Instruction& inst) {
(void)state;
(void)inst;
}
void EmitWaitcnt(EmitterState* state, const IR::Instruction& inst) {
(void)state;
(void)inst;
}
void EmitBarrier(EmitterState* state, const IR::Instruction& inst) {
(void)inst;
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
state->builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageHelpers.cpp
+314
View File
@@ -0,0 +1,314 @@
#include "common/assert.h"
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
return (inst.memory.image_sample_flags & flag) != 0;
}
ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) {
ImageSampleLayout layout;
uint32_t cursor = 0;
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
layout.offset = cursor++;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
layout.bias = cursor++;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) {
layout.dref = cursor++;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
layout.grad_x = cursor;
cursor += 2u;
layout.grad_y = cursor;
cursor += 2u;
}
layout.coord = cursor;
cursor += ImageViewCoordinateComponents(view);
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) {
layout.lod = cursor++;
}
return layout;
}
uint32_t EmitImageCoordF32(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout, ImageViewKind view) {
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
const auto y = inst.memory.image_address_components > layout.coord + 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
: EmitZeroF32(state);
const auto coord = state->builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > layout.coord + 2u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
: EmitZeroF32(state);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
return EmitZeroF32(state);
}
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
}
uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
return EmitZeroF32(state);
}
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
}
uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
return EmitZeroF32(state);
}
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
}
uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst,
uint32_t first_component) {
const auto x = inst.memory.image_address_components > first_component
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
: EmitZeroF32(state);
const auto y = inst.memory.image_address_components > first_component + 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
: EmitZeroF32(state);
const auto grad = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, grad, x, y});
return grad;
}
uint32_t EmitImagePackedOffset2I32(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout) {
if (layout.offset == NoImageComponent ||
inst.memory.image_address_components <= layout.offset) {
const auto zero = ConstantI32(state, 0);
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec2_int_type, ret, zero, zero});
return ret;
}
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
const auto packed_i32 = state->builder.AllocateId();
const auto offset_x = state->builder.AllocateId();
const auto offset_y = state->builder.AllocateId();
const auto offset = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, packed_i32, packed_bits});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_x, packed_i32,
ConstantI32(state, 0), ConstantI32(state, 6)});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_y, packed_i32,
ConstantI32(state, 8), ConstantI32(state, 6)});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec2_int_type, offset, offset_x, offset_y});
return offset;
}
uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout, uint32_t sampled_image,
uint32_t coord, ImageViewKind view) {
if (!HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
return coord;
}
const auto offset_i32 = EmitImagePackedOffset2I32(state, inst, layout);
const auto offset_f32 = state->builder.AllocateId();
const auto image = state->builder.AllocateId();
const auto size_u32 = state->builder.AllocateId();
const auto size_xy_u32 =
ImageViewCoordinateComponents(view) == 3u ? state->builder.AllocateId() : size_u32;
const auto size_f32 = state->builder.AllocateId();
const auto normalized = state->builder.AllocateId();
const auto adjusted = state->builder.AllocateId();
state->builder.AddFunction({OpConvertSToF, state->vec2_float_type, offset_f32, offset_i32});
state->builder.AddFunction(
{OpImage, ImageViewImageType(*state, view, inst.memory.kind == IR::ResourceKind::ImageUint),
image, sampled_image});
state->builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(*state, view), size_u32,
image, ConstantU32(state, 0)});
if (ImageViewCoordinateComponents(view) == 3u) {
const auto width = state->builder.AllocateId();
const auto height = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size_u32, 0});
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size_u32, 1});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec2_uint_type, size_xy_u32, width, height});
}
state->builder.AddFunction({OpConvertUToF, state->vec2_float_type, size_f32, size_xy_u32});
state->builder.AddFunction({OpFDiv, state->vec2_float_type, normalized, offset_f32, size_f32});
if (ImageViewCoordinateComponents(view) == 2u) {
state->builder.AddFunction({OpFAdd, state->vec2_float_type, adjusted, coord, normalized});
return adjusted;
}
const auto x = state->builder.AllocateId();
const auto y = state->builder.AllocateId();
const auto z = state->builder.AllocateId();
const auto normalized_x = state->builder.AllocateId();
const auto normalized_y = state->builder.AllocateId();
const auto adjusted_x = state->builder.AllocateId();
const auto adjusted_y = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->float_type, x, coord, 0});
state->builder.AddFunction({OpCompositeExtract, state->float_type, y, coord, 1});
state->builder.AddFunction({OpCompositeExtract, state->float_type, z, coord, 2});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, normalized_x, normalized, 0});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, normalized_y, normalized, 1});
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_x, x, normalized_x});
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_y, y, normalized_y});
state->builder.AddFunction(
{OpCompositeConstruct, state->vec3_float_type, adjusted, adjusted_x, adjusted_y, z});
return adjusted;
}
uint32_t EmitImageCoordU32(EmitterState* state, const IR::Instruction& inst, ImageViewKind view) {
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
: ConstantU32(state, 0);
const auto coord = state->builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
: ConstantU32(state, 0);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageLoadCoordU32(EmitterState* state, const IR::Instruction& inst,
ImageViewKind view) {
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
: ConstantU32(state, 0);
const auto coord = state->builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
: ConstantU32(state, 0);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
}
return coord;
}
uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst,
const IR::Operand& address, ImageViewKind view) {
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
return ConstantU32(state, 0);
}
const auto lod_component = ImageViewCoordinateComponents(view);
if (inst.memory.image_address_components <= lod_component) {
return ConstantU32(state, 0);
}
return EmitImageAddressValueLoad(state, inst, address, lod_component);
}
uint32_t EmitImageQueryCoordF32(EmitterState* state, const IR::Instruction& inst,
ImageViewKind view) {
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
const auto y = inst.memory.image_address_components > 1u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
: EmitZeroF32(state);
const auto coord = state->builder.AllocateId();
if (ImageViewCoordinateComponents(view) == 3u) {
const auto z = inst.memory.image_address_components > 2u
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
: EmitZeroF32(state);
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
} else {
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
}
return coord;
}
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
uint32_t index = 0;
for (uint32_t i = 0; i < component; i++) {
index += (dmask >> i) & 1u;
}
return index;
}
uint32_t EmitImageStoreComponentF32(EmitterState* state, const IR::Instruction& inst,
uint32_t component) {
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
if (((dmask >> component) & 1u) == 0) {
return EmitZeroF32(state);
}
return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
}
uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
const auto target = 4u + component;
for (uint32_t source = 0; source < 4u; source++) {
if (((swizzle >> (source * 3u)) & 0x7u) == target) {
return source;
}
}
return UINT32_MAX;
}
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
if (state.program == nullptr || inst.memory.resource >= state.program->info.images.size()) {
EXIT("storage image instruction has no specialized swizzle\n");
}
return state.program->info.images[inst.memory.resource].storage_swizzle;
}
uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(*state, inst);
const auto component_value = [&](uint32_t component) {
const auto source = InverseStorageSwizzleComponent(swizzle, component);
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
};
const auto x = component_value(0);
const auto y = component_value(1);
const auto z = component_value(2);
const auto w = component_value(3);
const auto texel = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, texel, x, y, z, w});
return texel;
}
uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction& inst,
uint32_t component) {
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
if (((dmask >> component) & 1u) == 0) {
return ConstantU32(state, 0);
}
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
}
uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst) {
const auto swizzle = StorageImageSwizzle(*state, inst);
const auto component_value = [&](uint32_t component) {
const auto source = InverseStorageSwizzleComponent(swizzle, component);
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
: ConstantU32(state, 0);
};
const auto x = component_value(0);
const auto y = component_value(1);
const auto z = component_value(2);
const auto w = component_value(3);
const auto texel = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeConstruct, state->vec4_uint_type, texel, x, y, z, w});
return texel;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterImageOps.cpp
+361
View File
@@ -0,0 +1,361 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
namespace {
uint32_t ConstantVec2I32(EmitterState* state, int32_t x, int32_t y) {
const auto value = state->builder.AllocateId();
state->builder.AddType({OpConstantComposite, state->vec2_int_type, value, ConstantI32(state, x),
ConstantI32(state, y)});
return value;
}
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState* state) {
const auto offset0 = ConstantVec2I32(state, -1, 0);
const auto offset1 = ConstantVec2I32(state, 0, 0);
const auto offset2 = ConstantVec2I32(state, 1, 0);
const auto offset3 = ConstantVec2I32(state, 2, 0);
const auto type = state->builder.AllocateId();
state->builder.AddType({OpTypeArray, type, state->vec2_int_type, ConstantU32(state, 4)});
const auto value = state->builder.AllocateId();
state->builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
return value;
}
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resource,
uint32_t array_index, bool uint_image,
ImageViewKind view) {
uint32_t pointer_type =
uint_image ? state->ptr_uniform_storage_image_uint : state->ptr_uniform_storage_image;
uint32_t variable =
uint_image ? state->storage_image_uint_variable : state->storage_image_variable;
uint32_t image_type = uint_image ? state->storage_image_uint_type : state->storage_image_type;
IR::DescriptorBindingKind kind = uint_image ? IR::DescriptorBindingKind::StorageUint2D
: IR::DescriptorBindingKind::Storage2D;
switch (view) {
case ImageViewKind::Dim2D: break;
case ImageViewKind::Dim2DArray:
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_2d_array
: state->ptr_uniform_storage_image_2d_array;
variable = uint_image ? state->storage_image_uint_2d_array_variable
: state->storage_image_2d_array_variable;
image_type = uint_image ? state->storage_image_uint_2d_array_type
: state->storage_image_2d_array_type;
kind = uint_image ? IR::DescriptorBindingKind::StorageUint2DArray
: IR::DescriptorBindingKind::Storage2DArray;
break;
case ImageViewKind::Dim3D:
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_3d
: state->ptr_uniform_storage_image_3d;
variable = uint_image ? state->storage_image_uint_3d_variable
: state->storage_image_3d_variable;
image_type =
uint_image ? state->storage_image_uint_3d_type : state->storage_image_3d_type;
kind = uint_image ? IR::DescriptorBindingKind::StorageUint3D
: IR::DescriptorBindingKind::Storage3D;
break;
}
const auto pointer =
DescriptorElementPointer(state, pointer_type, variable, array_index, kind, resource,
"storage image descriptor array was not emitted");
const auto image = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, image_type, image, pointer});
return image;
}
} // namespace
uint32_t EmitImageGetResinfoComponent(EmitterState* state, uint32_t image, uint32_t size,
uint32_t component_index, ImageViewKind view) {
switch (component_index) {
case 0: {
const auto width = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size, 0});
return width;
}
case 1: {
const auto height = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size, 1});
return height;
}
case 2:
if (ImageViewCoordinateComponents(view) == 3u) {
const auto depth = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, depth, size, 2});
return depth;
}
return ConstantU32(state, 1);
case 3: {
const auto levels = state->builder.AllocateId();
state->builder.AddFunction({OpImageQueryLevels, state->uint_type, levels, image});
return levels;
}
default: return ConstantU32(state, 0);
}
}
void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
const auto mip_level = EmitValueLoad(state, inst.src[0]);
const auto size = state->builder.AllocateId();
state->builder.AddFunction(
{OpImageQuerySizeLod, ImageViewSizeType(*state, view), size, image, mip_level});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component =
EmitImageGetResinfoComponent(state, image, size, component_index, view);
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component);
}
}
void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto lod = state->builder.AllocateId();
state->builder.AddFunction({OpImageQueryLod, state->vec2_float_type, lod, sampled_image,
EmitImageQueryCoordF32(state, inst, view)});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
for (uint32_t component_index = 0; component_index < 2u; component_index++) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, component, lod, component_index});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto color = state->builder.AllocateId();
state->builder.AddFunction(
{OpImageFetch, integer ? state->vec4_uint_type : state->vec4_float_type, color, image,
EmitImageLoadCoordU32(state, inst, view), ImageOperandsLodMask,
EmitImageMipLodU32(state, inst, inst.src[0], view)});
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
color, component_index});
const auto bits = integer ? component : state->builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
void EmitImageStore(EmitterState* state, const IR::Instruction& inst) {
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
const auto view = StorageImageViewKind(*state, inst.memory, uint_image, inst.pc);
const auto binding = ResourceForDescriptor(
*state,
uint_image
? (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::StorageUint2DArray
: view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::StorageUint3D
: IR::DescriptorBindingKind::StorageUint2D)
: (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::Storage2DArray
: view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::Storage3D
: IR::DescriptorBindingKind::Storage2D),
inst.memory.resource);
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
binding.array_index, uint_image, view);
state->builder.AddFunction(
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
}
void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uint32_t sample,
bool dref, bool integer) {
if (dref) {
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, sample});
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
}
return;
}
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
uint32_t dst_index = 0;
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
if (((dmask >> component_index) & 1u) == 0) {
continue;
}
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
sample, component_index});
const auto bits = integer ? component : state->builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
}
}
bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) {
// RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it
// to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information.
// SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need
// an explicit operand even for the unsuffixed sample opcode.
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) ||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) ||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) {
return true;
}
return state.stage != ShaderType::Pixel;
}
void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout, bool explicit_lod,
std::vector<uint32_t>* words) {
if (words == nullptr) {
return;
}
uint32_t mask = 0;
std::vector<uint32_t> operands;
// Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod
// operands belong to explicit-lod instructions; bias belongs to implicit-lod
// sampling because it modifies the hardware-computed LOD.
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
mask |= ImageOperandsGradMask;
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x));
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y));
} else if (explicit_lod) {
mask |= ImageOperandsLodMask;
operands.push_back(EmitImageLodF32(state, inst, layout));
} else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
mask |= ImageOperandsBiasMask;
operands.push_back(EmitImageBiasF32(state, inst, layout));
}
if (mask != 0) {
words->push_back(mask);
words->insert(words->end(), operands.begin(), operands.end());
}
}
uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) {
const auto flags = inst.memory.image_sample_flags;
const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0;
const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
if (explicit_lod) {
return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
}
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
}
void EmitImageSample(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto layout = MakeImageSampleLayout(inst, view);
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
const auto sample = state->builder.AllocateId();
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto result_type = dref ? state->float_type
: integer ? state->vec4_uint_type
: state->vec4_float_type;
const auto explicit_lod = ImageSampleNeedsExplicitLod(*state, inst);
const auto opcode = ImageSampleOpcode(*state, inst);
const auto coord =
EmitImageOffsetCoordF32(state, inst, layout, sampled_image, base_coord, view);
std::vector<uint32_t> words = {opcode, result_type, sample, sampled_image, coord};
if (dref) {
words.push_back(EmitImageDrefF32(state, inst, layout));
}
AddImageSampleOperands(state, inst, layout, explicit_lod, &words);
state->builder.AddFunction(words);
EmitImageSampleResult(state, inst, sample, dref, integer);
}
uint32_t ImageGatherComponent(uint32_t dmask) {
switch (dmask) {
case 0x2u: return 1;
case 0x4u: return 2;
case 0x8u: return 3;
default: return 0;
}
}
void AddImageGatherOperands(EmitterState* state, const IR::Instruction& inst,
const ImageSampleLayout& layout, std::vector<uint32_t>* words) {
if (words == nullptr) {
return;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
words->push_back(ImageOperandsConstOffsetsMask);
words->push_back(ConstantImageGatherHorizontalOffsets(state));
return;
}
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
words->push_back(ImageOperandsOffsetMask);
words->push_back(EmitImagePackedOffset2I32(state, inst, layout));
}
}
void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) {
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
const auto layout = MakeImageSampleLayout(inst, view);
const auto coord = EmitImageCoordF32(state, inst, layout, view);
const auto texels = state->builder.AllocateId();
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
const auto result_type = integer ? state->vec4_uint_type : state->vec4_float_type;
std::vector<uint32_t> words =
dref ? std::vector<uint32_t> {OpImageDrefGather,
state->vec4_float_type,
texels,
sampled_image,
coord,
EmitImageDrefF32(state, inst, layout)}
: std::vector<uint32_t> {
OpImageGather, result_type,
texels, sampled_image,
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
AddImageGatherOperands(state, inst, layout, &words);
state->builder.AddFunction(words);
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
const auto component = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract,
integer ? state->uint_type : state->float_type, component,
texels, i});
const auto bits = integer ? component : state->builder.AllocateId();
if (!integer) {
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
}
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h
+1555
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterMemory.cpp
+1587
View File
File diff suppressed because it is too large. Load diff
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterModule.cpp
+929
View File
@@ -0,0 +1,929 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state,
IR::DescriptorBindingKind kind) {
return IR::FindBinding(state.program->bindings, kind);
}
uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind) {
const auto* binding = DescriptorBinding(state, kind);
return binding != nullptr ? static_cast<uint32_t>(binding->resources.size()) : 0;
}
uint32_t ConstantU32(EmitterState* state, uint32_t value) {
if (auto it = state->constants.find(value); it != state->constants.end()) {
return it->second;
}
const auto id = state->builder.AllocateId();
state->constants.emplace(value, id);
state->builder.AddType({OpConstant, state->uint_type, id, value});
return id;
}
uint32_t ConstantI32(EmitterState* state, int32_t value) {
const auto bits = static_cast<uint32_t>(value);
if (auto it = state->signed_constants.find(bits); it != state->signed_constants.end()) {
return it->second;
}
const auto id = state->builder.AllocateId();
state->signed_constants.emplace(bits, id);
state->builder.AddType({OpConstant, state->int_type, id, bits});
return id;
}
uint32_t ConstantF32(EmitterState* state, uint32_t bits) {
if (auto it = state->float_constants.find(bits); it != state->float_constants.end()) {
return it->second;
}
const auto id = state->builder.AllocateId();
state->float_constants.emplace(bits, id);
state->builder.AddType({OpConstant, state->float_type, id, bits});
return id;
}
uint32_t FloatBits(float value) {
uint32_t bits = 0;
std::memcpy(&bits, &value, sizeof(bits));
return bits;
}
uint32_t ConstantF32Value(EmitterState* state, float value) {
return ConstantF32(state, FloatBits(value));
}
VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
if (state.stage != ShaderType::Vertex || state.vertex_input_info == nullptr ||
location >= ShaderVertexInputInfo::RES_MAX ||
location >= static_cast<uint32_t>(state.vertex_input_info->resources_num)) {
return VertexInputScalarKind::Float;
}
switch (state.vertex_input_info->resources[location].Format()) {
case 5:
case 11:
case 18:
case 20:
case 27:
case 60:
case 62:
case 69:
case 72:
case 75: return VertexInputScalarKind::Uint;
case 6:
case 12:
case 19:
case 21:
case 28:
case 61:
case 63:
case 70:
case 73:
case 76: return VertexInputScalarKind::Sint;
default: return VertexInputScalarKind::Float;
}
}
uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) {
uint32_t count = input.component_count;
if (state.stage == ShaderType::Vertex && state.vertex_input_info != nullptr &&
input.location < ShaderVertexInputInfo::RES_MAX &&
input.location < static_cast<uint32_t>(state.vertex_input_info->resources_num) &&
state.vertex_input_info->resources_dst[input.location].registers_num > 0) {
count = static_cast<uint32_t>(
state.vertex_input_info->resources_dst[input.location].registers_num);
}
return std::clamp(count, 1u, 4u);
}
uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind) {
switch (kind) {
case VertexInputScalarKind::Sint: return state.int_type;
case VertexInputScalarKind::Uint: return state.uint_type;
case VertexInputScalarKind::Float:
default: return state.float_type;
}
}
uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind) {
switch (kind) {
case VertexInputScalarKind::Sint: return state.ptr_input_int;
case VertexInputScalarKind::Uint: return state.ptr_input_uint;
case VertexInputScalarKind::Float:
default: return state.ptr_input_float;
}
}
uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind,
uint32_t components) {
switch (kind) {
case VertexInputScalarKind::Sint:
switch (components) {
case 1: return state.int_type;
case 2: return state.vec2_int_type;
case 3: return state.vec3_int_type;
default: return state.vec4_int_type;
}
case VertexInputScalarKind::Uint:
switch (components) {
case 1: return state.uint_type;
case 2: return state.vec2_uint_type;
case 3: return state.vec3_uint_type;
default: return state.vec4_uint_type;
}
case VertexInputScalarKind::Float:
default:
switch (components) {
case 1: return state.float_type;
case 2: return state.vec2_float_type;
case 3: return state.vec3_float_type;
default: return state.vec4_float_type;
}
}
}
uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind,
uint32_t components) {
switch (kind) {
case VertexInputScalarKind::Sint:
switch (components) {
case 1: return state.ptr_input_int;
case 2: return state.ptr_input_vec2_int;
case 3: return state.ptr_input_vec3_int;
default: return state.ptr_input_vec4_int;
}
case VertexInputScalarKind::Uint:
switch (components) {
case 1: return state.ptr_input_uint;
case 2: return state.ptr_input_vec2_uint;
case 3: return state.ptr_input_vec3_uint;
default: return state.ptr_input_vec4_uint;
}
case VertexInputScalarKind::Float:
default:
switch (components) {
case 1: return state.ptr_input_float;
case 2: return state.ptr_input_vec2_float;
case 3: return state.ptr_input_vec3_float;
default: return state.ptr_input_vec4_float;
}
}
}
void AllocateInputVariables(EmitterState* state) {
for (auto& binding: state->inputs) {
binding.variable_id = state->builder.AllocateId();
state->interface_variables.push_back(binding.variable_id);
}
if (state->needs_subgroup_local_invocation_id) {
state->subgroup_local_invocation_id_variable = state->builder.AllocateId();
state->interface_variables.push_back(state->subgroup_local_invocation_id_variable);
}
}
static uint32_t AllocateInterfaceVariable(EmitterState* state) {
const auto variable = state->builder.AllocateId();
state->interface_variables.push_back(variable);
return variable;
}
static uint32_t AllocateSharedOutputVariable(EmitterState* state, uint32_t& variable) {
if (variable == 0) {
variable = AllocateInterfaceVariable(state);
}
return variable;
}
void AllocateOutputVariables(EmitterState* state) {
for (auto& binding: state->outputs) {
switch (binding.kind) {
case IR::StageOutputKind::Position:
binding.variable_id =
AllocateSharedOutputVariable(state, state->per_vertex_variable);
break;
case IR::StageOutputKind::Depth:
binding.variable_id = AllocateSharedOutputVariable(state, state->depth_variable);
break;
case IR::StageOutputKind::SampleMask:
binding.variable_id =
AllocateSharedOutputVariable(state, state->sample_mask_variable);
break;
case IR::StageOutputKind::Parameter:
case IR::StageOutputKind::Mrt:
binding.variable_id = AllocateInterfaceVariable(state);
break;
}
}
}
uint32_t BuiltInForInput(IR::StageInputKind kind) {
switch (kind) {
case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex;
case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex;
case IR::StageInputKind::FragCoord: return BuiltInFragCoord;
case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing;
case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId;
case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId;
case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex;
case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId;
default: return UINT32_MAX;
}
}
void AddInputAnnotationsAndNames(EmitterState* state) {
if (state->subgroup_local_invocation_id_variable != 0) {
state->builder.AddName(state->subgroup_local_invocation_id_variable,
"gl_SubgroupInvocationID");
state->builder.AddAnnotation({OpDecorate, state->subgroup_local_invocation_id_variable,
DecorationBuiltIn, BuiltInSubgroupLocalInvocationId});
if (state->stage == ShaderType::Pixel) {
state->builder.AddAnnotation(
{OpDecorate, state->subgroup_local_invocation_id_variable, DecorationFlat});
}
}
for (const auto& input: state->inputs) {
state->builder.AddName(input.variable_id, input.debug_name.c_str());
if (input.kind == IR::StageInputKind::Parameter) {
const auto flat = PixelParameterIsFlat(*state, input.location);
if (flat) {
state->builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
}
if (state->stage == ShaderType::Pixel && state->pixel_input_info != nullptr &&
state->pixel_input_info->ps_no_perspective && !flat) {
state->builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationNoPerspective});
}
const auto location = PixelParameterLocation(*state, input.location);
state->builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationLocation, location});
continue;
}
const auto builtin = BuiltInForInput(input.kind);
if (builtin != UINT32_MAX) {
state->builder.AddAnnotation(
{OpDecorate, input.variable_id, DecorationBuiltIn, builtin});
}
}
}
void AddOutputAnnotationsAndNames(EmitterState* state) {
if (state->per_vertex_variable != 0) {
state->builder.AddName(state->per_vertex_type, "gl_PerVertex");
state->builder.AddName(state->per_vertex_variable, "outPerVertex");
state->builder.AddAnnotation(
{OpMemberDecorate, state->per_vertex_type, 0, DecorationBuiltIn, BuiltInPosition});
state->builder.AddAnnotation({OpDecorate, state->per_vertex_type, DecorationBlock});
}
if (state->depth_variable != 0) {
state->builder.AddName(state->depth_variable, "gl_FragDepth");
state->builder.AddAnnotation(
{OpDecorate, state->depth_variable, DecorationBuiltIn, BuiltInFragDepth});
}
if (state->sample_mask_variable != 0) {
state->builder.AddName(state->sample_mask_variable, "gl_SampleMask");
state->builder.AddAnnotation(
{OpDecorate, state->sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask});
}
for (const auto& binding: state->outputs) {
if (binding.kind == IR::StageOutputKind::Parameter ||
binding.kind == IR::StageOutputKind::Mrt) {
state->builder.AddName(binding.variable_id, binding.debug_name.c_str());
state->builder.AddAnnotation(
{OpDecorate, binding.variable_id, DecorationLocation, binding.location});
}
}
}
void DecorateDescriptor(EmitterState* state, uint32_t variable, const char* name, uint32_t set,
uint32_t binding) {
if (variable == 0) {
return;
}
state->builder.AddName(variable, name);
state->builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, set});
state->builder.AddAnnotation({OpDecorate, variable, DecorationBinding, binding});
}
void AddDescriptorAnnotationsAndNames(EmitterState* state) {
const auto set = state->program->bindings.descriptor_set;
auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) {
const auto* binding = DescriptorBinding(*state, kind);
if (binding != nullptr) {
DecorateDescriptor(state, variable, name, set, binding->binding);
}
};
if (state->storage_buffer_variable != 0) {
Decorate(state->storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
}
if (state->address_memory_variable != 0) {
Decorate(state->address_memory_variable, "address_memory",
IR::DescriptorBindingKind::AddressMemory);
}
constexpr const char* SampledNames[] = {
"sampled_2d", "sampled_2d_array", "sampled_3d",
"sampled_uint_2d", "sampled_uint_2d_array", "sampled_uint_3d"};
for (uint32_t i = 0; i < state->sampled_images.size(); i++) {
const auto view = static_cast<ImageViewKind>(i % 3u);
Decorate(state->sampled_images[i].variable, SampledNames[i],
SampledBindingKind(i >= 3u, view));
}
if (state->storage_image_variable != 0) {
Decorate(state->storage_image_variable, "textures2D_L",
IR::DescriptorBindingKind::Storage2D);
}
if (state->storage_image_uint_variable != 0) {
Decorate(state->storage_image_uint_variable, "textures2D_L_U",
IR::DescriptorBindingKind::StorageUint2D);
}
if (state->storage_image_2d_array_variable != 0) {
Decorate(state->storage_image_2d_array_variable, "textures2D_L_A",
IR::DescriptorBindingKind::Storage2DArray);
}
if (state->storage_image_uint_2d_array_variable != 0) {
Decorate(state->storage_image_uint_2d_array_variable, "textures2D_L_U_A",
IR::DescriptorBindingKind::StorageUint2DArray);
}
if (state->storage_image_3d_variable != 0) {
Decorate(state->storage_image_3d_variable, "textures2D_L_3D",
IR::DescriptorBindingKind::Storage3D);
}
if (state->storage_image_uint_3d_variable != 0) {
Decorate(state->storage_image_uint_3d_variable, "textures2D_L_U_3D",
IR::DescriptorBindingKind::StorageUint3D);
}
if (state->sampler_variable != 0) {
Decorate(state->sampler_variable, "samplers", IR::DescriptorBindingKind::Samplers);
}
if (state->gds_variable != 0) {
Decorate(state->gds_variable, "gds", IR::DescriptorBindingKind::Gds);
}
if (state->flattened_srt_variable != 0) {
Decorate(state->flattened_srt_variable, "flattened_srt",
IR::DescriptorBindingKind::FlattenedSrt);
}
}
void AddVsharpAnnotationsAndNames(EmitterState* state) {
const auto& bind = state->program->bindings;
if (state->push_constant_variable != 0) {
state->builder.AddName(state->push_constant_block_type, "BufferResource");
state->builder.AddName(state->push_constant_variable, "vsharp");
state->builder.AddAnnotation(
{OpDecorate, state->push_constant_u32x4_array_type, DecorationArrayStride, 4});
state->builder.AddAnnotation(
{OpDecorate, state->push_constant_rows_array_type, DecorationArrayStride, 16});
state->builder.AddAnnotation({OpMemberDecorate, state->push_constant_block_type, 0,
DecorationOffset, bind.push_constant_offset});
state->builder.AddAnnotation(
{OpDecorate, state->push_constant_block_type, DecorationBlock});
}
if (state->vsharp_storage_variable != 0) {
const auto* storage = DescriptorBinding(*state, IR::DescriptorBindingKind::UserData);
if (storage != nullptr) {
DecorateDescriptor(state, state->vsharp_storage_variable, "user_data",
bind.descriptor_set, storage->binding);
}
}
}
void EmitHeaderAndTypes(EmitterState* state) {
state->void_type = state->builder.AllocateId();
state->bool_type = state->builder.AllocateId();
state->uint_type = state->builder.AllocateId();
state->uint_pair_type = state->builder.AllocateId();
state->int_type = state->builder.AllocateId();
state->int_pair_type = state->builder.AllocateId();
state->float_type = state->builder.AllocateId();
state->vec2_uint_type = state->builder.AllocateId();
state->vec3_uint_type = state->builder.AllocateId();
state->vec4_uint_type = state->builder.AllocateId();
state->vec2_int_type = state->builder.AllocateId();
state->vec3_int_type = state->builder.AllocateId();
state->vec4_int_type = state->builder.AllocateId();
state->vec2_float_type = state->builder.AllocateId();
state->vec3_float_type = state->builder.AllocateId();
state->vec4_float_type = state->builder.AllocateId();
state->ptr_func_uint = state->builder.AllocateId();
state->ptr_input_float = state->builder.AllocateId();
state->ptr_input_bool = state->builder.AllocateId();
state->ptr_input_int = state->builder.AllocateId();
state->ptr_input_uint = state->builder.AllocateId();
state->ptr_input_vec2_float = state->builder.AllocateId();
state->ptr_input_vec3_float = state->builder.AllocateId();
state->ptr_input_vec2_int = state->builder.AllocateId();
state->ptr_input_vec3_int = state->builder.AllocateId();
state->ptr_input_vec4_int = state->builder.AllocateId();
state->ptr_input_vec2_uint = state->builder.AllocateId();
state->ptr_input_vec3_uint = state->builder.AllocateId();
state->ptr_input_vec4_uint = state->builder.AllocateId();
state->ptr_input_vec4_float = state->builder.AllocateId();
state->sample_mask_array_type = state->builder.AllocateId();
state->ptr_output_int = state->builder.AllocateId();
state->ptr_output_sample_mask_array = state->builder.AllocateId();
state->ptr_output_float = state->builder.AllocateId();
state->ptr_output_vec4_float = state->builder.AllocateId();
state->per_vertex_type = state->builder.AllocateId();
state->ptr_output_per_vertex = state->builder.AllocateId();
state->storage_runtime_array_type = state->builder.AllocateId();
state->storage_buffer_type = state->builder.AllocateId();
state->ptr_storage_buffer = state->builder.AllocateId();
state->ptr_storage_buffer_uint = state->builder.AllocateId();
state->storage_buffer_array_type = state->builder.AllocateId();
state->ptr_storage_buffer_array = state->builder.AllocateId();
if (state->address_memory_variable != 0) {
state->address_memory_array_type = state->builder.AllocateId();
state->ptr_address_memory_array = state->builder.AllocateId();
}
if (state->push_constant_variable != 0) {
state->push_constant_u32x4_array_type = state->builder.AllocateId();
state->push_constant_rows_array_type = state->builder.AllocateId();
state->push_constant_block_type = state->builder.AllocateId();
state->ptr_push_constant_block = state->builder.AllocateId();
state->ptr_push_constant_uint = state->builder.AllocateId();
}
state->lds_array_type = state->builder.AllocateId();
state->ptr_workgroup_array = state->builder.AllocateId();
state->ptr_workgroup_uint = state->builder.AllocateId();
state->lds_variable = state->builder.AllocateId();
for (auto& image: state->sampled_images) {
image.image_type = state->builder.AllocateId();
image.sampled_image_type = state->builder.AllocateId();
image.pointer_type = state->builder.AllocateId();
image.array_type = state->builder.AllocateId();
image.array_pointer_type = state->builder.AllocateId();
}
state->sampler_type = state->builder.AllocateId();
state->sampler_array_type = state->builder.AllocateId();
state->ptr_uniform_sampler = state->builder.AllocateId();
state->ptr_uniform_sampler_array = state->builder.AllocateId();
state->storage_image_type = state->builder.AllocateId();
state->ptr_uniform_storage_image = state->builder.AllocateId();
state->storage_image_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_array = state->builder.AllocateId();
state->storage_image_2d_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_2d_array = state->builder.AllocateId();
state->storage_image_2d_array_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_2d_array_array = state->builder.AllocateId();
state->storage_image_3d_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_3d = state->builder.AllocateId();
state->storage_image_3d_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_3d_array = state->builder.AllocateId();
state->storage_image_uint_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_uint = state->builder.AllocateId();
state->storage_image_uint_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_uint_array = state->builder.AllocateId();
state->storage_image_uint_2d_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_uint_2d_array = state->builder.AllocateId();
state->storage_image_uint_2d_array_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_uint_2d_array_array = state->builder.AllocateId();
state->storage_image_uint_3d_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_uint_3d = state->builder.AllocateId();
state->storage_image_uint_3d_array_type = state->builder.AllocateId();
state->ptr_uniform_storage_image_uint_3d_array = state->builder.AllocateId();
state->ptr_image_uint = state->builder.AllocateId();
state->func_type = state->builder.AllocateId();
state->main_func = state->builder.AllocateId();
state->entry_label = state->builder.AllocateId();
state->glsl_std450 = state->builder.AllocateId();
state->builder.AddCapability({CapabilityShader});
state->builder.AddCapability({CapabilityImageQuery});
if (state->needs_image_gather_extended) {
state->builder.AddCapability({CapabilityImageGatherExtended});
}
if (state->storage_image_variable != 0 || state->storage_image_2d_array_variable != 0 ||
state->storage_image_3d_variable != 0) {
state->builder.AddCapability({CapabilityStorageImageReadWithoutFormat});
state->builder.AddCapability({CapabilityStorageImageWriteWithoutFormat});
}
if (state->needs_subgroup_ballot || state->needs_subgroup_shuffle ||
state->needs_subgroup_local_invocation_id) {
state->builder.AddCapability({CapabilityGroupNonUniform});
}
if (state->needs_subgroup_ballot) {
state->builder.AddCapability({CapabilityGroupNonUniformBallot});
}
if (state->needs_subgroup_shuffle) {
state->builder.AddCapability({CapabilityGroupNonUniformShuffle});
}
if (state->needs_compute_derivatives && state->stage == ShaderType::Compute) {
state->builder.AddCapability({CapabilityComputeDerivativeGroupQuadsKHR});
state->builder.AddExtension("SPV_KHR_compute_shader_derivatives");
}
state->builder.AddExtInstImport(state->glsl_std450, "GLSL.std.450");
state->builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
state->builder.AddEntryPoint(ExecutionModelForStage(state->stage), state->main_func, "main",
state->interface_variables);
if (state->stage == ShaderType::Compute) {
uint32_t local_x = state->needs_compute_derivatives ? 2u : 1u;
uint32_t local_y = state->needs_compute_derivatives ? 2u : 1u;
uint32_t local_z = 1u;
if (state->compute_input_info != nullptr) {
const auto* cs = state->compute_input_info;
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
}
state->builder.AddExecutionMode(
{state->main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
}
if (state->stage == ShaderType::Pixel) {
state->builder.AddExecutionMode({state->main_func, ExecutionModeOriginUpperLeft});
if (state->depth_variable != 0) {
state->builder.AddExecutionMode({state->main_func, ExecutionModeDepthReplacing});
}
if (state->pixel_input_info != nullptr && state->pixel_input_info->ps_early_z &&
!state->pixel_input_info->ps_pixel_kill_enable &&
!state->pixel_input_info->ps_depth_export_enable &&
!state->pixel_input_info->ps_sample_mask_export_enable) {
state->builder.AddExecutionMode({state->main_func, ExecutionModeEarlyFragmentTests});
}
}
if (state->needs_compute_derivatives && state->stage == ShaderType::Compute) {
state->builder.AddExecutionMode({state->main_func, ExecutionModeDerivativeGroupQuadsKHR});
}
state->builder.AddName(state->main_func, "main");
if (state->stage == ShaderType::Compute || state->needs_function_lds) {
state->builder.AddName(state->lds_variable, "lds_dwords");
}
AddInputAnnotationsAndNames(state);
AddOutputAnnotationsAndNames(state);
AddDescriptorAnnotationsAndNames(state);
AddVsharpAnnotationsAndNames(state);
state->builder.AddAnnotation(
{OpDecorate, state->storage_runtime_array_type, DecorationArrayStride, 4});
state->builder.AddAnnotation(
{OpMemberDecorate, state->storage_buffer_type, 0, DecorationOffset, 0});
state->builder.AddAnnotation({OpDecorate, state->storage_buffer_type, DecorationBlock});
state->builder.AddType({OpTypeVoid, state->void_type});
state->builder.AddType({OpTypeBool, state->bool_type});
state->builder.AddType({OpTypeInt, state->uint_type, 32, 0});
state->builder.AddType(
{OpTypeStruct, state->uint_pair_type, state->uint_type, state->uint_type});
state->builder.AddType({OpTypeInt, state->int_type, 32, 1});
state->builder.AddType({OpTypeStruct, state->int_pair_type, state->int_type, state->int_type});
state->builder.AddType({OpTypeFloat, state->float_type, 32});
state->builder.AddType({OpTypeVector, state->vec2_uint_type, state->uint_type, 2});
state->builder.AddType({OpTypeVector, state->vec3_uint_type, state->uint_type, 3});
state->builder.AddType({OpTypeVector, state->vec4_uint_type, state->uint_type, 4});
state->builder.AddType({OpTypeVector, state->vec2_int_type, state->int_type, 2});
state->builder.AddType({OpTypeVector, state->vec3_int_type, state->int_type, 3});
state->builder.AddType({OpTypeVector, state->vec4_int_type, state->int_type, 4});
state->builder.AddType({OpTypeVector, state->vec2_float_type, state->float_type, 2});
state->builder.AddType({OpTypeVector, state->vec3_float_type, state->float_type, 3});
state->builder.AddType({OpTypeVector, state->vec4_float_type, state->float_type, 4});
state->builder.AddType(
{OpTypePointer, state->ptr_func_uint, StorageClassFunction, state->uint_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_float, StorageClassInput, state->float_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_bool, StorageClassInput, state->bool_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_int, StorageClassInput, state->int_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_uint, StorageClassInput, state->uint_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec2_float, StorageClassInput, state->vec2_float_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec3_float, StorageClassInput, state->vec3_float_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec2_int, StorageClassInput, state->vec2_int_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec3_int, StorageClassInput, state->vec3_int_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec4_int, StorageClassInput, state->vec4_int_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec2_uint, StorageClassInput, state->vec2_uint_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec3_uint, StorageClassInput, state->vec3_uint_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec4_uint, StorageClassInput, state->vec4_uint_type});
state->builder.AddType(
{OpTypePointer, state->ptr_input_vec4_float, StorageClassInput, state->vec4_float_type});
if (state->subgroup_local_invocation_id_variable != 0) {
state->builder.AddType({OpVariable, state->ptr_input_uint,
state->subgroup_local_invocation_id_variable, StorageClassInput});
}
for (const auto& input: state->inputs) {
uint32_t ptr_type = state->ptr_input_uint;
switch (input.kind) {
case IR::StageInputKind::VertexIndex:
case IR::StageInputKind::InstanceIndex: ptr_type = state->ptr_input_int; break;
case IR::StageInputKind::WorkgroupId:
case IR::StageInputKind::LocalInvocationId:
case IR::StageInputKind::GlobalInvocationId:
ptr_type = state->ptr_input_vec3_uint;
break;
case IR::StageInputKind::FragCoord: ptr_type = state->ptr_input_vec4_float; break;
case IR::StageInputKind::FrontFacing: ptr_type = state->ptr_input_bool; break;
case IR::StageInputKind::Parameter:
if (state->stage == ShaderType::Vertex) {
const auto kind = VertexParameterScalarKind(*state, input.location);
const auto components = VertexParameterComponentCount(*state, input);
ptr_type = VertexParameterInputPointerType(*state, kind, components);
} else {
ptr_type = state->ptr_input_vec4_float;
}
break;
default: break;
}
state->builder.AddType({OpVariable, ptr_type, input.variable_id, StorageClassInput});
}
state->builder.AddType(
{OpTypePointer, state->ptr_output_float, StorageClassOutput, state->float_type});
state->builder.AddType(
{OpTypePointer, state->ptr_output_int, StorageClassOutput, state->int_type});
state->builder.AddType(
{OpTypePointer, state->ptr_output_vec4_float, StorageClassOutput, state->vec4_float_type});
if (state->per_vertex_variable != 0) {
state->builder.AddType({OpTypeStruct, state->per_vertex_type, state->vec4_float_type});
state->builder.AddType({OpTypePointer, state->ptr_output_per_vertex, StorageClassOutput,
state->per_vertex_type});
state->builder.AddType({OpVariable, state->ptr_output_per_vertex,
state->per_vertex_variable, StorageClassOutput});
}
for (const auto& binding: state->outputs) {
if (binding.kind == IR::StageOutputKind::Parameter ||
binding.kind == IR::StageOutputKind::Mrt) {
state->builder.AddType({OpVariable, state->ptr_output_vec4_float, binding.variable_id,
StorageClassOutput});
}
}
if (state->depth_variable != 0) {
state->builder.AddType(
{OpVariable, state->ptr_output_float, state->depth_variable, StorageClassOutput});
}
if (state->sample_mask_variable != 0) {
state->builder.AddType(
{OpTypeArray, state->sample_mask_array_type, state->int_type, ConstantU32(state, 1)});
state->builder.AddType({OpTypePointer, state->ptr_output_sample_mask_array,
StorageClassOutput, state->sample_mask_array_type});
state->builder.AddType({OpVariable, state->ptr_output_sample_mask_array,
state->sample_mask_variable, StorageClassOutput});
}
state->builder.AddType(
{OpTypeRuntimeArray, state->storage_runtime_array_type, state->uint_type});
state->builder.AddType(
{OpTypeStruct, state->storage_buffer_type, state->storage_runtime_array_type});
state->builder.AddType({OpTypePointer, state->ptr_storage_buffer, StorageClassStorageBuffer,
state->storage_buffer_type});
state->builder.AddType({OpTypePointer, state->ptr_storage_buffer_uint,
StorageClassStorageBuffer, state->uint_type});
if (state->storage_buffer_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Buffers));
state->builder.AddType(
{OpTypeArray, state->storage_buffer_array_type, state->storage_buffer_type, count});
state->builder.AddType({OpTypePointer, state->ptr_storage_buffer_array,
StorageClassStorageBuffer, state->storage_buffer_array_type});
state->builder.AddType({OpVariable, state->ptr_storage_buffer_array,
state->storage_buffer_variable, StorageClassStorageBuffer});
}
if (state->gds_variable != 0) {
state->builder.AddType({OpVariable, state->ptr_storage_buffer, state->gds_variable,
StorageClassStorageBuffer});
}
if (state->push_constant_variable != 0) {
const auto row_width = ConstantU32(state, 4);
const auto row_count = ConstantU32(
state, std::max((state->program->bindings.push_constant_size + 15u) / 16u, 1u));
state->builder.AddType(
{OpTypeArray, state->push_constant_u32x4_array_type, state->uint_type, row_width});
state->builder.AddType({OpTypeArray, state->push_constant_rows_array_type,
state->push_constant_u32x4_array_type, row_count});
state->builder.AddType(
{OpTypeStruct, state->push_constant_block_type, state->push_constant_rows_array_type});
state->builder.AddType({OpTypePointer, state->ptr_push_constant_block,
StorageClassPushConstant, state->push_constant_block_type});
state->builder.AddType({OpTypePointer, state->ptr_push_constant_uint,
StorageClassPushConstant, state->uint_type});
state->builder.AddType({OpVariable, state->ptr_push_constant_block,
state->push_constant_variable, StorageClassPushConstant});
}
if (state->vsharp_storage_variable != 0) {
state->builder.AddType({OpVariable, state->ptr_storage_buffer,
state->vsharp_storage_variable, StorageClassStorageBuffer});
}
if (state->address_memory_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::AddressMemory));
state->builder.AddType(
{OpTypeArray, state->address_memory_array_type, state->storage_buffer_type, count});
state->builder.AddType({OpTypePointer, state->ptr_address_memory_array,
StorageClassStorageBuffer, state->address_memory_array_type});
state->builder.AddType({OpVariable, state->ptr_address_memory_array,
state->address_memory_variable, StorageClassStorageBuffer});
}
if (state->flattened_srt_variable != 0) {
state->builder.AddType({OpVariable, state->ptr_storage_buffer,
state->flattened_srt_variable, StorageClassStorageBuffer});
}
if (state->stage == ShaderType::Compute || state->needs_function_lds) {
const auto storage_class =
state->needs_function_lds ? StorageClassFunction : StorageClassWorkgroup;
const auto lds_size = ConstantU32(state, state->needs_function_lds ? 8192u : 1024u);
state->builder.AddType({OpTypeArray, state->lds_array_type, state->uint_type, lds_size});
state->builder.AddType(
{OpTypePointer, state->ptr_workgroup_array, storage_class, state->lds_array_type});
state->builder.AddType(
{OpTypePointer, state->ptr_workgroup_uint, storage_class, state->uint_type});
if (state->stage == ShaderType::Compute) {
state->builder.AddType({OpVariable, state->ptr_workgroup_array, state->lds_variable,
StorageClassWorkgroup});
}
}
for (uint32_t i = 0; i < state->sampled_images.size(); i++) {
auto& image = state->sampled_images[i];
const auto view = static_cast<ImageViewKind>(i % 3u);
const auto component = i >= 3u ? state->uint_type : state->float_type;
const auto dimension = view == ImageViewKind::Dim3D ? Dim3D : Dim2D;
const auto arrayed = view == ImageViewKind::Dim2DArray ? 1u : 0u;
state->builder.AddType({OpTypeImage, image.image_type, component, dimension, 0, arrayed, 0,
1, ImageFormatUnknown});
state->builder.AddType({OpTypeSampledImage, image.sampled_image_type, image.image_type});
state->builder.AddType(
{OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type});
if (image.variable != 0) {
const auto kind = SampledBindingKind(i >= 3u, view);
const auto count = ConstantU32(state, DescriptorCount(*state, kind));
state->builder.AddType({OpTypeArray, image.array_type, image.image_type, count});
state->builder.AddType({OpTypePointer, image.array_pointer_type,
StorageClassUniformConstant, image.array_type});
state->builder.AddType({OpVariable, image.array_pointer_type, image.variable,
StorageClassUniformConstant});
}
}
state->builder.AddType({OpTypeSampler, state->sampler_type});
state->builder.AddType({OpTypePointer, state->ptr_uniform_sampler, StorageClassUniformConstant,
state->sampler_type});
if (state->sampler_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Samplers));
state->builder.AddType(
{OpTypeArray, state->sampler_array_type, state->sampler_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_sampler_array,
StorageClassUniformConstant, state->sampler_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_sampler_array,
state->sampler_variable, StorageClassUniformConstant});
}
state->builder.AddType({OpTypeImage, state->storage_image_type, state->float_type, Dim2D, 0, 0,
0, 2, ImageFormatUnknown});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image,
StorageClassUniformConstant, state->storage_image_type});
if (state->storage_image_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage2D));
state->builder.AddType(
{OpTypeArray, state->storage_image_array_type, state->storage_image_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_array,
StorageClassUniformConstant, state->storage_image_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_array,
state->storage_image_variable, StorageClassUniformConstant});
}
state->builder.AddType({OpTypeImage, state->storage_image_2d_array_type, state->float_type,
Dim2D, 0, 1, 0, 2, ImageFormatUnknown});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_2d_array,
StorageClassUniformConstant, state->storage_image_2d_array_type});
if (state->storage_image_2d_array_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage2DArray));
state->builder.AddType({OpTypeArray, state->storage_image_2d_array_array_type,
state->storage_image_2d_array_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_2d_array_array,
StorageClassUniformConstant,
state->storage_image_2d_array_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_2d_array_array,
state->storage_image_2d_array_variable,
StorageClassUniformConstant});
}
state->builder.AddType({OpTypeImage, state->storage_image_3d_type, state->float_type, Dim3D, 0,
0, 0, 2, ImageFormatUnknown});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_3d,
StorageClassUniformConstant, state->storage_image_3d_type});
if (state->storage_image_3d_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage3D));
state->builder.AddType(
{OpTypeArray, state->storage_image_3d_array_type, state->storage_image_3d_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_3d_array,
StorageClassUniformConstant, state->storage_image_3d_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_3d_array,
state->storage_image_3d_variable, StorageClassUniformConstant});
}
state->builder.AddType({OpTypeImage, state->storage_image_uint_type, state->uint_type, Dim2D, 0,
0, 0, 2, ImageFormatR32ui});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint,
StorageClassUniformConstant, state->storage_image_uint_type});
if (state->storage_image_uint_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint2D));
state->builder.AddType({OpTypeArray, state->storage_image_uint_array_type,
state->storage_image_uint_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_array,
StorageClassUniformConstant, state->storage_image_uint_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_array,
state->storage_image_uint_variable, StorageClassUniformConstant});
}
state->builder.AddType({OpTypeImage, state->storage_image_uint_2d_array_type, state->uint_type,
Dim2D, 0, 1, 0, 2, ImageFormatR32ui});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_2d_array,
StorageClassUniformConstant, state->storage_image_uint_2d_array_type});
if (state->storage_image_uint_2d_array_variable != 0) {
const auto count = ConstantU32(
state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint2DArray));
state->builder.AddType({OpTypeArray, state->storage_image_uint_2d_array_array_type,
state->storage_image_uint_2d_array_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_2d_array_array,
StorageClassUniformConstant,
state->storage_image_uint_2d_array_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_2d_array_array,
state->storage_image_uint_2d_array_variable,
StorageClassUniformConstant});
}
state->builder.AddType({OpTypeImage, state->storage_image_uint_3d_type, state->uint_type, Dim3D,
0, 0, 0, 2, ImageFormatR32ui});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_3d,
StorageClassUniformConstant, state->storage_image_uint_3d_type});
if (state->storage_image_uint_3d_variable != 0) {
const auto count =
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint3D));
state->builder.AddType({OpTypeArray, state->storage_image_uint_3d_array_type,
state->storage_image_uint_3d_type, count});
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_3d_array,
StorageClassUniformConstant,
state->storage_image_uint_3d_array_type});
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_3d_array,
state->storage_image_uint_3d_variable,
StorageClassUniformConstant});
}
state->builder.AddType(
{OpTypePointer, state->ptr_image_uint, StorageClassImage, state->uint_type});
state->builder.AddType({OpTypeFunction, state->func_type, state->void_type});
}
void AllocateRegisterVariables(EmitterState* state) {
if (state->stage == ShaderType::Pixel && state->needs_pixel_valid_mask) {
state->pixel_valid_mask_variable = state->builder.AllocateId();
state->builder.AddName(state->pixel_valid_mask_variable, "pixel_valid_mask_active");
}
for (auto& binding: state->registers) {
binding.pointer_id = state->builder.AllocateId();
const auto name = IR::RegisterToString(binding.reg);
state->builder.AddName(binding.pointer_id, name.c_str());
}
}
void AllocateDescriptorVariables(EmitterState* state) {
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Buffers) != nullptr) {
state->storage_buffer_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::AddressMemory) != nullptr) {
state->address_memory_variable = state->builder.AllocateId();
}
if (state->program->bindings.push_constant_size != 0) {
state->push_constant_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::UserData) != nullptr) {
state->vsharp_storage_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
state->flattened_srt_variable = state->builder.AllocateId();
}
for (uint32_t i = 0; i < state->sampled_images.size(); i++) {
const auto view = static_cast<ImageViewKind>(i % 3u);
if (DescriptorBinding(*state, SampledBindingKind(i >= 3u, view)) != nullptr) {
state->sampled_images[i].variable = state->builder.AllocateId();
}
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage2D) != nullptr) {
state->storage_image_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint2D) != nullptr) {
state->storage_image_uint_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage2DArray) != nullptr) {
state->storage_image_2d_array_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint2DArray) != nullptr) {
state->storage_image_uint_2d_array_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage3D) != nullptr) {
state->storage_image_3d_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint3D) != nullptr) {
state->storage_image_uint_3d_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Samplers) != nullptr) {
state->sampler_variable = state->builder.AllocateId();
}
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Gds) != nullptr) {
state->gds_variable = state->builder.AllocateId();
}
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
src/graphics/shader/recompiler/spirvEmitter/spirvEmitterValues.cpp
+840
View File
@@ -0,0 +1,840 @@
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
if (sel <= 3u) {
offset = sel * 8u;
width = 8u;
return true;
}
if (sel == 4u || sel == 5u) {
offset = sel == 5u ? 16u : 0u;
width = 16u;
return true;
}
if (sel == 6u) {
offset = 0;
width = 32u;
return true;
}
return false;
}
uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
return value;
}
uint32_t offset = 0;
uint32_t width = 0;
if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) {
return value;
}
if (operand.sdwa_sext) {
const auto signed_value = state->builder.AllocateId();
const auto extracted = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->int_type, signed_value, value});
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_value,
ConstantU32(state, offset), ConstantU32(state, width)});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, extracted});
return bits;
}
const auto extracted = state->builder.AllocateId();
state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, extracted, value,
ConstantU32(state, offset), ConstantU32(state, width)});
return extracted;
}
uint32_t EmitTrueBool(EmitterState* state) {
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
return ret;
}
DppTargetLane EmitDppQuadPermTargetLane(EmitterState* state, uint32_t subid, uint32_t control) {
const auto quad_base = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto shift = state->builder.AllocateId();
const auto selected0 = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 3)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, shift, lane, ConstantU32(state, 1)});
state->builder.AddFunction(
{OpShiftRightLogical, state->uint_type, selected0, ConstantU32(state, control), shift});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, selected, selected0, ConstantU32(state, 3)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, quad_base, selected});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppRowShiftTargetLane(EmitterState* state, uint32_t subid, uint32_t amount,
bool left) {
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto lane_shifted = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
const auto valid = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
if (left) {
state->builder.AddFunction(
{OpIAdd, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpULessThan, state->bool_type, valid, lane, ConstantU32(state, 16u - amount)});
} else {
state->builder.AddFunction(
{OpISub, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, valid, lane, ConstantU32(state, amount)});
}
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, lane_shifted});
return {target, valid};
}
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState* state, uint32_t subid,
uint32_t amount) {
const auto row = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto in_high = state->builder.AllocateId();
const auto minus = state->builder.AllocateId();
const auto plus = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, amount)});
state->builder.AddFunction({OpISub, state->uint_type, minus, lane, ConstantU32(state, amount)});
state->builder.AddFunction(
{OpIAdd, state->uint_type, plus, lane, ConstantU32(state, 16u - amount)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, minus, plus});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, selected});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppMirrorTargetLane(EmitterState* state, uint32_t subid, bool half_row) {
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
const auto lane_mask = half_row ? 7u : 15u;
const auto base = state->builder.AllocateId();
const auto lane = state->builder.AllocateId();
const auto mirrored = state->builder.AllocateId();
const auto target = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, base_mask)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, lane_mask)});
state->builder.AddFunction(
{OpISub, state->uint_type, mirrored, ConstantU32(state, lane_mask), lane});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, mirrored});
return {target, EmitTrueBool(state)};
}
DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) {
const auto subid = EmitSubgroupLocalInvocationId(state);
if (control <= 0xffu) {
return EmitDppQuadPermTargetLane(state, subid, control);
}
if (control >= 0x101u && control <= 0x10fu) {
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
}
if (control >= 0x111u && control <= 0x11fu) {
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
}
if (control >= 0x121u && control <= 0x12fu) {
return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
}
if (control == 0x140u) {
return EmitDppMirrorTargetLane(state, subid, false);
}
if (control == 0x141u) {
return EmitDppMirrorTargetLane(state, subid, true);
}
return {subid, EmitTrueBool(state)};
}
uint32_t EmitDppValueU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
if (!operand.dpp) {
return value;
}
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
const auto shuffled = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
ConstantU32(state, ScopeSubgroup), value, target.lane});
if (operand.dpp_fetch_inactive) {
return shuffled;
}
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
const auto can_fetch = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpLogicalAnd, state->bool_type, can_fetch, target.valid, source_active});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) {
uint32_t value = 0;
switch (operand.kind) {
case IR::OperandKind::ImmediateU32:
case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
case IR::OperandKind::Register: {
const auto pointer = PointerForRegister(*state, operand.reg);
if (pointer == 0) {
value = ConstantU32(state, 0);
break;
}
value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
break;
}
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
default: value = ConstantU32(state, 0); break;
}
value = EmitDppValueU32(state, operand, value);
return EmitSdwaExtractU32(state, operand, value);
}
uint32_t EmitRegisterLoad(EmitterState* state, IR::Register reg) {
if (IsInactiveWave32ExecHigh(*state, reg)) {
return ConstantU32(state, 0);
}
const auto pointer = PointerForRegister(*state, reg);
if (pointer == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
return value;
}
uint32_t WaveMaskHighLoad(EmitterState* state, IR::RegisterFile file) {
if (state->wave_size == 32u && IsMaskRegisterFile(file)) {
return ConstantU32(state, 0);
}
return EmitRegisterLoad(state, {file, 1});
}
uint32_t EmitLaneMaskPartU32(EmitterState* state, uint32_t part) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto lane = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
if (part == 0) {
const auto in_low = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpULessThan, state->bool_type, in_low, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_low, bit, ConstantU32(state, 0)});
return ret;
}
if (state->wave_size == 32u) {
return ConstantU32(state, 0);
}
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, bit, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitThreadMaskBelowPartU32(EmitterState* state, uint32_t part) {
const auto subid = EmitSubgroupLocalInvocationId(state);
const auto lane = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
const auto below = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
state->builder.AddFunction({OpISub, state->uint_type, below, bit, ConstantU32(state, 1)});
if (part == 0) {
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
return ret;
}
if (state->wave_size == 32u) {
return ConstantU32(state, 0);
}
const auto in_high = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
state->builder.AddFunction(
{OpSelect, state->uint_type, ret, in_high, below, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitMaskActiveBool(EmitterState* state, IR::RegisterFile file) {
const auto low = EmitRegisterLoad(state, {file, 0});
const auto condition = state->builder.AllocateId();
if (state->per_invocation_masks) {
const auto combined = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseOr, state->uint_type, combined, low, WaveMaskHighLoad(state, file)});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, combined, ConstantU32(state, 0)});
return condition;
}
if (state->exact_subgroup_operations) {
const auto high = WaveMaskHighLoad(state, file);
const auto low_hit = state->builder.AllocateId();
const auto high_hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, active, low_hit, high_hit});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, active, ConstantU32(state, 0)});
return condition;
}
state->builder.AddFunction(
{OpINotEqual, state->bool_type, condition, low, ConstantU32(state, 0)});
return condition;
}
uint32_t EmitMaskZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
const auto active = EmitMaskActiveBool(state, file);
if (!zero) {
return active;
}
const auto cond = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, cond, active});
return cond;
}
uint32_t EmitMaskSummaryZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
if (state->per_invocation_masks) {
return EmitMaskZeroBool(state, file, zero);
}
if (state->exact_subgroup_operations) {
const auto low = EmitRegisterLoad(state, {file, 0});
const auto high = WaveMaskHighLoad(state, file);
const auto combined = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
if (!zero) {
return active;
}
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
return inactive;
}
const auto active = EmitMaskActiveBool(state, file);
if (!zero) {
return active;
}
const auto inactive = state->builder.AllocateId();
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
return inactive;
}
uint32_t EmitExecActiveBool(EmitterState* state) {
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
}
uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) {
if (operand.kind == IR::OperandKind::Register) {
if (operand.reg.file == IR::RegisterFile::Exec ||
operand.reg.file == IR::RegisterFile::Vcc) {
return EmitMaskActiveBool(state, operand.reg.file);
}
}
const auto cond_value = EmitValueLoad(state, operand);
const auto cond = state->builder.AllocateId();
state->builder.AddFunction(
{OpINotEqual, state->bool_type, cond, cond_value, ConstantU32(state, 0)});
return cond;
}
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state) {
if (state->subgroup_local_invocation_id_variable == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpLoad, state->uint_type, value, state->subgroup_local_invocation_id_variable});
return value;
}
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
for (const auto& input: state.inputs) {
if (input.kind == kind) {
return input.variable_id;
}
}
return 0;
}
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) {
for (const auto& input: state.inputs) {
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
return input.variable_id;
}
}
return 0;
}
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
for (const auto& input: state.inputs) {
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
return &input;
}
}
return nullptr;
}
uint32_t EmitInputComponentU32(EmitterState* state, IR::StageInputKind kind, uint32_t component) {
const auto variable = InputVariableForKind(*state, kind);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto pointer = state->builder.AllocateId();
const auto value = state->builder.AllocateId();
state->builder.AddFunction(
{OpAccessChain, state->ptr_input_uint, pointer, variable, ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
return value;
}
uint32_t EmitLocalInvocationIndex(EmitterState* state) {
const auto variable = InputVariableForKind(*state, IR::StageInputKind::LocalInvocationIndex);
if (variable == 0) {
return ConstantU32(state, 0);
}
const auto value = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->uint_type, value, variable});
return value;
}
uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKind kind,
uint32_t component) {
if ((component & 3u) != 3u) {
return ConstantU32(state, 0);
}
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
}
uint32_t EmitVertexParameterComponentU32(EmitterState* state, const InputBinding& input,
uint32_t component) {
const auto count = VertexParameterComponentCount(*state, input);
const auto kind = VertexParameterScalarKind(*state, input.location);
if (component >= count) {
return VertexInputDefaultComponentU32(state, kind, component);
}
const auto scalar_type = VertexParameterScalarType(*state, kind);
uint32_t raw = state->builder.AllocateId();
if (count == 1u) {
state->builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
} else {
const auto pointer_type = VertexParameterScalarPointerType(*state, kind);
const auto pointer = state->builder.AllocateId();
state->builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
ConstantU32(state, component)});
state->builder.AddFunction({OpLoad, scalar_type, raw, pointer});
}
if (kind == VertexInputScalarKind::Uint) {
return raw;
}
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw});
return bits;
}
void EmitLoadInputF32(EmitterState* state, const IR::Instruction& inst) {
const auto* input = InputBindingForParameter(*state, inst.input_info.attr);
if (input == nullptr || input->variable_id == 0) {
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
return;
}
if (state->stage == ShaderType::Vertex) {
EmitStoreU32(state, inst.dst,
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
return;
}
const auto input_value = state->builder.AllocateId();
const auto component = state->builder.AllocateId();
const auto bits = state->builder.AllocateId();
state->builder.AddFunction({OpLoad, state->vec4_float_type, input_value, input->variable_id});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, component, input_value, inst.input_info.chan & 3u});
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
EmitStoreU32(state, inst.dst, bits);
}
uint32_t EmitSccBool(EmitterState* state, bool non_zero) {
const auto value = EmitRegisterLoad(state, SccRegister());
const auto cond = state->builder.AllocateId();
state->builder.AddFunction(
{non_zero ? OpINotEqual : OpIEqual, state->bool_type, cond, value, ConstantU32(state, 0)});
return cond;
}
uint32_t EmitFloatLoad(EmitterState* state, const IR::Operand& operand) {
const auto uint_value = EmitValueLoad(state, operand);
auto float_value = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, float_value, uint_value});
if (operand.absolute) {
const auto abs_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, float_value});
float_value = abs_value;
}
if (operand.negate) {
const auto neg_value = state->builder.AllocateId();
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, float_value});
float_value = neg_value;
}
return float_value;
}
uint32_t ApplyResultModifiersF32(EmitterState* state, uint32_t value, const IR::Operand& dst) {
uint32_t result = value;
if (dst.omod != 0) {
const uint32_t bits = dst.omod == 1u ? 0x40000000u
: dst.omod == 2u ? 0x40800000u
: 0x3f000000u;
const auto scaled = state->builder.AllocateId();
state->builder.AddFunction(
{OpFMul, state->float_type, scaled, result, ConstantF32(state, bits)});
result = scaled;
}
if (!dst.clamp) {
return result;
}
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->float_type, ret, state->glsl_std450, GlslFClamp,
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
return ret;
}
uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) {
auto value_operand = operand;
value_operand.negate_hi = false;
if (operand.op_sel_hi) {
auto raw = operand;
raw.sdwa_sel = 6;
raw.sdwa_sext = false;
raw.op_sel = false;
raw.op_sel_hi = false;
raw.negate = false;
raw.negate_hi = false;
raw.absolute = false;
raw.dpp = false;
raw.dpp_fetch_inactive = false;
raw.dpp_bound_ctrl = false;
const auto bits = EmitValueLoad(state, raw);
const auto unpacked = state->builder.AllocateId();
auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450,
GlslUnpackHalf2x16, bits});
state->builder.AddFunction(
{OpCompositeExtract, state->float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
if (operand.absolute) {
const auto abs_value = state->builder.AllocateId();
state->builder.AddFunction(
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32});
f32 = abs_value;
}
if (value_operand.negate) {
const auto neg_value = state->builder.AllocateId();
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32});
f32 = neg_value;
}
return f32;
}
value_operand.op_sel = false;
value_operand.op_sel_hi = false;
return EmitFloatLoad(state, value_operand);
}
IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
if (offset == 0 || operand.kind != IR::OperandKind::Register) {
return operand;
}
auto ret = operand;
ret.reg.index += offset;
ret.sdwa_sel = 6;
ret.omod = 0;
ret.sdwa_sext = false;
ret.op_sel = false;
ret.op_sel_hi = false;
ret.negate = false;
ret.negate_hi = false;
ret.absolute = false;
ret.clamp = false;
ret.dpp_ctrl = 0;
ret.dpp_row_mask = 0xf;
ret.dpp_bank_mask = 0xf;
ret.dpp_fetch_inactive = false;
ret.dpp_bound_ctrl = false;
ret.dpp = false;
return ret;
}
uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& operand) {
const auto low = EmitValueLoad(state, operand);
const auto active = state->builder.AllocateId();
if (state->per_invocation_masks || state->exact_subgroup_operations) {
uint32_t high = ConstantU32(state, 0);
if (operand.kind == IR::OperandKind::Register &&
(operand.reg.file == IR::RegisterFile::Scalar ||
operand.reg.file == IR::RegisterFile::Vcc ||
operand.reg.file == IR::RegisterFile::Exec)) {
high = IsMaskRegisterFile(operand.reg.file)
? WaveMaskHighLoad(state, operand.reg.file)
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
}
if (state->per_invocation_masks) {
const auto combined = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
return active;
}
const auto low_hit = state->builder.AllocateId();
const auto high_hit = state->builder.AllocateId();
const auto hit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
state->builder.AddFunction({OpBitwiseOr, state->uint_type, hit, low_hit, high_hit});
state->builder.AddFunction(
{OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
return active;
}
state->builder.AddFunction({OpINotEqual, state->bool_type, active, low, ConstantU32(state, 0)});
return active;
}
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane) {
if (state->per_invocation_masks) {
const auto ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
return EmitBallotLaneActiveBool(state, ballot, lane);
}
const auto lane_low = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
if (state->wave_size == 64u) {
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
const auto in_high = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, mask});
mask = selected;
}
const auto hit = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, ret, hit, ConstantU32(state, 0)});
return ret;
}
uint32_t EmitSubgroupLaneActiveBool(EmitterState* state, uint32_t lane) {
const auto active_ballot = state->builder.AllocateId();
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, active_ballot,
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
return EmitBallotLaneActiveBool(state, active_ballot, lane);
}
uint32_t EmitBallotLaneActiveBool(EmitterState* state, uint32_t active_ballot, uint32_t lane) {
const auto low = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, active_ballot, 0});
uint32_t mask = low;
if (state->wave_size == 64u) {
const auto high = state->builder.AllocateId();
const auto in_high = state->builder.AllocateId();
const auto selected = state->builder.AllocateId();
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, active_ballot, 1});
state->builder.AddFunction(
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, low});
mask = selected;
}
const auto lane_low = state->builder.AllocateId();
const auto bit = state->builder.AllocateId();
state->builder.AddFunction(
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
state->builder.AddFunction(
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
const auto hit = state->builder.AllocateId();
const auto active = state->builder.AllocateId();
const auto in_range = state->builder.AllocateId();
const auto ret = state->builder.AllocateId();
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
state->builder.AddFunction(
{OpULessThan, state->bool_type, in_range, lane, ConstantU32(state, state->wave_size)});
state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, active, in_range});
return ret;
}
uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
}
if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) {
return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u);
}
return ConstantU32(state, 0);
}
uint32_t EmitSequentialFloatLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
const auto value = EmitSequentialValueLoad(state, operand, offset);
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
return f32;
}
bool ImageAddressUsesA16(const IR::Instruction& inst) {
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u;
}
bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) {
if (!ImageAddressUsesA16(inst)) {
return false;
}
uint32_t cursor = 0;
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) {
if (component == cursor) {
return false;
}
cursor++;
}
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) {
if (component == cursor) {
return true;
}
cursor++;
}
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u &&
component == cursor) {
return false;
}
return true;
}
uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) {
if (!ImageAddressUsesA16(inst)) {
return component * 2u;
}
uint32_t half_component = 0;
for (uint32_t i = 0; i < component; i++) {
const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u;
if (half_width == 2u && (half_component & 1u) != 0u) {
half_component++;
}
half_component += half_width;
}
if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) {
half_component++;
}
return half_component;
}
uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) {
return ImageAddressHalfComponent(inst, component) / 2u;
}
IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
uint32_t component) {
const auto packed_component = ImageAddressPackedComponent(inst, component);
if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) {
const auto nsa_index = packed_component - 1u;
const auto nsa_count = std::min<uint32_t>(inst.memory.image_nsa_dwords * 4u,
Decoder::MaxImageNsaAddressComponents);
if (nsa_index < nsa_count) {
return MakeRegisterOperand(IR::RegisterFile::Vector,
inst.memory.image_nsa_addr[nsa_index]);
}
}
return OffsetRegisterOperand(base, packed_component);
}
uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& inst,
const IR::Operand& base, uint32_t component) {
const auto operand = ImageAddressOperand(inst, base, component);
const auto packed_component = ImageAddressPackedComponent(inst, component);
if (packed_component == 0 || operand.kind == IR::OperandKind::Register) {
auto value = EmitValueLoad(state, operand);
if (ImageAddressComponentUsesA16(inst, component)) {
if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) {
value = EmitShiftRightConstant(state, value, 16u);
}
value = EmitAndConstant(state, value, 0xffffu);
}
return value;
}
return ConstantU32(state, 0);
}
uint32_t EmitImageAddressFloatLoad(EmitterState* state, const IR::Instruction& inst,
const IR::Operand& base, uint32_t component) {
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
if (ImageAddressComponentUsesA16(inst, component)) {
return EmitF16BitsToF32(state, value);
}
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
return f32;
}
uint32_t EmitZeroF32(EmitterState* state) {
const auto f32 = state->builder.AllocateId();
state->builder.AddFunction({OpBitcast, state->float_type, f32, ConstantU32(state, 0)});
return f32;
}
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter