mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-29 00:33:06 +00:00
Initial commit
This commit is contained in:
nmzik committed
commit
147188b143
427 files changed
+205923
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,752 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/guest_gpu/gpu_defs.h"
|
||||
#include "graphics/shader/recompiler/SpirvEmitter.h"
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
|
||||
const auto* ps = state.pixel_input_info;
|
||||
if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
|
||||
return attr;
|
||||
}
|
||||
// VINTRP ATTR selects the PS input slot. SPI_PS_INPUT_CNTL maps that slot to a
|
||||
// VS parameter export, which is the SPIR-V location we must link against.
|
||||
return ps->interpolator_settings[attr] & PsInputOffsetMask;
|
||||
}
|
||||
|
||||
uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
|
||||
bool used_locations[32] = {};
|
||||
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind != IR::StageInputKind::Parameter) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto location = PixelParameterMappedLocation(state, input.location);
|
||||
if (location < std::size(used_locations) && used_locations[location]) {
|
||||
auto fallback_location = input.location;
|
||||
while (fallback_location < std::size(used_locations) &&
|
||||
used_locations[fallback_location]) {
|
||||
fallback_location++;
|
||||
}
|
||||
EXIT_NOT_IMPLEMENTED(fallback_location >= std::size(used_locations));
|
||||
location = fallback_location;
|
||||
}
|
||||
|
||||
if (input.location == attr) {
|
||||
return location;
|
||||
}
|
||||
|
||||
if (location < std::size(used_locations)) {
|
||||
used_locations[location] = true;
|
||||
}
|
||||
}
|
||||
|
||||
return PixelParameterMappedLocation(state, attr);
|
||||
}
|
||||
|
||||
bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
|
||||
const auto* ps = state.pixel_input_info;
|
||||
if (state.stage != ShaderType::Pixel || ps == nullptr || attr >= ps->input_num) {
|
||||
return false;
|
||||
}
|
||||
return (ps->interpolator_settings[attr] & PsInputFlatShade) != 0;
|
||||
}
|
||||
|
||||
void SetError(std::string* error, const char* message) {
|
||||
if (error != nullptr) {
|
||||
*error = message;
|
||||
}
|
||||
}
|
||||
|
||||
void CollectRegister(std::vector<RegisterBinding>* registers, IR::Register reg) {
|
||||
if (std::any_of(registers->begin(), registers->end(),
|
||||
[reg](const RegisterBinding& binding) { return binding.reg == reg; })) {
|
||||
return;
|
||||
}
|
||||
registers->push_back({reg, 0});
|
||||
}
|
||||
|
||||
IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
|
||||
IR::Operand operand;
|
||||
operand.kind = IR::OperandKind::Register;
|
||||
operand.reg = {file, index};
|
||||
return operand;
|
||||
}
|
||||
|
||||
IR::Register SccRegister() {
|
||||
return {IR::RegisterFile::Scc, static_cast<uint32_t>(Decoder::OperandKind::Scc)};
|
||||
}
|
||||
|
||||
IR::Operand SccOperand() {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Scc,
|
||||
static_cast<uint32_t>(Decoder::OperandKind::Scc));
|
||||
}
|
||||
|
||||
bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
|
||||
return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
|
||||
}
|
||||
|
||||
bool IsMaskRegisterFile(IR::RegisterFile file) {
|
||||
return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc;
|
||||
}
|
||||
|
||||
bool IsSccOperand(const IR::Operand& operand) {
|
||||
return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
|
||||
}
|
||||
|
||||
bool IsCompareOpcode(IR::Opcode op);
|
||||
|
||||
void CollectMaskStateRegisters(std::vector<RegisterBinding>* registers) {
|
||||
CollectRegister(registers, {IR::RegisterFile::Exec, 0});
|
||||
CollectRegister(registers, {IR::RegisterFile::Exec, 1});
|
||||
CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
|
||||
CollectRegister(registers, {IR::RegisterFile::Vcc, 1});
|
||||
CollectRegister(registers, SccRegister());
|
||||
}
|
||||
|
||||
void CollectSequentialRegisters(std::vector<RegisterBinding>* registers, const IR::Operand& base,
|
||||
uint32_t count) {
|
||||
if (base.kind != IR::OperandKind::Register) {
|
||||
return;
|
||||
}
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
auto reg = base.reg;
|
||||
reg.index += i;
|
||||
CollectRegister(registers, reg);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& registers) {
|
||||
uint32_t max_end = 0;
|
||||
for (const auto& binding: registers) {
|
||||
if (binding.reg.file == IR::RegisterFile::Vector) {
|
||||
max_end = std::max(max_end, binding.reg.index + 1u);
|
||||
}
|
||||
}
|
||||
return std::min(max_end, 256u);
|
||||
}
|
||||
|
||||
void CollectMoveRelSourceRegisters(const IR::Program& program,
|
||||
std::vector<RegisterBinding>* registers) {
|
||||
const auto max_vector_end = MaxCollectedVectorRegisterEnd(*registers);
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op != IR::Opcode::MoveRelSourceU32 ||
|
||||
inst.src[0].kind != IR::OperandKind::Register ||
|
||||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
|
||||
continue;
|
||||
}
|
||||
const auto base_index = inst.src[0].reg.index;
|
||||
const auto limit = std::min(std::max(max_vector_end, base_index + 1u), 256u);
|
||||
CollectSequentialRegisters(registers, inst.src[0], limit - base_index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool IsPairDwordOpcode(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::MoveU64:
|
||||
case IR::Opcode::WqmB64:
|
||||
case IR::Opcode::SaveexecB64:
|
||||
case IR::Opcode::BitwiseAndU64:
|
||||
case IR::Opcode::BitwiseAndNotU64:
|
||||
case IR::Opcode::BitwiseOrU64:
|
||||
case IR::Opcode::BitwiseOrNotU64:
|
||||
case IR::Opcode::BitwiseXorU64:
|
||||
case IR::Opcode::BitwiseNandU64:
|
||||
case IR::Opcode::BitwiseNorU64:
|
||||
case IR::Opcode::BitwiseXnorU64:
|
||||
case IR::Opcode::BitwiseNotU64:
|
||||
case IR::Opcode::BitFieldMaskU64:
|
||||
case IR::Opcode::BitFieldExtractU64:
|
||||
case IR::Opcode::BitReplicateB64B32:
|
||||
case IR::Opcode::ShiftLeftLogicalU64:
|
||||
case IR::Opcode::ShiftRightLogicalU64:
|
||||
case IR::Opcode::SelectU64: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
|
||||
switch (op) {
|
||||
case IR::Opcode::BitFieldMaskU64: return 0;
|
||||
case IR::Opcode::BitReplicateB64B32: return 0;
|
||||
case IR::Opcode::BitFieldExtractU64: return 1;
|
||||
case IR::Opcode::ShiftLeftLogicalU64:
|
||||
case IR::Opcode::ShiftRightLogicalU64:
|
||||
case IR::Opcode::BitwiseNotU64:
|
||||
case IR::Opcode::SaveexecB64:
|
||||
case IR::Opcode::WqmB64:
|
||||
case IR::Opcode::MoveU64: return std::min<uint32_t>(src_count, 1u);
|
||||
default: return src_count;
|
||||
}
|
||||
}
|
||||
|
||||
void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>* registers) {
|
||||
CollectMaskStateRegisters(registers);
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.dst.kind == IR::OperandKind::Register) {
|
||||
CollectRegister(registers, inst.dst.reg);
|
||||
}
|
||||
if (inst.dst2.kind == IR::OperandKind::Register) {
|
||||
CollectRegister(registers, inst.dst2.reg);
|
||||
}
|
||||
for (uint32_t i = 0; i < inst.src_count; i++) {
|
||||
if (inst.src[i].kind == IR::OperandKind::Register) {
|
||||
CollectRegister(registers, inst.src[i].reg);
|
||||
}
|
||||
}
|
||||
if (IsPairDwordOpcode(inst.op)) {
|
||||
CollectSequentialRegisters(registers, inst.dst, 2);
|
||||
const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u;
|
||||
const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count);
|
||||
for (uint32_t i = first_pair_src; i < pair_src_count; i++) {
|
||||
CollectSequentialRegisters(registers, inst.src[i], 2);
|
||||
}
|
||||
}
|
||||
if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) {
|
||||
CollectSequentialRegisters(registers, inst.src[0], 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::CompareNeU64) {
|
||||
CollectSequentialRegisters(registers, inst.src[0], 2);
|
||||
CollectSequentialRegisters(registers, inst.src[1], 2);
|
||||
}
|
||||
if (IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file != IR::RegisterFile::Scc) {
|
||||
CollectSequentialRegisters(registers, inst.dst, 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) {
|
||||
CollectSequentialRegisters(registers, inst.dst2, 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::UMadU64U32) {
|
||||
CollectSequentialRegisters(registers, inst.dst, 2);
|
||||
CollectSequentialRegisters(registers, inst.dst2, 2);
|
||||
CollectSequentialRegisters(registers, inst.src[2], 2);
|
||||
}
|
||||
if (inst.op == IR::Opcode::SBufferLoadDword &&
|
||||
inst.memory.kind == IR::ResourceKind::ScalarBuffer) {
|
||||
CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u});
|
||||
}
|
||||
if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) &&
|
||||
inst.memory.image_address_components > 1u) {
|
||||
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[0],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageGetLod) {
|
||||
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[0],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageLoad) {
|
||||
CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[0],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.op == IR::Opcode::ImageStore) {
|
||||
CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords);
|
||||
CollectSequentialRegisters(registers, inst.src[1],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
if (inst.memory.kind == IR::ResourceKind::StorageImageUint) {
|
||||
CollectSequentialRegisters(registers, inst.src[1],
|
||||
inst.memory.image_address_components);
|
||||
}
|
||||
}
|
||||
}
|
||||
CollectMoveRelSourceRegisters(program, registers);
|
||||
}
|
||||
|
||||
bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
|
||||
uint32_t index) {
|
||||
return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
|
||||
return binding.kind == kind && binding.index == index;
|
||||
});
|
||||
}
|
||||
|
||||
void CopyProgramInputsAndOutputs(EmitterState* state, const IR::Program& program) {
|
||||
for (const auto& input: program.info.inputs) {
|
||||
state->inputs.push_back(
|
||||
{input.kind, input.location, input.component_count, 0, input.debug_name});
|
||||
}
|
||||
for (const auto& output: program.info.outputs) {
|
||||
if (HasOutput(state->outputs, output.kind, output.index)) {
|
||||
continue;
|
||||
}
|
||||
state->outputs.push_back(
|
||||
{output.kind, output.index, output.location, 0, output.debug_name});
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
|
||||
if (exp.kind == IR::ExportTargetKind::Position) {
|
||||
return state.per_vertex_variable;
|
||||
}
|
||||
if (exp.kind == IR::ExportTargetKind::MrtZ) {
|
||||
return state.depth_variable;
|
||||
}
|
||||
for (const auto& binding: state.outputs) {
|
||||
const auto expected_kind = exp.kind == IR::ExportTargetKind::Mrt
|
||||
? IR::StageOutputKind::Mrt
|
||||
: IR::StageOutputKind::Parameter;
|
||||
if (binding.kind == expected_kind && binding.index == exp.index) {
|
||||
return binding.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::ImageGetLod) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::ImageGather4) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IsLdsOpcode(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::DsReadUbyte:
|
||||
case IR::Opcode::DsReadSbyte:
|
||||
case IR::Opcode::DsReadUshort:
|
||||
case IR::Opcode::DsReadSshort:
|
||||
case IR::Opcode::DsReadB32:
|
||||
case IR::Opcode::DsWriteByte:
|
||||
case IR::Opcode::DsWriteShort:
|
||||
case IR::Opcode::DsWriteB32:
|
||||
case IR::Opcode::DsMinF32:
|
||||
case IR::Opcode::DsMaxF32:
|
||||
case IR::Opcode::DsSwizzleB32:
|
||||
case IR::Opcode::DsWriteAddtidB32:
|
||||
case IR::Opcode::DsReadAddtidB32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool ProgramNeedsFunctionLds(const IR::Program& program) {
|
||||
if (program.stage == ShaderType::Compute) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (IsLdsOpcode(inst.op)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsPixelValidMask(const IR::Program& program) {
|
||||
if (program.stage != ShaderType::Pixel) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::Export && inst.export_info.vm) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool InstructionHasDppSource(const IR::Instruction& inst) {
|
||||
for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) {
|
||||
if (inst.src[i].dpp) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
|
||||
if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::WqmB64) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ProgramRequiresExactSubgroupSize(program);
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 ||
|
||||
inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 ||
|
||||
inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume ||
|
||||
inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IsCompareOpcode(IR::Opcode op) {
|
||||
switch (op) {
|
||||
case IR::Opcode::CompareFalse:
|
||||
case IR::Opcode::CompareTrue:
|
||||
case IR::Opcode::CompareEqU32:
|
||||
case IR::Opcode::CompareNeU32:
|
||||
case IR::Opcode::CompareGtU32:
|
||||
case IR::Opcode::CompareGeU32:
|
||||
case IR::Opcode::CompareLtU32:
|
||||
case IR::Opcode::CompareLeU32:
|
||||
case IR::Opcode::CompareNeU64:
|
||||
case IR::Opcode::CompareMaskEqU32:
|
||||
case IR::Opcode::CompareMaskNeU32:
|
||||
case IR::Opcode::CompareMaskGtU32:
|
||||
case IR::Opcode::CompareMaskGeU32:
|
||||
case IR::Opcode::CompareMaskLtU32:
|
||||
case IR::Opcode::CompareMaskLeU32:
|
||||
case IR::Opcode::CompareEqI32:
|
||||
case IR::Opcode::CompareNeI32:
|
||||
case IR::Opcode::CompareGtI32:
|
||||
case IR::Opcode::CompareGeI32:
|
||||
case IR::Opcode::CompareLtI32:
|
||||
case IR::Opcode::CompareLeI32:
|
||||
case IR::Opcode::CompareEqI16:
|
||||
case IR::Opcode::CompareNeI16:
|
||||
case IR::Opcode::CompareGtI16:
|
||||
case IR::Opcode::CompareGeI16:
|
||||
case IR::Opcode::CompareLtI16:
|
||||
case IR::Opcode::CompareLeI16:
|
||||
case IR::Opcode::CompareMaskEqI32:
|
||||
case IR::Opcode::CompareMaskNeI32:
|
||||
case IR::Opcode::CompareMaskGtI32:
|
||||
case IR::Opcode::CompareMaskGeI32:
|
||||
case IR::Opcode::CompareMaskLtI32:
|
||||
case IR::Opcode::CompareMaskLeI32:
|
||||
case IR::Opcode::CompareEqU16:
|
||||
case IR::Opcode::CompareNeU16:
|
||||
case IR::Opcode::CompareGtU16:
|
||||
case IR::Opcode::CompareGeU16:
|
||||
case IR::Opcode::CompareLtU16:
|
||||
case IR::Opcode::CompareLeU16:
|
||||
case IR::Opcode::CompareEqF32:
|
||||
case IR::Opcode::CompareNeF32:
|
||||
case IR::Opcode::CompareGtF32:
|
||||
case IR::Opcode::CompareGeF32:
|
||||
case IR::Opcode::CompareLtF32:
|
||||
case IR::Opcode::CompareLeF32:
|
||||
case IR::Opcode::CompareOrderedF32:
|
||||
case IR::Opcode::CompareUnorderedF32:
|
||||
case IR::Opcode::CompareUnordEqF32:
|
||||
case IR::Opcode::CompareUnordNeF32:
|
||||
case IR::Opcode::CompareUnordGtF32:
|
||||
case IR::Opcode::CompareUnordGeF32:
|
||||
case IR::Opcode::CompareUnordLtF32:
|
||||
case IR::Opcode::CompareUnordLeF32:
|
||||
case IR::Opcode::CompareClassF32:
|
||||
case IR::Opcode::CompareEqF16:
|
||||
case IR::Opcode::CompareNeF16:
|
||||
case IR::Opcode::CompareGtF16:
|
||||
case IR::Opcode::CompareGeF16:
|
||||
case IR::Opcode::CompareLtF16:
|
||||
case IR::Opcode::CompareLeF16:
|
||||
case IR::Opcode::CompareUnordNeF16:
|
||||
case IR::Opcode::CompareMaskEqF16:
|
||||
case IR::Opcode::CompareMaskNeF16:
|
||||
case IR::Opcode::CompareMaskGtF16:
|
||||
case IR::Opcode::CompareMaskGeF16:
|
||||
case IR::Opcode::CompareMaskLtF16:
|
||||
case IR::Opcode::CompareMaskLeF16:
|
||||
case IR::Opcode::CompareMaskUnordNeF16:
|
||||
case IR::Opcode::CompareMaskUnordGeF16:
|
||||
case IR::Opcode::CompareMaskEqF32:
|
||||
case IR::Opcode::CompareMaskNeF32:
|
||||
case IR::Opcode::CompareMaskGtF32:
|
||||
case IR::Opcode::CompareMaskGeF32:
|
||||
case IR::Opcode::CompareMaskLtF32:
|
||||
case IR::Opcode::CompareMaskLeF32:
|
||||
case IR::Opcode::CompareMaskUnordEqF32:
|
||||
case IR::Opcode::CompareMaskUnordNeF32:
|
||||
case IR::Opcode::CompareMaskUnordGtF32:
|
||||
case IR::Opcode::CompareMaskUnordGeF32:
|
||||
case IR::Opcode::CompareMaskUnordLtF32:
|
||||
case IR::Opcode::CompareMaskUnordLeF32: return true;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
|
||||
for (const auto& block: program.blocks) {
|
||||
for (const auto& inst: block.instructions) {
|
||||
if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 ||
|
||||
inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend ||
|
||||
inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 ||
|
||||
inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export ||
|
||||
InstructionHasDppSource(inst)) {
|
||||
return true;
|
||||
}
|
||||
if (IsCompareOpcode(inst.op)) {
|
||||
return true;
|
||||
}
|
||||
if (inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file == IR::RegisterFile::Vector) {
|
||||
return true;
|
||||
}
|
||||
if (inst.dst2.kind == IR::OperandKind::Register &&
|
||||
inst.dst2.reg.file == IR::RegisterFile::Vector) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg == reg) {
|
||||
return binding.pointer_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState* state, uint32_t value);
|
||||
void EmitStoreU32(EmitterState* state, const IR::Operand& dst, uint32_t value);
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state);
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane);
|
||||
[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* reason) {
|
||||
EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
|
||||
" stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
|
||||
state.program->shader_hash, static_cast<unsigned>(state.stage), resource,
|
||||
static_cast<unsigned>(kind), reason);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
DescriptorResourceBinding ResourceForDescriptor(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource) {
|
||||
const auto* descriptor = IR::FindBinding(state.program->bindings, kind);
|
||||
if (descriptor == nullptr) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
|
||||
}
|
||||
const auto found =
|
||||
std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
|
||||
if (found == descriptor->resources.end()) {
|
||||
ExitDescriptorBindingFailure(state, kind, resource,
|
||||
"resource is absent from descriptor group");
|
||||
}
|
||||
return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
|
||||
}
|
||||
|
||||
uint32_t DescriptorElementPointer(EmitterState* state, uint32_t result_ptr_type,
|
||||
uint32_t variable_id, uint32_t array_index,
|
||||
IR::DescriptorBindingKind kind, uint32_t resource,
|
||||
const char* variable_name) {
|
||||
if (variable_id == 0) {
|
||||
ExitDescriptorBindingFailure(*state, kind, resource, variable_name);
|
||||
}
|
||||
const auto pointer = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
|
||||
return pointer;
|
||||
}
|
||||
|
||||
ImageViewKind ImageViewKindFromDimension(Decoder::ImageDimension dimension) {
|
||||
switch (dimension) {
|
||||
case Decoder::ImageDimension::Dim2DArray: return ImageViewKind::Dim2DArray;
|
||||
case Decoder::ImageDimension::Dim3D: return ImageViewKind::Dim3D;
|
||||
default: return ImageViewKind::Dim2D;
|
||||
}
|
||||
}
|
||||
|
||||
ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t use_pc) {
|
||||
(void)state;
|
||||
(void)use_pc;
|
||||
return ImageViewKindFromDimension(mem.image_dimension);
|
||||
}
|
||||
|
||||
ImageViewKind StorageImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
|
||||
bool uint_image, uint32_t use_pc) {
|
||||
(void)state;
|
||||
(void)uint_image;
|
||||
(void)use_pc;
|
||||
return ImageViewKindFromDimension(mem.image_dimension);
|
||||
}
|
||||
|
||||
uint32_t ImageViewCoordinateComponents(ImageViewKind view) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray:
|
||||
case ImageViewKind::Dim3D: return 3u;
|
||||
default: return 2u;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageViewImageType(const EmitterState& state, ImageViewKind view, bool integer) {
|
||||
return state.sampled_images[SampledImageIndex(integer, view)].image_type;
|
||||
}
|
||||
|
||||
uint32_t ImageViewSampledImageType(const EmitterState& state, ImageViewKind view, bool integer) {
|
||||
return state.sampled_images[SampledImageIndex(integer, view)].sampled_image_type;
|
||||
}
|
||||
|
||||
uint32_t ImageViewSizeType(const EmitterState& state, ImageViewKind view) {
|
||||
return ImageViewCoordinateComponents(view) == 3u ? state.vec3_uint_type : state.vec2_uint_type;
|
||||
}
|
||||
|
||||
uint32_t StorageImageType(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
if (uint_image) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return state.storage_image_uint_2d_array_type;
|
||||
case ImageViewKind::Dim3D: return state.storage_image_uint_3d_type;
|
||||
default: return state.storage_image_uint_type;
|
||||
}
|
||||
}
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return state.storage_image_2d_array_type;
|
||||
case ImageViewKind::Dim3D: return state.storage_image_3d_type;
|
||||
default: return state.storage_image_type;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t StorageImagePointerType(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
if (uint_image) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return state.ptr_uniform_storage_image_uint_2d_array;
|
||||
case ImageViewKind::Dim3D: return state.ptr_uniform_storage_image_uint_3d;
|
||||
default: return state.ptr_uniform_storage_image_uint;
|
||||
}
|
||||
}
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return state.ptr_uniform_storage_image_2d_array;
|
||||
case ImageViewKind::Dim3D: return state.ptr_uniform_storage_image_3d;
|
||||
default: return state.ptr_uniform_storage_image;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t StorageImageVariable(const EmitterState& state, bool uint_image, ImageViewKind view) {
|
||||
if (uint_image) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return state.storage_image_uint_2d_array_variable;
|
||||
case ImageViewKind::Dim3D: return state.storage_image_uint_3d_variable;
|
||||
default: return state.storage_image_uint_variable;
|
||||
}
|
||||
}
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return state.storage_image_2d_array_variable;
|
||||
case ImageViewKind::Dim3D: return state.storage_image_3d_variable;
|
||||
default: return state.storage_image_variable;
|
||||
}
|
||||
}
|
||||
|
||||
IR::DescriptorBindingKind StorageBindingKind(bool uint_image, ImageViewKind view) {
|
||||
if (uint_image) {
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::StorageUint2DArray;
|
||||
case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
|
||||
default: return IR::DescriptorBindingKind::StorageUint2D;
|
||||
}
|
||||
}
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
|
||||
case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
|
||||
default: return IR::DescriptorBindingKind::Storage2D;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t LoadSampledImageDescriptor(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
|
||||
ImageViewKind view) {
|
||||
(void)use_pc;
|
||||
const bool integer = mem.kind == IR::ResourceKind::ImageUint;
|
||||
const auto kind = SampledBindingKind(integer, view);
|
||||
const auto binding = ResourceForDescriptor(*state, kind, mem.resource);
|
||||
const auto& descriptors = state->sampled_images[SampledImageIndex(integer, view)];
|
||||
const auto pointer = DescriptorElementPointer(
|
||||
state, descriptors.pointer_type, descriptors.variable, binding.array_index, kind,
|
||||
mem.resource, "sampled image descriptor array was not emitted");
|
||||
const auto image = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, ImageViewImageType(*state, view, integer), image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
uint32_t LoadSamplerDescriptor(EmitterState* state, uint32_t sampler, uint32_t use_pc) {
|
||||
(void)use_pc;
|
||||
const auto binding =
|
||||
ResourceForDescriptor(*state, IR::DescriptorBindingKind::Samplers, sampler);
|
||||
const auto pointer = DescriptorElementPointer(
|
||||
state, state->ptr_uniform_sampler, state->sampler_variable, binding.array_index,
|
||||
IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
|
||||
const auto sampler_id = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, state->sampler_type, sampler_id, pointer});
|
||||
return sampler_id;
|
||||
}
|
||||
|
||||
uint32_t MakeSampledImage(EmitterState* state, const IR::MemoryInfo& mem, uint32_t use_pc,
|
||||
ImageViewKind view) {
|
||||
const auto image = LoadSampledImageDescriptor(state, mem, use_pc, view);
|
||||
const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
|
||||
if (image == 0 || sampler == 0) {
|
||||
ExitDescriptorBindingFailure(
|
||||
*state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
|
||||
"sampled image or sampler descriptor load failed");
|
||||
}
|
||||
const auto sampled_image = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpSampledImage,
|
||||
ImageViewSampledImageType(*state, view, mem.kind == IR::ResourceKind::ImageUint),
|
||||
sampled_image, image, sampler});
|
||||
return sampled_image;
|
||||
}
|
||||
|
||||
uint32_t StorageImageDescriptorPointer(EmitterState* state, uint32_t resource, bool uint_image,
|
||||
uint32_t use_pc, ImageViewKind view) {
|
||||
(void)use_pc;
|
||||
const auto kind = StorageBindingKind(uint_image, view);
|
||||
const auto binding = ResourceForDescriptor(*state, kind, resource);
|
||||
const auto ptr_type = StorageImagePointerType(*state, uint_image, view);
|
||||
const auto variable = StorageImageVariable(*state, uint_image, view);
|
||||
return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
|
||||
"storage image descriptor array was not emitted");
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptor(EmitterState* state, uint32_t resource, bool uint_image,
|
||||
uint32_t use_pc, ImageViewKind view) {
|
||||
const auto pointer = StorageImageDescriptorPointer(state, resource, uint_image, use_pc, view);
|
||||
if (pointer == 0) {
|
||||
ExitDescriptorBindingFailure(*state, StorageBindingKind(uint_image, view), resource,
|
||||
"storage image descriptor pointer creation failed");
|
||||
}
|
||||
const auto type = StorageImageType(*state, uint_image, view);
|
||||
const auto image = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
uint32_t ExecutionModelForStage(ShaderType stage) {
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex: return ExecutionModelVertex;
|
||||
case ShaderType::Pixel: return ExecutionModelFragment;
|
||||
default: return ExecutionModelGLCompute;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,154 @@
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
uint32_t EmitExportComponentF32(EmitterState* state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
|
||||
if (!enabled || component >= inst.src_count || component >= 4u) {
|
||||
return ConstantF32(state, component == 3u ? 0x3f800000u : 0u);
|
||||
}
|
||||
return EmitFloatLoad(state, inst.src[component]);
|
||||
}
|
||||
|
||||
uint32_t EmitExportVec4F32(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.compr) {
|
||||
uint32_t components[4] = {
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0u),
|
||||
ConstantF32(state, 0x3f800000u),
|
||||
};
|
||||
for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
|
||||
pair_index++) {
|
||||
const auto raw = EmitValueLoad(state, inst.src[pair_index]);
|
||||
const auto unpacked = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked,
|
||||
state->glsl_std450, GlslUnpackHalf2x16, raw});
|
||||
for (uint32_t lane = 0; lane < 2u; lane++) {
|
||||
const auto component = pair_index * 2u + lane;
|
||||
if (((inst.export_info.en >> component) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
components[component] = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeExtract, state->float_type, components[component], unpacked, lane});
|
||||
}
|
||||
}
|
||||
const auto vec = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec,
|
||||
components[0], components[1], components[2], components[3]});
|
||||
return vec;
|
||||
}
|
||||
|
||||
const auto x = EmitExportComponentF32(state, inst, 0);
|
||||
const auto y = EmitExportComponentF32(state, inst, 1);
|
||||
const auto z = EmitExportComponentF32(state, inst, 2);
|
||||
const auto w = EmitExportComponentF32(state, inst, 3);
|
||||
const auto vec = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, vec, x, y, z, w});
|
||||
return vec;
|
||||
}
|
||||
|
||||
bool ExportWritesData(const IR::Instruction& inst) {
|
||||
switch (inst.export_info.kind) {
|
||||
case IR::ExportTargetKind::Null:
|
||||
case IR::ExportTargetKind::Primitive: return false;
|
||||
case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0;
|
||||
default: return inst.export_info.en != 0;
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMrtZExport(EmitterState* state, const IR::Instruction& inst) {
|
||||
if ((inst.export_info.en & 0x1u) != 0 && state->depth_variable != 0) {
|
||||
state->builder.AddFunction(
|
||||
{OpStore, state->depth_variable, EmitExportComponentF32(state, inst, 0)});
|
||||
}
|
||||
|
||||
if ((inst.export_info.en & 0x4u) != 0 && state->sample_mask_variable != 0) {
|
||||
const auto raw =
|
||||
inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
|
||||
const auto mask = state->builder.AllocateId();
|
||||
const auto ptr = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->int_type, mask, raw});
|
||||
state->builder.AddFunction({OpAccessChain, state->ptr_output_int, ptr,
|
||||
state->sample_mask_variable, ConstantU32(state, 0)});
|
||||
state->builder.AddFunction({OpStore, ptr, mask});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitExport(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Null ||
|
||||
inst.export_info.kind == IR::ExportTargetKind::Primitive) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ExportWritesData(inst)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) {
|
||||
EmitMrtZExport(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto variable = OutputVariableForExport(*state, inst.export_info);
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto value = EmitExportVec4F32(state, inst);
|
||||
if (inst.export_info.kind == IR::ExportTargetKind::Position) {
|
||||
const auto pointer = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpAccessChain, state->ptr_output_vec4_float, pointer, variable,
|
||||
ConstantU32(state, 0)});
|
||||
state->builder.AddFunction({OpStore, pointer, value});
|
||||
return;
|
||||
}
|
||||
|
||||
state->builder.AddFunction({OpStore, variable, value});
|
||||
}
|
||||
|
||||
bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
|
||||
return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
|
||||
}
|
||||
|
||||
void EmitKillIfBoolFalse(EmitterState* state, uint32_t active) {
|
||||
const auto kill_label = state->builder.AllocateId();
|
||||
const auto merge_label = state->builder.AllocateId();
|
||||
const auto inactive = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
|
||||
state->builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
|
||||
state->builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
|
||||
state->builder.AddFunction({OpLabel, kill_label});
|
||||
state->builder.AddFunction({OpKill});
|
||||
state->builder.AddFunction({OpLabel, merge_label});
|
||||
}
|
||||
|
||||
void EmitUpdatePixelValidMask(EmitterState* state) {
|
||||
if (state->pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
state->builder.AddFunction({OpStore, state->pixel_valid_mask_variable, value});
|
||||
}
|
||||
|
||||
void EmitKillIfPixelValidMaskInactive(EmitterState* state) {
|
||||
if (state->pixel_valid_mask_variable == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto mask_value = state->builder.AllocateId();
|
||||
const auto active = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpLoad, state->uint_type, mask_value, state->pixel_valid_mask_variable});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, active, mask_value, ConstantU32(state, 0)});
|
||||
EmitKillIfBoolFalse(state, active);
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,381 @@
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
void EmitMoveU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitMoveF32Bits(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto value = EmitFloatLoad(state, inst.src[0]);
|
||||
const auto bits = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, value});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
|
||||
uint32_t limit = base.index + 1u;
|
||||
for (const auto& binding: state.registers) {
|
||||
if (binding.reg.file == IR::RegisterFile::Vector) {
|
||||
limit = std::max(limit, binding.reg.index + 1u);
|
||||
}
|
||||
}
|
||||
return std::min(limit, 256u);
|
||||
}
|
||||
|
||||
void EmitMoveRelSourceU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (inst.src[0].kind != IR::OperandKind::Register ||
|
||||
inst.src[0].reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.src[0].reg;
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
auto selected = EmitRegisterLoad(state, base_reg);
|
||||
const auto limit = MoveRelRegisterLimit(*state, base_reg);
|
||||
for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
|
||||
const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
|
||||
const auto match = state->builder.AllocateId();
|
||||
const auto next = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, next, match, candidate, selected});
|
||||
selected = next;
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, selected);
|
||||
}
|
||||
|
||||
void EmitMoveRelDestU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (inst.dst.kind != IR::OperandKind::Register ||
|
||||
inst.dst.reg.file != IR::RegisterFile::Vector) {
|
||||
EmitMoveU32(state, inst);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto base_reg = inst.dst.reg;
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
|
||||
const auto exec_active = EmitExecActiveBool(state);
|
||||
const auto limit = MoveRelRegisterLimit(*state, base_reg);
|
||||
for (uint32_t reg = base_reg.index; reg < limit; reg++) {
|
||||
const auto pointer = PointerForRegister(*state, {IR::RegisterFile::Vector, reg});
|
||||
if (pointer == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto old_value = state->builder.AllocateId();
|
||||
const auto match = state->builder.AllocateId();
|
||||
const auto write = state->builder.AllocateId();
|
||||
const auto merged = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, state->uint_type, old_value, pointer});
|
||||
state->builder.AddFunction(
|
||||
{OpIEqual, state->bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
|
||||
state->builder.AddFunction({OpLogicalAnd, state->bool_type, write, exec_active, match});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, merged, write, value, old_value});
|
||||
state->builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitMoveU64(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
IsMaskRegisterFile(inst.dst.reg.file)) {
|
||||
EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
|
||||
return;
|
||||
}
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
EmitStoreU32(state, inst.dst, low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
|
||||
}
|
||||
|
||||
uint32_t EmitWqmLaneU32(EmitterState* state, uint32_t src) {
|
||||
uint32_t ret = ConstantU32(state, 0);
|
||||
for (uint32_t i = 0; i < 8u; i++) {
|
||||
const auto mask = 0x0fu << (i * 4u);
|
||||
const auto masked = state->builder.AllocateId();
|
||||
const auto non_zero = state->builder.AllocateId();
|
||||
const auto expanded = state->builder.AllocateId();
|
||||
const auto combined = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, masked, src, ConstantU32(state, mask)});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, non_zero, masked, ConstantU32(state, 0)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, expanded, non_zero,
|
||||
ConstantU32(state, mask), ConstantU32(state, 0)});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, ret, expanded});
|
||||
ret = combined;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void EmitWqmB64(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (!state->per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
|
||||
inst.dst.reg.file == IR::RegisterFile::Scalar) {
|
||||
const auto low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
const auto ret_low = EmitWqmLaneU32(state, low);
|
||||
const auto ret_high = EmitWqmLaneU32(state, high);
|
||||
EmitStoreU32(state, inst.dst, ret_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
|
||||
return;
|
||||
}
|
||||
const auto ballot = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup),
|
||||
EmitLaneMaskOperandActiveBool(state, inst.src[0])});
|
||||
const auto low = state->builder.AllocateId();
|
||||
const auto high = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, ballot, 0});
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, ballot, 1});
|
||||
const auto wqm_low = EmitWqmLaneU32(state, low);
|
||||
const auto wqm_high = EmitWqmLaneU32(state, high);
|
||||
const auto lane = EmitSubgroupLocalInvocationId(state);
|
||||
const auto high_half = state->builder.AllocateId();
|
||||
const auto mask = state->builder.AllocateId();
|
||||
const auto bit_index = state->builder.AllocateId();
|
||||
const auto bit = state->builder.AllocateId();
|
||||
const auto hit = state->builder.AllocateId();
|
||||
const auto active = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, high_half, lane, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, mask, high_half, wqm_high, wqm_low});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, bit_index, lane, ConstantU32(state, 31)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), bit_index});
|
||||
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
|
||||
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
|
||||
if (state->per_invocation_masks) {
|
||||
EmitPerInvocationMask(state, inst.dst, active);
|
||||
} else {
|
||||
const auto result = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, result, active,
|
||||
ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, inst.dst, result);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSaveexecPerInvocation(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto old_exec = EmitExecActiveBool(state);
|
||||
const auto src = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
|
||||
EmitPerInvocationMask(state, inst.dst, old_exec);
|
||||
|
||||
uint32_t lhs = old_exec;
|
||||
uint32_t rhs = src;
|
||||
bool use_or = false;
|
||||
switch (inst.saveexec_mode) {
|
||||
case IR::SaveexecMode::And: break;
|
||||
case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break;
|
||||
case IR::SaveexecMode::Orn2:
|
||||
lhs = EmitLogicalNotBool(state, old_exec);
|
||||
use_or = true;
|
||||
break;
|
||||
}
|
||||
const auto result =
|
||||
use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs);
|
||||
EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result);
|
||||
EmitStoreSccBool(state, result);
|
||||
}
|
||||
|
||||
void EmitSaveexecB32(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (state->per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto src_low = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
|
||||
uint32_t rhs_low = src_low;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
|
||||
}
|
||||
|
||||
const auto new_low = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, new_low, rhs_low, old_low});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
|
||||
if (state->wave_size == 32u) {
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
|
||||
}
|
||||
|
||||
const auto cond = state->builder.AllocateId();
|
||||
const auto scc = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, cond, new_low, ConstantU32(state, 0)});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitSaveexecB64(EmitterState* state, const IR::Instruction& inst) {
|
||||
if (state->per_invocation_masks) {
|
||||
EmitSaveexecPerInvocation(state, inst);
|
||||
return;
|
||||
}
|
||||
const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto src_low = EmitSequentialValueLoad(state, inst.src[0], 0);
|
||||
const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1);
|
||||
|
||||
EmitStoreU32(state, inst.dst, old_low);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high);
|
||||
|
||||
uint32_t lhs_low = old_low;
|
||||
uint32_t lhs_high = old_high;
|
||||
uint32_t rhs_low = src_low;
|
||||
uint32_t rhs_high = src_high;
|
||||
uint32_t opcode = OpBitwiseAnd;
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
|
||||
rhs_low = state->builder.AllocateId();
|
||||
rhs_high = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpNot, state->uint_type, rhs_low, src_low});
|
||||
state->builder.AddFunction({OpNot, state->uint_type, rhs_high, src_high});
|
||||
}
|
||||
if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
|
||||
lhs_low = state->builder.AllocateId();
|
||||
lhs_high = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpNot, state->uint_type, lhs_low, old_low});
|
||||
state->builder.AddFunction({OpNot, state->uint_type, lhs_high, old_high});
|
||||
opcode = OpBitwiseOr;
|
||||
}
|
||||
|
||||
const auto new_low = state->builder.AllocateId();
|
||||
const auto new_high = state->builder.AllocateId();
|
||||
state->builder.AddFunction({opcode, state->uint_type, new_low, lhs_low, rhs_low});
|
||||
state->builder.AddFunction({opcode, state->uint_type, new_high, lhs_high, rhs_high});
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
|
||||
EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
|
||||
|
||||
const auto active_new_high = state->wave_size == 32u ? ConstantU32(state, 0) : new_high;
|
||||
const auto mask = state->builder.AllocateId();
|
||||
const auto cond = state->builder.AllocateId();
|
||||
const auto scc = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, mask, new_low, active_new_high});
|
||||
state->builder.AddFunction({OpINotEqual, state->bool_type, cond, mask, ConstantU32(state, 0)});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
EmitStoreU32(state, SccOperand(), scc);
|
||||
}
|
||||
|
||||
void EmitReadFirstLaneU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto active = EmitExecActiveBool(state);
|
||||
const auto ballot = state->builder.AllocateId();
|
||||
const auto first_lane = state->builder.AllocateId();
|
||||
const auto first_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), active});
|
||||
state->builder.AddFunction({OpGroupNonUniformBallotFindLSB, state->uint_type, first_lane,
|
||||
ConstantU32(state, ScopeSubgroup), ballot});
|
||||
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, first_value,
|
||||
ConstantU32(state, ScopeSubgroup), src, first_lane});
|
||||
EmitStoreU32(state, inst.dst, first_value);
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndex(EmitterState* state, const IR::Operand& operand) {
|
||||
const auto lane = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, lane, EmitValueLoad(state, operand),
|
||||
ConstantU32(state, 63)});
|
||||
return lane;
|
||||
}
|
||||
|
||||
void EmitReadLaneU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto src = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, value,
|
||||
ConstantU32(state, ScopeSubgroup), src, lane});
|
||||
EmitStoreU32(state, inst.dst, value);
|
||||
}
|
||||
|
||||
void EmitWriteLaneU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto lane = EmitLaneIndex(state, inst.src[1]);
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto hit = state->builder.AllocateId();
|
||||
const auto pointer =
|
||||
inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(*state, inst.dst.reg) : 0;
|
||||
const auto old = pointer != 0 ? state->builder.AllocateId() : ConstantU32(state, 0);
|
||||
const auto merged = state->builder.AllocateId();
|
||||
if (pointer != 0) {
|
||||
state->builder.AddFunction({OpLoad, state->uint_type, old, pointer});
|
||||
}
|
||||
state->builder.AddFunction({OpIEqual, state->bool_type, hit, subid, lane});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, merged, hit, value, old});
|
||||
if (pointer != 0) {
|
||||
// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
|
||||
state->builder.AddFunction({OpStore, pointer, merged});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitPermlaneB32(EmitterState* state, const IR::Instruction& inst, bool x16) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto row = state->builder.AllocateId();
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto lane8 = state->builder.AllocateId();
|
||||
const auto shift = state->builder.AllocateId();
|
||||
const auto upper = state->builder.AllocateId();
|
||||
const auto selected = state->builder.AllocateId();
|
||||
const auto index0 = state->builder.AllocateId();
|
||||
const auto index1 = state->builder.AllocateId();
|
||||
const auto target = state->builder.AllocateId();
|
||||
const auto shuffled = state->builder.AllocateId();
|
||||
const auto value = EmitValueLoad(state, inst.src[0]);
|
||||
const auto src1 = EmitValueLoad(state, inst.src[1]);
|
||||
const auto src2 = EmitValueLoad(state, inst.src[2]);
|
||||
uint32_t row_value = row;
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
if (x16) {
|
||||
row_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseXor, state->uint_type, row_value, row, ConstantU32(state, 16)});
|
||||
}
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane8, lane, ConstantU32(state, 7)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, shift, lane8, ConstantU32(state, 2)});
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, upper, lane, ConstantU32(state, 8)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, selected, upper, src2, src1});
|
||||
state->builder.AddFunction({OpShiftRightLogical, state->uint_type, index0, selected, shift});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, index1, index0, ConstantU32(state, 15)});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row_value, index1});
|
||||
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target});
|
||||
uint32_t ret = shuffled;
|
||||
if (!inst.dst.op_sel) {
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target);
|
||||
ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
|
||||
}
|
||||
EmitStoreU32(state, inst.dst, ret);
|
||||
}
|
||||
|
||||
void EmitControlNop(EmitterState* state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitWaitcnt(EmitterState* state, const IR::Instruction& inst) {
|
||||
(void)state;
|
||||
(void)inst;
|
||||
}
|
||||
|
||||
void EmitBarrier(EmitterState* state, const IR::Instruction& inst) {
|
||||
(void)inst;
|
||||
const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
|
||||
state->builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
|
||||
ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,314 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
|
||||
return (inst.memory.image_sample_flags & flag) != 0;
|
||||
}
|
||||
|
||||
ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) {
|
||||
ImageSampleLayout layout;
|
||||
uint32_t cursor = 0;
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
layout.offset = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
layout.bias = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) {
|
||||
layout.dref = cursor++;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
layout.grad_x = cursor;
|
||||
cursor += 2u;
|
||||
layout.grad_y = cursor;
|
||||
cursor += 2u;
|
||||
}
|
||||
layout.coord = cursor;
|
||||
cursor += ImageViewCoordinateComponents(view);
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) {
|
||||
layout.lod = cursor++;
|
||||
}
|
||||
return layout;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordF32(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, ImageViewKind view) {
|
||||
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord);
|
||||
const auto y = inst.memory.image_address_components > layout.coord + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 1u)
|
||||
: EmitZeroF32(state);
|
||||
const auto coord = state->builder.AllocateId();
|
||||
if (ImageViewCoordinateComponents(view) == 3u) {
|
||||
const auto z = inst.memory.image_address_components > layout.coord + 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.coord + 2u)
|
||||
: EmitZeroF32(state);
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
|
||||
} else {
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageLodF32(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
|
||||
layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
|
||||
}
|
||||
|
||||
uint32_t EmitImageDrefF32(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
|
||||
}
|
||||
|
||||
uint32_t EmitImageBiasF32(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
|
||||
}
|
||||
|
||||
uint32_t EmitImageGradientF32(EmitterState* state, const IR::Instruction& inst,
|
||||
uint32_t first_component) {
|
||||
const auto x = inst.memory.image_address_components > first_component
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
|
||||
: EmitZeroF32(state);
|
||||
const auto y = inst.memory.image_address_components > first_component + 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
|
||||
: EmitZeroF32(state);
|
||||
const auto grad = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, grad, x, y});
|
||||
return grad;
|
||||
}
|
||||
|
||||
uint32_t EmitImagePackedOffset2I32(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout) {
|
||||
if (layout.offset == NoImageComponent ||
|
||||
inst.memory.image_address_components <= layout.offset) {
|
||||
const auto zero = ConstantI32(state, 0);
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec2_int_type, ret, zero, zero});
|
||||
return ret;
|
||||
}
|
||||
|
||||
const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
|
||||
const auto packed_i32 = state->builder.AllocateId();
|
||||
const auto offset_x = state->builder.AllocateId();
|
||||
const auto offset_y = state->builder.AllocateId();
|
||||
const auto offset = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->int_type, packed_i32, packed_bits});
|
||||
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_x, packed_i32,
|
||||
ConstantI32(state, 0), ConstantI32(state, 6)});
|
||||
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, offset_y, packed_i32,
|
||||
ConstantI32(state, 8), ConstantI32(state, 6)});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeConstruct, state->vec2_int_type, offset, offset_x, offset_y});
|
||||
return offset;
|
||||
}
|
||||
|
||||
uint32_t EmitImageOffsetCoordF32(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, uint32_t sampled_image,
|
||||
uint32_t coord, ImageViewKind view) {
|
||||
if (!HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
return coord;
|
||||
}
|
||||
|
||||
const auto offset_i32 = EmitImagePackedOffset2I32(state, inst, layout);
|
||||
const auto offset_f32 = state->builder.AllocateId();
|
||||
const auto image = state->builder.AllocateId();
|
||||
const auto size_u32 = state->builder.AllocateId();
|
||||
const auto size_xy_u32 =
|
||||
ImageViewCoordinateComponents(view) == 3u ? state->builder.AllocateId() : size_u32;
|
||||
const auto size_f32 = state->builder.AllocateId();
|
||||
const auto normalized = state->builder.AllocateId();
|
||||
const auto adjusted = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpConvertSToF, state->vec2_float_type, offset_f32, offset_i32});
|
||||
state->builder.AddFunction(
|
||||
{OpImage, ImageViewImageType(*state, view, inst.memory.kind == IR::ResourceKind::ImageUint),
|
||||
image, sampled_image});
|
||||
state->builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(*state, view), size_u32,
|
||||
image, ConstantU32(state, 0)});
|
||||
if (ImageViewCoordinateComponents(view) == 3u) {
|
||||
const auto width = state->builder.AllocateId();
|
||||
const auto height = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size_u32, 0});
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size_u32, 1});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeConstruct, state->vec2_uint_type, size_xy_u32, width, height});
|
||||
}
|
||||
state->builder.AddFunction({OpConvertUToF, state->vec2_float_type, size_f32, size_xy_u32});
|
||||
state->builder.AddFunction({OpFDiv, state->vec2_float_type, normalized, offset_f32, size_f32});
|
||||
if (ImageViewCoordinateComponents(view) == 2u) {
|
||||
state->builder.AddFunction({OpFAdd, state->vec2_float_type, adjusted, coord, normalized});
|
||||
return adjusted;
|
||||
}
|
||||
|
||||
const auto x = state->builder.AllocateId();
|
||||
const auto y = state->builder.AllocateId();
|
||||
const auto z = state->builder.AllocateId();
|
||||
const auto normalized_x = state->builder.AllocateId();
|
||||
const auto normalized_y = state->builder.AllocateId();
|
||||
const auto adjusted_x = state->builder.AllocateId();
|
||||
const auto adjusted_y = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->float_type, x, coord, 0});
|
||||
state->builder.AddFunction({OpCompositeExtract, state->float_type, y, coord, 1});
|
||||
state->builder.AddFunction({OpCompositeExtract, state->float_type, z, coord, 2});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeExtract, state->float_type, normalized_x, normalized, 0});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeExtract, state->float_type, normalized_y, normalized, 1});
|
||||
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_x, x, normalized_x});
|
||||
state->builder.AddFunction({OpFAdd, state->float_type, adjusted_y, y, normalized_y});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeConstruct, state->vec3_float_type, adjusted, adjusted_x, adjusted_y, z});
|
||||
return adjusted;
|
||||
}
|
||||
|
||||
uint32_t EmitImageCoordU32(EmitterState* state, const IR::Instruction& inst, ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state->builder.AllocateId();
|
||||
if (ImageViewCoordinateComponents(view) == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageLoadCoordU32(EmitterState* state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
const auto x = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
|
||||
: ConstantU32(state, 0);
|
||||
const auto coord = state->builder.AllocateId();
|
||||
if (ImageViewCoordinateComponents(view) == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
|
||||
: ConstantU32(state, 0);
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec3_uint_type, coord, x, y, z});
|
||||
} else {
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec2_uint_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t EmitImageMipLodU32(EmitterState* state, const IR::Instruction& inst,
|
||||
const IR::Operand& address, ImageViewKind view) {
|
||||
if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto lod_component = ImageViewCoordinateComponents(view);
|
||||
if (inst.memory.image_address_components <= lod_component) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitImageAddressValueLoad(state, inst, address, lod_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageQueryCoordF32(EmitterState* state, const IR::Instruction& inst,
|
||||
ImageViewKind view) {
|
||||
const auto x = EmitImageAddressFloatLoad(state, inst, inst.src[0], 0);
|
||||
const auto y = inst.memory.image_address_components > 1u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 1)
|
||||
: EmitZeroF32(state);
|
||||
const auto coord = state->builder.AllocateId();
|
||||
if (ImageViewCoordinateComponents(view) == 3u) {
|
||||
const auto z = inst.memory.image_address_components > 2u
|
||||
? EmitImageAddressFloatLoad(state, inst, inst.src[0], 2)
|
||||
: EmitZeroF32(state);
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec3_float_type, coord, x, y, z});
|
||||
} else {
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec2_float_type, coord, x, y});
|
||||
}
|
||||
return coord;
|
||||
}
|
||||
|
||||
uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
|
||||
uint32_t index = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
index += (dmask >> i) & 1u;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentF32(EmitterState* state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return EmitZeroF32(state);
|
||||
}
|
||||
return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
|
||||
const auto target = 4u + component;
|
||||
for (uint32_t source = 0; source < 4u; source++) {
|
||||
if (((swizzle >> (source * 3u)) & 0x7u) == target) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return UINT32_MAX;
|
||||
}
|
||||
|
||||
uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
|
||||
if (state.program == nullptr || inst.memory.resource >= state.program->info.images.size()) {
|
||||
EXIT("storage image instruction has no specialized swizzle\n");
|
||||
}
|
||||
return state.program->info.images[inst.memory.resource].storage_swizzle;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelF32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(*state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec4_float_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreComponentU32(EmitterState* state, const IR::Instruction& inst,
|
||||
uint32_t component) {
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
if (((dmask >> component) & 1u) == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
|
||||
}
|
||||
|
||||
uint32_t EmitImageStoreTexelU32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto swizzle = StorageImageSwizzle(*state, inst);
|
||||
const auto component_value = [&](uint32_t component) {
|
||||
const auto source = InverseStorageSwizzleComponent(swizzle, component);
|
||||
return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
|
||||
: ConstantU32(state, 0);
|
||||
};
|
||||
const auto x = component_value(0);
|
||||
const auto y = component_value(1);
|
||||
const auto z = component_value(2);
|
||||
const auto w = component_value(3);
|
||||
const auto texel = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeConstruct, state->vec4_uint_type, texel, x, y, z, w});
|
||||
return texel;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,361 @@
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
namespace {
|
||||
|
||||
uint32_t ConstantVec2I32(EmitterState* state, int32_t x, int32_t y) {
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddType({OpConstantComposite, state->vec2_int_type, value, ConstantI32(state, x),
|
||||
ConstantI32(state, y)});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t ConstantImageGatherHorizontalOffsets(EmitterState* state) {
|
||||
const auto offset0 = ConstantVec2I32(state, -1, 0);
|
||||
const auto offset1 = ConstantVec2I32(state, 0, 0);
|
||||
const auto offset2 = ConstantVec2I32(state, 1, 0);
|
||||
const auto offset3 = ConstantVec2I32(state, 2, 0);
|
||||
const auto type = state->builder.AllocateId();
|
||||
state->builder.AddType({OpTypeArray, type, state->vec2_int_type, ConstantU32(state, 4)});
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t LoadStorageImageDescriptorAtIndex(EmitterState* state, uint32_t resource,
|
||||
uint32_t array_index, bool uint_image,
|
||||
ImageViewKind view) {
|
||||
uint32_t pointer_type =
|
||||
uint_image ? state->ptr_uniform_storage_image_uint : state->ptr_uniform_storage_image;
|
||||
uint32_t variable =
|
||||
uint_image ? state->storage_image_uint_variable : state->storage_image_variable;
|
||||
uint32_t image_type = uint_image ? state->storage_image_uint_type : state->storage_image_type;
|
||||
IR::DescriptorBindingKind kind = uint_image ? IR::DescriptorBindingKind::StorageUint2D
|
||||
: IR::DescriptorBindingKind::Storage2D;
|
||||
switch (view) {
|
||||
case ImageViewKind::Dim2D: break;
|
||||
case ImageViewKind::Dim2DArray:
|
||||
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_2d_array
|
||||
: state->ptr_uniform_storage_image_2d_array;
|
||||
variable = uint_image ? state->storage_image_uint_2d_array_variable
|
||||
: state->storage_image_2d_array_variable;
|
||||
image_type = uint_image ? state->storage_image_uint_2d_array_type
|
||||
: state->storage_image_2d_array_type;
|
||||
kind = uint_image ? IR::DescriptorBindingKind::StorageUint2DArray
|
||||
: IR::DescriptorBindingKind::Storage2DArray;
|
||||
break;
|
||||
case ImageViewKind::Dim3D:
|
||||
pointer_type = uint_image ? state->ptr_uniform_storage_image_uint_3d
|
||||
: state->ptr_uniform_storage_image_3d;
|
||||
variable = uint_image ? state->storage_image_uint_3d_variable
|
||||
: state->storage_image_3d_variable;
|
||||
image_type =
|
||||
uint_image ? state->storage_image_uint_3d_type : state->storage_image_3d_type;
|
||||
kind = uint_image ? IR::DescriptorBindingKind::StorageUint3D
|
||||
: IR::DescriptorBindingKind::Storage3D;
|
||||
break;
|
||||
}
|
||||
const auto pointer =
|
||||
DescriptorElementPointer(state, pointer_type, variable, array_index, kind, resource,
|
||||
"storage image descriptor array was not emitted");
|
||||
const auto image = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, image_type, image, pointer});
|
||||
return image;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
uint32_t EmitImageGetResinfoComponent(EmitterState* state, uint32_t image, uint32_t size,
|
||||
uint32_t component_index, ImageViewKind view) {
|
||||
switch (component_index) {
|
||||
case 0: {
|
||||
const auto width = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, width, size, 0});
|
||||
return width;
|
||||
}
|
||||
case 1: {
|
||||
const auto height = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, height, size, 1});
|
||||
return height;
|
||||
}
|
||||
case 2:
|
||||
if (ImageViewCoordinateComponents(view) == 3u) {
|
||||
const auto depth = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, depth, size, 2});
|
||||
return depth;
|
||||
}
|
||||
return ConstantU32(state, 1);
|
||||
case 3: {
|
||||
const auto levels = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpImageQueryLevels, state->uint_type, levels, image});
|
||||
return levels;
|
||||
}
|
||||
default: return ConstantU32(state, 0);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetResinfo(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const auto mip_level = EmitValueLoad(state, inst.src[0]);
|
||||
|
||||
const auto size = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpImageQuerySizeLod, ImageViewSizeType(*state, view), size, image, mip_level});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component =
|
||||
EmitImageGetResinfoComponent(state, image, size, component_index, view);
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGetLod(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto lod = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpImageQueryLod, state->vec2_float_type, lod, sampled_image,
|
||||
EmitImageQueryCoordF32(state, inst, view)});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 2u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state->builder.AllocateId();
|
||||
const auto bits = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeExtract, state->float_type, component, lod, component_index});
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageLoad(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
|
||||
const auto image = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
|
||||
const auto color = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpImageFetch, integer ? state->vec4_uint_type : state->vec4_float_type, color, image,
|
||||
EmitImageLoadCoordU32(state, inst, view), ImageOperandsLodMask,
|
||||
EmitImageMipLodU32(state, inst, inst.src[0], view)});
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract,
|
||||
integer ? state->uint_type : state->float_type, component,
|
||||
color, component_index});
|
||||
const auto bits = integer ? component : state->builder.AllocateId();
|
||||
if (!integer) {
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageStore(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
|
||||
const auto view = StorageImageViewKind(*state, inst.memory, uint_image, inst.pc);
|
||||
const auto binding = ResourceForDescriptor(
|
||||
*state,
|
||||
uint_image
|
||||
? (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::StorageUint2DArray
|
||||
: view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::StorageUint3D
|
||||
: IR::DescriptorBindingKind::StorageUint2D)
|
||||
: (view == ImageViewKind::Dim2DArray ? IR::DescriptorBindingKind::Storage2DArray
|
||||
: view == ImageViewKind::Dim3D ? IR::DescriptorBindingKind::Storage3D
|
||||
: IR::DescriptorBindingKind::Storage2D),
|
||||
inst.memory.resource);
|
||||
const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
|
||||
binding.array_index, uint_image, view);
|
||||
|
||||
state->builder.AddFunction(
|
||||
{OpImageWrite, image, EmitImageCoordU32(state, inst, view),
|
||||
uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
|
||||
}
|
||||
|
||||
void EmitImageSampleResult(EmitterState* state, const IR::Instruction& inst, uint32_t sample,
|
||||
bool dref, bool integer) {
|
||||
if (dref) {
|
||||
const auto bits = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, sample});
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
|
||||
uint32_t dst_index = 0;
|
||||
for (uint32_t component_index = 0; component_index < 4u; component_index++) {
|
||||
if (((dmask >> component_index) & 1u) == 0) {
|
||||
continue;
|
||||
}
|
||||
const auto component = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract,
|
||||
integer ? state->uint_type : state->float_type, component,
|
||||
sample, component_index});
|
||||
const auto bits = integer ? component : state->builder.AllocateId();
|
||||
if (!integer) {
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
|
||||
}
|
||||
}
|
||||
|
||||
bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) {
|
||||
// RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it
|
||||
// to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information.
|
||||
// SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need
|
||||
// an explicit operand even for the unsuffixed sample opcode.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) ||
|
||||
HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) {
|
||||
return true;
|
||||
}
|
||||
return state.stage != ShaderType::Pixel;
|
||||
}
|
||||
|
||||
void AddImageSampleOperands(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, bool explicit_lod,
|
||||
std::vector<uint32_t>* words) {
|
||||
if (words == nullptr) {
|
||||
return;
|
||||
}
|
||||
uint32_t mask = 0;
|
||||
std::vector<uint32_t> operands;
|
||||
|
||||
// Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod
|
||||
// operands belong to explicit-lod instructions; bias belongs to implicit-lod
|
||||
// sampling because it modifies the hardware-computed LOD.
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
|
||||
mask |= ImageOperandsGradMask;
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x));
|
||||
operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y));
|
||||
} else if (explicit_lod) {
|
||||
mask |= ImageOperandsLodMask;
|
||||
operands.push_back(EmitImageLodF32(state, inst, layout));
|
||||
} else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
|
||||
mask |= ImageOperandsBiasMask;
|
||||
operands.push_back(EmitImageBiasF32(state, inst, layout));
|
||||
}
|
||||
if (mask != 0) {
|
||||
words->push_back(mask);
|
||||
words->insert(words->end(), operands.begin(), operands.end());
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) {
|
||||
const auto flags = inst.memory.image_sample_flags;
|
||||
const bool dref = (flags & Decoder::ImageSampleFlagCompare) != 0;
|
||||
const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
|
||||
if (explicit_lod) {
|
||||
return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
|
||||
}
|
||||
return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
|
||||
}
|
||||
|
||||
void EmitImageSample(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto base_coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto sample = state->builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
const auto result_type = dref ? state->float_type
|
||||
: integer ? state->vec4_uint_type
|
||||
: state->vec4_float_type;
|
||||
const auto explicit_lod = ImageSampleNeedsExplicitLod(*state, inst);
|
||||
const auto opcode = ImageSampleOpcode(*state, inst);
|
||||
const auto coord =
|
||||
EmitImageOffsetCoordF32(state, inst, layout, sampled_image, base_coord, view);
|
||||
|
||||
std::vector<uint32_t> words = {opcode, result_type, sample, sampled_image, coord};
|
||||
if (dref) {
|
||||
words.push_back(EmitImageDrefF32(state, inst, layout));
|
||||
}
|
||||
AddImageSampleOperands(state, inst, layout, explicit_lod, &words);
|
||||
state->builder.AddFunction(words);
|
||||
EmitImageSampleResult(state, inst, sample, dref, integer);
|
||||
}
|
||||
|
||||
uint32_t ImageGatherComponent(uint32_t dmask) {
|
||||
switch (dmask) {
|
||||
case 0x2u: return 1;
|
||||
case 0x4u: return 2;
|
||||
case 0x8u: return 3;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
void AddImageGatherOperands(EmitterState* state, const IR::Instruction& inst,
|
||||
const ImageSampleLayout& layout, std::vector<uint32_t>* words) {
|
||||
if (words == nullptr) {
|
||||
return;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
|
||||
words->push_back(ImageOperandsConstOffsetsMask);
|
||||
words->push_back(ConstantImageGatherHorizontalOffsets(state));
|
||||
return;
|
||||
}
|
||||
if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
|
||||
words->push_back(ImageOperandsOffsetMask);
|
||||
words->push_back(EmitImagePackedOffset2I32(state, inst, layout));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitImageGather4(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto view = SampledImageViewKind(*state, inst.memory, inst.pc);
|
||||
const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
|
||||
|
||||
const auto layout = MakeImageSampleLayout(inst, view);
|
||||
const auto coord = EmitImageCoordF32(state, inst, layout, view);
|
||||
const auto texels = state->builder.AllocateId();
|
||||
const auto dref = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
|
||||
const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
|
||||
const auto result_type = integer ? state->vec4_uint_type : state->vec4_float_type;
|
||||
std::vector<uint32_t> words =
|
||||
dref ? std::vector<uint32_t> {OpImageDrefGather,
|
||||
state->vec4_float_type,
|
||||
texels,
|
||||
sampled_image,
|
||||
coord,
|
||||
EmitImageDrefF32(state, inst, layout)}
|
||||
: std::vector<uint32_t> {
|
||||
OpImageGather, result_type,
|
||||
texels, sampled_image,
|
||||
coord, ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
|
||||
AddImageGatherOperands(state, inst, layout, &words);
|
||||
state->builder.AddFunction(words);
|
||||
|
||||
for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
|
||||
const auto component = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract,
|
||||
integer ? state->uint_type : state->float_type, component,
|
||||
texels, i});
|
||||
const auto bits = integer ? component : state->builder.AllocateId();
|
||||
if (!integer) {
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
|
||||
}
|
||||
EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,929 @@
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
const IR::DescriptorBinding* DescriptorBinding(const EmitterState& state,
|
||||
IR::DescriptorBindingKind kind) {
|
||||
return IR::FindBinding(state.program->bindings, kind);
|
||||
}
|
||||
|
||||
uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind) {
|
||||
const auto* binding = DescriptorBinding(state, kind);
|
||||
return binding != nullptr ? static_cast<uint32_t>(binding->resources.size()) : 0;
|
||||
}
|
||||
|
||||
uint32_t ConstantU32(EmitterState* state, uint32_t value) {
|
||||
if (auto it = state->constants.find(value); it != state->constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state->builder.AllocateId();
|
||||
state->constants.emplace(value, id);
|
||||
state->builder.AddType({OpConstant, state->uint_type, id, value});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantI32(EmitterState* state, int32_t value) {
|
||||
const auto bits = static_cast<uint32_t>(value);
|
||||
if (auto it = state->signed_constants.find(bits); it != state->signed_constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state->builder.AllocateId();
|
||||
state->signed_constants.emplace(bits, id);
|
||||
state->builder.AddType({OpConstant, state->int_type, id, bits});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t ConstantF32(EmitterState* state, uint32_t bits) {
|
||||
if (auto it = state->float_constants.find(bits); it != state->float_constants.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
const auto id = state->builder.AllocateId();
|
||||
state->float_constants.emplace(bits, id);
|
||||
state->builder.AddType({OpConstant, state->float_type, id, bits});
|
||||
return id;
|
||||
}
|
||||
|
||||
uint32_t FloatBits(float value) {
|
||||
uint32_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
return bits;
|
||||
}
|
||||
|
||||
uint32_t ConstantF32Value(EmitterState* state, float value) {
|
||||
return ConstantF32(state, FloatBits(value));
|
||||
}
|
||||
|
||||
VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
|
||||
if (state.stage != ShaderType::Vertex || state.vertex_input_info == nullptr ||
|
||||
location >= ShaderVertexInputInfo::RES_MAX ||
|
||||
location >= static_cast<uint32_t>(state.vertex_input_info->resources_num)) {
|
||||
return VertexInputScalarKind::Float;
|
||||
}
|
||||
|
||||
switch (state.vertex_input_info->resources[location].Format()) {
|
||||
case 5:
|
||||
case 11:
|
||||
case 18:
|
||||
case 20:
|
||||
case 27:
|
||||
case 60:
|
||||
case 62:
|
||||
case 69:
|
||||
case 72:
|
||||
case 75: return VertexInputScalarKind::Uint;
|
||||
case 6:
|
||||
case 12:
|
||||
case 19:
|
||||
case 21:
|
||||
case 28:
|
||||
case 61:
|
||||
case 63:
|
||||
case 70:
|
||||
case 73:
|
||||
case 76: return VertexInputScalarKind::Sint;
|
||||
default: return VertexInputScalarKind::Float;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) {
|
||||
uint32_t count = input.component_count;
|
||||
if (state.stage == ShaderType::Vertex && state.vertex_input_info != nullptr &&
|
||||
input.location < ShaderVertexInputInfo::RES_MAX &&
|
||||
input.location < static_cast<uint32_t>(state.vertex_input_info->resources_num) &&
|
||||
state.vertex_input_info->resources_dst[input.location].registers_num > 0) {
|
||||
count = static_cast<uint32_t>(
|
||||
state.vertex_input_info->resources_dst[input.location].registers_num);
|
||||
}
|
||||
return std::clamp(count, 1u, 4u);
|
||||
}
|
||||
|
||||
uint32_t VertexParameterScalarType(const EmitterState& state, VertexInputScalarKind kind) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint: return state.int_type;
|
||||
case VertexInputScalarKind::Uint: return state.uint_type;
|
||||
case VertexInputScalarKind::Float:
|
||||
default: return state.float_type;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterScalarPointerType(const EmitterState& state, VertexInputScalarKind kind) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint: return state.ptr_input_int;
|
||||
case VertexInputScalarKind::Uint: return state.ptr_input_uint;
|
||||
case VertexInputScalarKind::Float:
|
||||
default: return state.ptr_input_float;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterVectorOrScalarType(const EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t components) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint:
|
||||
switch (components) {
|
||||
case 1: return state.int_type;
|
||||
case 2: return state.vec2_int_type;
|
||||
case 3: return state.vec3_int_type;
|
||||
default: return state.vec4_int_type;
|
||||
}
|
||||
case VertexInputScalarKind::Uint:
|
||||
switch (components) {
|
||||
case 1: return state.uint_type;
|
||||
case 2: return state.vec2_uint_type;
|
||||
case 3: return state.vec3_uint_type;
|
||||
default: return state.vec4_uint_type;
|
||||
}
|
||||
case VertexInputScalarKind::Float:
|
||||
default:
|
||||
switch (components) {
|
||||
case 1: return state.float_type;
|
||||
case 2: return state.vec2_float_type;
|
||||
case 3: return state.vec3_float_type;
|
||||
default: return state.vec4_float_type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputScalarKind kind,
|
||||
uint32_t components) {
|
||||
switch (kind) {
|
||||
case VertexInputScalarKind::Sint:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_int;
|
||||
case 2: return state.ptr_input_vec2_int;
|
||||
case 3: return state.ptr_input_vec3_int;
|
||||
default: return state.ptr_input_vec4_int;
|
||||
}
|
||||
case VertexInputScalarKind::Uint:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_uint;
|
||||
case 2: return state.ptr_input_vec2_uint;
|
||||
case 3: return state.ptr_input_vec3_uint;
|
||||
default: return state.ptr_input_vec4_uint;
|
||||
}
|
||||
case VertexInputScalarKind::Float:
|
||||
default:
|
||||
switch (components) {
|
||||
case 1: return state.ptr_input_float;
|
||||
case 2: return state.ptr_input_vec2_float;
|
||||
case 3: return state.ptr_input_vec3_float;
|
||||
default: return state.ptr_input_vec4_float;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AllocateInputVariables(EmitterState* state) {
|
||||
for (auto& binding: state->inputs) {
|
||||
binding.variable_id = state->builder.AllocateId();
|
||||
state->interface_variables.push_back(binding.variable_id);
|
||||
}
|
||||
if (state->needs_subgroup_local_invocation_id) {
|
||||
state->subgroup_local_invocation_id_variable = state->builder.AllocateId();
|
||||
state->interface_variables.push_back(state->subgroup_local_invocation_id_variable);
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t AllocateInterfaceVariable(EmitterState* state) {
|
||||
const auto variable = state->builder.AllocateId();
|
||||
state->interface_variables.push_back(variable);
|
||||
return variable;
|
||||
}
|
||||
|
||||
static uint32_t AllocateSharedOutputVariable(EmitterState* state, uint32_t& variable) {
|
||||
if (variable == 0) {
|
||||
variable = AllocateInterfaceVariable(state);
|
||||
}
|
||||
return variable;
|
||||
}
|
||||
|
||||
void AllocateOutputVariables(EmitterState* state) {
|
||||
for (auto& binding: state->outputs) {
|
||||
switch (binding.kind) {
|
||||
case IR::StageOutputKind::Position:
|
||||
binding.variable_id =
|
||||
AllocateSharedOutputVariable(state, state->per_vertex_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::Depth:
|
||||
binding.variable_id = AllocateSharedOutputVariable(state, state->depth_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::SampleMask:
|
||||
binding.variable_id =
|
||||
AllocateSharedOutputVariable(state, state->sample_mask_variable);
|
||||
break;
|
||||
case IR::StageOutputKind::Parameter:
|
||||
case IR::StageOutputKind::Mrt:
|
||||
binding.variable_id = AllocateInterfaceVariable(state);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t BuiltInForInput(IR::StageInputKind kind) {
|
||||
switch (kind) {
|
||||
case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex;
|
||||
case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex;
|
||||
case IR::StageInputKind::FragCoord: return BuiltInFragCoord;
|
||||
case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing;
|
||||
case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId;
|
||||
case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId;
|
||||
case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex;
|
||||
case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId;
|
||||
default: return UINT32_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
void AddInputAnnotationsAndNames(EmitterState* state) {
|
||||
if (state->subgroup_local_invocation_id_variable != 0) {
|
||||
state->builder.AddName(state->subgroup_local_invocation_id_variable,
|
||||
"gl_SubgroupInvocationID");
|
||||
state->builder.AddAnnotation({OpDecorate, state->subgroup_local_invocation_id_variable,
|
||||
DecorationBuiltIn, BuiltInSubgroupLocalInvocationId});
|
||||
if (state->stage == ShaderType::Pixel) {
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->subgroup_local_invocation_id_variable, DecorationFlat});
|
||||
}
|
||||
}
|
||||
for (const auto& input: state->inputs) {
|
||||
state->builder.AddName(input.variable_id, input.debug_name.c_str());
|
||||
if (input.kind == IR::StageInputKind::Parameter) {
|
||||
const auto flat = PixelParameterIsFlat(*state, input.location);
|
||||
if (flat) {
|
||||
state->builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
|
||||
}
|
||||
if (state->stage == ShaderType::Pixel && state->pixel_input_info != nullptr &&
|
||||
state->pixel_input_info->ps_no_perspective && !flat) {
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationNoPerspective});
|
||||
}
|
||||
const auto location = PixelParameterLocation(*state, input.location);
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationLocation, location});
|
||||
continue;
|
||||
}
|
||||
const auto builtin = BuiltInForInput(input.kind);
|
||||
if (builtin != UINT32_MAX) {
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, input.variable_id, DecorationBuiltIn, builtin});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AddOutputAnnotationsAndNames(EmitterState* state) {
|
||||
if (state->per_vertex_variable != 0) {
|
||||
state->builder.AddName(state->per_vertex_type, "gl_PerVertex");
|
||||
state->builder.AddName(state->per_vertex_variable, "outPerVertex");
|
||||
state->builder.AddAnnotation(
|
||||
{OpMemberDecorate, state->per_vertex_type, 0, DecorationBuiltIn, BuiltInPosition});
|
||||
state->builder.AddAnnotation({OpDecorate, state->per_vertex_type, DecorationBlock});
|
||||
}
|
||||
if (state->depth_variable != 0) {
|
||||
state->builder.AddName(state->depth_variable, "gl_FragDepth");
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->depth_variable, DecorationBuiltIn, BuiltInFragDepth});
|
||||
}
|
||||
if (state->sample_mask_variable != 0) {
|
||||
state->builder.AddName(state->sample_mask_variable, "gl_SampleMask");
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask});
|
||||
}
|
||||
for (const auto& binding: state->outputs) {
|
||||
if (binding.kind == IR::StageOutputKind::Parameter ||
|
||||
binding.kind == IR::StageOutputKind::Mrt) {
|
||||
state->builder.AddName(binding.variable_id, binding.debug_name.c_str());
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, binding.variable_id, DecorationLocation, binding.location});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DecorateDescriptor(EmitterState* state, uint32_t variable, const char* name, uint32_t set,
|
||||
uint32_t binding) {
|
||||
if (variable == 0) {
|
||||
return;
|
||||
}
|
||||
state->builder.AddName(variable, name);
|
||||
state->builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, set});
|
||||
state->builder.AddAnnotation({OpDecorate, variable, DecorationBinding, binding});
|
||||
}
|
||||
|
||||
void AddDescriptorAnnotationsAndNames(EmitterState* state) {
|
||||
const auto set = state->program->bindings.descriptor_set;
|
||||
auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) {
|
||||
const auto* binding = DescriptorBinding(*state, kind);
|
||||
if (binding != nullptr) {
|
||||
DecorateDescriptor(state, variable, name, set, binding->binding);
|
||||
}
|
||||
};
|
||||
if (state->storage_buffer_variable != 0) {
|
||||
Decorate(state->storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
|
||||
}
|
||||
if (state->address_memory_variable != 0) {
|
||||
Decorate(state->address_memory_variable, "address_memory",
|
||||
IR::DescriptorBindingKind::AddressMemory);
|
||||
}
|
||||
constexpr const char* SampledNames[] = {
|
||||
"sampled_2d", "sampled_2d_array", "sampled_3d",
|
||||
"sampled_uint_2d", "sampled_uint_2d_array", "sampled_uint_3d"};
|
||||
for (uint32_t i = 0; i < state->sampled_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % 3u);
|
||||
Decorate(state->sampled_images[i].variable, SampledNames[i],
|
||||
SampledBindingKind(i >= 3u, view));
|
||||
}
|
||||
if (state->storage_image_variable != 0) {
|
||||
Decorate(state->storage_image_variable, "textures2D_L",
|
||||
IR::DescriptorBindingKind::Storage2D);
|
||||
}
|
||||
if (state->storage_image_uint_variable != 0) {
|
||||
Decorate(state->storage_image_uint_variable, "textures2D_L_U",
|
||||
IR::DescriptorBindingKind::StorageUint2D);
|
||||
}
|
||||
if (state->storage_image_2d_array_variable != 0) {
|
||||
Decorate(state->storage_image_2d_array_variable, "textures2D_L_A",
|
||||
IR::DescriptorBindingKind::Storage2DArray);
|
||||
}
|
||||
if (state->storage_image_uint_2d_array_variable != 0) {
|
||||
Decorate(state->storage_image_uint_2d_array_variable, "textures2D_L_U_A",
|
||||
IR::DescriptorBindingKind::StorageUint2DArray);
|
||||
}
|
||||
if (state->storage_image_3d_variable != 0) {
|
||||
Decorate(state->storage_image_3d_variable, "textures2D_L_3D",
|
||||
IR::DescriptorBindingKind::Storage3D);
|
||||
}
|
||||
if (state->storage_image_uint_3d_variable != 0) {
|
||||
Decorate(state->storage_image_uint_3d_variable, "textures2D_L_U_3D",
|
||||
IR::DescriptorBindingKind::StorageUint3D);
|
||||
}
|
||||
if (state->sampler_variable != 0) {
|
||||
Decorate(state->sampler_variable, "samplers", IR::DescriptorBindingKind::Samplers);
|
||||
}
|
||||
if (state->gds_variable != 0) {
|
||||
Decorate(state->gds_variable, "gds", IR::DescriptorBindingKind::Gds);
|
||||
}
|
||||
if (state->flattened_srt_variable != 0) {
|
||||
Decorate(state->flattened_srt_variable, "flattened_srt",
|
||||
IR::DescriptorBindingKind::FlattenedSrt);
|
||||
}
|
||||
}
|
||||
|
||||
void AddVsharpAnnotationsAndNames(EmitterState* state) {
|
||||
const auto& bind = state->program->bindings;
|
||||
if (state->push_constant_variable != 0) {
|
||||
state->builder.AddName(state->push_constant_block_type, "BufferResource");
|
||||
state->builder.AddName(state->push_constant_variable, "vsharp");
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->push_constant_u32x4_array_type, DecorationArrayStride, 4});
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->push_constant_rows_array_type, DecorationArrayStride, 16});
|
||||
state->builder.AddAnnotation({OpMemberDecorate, state->push_constant_block_type, 0,
|
||||
DecorationOffset, bind.push_constant_offset});
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->push_constant_block_type, DecorationBlock});
|
||||
}
|
||||
if (state->vsharp_storage_variable != 0) {
|
||||
const auto* storage = DescriptorBinding(*state, IR::DescriptorBindingKind::UserData);
|
||||
if (storage != nullptr) {
|
||||
DecorateDescriptor(state, state->vsharp_storage_variable, "user_data",
|
||||
bind.descriptor_set, storage->binding);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void EmitHeaderAndTypes(EmitterState* state) {
|
||||
state->void_type = state->builder.AllocateId();
|
||||
state->bool_type = state->builder.AllocateId();
|
||||
state->uint_type = state->builder.AllocateId();
|
||||
state->uint_pair_type = state->builder.AllocateId();
|
||||
state->int_type = state->builder.AllocateId();
|
||||
state->int_pair_type = state->builder.AllocateId();
|
||||
state->float_type = state->builder.AllocateId();
|
||||
state->vec2_uint_type = state->builder.AllocateId();
|
||||
state->vec3_uint_type = state->builder.AllocateId();
|
||||
state->vec4_uint_type = state->builder.AllocateId();
|
||||
state->vec2_int_type = state->builder.AllocateId();
|
||||
state->vec3_int_type = state->builder.AllocateId();
|
||||
state->vec4_int_type = state->builder.AllocateId();
|
||||
state->vec2_float_type = state->builder.AllocateId();
|
||||
state->vec3_float_type = state->builder.AllocateId();
|
||||
state->vec4_float_type = state->builder.AllocateId();
|
||||
state->ptr_func_uint = state->builder.AllocateId();
|
||||
state->ptr_input_float = state->builder.AllocateId();
|
||||
state->ptr_input_bool = state->builder.AllocateId();
|
||||
state->ptr_input_int = state->builder.AllocateId();
|
||||
state->ptr_input_uint = state->builder.AllocateId();
|
||||
state->ptr_input_vec2_float = state->builder.AllocateId();
|
||||
state->ptr_input_vec3_float = state->builder.AllocateId();
|
||||
state->ptr_input_vec2_int = state->builder.AllocateId();
|
||||
state->ptr_input_vec3_int = state->builder.AllocateId();
|
||||
state->ptr_input_vec4_int = state->builder.AllocateId();
|
||||
state->ptr_input_vec2_uint = state->builder.AllocateId();
|
||||
state->ptr_input_vec3_uint = state->builder.AllocateId();
|
||||
state->ptr_input_vec4_uint = state->builder.AllocateId();
|
||||
state->ptr_input_vec4_float = state->builder.AllocateId();
|
||||
state->sample_mask_array_type = state->builder.AllocateId();
|
||||
state->ptr_output_int = state->builder.AllocateId();
|
||||
state->ptr_output_sample_mask_array = state->builder.AllocateId();
|
||||
state->ptr_output_float = state->builder.AllocateId();
|
||||
state->ptr_output_vec4_float = state->builder.AllocateId();
|
||||
state->per_vertex_type = state->builder.AllocateId();
|
||||
state->ptr_output_per_vertex = state->builder.AllocateId();
|
||||
state->storage_runtime_array_type = state->builder.AllocateId();
|
||||
state->storage_buffer_type = state->builder.AllocateId();
|
||||
state->ptr_storage_buffer = state->builder.AllocateId();
|
||||
state->ptr_storage_buffer_uint = state->builder.AllocateId();
|
||||
state->storage_buffer_array_type = state->builder.AllocateId();
|
||||
state->ptr_storage_buffer_array = state->builder.AllocateId();
|
||||
if (state->address_memory_variable != 0) {
|
||||
state->address_memory_array_type = state->builder.AllocateId();
|
||||
state->ptr_address_memory_array = state->builder.AllocateId();
|
||||
}
|
||||
if (state->push_constant_variable != 0) {
|
||||
state->push_constant_u32x4_array_type = state->builder.AllocateId();
|
||||
state->push_constant_rows_array_type = state->builder.AllocateId();
|
||||
state->push_constant_block_type = state->builder.AllocateId();
|
||||
state->ptr_push_constant_block = state->builder.AllocateId();
|
||||
state->ptr_push_constant_uint = state->builder.AllocateId();
|
||||
}
|
||||
state->lds_array_type = state->builder.AllocateId();
|
||||
state->ptr_workgroup_array = state->builder.AllocateId();
|
||||
state->ptr_workgroup_uint = state->builder.AllocateId();
|
||||
state->lds_variable = state->builder.AllocateId();
|
||||
for (auto& image: state->sampled_images) {
|
||||
image.image_type = state->builder.AllocateId();
|
||||
image.sampled_image_type = state->builder.AllocateId();
|
||||
image.pointer_type = state->builder.AllocateId();
|
||||
image.array_type = state->builder.AllocateId();
|
||||
image.array_pointer_type = state->builder.AllocateId();
|
||||
}
|
||||
state->sampler_type = state->builder.AllocateId();
|
||||
state->sampler_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_sampler = state->builder.AllocateId();
|
||||
state->ptr_uniform_sampler_array = state->builder.AllocateId();
|
||||
state->storage_image_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image = state->builder.AllocateId();
|
||||
state->storage_image_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_array = state->builder.AllocateId();
|
||||
state->storage_image_2d_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_2d_array = state->builder.AllocateId();
|
||||
state->storage_image_2d_array_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_2d_array_array = state->builder.AllocateId();
|
||||
state->storage_image_3d_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_3d = state->builder.AllocateId();
|
||||
state->storage_image_3d_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_3d_array = state->builder.AllocateId();
|
||||
state->storage_image_uint_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_uint = state->builder.AllocateId();
|
||||
state->storage_image_uint_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_uint_array = state->builder.AllocateId();
|
||||
state->storage_image_uint_2d_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_uint_2d_array = state->builder.AllocateId();
|
||||
state->storage_image_uint_2d_array_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_uint_2d_array_array = state->builder.AllocateId();
|
||||
state->storage_image_uint_3d_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_uint_3d = state->builder.AllocateId();
|
||||
state->storage_image_uint_3d_array_type = state->builder.AllocateId();
|
||||
state->ptr_uniform_storage_image_uint_3d_array = state->builder.AllocateId();
|
||||
state->ptr_image_uint = state->builder.AllocateId();
|
||||
state->func_type = state->builder.AllocateId();
|
||||
state->main_func = state->builder.AllocateId();
|
||||
state->entry_label = state->builder.AllocateId();
|
||||
state->glsl_std450 = state->builder.AllocateId();
|
||||
|
||||
state->builder.AddCapability({CapabilityShader});
|
||||
state->builder.AddCapability({CapabilityImageQuery});
|
||||
if (state->needs_image_gather_extended) {
|
||||
state->builder.AddCapability({CapabilityImageGatherExtended});
|
||||
}
|
||||
if (state->storage_image_variable != 0 || state->storage_image_2d_array_variable != 0 ||
|
||||
state->storage_image_3d_variable != 0) {
|
||||
state->builder.AddCapability({CapabilityStorageImageReadWithoutFormat});
|
||||
state->builder.AddCapability({CapabilityStorageImageWriteWithoutFormat});
|
||||
}
|
||||
if (state->needs_subgroup_ballot || state->needs_subgroup_shuffle ||
|
||||
state->needs_subgroup_local_invocation_id) {
|
||||
state->builder.AddCapability({CapabilityGroupNonUniform});
|
||||
}
|
||||
if (state->needs_subgroup_ballot) {
|
||||
state->builder.AddCapability({CapabilityGroupNonUniformBallot});
|
||||
}
|
||||
if (state->needs_subgroup_shuffle) {
|
||||
state->builder.AddCapability({CapabilityGroupNonUniformShuffle});
|
||||
}
|
||||
if (state->needs_compute_derivatives && state->stage == ShaderType::Compute) {
|
||||
state->builder.AddCapability({CapabilityComputeDerivativeGroupQuadsKHR});
|
||||
state->builder.AddExtension("SPV_KHR_compute_shader_derivatives");
|
||||
}
|
||||
state->builder.AddExtInstImport(state->glsl_std450, "GLSL.std.450");
|
||||
state->builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
|
||||
state->builder.AddEntryPoint(ExecutionModelForStage(state->stage), state->main_func, "main",
|
||||
state->interface_variables);
|
||||
if (state->stage == ShaderType::Compute) {
|
||||
uint32_t local_x = state->needs_compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_y = state->needs_compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_z = 1u;
|
||||
if (state->compute_input_info != nullptr) {
|
||||
const auto* cs = state->compute_input_info;
|
||||
local_x = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
|
||||
local_y = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
|
||||
local_z = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
|
||||
}
|
||||
state->builder.AddExecutionMode(
|
||||
{state->main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
|
||||
}
|
||||
if (state->stage == ShaderType::Pixel) {
|
||||
state->builder.AddExecutionMode({state->main_func, ExecutionModeOriginUpperLeft});
|
||||
if (state->depth_variable != 0) {
|
||||
state->builder.AddExecutionMode({state->main_func, ExecutionModeDepthReplacing});
|
||||
}
|
||||
if (state->pixel_input_info != nullptr && state->pixel_input_info->ps_early_z &&
|
||||
!state->pixel_input_info->ps_pixel_kill_enable &&
|
||||
!state->pixel_input_info->ps_depth_export_enable &&
|
||||
!state->pixel_input_info->ps_sample_mask_export_enable) {
|
||||
state->builder.AddExecutionMode({state->main_func, ExecutionModeEarlyFragmentTests});
|
||||
}
|
||||
}
|
||||
if (state->needs_compute_derivatives && state->stage == ShaderType::Compute) {
|
||||
state->builder.AddExecutionMode({state->main_func, ExecutionModeDerivativeGroupQuadsKHR});
|
||||
}
|
||||
state->builder.AddName(state->main_func, "main");
|
||||
if (state->stage == ShaderType::Compute || state->needs_function_lds) {
|
||||
state->builder.AddName(state->lds_variable, "lds_dwords");
|
||||
}
|
||||
AddInputAnnotationsAndNames(state);
|
||||
AddOutputAnnotationsAndNames(state);
|
||||
AddDescriptorAnnotationsAndNames(state);
|
||||
AddVsharpAnnotationsAndNames(state);
|
||||
state->builder.AddAnnotation(
|
||||
{OpDecorate, state->storage_runtime_array_type, DecorationArrayStride, 4});
|
||||
state->builder.AddAnnotation(
|
||||
{OpMemberDecorate, state->storage_buffer_type, 0, DecorationOffset, 0});
|
||||
state->builder.AddAnnotation({OpDecorate, state->storage_buffer_type, DecorationBlock});
|
||||
|
||||
state->builder.AddType({OpTypeVoid, state->void_type});
|
||||
state->builder.AddType({OpTypeBool, state->bool_type});
|
||||
state->builder.AddType({OpTypeInt, state->uint_type, 32, 0});
|
||||
state->builder.AddType(
|
||||
{OpTypeStruct, state->uint_pair_type, state->uint_type, state->uint_type});
|
||||
state->builder.AddType({OpTypeInt, state->int_type, 32, 1});
|
||||
state->builder.AddType({OpTypeStruct, state->int_pair_type, state->int_type, state->int_type});
|
||||
state->builder.AddType({OpTypeFloat, state->float_type, 32});
|
||||
state->builder.AddType({OpTypeVector, state->vec2_uint_type, state->uint_type, 2});
|
||||
state->builder.AddType({OpTypeVector, state->vec3_uint_type, state->uint_type, 3});
|
||||
state->builder.AddType({OpTypeVector, state->vec4_uint_type, state->uint_type, 4});
|
||||
state->builder.AddType({OpTypeVector, state->vec2_int_type, state->int_type, 2});
|
||||
state->builder.AddType({OpTypeVector, state->vec3_int_type, state->int_type, 3});
|
||||
state->builder.AddType({OpTypeVector, state->vec4_int_type, state->int_type, 4});
|
||||
state->builder.AddType({OpTypeVector, state->vec2_float_type, state->float_type, 2});
|
||||
state->builder.AddType({OpTypeVector, state->vec3_float_type, state->float_type, 3});
|
||||
state->builder.AddType({OpTypeVector, state->vec4_float_type, state->float_type, 4});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_func_uint, StorageClassFunction, state->uint_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_float, StorageClassInput, state->float_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_bool, StorageClassInput, state->bool_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_int, StorageClassInput, state->int_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_uint, StorageClassInput, state->uint_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec2_float, StorageClassInput, state->vec2_float_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec3_float, StorageClassInput, state->vec3_float_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec2_int, StorageClassInput, state->vec2_int_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec3_int, StorageClassInput, state->vec3_int_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec4_int, StorageClassInput, state->vec4_int_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec2_uint, StorageClassInput, state->vec2_uint_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec3_uint, StorageClassInput, state->vec3_uint_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec4_uint, StorageClassInput, state->vec4_uint_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_input_vec4_float, StorageClassInput, state->vec4_float_type});
|
||||
if (state->subgroup_local_invocation_id_variable != 0) {
|
||||
state->builder.AddType({OpVariable, state->ptr_input_uint,
|
||||
state->subgroup_local_invocation_id_variable, StorageClassInput});
|
||||
}
|
||||
for (const auto& input: state->inputs) {
|
||||
uint32_t ptr_type = state->ptr_input_uint;
|
||||
switch (input.kind) {
|
||||
case IR::StageInputKind::VertexIndex:
|
||||
case IR::StageInputKind::InstanceIndex: ptr_type = state->ptr_input_int; break;
|
||||
case IR::StageInputKind::WorkgroupId:
|
||||
case IR::StageInputKind::LocalInvocationId:
|
||||
case IR::StageInputKind::GlobalInvocationId:
|
||||
ptr_type = state->ptr_input_vec3_uint;
|
||||
break;
|
||||
case IR::StageInputKind::FragCoord: ptr_type = state->ptr_input_vec4_float; break;
|
||||
case IR::StageInputKind::FrontFacing: ptr_type = state->ptr_input_bool; break;
|
||||
case IR::StageInputKind::Parameter:
|
||||
if (state->stage == ShaderType::Vertex) {
|
||||
const auto kind = VertexParameterScalarKind(*state, input.location);
|
||||
const auto components = VertexParameterComponentCount(*state, input);
|
||||
ptr_type = VertexParameterInputPointerType(*state, kind, components);
|
||||
} else {
|
||||
ptr_type = state->ptr_input_vec4_float;
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
state->builder.AddType({OpVariable, ptr_type, input.variable_id, StorageClassInput});
|
||||
}
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_output_float, StorageClassOutput, state->float_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_output_int, StorageClassOutput, state->int_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_output_vec4_float, StorageClassOutput, state->vec4_float_type});
|
||||
if (state->per_vertex_variable != 0) {
|
||||
state->builder.AddType({OpTypeStruct, state->per_vertex_type, state->vec4_float_type});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_output_per_vertex, StorageClassOutput,
|
||||
state->per_vertex_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_output_per_vertex,
|
||||
state->per_vertex_variable, StorageClassOutput});
|
||||
}
|
||||
for (const auto& binding: state->outputs) {
|
||||
if (binding.kind == IR::StageOutputKind::Parameter ||
|
||||
binding.kind == IR::StageOutputKind::Mrt) {
|
||||
state->builder.AddType({OpVariable, state->ptr_output_vec4_float, binding.variable_id,
|
||||
StorageClassOutput});
|
||||
}
|
||||
}
|
||||
if (state->depth_variable != 0) {
|
||||
state->builder.AddType(
|
||||
{OpVariable, state->ptr_output_float, state->depth_variable, StorageClassOutput});
|
||||
}
|
||||
if (state->sample_mask_variable != 0) {
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->sample_mask_array_type, state->int_type, ConstantU32(state, 1)});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_output_sample_mask_array,
|
||||
StorageClassOutput, state->sample_mask_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_output_sample_mask_array,
|
||||
state->sample_mask_variable, StorageClassOutput});
|
||||
}
|
||||
state->builder.AddType(
|
||||
{OpTypeRuntimeArray, state->storage_runtime_array_type, state->uint_type});
|
||||
state->builder.AddType(
|
||||
{OpTypeStruct, state->storage_buffer_type, state->storage_runtime_array_type});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_storage_buffer, StorageClassStorageBuffer,
|
||||
state->storage_buffer_type});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_storage_buffer_uint,
|
||||
StorageClassStorageBuffer, state->uint_type});
|
||||
if (state->storage_buffer_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Buffers));
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->storage_buffer_array_type, state->storage_buffer_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_storage_buffer_array,
|
||||
StorageClassStorageBuffer, state->storage_buffer_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_storage_buffer_array,
|
||||
state->storage_buffer_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state->gds_variable != 0) {
|
||||
state->builder.AddType({OpVariable, state->ptr_storage_buffer, state->gds_variable,
|
||||
StorageClassStorageBuffer});
|
||||
}
|
||||
if (state->push_constant_variable != 0) {
|
||||
const auto row_width = ConstantU32(state, 4);
|
||||
const auto row_count = ConstantU32(
|
||||
state, std::max((state->program->bindings.push_constant_size + 15u) / 16u, 1u));
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->push_constant_u32x4_array_type, state->uint_type, row_width});
|
||||
state->builder.AddType({OpTypeArray, state->push_constant_rows_array_type,
|
||||
state->push_constant_u32x4_array_type, row_count});
|
||||
state->builder.AddType(
|
||||
{OpTypeStruct, state->push_constant_block_type, state->push_constant_rows_array_type});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_push_constant_block,
|
||||
StorageClassPushConstant, state->push_constant_block_type});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_push_constant_uint,
|
||||
StorageClassPushConstant, state->uint_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_push_constant_block,
|
||||
state->push_constant_variable, StorageClassPushConstant});
|
||||
}
|
||||
if (state->vsharp_storage_variable != 0) {
|
||||
state->builder.AddType({OpVariable, state->ptr_storage_buffer,
|
||||
state->vsharp_storage_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state->address_memory_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::AddressMemory));
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->address_memory_array_type, state->storage_buffer_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_address_memory_array,
|
||||
StorageClassStorageBuffer, state->address_memory_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_address_memory_array,
|
||||
state->address_memory_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state->flattened_srt_variable != 0) {
|
||||
state->builder.AddType({OpVariable, state->ptr_storage_buffer,
|
||||
state->flattened_srt_variable, StorageClassStorageBuffer});
|
||||
}
|
||||
if (state->stage == ShaderType::Compute || state->needs_function_lds) {
|
||||
const auto storage_class =
|
||||
state->needs_function_lds ? StorageClassFunction : StorageClassWorkgroup;
|
||||
const auto lds_size = ConstantU32(state, state->needs_function_lds ? 8192u : 1024u);
|
||||
state->builder.AddType({OpTypeArray, state->lds_array_type, state->uint_type, lds_size});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_workgroup_array, storage_class, state->lds_array_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_workgroup_uint, storage_class, state->uint_type});
|
||||
if (state->stage == ShaderType::Compute) {
|
||||
state->builder.AddType({OpVariable, state->ptr_workgroup_array, state->lds_variable,
|
||||
StorageClassWorkgroup});
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < state->sampled_images.size(); i++) {
|
||||
auto& image = state->sampled_images[i];
|
||||
const auto view = static_cast<ImageViewKind>(i % 3u);
|
||||
const auto component = i >= 3u ? state->uint_type : state->float_type;
|
||||
const auto dimension = view == ImageViewKind::Dim3D ? Dim3D : Dim2D;
|
||||
const auto arrayed = view == ImageViewKind::Dim2DArray ? 1u : 0u;
|
||||
state->builder.AddType({OpTypeImage, image.image_type, component, dimension, 0, arrayed, 0,
|
||||
1, ImageFormatUnknown});
|
||||
state->builder.AddType({OpTypeSampledImage, image.sampled_image_type, image.image_type});
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, image.pointer_type, StorageClassUniformConstant, image.image_type});
|
||||
if (image.variable != 0) {
|
||||
const auto kind = SampledBindingKind(i >= 3u, view);
|
||||
const auto count = ConstantU32(state, DescriptorCount(*state, kind));
|
||||
state->builder.AddType({OpTypeArray, image.array_type, image.image_type, count});
|
||||
state->builder.AddType({OpTypePointer, image.array_pointer_type,
|
||||
StorageClassUniformConstant, image.array_type});
|
||||
state->builder.AddType({OpVariable, image.array_pointer_type, image.variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
}
|
||||
state->builder.AddType({OpTypeSampler, state->sampler_type});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_sampler, StorageClassUniformConstant,
|
||||
state->sampler_type});
|
||||
if (state->sampler_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Samplers));
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->sampler_array_type, state->sampler_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_sampler_array,
|
||||
StorageClassUniformConstant, state->sampler_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_sampler_array,
|
||||
state->sampler_variable, StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType({OpTypeImage, state->storage_image_type, state->float_type, Dim2D, 0, 0,
|
||||
0, 2, ImageFormatUnknown});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image,
|
||||
StorageClassUniformConstant, state->storage_image_type});
|
||||
if (state->storage_image_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage2D));
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->storage_image_array_type, state->storage_image_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_array,
|
||||
StorageClassUniformConstant, state->storage_image_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_array,
|
||||
state->storage_image_variable, StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType({OpTypeImage, state->storage_image_2d_array_type, state->float_type,
|
||||
Dim2D, 0, 1, 0, 2, ImageFormatUnknown});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_2d_array,
|
||||
StorageClassUniformConstant, state->storage_image_2d_array_type});
|
||||
if (state->storage_image_2d_array_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage2DArray));
|
||||
state->builder.AddType({OpTypeArray, state->storage_image_2d_array_array_type,
|
||||
state->storage_image_2d_array_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_2d_array_array,
|
||||
StorageClassUniformConstant,
|
||||
state->storage_image_2d_array_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_2d_array_array,
|
||||
state->storage_image_2d_array_variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType({OpTypeImage, state->storage_image_3d_type, state->float_type, Dim3D, 0,
|
||||
0, 0, 2, ImageFormatUnknown});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_3d,
|
||||
StorageClassUniformConstant, state->storage_image_3d_type});
|
||||
if (state->storage_image_3d_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::Storage3D));
|
||||
state->builder.AddType(
|
||||
{OpTypeArray, state->storage_image_3d_array_type, state->storage_image_3d_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_3d_array,
|
||||
StorageClassUniformConstant, state->storage_image_3d_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_3d_array,
|
||||
state->storage_image_3d_variable, StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType({OpTypeImage, state->storage_image_uint_type, state->uint_type, Dim2D, 0,
|
||||
0, 0, 2, ImageFormatR32ui});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint,
|
||||
StorageClassUniformConstant, state->storage_image_uint_type});
|
||||
if (state->storage_image_uint_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint2D));
|
||||
state->builder.AddType({OpTypeArray, state->storage_image_uint_array_type,
|
||||
state->storage_image_uint_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_array,
|
||||
StorageClassUniformConstant, state->storage_image_uint_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_array,
|
||||
state->storage_image_uint_variable, StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType({OpTypeImage, state->storage_image_uint_2d_array_type, state->uint_type,
|
||||
Dim2D, 0, 1, 0, 2, ImageFormatR32ui});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_2d_array,
|
||||
StorageClassUniformConstant, state->storage_image_uint_2d_array_type});
|
||||
if (state->storage_image_uint_2d_array_variable != 0) {
|
||||
const auto count = ConstantU32(
|
||||
state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint2DArray));
|
||||
state->builder.AddType({OpTypeArray, state->storage_image_uint_2d_array_array_type,
|
||||
state->storage_image_uint_2d_array_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_2d_array_array,
|
||||
StorageClassUniformConstant,
|
||||
state->storage_image_uint_2d_array_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_2d_array_array,
|
||||
state->storage_image_uint_2d_array_variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType({OpTypeImage, state->storage_image_uint_3d_type, state->uint_type, Dim3D,
|
||||
0, 0, 0, 2, ImageFormatR32ui});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_3d,
|
||||
StorageClassUniformConstant, state->storage_image_uint_3d_type});
|
||||
if (state->storage_image_uint_3d_variable != 0) {
|
||||
const auto count =
|
||||
ConstantU32(state, DescriptorCount(*state, IR::DescriptorBindingKind::StorageUint3D));
|
||||
state->builder.AddType({OpTypeArray, state->storage_image_uint_3d_array_type,
|
||||
state->storage_image_uint_3d_type, count});
|
||||
state->builder.AddType({OpTypePointer, state->ptr_uniform_storage_image_uint_3d_array,
|
||||
StorageClassUniformConstant,
|
||||
state->storage_image_uint_3d_array_type});
|
||||
state->builder.AddType({OpVariable, state->ptr_uniform_storage_image_uint_3d_array,
|
||||
state->storage_image_uint_3d_variable,
|
||||
StorageClassUniformConstant});
|
||||
}
|
||||
state->builder.AddType(
|
||||
{OpTypePointer, state->ptr_image_uint, StorageClassImage, state->uint_type});
|
||||
state->builder.AddType({OpTypeFunction, state->func_type, state->void_type});
|
||||
}
|
||||
|
||||
void AllocateRegisterVariables(EmitterState* state) {
|
||||
if (state->stage == ShaderType::Pixel && state->needs_pixel_valid_mask) {
|
||||
state->pixel_valid_mask_variable = state->builder.AllocateId();
|
||||
state->builder.AddName(state->pixel_valid_mask_variable, "pixel_valid_mask_active");
|
||||
}
|
||||
for (auto& binding: state->registers) {
|
||||
binding.pointer_id = state->builder.AllocateId();
|
||||
const auto name = IR::RegisterToString(binding.reg);
|
||||
state->builder.AddName(binding.pointer_id, name.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
void AllocateDescriptorVariables(EmitterState* state) {
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Buffers) != nullptr) {
|
||||
state->storage_buffer_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::AddressMemory) != nullptr) {
|
||||
state->address_memory_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (state->program->bindings.push_constant_size != 0) {
|
||||
state->push_constant_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::UserData) != nullptr) {
|
||||
state->vsharp_storage_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
|
||||
state->flattened_srt_variable = state->builder.AllocateId();
|
||||
}
|
||||
for (uint32_t i = 0; i < state->sampled_images.size(); i++) {
|
||||
const auto view = static_cast<ImageViewKind>(i % 3u);
|
||||
if (DescriptorBinding(*state, SampledBindingKind(i >= 3u, view)) != nullptr) {
|
||||
state->sampled_images[i].variable = state->builder.AllocateId();
|
||||
}
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage2D) != nullptr) {
|
||||
state->storage_image_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint2D) != nullptr) {
|
||||
state->storage_image_uint_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage2DArray) != nullptr) {
|
||||
state->storage_image_2d_array_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint2DArray) != nullptr) {
|
||||
state->storage_image_uint_2d_array_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Storage3D) != nullptr) {
|
||||
state->storage_image_3d_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::StorageUint3D) != nullptr) {
|
||||
state->storage_image_uint_3d_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Samplers) != nullptr) {
|
||||
state->sampler_variable = state->builder.AllocateId();
|
||||
}
|
||||
if (DescriptorBinding(*state, IR::DescriptorBindingKind::Gds) != nullptr) {
|
||||
state->gds_variable = state->builder.AllocateId();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
@@ -0,0 +1,840 @@
|
||||
#include "graphics/shader/recompiler/spirvEmitter/spirvEmitterInternal.h"
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
|
||||
bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
|
||||
if (sel <= 3u) {
|
||||
offset = sel * 8u;
|
||||
width = 8u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 4u || sel == 5u) {
|
||||
offset = sel == 5u ? 16u : 0u;
|
||||
width = 16u;
|
||||
return true;
|
||||
}
|
||||
if (sel == 6u) {
|
||||
offset = 0;
|
||||
width = 32u;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t EmitSdwaExtractU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
|
||||
if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t offset = 0;
|
||||
uint32_t width = 0;
|
||||
if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (operand.sdwa_sext) {
|
||||
const auto signed_value = state->builder.AllocateId();
|
||||
const auto extracted = state->builder.AllocateId();
|
||||
const auto bits = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->int_type, signed_value, value});
|
||||
state->builder.AddFunction({OpBitFieldSExtract, state->int_type, extracted, signed_value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, extracted});
|
||||
return bits;
|
||||
}
|
||||
|
||||
const auto extracted = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitFieldUExtract, state->uint_type, extracted, value,
|
||||
ConstantU32(state, offset), ConstantU32(state, width)});
|
||||
return extracted;
|
||||
}
|
||||
|
||||
uint32_t EmitTrueBool(EmitterState* state) {
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppQuadPermTargetLane(EmitterState* state, uint32_t subid, uint32_t control) {
|
||||
const auto quad_base = state->builder.AllocateId();
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto shift = state->builder.AllocateId();
|
||||
const auto selected0 = state->builder.AllocateId();
|
||||
const auto selected = state->builder.AllocateId();
|
||||
const auto target = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 3)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, shift, lane, ConstantU32(state, 1)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftRightLogical, state->uint_type, selected0, ConstantU32(state, control), shift});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, selected, selected0, ConstantU32(state, 3)});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, quad_base, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowShiftTargetLane(EmitterState* state, uint32_t subid, uint32_t amount,
|
||||
bool left) {
|
||||
const auto row = state->builder.AllocateId();
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto lane_shifted = state->builder.AllocateId();
|
||||
const auto target = state->builder.AllocateId();
|
||||
const auto valid = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
if (left) {
|
||||
state->builder.AddFunction(
|
||||
{OpIAdd, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state->builder.AddFunction(
|
||||
{OpULessThan, state->bool_type, valid, lane, ConstantU32(state, 16u - amount)});
|
||||
} else {
|
||||
state->builder.AddFunction(
|
||||
{OpISub, state->uint_type, lane_shifted, lane, ConstantU32(state, amount)});
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, valid, lane, ConstantU32(state, amount)});
|
||||
}
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, lane_shifted});
|
||||
return {target, valid};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState* state, uint32_t subid,
|
||||
uint32_t amount) {
|
||||
const auto row = state->builder.AllocateId();
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto in_high = state->builder.AllocateId();
|
||||
const auto minus = state->builder.AllocateId();
|
||||
const auto plus = state->builder.AllocateId();
|
||||
const auto selected = state->builder.AllocateId();
|
||||
const auto target = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 15)});
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, amount)});
|
||||
state->builder.AddFunction({OpISub, state->uint_type, minus, lane, ConstantU32(state, amount)});
|
||||
state->builder.AddFunction(
|
||||
{OpIAdd, state->uint_type, plus, lane, ConstantU32(state, 16u - amount)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, minus, plus});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, row, selected});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppMirrorTargetLane(EmitterState* state, uint32_t subid, bool half_row) {
|
||||
const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
|
||||
const auto lane_mask = half_row ? 7u : 15u;
|
||||
const auto base = state->builder.AllocateId();
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto mirrored = state->builder.AllocateId();
|
||||
const auto target = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, base, subid, ConstantU32(state, base_mask)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, lane_mask)});
|
||||
state->builder.AddFunction(
|
||||
{OpISub, state->uint_type, mirrored, ConstantU32(state, lane_mask), lane});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, target, base, mirrored});
|
||||
return {target, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
DppTargetLane EmitDppTargetLane(EmitterState* state, uint32_t control) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
if (control <= 0xffu) {
|
||||
return EmitDppQuadPermTargetLane(state, subid, control);
|
||||
}
|
||||
if (control >= 0x101u && control <= 0x10fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
|
||||
}
|
||||
if (control >= 0x111u && control <= 0x11fu) {
|
||||
return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
|
||||
}
|
||||
if (control >= 0x121u && control <= 0x12fu) {
|
||||
return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
|
||||
}
|
||||
if (control == 0x140u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, false);
|
||||
}
|
||||
if (control == 0x141u) {
|
||||
return EmitDppMirrorTargetLane(state, subid, true);
|
||||
}
|
||||
return {subid, EmitTrueBool(state)};
|
||||
}
|
||||
|
||||
uint32_t EmitDppValueU32(EmitterState* state, const IR::Operand& operand, uint32_t value) {
|
||||
if (!operand.dpp) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const auto target = EmitDppTargetLane(state, operand.dpp_ctrl);
|
||||
const auto shuffled = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpGroupNonUniformShuffle, state->uint_type, shuffled,
|
||||
ConstantU32(state, ScopeSubgroup), value, target.lane});
|
||||
if (operand.dpp_fetch_inactive) {
|
||||
return shuffled;
|
||||
}
|
||||
|
||||
const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
|
||||
const auto can_fetch = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpLogicalAnd, state->bool_type, can_fetch, target.valid, source_active});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitValueLoad(EmitterState* state, const IR::Operand& operand) {
|
||||
uint32_t value = 0;
|
||||
switch (operand.kind) {
|
||||
case IR::OperandKind::ImmediateU32:
|
||||
case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
|
||||
case IR::OperandKind::Register: {
|
||||
const auto pointer = PointerForRegister(*state, operand.reg);
|
||||
if (pointer == 0) {
|
||||
value = ConstantU32(state, 0);
|
||||
break;
|
||||
}
|
||||
value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
|
||||
break;
|
||||
}
|
||||
case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
|
||||
default: value = ConstantU32(state, 0); break;
|
||||
}
|
||||
value = EmitDppValueU32(state, operand, value);
|
||||
return EmitSdwaExtractU32(state, operand, value);
|
||||
}
|
||||
|
||||
uint32_t EmitRegisterLoad(EmitterState* state, IR::Register reg) {
|
||||
if (IsInactiveWave32ExecHigh(*state, reg)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = PointerForRegister(*state, reg);
|
||||
if (pointer == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t WaveMaskHighLoad(EmitterState* state, IR::RegisterFile file) {
|
||||
if (state->wave_size == 32u && IsMaskRegisterFile(file)) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return EmitRegisterLoad(state, {file, 1});
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskPartU32(EmitterState* state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto bit = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
|
||||
if (part == 0) {
|
||||
const auto in_low = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpULessThan, state->bool_type, in_low, subid, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, ret, in_low, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
if (state->wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, ret, in_high, bit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitThreadMaskBelowPartU32(EmitterState* state, uint32_t part) {
|
||||
const auto subid = EmitSubgroupLocalInvocationId(state);
|
||||
const auto lane = state->builder.AllocateId();
|
||||
const auto bit = state->builder.AllocateId();
|
||||
const auto below = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane, subid, ConstantU32(state, 31)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane});
|
||||
state->builder.AddFunction({OpISub, state->uint_type, below, bit, ConstantU32(state, 1)});
|
||||
|
||||
if (part == 0) {
|
||||
const auto in_high = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
|
||||
return ret;
|
||||
}
|
||||
if (state->wave_size == 32u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto in_high = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, in_high, subid, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction(
|
||||
{OpSelect, state->uint_type, ret, in_high, below, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskActiveBool(EmitterState* state, IR::RegisterFile file) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto condition = state->builder.AllocateId();
|
||||
if (state->per_invocation_masks) {
|
||||
const auto combined = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseOr, state->uint_type, combined, low, WaveMaskHighLoad(state, file)});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, condition, combined, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
if (state->exact_subgroup_operations) {
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto low_hit = state->builder.AllocateId();
|
||||
const auto high_hit = state->builder.AllocateId();
|
||||
const auto active = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, active, low_hit, high_hit});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, condition, active, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, condition, low, ConstantU32(state, 0)});
|
||||
return condition;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto cond = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLogicalNot, state->bool_type, cond, active});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitMaskSummaryZeroBool(EmitterState* state, IR::RegisterFile file, bool zero) {
|
||||
if (state->per_invocation_masks) {
|
||||
return EmitMaskZeroBool(state, file, zero);
|
||||
}
|
||||
if (state->exact_subgroup_operations) {
|
||||
const auto low = EmitRegisterLoad(state, {file, 0});
|
||||
const auto high = WaveMaskHighLoad(state, file);
|
||||
const auto combined = state->builder.AllocateId();
|
||||
const auto active = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
const auto active = EmitMaskActiveBool(state, file);
|
||||
if (!zero) {
|
||||
return active;
|
||||
}
|
||||
const auto inactive = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLogicalNot, state->bool_type, inactive, active});
|
||||
return inactive;
|
||||
}
|
||||
|
||||
uint32_t EmitExecActiveBool(EmitterState* state) {
|
||||
return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
|
||||
}
|
||||
|
||||
uint32_t EmitConditionBool(EmitterState* state, const IR::Operand& operand) {
|
||||
if (operand.kind == IR::OperandKind::Register) {
|
||||
if (operand.reg.file == IR::RegisterFile::Exec ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc) {
|
||||
return EmitMaskActiveBool(state, operand.reg.file);
|
||||
}
|
||||
}
|
||||
const auto cond_value = EmitValueLoad(state, operand);
|
||||
const auto cond = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, cond, cond_value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLocalInvocationId(EmitterState* state) {
|
||||
if (state->subgroup_local_invocation_id_variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpLoad, state->uint_type, value, state->subgroup_local_invocation_id_variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == kind) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return input.variable_id;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
|
||||
for (const auto& input: state.inputs) {
|
||||
if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
|
||||
return &input;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint32_t EmitInputComponentU32(EmitterState* state, IR::StageInputKind kind, uint32_t component) {
|
||||
const auto variable = InputVariableForKind(*state, kind);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto pointer = state->builder.AllocateId();
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpAccessChain, state->ptr_input_uint, pointer, variable, ConstantU32(state, component)});
|
||||
state->builder.AddFunction({OpLoad, state->uint_type, value, pointer});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t EmitLocalInvocationIndex(EmitterState* state) {
|
||||
const auto variable = InputVariableForKind(*state, IR::StageInputKind::LocalInvocationIndex);
|
||||
if (variable == 0) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
const auto value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, state->uint_type, value, variable});
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t VertexInputDefaultComponentU32(EmitterState* state, VertexInputScalarKind kind,
|
||||
uint32_t component) {
|
||||
if ((component & 3u) != 3u) {
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
|
||||
}
|
||||
|
||||
uint32_t EmitVertexParameterComponentU32(EmitterState* state, const InputBinding& input,
|
||||
uint32_t component) {
|
||||
const auto count = VertexParameterComponentCount(*state, input);
|
||||
const auto kind = VertexParameterScalarKind(*state, input.location);
|
||||
if (component >= count) {
|
||||
return VertexInputDefaultComponentU32(state, kind, component);
|
||||
}
|
||||
|
||||
const auto scalar_type = VertexParameterScalarType(*state, kind);
|
||||
uint32_t raw = state->builder.AllocateId();
|
||||
if (count == 1u) {
|
||||
state->builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
|
||||
} else {
|
||||
const auto pointer_type = VertexParameterScalarPointerType(*state, kind);
|
||||
const auto pointer = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
|
||||
ConstantU32(state, component)});
|
||||
state->builder.AddFunction({OpLoad, scalar_type, raw, pointer});
|
||||
}
|
||||
|
||||
if (kind == VertexInputScalarKind::Uint) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
const auto bits = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, raw});
|
||||
return bits;
|
||||
}
|
||||
|
||||
void EmitLoadInputF32(EmitterState* state, const IR::Instruction& inst) {
|
||||
const auto* input = InputBindingForParameter(*state, inst.input_info.attr);
|
||||
if (input == nullptr || input->variable_id == 0) {
|
||||
EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state->stage == ShaderType::Vertex) {
|
||||
EmitStoreU32(state, inst.dst,
|
||||
EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
|
||||
return;
|
||||
}
|
||||
|
||||
const auto input_value = state->builder.AllocateId();
|
||||
const auto component = state->builder.AllocateId();
|
||||
const auto bits = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpLoad, state->vec4_float_type, input_value, input->variable_id});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeExtract, state->float_type, component, input_value, inst.input_info.chan & 3u});
|
||||
state->builder.AddFunction({OpBitcast, state->uint_type, bits, component});
|
||||
EmitStoreU32(state, inst.dst, bits);
|
||||
}
|
||||
|
||||
uint32_t EmitSccBool(EmitterState* state, bool non_zero) {
|
||||
const auto value = EmitRegisterLoad(state, SccRegister());
|
||||
const auto cond = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{non_zero ? OpINotEqual : OpIEqual, state->bool_type, cond, value, ConstantU32(state, 0)});
|
||||
return cond;
|
||||
}
|
||||
|
||||
uint32_t EmitFloatLoad(EmitterState* state, const IR::Operand& operand) {
|
||||
const auto uint_value = EmitValueLoad(state, operand);
|
||||
auto float_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->float_type, float_value, uint_value});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, float_value});
|
||||
float_value = abs_value;
|
||||
}
|
||||
if (operand.negate) {
|
||||
const auto neg_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, float_value});
|
||||
float_value = neg_value;
|
||||
}
|
||||
return float_value;
|
||||
}
|
||||
|
||||
uint32_t ApplyResultModifiersF32(EmitterState* state, uint32_t value, const IR::Operand& dst) {
|
||||
uint32_t result = value;
|
||||
if (dst.omod != 0) {
|
||||
const uint32_t bits = dst.omod == 1u ? 0x40000000u
|
||||
: dst.omod == 2u ? 0x40800000u
|
||||
: 0x3f000000u;
|
||||
const auto scaled = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpFMul, state->float_type, scaled, result, ConstantF32(state, bits)});
|
||||
result = scaled;
|
||||
}
|
||||
if (!dst.clamp) {
|
||||
return result;
|
||||
}
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpExtInst, state->float_type, ret, state->glsl_std450, GlslFClamp,
|
||||
result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitMixF32Load(EmitterState* state, const IR::Operand& operand) {
|
||||
auto value_operand = operand;
|
||||
value_operand.negate_hi = false;
|
||||
|
||||
if (operand.op_sel_hi) {
|
||||
auto raw = operand;
|
||||
raw.sdwa_sel = 6;
|
||||
raw.sdwa_sext = false;
|
||||
raw.op_sel = false;
|
||||
raw.op_sel_hi = false;
|
||||
raw.negate = false;
|
||||
raw.negate_hi = false;
|
||||
raw.absolute = false;
|
||||
raw.dpp = false;
|
||||
raw.dpp_fetch_inactive = false;
|
||||
raw.dpp_bound_ctrl = false;
|
||||
|
||||
const auto bits = EmitValueLoad(state, raw);
|
||||
const auto unpacked = state->builder.AllocateId();
|
||||
auto f32 = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpExtInst, state->vec2_float_type, unpacked, state->glsl_std450,
|
||||
GlslUnpackHalf2x16, bits});
|
||||
state->builder.AddFunction(
|
||||
{OpCompositeExtract, state->float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
|
||||
if (operand.absolute) {
|
||||
const auto abs_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpExtInst, state->float_type, abs_value, state->glsl_std450, GlslFAbs, f32});
|
||||
f32 = abs_value;
|
||||
}
|
||||
if (value_operand.negate) {
|
||||
const auto neg_value = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpFNegate, state->float_type, neg_value, f32});
|
||||
f32 = neg_value;
|
||||
}
|
||||
return f32;
|
||||
}
|
||||
|
||||
value_operand.op_sel = false;
|
||||
value_operand.op_sel_hi = false;
|
||||
return EmitFloatLoad(state, value_operand);
|
||||
}
|
||||
|
||||
IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind != IR::OperandKind::Register) {
|
||||
return operand;
|
||||
}
|
||||
auto ret = operand;
|
||||
ret.reg.index += offset;
|
||||
ret.sdwa_sel = 6;
|
||||
ret.omod = 0;
|
||||
ret.sdwa_sext = false;
|
||||
ret.op_sel = false;
|
||||
ret.op_sel_hi = false;
|
||||
ret.negate = false;
|
||||
ret.negate_hi = false;
|
||||
ret.absolute = false;
|
||||
ret.clamp = false;
|
||||
ret.dpp_ctrl = 0;
|
||||
ret.dpp_row_mask = 0xf;
|
||||
ret.dpp_bank_mask = 0xf;
|
||||
ret.dpp_fetch_inactive = false;
|
||||
ret.dpp_bound_ctrl = false;
|
||||
ret.dpp = false;
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneMaskOperandActiveBool(EmitterState* state, const IR::Operand& operand) {
|
||||
const auto low = EmitValueLoad(state, operand);
|
||||
const auto active = state->builder.AllocateId();
|
||||
if (state->per_invocation_masks || state->exact_subgroup_operations) {
|
||||
uint32_t high = ConstantU32(state, 0);
|
||||
if (operand.kind == IR::OperandKind::Register &&
|
||||
(operand.reg.file == IR::RegisterFile::Scalar ||
|
||||
operand.reg.file == IR::RegisterFile::Vcc ||
|
||||
operand.reg.file == IR::RegisterFile::Exec)) {
|
||||
high = IsMaskRegisterFile(operand.reg.file)
|
||||
? WaveMaskHighLoad(state, operand.reg.file)
|
||||
: EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
|
||||
}
|
||||
if (state->per_invocation_masks) {
|
||||
const auto combined = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, combined, low, high});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, active, combined, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
const auto low_hit = state->builder.AllocateId();
|
||||
const auto high_hit = state->builder.AllocateId();
|
||||
const auto hit = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
|
||||
state->builder.AddFunction({OpBitwiseOr, state->uint_type, hit, low_hit, high_hit});
|
||||
state->builder.AddFunction(
|
||||
{OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
state->builder.AddFunction({OpINotEqual, state->bool_type, active, low, ConstantU32(state, 0)});
|
||||
return active;
|
||||
}
|
||||
|
||||
uint32_t EmitLaneIndexActiveBool(EmitterState* state, uint32_t lane) {
|
||||
if (state->per_invocation_masks) {
|
||||
const auto ballot = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, ballot, lane);
|
||||
}
|
||||
const auto lane_low = state->builder.AllocateId();
|
||||
const auto bit = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
|
||||
if (state->wave_size == 64u) {
|
||||
const auto high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
|
||||
const auto in_high = state->builder.AllocateId();
|
||||
const auto selected = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, mask});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto hit = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
|
||||
state->builder.AddFunction({OpINotEqual, state->bool_type, ret, hit, ConstantU32(state, 0)});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSubgroupLaneActiveBool(EmitterState* state, uint32_t lane) {
|
||||
const auto active_ballot = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpGroupNonUniformBallot, state->vec4_uint_type, active_ballot,
|
||||
ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
|
||||
return EmitBallotLaneActiveBool(state, active_ballot, lane);
|
||||
}
|
||||
uint32_t EmitBallotLaneActiveBool(EmitterState* state, uint32_t active_ballot, uint32_t lane) {
|
||||
const auto low = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, low, active_ballot, 0});
|
||||
uint32_t mask = low;
|
||||
if (state->wave_size == 64u) {
|
||||
const auto high = state->builder.AllocateId();
|
||||
const auto in_high = state->builder.AllocateId();
|
||||
const auto selected = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpCompositeExtract, state->uint_type, high, active_ballot, 1});
|
||||
state->builder.AddFunction(
|
||||
{OpUGreaterThanEqual, state->bool_type, in_high, lane, ConstantU32(state, 32)});
|
||||
state->builder.AddFunction({OpSelect, state->uint_type, selected, in_high, high, low});
|
||||
mask = selected;
|
||||
}
|
||||
|
||||
const auto lane_low = state->builder.AllocateId();
|
||||
const auto bit = state->builder.AllocateId();
|
||||
state->builder.AddFunction(
|
||||
{OpBitwiseAnd, state->uint_type, lane_low, lane, ConstantU32(state, 31)});
|
||||
state->builder.AddFunction(
|
||||
{OpShiftLeftLogical, state->uint_type, bit, ConstantU32(state, 1), lane_low});
|
||||
|
||||
const auto hit = state->builder.AllocateId();
|
||||
const auto active = state->builder.AllocateId();
|
||||
const auto in_range = state->builder.AllocateId();
|
||||
const auto ret = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitwiseAnd, state->uint_type, hit, mask, bit});
|
||||
state->builder.AddFunction({OpINotEqual, state->bool_type, active, hit, ConstantU32(state, 0)});
|
||||
state->builder.AddFunction(
|
||||
{OpULessThan, state->bool_type, in_range, lane, ConstantU32(state, state->wave_size)});
|
||||
state->builder.AddFunction({OpLogicalAnd, state->bool_type, ret, active, in_range});
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialValueLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
|
||||
if (offset == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
|
||||
}
|
||||
if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) {
|
||||
return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u);
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitSequentialFloatLoad(EmitterState* state, const IR::Operand& operand, uint32_t offset) {
|
||||
const auto value = EmitSequentialValueLoad(state, operand, offset);
|
||||
const auto f32 = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
bool ImageAddressUsesA16(const IR::Instruction& inst) {
|
||||
return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u;
|
||||
}
|
||||
|
||||
bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t cursor = 0;
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) {
|
||||
if (component == cursor) {
|
||||
return false;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) {
|
||||
if (component == cursor) {
|
||||
return true;
|
||||
}
|
||||
cursor++;
|
||||
}
|
||||
if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u &&
|
||||
component == cursor) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
if (!ImageAddressUsesA16(inst)) {
|
||||
return component * 2u;
|
||||
}
|
||||
|
||||
uint32_t half_component = 0;
|
||||
for (uint32_t i = 0; i < component; i++) {
|
||||
const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u;
|
||||
if (half_width == 2u && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
half_component += half_width;
|
||||
}
|
||||
if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) {
|
||||
half_component++;
|
||||
}
|
||||
return half_component;
|
||||
}
|
||||
|
||||
uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) {
|
||||
return ImageAddressHalfComponent(inst, component) / 2u;
|
||||
}
|
||||
|
||||
IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
|
||||
uint32_t component) {
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) {
|
||||
const auto nsa_index = packed_component - 1u;
|
||||
const auto nsa_count = std::min<uint32_t>(inst.memory.image_nsa_dwords * 4u,
|
||||
Decoder::MaxImageNsaAddressComponents);
|
||||
if (nsa_index < nsa_count) {
|
||||
return MakeRegisterOperand(IR::RegisterFile::Vector,
|
||||
inst.memory.image_nsa_addr[nsa_index]);
|
||||
}
|
||||
}
|
||||
return OffsetRegisterOperand(base, packed_component);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressValueLoad(EmitterState* state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto operand = ImageAddressOperand(inst, base, component);
|
||||
const auto packed_component = ImageAddressPackedComponent(inst, component);
|
||||
if (packed_component == 0 || operand.kind == IR::OperandKind::Register) {
|
||||
auto value = EmitValueLoad(state, operand);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) {
|
||||
value = EmitShiftRightConstant(state, value, 16u);
|
||||
}
|
||||
value = EmitAndConstant(state, value, 0xffffu);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
return ConstantU32(state, 0);
|
||||
}
|
||||
|
||||
uint32_t EmitImageAddressFloatLoad(EmitterState* state, const IR::Instruction& inst,
|
||||
const IR::Operand& base, uint32_t component) {
|
||||
const auto value = EmitImageAddressValueLoad(state, inst, base, component);
|
||||
if (ImageAddressComponentUsesA16(inst, component)) {
|
||||
return EmitF16BitsToF32(state, value);
|
||||
}
|
||||
const auto f32 = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->float_type, f32, value});
|
||||
return f32;
|
||||
}
|
||||
|
||||
uint32_t EmitZeroF32(EmitterState* state) {
|
||||
const auto f32 = state->builder.AllocateId();
|
||||
state->builder.AddFunction({OpBitcast, state->float_type, f32, ConstantU32(state, 0)});
|
||||
return f32;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
|
||||
Reference in new issue
Block a user