#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h" #include "common/assert.h" #include "common/debugCounters.h" #include "common/emulatorConfig.h" #include "common/file.h" #include "common/logging/log.h" #include "common/profiler.h" #include "common/threads.h" #include "graphics/guest_gpu/gpu_format.h" #include "graphics/guest_gpu/hardwareContext.h" #include "graphics/host_gpu/renderer/debug.h" #include "graphics/host_gpu/renderer/image/imageView.h" #include "graphics/host_gpu/renderer/image/textureCommon.h" #include "graphics/host_gpu/renderer/pipeline/blendMapping.h" #include "graphics/host_gpu/renderer/render.h" #include "graphics/host_gpu/renderer/renderContext.h" #include "graphics/shader/recompiler/BufferFormat.h" #include "graphics/shader/recompiler/ShaderRecompiler.h" #include "graphics/shader/recompiler/Tessellation.h" #include "graphics/shader/recompiler/ir/Serialize.h" #include "graphics/shader/shaderCompiler.h" #include "kernel/memory.h" #include "loader/systemContent.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace Libs::Graphics { namespace { constexpr auto DriverCacheSaveInterval = std::chrono::minutes {5}; uint8_t RemapSourceAlphaFactor(uint8_t factor) { switch (static_cast(factor)) { case Prospero::BlendFactor::kSrcAlpha: return static_cast(Prospero::BlendFactor::kSrc1Alpha); case Prospero::BlendFactor::kOneMinusSrcAlpha: return static_cast(Prospero::BlendFactor::kOneMinusSrc1Alpha); default: return factor; } } vk::PolygonMode ResolvePolygonMode(const HW::ModeControl& mode, bool cull_front, bool cull_back) { // CxPrimitiveSetup::PolygonMode disables both per-face modes when it is zero. if (mode.poly_mode == 0) { return vk::PolygonMode::eFill; } EXIT_NOT_IMPLEMENTED(mode.poly_mode != 1); if (cull_front && cull_back) { return vk::PolygonMode::eFill; } if (!cull_front && !cull_back && mode.polymode_front_ptype != mode.polymode_back_ptype) { EXIT("Pipeline: different polygon modes for two visible faces are unsupported\n"); } // Vulkan has one polygon mode. A culled face does not constrain that mode. const auto polygon_mode = cull_front ? mode.polymode_back_ptype : mode.polymode_front_ptype; switch (polygon_mode) { case 0: return vk::PolygonMode::ePoint; case 1: return vk::PolygonMode::eLine; case 2: return vk::PolygonMode::eFill; default: EXIT("Pipeline: invalid polygon mode %u\n", polygon_mode); } } std::string DriverCacheSignature(const vk::PhysicalDeviceProperties& properties) { constexpr char hex[] = "0123456789abcdef"; std::string uuid(VK_UUID_SIZE * 2, '0'); for (size_t i = 0; i < VK_UUID_SIZE; i++) { uuid[i * 2] = hex[properties.pipelineCacheUUID[i] >> 4u]; uuid[i * 2 + 1] = hex[properties.pipelineCacheUUID[i] & 0xfu]; } // Driver cache entries already distinguish shader code and pipeline state across builds. return fmt::format("KytyPC2:{:08x}:{:08x}:{:08x}:{}\n", properties.vendorID, properties.deviceID, properties.driverVersion, uuid); } std::string PipelineCacheTitleId() { std::string title_id; if ((!Loader::SystemContentParamSfoGetString("TITLE_ID", &title_id) || title_id.empty()) && (!Loader::SystemContentParamSfoGetString("CONTENT_ID", &title_id) || title_id.empty())) { return {}; } if (!std::ranges::all_of(title_id, [](unsigned char c) { return std::isalnum(c) != 0 || c == '-' || c == '_'; })) { return {}; } return title_id; } template void PipelineCacheLog(fmt::format_string format, Args&&... args) { auto message = fmt::format(format, std::forward(args)...); message += '\n'; Log::WriteToConsoleAndLog(message); } bool ReadShaderGuestMemory(void*, uint64_t address, std::span values) { // Scalar and unformatted buffer dependencies use the same backing as native raw loads. // Image synchronization belongs to formatted buffer bindings, not these reads. return !values.empty() && Libs::LibKernel::Memory::TryReadBufferBacking(address, values.data(), values.size_bytes()); } void DumpShader(std::span words, const std::string& name) { if (!Config::GraphicsDebugDumpEnabled()) { return; } const auto path = Config::GetShaderLogFolder() / name; Common::File::CreateDirectories(path.parent_path()); Common::File file(path); if (file.IsInvalid()) { const auto path_text = Common::PathToString(path); LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str()); return; } file.Write(words.data(), words.size_bytes()); } void ValidateShaderSpirv(const char* stage, uint64_t shader_hash, const std::vector& spirv, bool maintenance8_enabled) { if (!Config::ShaderValidationEnabled()) { return; } spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_3); std::string messages; tools.SetMessageConsumer([&messages](spv_message_level_t, const char*, const spv_position_t& position, const char* message) { messages += fmt::format("{}: {} ({}) {}\n", static_cast(position.line), static_cast(position.column), static_cast(position.index), message); }); spvtools::ValidatorOptions validator_options; validator_options.SetAllowOffsetTextureOperand(maintenance8_enabled); if (tools.Validate(spirv.data(), spirv.size(), validator_options)) { return; } spvtools::SpirvTools disassembler(SPV_ENV_VULKAN_1_2); std::string text; disassembler.Disassemble(spirv, &text, static_cast(SPV_BINARY_TO_TEXT_OPTION_NO_HEADER) | static_cast(SPV_BINARY_TO_TEXT_OPTION_FRIENDLY_NAMES) | static_cast(SPV_BINARY_TO_TEXT_OPTION_COMMENT) | static_cast(SPV_BINARY_TO_TEXT_OPTION_INDENT) | static_cast(SPV_BINARY_TO_TEXT_OPTION_COLOR)); LOGF("%s\n", text.c_str()); EXIT("ShaderRecompiler %s SPIR-V validation failed hash=0x%016" PRIx64 ":\n%s", stage, shader_hash, messages.c_str()); } // Formats with a narrower host variant drop the components the shader does not read. vk::Format NarrowInputFormat(vk::Format format, uint32_t used_components) { switch (format) { case vk::Format::eR32G32B32A32Sfloat: switch (used_components) { case 1: return vk::Format::eR32Sfloat; case 2: return vk::Format::eR32G32Sfloat; case 3: return vk::Format::eR32G32B32Sfloat; default: return format; } case vk::Format::eR32G32B32Sfloat: switch (used_components) { case 1: return vk::Format::eR32Sfloat; case 2: return vk::Format::eR32G32Sfloat; default: return format; } case vk::Format::eR16G16B16A16Sfloat: switch (used_components) { case 1: return vk::Format::eR16Sfloat; case 2: return vk::Format::eR16G16Sfloat; default: return format; } case vk::Format::eR8G8B8A8Unorm: switch (used_components) { case 1: return vk::Format::eR8Unorm; case 2: return vk::Format::eR8G8Unorm; default: return format; } case vk::Format::eR8G8B8A8Snorm: return used_components == 2 ? vk::Format::eR8G8Snorm : format; case vk::Format::eR8G8B8A8Uint: switch (used_components) { case 1: return vk::Format::eR8Uint; case 2: return vk::Format::eR8G8Uint; default: return format; } default: return format; } } vk::Format InputFormat(const ShaderBufferResource& resource, uint32_t used_components) { switch (resource.RawFormat()) { // Temporary: invalid PS5 buffer format 113 (VertexAttribFormat::k16_16SInt). case 113: return NarrowInputFormat(vk::Format::eR32G32B32A32Sfloat, used_components); case 121: return vk::Format::eR16G16Sfloat; default: break; } const auto format = VulkanFormat(resource.Format()); if (format == vk::Format::eUndefined || ShaderRecompiler::Format::GetFormatInfo(resource.Format()).component_count == 0) { EXIT("unknown vertex format: fmt = %u\n", resource.RawFormat()); } return NarrowInputFormat(format, used_components); } // The host format of each vertex input attribute, narrowed to the components the stage reads. std::vector VertexInputFormats(const ShaderVertexInputInfo& input_info, const ShaderRecompiler::IR::CompiledShaderInfo& program) { std::vector formats(input_info.resources_num); for (int index = 0; index < input_info.resources_num; index++) { const auto registers_num = input_info.resources_dst[index].registers_num; EXIT_IF(registers_num < 1 || registers_num > 4); const auto fetched = program.info.vertex_fetch_components[index]; formats[index] = InputFormat(input_info.resources[index], fetched > 0 ? fetched : static_cast(registers_num)); } return formats; } } // namespace ShaderRecompiler::Spirv::Profile SpirvProfile(const GraphicContext& graphics) { // AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and // signed-zero results; GLSL.std.450 alone does not order -0 below +0. return {.maintenance8 = graphics.maintenance8_enabled, .native_float_min_max = graphics.physical_device_properties.vendorID == 0x1002u, .device_clock_period_ps = graphics.device_clock_period_ps}; } std::filesystem::path ShaderStorePath() { if (KYTY_BUILD != KYTY_BUILD_RELEASE || !Config::ShaderCacheEnabled() || Config::GraphicsDebugDumpEnabled()) { return {}; } const auto title_id = PipelineCacheTitleId(); if (title_id.empty()) { return {}; } return std::filesystem::path("_PipelineCache") / (title_id + ".shaders"); } bool GraphicsPipelineKey::operator==(const GraphicsPipelineKey& other) const { return std::memcmp(this, &other, sizeof(*this)) == 0; } std::size_t PipelineCache::GraphicsPipelineKeyHash::operator()(const GraphicsPipelineKey& key) const { return XXH3_64bits(&key, sizeof(key)); } std::span PipelineCache::Program::FunctionCode() const { return call_source ? std::span(call_source->linked->code).subspan(call_source->code.size()) : std::span{}; } bool PipelineCache::Program::RefreshCallSource(const ShaderRecompiler::IR::SrtRuntime& runtime) { const auto revision = call_source->revision; ShaderRecompiler::RefreshShaderSource(*call_source, runtime); resource_plan.source_reads = call_source->reads; return revision != call_source->revision; } template ShaderModule* PipelineCache::Program::FindModule(const ShaderRecompiler::IR::SrtRuntime& runtime, uint32_t push_data_cursor) { EXIT_IF(!ShaderRecompiler::IR::MaterializeResources(resource_plan, runtime, resources, specialization)); for (auto& candidate: modules) { const auto& layout = candidate.info.bindings; if (layout.push_data_start_dword != ShaderRecompiler::IR::PushData::StartFor( push_data_cursor, layout.ShaderDataDwords()) || candidate.specialization != specialization) { continue; } if constexpr (HasCalls) { if (!std::ranges::equal(candidate.function_code, FunctionCode())) continue; } return &candidate; } return nullptr; } ShaderModule& PipelineCache::CompileModule(Program& program, const ProgramKey& key, ShaderRecompiler::CompileOptions options, ShaderRecompiler::TranslateResult translated, uint32_t push_data_cursor) { Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::ShadersCompiled, Common::DebugCounters::Counter::ShaderCompileNs, Common::DebugCounters::Peak::ShaderCompileNs); const auto profile = SpirvProfile(m_graphics); options.profile = &profile; auto result = ShaderRecompiler::CompileProgram(std::move(translated), options, program.specialization, push_data_cursor); auto info = std::move(result.program).TakeCompiledInfo(); // The same inputs give the same module id across runs. const auto function_code = program.FunctionCode(); ShaderRecompiler::IR::Writer inputs; ShaderRecompiler::IR::IoAll(inputs, key.stage, key.hash, key.user_data_count, key.static_state, info.bindings.push_data_start_dword, program.specialization); inputs.Raw(function_code.data(), function_code.size_bytes()); // Last, so it needs no size. const auto id = XXH3_64bits(inputs.bytes.data(), inputs.bytes.size()); const auto* stage = ShaderRecompiler::StageName(key.stage); DumpShader(result.spirv, fmt::format("{}_{:016x}_{:016x}.spv", stage, key.hash, id)); ValidateShaderSpirv(stage, key.hash, result.spirv, profile.maintenance8); auto& module = program.modules.emplace_back(ShaderModule { .id = id, .module = CompileSPV(result.spirv, m_graphics.device), .info = std::move(info), .specialization = program.specialization, .function_code = {function_code.begin(), function_code.end()}, }); EXIT_IF(module.module == nullptr); m_compiled[static_cast(key.stage)]++; // Guest geometry shaders are compiled through the host mesh stage. std::printf("Shaders: VS %u | PS %u | CS %u | GS %u | LS %u | HS %u | TES %u\n", m_compiled[static_cast(ShaderType::Vertex)], m_compiled[static_cast(ShaderType::Pixel)], m_compiled[static_cast(ShaderType::Compute)], m_compiled[static_cast(ShaderType::Mesh)], m_compiled[static_cast(ShaderType::Local)], m_compiled[static_cast(ShaderType::TessellationControl)], m_compiled[static_cast(ShaderType::TessellationEvaluation)]); // The inputs of these come from the stage's static key. if (key.stage == ShaderType::Vertex || key.stage == ShaderType::Local) { module.vertex_formats = VertexInputFormats(*options.input_info.vertex, module.info); } else if (key.stage == ShaderType::Pixel && options.input_info.pixel->parameter_mode == ShaderPixelParameterMode::Rectangle) { module.rect_parameters = RectListParameters(*options.input_info.pixel, module.info); } // Programs with GPU-selected calls depend on their callees' current code. if (program.call_source == nullptr) { StoreModule(key, program, module, result.spirv); } return module; } template const ShaderModule* PipelineCache::GetProgram(const ShaderParams& params, InputInfo& input_info, uint32_t& push_data_cursor) { const auto user_data = std::span(params.user_data).first(params.user_data_count); ShaderRecompiler::IR::SrtRuntime runtime { .user_data = user_data, .shader_base = params.Base(), .read_specialization_memory = ReadShaderGuestMemory, .image_heap = m_image_heap, }; ShaderRecompiler::CompileOptions options {.wave_size = input_info.wave_size}; if constexpr (std::is_same_v) { options.stage = input_info.logical_stage; options.input_info.vertex = &input_info; options.user_data_base = 8; if (options.stage == ShaderType::Mesh || options.stage == ShaderType::TessellationControl) { options.user_data_base = 0; options.wave_size = options.stage == ShaderType::Mesh ? input_info.mesh.wave_size : 64u; } } else if constexpr (std::is_same_v) { options.stage = ShaderType::Pixel; options.input_info.pixel = &input_info; } else { static_assert(std::is_same_v); options.stage = ShaderType::Compute; options.input_info.compute = &input_info; runtime.workgroup_counts = input_info.workgroup_counts; runtime.indirect_args_address = input_info.indirect_args_address; } m_lookup_key.stage = options.stage; m_lookup_key.hash = params.hash; m_lookup_key.user_data_count = params.user_data_count; BuildStageStaticKey(input_info, m_lookup_key.static_state); auto entry = m_program_cache.find(m_lookup_key); const auto bind = [&](const ShaderModule& module) { input_info.stage = {.module = &module, .resources = &entry->second.resources}; module.info.bindings.AdvancePushData(push_data_cursor); return &module; }; bool source_changed = false; if (entry != m_program_cache.end()) { auto& cached = entry->second; if (cached.skip_dispatch) return nullptr; ShaderModule* module = nullptr; if (cached.call_source) { source_changed = cached.RefreshCallSource(runtime); if (!source_changed) module = cached.FindModule(runtime, push_data_cursor); } else { module = cached.FindModule(runtime, push_data_cursor); } if (module != nullptr) return bind(*module); } options.shader_hash = params.hash; options.user_data = user_data; options.back_code = params.back_code; options.dump_ir = Config::GetShaderLogDirection() != Config::LogDirection::Silent; options.emit_debug_info = Config::GraphicsDebugDumpEnabled(); ShaderRecompiler::ShaderSource source; if (entry == m_program_cache.end()) { DumpShader(params.code, fmt::format("original/{}_{:016x}.bin", ShaderRecompiler::StageName(options.stage), params.hash)); source = ShaderRecompiler::PrepareShaderSource(params.code, options); if (source.unsupported_call_pc != UINT32_MAX) { PipelineCacheLog("!!! TEMPORARY COMPUTE DISPATCH SKIP !!! shader=0x{:016x} pc=0x{:08x} " "reason=unsupported GPU-selected callable target; rendering is incomplete", params.hash, source.unsupported_call_pc); m_program_cache.emplace(m_lookup_key, Program {.skip_dispatch = true}); return nullptr; } } auto* current = entry == m_program_cache.end() ? &source : entry->second.call_source.get(); ShaderRecompiler::TranslateResult translated; if (current != nullptr) { if (!current->calls.empty()) { if (entry == m_program_cache.end()) ShaderRecompiler::RefreshShaderSource(*current, runtime); translated = ShaderRecompiler::TranslateProgram(current->linked->decoded, options); translated.program.source_reads = current->reads; } else { translated = ShaderRecompiler::TranslateProgram(current->decoded, options); } } else { translated = ShaderRecompiler::TranslateProgram(params.code, options); } if (entry == m_program_cache.end()) { auto retained = !source.calls.empty() ? std::make_unique(std::move(source)) : nullptr; entry = m_program_cache.emplace(m_lookup_key, Program { .call_source = std::move(retained), .resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program), }).first; auto& program = entry->second; EXIT_IF(!ShaderRecompiler::IR::MaterializeResources( program.resource_plan, runtime, program.resources, program.specialization)); } else if (source_changed) { entry->second.resource_plan = ShaderRecompiler::IR::ExtractResourcePlan(translated.program); if (const auto* module = entry->second.FindModule(runtime, push_data_cursor)) { return bind(*module); } } return bind(CompileModule(entry->second, entry->first, options, std::move(translated), push_data_cursor)); } PipelineCache::PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap, const std::filesystem::path& shader_store) : m_graphics(graphics), m_image_heap(image_heap), m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); InitializeDriverCache(); WarmUp(shader_store); } PipelineCache::~PipelineCache() { Save(); auto destroy = [this](const auto& pipelines) { for (const auto& pipeline: pipelines | std::views::values) { m_graphics.device.destroyPipeline(pipeline.pipeline, nullptr); m_graphics.device.destroyPipelineLayout(pipeline.pipeline_layout, nullptr); m_graphics.device.destroyDescriptorSetLayout(pipeline.descriptor_set_layout, nullptr); } }; destroy(m_graphics_pipelines); destroy(m_compute_pipelines); for (const auto& program: m_program_cache | std::views::values) { for (const auto& module: program.modules) { m_graphics.device.destroyShaderModule(module.module, nullptr); } } if (m_driver_cache != nullptr) { m_graphics.device.destroyPipelineCache(m_driver_cache, nullptr); } } void PipelineCache::InitializeDriverCache() { const auto title_id = PipelineCacheTitleId(); if (title_id.empty()) { return; } if (KYTY_BUILD != KYTY_BUILD_RELEASE) { PipelineCacheLog("Vulkan pipeline cache: disabled (non-Release build)"); return; } m_driver_cache_path = std::filesystem::path("_PipelineCache") / (title_id + ".bin"); const auto path = Common::PathToString(m_driver_cache_path); const bool cache_exists = Common::File::IsFileExisting(m_driver_cache_path); if (cache_exists) { PipelineCacheLog("Vulkan pipeline cache: loading {}", path); } else { PipelineCacheLog("Vulkan pipeline cache: initializing {}", path); } std::vector initial_data; if (cache_exists) { Common::File file(m_driver_cache_path, Common::File::Mode::Read); const auto file_size = file.IsInvalid() ? 0 : file.Size(); const auto signature = DriverCacheSignature(m_graphics.GetPhysicalDeviceProperties()); if (file_size >= signature.size() + sizeof(uint64_t) && file_size <= std::numeric_limits::max()) { std::string cached_signature(signature.size(), '\0'); uint64_t payload_hash = 0; initial_data.resize(file_size - signature.size() - sizeof(payload_hash)); uint32_t signature_read = 0; uint32_t hash_read = 0; uint32_t payload_read = 0; file.Read(cached_signature.data(), static_cast(cached_signature.size()), &signature_read); file.Read(&payload_hash, sizeof(payload_hash), &hash_read); file.Read(initial_data.data(), static_cast(initial_data.size()), &payload_read); file.Close(); if (signature_read != cached_signature.size() || hash_read != sizeof(payload_hash) || payload_read != initial_data.size() || cached_signature != signature || XXH3_64bits(initial_data.data(), initial_data.size()) != payload_hash) { initial_data.clear(); PipelineCacheLog( "Vulkan pipeline cache: invalidating {} (driver or data mismatch)", path); } } else { file.Close(); PipelineCacheLog("Vulkan pipeline cache: invalidating {} (invalid file size)", path); } } vk::PipelineCacheCreateInfo create {}; create.initialDataSize = initial_data.size(); create.pInitialData = initial_data.empty() ? nullptr : initial_data.data(); auto result = m_graphics.device.createPipelineCache(&create, nullptr, &m_driver_cache); if (result != vk::Result::eSuccess && !initial_data.empty()) { PipelineCacheLog("Vulkan pipeline cache: driver rejected {} ({}); starting empty", path, vk::to_string(result)); initial_data.clear(); create.initialDataSize = 0; create.pInitialData = nullptr; result = m_graphics.device.createPipelineCache(&create, nullptr, &m_driver_cache); } if (result != vk::Result::eSuccess) { PipelineCacheLog("Vulkan pipeline cache: disabled ({})", vk::to_string(result)); m_driver_cache = nullptr; return; } if (!initial_data.empty()) { PipelineCacheLog("Vulkan pipeline cache: loaded {:.2f} MB from {}", static_cast(initial_data.size()) / 1'000'000.0, path); } else { PipelineCacheLog("Vulkan pipeline cache: initialized empty"); } } void PipelineCache::CheckpointDriverCache() { if (m_driver_cache == nullptr) { return; } m_driver_cache_dirty = true; if (std::chrono::steady_clock::now() >= m_next_driver_cache_save) { Save(); } } void PipelineCache::Save() { if (m_driver_cache == nullptr || !m_driver_cache_dirty) { return; } m_next_driver_cache_save = std::chrono::steady_clock::now() + DriverCacheSaveInterval; size_t size = 0; vk::Result result; std::vector payload; for (uint32_t attempt = 0; attempt < 3; attempt++) { size = 0; result = m_graphics.device.getPipelineCacheData(m_driver_cache, &size, nullptr); if (result != vk::Result::eSuccess || size == 0 || size > std::numeric_limits::max()) { break; } payload.resize(size); result = m_graphics.device.getPipelineCacheData(m_driver_cache, &size, payload.data()); if (result != vk::Result::eIncomplete) { break; } } if (result != vk::Result::eSuccess || size == 0 || size > std::numeric_limits::max()) { PipelineCacheLog("Vulkan pipeline cache: save failed ({}, {:.2f} MB)", vk::to_string(result), static_cast(size) / 1'000'000.0); return; } payload.resize(size); auto prefix = DriverCacheSignature(m_graphics.GetPhysicalDeviceProperties()); const auto payload_hash = XXH3_64bits(payload.data(), payload.size()); prefix.append(reinterpret_cast(&payload_hash), sizeof(payload_hash)); if (!Common::File::CreateDirectories(m_driver_cache_path.parent_path())) { PipelineCacheLog("Vulkan pipeline cache: failed to create cache directory"); return; } auto temp_path = m_driver_cache_path; temp_path += ".tmp"; Common::File file; uint32_t prefix_written = 0; uint32_t payload_written = 0; if (file.Create(temp_path)) { file.Write(prefix.data(), static_cast(prefix.size()), &prefix_written); file.Write(payload.data(), static_cast(payload.size()), &payload_written); } const bool flushed = !file.IsInvalid() && file.Flush(); file.Close(); const bool written = prefix_written == prefix.size() && payload_written == payload.size() && flushed; std::error_code error; if (written) { std::filesystem::rename(temp_path, m_driver_cache_path, error); } if (!written || error) { PipelineCacheLog("Vulkan pipeline cache: failed to write {}", Common::PathToString(m_driver_cache_path)); return; } PipelineCacheLog("Vulkan pipeline cache: saved {:.2f} MB to {}", static_cast(payload.size()) / 1'000'000.0, Common::PathToString(m_driver_cache_path)); m_driver_cache_dirty = false; } void PipelineCache::GetGraphicsPrograms(const CommandBuffer& command, DrawRenderState& state) { const auto& context = command.GetRegisters(); const auto& user_config = command.GetUserConfig(); const auto& sh = context.GetShaderRegisters(); const auto& vertex_regs = command.GetShaders().GetVs(); const auto& pixel_regs = command.GetShaders().GetPs(); auto& vertex_info = state.vertex_info; auto& pixel_info = state.ps_input_info; const bool object_primitive_id = context.HasObjectPrimitiveIdOutput(); const auto write_mask = context.GetRenderTargetMask() & sh.m_cbShaderMask; uint32_t color_output_mask = object_primitive_id ? 1u << 7u : 0; for (uint32_t index = 0; index < RENDER_COLOR_ATTACHMENTS_MAX; index++) { const auto slot = ShaderPixelExportTarget(sh.m_cbShaderMask, index); if (slot >= RENDER_COLOR_ATTACHMENTS_MAX) { break; } if (sh.target_output_mode[index] != 0 && render_target_mask_slot(write_mask, slot) != 0) { color_output_mask |= 1u << slot; } } const auto& db = sh.db_shader_control; // Without color outputs, a pixel shader still runs for its depth or coverage side effects. const bool pixel_active = pixel_regs.ps_regs.data_addr != 0 && (color_output_mask != 0 || db.shader_kill_enable || db.shader_z_export_enable || db.shader_mask_export_enable || db.shader_dual_export_enable || db.shader_execute_on_noop || sh.AlphaToMask()); state.ps_active = pixel_active; const bool tess_active = user_config.GetPrimType() == Prospero::PrimitiveType::kPatch; std::array vertex_params; if (tess_active) { vertex_params = PrepareTessellationPrograms(vertex_regs, context, vertex_info); // Reflection decodes both programs; the key also covers the VGT fields in tess. const uint64_t layout_key[] = {vertex_params[0].hash, vertex_params[1].hash, uint64_t {sh.m_vgtLsHsConfig & 0xfff00u} << 32u | sh.m_vgtTfParam}; const auto [layout, inserted] = m_tess_layouts.try_emplace( XXH3_64bits(layout_key, sizeof(layout_key)), vertex_info[0].tess); if (inserted) { ShaderRecompiler::AnalyzeTessellationPrograms(vertex_params[0].code, vertex_params[1].code, layout->second); StoreLayout(layout->first, layout->second); } for (auto& stage: vertex_info) { stage.tess = layout->second; } } else { vertex_params[0] = PrepareProgram(vertex_regs, context, user_config, vertex_info[0]); } const bool mesh_active = vertex_info[0].logical_stage == ShaderType::Mesh; if (object_primitive_id) { // CxObjectIdMode: register source plus a 32-bit pipelined primitive-ID addition. EXIT_NOT_IMPLEMENTED(sh.m_paClObjprimIdCntl != 2u || mesh_active || tess_active || !pixel_active || (sh.m_vgtDrawPayloadCntl & 8u) == 0 || user_config.GetPrimType() != Prospero::PrimitiveType::kTriList); const auto& target = context.GetRenderTarget(7); EXIT_NOT_IMPLEMENTED(Prospero::ResolveRenderTargetFormat(target.info.format, target.info.channel_type) .buffer_format != Prospero::BufferFormat::k32UInt); } if (mesh_active) { EXIT_NOT_IMPLEMENTED(!m_graphics.mesh_shader_enabled); auto& mesh = vertex_info[0].mesh; mesh.host_subgroup_size = m_graphics.subgroup_size; const auto& limits = m_graphics.mesh_shader_properties; const auto logical_threads = mesh.threads_num[0] * mesh.threads_num[1] * mesh.threads_num[2]; const auto host_threads = ((logical_threads + mesh.wave_size - 1u) / mesh.wave_size) * std::min(mesh.host_subgroup_size, mesh.wave_size); if (host_threads > limits.maxMeshWorkGroupInvocations || host_threads > limits.maxMeshWorkGroupSize[0] || mesh.max_vertices > limits.maxMeshOutputVertices || mesh.max_primitives > limits.maxMeshOutputPrimitives || mesh.lds_size_dwords * sizeof(uint32_t) > limits.maxMeshSharedMemorySize) { EXIT("mesh shader exceeds host limits: threads=%u vertices=%u primitives=%u LDS=%u\n", host_threads, mesh.max_vertices, mesh.max_primitives, mesh.lds_size_dwords); } } ShaderParams pixel_params; if (pixel_active) { std::array target_export_mapping {}; for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) { const auto& rt = context.GetRenderTarget(slot); if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) { target_export_mapping[slot] = TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type, rt.info.channel_order) .export_mapping; } } pixel_params = PrepareProgram(pixel_regs, sh, target_export_mapping, pixel_info); pixel_info.object_primitive_id = object_primitive_id; for (uint32_t export_index = 0; export_index < 8u; export_index++) { const auto slot = ShaderPixelExportTarget(pixel_info.target_shader_mask, export_index); if (slot >= 8u || pixel_info.target_output_mode[export_index] == 0u) continue; const auto& rt = context.GetRenderTarget(slot); const auto target_mask = render_target_mask_slot(context.GetRenderTargetMask(), slot); if (rt.base.addr == 0 || target_mask == 0u) continue; const auto format = Prospero::ResolveRenderTargetFormat(rt.info.format, rt.info.channel_type); if (format.buffer_format != Prospero::BufferFormat::k11_11_10UNorm) continue; const auto mapping = pixel_info.target_export_mapping[export_index]; const auto shader_mask = render_target_mask_slot(pixel_info.target_shader_mask, slot); const bool blend = context.GetBlendControl(slot).enable && !rt.info.blend_bypass; if (blend || (mapping.ApplyMask(target_mask) & 7u) != 7u || (mapping.ApplyMask(shader_mask) & 7u) != 7u || pixel_info.target_output_mode[export_index] != 9u) { EXIT("unsupported packed UNorm render-target export: slot=%u export=%u mode=%u " "target_mask=0x%x shader_mask=0x%x mapping=0x%02x blend=%u\n", slot, export_index, pixel_info.target_output_mode[export_index], target_mask, shader_mask, mapping.packed, static_cast(blend)); } pixel_info.target_packed_unorm_mask |= 1u << export_index; pixel_info.target_round_mode |= static_cast(rt.info.round_mode) << export_index; } if (!mesh_active && !tess_active && Prospero::IsRectList(user_config.GetPrimType())) { pixel_info.parameter_mode = ShaderPixelParameterMode::Rectangle; } else if (context.GetModeControl().provoking_vtx_last) { pixel_info.parameter_mode = ShaderPixelParameterMode::LastVertex; } else { pixel_info.parameter_mode = ShaderPixelParameterMode::FirstVertex; } const auto& blend = context.GetBlendControl(0); pixel_info.dual_source_blending = blend.enable && !context.GetRenderTarget(0).info.blend_bypass && (BlendFactorIsDualSource(blend.color_srcblend) || BlendFactorIsDualSource(blend.color_destblend) || (blend.separate_alpha_blend && (BlendFactorIsDualSource(blend.alpha_srcblend) || BlendFactorIsDualSource(blend.alpha_destblend)))); if (pixel_info.dual_source_blending) { // MRT1 supplies the second blend source for target 0. pixel_info.target_output_mode[1] = pixel_info.target_output_mode[0]; pixel_info.target_export_mapping[1] = pixel_info.target_export_mapping[0]; } else if (blend.enable && !context.GetRenderTarget(0).info.blend_bypass && pixel_info.target_output_mode[0] != 0 && pixel_info.target_output_mode[0] != 7 && std::all_of(std::begin(pixel_info.target_output_mode) + 1, std::end(pixel_info.target_output_mode), [](uint8_t mode) { return mode == 0; })) { switch (ClassifyBlendMapping(blend, pixel_info.target_export_mapping[0])) { case BlendMappingSupport::SourceAlpha: pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlpha; break; case BlendMappingSupport::SourceAlphaOne: pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlphaOne; break; case BlendMappingSupport::SourceAlphaZero: pixel_info.alpha_blend_source = ShaderAlphaBlendSource::SourceAlphaZero; break; default: break; } if (pixel_info.alpha_blend_source != ShaderAlphaBlendSource::None) { pixel_info.dual_source_blending = true; pixel_info.target_output_mode[1] = pixel_info.target_output_mode[0]; pixel_info.target_export_mapping[1] = {}; } } } auto& last_stage = vertex_info[tess_active ? 2u : 0u]; const auto& clip_control = context.GetClipControl(); if (clip_control.clip_disable) { const auto& viewport = context.GetScreenViewport().viewports[0]; const auto& limits = m_graphics.GetPhysicalDeviceProperties().limits; auto& clip = last_stage.clip_space; clip.scale[0] = viewport.xscale; clip.scale[1] = viewport.yscale; clip.offset[0] = viewport.xoffset; clip.offset[1] = viewport.yoffset; clip.half_extent[0] = static_cast(std::min(limits.maxViewportDimensions[0], 16384u)) * 0.5f; clip.half_extent[1] = static_cast(std::min(limits.maxViewportDimensions[1], 16384u)) * 0.5f; clip.enabled = true; } else if (clip_control.user_clip_planes != 0) { // CLIP_DISABLE also disables user planes. How they combine with shader clip and cull // distances is unverified; draw push constants only reach VS/mesh and PS, and the // generated RECTLIST stages do not forward clip distances. EXIT_NOT_IMPLEMENTED((last_stage.pa_cl_vs_out_cntl & 0xffffu) != 0 || tess_active || Prospero::IsRectList(user_config.GetPrimType())); last_stage.user_clip_planes = clip_control.user_clip_planes; } uint32_t push_data_cursor = ShaderRecompiler::IR::PushData::GraphicsDrawDwords( mesh_active, object_primitive_id, last_stage.user_clip_planes); if (pixel_active) { const auto& pixel_module = *GetProgram(pixel_params, pixel_info, push_data_cursor); for (const auto& output: pixel_module.info.info.outputs) { if (output.kind == ShaderRecompiler::IR::StageOutputKind::Mrt || output.kind == ShaderRecompiler::IR::StageOutputKind::ObjectPrimitiveId) { state.mrt_mask |= 1u << output.location; } } state.mrt_mask &= color_output_mask; } for (uint32_t i = 0; i < (tess_active ? 3u : 1u); i++) { GetProgram(vertex_params[i], vertex_info[i], push_data_cursor); } } const ShaderModule* PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs, ShaderComputeInputInfo& input_info) { input_info.host_subgroup_size = m_graphics.SupportsComputeWave64() ? 64u : 32u; const auto params = PrepareProgram(regs, input_info); input_info.lds_storage = input_info.lds_size_dwords * 4u > m_graphics.GetPhysicalDeviceProperties().limits.maxComputeSharedMemorySize; uint32_t push_data_cursor = 0; return GetProgram(params, input_info, push_data_cursor); } PipelineCache::Pipeline& PipelineCache::GetGraphicsPipeline(const CommandBuffer& command, const DrawRenderState& state) { const auto* colors = state.color_info; const auto& depth = state.depth_info; const auto vertex_info = state.VertexStages(); const auto& ps_input_info = state.ps_input_info; const auto& vs_input_info = vertex_info.front(); KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200); const bool ps_active = state.ps_active; const bool tessellation = vertex_info.size() == 3; std::array stages {}; for (uint32_t i = 0; i < vertex_info.size(); i++) { stages[i] = vertex_info[i].stage.module; } if (ps_active) { stages[3] = ps_input_info.stage.module; } EXIT_IF(stages[0] == nullptr || (ps_active && stages[3] == nullptr)); const bool mesh = stages[0]->info.stage == ShaderType::Mesh; const auto color_count = state.color_count; auto& ctx = command.GetRegisters(); const HW::ModeControl& mc = ctx.GetModeControl(); GraphicsPipelineKey key {}; for (uint32_t i = 0; i < stages.size(); i++) { if (stages[i] != nullptr) { key.stage_ids[i] = stages[i]->id; } } auto& static_params = key.static_params; auto& rendering = key.rendering; rendering.color_count = 0; uint32_t attachment_samples = 0; for (uint32_t i = 0; i < color_count; i++) { const auto slot = colors[i].target_slot; EXIT_IF(slot >= RENDER_COLOR_ATTACHMENTS_MAX); rendering.color_count = std::max(rendering.color_count, slot + 1); EXIT_IF(!colors[i].image_id || colors[i].desc.view_info.format == vk::Format::eUndefined); static_params.color_mask[slot] = colors[i].export_mapping.ApplyMask( render_target_mask_slot(ctx.GetRenderTargetMask(), colors[i].target_slot)); if (colors[i].desc.info.guest_format == Prospero::BufferFormat::k11_11_10UNorm) { static_params.color_mask[slot] = 1u; } rendering.color_formats[slot] = colors[i].desc.view_info.format; if (attachment_samples == 0) { attachment_samples = colors[i].desc.info.samples; } else if (attachment_samples != colors[i].desc.info.samples) { EXIT("mixed color attachment sample counts are unsupported: %u and %u\n", attachment_samples, colors[i].desc.info.samples); } const auto& rt = ctx.GetRenderTarget(colors[i].target_slot); const auto& bc = ctx.GetBlendControl(colors[i].target_slot); auto alpha_source = ShaderAlphaBlendSource::None; if (slot == 0 && ps_active) { alpha_source = ps_input_info.alpha_blend_source; } static_params.blend_enable[slot] = bc.enable && !rt.info.blend_bypass; if (static_params.blend_enable[slot] && alpha_source == ShaderAlphaBlendSource::None && ClassifyBlendMapping(bc, colors[i].export_mapping) != BlendMappingSupport::Direct) { static_params.blend_enable[slot] = false; static std::atomic_bool warned = false; if (!warned.exchange(true, std::memory_order_relaxed)) { Log::WriteToConsoleAndLog(fmt::format( "Warning: blending disabled for unsupported color mapping " "(slot={} mapping=0x{:02x} color={}/{} alpha={}/{} separate={}).\n", slot, colors[i].export_mapping.packed, bc.color_srcblend, bc.color_destblend, bc.alpha_srcblend, bc.alpha_destblend, bc.separate_alpha_blend ? 1 : 0)); } } if (static_params.blend_enable[slot]) { auto blend = bc; switch (alpha_source) { case ShaderAlphaBlendSource::SourceAlpha: blend.color_srcblend = RemapSourceAlphaFactor(blend.color_srcblend); blend.color_destblend = RemapSourceAlphaFactor(blend.color_destblend); blend.separate_alpha_blend = false; break; case ShaderAlphaBlendSource::SourceAlphaOne: case ShaderAlphaBlendSource::SourceAlphaZero: // The second source carries the mapped source factor; its alpha stays logical Sa. blend.color_srcblend = static_cast(Prospero::BlendFactor::kSrc1Color); blend.color_destblend = static_cast(Prospero::BlendFactor::kOneMinusSrc1Alpha); blend.separate_alpha_blend = false; break; case ShaderAlphaBlendSource::None: break; } static_params.color_srcblend[slot] = blend.color_srcblend; static_params.color_comb_fcn[slot] = blend.color_comb_fcn; static_params.color_destblend[slot] = blend.color_destblend; static_params.separate_alpha_blend[slot] = blend.separate_alpha_blend; if (blend.separate_alpha_blend) { static_params.alpha_srcblend[slot] = blend.alpha_srcblend; static_params.alpha_comb_fcn[slot] = blend.alpha_comb_fcn; static_params.alpha_destblend[slot] = blend.alpha_destblend; } } } const bool with_depth = depth.desc.view_info.format != vk::Format::eUndefined && static_cast(depth.image_id); if (with_depth) { const auto aspects = ImageViewOps::DepthAspectMask(depth.desc.view_info.format); rendering.depth_format = aspects & vk::ImageAspectFlagBits::eDepth ? depth.desc.view_info.format : vk::Format::eUndefined; rendering.stencil_format = aspects & vk::ImageAspectFlagBits::eStencil ? depth.desc.view_info.format : vk::Format::eUndefined; if (attachment_samples == 0) { attachment_samples = depth.desc.info.samples; } else if (attachment_samples != depth.desc.info.samples) { EXIT("mixed color/depth sample counts are unsupported: %u and %u\n", attachment_samples, depth.desc.info.samples); } } if (color_count == 0 && !with_depth) { attachment_samples = render_sample_count(ctx.GetAaConfig().msaa_num_samples); EXIT_IF(!static_cast( m_graphics.GetPhysicalDeviceProperties().limits.framebufferNoAttachmentsSampleCounts & vulkan_sample_count(attachment_samples))); } EXIT_IF(attachment_samples == 0 || vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}); if (ps_active && depth.depth_test_enable && ps_input_info.ps_execute_on_noop) { static std::atomic log_count {0}; if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) { LOGF("Pipeline: temporary: accepting EXEC_ON_NOOP with depth test enabled\n"); } } const auto& clip_control = ctx.GetClipControl(); static_params.negative_one_to_one = !clip_control.dx_clip_space; static_params.depth_clip_enable = clip_control.IsZClipEnabled(); static_params.topology = state.topology; static_params.primitive_restart_enable = state.primitive_restart_enable; static_params.samples = attachment_samples; static_params.sample_shading_enable = ps_active && attachment_samples > 1 && ps_input_info.ps_sample_shading; if (static_params.sample_shading_enable && !m_graphics.sample_rate_shading_enabled) { EXIT("Pipeline: sample-rate shading is required but unsupported by the host\n"); } static_params.alpha_to_coverage_enable = ps_active && ctx.GetShaderRegisters().AlphaToMask(); const bool rect_list = Prospero::IsRectList(command.GetUserConfig().GetPrimType()); static_params.cull_back = !rect_list && mc.cull_back; static_params.cull_front = !rect_list && mc.cull_front; static_params.face = mc.face; static_params.provoking_vtx_last = mc.provoking_vtx_last; static_params.polygon_mode = ResolvePolygonMode(mc, static_params.cull_front, static_params.cull_back); if (tessellation) { static_params.patch_control_points = static_cast(vs_input_info.tess.input_control_points); } else if (!mesh && static_params.topology == vk::PrimitiveTopology::ePatchList) { // Rectangle lists expand through generated tessellation stages. static_params.patch_control_points = 3; } if (!mesh) { EXIT_IF(vs_input_info.buffers_num < 0 || vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX || vs_input_info.resources_num < 0 || vs_input_info.resources_num > ShaderVertexInputInfo::RES_MAX); key.vertex_input.binding_count = static_cast(vs_input_info.buffers_num); key.vertex_input.attribute_count = static_cast(vs_input_info.resources_num); for (int binding = 0; binding < vs_input_info.buffers_num; binding++) { const auto& buffer = vs_input_info.buffers[binding]; key.vertex_input.bindings[binding] = {.stride = buffer.stride, .instance = buffer.fetch_index != 0}; } for (int attribute = 0; attribute < vs_input_info.resources_num; attribute++) { const auto binding = vs_input_info.resources_dst[attribute].buffer_index; EXIT_IF(binding < 0 || binding >= vs_input_info.buffers_num); key.vertex_input.attributes[attribute] = { .offset = static_cast(vs_input_info.resources[attribute].Base48() - vs_input_info.buffers[binding].addr), .binding = static_cast(binding), }; } } auto [iter, inserted] = m_graphics_pipelines.try_emplace(key); auto& pipeline = iter->second; if (!inserted) { return pipeline; } if (graphics_debug_dump_enabled()) { ShaderDbgDumpInputInfo(vs_input_info); if (ps_active) { ShaderDbgDumpInputInfo(ps_input_info); } LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64 " module=%p\n", key.stage_ids[0], static_cast(stages[0]->module), key.stage_ids[3], ps_active ? static_cast(stages[3]->module) : nullptr); } { Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::PipelinesCreated, Common::DebugCounters::Counter::PipelineCreateNs, Common::DebugCounters::Peak::PipelineCreateNs); CreatePipelineInternal(m_graphics, pipeline, key, stages, m_driver_cache); } StorePipeline(key); CheckpointDriverCache(); return pipeline; } PipelineCache::Pipeline& PipelineCache::GetComputePipeline(const ShaderComputeInputInfo& input_info) { KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100); EXIT_IF(input_info.stage.module == nullptr); const auto& module = *input_info.stage.module; auto [iter, inserted] = m_compute_pipelines.try_emplace(module.id); auto& pipeline = iter->second; if (!inserted) { return pipeline; } if (graphics_debug_dump_enabled()) { ShaderDbgDumpInputInfo(input_info); } { Common::DebugCounters::ScopedTimer timer(Common::DebugCounters::Counter::PipelinesCreated, Common::DebugCounters::Counter::PipelineCreateNs, Common::DebugCounters::Peak::PipelineCreateNs); CreatePipelineInternal(m_graphics, pipeline, module, m_driver_cache); } CheckpointDriverCache(); return pipeline; } } // namespace Libs::Graphics