Compare commits

...
47 changed files with 2074 additions and 145 deletions

No files matched your search

+6
View File
@@ -676,6 +676,12 @@ if(BUILD_TESTING)
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --stream-buffer-only)
add_test(NAME gpu_command_lane
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
add_test(NAME wait_until_safe_for_rendering
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --wait-until-safe-for-rendering-only)
add_test(NAME depth_bits
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-bits-only)
add_test(NAME msaa_storage
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --msaa-storage-only)
add_test(NAME pm4_context_state
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --context-state-only)
add_test(NAME gpu_tiler
+10
View File
@@ -46,6 +46,16 @@ or graphical glitches, so please include the version you tested when reporting a
## Screenshots
<table align="center">
<tr>
<td align="center">
<strong>Cyberpunk 2077</strong><br>
<img src="docs/screenshots/ps5-07.png" width="300" alt="Cyberpunk 2077 running in KytyPS5">
</td>
<td align="center">
<strong>MLB The Show 23</strong><br>
<img src="docs/screenshots/ps5-08.png" width="300" alt="MLB The Show 23 running in KytyPS5">
</td>
</tr>
<tr>
<td align="center">
<strong>Astro Bot</strong><br>
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.0 MiB

+16 -5
View File
@@ -30,6 +30,7 @@
#include <mutex>
#include <semaphore>
#include <thread>
#include <utility>
#include <vector>
namespace Libs::Graphics {
@@ -91,6 +92,12 @@ void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
m_work_available.Signal();
}
void GuestGpu::Wake() {
Common::LockGuard lock(m_queue_mutex);
m_wake_pending = true;
m_work_available.Signal();
}
void GuestGpu::ProcessCommands() {
EXIT_IF(!IsGpuThread());
while (m_pending_commands.load(std::memory_order_acquire) != 0) {
@@ -464,7 +471,10 @@ void GuestGpu::ThreadRun(void* data) {
}
if (selected_queue < 0) {
gpu->m_processing = false;
gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
// A wake that came before this wait must not be lost.
if (!std::exchange(gpu->m_wake_pending, false)) {
gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
}
for (auto& queue: gpu->m_queues) {
if (!queue.empty()) {
queue.front().blocked = false;
@@ -998,10 +1008,11 @@ void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {
}
void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
BufferFlush();
m_renderer.GetVideoOut().WaitFlipDone(static_cast<int>(video_out_handle),
static_cast<int>(display_buffer_index));
// Stall only this command processor, not the host command lane or other queues.
if (!m_renderer.GetVideoOut().IsSafeForRendering(static_cast<int>(video_out_handle),
static_cast<int>(display_buffer_index))) {
SuspendPm4();
}
}
template <typename T>
+3
View File
@@ -31,6 +31,8 @@ public:
[[nodiscard]] bool IsStopping();
void SendCommand(Common::UniqueFunction<void>&& command);
void SendCommandSync(Common::UniqueFunction<void>&& command);
// Retries blocked submissions now instead of after the 100 ms stall timeout.
void Wake();
// Submitted command memory is borrowed and must remain valid until GPU execution completes.
void Submit(std::span<const uint32_t> draw_commands,
@@ -87,6 +89,7 @@ private:
bool m_processing = false;
bool m_accepting = true;
bool m_stopping = false;
bool m_wake_pending = false;
bool m_shutdown_complete = false;
// Completion callbacks can outlive GuestGpu during renderer shutdown.
std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =
+1
View File
@@ -43,6 +43,7 @@ struct GraphicContext {
uint32_t min_subgroup_size = 0;
uint32_t max_subgroup_size = 0;
uint32_t max_push_descriptors = 0;
vk::DescriptorSetLayout image_heap_layout = nullptr;
vk::ShaderStageFlags required_subgroup_size_stages = {};
Common::Mutex queue_mutex;
uint32_t queue_family = static_cast<uint32_t>(-1);
+10 -1
View File
@@ -229,6 +229,7 @@ void TextureCache::UnregisterImage(ImageId id) {
if (!image.registered) {
return;
}
image.HeapChanged();
UntrackImage(id);
ImagePageTable::PageRange pages {};
if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
@@ -378,6 +379,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE);
const auto size = address - begin;
image.track_addr = address;
image.HeapChanged();
if (image.track_addr == image.track_addr_end) {
MarkAsMaybeDirty(id, image);
}
@@ -395,6 +397,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
const auto address = Common::AlignDown(end, TRACKER_PAGE_SIZE);
const auto size = end - address;
image.track_addr_end = address;
image.HeapChanged();
if (image.track_addr == image.track_addr_end) {
MarkAsMaybeDirty(id, image);
}
@@ -795,7 +798,8 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
auto& cached = *owner;
const auto current_tick = m_scheduler.CurrentTick();
const bool safe_to_delete =
current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval;
current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval &&
cached.heap_refs.load(std::memory_order_relaxed) == 0u;
const uint32_t requested_block = requested.bytes_per_block * requested.samples;
const uint32_t cached_block = cached.info.bytes_per_block * cached.info.samples;
@@ -2015,6 +2019,11 @@ void TextureCache::RunGarbageCollector() {
if (owner == nullptr || !owner->registered || owner->depth_id) {
continue;
}
if (owner->heap_refs.load(std::memory_order_relaxed) != 0u) {
// Resident heaps sample it without touching it.
TouchImage(*owner);
continue;
}
if (owner->IsGpuModified()) {
const bool safe = owner->SafeToDownload();
if (safe && owner->info.IsTiled()) {
+1
View File
@@ -191,6 +191,7 @@ private:
friend struct TextureCacheTestAccess;
friend class BufferCache;
friend class RenderExecutor;
friend class ImageHeap;
friend bool BcAlias::Matches(const ImageInfo&, const ImageInfo&);
friend bool BcAlias::Publish(TextureCache&, Image&);
friend void BcAlias::Resolve(TextureCache&, ImageId, ImageId);
@@ -90,7 +90,8 @@ ImageCreateFlags(const GraphicContext& graphics, const ImageInfo& info, vk::Imag
if (HasFormatFeature(properties, vk::FormatFeatureFlagBits::eColorAttachment)) {
usage |= vk::ImageUsageFlagBits::eColorAttachment;
}
if (info.samples == 1) {
if (graphics.GetPhysicalDeviceProperties().limits.storageImageSampleCounts &
static_cast<vk::SampleCountFlagBits>(info.samples)) {
usage |= vk::ImageUsageFlagBits::eStorage;
}
return usage;
@@ -211,6 +212,13 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destinat
barriers.push_back(barrier);
}
// Resident heaps sample their images in the read state; only leaving it invalidates them.
if (std::ranges::any_of(barriers, [&](const vk::ImageMemoryBarrier2& barrier) {
return barrier.oldLayout == ReadLayout() &&
barrier.srcAccessMask == vk::AccessFlagBits2::eShaderRead;
})) {
HeapChanged();
}
state = {destination_stage, destination_access, destination_layout};
return barriers;
}
+22 -1
View File
@@ -7,6 +7,7 @@
#include "graphics/host_gpu/graphicContext.h"
#include "graphics/host_gpu/renderer/image/imageInfo.h"
#include <atomic>
#include <compare>
#include <limits>
#include <optional>
@@ -71,6 +72,7 @@ public:
void MarkCpuDirty() {
m_cpu_dirty = true;
m_maybe_cpu_dirty = false;
HeapChanged();
}
[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
@@ -80,6 +82,7 @@ public:
if (!IsCpuDirty()) {
m_maybe_cpu_hash = HashGuestEdges();
m_maybe_cpu_dirty = true;
HeapChanged();
}
}
[[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) {
@@ -104,7 +107,10 @@ public:
void ClearGpuModified() noexcept { m_gpu_modified = false; }
[[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; }
void MarkBufferModified() noexcept { m_buffer_modified = true; }
void MarkBufferModified() noexcept {
m_buffer_modified = true;
HeapChanged();
}
void ClearBufferModified() noexcept { m_buffer_modified = false; }
[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
@@ -120,6 +126,19 @@ public:
return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024);
}
[[nodiscard]] uint64_t HashGuestEdges() const;
[[nodiscard]] vk::ImageLayout ReadLayout() const noexcept {
return info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
: vk::ImageLayout::eShaderReadOnlyOptimal;
}
// Resident image heaps sample an image without rediscovering it until one changes its
// contents, tracking, identity or layout.
void HeapChanged() const noexcept {
if (heap_refs.load(std::memory_order_relaxed) != 0u) {
heap_epoch.fetch_add(1u, std::memory_order_relaxed);
}
}
static inline std::atomic<uint64_t> heap_epoch {0};
ImageInfo info;
VulkanImage backing;
@@ -137,6 +156,8 @@ public:
ImageSubresourceRange stencil_subresources;
uint64_t tick_accessed_last = 0;
size_t lru_id = 0;
// Resident image heap elements that select this image.
std::atomic<uint32_t> heap_refs {0};
private:
friend struct ImageTestAccess;
@@ -101,6 +101,7 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
(resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
@@ -254,6 +254,10 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
const bool is_3d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D &&
descriptor.Type() == Prospero::ImageType::kColor3D &&
descriptor.BaseArray5() == 0;
const bool is_2d_msaa =
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
descriptor.Type() == Prospero::ImageType::kColor2DMsaa && descriptor.Depth() == 0 &&
descriptor.BaseArray5() == 0;
TileTextureBlockLayout tile_layout {};
bool supported_tile = false;
switch (tile) {
@@ -284,8 +288,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
const bool supported_swizzle =
IsValidImageSwizzle(swizzle) &&
(swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
descriptor.BaseLevel() <= descriptor.LastLevel() &&
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d || is_2d_msaa) &&
supported_tile && descriptor.BaseLevel() <= descriptor.LastLevel() &&
descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
!descriptor.MsaaDepth();
}
@@ -356,7 +360,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
}
view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
view.mapping =
storage || surface_format.conversion_format != Prospero::BufferFormat::kInvalid
storage || resource.conversion_format != Prospero::BufferFormat::kInvalid
? vk::ComponentMapping {}
: TextureGetComponentMapping(descriptor.DstSelXYZW(), surface_format.host_to_storage);
switch (resource.dimension) {
@@ -515,7 +519,9 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
}
const auto samples = multisampled ? 1u << last_level : 1u;
const auto depth = static_cast<uint32_t>(descriptor.Depth()) + 1u;
const auto format = descriptor.Format();
const auto format = ShaderRecompiler::IR::ReadsDepthBits(resource)
? Prospero::BufferFormat::k16UNorm
: descriptor.Format();
const auto surface_format = TextureGetSurfaceFormatInfo(format);
const bool shader_conversion =
surface_format.conversion_format != Prospero::BufferFormat::kInvalid;
@@ -579,6 +585,10 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
const auto alignment = desc.info.DataAlignment();
EXIT_NOT_IMPLEMENTED(desc.info.data.size == 0 || alignment == 0 ||
(address & (static_cast<uint64_t>(alignment) - 1u)) != 0);
// Image heaps keep the descriptors of unmapped textures, which PS5 never samples.
if (resource.heap && !m_context.IsMapped(address, desc.info.data.size)) {
return {.layout = vk::ImageLayout::eGeneral};
}
if (storage) {
ValidateStorageTexture(resource, descriptor, desc.info.data.size);
}
@@ -684,6 +694,10 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
prepared.samplers.clear();
prepared.shader_data.clear();
for (uint32_t i = 0; i < program.info.images.size(); i++) {
if (program.info.images[i].heap) {
prepared.images[i] = {};
continue;
}
auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]);
BindImage(binding.image_id, binding.desc.type == TextureCache::BindingType::Storage);
binding.mip_views.swap(prepared.images[i].mip_views);
@@ -846,14 +860,19 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
std::span<RenderColorInfo> colors) {
bool uses_dma = false;
bool uses_dma = false;
bool uses_image_heap = false;
FindBuffers(stages);
for (auto* stage: stages) {
uses_dma |= stage->runtime->program->info.uses_dma;
uses_image_heap |= stage->runtime->program->info.uses_image_heap;
}
if (uses_dma) {
m_context.PrepareBda();
}
if (uses_image_heap) {
m_context.GetImageHeap().Rebind(*this);
}
for (auto* stage: stages) {
RebindImages(*stage);
}
@@ -953,6 +972,9 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
}
}
}
if (pipeline.uses_image_heap) {
m_context.GetImageHeap().Commit(buffer, pipeline_bind_point, pipeline.pipeline_layout);
}
m_descriptor_images.resize(image_descriptor_count);
m_descriptor_writes.clear();
m_descriptor_writes.reserve(write_count);
@@ -1010,9 +1032,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
range, vk_buffer);
} else {
image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
: vk::ImageLayout::eShaderReadOnlyOptimal,
vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
image.Transit(image.ReadLayout(), vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
}
binding.layout = image.backing.state.layout;
}
@@ -1083,6 +1103,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
for (uint32_t i = 0; i < program.info.images.size(); ++i) {
const auto& resource = program.info.images[i];
const auto& texture = descriptors.images[i];
if (resource.heap) continue;
EXIT_IF(texture.image_id &&
resource.mip_count != (texture.mip_views.empty() ? 1u : texture.mip_views.size()));
const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(resource);
@@ -0,0 +1,544 @@
#include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
#include "common/alignment.h"
#include "common/assert.h"
#include "common/profiler.h"
#include "graphics/host_gpu/pageManager.h"
#include "graphics/host_gpu/regionDefinitions.h"
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
#include "graphics/host_gpu/renderer/render.h"
#include "graphics/host_gpu/renderer/renderContext.h"
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
#include "kernel/memory.h"
#include <algorithm>
#include <xxhash.h>
namespace Libs::Graphics {
namespace {
namespace IR = ShaderRecompiler::IR;
// Smaller tables are cheap to materialize on each draw and are often transient.
constexpr uint64_t MinTableBytes = 64ull * 1024ull;
constexpr uint32_t ElementMask = (1u << IR::ImageHeapKindShift) - 1u;
template <typename Pred, typename Fn>
void ForEachRun(size_t begin, size_t end, Pred&& pred, Fn&& fn) {
while (begin < end) {
if (!pred(begin)) {
++begin;
continue;
}
auto run = begin + 1u;
while (run < end && pred(run)) ++run;
fn(begin, run);
begin = run;
}
}
// Grows [begin, end) to cover [first, last).
void Widen(size_t& begin, size_t& end, size_t first, size_t last) {
const bool empty = begin == end;
begin = empty ? first : std::min(begin, first);
end = empty ? last : std::max(end, last);
}
// Guest bytes of a table that its tracker pages [begin, end) cover.
std::pair<uint64_t, uint64_t> PageRange(uint64_t base, uint64_t size, size_t begin, size_t end) {
const auto origin = Common::AlignDown(base, TRACKER_PAGE_SIZE);
return {std::max(base, origin + begin * TRACKER_PAGE_SIZE),
std::min(base + size, origin + end * TRACKER_PAGE_SIZE)};
}
} // namespace
size_t ImageHeap::DescriptorHash::operator()(const DescriptorValue& value) const noexcept {
return XXH3_64bits(value.dwords.data(), sizeof(value.dwords));
}
ImageHeap::ImageHeap(RenderContext& context, PageManager& page_manager)
: m_context(context), m_page_manager(page_manager) {
auto& graphics = context.GetGraphics();
vk::PhysicalDeviceVulkan12Properties limits {};
vk::PhysicalDeviceProperties2 properties {};
properties.pNext = &limits;
graphics.physical_device.getProperties2(&properties);
EXIT_NOT_IMPLEMENTED(std::min({limits.maxDescriptorSetUpdateAfterBindSampledImages,
limits.maxPerStageDescriptorUpdateAfterBindSampledImages,
limits.maxPerStageUpdateAfterBindResources}) <
IR::ImageHeapCapacity);
const vk::DescriptorBindingFlags binding_flags =
vk::DescriptorBindingFlagBits::ePartiallyBound |
vk::DescriptorBindingFlagBits::eUpdateAfterBind |
vk::DescriptorBindingFlagBits::eUpdateUnusedWhilePending;
vk::DescriptorSetLayoutBindingFlagsCreateInfo flags_info {};
flags_info.bindingCount = 1;
flags_info.pBindingFlags = &binding_flags;
const vk::DescriptorSetLayoutBinding binding {0, vk::DescriptorType::eSampledImage,
IR::ImageHeapCapacity,
vk::ShaderStageFlagBits::eAll, nullptr};
vk::DescriptorSetLayoutCreateInfo layout_info {};
layout_info.pNext = &flags_info;
layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::eUpdateAfterBindPool;
layout_info.bindingCount = 1;
layout_info.pBindings = &binding;
EXIT_IF(graphics.device.createDescriptorSetLayout(&layout_info, nullptr, &m_layout) !=
vk::Result::eSuccess);
const vk::DescriptorPoolSize pool_size {vk::DescriptorType::eSampledImage,
IR::ImageHeapCapacity};
vk::DescriptorPoolCreateInfo pool_info {};
pool_info.flags = vk::DescriptorPoolCreateFlagBits::eUpdateAfterBind;
pool_info.maxSets = 1;
pool_info.poolSizeCount = 1;
pool_info.pPoolSizes = &pool_size;
EXIT_IF(graphics.device.createDescriptorPool(&pool_info, nullptr, &m_pool) !=
vk::Result::eSuccess);
vk::DescriptorSetAllocateInfo allocate {};
allocate.descriptorPool = m_pool;
allocate.descriptorSetCount = 1;
allocate.pSetLayouts = &m_layout;
EXIT_IF(graphics.device.allocateDescriptorSets(&allocate, &m_set) != vk::Result::eSuccess);
graphics.image_heap_layout = m_layout;
// Slot word 0 selects element 0, which stays a null image.
m_elements.push_back({.binding = {.layout = vk::ImageLayout::eShaderReadOnlyOptimal}});
m_writes.push_back(0u);
WriteDescriptors();
}
ImageHeap::~ImageHeap() {
for (const auto& memory: m_memories) Unwatch(*memory);
auto& graphics = m_context.GetGraphics();
graphics.image_heap_layout = nullptr;
graphics.device.destroyDescriptorPool(m_pool, nullptr);
graphics.device.destroyDescriptorSetLayout(m_layout, nullptr);
}
bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) {
KYTY_PROFILER_FUNCTION();
const auto size = request.size & ~uint64_t {3};
if (size < MinTableBytes || request.count == 0u || request.step == 0u ||
((request.base | request.first | request.step) & 3u) != 0u) {
return false;
}
auto memory = std::ranges::find_if(m_memories, [&](const auto& candidate) {
return candidate->base == request.base && candidate->size == size;
});
if (memory == m_memories.end()) {
if (!m_context.IsMapped(request.base, size)) return false;
auto created = std::make_unique<Memory>();
created->base = request.base;
created->size = size;
created->words.resize(size / 4u);
created->dead.resize(size / 4u);
created->dirty_pages.assign((Common::AlignUp(request.base + size, TRACKER_PAGE_SIZE) -
Common::AlignDown(request.base, TRACKER_PAGE_SIZE)) /
TRACKER_PAGE_SIZE,
uint8_t {1});
std::scoped_lock lock {m_mutex};
memory = m_memories.insert(m_memories.end(), std::move(created));
}
// A draw refreshes a table and a view takes its changes once, so all programs of the draw see
// one snapshot and one set of kinds.
if ((*memory)->draw != m_draw) {
if (!Refresh(**memory)) return false;
(*memory)->draw = m_draw;
}
auto& views = (*memory)->views;
const auto profile = IR::ImageHeapResource(*request.root, 0u);
auto view = std::ranges::find_if(views, [&](const auto& candidate) {
return candidate->step == request.step && candidate->first == request.first &&
candidate->count == request.count && candidate->profile == profile;
});
if (view == views.end()) {
const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
auto created = std::make_unique<View>(View {.memory = **memory,
.step = request.step,
.first = request.first,
.count = request.count,
.profile = profile,
.resolve_epoch = epoch,
.layout_epoch = epoch});
created->slots.resize(size / 4u + 1u);
created->buffer = std::make_unique<Buffer>(
m_context.GetGraphics(), m_context.GetCommandScheduler(), MemoryUsage::DeviceLocal, 0,
vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer |
vk::BufferUsageFlagBits::eShaderDeviceAddress,
size + 4u);
UpdateProbes(*created, 0u, size);
// The first upload also clears the slots no probe selects.
created->dirty_begin = 0;
created->dirty_end = created->slots.size();
view = views.insert(views.end(), std::move(created));
} else {
// Empty after the first acquire in a draw, since the table refreshes once per draw.
UpdateProbes(**view, (*view)->changed_begin, (*view)->changed_end);
(*view)->changed_begin = (*view)->changed_end = 0;
}
if (std::ranges::find(m_active, view->get()) == m_active.end()) m_active.push_back(view->get());
const auto& selected = **view;
result = {.slots = selected.buffer->BufferDeviceAddress(),
.slot_count = static_cast<uint32_t>(selected.slots.size() - 1u),
.kinds = selected.kinds};
return true;
}
bool ImageHeap::Refresh(Memory& memory) {
// GPU writes leave no fault; reading the table downloads them.
const bool gpu_dirty = m_context.GetBufferCache().HasGpuDirtyBytes(memory.base, memory.size);
const auto pages = memory.dirty_pages.size();
m_runs.clear();
{
std::scoped_lock lock {m_mutex};
// Watch before reading, so a write during the read faults the page again.
ForEachRun(
0, pages, [&](size_t page) { return memory.dirty_pages[page] != 0u; },
[&](size_t begin, size_t end) {
std::fill(memory.dirty_pages.begin() + begin, memory.dirty_pages.begin() + end,
uint8_t {0});
Watch<true>(memory, begin, end);
m_runs.emplace_back(begin, end);
});
}
if (gpu_dirty) m_runs.assign(1u, {0u, pages});
for (size_t run = 0; run < m_runs.size(); ++run) {
const auto [address, end] = PageRange(memory.base, memory.size, m_runs[run].first,
m_runs[run].second);
m_scratch.resize((end - address) / 4u);
if (!LibKernel::Memory::TryReadBufferBacking(address, m_scratch.data(), end - address)) {
std::scoped_lock lock {m_mutex};
for (; run < m_runs.size(); ++run) {
MarkDirty(memory, m_runs[run].first, m_runs[run].second);
}
return false;
}
const auto words = std::span(memory.words).subspan((address - memory.base) / 4u,
m_scratch.size());
size_t first = 0;
size_t last = words.size();
while (first < last && words[first] == m_scratch[first]) ++first;
while (first < last && words[last - 1u] == m_scratch[last - 1u]) --last;
if (first == last) continue;
std::copy(m_scratch.begin() + first, m_scratch.begin() + last, words.begin() + first);
const auto offset = address - memory.base;
for (auto& view: memory.views) {
Widen(view->changed_begin, view->changed_end, offset + first * 4u, offset + last * 4u);
}
}
return true;
}
void ImageHeap::MarkDirty(Memory& memory, size_t begin, size_t end) {
ForEachRun(
begin, end, [&](size_t page) { return memory.dirty_pages[page] == 0u; },
[&](size_t first, size_t last) {
std::fill(memory.dirty_pages.begin() + first, memory.dirty_pages.begin() + last,
uint8_t {1});
Watch<false>(memory, first, last);
});
}
template <bool track>
void ImageHeap::Watch(const Memory& memory, size_t begin, size_t end) {
const auto [address, limit] = PageRange(memory.base, memory.size, begin, end);
m_page_manager.UpdatePageWatchers<track>(address, limit - address);
}
void ImageHeap::Unwatch(const Memory& memory) {
ForEachRun(
0, memory.dirty_pages.size(), [&](size_t page) { return memory.dirty_pages[page] == 0u; },
[&](size_t begin, size_t end) { Watch<false>(memory, begin, end); });
}
void ImageHeap::InvalidateMemory(uint64_t address, uint64_t size) {
std::scoped_lock lock {m_mutex};
for (auto& memory: m_memories) {
const auto begin = std::max(address, memory->base);
const auto end = std::min(address + size, memory->base + memory->size);
if (begin >= end) continue;
const auto origin = Common::AlignDown(memory->base, TRACKER_PAGE_SIZE);
MarkDirty(*memory, (begin - origin) / TRACKER_PAGE_SIZE,
(end - 1u - origin) / TRACKER_PAGE_SIZE + 1u);
}
}
void ImageHeap::UpdateProbes(View& view, uint64_t begin, uint64_t end) {
// A probe reads its descriptor from first + probe * step.
const uint64_t bytes = view.profile.r128 ? 16u : 32u;
auto probe = begin < view.first + bytes ? 0u : (begin - view.first - bytes) / view.step + 1u;
for (; probe < view.count && view.first + probe * view.step < end; ++probe) {
UpdateProbe(view, probe);
}
}
void ImageHeap::UpdateProbe(View& view, uint64_t probe) {
const auto& words = view.memory.words;
const auto offset = (view.first + probe * view.step) / 4u;
DescriptorValue value {.dword_count = 8u};
std::copy_n(words.begin() + offset,
std::min<size_t>(view.profile.r128 ? 4u : 8u, words.size() - offset),
value.dwords.begin());
// A buried word stays null in every view until its descriptor changes.
auto& dead = view.memory.dead[offset];
if (dead != 0u && dead != DescriptorHash {}(value)) dead = 0u;
const auto kind = dead != 0u ? 0u : IR::ImageHeapKind(view.profile, value);
Assign(view, offset, kind == 0u ? 0u : Intern(view, kind, value));
}
void ImageHeap::Assign(View& view, size_t slot, uint32_t entry) {
const auto previous = std::exchange(view.slots[slot], entry);
Release(view, previous);
if (previous != entry) Widen(view.dirty_begin, view.dirty_end, slot, slot + 1u);
}
void ImageHeap::Bury() {
if (m_unmapped.empty()) return;
for (const auto& memory: m_memories) {
for (const auto& view: memory->views) {
bool buried = false;
for (const auto& [value, index]: view->elements) {
const auto& binding = m_elements[index].binding;
if (binding.image_id && std::ranges::any_of(m_unmapped, [&](GuestRange range) {
return ImageRangeOverlaps(binding.desc.info.data, range);
})) {
m_elements[index].stale = buried = true;
}
}
if (!buried) continue;
for (uint64_t probe = 0; probe < view->count; ++probe) {
const auto slot = (view->first + probe * view->step) / 4u;
const auto& element = m_elements[view->slots[slot] & ElementMask];
if (!element.stale) continue;
memory->dead[slot] = DescriptorHash {}(element.value);
Assign(*view, slot, 0u);
}
}
}
m_unmapped.clear();
}
uint32_t ImageHeap::Intern(View& view, uint32_t kind, const DescriptorValue& value) {
const auto arm = static_cast<uint32_t>(std::ranges::find(view.kind_keys, kind) -
view.kind_keys.begin());
if (arm == view.kind_keys.size()) {
view.kind_keys.push_back(kind);
view.kinds.push_back(value);
}
auto [found, inserted] = view.elements.try_emplace(value, 0u);
if (inserted) {
found->second = Allocate();
auto& element = m_elements[found->second];
element.value = value;
element.kind = kind;
view.pending.push_back(found->second);
}
++m_elements[found->second].refs;
return arm << IR::ImageHeapKindShift | found->second;
}
void ImageHeap::Release(View& view, uint32_t entry) {
const auto index = entry & ElementMask;
if (index == 0u || --m_elements[index].refs != 0u) return;
auto& element = m_elements[index];
if (const auto found = view.elements.find(element.value);
found != view.elements.end() && found->second == index) {
view.elements.erase(found);
}
Unreference(element);
view.released.push_back(index);
}
void ImageHeap::Unreference(Element& element) {
if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(element.binding.image_id)) {
image->heap_refs.fetch_sub(1u, std::memory_order_relaxed);
image->tick_accessed_last = m_context.GetCommandScheduler().CurrentTick();
}
}
uint32_t ImageHeap::Allocate() {
auto& scheduler = m_context.GetCommandScheduler();
while (!m_retired.empty() && scheduler.IsFree(m_retired.front().first)) {
m_free.push_back(m_retired.front().second);
m_retired.pop_front();
}
if (m_free.empty()) {
if (m_elements.size() == IR::ImageHeapCapacity) {
EXIT("image heaps exceed %u resident images\n", IR::ImageHeapCapacity);
}
m_elements.emplace_back();
return static_cast<uint32_t>(m_elements.size() - 1u);
}
const auto index = m_free.back();
m_free.pop_back();
m_elements[index] = {};
return index;
}
void ImageHeap::Retire(View& view) {
const auto tick = m_context.GetCommandScheduler().CurrentTick();
for (const auto index: view.released) m_retired.emplace_back(tick, index);
view.released.clear();
}
void ImageHeap::Rebind(RenderExecutor& executor) {
KYTY_PROFILER_FUNCTION();
auto& textures = m_context.GetTextureCache();
const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
for (auto* view: m_active) {
bool stale = false;
const auto mark = [&](uint32_t index) { m_elements[index].stale = stale = true; };
if (view->resolve_epoch != epoch) {
view->resolve_epoch = epoch;
for (const auto& [value, index]: view->elements) {
const auto& binding = m_elements[index].binding;
if (!binding.image_id) continue;
const auto* image = textures.m_slot_images.try_get(binding.image_id);
if (image == nullptr || !image->registered || image->depth_id ||
image->binding.needs_rebind) {
mark(index);
} else {
(void)textures.FindTexture(binding.image_id, binding.desc);
}
}
}
// Discovery applies the color metadata clears written since the last draw.
std::erase_if(view->dcc, [&](uint32_t index) {
const auto& element = m_elements[index];
if (element.refs == 0u || element.stale) return true;
if (executor.ResolveTexture(IR::ImageHeapResource(view->profile, element.kind),
element.value).image_id == element.binding.image_id) {
return false;
}
mark(index);
return true;
});
if (stale) {
// A changed image needs a new element; pending draws still read the old one. The new
// element keeps the old value, since a view takes table changes only when acquired.
std::erase_if(view->elements,
[&](const auto& item) { return m_elements[item.second].stale; });
for (uint64_t probe = 0; probe < view->count; ++probe) {
const auto slot = (view->first + probe * view->step) / 4u;
const auto& element = m_elements[view->slots[slot] & ElementMask];
if (!element.stale) continue;
const auto value = element.value; // Intern may grow m_elements.
Assign(*view, slot, Intern(*view, element.kind, value));
}
}
for (const auto index: view->pending) {
if (m_elements[index].refs == 0u) continue;
auto binding = executor.ResolveTexture(
IR::ImageHeapResource(view->profile, m_elements[index].kind),
m_elements[index].value);
if (binding.image_id) {
auto& image = textures.GetImage(binding.image_id);
image.heap_refs.fetch_add(1u, std::memory_order_relaxed);
binding.layout = image.ReadLayout();
binding.image_view = textures.FindTexture(binding.image_id, binding.desc);
if (binding.desc.info.metadata.kind == ImageMetadataKind::Dcc) {
view->dcc.push_back(index);
}
view->away.push_back(index);
}
m_elements[index].binding = std::move(binding);
m_writes.push_back(index);
}
view->pending.clear();
}
WriteDescriptors();
}
void ImageHeap::WriteDescriptors() {
if (m_writes.empty()) return;
std::vector<vk::DescriptorImageInfo> infos;
std::vector<vk::WriteDescriptorSet> writes(m_writes.size());
infos.reserve(m_writes.size());
for (size_t write = 0; write < m_writes.size(); ++write) {
const auto& binding = m_elements[m_writes[write]].binding;
infos.push_back({nullptr, binding.image_view, binding.layout});
writes[write].dstSet = m_set;
writes[write].dstArrayElement = m_writes[write];
writes[write].descriptorCount = 1;
writes[write].descriptorType = vk::DescriptorType::eSampledImage;
writes[write].pImageInfo = &infos.back();
}
m_context.GetGraphics().device.updateDescriptorSets(static_cast<uint32_t>(writes.size()),
writes.data(), 0, nullptr);
m_writes.clear();
}
void ImageHeap::Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point,
vk::PipelineLayout layout) {
KYTY_PROFILER_FUNCTION();
auto& textures = m_context.GetTextureCache();
const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
for (auto* view: m_active) {
if (view->dirty_begin != view->dirty_end) {
const auto slots = std::span(view->slots).subspan(view->dirty_begin,
view->dirty_end - view->dirty_begin);
auto& stream = m_context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
const auto offset = stream.Copy(slots.data(), slots.size_bytes(), sizeof(uint32_t));
view->buffer->CopyFrom(buffer, stream, offset, view->dirty_begin * sizeof(uint32_t),
slots.size_bytes());
view->dirty_begin = view->dirty_end = 0;
}
Retire(*view);
if (view->layout_epoch != epoch) {
view->layout_epoch = epoch;
view->away.clear();
for (const auto& [value, index]: view->elements) view->away.push_back(index);
}
// This draw's own bindings own the layouts of its attachments and storage images. An image
// this draw's discovery replaced lives until the tick ends, and its slots still select it.
std::erase_if(view->away, [&](uint32_t index) {
const auto& element = m_elements[index];
auto* image = textures.m_slot_images.try_get(element.binding.image_id);
if (element.refs == 0u || image == nullptr) return true;
if (image->binding.is_target || image->binding.shader_write ||
image->binding.force_general) {
return false;
}
const auto& view_info = element.binding.desc.view_info;
image->Transit(image->ReadLayout(), vk::AccessFlagBits2::eShaderRead,
ImageSubresourceRange {view_info.base_level, view_info.level_count,
view_info.base_layer, view_info.layer_count},
buffer.Handle());
return true;
});
}
m_active.clear();
buffer.Handle().bindDescriptorSets(bind_point, layout, IR::ImageHeapSet, 1, &m_set, 0,
nullptr);
}
void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) {
// Descriptors of textures in the range stay null even once it is mapped again.
if (!m_unmapped.empty() && m_unmapped.back().End() == address) {
m_unmapped.back().size += size;
} else {
m_unmapped.push_back({address, size});
}
auto& scheduler = m_context.GetCommandScheduler();
std::scoped_lock lock {m_mutex};
std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
if (address >= memory->base + memory->size || memory->base >= address + size) {
return false;
}
Unwatch(*memory);
for (auto& view: memory->views) {
std::erase(m_active, view.get());
for (const auto& [value, index]: view->elements) {
Unreference(m_elements[index]);
view->released.push_back(index);
}
Retire(*view);
if (scheduler.Active()) {
scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
}
}
return true;
});
}
} // namespace Libs::Graphics
@@ -0,0 +1,138 @@
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
#include "common/common.h"
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
#include <deque>
#include <memory>
#include <mutex>
#include <unordered_map>
#include <utility>
#include <vector>
namespace Libs::Graphics {
class Buffer;
class CommandBuffer;
class PageManager;
class RenderContext;
class RenderExecutor;
struct ImageHeapTestAccess;
// Guest image tables that shaders index by GPU-computed offsets, kept resident in one bindless
// sampled-image array. Each view of a table owns device slot words, one per table dword, that
// select an array element. CPU writes to a table fault in its snapshot, which each draw refreshes
// once; a view takes the changes when a draw first acquires it. Changes to the images behind
// elements bump Image::heap_epoch.
class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource {
public:
ImageHeap(RenderContext& context, PageManager& page_manager);
~ImageHeap();
KYTY_CLASS_NO_COPY(ImageHeap);
// Starts a draw or dispatch, before its programs acquire views.
void BeginDraw() {
++m_draw;
Bury();
}
bool Acquire(const ShaderRecompiler::IR::ImageHeapRequest& request,
ShaderRecompiler::IR::ImageHeapView& result) override;
// Image discovery: resolves new elements and rediscovers changed images.
void Rebind(RenderExecutor& executor);
// Recording: uploads slots, moves resident images to their read layout and binds the array.
void Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point, vk::PipelineLayout layout);
// CPU write fault, on any thread.
void InvalidateMemory(uint64_t address, uint64_t size);
void UnmapMemory(uint64_t address, uint64_t size);
private:
using DescriptorValue = ShaderRecompiler::IR::DescriptorValue;
struct DescriptorHash {
size_t operator()(const DescriptorValue& value) const noexcept;
};
struct Element {
DescriptorValue value;
TextureBinding binding;
uint32_t kind = 0;
uint32_t refs = 0; // Slots of the owning view that select it.
bool stale = false;
};
struct Memory;
struct View {
Memory& memory;
uint64_t step = 0;
uint64_t first = 0;
uint64_t count = 0;
ShaderRecompiler::IR::ImageResource profile;
std::vector<uint32_t> slots; // Plus a trailing null slot.
std::vector<uint32_t> kind_keys;
std::vector<DescriptorValue> kinds; // One representative per kind_keys entry.
std::unordered_map<DescriptorValue, uint32_t, DescriptorHash> elements;
std::vector<uint32_t> pending; // Not yet resolved.
std::vector<uint32_t> released; // No slot selects them after the next upload.
std::vector<uint32_t> away; // Not yet moved to the read layout.
std::vector<uint32_t> dcc; // Color metadata clears apply on each discovery.
std::unique_ptr<Buffer> buffer;
size_t dirty_begin = 0; // Slots to upload.
size_t dirty_end = 0;
size_t changed_begin = 0; // Table bytes changed since it was probed.
size_t changed_end = 0;
uint64_t resolve_epoch = 0;
uint64_t layout_epoch = 0;
};
struct Memory {
uint64_t base = 0;
uint64_t size = 0;
std::vector<uint32_t> words;
std::vector<uint8_t> dirty_pages; // Unwatched, so unread since a write.
std::vector<uint64_t> dead; // Per word, a buried descriptor's hash.
uint64_t draw = 0; // Last draw that refreshed it.
std::vector<std::unique_ptr<View>> views;
};
[[nodiscard]] bool Refresh(Memory& memory);
void MarkDirty(Memory& memory, size_t begin, size_t end);
template <bool track>
void Watch(const Memory& memory, size_t begin, size_t end);
void Unwatch(const Memory& memory);
void UpdateProbes(View& view, uint64_t begin, uint64_t end);
void UpdateProbe(View& view, uint64_t probe);
void Assign(View& view, size_t slot, uint32_t entry);
void Bury();
uint32_t Intern(View& view, uint32_t kind, const DescriptorValue& value);
void Release(View& view, uint32_t entry);
void Unreference(Element& element);
uint32_t Allocate();
void Retire(View& view);
void WriteDescriptors();
RenderContext& m_context;
PageManager& m_page_manager;
std::mutex m_mutex; // Fault-visible Memory state and m_memories.
std::vector<std::unique_ptr<Memory>> m_memories;
std::vector<View*> m_active; // Acquired since the last commit.
uint64_t m_draw = 1;
std::vector<Element> m_elements;
std::vector<uint32_t> m_free;
std::deque<std::pair<uint64_t, uint32_t>> m_retired; // Tick, element.
std::vector<std::pair<size_t, size_t>> m_runs;
std::vector<uint32_t> m_scratch;
std::vector<uint32_t> m_writes; // Elements whose descriptors changed.
std::vector<GuestRange> m_unmapped; // Since the last Bury.
vk::DescriptorSetLayout m_layout = nullptr;
vk::DescriptorPool m_pool = nullptr;
vk::DescriptorSet m_set = nullptr;
friend struct ImageHeapTestAccess;
};
} // namespace Libs::Graphics
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
@@ -353,6 +353,7 @@ struct PipelineCache::ProgramCache {
.user_data = user_data,
.shader_base = params.Base(),
.read_specialization_memory = ReadShaderGuestMemory,
.image_heap = image_heap,
};
if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
runtime.workgroup_counts = input_info.workgroup_counts;
@@ -480,8 +481,8 @@ struct PipelineCache::ProgramCache {
return permutation.handle;
}
explicit ProgramCache(GraphicContext& graphics)
: graphics(graphics),
ProgramCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap)
: graphics(graphics), image_heap(image_heap),
// AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
// signed-zero results; GLSL.std.450 alone does not order -0 below +0.
profile {.maintenance8 = graphics.maintenance8_enabled,
@@ -501,12 +502,14 @@ struct PipelineCache::ProgramCache {
std::unordered_map<ProgramKey, SourceEntry, ProgramKeyHash> programs;
ProgramKey lookup_key;
GraphicContext& graphics;
ShaderRecompiler::IR::ImageHeapSource* image_heap;
const ShaderRecompiler::Spirv::Profile profile;
uint64_t next_shader_id = 0;
};
PipelineCache::PipelineCache(GraphicContext& graphics)
: m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics)),
PipelineCache::PipelineCache(GraphicContext& graphics,
ShaderRecompiler::IR::ImageHeapSource* image_heap)
: m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics, image_heap)),
m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) {
EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
InitializeDriverCache();
@@ -30,6 +30,10 @@ class UserConfig;
struct ComputeShaderInfo;
} // namespace HW
namespace ShaderRecompiler::IR {
class ImageHeapSource;
} // namespace ShaderRecompiler::IR
#pragma pack(push, 1)
struct PipelineStaticParameters {
@@ -104,7 +108,7 @@ struct ShaderProgram {
// The owning renderer serializes access, including saves while the GPU is running.
class PipelineCache {
public:
explicit PipelineCache(GraphicContext& graphics);
PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap);
~PipelineCache();
KYTY_CLASS_NO_COPY(PipelineCache);
void Save();
@@ -114,6 +118,7 @@ public:
vk::Pipeline pipeline = nullptr;
vk::DescriptorSetLayout descriptor_set_layout = nullptr;
bool uses_push_descriptors = false;
bool uses_image_heap = false;
};
struct GraphicsPrograms {
@@ -473,20 +473,23 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
for (const auto& stage: vertex_info) {
const auto native_stage = NativeShaderStage(stage.logical_stage);
AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage);
pipeline.uses_image_heap |= stage.stage.program->info.uses_image_heap;
graphics_stages |= native_stage;
}
if (ps_active) {
EXIT_IF(!ps_input_info->stage);
AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
vk::ShaderStageFlagBits::eFragment);
pipeline.uses_image_heap |= ps_input_info->stage.program->info.uses_image_heap;
}
CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
const vk::PushConstantRange push_constants {graphics_stages, 0,
ShaderRecompiler::IR::NativePushConstantSize};
const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout};
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout;
pipeline_layout_info.setLayoutCount = pipeline.uses_image_heap ? 2u : 1u;
pipeline_layout_info.pSetLayouts = set_layouts.data();
pipeline_layout_info.pushConstantRangeCount = 1;
pipeline_layout_info.pPushConstantRanges = &push_constants;
@@ -615,13 +618,15 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
vk::ShaderStageFlagBits::eCompute);
pipeline.uses_image_heap = input_info.stage.program->info.uses_image_heap;
CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
ShaderRecompiler::IR::NativePushConstantSize};
const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout};
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
pipeline_layout_info.setLayoutCount = 1;
pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout;
pipeline_layout_info.setLayoutCount = pipeline.uses_image_heap ? 2u : 1u;
pipeline_layout_info.pSetLayouts = set_layouts.data();
pipeline_layout_info.pushConstantRangeCount = 1;
pipeline_layout_info.pPushConstantRanges = &push_constants;
+2 -2
View File
@@ -171,6 +171,8 @@ public:
void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
const PipelineCache::Pipeline& pipeline,
std::span<PreparedBindings* const> bindings);
[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
const ShaderRecompiler::IR::DescriptorValue& value);
private:
// False, with nothing drawn, when the draw's indirect records need CPU counts.
@@ -182,8 +184,6 @@ private:
std::optional<PreparedBindings> pixel;
};
[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
const ShaderRecompiler::IR::DescriptorValue& value);
void PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
std::span<RenderColorInfo> colors);
void ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& target,
@@ -311,6 +311,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
input_info.workgroup_counts[axis] = threads / size + (threads % size != 0u);
}
}
m_context.GetImageHeap().BeginDraw();
const auto compute_program =
m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
if (!compute_program) return;
@@ -363,6 +364,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
if (program.info.uses_dma) {
m_context.PrepareBda();
}
if (program.info.uses_image_heap) {
m_context.GetImageHeap().Rebind(*this);
}
RebindImages(bindings);
BindSharedMemory(m_context, input_info, bindings);
RebindBuffers(bindings);
@@ -497,6 +501,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
ShaderComputeInputInfo input_info {};
input_info.async_compute = async_compute;
input_info.indirect_args_address = args_addr;
m_context.GetImageHeap().BeginDraw();
const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
if (!compute_program) return;
@@ -511,6 +516,9 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
m_context.PrepareBda();
}
BindSharedMemory(m_context, input_info, bindings, args_addr);
if (program.info.uses_image_heap) {
m_context.GetImageHeap().Rebind(*this);
}
RebindImages(bindings);
// Acquiring arguments can merge cache buffers; finalize shader bindings afterward.
const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer(
@@ -13,9 +13,10 @@ namespace Libs::Graphics {
RenderContext::RenderContext(GraphicContext& graphics)
: m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
m_pipeline_cache(graphics), m_sampler_cache(graphics),
m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics),
m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) {
m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache),
m_image_heap(*this, m_page_manager) {
EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
}
@@ -64,6 +65,7 @@ bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) no
if (access == PageFaultAccess::Write) {
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
m_image_heap.InvalidateMemory(fault_vaddr, fault_size);
} else {
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
}
@@ -76,6 +78,7 @@ bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) {
}
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.InvalidateMemory(vaddr, size);
m_image_heap.InvalidateMemory(vaddr, size);
return true;
}
@@ -112,6 +115,7 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
}
m_buffer_cache.InvalidateMemory(vaddr, size);
m_texture_cache.UnmapMemory(vaddr, size);
m_image_heap.UnmapMemory(vaddr, size);
std::lock_guard lock(m_mapped_ranges_mutex);
m_mapped_ranges.Subtract(vaddr, size);
};
@@ -12,6 +12,7 @@
#include "graphics/host_gpu/renderer/cache/textureCache.h"
#include "graphics/host_gpu/renderer/commandScheduler.h"
#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
#include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
#include "kernel/eventQueue.h"
@@ -43,6 +44,7 @@ public:
CommandScheduler& GetCommandScheduler() { return m_command_scheduler; }
PipelineCache& GetPipelineCache() { return m_pipeline_cache; }
DescriptorHeap& GetDescriptorHeap() { return m_descriptor_heap; }
ImageHeap& GetImageHeap() { return m_image_heap; }
SamplerCache& GetSamplerCache() { return m_sampler_cache; }
BufferCache& GetBufferCache() { return m_buffer_cache; }
TextureCache& GetTextureCache() { return m_texture_cache; }
@@ -76,6 +78,7 @@ private:
PageManager m_page_manager;
BufferCache m_buffer_cache;
TextureCache m_texture_cache;
ImageHeap m_image_heap;
mutable std::shared_mutex m_mapped_ranges_mutex;
RangeSet m_mapped_ranges;
std::unique_ptr<GuestGpu> m_gpu;
@@ -893,6 +893,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
}
}
auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
buffer.GetContext().GetImageHeap().BeginDraw();
if (draw.IsIndexed()) {
LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
}
+25 -16
View File
@@ -221,7 +221,7 @@ public:
void WaitForSubmitSlot(VideoOutConfig& cfg);
bool Flip(uint32_t micros);
void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out);
void Wait(VideoOutConfig& cfg, int index);
bool IsSafeForRendering(VideoOutConfig& cfg, int index);
private:
enum class RequestState { Reserved, Recording, Ready, Presenting };
@@ -240,6 +240,7 @@ private:
Graphics::Presenter::Frame* frame;
bool premultiplied_alpha;
};
void SignalDoneLocked();
Graphics::Presenter& m_presenter;
Common::Mutex m_mutex;
@@ -250,6 +251,7 @@ private:
std::list<Request> m_cpu_requests;
std::list<Request> m_cancelled_requests;
bool m_processing = false;
bool m_gpu_waiting = false;
uint64_t m_next_request_id = 1;
};
@@ -953,7 +955,7 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe
}
}
}
m_done_cond_var.SignalAll();
SignalDoneLocked();
}
if (cfg.flip_status.flipPendingNum >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
Common::DebugCounters::Add(Common::DebugCounters::Counter::FlipsRejected);
@@ -1033,7 +1035,7 @@ void FlipQueue::Cancel(VideoOutConfig& cfg) {
it = queue->erase(it);
}
}
m_done_cond_var.SignalAll();
SignalDoneLocked();
m_submit_slot_cond_var.SignalAll();
m_submit_cond_var.SignalAll();
m_mutex.Unlock();
@@ -1205,19 +1207,26 @@ void FlipQueue::WaitForSubmitSlot(VideoOutConfig& cfg) {
}
}
void FlipQueue::Wait(VideoOutConfig& cfg, int index) {
Common::LockGuard lock(m_mutex);
auto has_request = [this, &cfg, index] {
auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
return std::any_of(m_requests.begin(), m_requests.end(), matches) ||
std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
};
while (has_request()) {
m_done_cond_var.Wait(&m_mutex);
void FlipQueue::SignalDoneLocked() {
m_done_cond_var.SignalAll();
// A stalled GPU cannot drain and shut down without re-checking under m_mutex, so it is alive here.
if (m_gpu_waiting) {
m_gpu_waiting = false;
m_presenter.Renderer().GetGpu().Wake();
}
}
// A display buffer is safe for rendering once it is neither displayed nor waiting to be.
bool FlipQueue::IsSafeForRendering(VideoOutConfig& cfg, int index) {
Common::LockGuard lock(m_mutex);
auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
const bool safe = cfg.flip_status.currentBuffer != index &&
std::none_of(m_requests.begin(), m_requests.end(), matches) &&
std::none_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
m_gpu_waiting = !safe;
return safe;
}
bool FlipQueue::Flip(uint32_t micros) {
KYTY_PROFILER_BLOCK("FlipQueue::Flip");
@@ -1311,7 +1320,7 @@ bool FlipQueue::Flip(uint32_t micros) {
it = m_requests.erase(it);
}
m_processing = false;
m_done_cond_var.SignalAll();
SignalDoneLocked();
m_submit_slot_cond_var.SignalAll();
m_mutex.Unlock();
for (size_t i = count; i != 0; i--) {
@@ -1694,12 +1703,12 @@ void VideoOutDriver::WaitForSubmitSlot(int handle) {
m_impl->GetFlipQueue().WaitForSubmitSlot(*cfg);
}
void VideoOutDriver::WaitFlipDone(int handle, int index) {
bool VideoOutDriver::IsSafeForRendering(int handle, int index) {
auto* ctx = m_impl->Get(handle);
EXIT_IF(ctx == nullptr);
EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index));
m_impl->GetFlipQueue().Wait(*ctx, index);
return m_impl->GetFlipQueue().IsSafeForRendering(*ctx, index);
}
KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) {
+1 -1
View File
@@ -38,7 +38,7 @@ public:
void CompleteFlip(uint64_t request_id);
void SubmitFlipPreparation(uint64_t request_id);
void WaitForSubmitSlot(int handle);
void WaitFlipDone(int handle, int index);
[[nodiscard]] bool IsSafeForRendering(int handle, int index);
[[nodiscard]] Impl& State() noexcept;
@@ -65,6 +65,9 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
features.shaderStorageImageArrayNonUniformIndexing = VK_TRUE;
features.shaderStorageBufferArrayNonUniformIndexing = VK_TRUE;
features.descriptorBindingSampledImageUpdateAfterBind = VK_TRUE;
features.descriptorBindingUpdateUnusedWhilePending = VK_TRUE;
features.descriptorBindingPartiallyBound = VK_TRUE;
return features;
}
@@ -295,6 +298,14 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
check_feature(features12.shaderStorageBufferArrayNonUniformIndexing,
"shaderStorageBufferArrayNonUniformIndexing",
required_features12.shaderStorageBufferArrayNonUniformIndexing);
check_feature(features12.descriptorBindingSampledImageUpdateAfterBind,
"descriptorBindingSampledImageUpdateAfterBind",
required_features12.descriptorBindingSampledImageUpdateAfterBind);
check_feature(features12.descriptorBindingUpdateUnusedWhilePending,
"descriptorBindingUpdateUnusedWhilePending",
required_features12.descriptorBindingUpdateUnusedWhilePending);
check_feature(features12.descriptorBindingPartiallyBound, "descriptorBindingPartiallyBound",
required_features12.descriptorBindingPartiallyBound);
check_feature(features13.robustImageAccess, "robustImageAccess");
check_feature(robustness2.nullDescriptor, "nullDescriptor");
check_feature(features13.dynamicRendering, "dynamicRendering",
@@ -609,6 +620,8 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
device_features.depthClamp = VK_TRUE;
#endif
device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
device_features.shaderStorageImageMultisample =
supported_features2.features.shaderStorageImageMultisample;
device_features.shaderImageGatherExtended = VK_TRUE;
device_features.independentBlend = VK_TRUE;
device_features.dualSrcBlend = VK_TRUE;
@@ -623,6 +636,11 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
device_features.multiViewport = VK_TRUE;
device_features.fillModeNonSolid = VK_TRUE;
device_features.vertexPipelineStoresAndAtomics = VK_TRUE;
// Storage image sample counts beyond 1x require the feature.
if (!device_features.shaderStorageImageMultisample) {
graphics.physical_device_properties.limits.storageImageSampleCounts =
vk::SampleCountFlagBits::e1;
}
graphics.sample_rate_shading_enabled = true;
device_features.shaderInt64 = VK_TRUE;
device_features.shaderFloat64 =
@@ -169,6 +169,9 @@ uint32_t Builder::AddString(const char* text) {
}
void Builder::AddName(uint32_t target, const char* name) {
if (!m_emit_names) {
return;
}
std::vector<uint32_t> operands = {target};
AppendString(operands, name);
AppendInstruction(m_debug, spv::OpName, operands);
@@ -67,6 +67,7 @@ public:
}
uint32_t AddString(const char* text);
void EnableNames() { m_emit_names = true; }
void AddName(uint32_t target, const char* name);
template <typename... Args>
void AddAnnotation(spv::Op opcode, const Args&... operands) {
@@ -159,6 +160,7 @@ private:
std::map<std::string, uint32_t> m_import_ids;
std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
size_t m_unpatched_phi_incomings = 0;
bool m_emit_names = false;
};
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
@@ -161,6 +161,7 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
if (!kind.has_value()) {
Fail(program, "native shader plan has an invalid image class");
}
if (image.heap) continue;
if (image.mip_count == 0u ||
(image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
Fail(program, "native shader plan has an invalid image mip descriptor count");
@@ -226,6 +227,7 @@ std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo inp
std::snprintf(name, sizeof(name), "rdna2-%016" PRIx64 "-block-byte-pc-plus-one",
program.shader_hash);
state.debug_source = state.builder.AddString(name);
state.builder.EnableNames();
}
const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
state.lane_count =
@@ -120,7 +120,14 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
} else {
EXIT("invalid image resource class");
}
if (image.dimension == ImageDimension::Dim1D || image.dimension == ImageDimension::Dim1DArray) {
state.builder.RequireCapability(sampled == 1 ? spv::CapabilitySampled1D
: spv::CapabilityImage1D);
}
const auto& info = ImageDimensionInfoFor(image.dimension);
if (sampled == 2 && info.multisampled != 0u) {
state.builder.RequireCapability(spv::CapabilityStorageImageMultisample);
}
const auto scalar_type =
image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
return state.builder.Type(spv::OpTypeImage, scalar_type,
@@ -144,14 +151,20 @@ void DecorateNonUniformDescriptor(EmitterState& state, uint32_t id, spv::Capabil
state.builder.RequireCapability(capability);
state.builder.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform);
}
ImageDefinition& ImageDescriptorDefinition(EmitterState& state, const IR::ImageResource& image,
IR::DescriptorBindingKind kind) {
return (image.heap ? state.heap_images : state.images)[IR::ImageBindingIndex(kind)];
}
} // namespace
uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip,
uint32_t array_index) {
const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index);
const auto image = state.builder.AllocateId();
const auto kind = *IR::DescriptorBindingForImage(state.program.info.images[resource]);
state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type,
const auto& resource_info = state.program.info.images[resource];
const auto kind = *IR::DescriptorBindingForImage(resource_info);
state.builder.AddFunction(spv::OpLoad, ImageDescriptorDefinition(state, resource_info, kind).type,
image, pointer);
if (array_index != 0u) {
DecorateNonUniformDescriptor(state, image,
@@ -184,13 +197,13 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
const auto image = LoadImageDescriptor(state, resource, mip, array_index);
const auto sampled_image = state.builder.AllocateId();
const auto kind = *IR::DescriptorBindingForImage(image_resource);
auto& sampled_type = state.images[IR::ImageBindingIndex(kind)].sampled_type;
if (sampled_type == 0) {
sampled_type = state.builder.Type(
spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type);
auto& definition = ImageDescriptorDefinition(state, image_resource,
*IR::DescriptorBindingForImage(image_resource));
if (definition.sampled_type == 0) {
definition.sampled_type = state.builder.Type(spv::OpTypeSampledImage, definition.type);
}
state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
state.builder.AddFunction(spv::OpSampledImage, definition.sampled_type, sampled_image, image,
sampler_id);
if (array_index != 0u || sampler_dynamic) {
DecorateNonUniformDescriptor(state, sampled_image, spv::CapabilitySampledImageArrayNonUniformIndexing);
}
@@ -202,26 +215,31 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t
const auto& image = state.program.info.images.at(resource);
EXIT_IF(mip >= image.mip_count);
const auto kind = IR::DescriptorBindingForImage(image);
EXIT_IF(!kind.has_value());
EXIT_IF(!kind.has_value() || (image.heap && array_index == 0u));
if (array_index == 0u) {
array_index = ConstantU32(state, image.descriptor_index + mip);
}
const auto& definition = state.images[IR::ImageBindingIndex(*kind)];
const auto& definition = ImageDescriptorDefinition(state, image, *kind);
return DescriptorElementPointer(state, definition.pointer_type, definition.variable,
array_index, *kind, resource,
"image descriptor array was not emitted");
}
void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
uint32_t texel, uint32_t array_index) {
uint32_t texel, uint32_t array_index, uint32_t sample) {
const auto& image = state.program.info.images.at(resource);
EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
if (!image.atomic) {
state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
}
const auto EmitWrite = [&](uint32_t mip) {
state.builder.AddFunction(spv::OpImageWrite, LoadImageDescriptor(state, resource, mip, array_index),
coord, texel);
const auto descriptor = LoadImageDescriptor(state, resource, mip, array_index);
if (sample == 0u) {
state.builder.AddFunction(spv::OpImageWrite, descriptor, coord, texel);
} else {
state.builder.AddFunction(spv::OpImageWrite, descriptor, coord, texel,
spv::ImageOperandsSampleMask, sample);
}
};
if (image.mip_mode != IR::ImageMipMode::Dynamic) {
EmitWrite(0);
@@ -389,6 +389,7 @@ Format::BufferFormatInfo ImageConversionFormat(const EmitterState& state,
const auto format = image.conversion_format;
if (format == Prospero::BufferFormat::kInvalid) return {};
const auto info = Format::GetFormatInfo(format);
if (IR::ReadsDepthBits(image)) return info;
const bool raw_signed = IR::RawSignedStorageFormat(image, format) != Prospero::BufferFormat::kInvalid;
EXIT_IF((Prospero::RemapTextureFormat(format) == format && !raw_signed) || info.component_count == 0u ||
info.component_count > 4u);
@@ -424,6 +425,7 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
const auto scalar_type = ImageScalarType(ctx.state, numeric_class);
const bool integer = info.packed_bitfield || info.type == Format::ComponentType::Uint;
uint32_t packed = 0;
if (info.packed_bitfield) {
packed = ctx.state.builder.AllocateId();
@@ -441,8 +443,16 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type,
components[component], texel, component);
// UNorm backing preserves the guest's filtering; scaling precedes swizzle constants.
const auto unorm_max = static_cast<float>((1u << info.component_bits[component]) - 1u);
components[component] = Binary(ctx.state, spv::OpFMul, scalar_type, components[component],
ConstantF32Value(ctx.state, 255.0f));
ConstantF32Value(ctx.state, unorm_max));
if (integer) {
// Depth bits: round back to the stored UNorm code.
components[component] =
Unary(ctx.state, spv::OpConvertFToU, TypeU32(ctx.state),
Binary(ctx.state, spv::OpFAdd, scalar_type, components[component],
ConstantF32Value(ctx.state, 0.5f)));
}
}
}
for (uint32_t component = info.component_count; component < 4u; component++) {
@@ -454,20 +464,19 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
for (uint32_t component = 0; component < 4u; component++) {
const auto selector = (swizzle >> (component * 3u)) & 7u;
if (selector == 1u) {
selected[component] = info.packed_bitfield
selected[component] = integer
? ConstantU32(ctx.state, Format::FormattedConstantBits(info, Format::FormattedSourceKind::One))
: ConstantF32Value(ctx.state, 1.0f);
} else if (selector >= 4u) {
selected[component] = components[selector - 4u];
} else {
selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 0u)
: ZeroF32(ctx.state);
selected[component] = integer ? ConstantU32(ctx.state, 0u) : ZeroF32(ctx.state);
}
}
const auto result = ctx.state.builder.AllocateId();
ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
ImageVectorType(ctx.state, numeric_class, 4), result, selected[0],
selected[1], selected[2], selected[3]);
integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
result, selected[0], selected[1], selected[2], selected[3]);
return info.type == Format::ComponentType::Unorm
? Unary(ctx.state, spv::OpBitcast, TypeF32Vector(ctx.state, 4), result) : result;
}
@@ -691,6 +700,21 @@ auto EmitImageAccess(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t resul
image.indirect_resources.size() < 2u) {
ctx.Fail(inst, "has no indirect image runtime mapping");
}
if (image.heap) {
const auto entry = EmitImageHeapEntry(state, key, image.indirect_mapping_offset);
const auto kinds = static_cast<uint32_t>(image.indirect_resources.size()) - 1u;
// One kind leaves every slot word a bare element.
if (kinds == 1u) return emit(image.indirect_resources[1], entry);
const auto element = Binary(state, spv::OpBitwiseAnd, TypeU32(state), entry,
ConstantU32(state, (1u << IR::ImageHeapKindShift) - 1u));
const auto EmitKind = [&](uint32_t kind) {
return emit(image.indirect_resources[kind + 1u], element);
};
return EmitIndexSwitch(state,
Binary(state, spv::OpShiftRightLogical, TypeU32(state), entry,
ConstantU32(state, IR::ImageHeapKindShift)),
kinds, result_type, EmitKind);
}
const auto selected = EmitIndirectResourceIndex(
state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
const auto SameType = [&](uint32_t lhs, uint32_t rhs) {
@@ -870,14 +894,19 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
}
if (op == IR::ValueOpcode::ImageQueryLod) {
state.builder.RequireCapability(spv::CapabilityImageQuery);
const auto dimension = image.dimension;
const auto sampled = MakeSampledImage(state, mem.resource,
LoadSamplerDescriptor(state, mem.sampler));
const auto lod = state.builder.AllocateId();
state.builder.AddFunction(
spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components,
image.cube));
const auto sampler = LoadSamplerDescriptor(state, mem.sampler);
const auto lod = EmitImageAccess(ctx, inst, TypeF32Vector(state, 2),
[&](uint32_t resource, uint32_t array_index) {
const auto& candidate = state.program.info.images[resource];
const auto query = state.builder.AllocateId();
state.builder.AddFunction(
spv::OpImageQueryLod, TypeF32Vector(state, 2), query,
MakeSampledImage(state, resource, sampler, 0u, array_index),
CoordF32(ctx, mem, *address, 0,
ImageDimensionInfoFor(candidate.dimension).spatial_components,
candidate.cube));
return query;
});
uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
ConstantU32(state, 0)};
for (uint32_t index = 0; index < 2u; index++) {
@@ -932,9 +961,12 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
const auto mip_lod = candidate.mip_mode == IR::ImageMipMode::Dynamic
? LodU32(ctx, mem, *address, mem.image_dimension) : 0u;
const auto coord = CoordU32(ctx, mem, *address, candidate.dimension);
const auto& dimension_info = ImageDimensionInfoFor(candidate.dimension);
const auto sample = dimension_info.multisampled != 0u
? AddressU32(ctx, mem, *address, dimension_info.coordinate_components) : 0u;
const auto texel = StoreTexel(ctx, selected, ctx.Arg(inst, 2),
candidate.numeric_class == Prospero::TextureNumericClass::Uint);
EmitStorageImageWrite(state, resource, mip_lod, coord, texel, array_index);
EmitStorageImageWrite(state, resource, mip_lod, coord, texel, array_index, sample);
});
});
return;
@@ -130,6 +130,7 @@ struct EmitterState {
uint32_t lds_u64_variable = 0;
std::array<uint32_t, 2> scratch_variable {};
std::array<ImageDefinition, IR::ImageBindingCount> images {};
std::array<ImageDefinition, IR::ImageBindingCount> heap_images {};
uint32_t sampler_variable = 0;
uint32_t sampler_type = 0;
uint32_t sampler_pointer_type = 0;
@@ -339,8 +340,9 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
uint32_t mip = 0, uint32_t array_index = 0,
bool sampler_dynamic = false);
// sample is 0 (no id) for single-sampled images.
void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
uint32_t texel, uint32_t array_index);
uint32_t texel, uint32_t array_index, uint32_t sample);
spv::ExecutionModel ExecutionModelForStage(ShaderType stage);
@@ -430,6 +432,8 @@ struct MemoryResourceAccess {
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
uint32_t search_iterations, uint32_t default_resource);
// Slot word of an image heap byte offset; offsets past the heap select the null element.
uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset);
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem,
uint32_t array_index = 0u);
@@ -38,19 +38,39 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
return ret;
}
static uint32_t LoadFlattenedSrt(EmitterState& state, uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
state.flattened_srt_variable, ConstantU32(state, 0), index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
}
uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset) {
const auto Mapping = [&](uint32_t word) {
return LoadFlattenedSrt(state, ConstantU32(state, mapping_offset + word));
};
// Keys past the table select its trailing null slot.
const auto index = EmitUMin32(state, Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
ConstantU32(state, 2u)),
Mapping(0u));
const auto offset = Unary(state, spv::OpUConvert, TypeU64(state),
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
ConstantU32(state, 2u)));
const auto pointer = Unary(state, spv::OpConvertUToPtr, TypePhysicalU32Pointer(state),
Binary(state, spv::OpIAdd, TypeU64(state),
PackU64(state, Mapping(1u), Mapping(2u)), offset));
const auto entry = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, TypeU32(state), entry, pointer,
spv::MemoryAccessAlignedMask, 4u);
return entry;
}
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
uint32_t search_iterations, uint32_t default_resource) {
const auto LoadMapping = [&](uint32_t index) {
const auto pointer = state.builder.AllocateId();
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
pointer, state.flattened_srt_variable, ConstantU32(state, 0),
index);
const auto value = state.builder.AllocateId();
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
return value;
};
const auto mapping = ConstantU32(state, mapping_offset);
const auto count = LoadMapping(mapping);
const auto count = LoadFlattenedSrt(state, mapping);
if (search_iterations == 0u) {
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
ConstantU32(state, 2u));
@@ -58,7 +78,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u));
const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe,
ConstantU32(state, mapping_offset + 1u));
return Select(state, TypeU32(state), valid, LoadMapping(entry),
return Select(state, TypeU32(state), valid, LoadFlattenedSrt(state, entry),
ConstantU32(state, default_resource));
}
const auto entry_at = [&](uint32_t index) {
@@ -76,7 +96,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
ConstantU32(state, 1u));
const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
LoadMapping(entry_at(probe)), key);
LoadFlattenedSrt(state, entry_at(probe)), key);
low = Select(state, TypeU32(state),
Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
@@ -87,9 +107,10 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
ConstantU32(state, 1u)));
Binary(state, spv::OpIEqual, TypeBool(state),
LoadFlattenedSrt(state, entry), key));
const auto resource = LoadFlattenedSrt(
state, Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
}
@@ -206,6 +206,17 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
return state.builder.Type(spv::OpTypeArray, TypeF32(state), ConstantU32(state, count));
}
void DefineImageArray(EmitterState& state, ImageDefinition& definition, const IR::ImageResource& image,
uint32_t count, uint32_t set, uint32_t binding) {
definition.type = ImageType(state, image);
const auto array = state.builder.Type(spv::OpTypeArray, definition.type, ConstantU32(state, count));
definition.variable = state.builder.DefineGlobalVariable(
TypePointer(state, spv::StorageClassUniformConstant, array), spv::StorageClassUniformConstant);
state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationDescriptorSet, set);
state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationBinding, binding);
definition.pointer_type = TypePointer(state, spv::StorageClassUniformConstant, definition.type);
}
void DefineDescriptors(EmitterState& state) {
using Kind = IR::DescriptorBindingKind;
auto& info = state.program.info;
@@ -222,6 +233,14 @@ void DefineDescriptors(EmitterState& state) {
(image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
EXIT("shader image has an invalid descriptor type or mip count");
}
if (image.heap) {
// Heap kinds alias one array; the root itself is never accessed.
auto& definition = state.heap_images[IR::ImageBindingIndex(*kind)];
if (image.indirect_resources.empty() && definition.variable == 0) {
DefineImageArray(state, definition, image, IR::ImageHeapCapacity, IR::ImageHeapSet, 0u);
}
continue;
}
auto& count = counts[static_cast<size_t>(*kind)];
image.descriptor_index = count;
if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = &image;
@@ -331,21 +350,11 @@ void DefineDescriptors(EmitterState& state) {
default: {
EXIT_IF(IR::ImageBindingResourceClass(kind) ==
IR::ImageResourceClass::None);
const auto& image = *image_types[IR::ImageBindingIndex(kind)];
const auto name = "image_" + std::to_string(static_cast<uint32_t>(kind));
auto& definition = state.images[IR::ImageBindingIndex(kind)];
definition.type = ImageType(state, image);
definition.variable = Define(ArrayType(definition.type), name.c_str(),
spv::StorageClassUniformConstant);
definition.pointer_type = TypePointer(
state, spv::StorageClassUniformConstant, definition.type);
if (image.dimension == ImageDimension::Dim1D ||
image.dimension == ImageDimension::Dim1DArray) {
state.builder.RequireCapability(image.resource_class ==
IR::ImageResourceClass::Sampled
? spv::CapabilitySampled1D
: spv::CapabilityImage1D);
}
DefineImageArray(state, definition, *image_types[IR::ImageBindingIndex(kind)],
counts[index], 0u, IR::NativeBinding(state.program.stage, kind));
state.builder.AddName(definition.variable,
("image_" + std::to_string(static_cast<uint32_t>(kind))).c_str());
break;
}
}
@@ -752,7 +761,7 @@ void DefineModule(EmitterState& state) {
if (state.program.info.buffer_u16) {
state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
}
if (state.program.info.uses_dma) {
if (state.program.info.uses_dma || state.program.info.uses_image_heap) {
state.builder.RequireCapability(spv::CapabilityInt64);
state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
@@ -820,7 +829,7 @@ void DefineModule(EmitterState& state) {
state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
}
state.builder.RequireExtension("SPV_KHR_float_controls");
state.builder.AddMemoryModel(state.program.info.uses_dma
state.builder.AddMemoryModel(state.program.info.uses_dma || state.program.info.uses_image_heap
? spv::AddressingModelPhysicalStorageBuffer64
: spv::AddressingModelLogical,
spv::MemoryModelGLSL450);
+20 -7
View File
@@ -169,6 +169,7 @@ struct ImageResource {
bool depth_compare = false;
bool cube = false;
bool r128 = false;
bool heap = false; // Resident image heap root or kind.
uint32_t indirect_root = NoIndirectImage;
uint32_t indirect_mapping_offset = 0;
uint32_t indirect_search_iterations = 0;
@@ -188,6 +189,12 @@ RawSignedStorageFormat(const ImageResource& image, Prospero::BufferFormat format
return Prospero::BufferFormat::kInvalid;
}
// A raw load of a kDepth k16UInt T# reads a D16 surface's codes. Vulkan cannot view D16 as
// R16Uint, so the host binds it as UNorm and the shader converts back.
[[nodiscard]] constexpr bool ReadsDepthBits(const ImageResource& image) {
return image.conversion_format == Prospero::BufferFormat::k16UInt;
}
struct SamplerResource {
static constexpr uint32_t NoIndirectSampler = UINT32_MAX;
@@ -337,7 +344,12 @@ struct StageOutput {
inline constexpr uint32_t FirstImageBinding = 1u;
inline constexpr uint32_t FirstComparisonImageBinding = 22u;
inline constexpr uint32_t FirstStorageImageBinding = 29u;
inline constexpr uint32_t ImageBindingCount = 48u;
inline constexpr uint32_t ImageBindingCount = 50u;
// Resident image heaps share one sampled-image array at binding 0 of this set. A heap slot
// word is kind << ImageHeapKindShift | element; element 0 is the null descriptor.
inline constexpr uint32_t ImageHeapSet = 1u;
inline constexpr uint32_t ImageHeapCapacity = 65536u;
inline constexpr uint32_t ImageHeapKindShift = 20u;
enum class DescriptorBindingKind : uint32_t {
Buffers = 0u,
@@ -351,8 +363,8 @@ enum class DescriptorBindingKind : uint32_t {
Count,
};
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u);
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u);
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 51u);
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 58u);
struct PushData {
static constexpr uint32_t DwordCount = 32;
@@ -413,8 +425,8 @@ DescriptorBindingForImage(const ImageResource& image) {
constexpr uint32_t SampledUintBinding = 8u;
constexpr uint32_t SampledSintBinding = 15u;
constexpr uint32_t StorageFloatBinding = FirstStorageImageBinding;
constexpr uint32_t StorageUintBinding = StorageFloatBinding + 5u;
constexpr uint32_t AtomicUintBinding = StorageUintBinding + 5u;
constexpr uint32_t StorageUintBinding = StorageFloatBinding + 6u;
constexpr uint32_t AtomicUintBinding = StorageUintBinding + 6u;
uint32_t base = 0;
bool sampled = false;
@@ -461,10 +473,10 @@ DescriptorBindingForImage(const ImageResource& image) {
case Decoder::ImageDimension::Dim2D: dimension = 2u; break;
case Decoder::ImageDimension::Dim2DArray: dimension = 3u; break;
case Decoder::ImageDimension::Dim2DMsaa:
if (!sampled) {
if (image.atomic) {
return std::nullopt;
}
dimension = 4u;
dimension = sampled ? 4u : 5u;
break;
case Decoder::ImageDimension::Dim2DMsaaArray:
if (!sampled) {
@@ -523,6 +535,7 @@ struct ShaderInfo {
bool uses_lds = false;
bool uses_gds = false;
bool uses_flattened_srt = false;
bool uses_image_heap = false;
bool uses_dispatch_threads = false;
bool bvh = false;
bool subgroup_ballot = false;
@@ -681,6 +681,32 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
const auto count = table_size >= first + 4u
? std::min((table_size - first - 4u) / step + 1u, domain)
: 0u;
descriptors[resource_index] = {.dword_count = dword_count};
if constexpr (std::is_same_v<Specialization, ResourceSpecialization::Image>) {
const auto& image = program.info.images[resource_index];
// A resident heap maps slot words instead of materializing its entries.
ImageHeapView view;
if (image_heap && runtime.image_heap != nullptr &&
image.resource_class == ImageResourceClass::Sampled &&
image.mip_mode != ImageMipMode::Dynamic &&
runtime.image_heap->Acquire({table_base, table_size, step, first, count, &image},
view)) {
snapshot.specialization_reads.emplace_back(table_base, table_size);
if (view.kinds.empty()) return true;
auto root = root_resource;
root.indirect_root = resource_index;
root.indirect_mapping_offset = static_cast<uint32_t>(snapshot.flattened_srt.size());
root.heap = true;
specializations[resource_index] = root;
snapshot.flattened_srt.insert(snapshot.flattened_srt.end(),
{view.slot_count, static_cast<uint32_t>(view.slots), static_cast<uint32_t>(view.slots >> 32u)});
for (const auto& kind: view.kinds) {
descriptors.push_back(kind);
specializations.push_back(root);
}
return true;
}
}
if (child_count == 0u || (!image_heap && count * child_count > MaxIndirectDescriptorProbes))
return failed("candidate count", fmt::format("rows={} limit={}", count, MaxIndirectDescriptorProbes));
// Affine SMEM offsets wrap in U32 before adding the aligned immediate.
@@ -692,7 +718,6 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
return read_failed("image heap read", 0u, 0u, table_base, 0u, heap_words.size() * sizeof(uint32_t));
}
const auto mapping_offset = snapshot.flattened_srt.size();
descriptors[resource_index] = {.dword_count = dword_count};
snapshot.flattened_srt.resize(mapping_offset + 2u, 0u);
for (uint32_t word = 0; word < count; ++word) {
const auto table_offset = affine ? first + word * step : uint64_t {word} * 4u;
@@ -932,7 +957,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
return SpecializationFail("FMASK requires a direct image load");
}
}
image.conversion_format = ImageConversionFormat(format);
// See ReadsDepthBits; raw loads only, since filtering would blend the codes.
const bool depth_bits =
!storage && base.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt &&
static_cast<Prospero::TileMode>((descriptor.dwords[3] >> 20u) & 0x1fu) ==
Prospero::TileMode::kDepth;
image.conversion_format = depth_bits ? format : ImageConversionFormat(format);
if (storage || image.conversion_format != Prospero::BufferFormat::kInvalid) {
image.shader_swizzle = DescriptorImageSwizzle(descriptor);
}
@@ -940,7 +970,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
const bool raw_sint_storage = raw_sint_format != Prospero::BufferFormat::kInvalid;
if (raw_sint_format == Prospero::BufferFormat::k16UInt) image.conversion_format = format;
image.numeric_class = Prospero::SampledTextureNumericClass(
Prospero::RemapTextureFormat(format));
depth_bits ? Prospero::BufferFormat::k16UNorm : Prospero::RemapTextureFormat(format));
if (!storage && format == Prospero::BufferFormat::k11_11_10UNorm) {
if (image.indirect_root != ImageResource::NoIndirectImage) {
return SpecializationFail("packed UNorm sampling requires a direct image");
@@ -1005,10 +1035,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size()
? snapshot.flattened_srt[root.indirect_mapping_offset]
: 0u;
// A heap maps its slot count and slot address; tables map each key.
size_t mapping_words = 3u;
if (!root.heap)
mapping_words = 1u + size_t {key_count} * (root.indirect_search_iterations == 0u ? 1u : 2u);
if (key_count == 0u ||
static_cast<size_t>(root.indirect_mapping_offset) + 1u +
static_cast<size_t>(key_count) * (root.indirect_search_iterations == 0u ? 1u : 2u) >
snapshot.flattened_srt.size()) {
root.indirect_mapping_offset + mapping_words > snapshot.flattened_srt.size()) {
return SpecializationFail("indirect image specialization has an invalid key mapping");
}
uint32_t exemplar = ImageResource::NoIndirectImage;
@@ -1744,6 +1776,43 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
return BuildResourceSpecialization(program, snapshot, specialization);
}
uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value) {
// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, and
// entries that need shader-side conversion, read as null instead of failing the draw.
const auto format = static_cast<Prospero::BufferFormat>((value.dwords[1] >> 20u) & 0x1ffu);
const auto tile = static_cast<Prospero::TileMode>((value.dwords[3] >> 20u) & 0x1fu);
const auto numeric_class =
Prospero::SampledTextureNumericClass(Prospero::RemapTextureFormat(format));
if (NullImageDescriptor(value) || !ValidImageDescriptor(value, root.r128) ||
DescriptorDimension(value, root.dimension) != root.dimension ||
Prospero::IsFmaskTextureFormat(format) || format == Prospero::BufferFormat::k11_11_10UNorm ||
ImageConversionFormat(format) != Prospero::BufferFormat::kInvalid ||
(root.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt &&
tile == Prospero::TileMode::kDepth) ||
numeric_class == Prospero::TextureNumericClass::Unsupported ||
(root.depth_compare && numeric_class != Prospero::TextureNumericClass::Float)) {
value.dwords.fill(0u);
return 0u;
}
return 1u + static_cast<uint32_t>(numeric_class) * 2u + (DescriptorIsCube(value) ? 1u : 0u);
}
ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind) {
ImageResource resource;
resource.resource_class = ImageResourceClass::Sampled;
resource.read = true;
resource.heap = true;
resource.dimension = root.dimension;
resource.unfiltered_access_only = root.unfiltered_access_only;
resource.depth_compare = root.depth_compare;
resource.r128 = root.r128;
if (kind != 0u) {
resource.numeric_class = static_cast<Prospero::TextureNumericClass>((kind - 1u) / 2u);
resource.cube = ((kind - 1u) & 1u) != 0u;
}
return resource;
}
void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete);
EXIT_IF(program.info.buffers.size() > specialization.buffers.size() ||
@@ -1794,6 +1863,7 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
image.indirect_mapping_offset = source.indirect_mapping_offset;
image.indirect_search_iterations = source.indirect_search_iterations;
image.cube = source.cube;
image.heap = source.heap;
image.indirect_resources.clear();
}
for (uint32_t index = 0; index < images.size(); index++) {
@@ -31,6 +31,7 @@ struct ResourceSpecialization {
uint32_t indirect_search_iterations = 0;
bool cube = false;
bool fmask = false;
bool heap = false;
bool operator==(const Image&) const = default;
};
@@ -59,6 +60,13 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
// Applies an already-derived specialization to native IR before layout and emission.
void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization);
// Kind of an image heap entry as root sees it, or 0 (and a zeroed value) when root reads it as
// null. Entries of one kind specialize identically.
uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value);
// The sampled resource that root resolves entries of kind as; kind 0 gives the identity of
// root's view of a heap.
ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind);
} // namespace Libs::Graphics::ShaderRecompiler::IR
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */
@@ -594,6 +594,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
};
info.uses_flattened_srt |= std::ranges::any_of(info.buffers, uses_mapping) ||
std::ranges::any_of(info.images, uses_mapping);
info.uses_image_heap = std::ranges::any_of(info.images, &ImageResource::heap);
switch (program.stage) {
case ShaderType::Vertex:
case ShaderType::Local: CollectVertexInputs(input_info.vertex, info, inputs.components); break;
@@ -12,6 +12,33 @@ class Value;
using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span<uint32_t> values);
// Image descriptors at base + first + k * step, k < count, within size bytes that a sampled
// root selects by GPU-computed byte offset. The host keeps them resident instead of
// materializing them.
struct ImageHeapRequest {
uint64_t base = 0;
uint64_t size = 0;
uint64_t step = 0;
uint64_t first = 0;
uint64_t count = 0;
const ImageResource* root = nullptr;
};
struct ImageHeapView {
uint64_t slots = 0; // Device address of one slot per heap dword.
uint32_t slot_count = 0; // Keys past it select a trailing null slot.
std::span<const DescriptorValue> kinds; // One representative descriptor per kind.
};
class ImageHeapSource {
public:
// False when the host does not keep the table resident.
virtual bool Acquire(const ImageHeapRequest& request, ImageHeapView& view) = 0;
protected:
~ImageHeapSource() = default;
};
struct SrtRuntime {
std::span<const uint32_t> user_data;
uint64_t shader_base = 0;
@@ -20,6 +47,7 @@ struct SrtRuntime {
SrtMemoryReader read_specialization_memory = nullptr;
std::span<const uint32_t> workgroup_counts;
uint64_t indirect_args_address = 0;
ImageHeapSource* image_heap = nullptr;
};
class SrtReadCapture {
+5
View File
@@ -410,6 +410,11 @@ std::filesystem::path GetRealFilename(const std::string& mounted_file_name) {
return g_mount_points->ResolvePath(mounted_file_name);
}
bool IsOpenDescriptor(int d) {
const auto* file = g_files->GetFile(d);
return file != nullptr && file->opened;
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
PRINT_NAME();
+2
View File
@@ -56,6 +56,8 @@ void Mount(const std::filesystem::path& folder, const std::stri
void Umount(const std::string& folder_or_point);
// Returns an empty path when the guest path has no mounted filesystem.
std::filesystem::path GetRealFilename(const std::string& mounted_file_name);
// Returns whether d is an open file, directory or device descriptor.
bool IsOpenDescriptor(int d);
int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode);
int KYTY_SYSV_ABI KernelClose(int d);
+21 -1
View File
@@ -30,6 +30,7 @@
#include "common/emulatorConfig.h"
#include "common/logging/log.h"
#include "common/threads.h"
#include "kernel/fileSystem.h"
#include "kernel/pthread.h"
#include "libs/errno.h"
#include "libs/libs.h"
@@ -465,7 +466,7 @@ Network::Id Network::HttpCreateTemplate(Id http_ctx_id, const char* user_agent,
HttpTemplate tn {};
tn.used = true;
tn.http_ver = http_ver;
tn.user_agent = std::string(user_agent);
tn.user_agent = user_agent != nullptr ? user_agent : "";
tn.is_auto_proxy_conf = is_auto_proxy_conf;
tn.http_ctx_id = http_ctx_id.GetId();
tn.nonblock = false;
@@ -2577,6 +2578,7 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
std::vector<std::pair<int, NativeSocket>> descriptors;
std::vector<std::shared_ptr<SocketTransport>> stream_reads;
std::vector<std::pair<void*, int>> ready_files;
int host_nfds = 0;
for (int fd = 0; fd < nfds; fd++) {
const bool read = GuestFdIsSet(readfds, fd);
@@ -2585,6 +2587,16 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
if (!read && !write && !except) {
continue;
}
// FreeBSD polls files and devices ready to read and write at once.
if (!IsSocket(fd) && LibKernel::FileSystem::IsOpenDescriptor(fd)) {
if (read) {
ready_files.emplace_back(readfds, fd);
}
if (write) {
ready_files.emplace_back(writefds, fd);
}
continue;
}
NativeSocket socket = INVALID_NATIVE_SOCKET;
std::shared_ptr<SocketTransport> transport;
#if !defined(_WIN32)
@@ -2636,6 +2648,10 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
host_timeout.tv_usec = static_cast<decltype(host_timeout.tv_usec)>(guest_timeout->tv_usec);
host_timeout_ptr = &host_timeout;
}
if (!ready_files.empty()) {
host_timeout = {};
host_timeout_ptr = &host_timeout;
}
int result = 0;
#if defined(_WIN32)
@@ -2672,6 +2688,10 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
result++;
}
}
for (const auto& [fds, fd]: ready_files) {
GuestFdSet(fds, fd);
result++;
}
return result;
}
+13
View File
@@ -1,4 +1,5 @@
#include "common/abi.h"
#include "libs/network.h"
#include "loader/symbolDatabase.h"
#include <array>
@@ -162,6 +163,17 @@ void TestBuildHonoursOption(HttpUriParse parse, HttpUriBuild build) {
CHECK(BuildWith(build, element, 0) == url);
}
// Cyberpunk 2077 (PPSA03974) creates a template with a NULL user agent.
void TestTemplateWithoutUserAgent() {
using namespace Libs::Network;
Initialize();
const int pool = Net::NetPoolCreate("http", 0x10000, 0);
const int ssl = Ssl::SslInit(0x10000);
const int http = Http::HttpInit(pool, ssl, 0x10000);
CHECK(Http::HttpCreateTemplate(http, nullptr, 2, 1) > 0);
Shutdown();
}
} // namespace
int main() {
@@ -176,5 +188,6 @@ int main() {
TestAbsentQuery(parse);
TestEmptyUri(parse);
TestPresentQuery(parse);
TestTemplateWithoutUserAgent();
return failures == 0 ? 0 : 1;
}
+13 -1
View File
@@ -324,6 +324,7 @@ void CheckSaveRename(const std::filesystem::path &root,
}
void TestRandomDevices() {
namespace Net = Libs::Network::Net;
for (const auto* path : {"/dev/urandom", "/dev/random"}) {
const int fd = FileSystem::KernelOpen(path, 0, 0);
FileSystem::FileStat stat {};
@@ -333,7 +334,18 @@ void TestRandomDevices() {
std::array<uint8_t, 32> entropy {};
Check(FileSystem::KernelRead(fd, entropy.data(), entropy.size()) == entropy.size(),
"character device supplies requested entropy");
// OpenSSL RAND_poll selects the device before reading it; a null timeout must not block.
std::array<uint64_t, 16> readable {};
readable[fd / 64] = uint64_t {1} << (fd % 64);
auto writable = readable;
const auto requested = readable;
Check(Net::Select(fd + 1, readable.data(), writable.data(), nullptr, nullptr) == 2 &&
readable == requested && writable == requested,
"select reports an entropy source ready at once");
Check(FileSystem::KernelClose(fd) == OK, "close entropy source");
Check(Net::Select(fd + 1, readable.data(), nullptr, nullptr, nullptr) == -1 &&
*Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EBADF,
"select rejects a closed descriptor");
}
}
@@ -2453,8 +2465,8 @@ int main(int argc, char** argv) {
TestAioBatches();
CheckSaveRename(temporary.Path(), "first-save");
CheckSaveRename(temporary.Path(), "replacement-save");
FileSystem::Shutdown();
CheckSocketWakeup();
FileSystem::Shutdown();
CheckEtherAddressFormatting();
#if defined(__linux__)
TestNetCtlHostIpv4();
+142 -12
View File
@@ -436,6 +436,128 @@ void TestInvariantIndirectImageMaterialization(bool nested) {
"malformed image table tracking was not transactional");
}
void TestResidentImageHeap() {
struct FakeImageHeap final : ImageHeapSource {
bool accept = true;
std::vector<DescriptorValue> kinds;
std::vector<ImageHeapRequest> requests;
bool Acquire(const ImageHeapRequest &request, ImageHeapView &view) override {
requests.push_back(request);
if (!accept) return false;
view = {.slots = 0x123456789abcull, .slot_count = 36u, .kinds = kinds};
return true;
}
};
auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u);
fixture->PlanAndTrack();
const auto plan = ExtractResourcePlan(fixture->program);
std::array<uint32_t, 8> descriptor{};
descriptor[0] = 0x20u;
descriptor[1] = static_cast<uint32_t>(Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
descriptor[2] = 3u | (3u << 14u);
descriptor[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
auto cube = descriptor;
cube[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kCube) << 28u);
std::array<uint32_t, 16> data{0x3000u, 4u << 16u, 0x100000u, 0u,
0x2000u, 48u << 16u, 3u, 0u};
std::copy_n(data.begin(), 4u, data.begin() + 12u);
LinearTestMemory memory;
memory.fail_address = 0x3000u;
memory.watched_address = 0x2000u;
std::copy(descriptor.begin(), descriptor.end(), memory.words.begin() + 0x1010u / 4u);
FakeImageHeap heap;
heap.kinds = {{descriptor, 8u}, {cube, 8u}};
SrtRuntime runtime{.user_data = data, .userdata = &memory,
.read_specialization_memory = ReadLinearTestMemory, .image_heap = &heap};
ResourceSnapshot snapshot;
ResourceSpecialization specialization;
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
heap.requests.size() == 1u && heap.requests[0].base == 0x2000u &&
heap.requests[0].size == 144u && heap.requests[0].step == 16u &&
heap.requests[0].first == 0u && heap.requests[0].count == 9u &&
heap.requests[0].root == &plan.info.images[0] &&
memory.watched_reads == 0u && snapshot.images.size() == 3u,
"resident image heap was materialized entry by entry");
const auto &root = specialization.images[0];
const auto mapping = root.indirect_mapping_offset;
Check(root.heap && root.indirect_root == 0u && root.indirect_search_iterations == 0u &&
snapshot.flattened_srt.size() == mapping + 3u &&
snapshot.flattened_srt[mapping] == 36u &&
snapshot.flattened_srt[mapping + 1u] == 0x56789abcu &&
snapshot.flattened_srt[mapping + 2u] == 0x1234u &&
specialization.images[1].heap && specialization.images[1].indirect_root == 0u &&
!specialization.images[1].cube && specialization.images[2].heap &&
specialization.images[2].cube && snapshot.images[2].dwords == cube &&
std::ranges::find(snapshot.specialization_reads,
std::pair<uint64_t, uint64_t>{0x2000u, 144u}) !=
snapshot.specialization_reads.end(),
"resident image heap lost its slot mapping, typed kinds or read dependency");
auto &program = fixture->program;
ApplyResourceSpecialization(program, specialization);
ShaderComputeInputInfo compute{};
CollectShaderInfo(program, {.compute = &compute});
Check(program.info.uses_image_heap && program.info.images.size() == 3u &&
program.info.images[0].indirect_resources == std::vector<uint32_t>({0u, 1u, 2u}) &&
std::ranges::all_of(program.info.images, &ImageResource::heap),
"applied heap kinds lost their root or heap flag");
heap.kinds.clear();
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
snapshot.images.size() == 1u && !specialization.images[0].heap &&
specialization.images[0].indirect_root == ImageResource::NoIndirectImage,
"a resident heap without kinds did not read as one null image");
heap.accept = false;
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
heap.requests.size() == 3u && memory.watched_reads == 1u &&
snapshot.images.size() == 2u && !specialization.images[0].heap &&
snapshot.images[1].dwords == descriptor,
"a declined heap was not materialized entry by entry");
}
void TestImageHeapKinds() {
using Format = Libs::Graphics::Prospero::BufferFormat;
using Type = Libs::Graphics::Prospero::ImageType;
ImageResource root;
root.resource_class = ImageResourceClass::Sampled;
root.dimension = Decoder::ImageDimension::Dim2DArray;
root.source = 3u;
const auto descriptor = [](Format format, Type type) {
DescriptorValue value{.dword_count = 8u};
value.dwords[0] = 0x20u;
value.dwords[1] = static_cast<uint32_t>(format) << 20u;
value.dwords[2] = 3u | (3u << 14u);
value.dwords[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(type) << 28u);
return value;
};
const auto kind_of = [&](Format format, Type type) {
auto value = descriptor(format, type);
return ImageHeapKind(root, value);
};
const auto float_kind = kind_of(Format::k32_32_32_32Float, Type::kColor2DArray);
const auto uint_kind = kind_of(Format::k32UInt, Type::kColor2DArray);
const auto cube_kind = kind_of(Format::k32_32_32_32Float, Type::kCube);
auto volume = descriptor(Format::k32_32_32_32Float, Type::kColor3D);
Check(float_kind != 0u && uint_kind != 0u && cube_kind != 0u && float_kind != uint_kind &&
float_kind != cube_kind && kind_of(Format::kInvalid, Type::kColor2DArray) == 0u &&
ImageHeapKind(root, volume) == 0u &&
std::ranges::all_of(volume.dwords, [](uint32_t word) { return word == 0u; }),
"heap kinds did not separate typed entries or null mismatched ones");
const auto resource = ImageHeapResource(root, uint_kind);
const auto cube_resource = ImageHeapResource(root, cube_kind);
auto other = root;
other.source = 9u;
Check(resource.numeric_class == Libs::Graphics::Prospero::TextureNumericClass::Uint &&
!resource.cube && cube_resource.cube &&
cube_resource.numeric_class == Libs::Graphics::Prospero::TextureNumericClass::Float &&
resource.resource_class == ImageResourceClass::Sampled && resource.read &&
ImageHeapResource(root, 0u) == ImageHeapResource(other, 0u),
"heap kind resources lost their type or kept root identity");
root.depth_compare = true;
Check(kind_of(Format::k32UInt, Type::kColor2DArray) == 0u,
"depth comparison accepted an integer heap entry");
}
void TestBoundedImageViewEligibility() {
using Type = Libs::Graphics::Prospero::ImageType;
auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u);
@@ -8547,16 +8669,16 @@ void TestShaderInfoCollection() {
void TestImageBindingAbi() {
using NumericClass = Libs::Graphics::Prospero::TextureNumericClass;
Check(ImageBindingCount == 48u &&
Check(ImageBindingCount == 50u &&
static_cast<uint32_t>(DescriptorBindingKind::Buffers) == 0u &&
static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u &&
static_cast<uint32_t>(DescriptorBindingKind::Gds) == 50u &&
static_cast<uint32_t>(DescriptorBindingKind::BdaPagetable) == 51u &&
static_cast<uint32_t>(DescriptorBindingKind::FaultBuffer) == 52u &&
static_cast<uint32_t>(DescriptorBindingKind::FlattenedSrt) == 53u &&
static_cast<uint32_t>(DescriptorBindingKind::ShaderData) == 54u &&
static_cast<uint32_t>(DescriptorBindingKind::SharedMemory) == 55u &&
static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u,
static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 51u &&
static_cast<uint32_t>(DescriptorBindingKind::Gds) == 52u &&
static_cast<uint32_t>(DescriptorBindingKind::BdaPagetable) == 53u &&
static_cast<uint32_t>(DescriptorBindingKind::FaultBuffer) == 54u &&
static_cast<uint32_t>(DescriptorBindingKind::FlattenedSrt) == 55u &&
static_cast<uint32_t>(DescriptorBindingKind::ShaderData) == 56u &&
static_cast<uint32_t>(DescriptorBindingKind::SharedMemory) == 57u &&
static_cast<uint32_t>(DescriptorBindingKind::Count) == 58u,
"native descriptor binding anchors changed");
const std::array sampled_dimensions{
@@ -8615,6 +8737,8 @@ void TestImageBindingAbi() {
CheckBinding(ImageResourceClass::Storage, numeric_class, dimension,
false);
}
CheckBinding(ImageResourceClass::Storage, numeric_class,
Decoder::ImageDimension::Dim2DMsaa, false);
}
for (const auto dimension : storage_dimensions) {
CheckBinding(ImageResourceClass::Storage, NumericClass::Uint, dimension,
@@ -8658,11 +8782,15 @@ void TestImageBindingAbi() {
image.numeric_class = NumericClass::Sint;
Check(Invalid(image), "signed storage image received a descriptor binding");
image.numeric_class = NumericClass::Float;
image.dimension = Decoder::ImageDimension::Dim2DMsaa;
image.dimension = Decoder::ImageDimension::Dim2DMsaaArray;
Check(Invalid(image),
"multisampled storage image received a descriptor binding");
image.dimension = Decoder::ImageDimension::Dim2D;
"multisampled storage array received a descriptor binding");
image.dimension = Decoder::ImageDimension::Dim2DMsaa;
image.numeric_class = NumericClass::Uint;
image.atomic = true;
Check(Invalid(image), "multisampled atomic image received a descriptor binding");
image.numeric_class = NumericClass::Float;
image.dimension = Decoder::ImageDimension::Dim2D;
Check(Invalid(image), "float atomic image received a descriptor binding");
}
@@ -8790,6 +8918,8 @@ int main() {
Run("dynamic storage mips", TestDynamicStorageMipTracking);
Run("invariant indirect images", [] { TestInvariantIndirectImageMaterialization(true); });
Run("unbounded material image keys", [] { TestInvariantIndirectImageMaterialization(false); });
Run("resident image heap", TestResidentImageHeap);
Run("image heap kinds", TestImageHeapKinds);
Run("bounded image view eligibility", TestBoundedImageViewEligibility);
Run("exact unfiltered image candidates", TestExactUnfilteredImageCandidates);
Run("waterfall image table", TestWaterfallImageTable);
+700 -6
View File
@@ -39,6 +39,7 @@
#include "graphics/host_gpu/renderer/sync.h"
#include "graphics/host_gpu/vulkanCommon.h"
#include <vulkan/vulkan_format_traits.hpp>
#include "graphics/presentation/videoOut.h"
#include "graphics/presentation/window/windowInternal.h"
#include "graphics/shader/recompiler/BufferFormat.h"
#include "graphics/shader/recompiler/ShaderRecompiler.h"
@@ -227,6 +228,18 @@ struct TileManagerTestAccess {
}
};
struct ImageHeapTestAccess {
static size_t ElementCount(const ImageHeap &heap) { return heap.m_elements.size(); }
static size_t MemoryCount(const ImageHeap &heap) { return heap.m_memories.size(); }
static const auto &Views(const ImageHeap &heap) { return heap.m_memories.at(0)->views; }
static const auto &Element(const ImageHeap &heap, uint32_t entry) {
return heap.m_elements[entry & ((1u << ShaderRecompiler::IR::ImageHeapKindShift) - 1u)];
}
static ImageId ImageOf(const ImageHeap &heap, uint32_t entry) {
return Element(heap, entry).binding.image_id;
}
};
struct TextureCacheTestAccess {
static_assert(std::same_as<decltype(TextureCache::m_slot_images),
Common::SlotVector<Image>>);
@@ -1619,6 +1632,17 @@ void CheckRectListShaders() {
ValidateSpirv(name, position_only.evaluation);
}
// Words without OpName/OpString/OpLine, which only graphics debug dumps emit.
size_t SpirvCodeWords(const std::vector<u32> &spirv) {
size_t words = spirv.size();
for (size_t offset = 5; offset < spirv.size(); offset += spirv[offset] >> 16u) {
const auto opcode = static_cast<spv::Op>(spirv[offset] & 0xffffu);
if (opcode == spv::OpName || opcode == spv::OpString || opcode == spv::OpLine)
words -= spirv[offset] >> 16u;
}
return words;
}
void CheckSpirvEmission(const TestCase &test, const std::vector<u32> &spirv) {
std::vector<u32> result_types(spirv[3]);
for (size_t offset = 5; offset < spirv.size();) {
@@ -1679,6 +1703,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
options.stage = ShaderType::Compute;
options.profile = test.profile;
options.dump_ir = true;
options.emit_debug_info = true; // Checks find variables and functions by OpName.
auto compute_info = test.compute_info;
compute_info.host_subgroup_size = host_subgroup_size;
options.input_info.compute = &compute_info;
@@ -1750,7 +1775,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
result.program.shader_info_complete = false;
ShaderRecompiler::IR::CollectShaderInfo(result.program, options.input_info);
result.spirv = ShaderRecompiler::Spirv::EmitProgram(
result.program, options.input_info, 0u, *options.profile);
result.program, options.input_info, 0u, *options.profile, true);
const auto shader_data = result.program.bindings.descriptor_counts[
static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)];
Require(test.name, "binding layout",
@@ -1764,11 +1789,12 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
ValidateSpirv(test.name, result.spirv);
CheckSpirvEmission(test, result.spirv);
if (test.max_spirv_words != 0u) {
Require(test.name, "emitted shader size", result.spirv.size() <= test.max_spirv_words,
"SPIR-V words=" + std::to_string(result.spirv.size()) +
const auto words = SpirvCodeWords(result.spirv);
Require(test.name, "emitted shader size", words <= test.max_spirv_words,
"SPIR-V words=" + std::to_string(words) +
", limit=" + std::to_string(test.max_spirv_words));
std::printf("[compute] %s wave=%u SPIR-V words=%zu\n", test.name,
result.program.wave_size, result.spirv.size());
result.program.wave_size, words);
}
if (test.expected_buffer_resources) {
std::vector<u32> resources;
@@ -5136,6 +5162,289 @@ public:
std::printf("[host] %-32s ok\n", name);
}
void CheckResidentImageHeap() {
constexpr const char *name = "ResidentImageHeap";
constexpr uintptr_t base = 0x0000000206800000ull;
constexpr uint64_t allocation_size = 0x200000;
constexpr uint64_t table = base + 0x100000;
constexpr uint64_t idle_table = base + 0x180000;
constexpr uint64_t table_size = 0x10000;
namespace IR = ShaderRecompiler::IR;
using Format = Prospero::BufferFormat;
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, allocation_size, 0, &direct_offset) == 0,
"image heap allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset,
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
"image heap mapping failed");
std::memset(mapped, 0, allocation_size);
// A 4x4 linear 2D texture per index.
const auto descriptor = [](uint32_t texture, Format format) {
const uint64_t address = base + 0x10000u + texture * 0x1000u;
std::array<uint32_t, 8> words{};
words[0] = static_cast<uint32_t>(address >> 8u);
words[1] = static_cast<uint32_t>(address >> 40u) | (static_cast<uint32_t>(format) << 20u);
words[2] = 3u | (3u << 14u);
words[3] = DstSel(4, 5, 6, 7) |
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
return words;
};
const auto put = [&](uint64_t probe, const std::array<uint32_t, 8> &words) {
std::memcpy(reinterpret_cast<void *>(table + probe * 32u), words.data(), sizeof(words));
};
put(0, descriptor(0, Format::k8_8_8_8UNorm));
put(1, descriptor(0, Format::k8_8_8_8UNorm));
put(5, descriptor(1, Format::k8_8_8_8UNorm));
{
RenderContext context(m_runtime_context);
auto &scheduler = context.GetCommandScheduler();
HW::Context registers{};
HW::UserConfig user_config{};
HW::Shader shaders{};
scheduler.Begin(registers, user_config, shaders);
context.MapMemory(base, allocation_size);
auto &heap = context.GetImageHeap();
auto &executor = context.GetRenderExecutor();
auto &textures = context.GetTextureCache();
auto &device = context.GetGraphics().device;
const vk::DescriptorSetLayoutCreateInfo empty_info{};
vk::DescriptorSetLayout empty = nullptr;
vk::PipelineLayout layout = nullptr;
Require(name, "set 0 layout",
device.createDescriptorSetLayout(&empty_info, nullptr, &empty) ==
vk::Result::eSuccess,
"could not create the set 0 layout");
const std::array set_layouts{empty, context.GetGraphics().image_heap_layout};
vk::PipelineLayoutCreateInfo layout_info{};
layout_info.setLayoutCount = 2;
layout_info.pSetLayouts = set_layouts.data();
Require(name, "pipeline layout",
device.createPipelineLayout(&layout_info, nullptr, &layout) == vk::Result::eSuccess,
"could not create a pipeline layout with the heap set");
IR::ImageResource sampled{};
sampled.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
sampled.unfiltered_access_only = false;
auto unfiltered = sampled;
unfiltered.unfiltered_access_only = true;
IR::ImageHeapView view{};
const auto acquire = [&](const IR::ImageResource &root, uint64_t address = table,
uint64_t count = table_size / 32u) {
return heap.Acquire({address, table_size, 32u, 0u, count, &root}, view);
};
const auto draw = [&] {
heap.Rebind(executor);
heap.Commit(scheduler.Current(), vk::PipelineBindPoint::eCompute, layout);
scheduler.Finish();
heap.BeginDraw();
};
const auto write = [&](uint64_t probe, const std::array<uint32_t, 8> &words) {
Libs::LibKernel::Memory::WriteBacking(table + probe * 32u, words.data(), sizeof(words));
Require(name, "table write", context.InvalidateMemory(table + probe * 32u, sizeof(words)),
"the table was not mapped");
};
Require(name, "first view",
acquire(sampled) && view.kinds.size() == 1u && view.slot_count == table_size / 4u,
"a 64-KiB table was not kept resident");
const auto &views = ImageHeapTestAccess::Views(heap);
const auto &filtered = *views[0];
Require(name, "slot words",
filtered.slots[0] != 0u && filtered.slots[8] == filtered.slots[0] &&
filtered.slots[40] != 0u && filtered.slots[40] != filtered.slots[0] &&
filtered.slots[1] == 0u && filtered.slots.back() == 0u &&
ImageHeapTestAccess::ElementCount(heap) == 3u &&
ImageHeapTestAccess::Element(heap, 0u).binding.layout ==
vk::ImageLayout::eShaderReadOnlyOptimal,
"probes did not share equal descriptors, left a non-null gap or wrote the null "
"element without a sampled layout");
Require(name, "second view",
acquire(unfiltered) && views.size() == 2u &&
ImageHeapTestAccess::ElementCount(heap) == 5u,
"a second root profile did not get its own view");
const auto &idle = *views[1];
draw();
const auto resident = ImageHeapTestAccess::ImageOf(heap, filtered.slots[0]);
Require(name, "resident images",
resident && textures.GetImage(resident).heap_refs.load() == 2u,
"resolved elements did not pin their image");
// Only acquired views take table changes; the idle one must not allocate.
const auto idle_slot = idle.slots[40];
for (uint32_t texture = 2; texture < 4u; ++texture) {
write(5, descriptor(texture, Format::k8_8_8_8UNorm));
Require(name, "changed table", acquire(sampled), "a changed table was declined");
draw();
}
Require(name, "idle view",
idle.slots[40] == idle_slot && ImageHeapTestAccess::ElementCount(heap) <= 6u,
"an idle view took table changes or leaked released elements");
Require(name, "reactivated view", acquire(unfiltered), "the idle view was declined");
draw();
Require(name, "deferred probes",
ImageHeapTestAccess::ImageOf(heap, idle.slots[40]) ==
ImageHeapTestAccess::ImageOf(heap, filtered.slots[40]) &&
ImageHeapTestAccess::ElementCount(heap) <= 7u,
"a reactivated view missed the changes made while it was idle");
// All programs of one draw see the table and kinds of its first acquire.
Require(name, "draw kinds", acquire(sampled), "the view was declined");
write(7, descriptor(12, Format::k8_8_8_8UInt));
Require(name, "stable kinds",
acquire(sampled) && view.kinds.size() == 1u && filtered.slots[56] == 0u,
"a later program of a draw saw a kind the first one lacked");
draw();
Require(name, "next draw kinds",
acquire(sampled) && view.kinds.size() == 2u &&
filtered.slots[56] >> IR::ImageHeapKindShift == 1u,
"the next draw did not take the new kind");
draw();
Require(name, "bounded view",
acquire(sampled, table, 4u) && views.size() == 3u && views[2]->slots[0] != 0u &&
views[2]->slots[40] == 0u,
"a view probed descriptors past the guest table bound");
draw();
// Only leaving the read state invalidates resident images.
auto &image = textures.GetImage(resident);
const auto epoch = Libs::Graphics::Image::heap_epoch.load();
for (uint32_t pass = 0; pass < 2u; ++pass) {
image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite, {},
scheduler.Current().Handle());
}
Require(name, "write epoch", Libs::Graphics::Image::heap_epoch.load() == epoch + 1u,
"repeated writes after leaving the read state bumped the heap epoch again");
Require(name, "read again", acquire(sampled), "the view was declined");
draw();
Require(name, "read layout",
image.backing.state.layout == vk::ImageLayout::eShaderReadOnlyOptimal &&
Libs::Graphics::Image::heap_epoch.load() == epoch + 1u,
"commit did not return a resident image to its read layout");
// An image freed during a draw's discovery lives until the tick ends, and the draw's
// slots still select it.
write(9, descriptor(13, Format::k8_8_8_8UNorm));
Require(name, "freed image", acquire(sampled), "the view was declined");
heap.Rebind(executor);
const auto freed = ImageHeapTestAccess::ImageOf(heap, filtered.slots[72]);
textures.UnmapMemory(base + 0x10000u + 13u * 0x1000u, 0x1000u);
heap.Commit(scheduler.Current(), vk::PipelineBindPoint::eCompute, layout);
Require(name, "freed image layout",
!textures.GetImage(freed).registered &&
textures.GetImage(freed).backing.state.layout ==
vk::ImageLayout::eShaderReadOnlyOptimal,
"commit skipped the read transition of a freed but sampled image");
scheduler.Finish();
heap.BeginDraw();
// A draw skipped after its programs acquired views still leaves the next draw a fresh
// table.
Require(name, "skipped draw", acquire(sampled), "the view was declined");
heap.BeginDraw();
write(11, descriptor(16, Format::k8_8_8_8UNorm));
Require(name, "draw after a skipped one", acquire(sampled) && filtered.slots[88] != 0u,
"a draw sampled the table of a skipped draw");
// The skipped draw's view stays acquired. Another view refreshing the table must not let
// a stale element take a newer kind from it.
heap.BeginDraw();
write(5, descriptor(15, Format::k8_8_8_8SInt));
Require(name, "other view", acquire(unfiltered), "the view was declined");
textures.UnmapMemory(base + 0x10000u + 3u * 0x1000u, 0x1000u);
draw();
Require(name, "stale rescan",
filtered.kinds.size() == 2u &&
ImageHeapTestAccess::Element(heap, filtered.slots[40]).value.dwords ==
descriptor(3, Format::k8_8_8_8UNorm),
"a stale element's rescan took a kind its view's programs lack");
// Tables keep the descriptors of unmapped textures, which PS5 never samples. Their words
// stay null in every view, even when the memory is mapped again before the next draw,
// until one of their own words changes.
const auto texture = [](uint32_t index) { return base + 0x10000u + index * 0x1000u; };
write(20, descriptor(20, Format::k8_8_8_8UNorm));
Require(name, "live texture", acquire(sampled), "the view was declined");
draw();
Require(name, "live image",
static_cast<bool>(ImageHeapTestAccess::ImageOf(heap, filtered.slots[160])),
"a mapped texture was not resolved");
context.UnmapMemory(texture(20), 0x1000u);
context.MapMemory(texture(20), 0x1000u);
heap.BeginDraw();
Require(name, "dead in a new view",
acquire(sampled, table, 64u) && views.back()->slots[160] == 0u,
"a view created after the remap resolved a dead descriptor");
Require(name, "remapped texture", acquire(sampled), "the view was declined");
draw();
Require(name, "dead descriptor",
filtered.slots[160] == 0u && !textures.IsRegionRegistered(texture(20), 0x1000u),
"the descriptor of an unmapped texture was resolved again after a remap");
write(19, descriptor(19, Format::k8_8_8_8UNorm));
write(21, descriptor(21, Format::k8_8_8_8UNorm));
Require(name, "neighbour writes", acquire(sampled), "the view was declined");
draw();
Require(name, "dead reprobe", filtered.slots[160] == 0u,
"rescanning changed neighbours revived a dead descriptor");
write(24, descriptor(20, Format::k8_8_8_8UNorm));
Require(name, "republished texture", acquire(sampled), "the view was declined");
draw();
Require(name, "republished descriptor",
filtered.slots[160] == 0u && ImageHeapTestAccess::ImageOf(heap, filtered.slots[192]),
"the same descriptor in another word was not resolved");
write(20, descriptor(22, Format::k8_8_8_8UNorm));
Require(name, "rewritten word", acquire(sampled), "the view was declined");
draw();
Require(name, "rewritten descriptor",
static_cast<bool>(ImageHeapTestAccess::ImageOf(heap, filtered.slots[160])),
"a new descriptor in a dead word was not resolved");
// A view that a skipped draw left acquired buries before its rescan.
Require(name, "skipped draw view", acquire(sampled), "the view was declined");
context.UnmapMemory(texture(22), 0x1000u);
context.MapMemory(texture(22), 0x1000u);
heap.BeginDraw();
draw();
Require(name, "dead after a skipped draw", filtered.slots[160] == 0u,
"the rescan of a view left acquired resolved a dead descriptor");
// A new descriptor over unmapped memory is never resolved.
context.UnmapMemory(texture(23), 0x1000u);
write(23, descriptor(23, Format::k8_8_8_8UNorm));
Require(name, "unmapped texture", acquire(sampled), "the view was declined");
draw();
Require(name, "unmapped descriptor", !textures.IsRegionRegistered(texture(23), 0x1000u),
"a descriptor over unmapped memory was resolved");
heap.UnmapMemory(table, table_size);
Require(name, "unmapped table",
ImageHeapTestAccess::MemoryCount(heap) == 0u &&
textures.GetImage(resident).heap_refs.load() == 0u,
"unmapping the table kept its images pinned");
// The idle table stays watched; destroying the heap must unwatch it before the page
// manager fails on live watches.
Require(name, "idle table",
acquire(sampled, idle_table) && view.kinds.empty() &&
ImageHeapTestAccess::MemoryCount(heap) == 1u,
"an all-null table was not kept resident");
draw();
context.UnmapMemory(base, idle_table - base);
scheduler.Finish();
device.destroyPipelineLayout(layout, nullptr);
device.destroyDescriptorSetLayout(empty, nullptr);
}
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"image heap unmap failed");
Require(name, "release", Libs::LibKernel::Memory::KernelReleaseDirectMemory(
direct_offset, allocation_size) == 0,
"image heap release failed");
std::printf("[host] %-32s ok\n", name);
}
void CheckComputeMetaClearClassification() {
constexpr const char *name = "ComputeMetaClearClassification";
constexpr uint64_t read_only_meta = 0x0000000204201f00ull;
@@ -12464,6 +12773,212 @@ public:
std::printf("[gpu] %-32s ok\n", name);
}
// Cyberpunk 2077 copies its HTile D16 depth target through a kDepth k16UInt image load.
void CheckDepthBitsCopy() {
constexpr const char *name = "DepthBitsCopy";
constexpr uintptr_t base = 0x0000000205000000ull;
constexpr uint64_t allocation_size = 0x30000;
constexpr uint64_t htile_address = base + 0x10000;
constexpr uint64_t copy_address = base + 0x20000;
constexpr u32 depth_code = 0x2000u;
std::vector<u32> copy{EncodeMimg0(0x00, 0x1), EncodeMimg1(4, 0, 2),
EncodeMimg0(0x08, 0x1), EncodeMimg1(4, 0, 0)};
AppendEnd(&copy);
ShaderMapUserData(reinterpret_cast<uint64_t>(copy.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(copy.size() * sizeof(u32))});
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, 0x10000, 0, &direct_offset) == 0,
"depth allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset, 0x10000) == 0 &&
mapped == reinterpret_cast<void *>(base),
"depth mapping failed");
std::memset(mapped, 0, allocation_size);
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
CommandProcessor processor(context, 0);
processor.BufferInit();
auto &scheduler = context.GetCommandScheduler();
auto &textures = context.GetTextureCache();
auto &shaders = processor.GetShCtx();
context.MapMemory(base, allocation_size);
ImageDesc depth{};
depth.type = BindingType::DepthTarget;
depth.info.data = {base, 0x10000};
depth.info.pixel_format = vk::Format::eD16Unorm;
depth.info.guest_format = Prospero::BufferFormat::k16UNorm;
depth.info.type = Prospero::ImageType::kColor2D;
depth.info.extent = {256, 128, 1};
depth.info.resources = {1, 1};
depth.info.pitch = 256;
depth.info.bytes_per_block = 2;
depth.info.samples = 1;
depth.info.tile_mode = Prospero::TileMode::kDepth;
depth.info.UpdateSize();
depth.info.metadata.kind = ImageMetadataKind::Htile;
depth.info.metadata.range = {htile_address, 0x8000};
depth.info.htile_initial_clear = false;
depth.view_info.format = vk::Format::eD16Unorm;
depth.view_info.type = vk::ImageViewType::e2D;
depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
const auto depth_id = textures.FindImage(depth);
Require(name, "HTile depth target", textures.FindDepthTarget(depth_id, depth) != nullptr,
"the D16 depth target was not created");
vk::ClearValue clear{};
clear.depthStencil.depth = static_cast<float>(depth_code) / 65535.0f;
TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), depth_id,
{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, 1}, clear);
// The captured T# with this depth target's size and HTile; the copy writes 2x2 texels.
const ShaderTextureResource depth_bits{{static_cast<u32>(base >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k16UInt) << 20u) | (3u << 30u),
0x3fu | (127u << 14u),
DstSel(4, 0, 0, 1) | (static_cast<u32>(Prospero::TileMode::kDepth) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0x00280000u | (static_cast<u32>(htile_address >> 8u) << 24u),
static_cast<u32>(htile_address >> 16u)}};
const ShaderTextureResource target{{static_cast<u32>(copy_address >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k16UInt) << 20u) | (1u << 30u), 1u << 14u,
DstSel(4, 5, 6, 7) | (static_cast<u32>(Prospero::TileMode::kLinear) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
0, 0x00700000u, 0, 0}};
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(copy.data()),
.num_thread_x = 2, .num_thread_y = 2, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 16, .tidig_comp_cnt = 1});
for (u32 i = 0; i < 8; i++) {
shaders.SetCsUserSgpr(i, target.fields[i], HW::UserSgprType::Unknown);
shaders.SetCsUserSgpr(8u + i, depth_bits.fields[i], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(target.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Uint;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
resource.written = true;
const auto binding =
RenderExecutorTestAccess::ResolveTexture(context.GetRenderExecutor(), resource, value);
Require(name, "raw depth bits",
ReadCachedTexel(name, context, binding.image_id, {}, {2, 2, 1}) ==
std::vector<u32>(2, depth_code | depth_code << 16u),
"the copy did not read the stored D16 codes");
Require(name, "depth target retained", TextureCacheTestAccess::Contains(textures, depth_id),
"reading depth bits replaced the HTile depth target");
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"depth mapping release failed");
Require(name, "release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0,
"depth allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
// Cyberpunk 2077 clears a rect of an MSAA color target from compute, one sample per store.
void CheckMsaaStorageClear() {
constexpr const char *name = "MsaaStorageClear";
constexpr uintptr_t base = 0x0000000205000000ull;
constexpr uint64_t allocation_size = 0x10000;
// Captured CS 0xad6c86aca9c0916e: s[8:11] color, s[12:13] origin, s14 width,
// s15 height << 16 | samples.
std::vector<u32> clear{0xbfa00001u, 0xd7460001u, 0x04050611u, 0xd7460000u, 0x04010610u,
0x7da21ef9u, 0x85060001u, 0x7da8000eu, 0xbf88000fu, 0x4a08000cu,
0x4a0a020du, 0x996b800fu, 0xbeea0380u, 0xbf0a6b6au, 0xbf840009u,
0x7e0c026au, 0x7e000208u, 0x7e020209u, 0x7e04020au, 0x7e06020bu,
0x816a816au, 0xf0200f30u, 0x00000004u, 0xbf82fff5u, 0xbf810000u};
ShaderMapUserData(reinterpret_cast<uint64_t>(clear.data()),
{.type = Prospero::ShaderBinaryType::kCs,
.code_size_bytes = static_cast<u32>(clear.size() * sizeof(u32))});
EnsureRuntimeContext();
int64_t direct_offset = -1;
Require(name, "direct allocation",
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
allocation_size, 0x10000, 0, &direct_offset) == 0,
"MSAA allocation failed");
void *mapped = reinterpret_cast<void *>(base);
Require(name, "direct mapping",
Libs::LibKernel::Memory::KernelMapDirectMemory(
&mapped, allocation_size, 0x3, 0x10, direct_offset, 0x10000) == 0 &&
mapped == reinterpret_cast<void *>(base),
"MSAA mapping failed");
RenderContext context(m_runtime_context);
context.InitializeGpu(nullptr);
LibKernel::Memory::InstallGpuResources(&context);
context.GetGpu().SendCommandSync([&] {
CommandProcessor processor(context, 0);
processor.BufferInit();
auto &scheduler = context.GetCommandScheduler();
auto &textures = context.GetTextureCache();
auto &shaders = processor.GetShCtx();
context.MapMemory(base, allocation_size);
// 8x8 4x MSAA RGBA8: LAST_LEVEL and MAX_MIP hold log2(samples).
const ShaderTextureResource target{{static_cast<u32>(base >> 8u),
(static_cast<u32>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u) | (3u << 30u),
1u | (7u << 14u),
DstSel(4, 5, 6, 7) | (2u << 16u) |
(static_cast<u32>(Prospero::TileMode::kRenderTarget) << 20u) |
(static_cast<u32>(Prospero::ImageType::kColor2DMsaa) << 28u),
0, 0x00700020u, 0, 0}};
ShaderRecompiler::IR::DescriptorValue value{};
value.dword_count = 8;
std::copy_n(target.fields, 8, value.dwords.begin());
ShaderRecompiler::IR::ImageResource resource{};
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
resource.numeric_class = Prospero::TextureNumericClass::Float;
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
resource.written = true;
const auto binding =
RenderExecutorTestAccess::ResolveTexture(context.GetRenderExecutor(), resource, value);
TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), binding.image_id,
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(clear.data()),
.num_thread_x = 8, .num_thread_y = 8, .num_thread_z = 1,
.wave_size = 64, .user_sgpr = 16, .tgid_x_en = true,
.tgid_y_en = true, .tidig_comp_cnt = 1});
const std::array<u32, 8> args{std::bit_cast<u32>(1.0f), 0, 0, std::bit_cast<u32>(1.0f),
1, 1, 2, (2u << 16u) | 4u};
for (u32 i = 0; i < 8; i++) {
shaders.SetCsUserSgpr(i, target.fields[i], HW::UserSgprType::Unknown);
shaders.SetCsUserSgpr(8u + i, args[i], HW::UserSgprType::Unknown);
}
processor.DispatchDirect(1, 1, 1, 0x41u);
for (const int row : {1, 2, 3}) {
const u32 inside = row == 3 ? 0u : 0xff0000ffu;
Require(name, "every sample in the rect",
ReadCachedTexel(name, context, binding.image_id, {0, row, 0}, {4, 1, 1}) ==
std::vector<u32>{0, inside, inside, 0},
"the compute clear missed samples or wrote outside its rect");
}
context.UnmapMemory(base, allocation_size);
scheduler.Finish();
});
context.ShutdownGpu();
LibKernel::Memory::InstallGpuResources(nullptr);
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
"MSAA mapping release failed");
Require(name, "release",
Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0,
"MSAA allocation release failed");
std::printf("[gpu] %-32s ok\n", name);
}
void CheckSampledDccClear() {
constexpr const char *name = "SampledDccClear";
constexpr uintptr_t base = 0x0000000205000000ull;
@@ -20872,6 +21387,7 @@ private:
device_info.pNext = &conditional_rendering;
vk::PhysicalDeviceFeatures device_features{};
device_features.shaderStorageImageWriteWithoutFormat = true;
device_features.shaderStorageImageMultisample = available_features.shaderStorageImageMultisample;
device_features.shaderImageGatherExtended = true;
device_features.sampleRateShading = true;
device_features.shaderInt64 = true;
@@ -37759,6 +38275,39 @@ template <bool rg, bool snorm = false> TestCase ImageSamplePacked8() {
return test;
}
// A kDepth k16UInt image load returns the stored D16 codes and integer swizzle constants.
TestCase ImageLoadDepthBits() {
using O = ShaderOpcode;
TestCase test;
test.name = "ImageLoadDepthBits";
constexpr std::array<u32, 4> depth{0x2000u, 0xc000u, 0xffffu, 0x0001u};
for (u32 texel = 0; texel < depth.size(); ++texel) {
AppendVMovU32(&test.code, 20, texel % 2u);
AppendVMovU32(&test.code, 21, texel / 2u);
test.code.push_back(EncodeMimg0(0x00, 0xf));
test.code.push_back(EncodeMimg1(0, 20));
for (u32 component = 0; component < 4; ++component) {
AppendStoreVgpr(&test.code, component, texel * 4u + component);
}
test.expected.insert(test.expected.end(), {depth[texel], 0u, 0u, 1u});
}
AppendEnd(&test.code);
test.image_width = test.image_height = 2;
test.sampled_image_rgba = {depth[0] | depth[1] << 16u, depth[2] | depth[3] << 16u};
test.sampled_image_format = vk::Format::eR16Unorm;
test.sampled_image_dwords_per_pixel = 1;
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k16UInt);
test.user_data[1] |= 1u << 30u;
test.user_data[2] = 1u << 14u;
test.user_data[3] |= static_cast<u32>(Prospero::TileMode::kDepth) << 20u;
test.user_data[50] = test.expected.size() * sizeof(u32);
test.user_data[51] = 3u << 28u;
test.has_user_data = true;
test.image_descriptor_swizzle = DstSel(4, 0, 0, 1);
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
return test;
}
TestCase ImageLoadR128IgnoresAdjacentMaskSgprs() {
using O = ShaderOpcode;
@@ -40580,7 +41129,7 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) {
compiled.program.info.samplers.size() == rows + 1u &&
tools.Disassemble(compiled.spirv, &text) &&
CountText(text, "OpImageSampleExplicitLod") == halves &&
compiled.spirv.size() <= 8192u * halves,
SpirvCodeWords(compiled.spirv) <= 8192u * halves,
"59 sampler records hit the SGPR root limit or duplicated image sampling bodies");
}
if (large_samplers)
@@ -41035,6 +41584,41 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) {
Require(name, "alternating image emitted size", spirv.size() <= 8192u,
"alternating descriptors exceeded the 32 KiB SPIR-V regression budget");
// A resident heap selects its typed kind and element from a device slot word.
auto heap_root = root;
heap_root.heap = true;
heap_root.indirect_search_iterations = 0;
heap_root.indirect_resources = {0u, 1u, 2u};
auto heap_kind = candidate;
heap_kind.heap = true;
auto heap_uint = heap_kind;
heap_uint.numeric_class = Prospero::TextureNumericClass::Uint;
for (const bool mixed : {true, false}) {
program.info.images = {heap_root, heap_kind};
if (mixed) program.info.images.push_back(heap_uint);
program.info.images[0].indirect_resources.resize(program.info.images.size());
// ApplyResourceSpecialization adds the integer sampler variant of a Uint kind.
program.info.samplers.resize(1);
if (mixed) {
program.info.samplers.push_back(program.info.samplers[0]);
program.info.samplers[1].integer_border = true;
}
program.shader_info_complete = false;
CollectShaderInfo(program, {.compute = &compute});
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
ValidateSpirv("ResidentImageHeap", spirv);
Require(name, "image heap disassembly", tools.Disassemble(spirv, &text),
"failed to disassemble the image heap shader");
Require(name, "image heap selection",
program.info.uses_image_heap && CountText(text, "OpConvertUToPtr") == 1 &&
CountText(text, "UMin") == 1 &&
CountText(text, "OpSwitch") == (mixed ? 1 : 0) &&
CountText(text, "OpImageSampleExplicitLod") == (mixed ? 2 : 1) &&
CountText(text, "DescriptorSet 1") == (mixed ? 2 : 1) &&
text.find("PhysicalStorageBuffer64") != std::string::npos,
"image heap kinds did not sample the slot-selected heap element");
}
program.memory_info[0].image_dimension =
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
program.memory_info[0].image_address_components = 4;
@@ -43665,6 +44249,7 @@ std::vector<TestCase> MakeCases() {
AddCase(ImageSamplePacked8<false>);
AddCase(ImageSamplePacked8<true>);
AddCase(ImageSamplePacked8<false, true>);
AddCase(ImageLoadDepthBits);
AddCase(ImageLoadR128IgnoresAdjacentMaskSgprs);
AddCase(ImageLoad1DUsesScalarCoordinate);
AddCase(ImageGather2DInstructionWith1DDescriptor);
@@ -43799,7 +44384,7 @@ void CheckComputeLdsLimit(VulkanHarness &vulkan) {
regs.cs_regs.num_thread_y = 1;
regs.cs_regs.num_thread_z = 1;
regs.cs_regs.wave_size = 32;
PipelineCache cache(graphics);
PipelineCache cache(graphics, nullptr);
ShaderProgram previous;
uint16_t previous_units = 0;
for (const auto units : {static_cast<uint16_t>(limit_units - 1u), limit_units,
@@ -49095,6 +49680,94 @@ void CheckPm4IndirectControlFlow(RenderContext &renderer) {
std::printf("[host] %-32s ok\n", name);
}
// waitUntilSafeForRendering stalls only its own command processor, and a display buffer
// stays busy while its flip is pending or while it is on screen.
void CheckWaitUntilSafeForRendering() {
constexpr const char *name = "WaitUntilSafeForRendering";
Require(name, "video", SDL_InitSubSystem(SDL_INIT_VIDEO), "SDL video init failed");
WindowContext window;
window.graphic_ctx.screen_width = 64;
window.graphic_ctx.screen_height = 64;
window.window = SDL_CreateWindow(name, 64, 64, SDL_WINDOW_VULKAN | SDL_WINDOW_HIDDEN);
Require(name, "window", window.window != nullptr, "hidden Vulkan window failed");
window.CreateVulkan();
auto &video_out = VideoOut::VideoOutInit(64, 64, *window.presenter);
auto &renderer = *window.render_context;
renderer.InitializeGpu(&video_out);
auto &gpu = renderer.GetGpu();
constexpr int user_system = 255;
constexpr int bus_main = 0;
constexpr int flip_mode_vsync = 1;
// Special buffer indices flip without a registered surface.
constexpr int index_blank = -1;
constexpr int index_black = -2;
const int handle = VideoOut::VideoOutOpen(user_system, bus_main, 0, nullptr);
Require(name, "open", handle > 0, "video-out open failed");
const auto wait_until = [](const auto &done) {
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
while (!done() && std::chrono::steady_clock::now() < deadline) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
return done();
};
const auto address = [](const void *value) { return reinterpret_cast<uint64_t>(value); };
// Hold the graphics queue so the CPU flip of BLACK, prepared behind it, stays pending.
std::atomic_uint32_t gate{0};
std::array<uint32_t, 7> hold{};
hold[0] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
hold[1] = 0x10u | 3u;
hold[2] = static_cast<uint32_t>(address(&gate));
hold[3] = static_cast<uint32_t>(address(&gate) >> 32u);
hold[4] = 1;
hold[5] = UINT32_MAX;
gpu.Submit(hold, {});
Require(name, "flip black",
VideoOut::VideoOutSubmitFlip(handle, index_black, flip_mode_vsync, 0) == 0,
"CPU flip failed");
std::atomic_uint32_t rendered{0};
std::array<uint32_t, 12> render{};
render[0] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_FLIP_DONE);
render[1] = static_cast<uint32_t>(handle);
render[2] = static_cast<uint32_t>(index_black);
render[7] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
render[9] = static_cast<uint32_t>(address(&rendered));
render[10] = static_cast<uint32_t>(address(&rendered) >> 32u);
render[11] = 1;
gpu.SubmitCompute(0x20, render);
std::binary_semaphore lane{0};
std::jthread command_lane([&] {
gpu.SendCommandSync([] {});
lane.release();
});
Require(name, "command lane", lane.try_acquire_for(std::chrono::seconds(2)),
"a busy display buffer blocked the GPU thread");
Require(name, "pending", rendered.load() == 0,
"rendering started while its display buffer had a pending flip");
gate.store(1);
Require(name, "flip shown", wait_until([handle] { return VideoOut::VideoOutIsFlipPending(handle) == 0; }),
"the BLACK flip was not presented");
std::this_thread::sleep_for(std::chrono::milliseconds(100));
Require(name, "displayed", rendered.load() == 0,
"rendering started while its display buffer was on screen");
Require(name, "flip blank",
VideoOut::VideoOutSubmitFlip(handle, index_blank, flip_mode_vsync, 0) == 0,
"CPU flip failed");
Require(name, "resume", wait_until([&] { return rendered.load() == 1; }),
"rendering did not resume once its display buffer left the screen");
gpu.WaitForIdle();
Require(name, "close", VideoOut::VideoOutClose(handle) == 0, "video-out close failed");
renderer.ShutdownGpu();
VideoOut::VideoOutShutdown();
std::printf("[host] %-32s ok\n", name);
}
void CheckPm4WaitResume(RenderContext &renderer) {
GraphicsInitJmpTables();
CommandProcessor processor(renderer, 0);
@@ -49617,6 +50290,17 @@ int main(int argc, char **argv) {
RunCase(&vulkan, ImageSamplePacked8<false, true>());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--msaa-storage-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckMsaaStorageClear();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--depth-bits-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckDepthBitsCopy();
RunCase(&vulkan, ImageLoadDepthBits());
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--packed-texture-only") == 0) {
VulkanHarness vulkan;
RunCase(&vulkan, ImageLoadSintUsesSignedSampledImage<false>());
@@ -49974,6 +50658,10 @@ int main(int argc, char **argv) {
vulkan.CheckScalarLeafCall();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--wait-until-safe-for-rendering-only") == 0) {
CheckWaitUntilSafeForRendering();
return 0;
}
if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
VulkanHarness vulkan;
vulkan.CheckGpuCommandLane();
@@ -50295,6 +50983,7 @@ int main(int argc, char **argv) {
CheckNestedPointerImageTable(vulkan);
CheckGuardedRecordImageTable(vulkan);
CheckIndirectImageKeySwitch(vulkan);
vulkan.CheckResidentImageHeap();
CheckFiniteFirstLaneImageChoice(vulkan);
CheckFiniteInlineSamplerPhi(vulkan);
RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives());
@@ -50473,6 +51162,7 @@ int main(int argc, char **argv) {
CheckNestedPointerImageTable(vulkan);
CheckGuardedRecordImageTable(vulkan);
CheckIndirectImageKeySwitch(vulkan);
vulkan.CheckResidentImageHeap();
CheckDynamicSampleOffsets(vulkan, 32u);
CheckDynamicSampleOffsets(vulkan, 64u);
CheckFiniteFirstLaneImageChoice(vulkan);
@@ -50504,6 +51194,8 @@ int main(int argc, char **argv) {
vulkan.CheckRenderExecutorColorVolumeDiscovery();
vulkan.CheckRenderExecutorColorMetadataClear();
vulkan.CheckSampledDccClear();
vulkan.CheckDepthBitsCopy();
vulkan.CheckMsaaStorageClear();
vulkan.CheckRenderExecutorColorStandardTileDiscovery();
vulkan.CheckRenderExecutorColorDepthTileDiscovery();
vulkan.CheckRenderExecutorStencilBindingDiscovery();
@@ -50527,6 +51219,8 @@ int main(int argc, char **argv) {
vulkan.CheckGpuSuspendPoint();
vulkan.CheckGpuCommandLane();
vulkan.CheckScalarLeafCall();
// Last: its window installs global GPU resources.
CheckWaitUntilSafeForRendering();
if (skipped_device_checks) {
std::printf(
"ShaderRecompilerComputeTests: device rasterization checks skipped, this "
+18 -6
View File
@@ -99,6 +99,7 @@ ShaderRecompiler::CompileOptions MakeCompileOptions(ShaderType stage) {
ShaderRecompiler::CompileOptions options;
options.stage = stage;
options.user_data = user_data;
options.emit_debug_info = true; // Checks find variables and functions by OpName.
switch (stage) {
case ShaderType::Vertex:
options.input_info.vertex = &vertex;
@@ -403,6 +404,11 @@ SpirvMetrics MeasureSpirv(const std::vector<uint32_t> &binary) {
if (word_count == 0 || i + word_count > binary.size()) {
break;
}
if (opcode == spv::OpName || opcode == spv::OpString || opcode == spv::OpLine) {
metrics.words -= word_count; // Budgets measure the module without debug info.
i += word_count;
continue;
}
metrics.instructions++;
switch (opcode) {
case 11u: // OpExtInstImport
@@ -10354,7 +10360,7 @@ void TestNewShaderRecompilerPlanningOnlyLoads() {
ir.Emit(IR::ValueOpcode::ReferenceU32, {read});
program.shader_info_complete = false;
IR::CollectShaderInfo(program, options.input_info);
const auto spirv = Spirv::EmitProgram(program, options.input_info);
const auto spirv = Spirv::EmitProgram(program, options.input_info, 0, {}, true);
CheckSpirvBinaryValidates(spirv);
Check(spirv == result.spirv,
"planning-only loads or references emitted GPU operands/instructions");
@@ -11100,6 +11106,7 @@ void TestMeshExportStorage() {
options.stage = ShaderType::Mesh;
options.input_info.vertex = &input;
options.user_data = user_data;
options.emit_debug_info = true; // The checks find push data by its OpName.
struct Case {
uint32_t subgroup_size, push_data_start;
bool split;
@@ -14352,7 +14359,8 @@ void TestNewShaderRecompilerNativeBindingPlan() {
const auto bindings = result.program.bindings;
result.program.bindings = {};
result.program.info.buffers[0].descriptor_index = UINT32_MAX;
Check(ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info) == result.spirv &&
Check(ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info, 0, {}, true) ==
result.spirv &&
result.program.bindings == bindings &&
result.program.info.buffers[0].descriptor_index == 0,
"repeated emission did not rebuild descriptor slots deterministically");
@@ -16423,11 +16431,15 @@ void TestRepeatedExportsHaveOneInterface() {
void CheckSpirvBlockSourceMapping(TestCompileResult &result) {
using namespace ShaderRecompiler;
const auto options = MakeCompileOptions(ShaderType::Compute);
Check(SpirvInstructionOpcodeCount(result.spirv, spv::OpString) == 0 &&
SpirvInstructionOpcodeCount(result.spirv, spv::OpLine) == 0,
"default emission unexpectedly contains source mapping");
const auto plain = Spirv::EmitProgram(result.program, options.input_info);
Check(SpirvInstructionOpcodeCount(plain, spv::OpString) == 0 &&
SpirvInstructionOpcodeCount(plain, spv::OpLine) == 0 &&
SpirvInstructionOpcodeCount(plain, spv::OpName) == 0,
"default emission unexpectedly contains debug info");
const auto mapped =
Spirv::EmitProgram(result.program, options.input_info, 0, {}, true);
Check(SpirvInstructionOpcodeCount(mapped, spv::OpName) != 0,
"debug emission omitted names");
CheckSpirvBinaryValidates(mapped);
uint32_t source = 0;
uint32_t label = 0;
@@ -16456,7 +16468,7 @@ void CheckSpirvBlockSourceMapping(TestCompileResult &result) {
}
Check(source != 0 && lines >= result.program.blocks.size(),
"debug emission omitted native block source mapping");
Check(Spirv::EmitProgram(result.program, options.input_info) == result.spirv,
Check(Spirv::EmitProgram(result.program, options.input_info) == plain,
"debug re-emission changed the default binary");
}