mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-29 08:41:20 +00:00
Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
719c44310a | ||
|
|
7bb618d41a | ||
|
|
87e815b250 | ||
|
|
f9e14084d6 | ||
|
|
38c884e8c1 | ||
|
|
27dc7c281f | ||
|
|
79837b423d | ||
|
|
aafe014d0c | ||
|
|
a358b847c1 | ||
|
|
a951a91703 | ||
|
|
a875d1b205 |
No files matched your search
@@ -676,6 +676,12 @@ if(BUILD_TESTING)
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --stream-buffer-only)
|
||||
add_test(NAME gpu_command_lane
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --gpu-command-lane-only)
|
||||
add_test(NAME wait_until_safe_for_rendering
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --wait-until-safe-for-rendering-only)
|
||||
add_test(NAME depth_bits
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --depth-bits-only)
|
||||
add_test(NAME msaa_storage
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --msaa-storage-only)
|
||||
add_test(NAME pm4_context_state
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --context-state-only)
|
||||
add_test(NAME gpu_tiler
|
||||
|
||||
@@ -46,6 +46,16 @@ or graphical glitches, so please include the version you tested when reporting a
|
||||
## Screenshots
|
||||
|
||||
<table align="center">
|
||||
<tr>
|
||||
<td align="center">
|
||||
<strong>Cyberpunk 2077</strong><br>
|
||||
<img src="docs/screenshots/ps5-07.png" width="300" alt="Cyberpunk 2077 running in KytyPS5">
|
||||
</td>
|
||||
<td align="center">
|
||||
<strong>MLB The Show 23</strong><br>
|
||||
<img src="docs/screenshots/ps5-08.png" width="300" alt="MLB The Show 23 running in KytyPS5">
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<strong>Astro Bot</strong><br>
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.5 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.0 MiB |
@@ -30,6 +30,7 @@
|
||||
#include <mutex>
|
||||
#include <semaphore>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
@@ -91,6 +92,12 @@ void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
|
||||
m_work_available.Signal();
|
||||
}
|
||||
|
||||
void GuestGpu::Wake() {
|
||||
Common::LockGuard lock(m_queue_mutex);
|
||||
m_wake_pending = true;
|
||||
m_work_available.Signal();
|
||||
}
|
||||
|
||||
void GuestGpu::ProcessCommands() {
|
||||
EXIT_IF(!IsGpuThread());
|
||||
while (m_pending_commands.load(std::memory_order_acquire) != 0) {
|
||||
@@ -464,7 +471,10 @@ void GuestGpu::ThreadRun(void* data) {
|
||||
}
|
||||
if (selected_queue < 0) {
|
||||
gpu->m_processing = false;
|
||||
gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
|
||||
// A wake that came before this wait must not be lost.
|
||||
if (!std::exchange(gpu->m_wake_pending, false)) {
|
||||
gpu->m_work_available.WaitFor(&gpu->m_queue_mutex, 100);
|
||||
}
|
||||
for (auto& queue: gpu->m_queues) {
|
||||
if (!queue.empty()) {
|
||||
queue.front().blocked = false;
|
||||
@@ -998,10 +1008,11 @@ void CommandProcessor::DrawIndexAuto(DrawAutoArgs args) {
|
||||
}
|
||||
|
||||
void CommandProcessor::WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index) {
|
||||
BufferFlush();
|
||||
|
||||
m_renderer.GetVideoOut().WaitFlipDone(static_cast<int>(video_out_handle),
|
||||
static_cast<int>(display_buffer_index));
|
||||
// Stall only this command processor, not the host command lane or other queues.
|
||||
if (!m_renderer.GetVideoOut().IsSafeForRendering(static_cast<int>(video_out_handle),
|
||||
static_cast<int>(display_buffer_index))) {
|
||||
SuspendPm4();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
|
||||
@@ -31,6 +31,8 @@ public:
|
||||
[[nodiscard]] bool IsStopping();
|
||||
void SendCommand(Common::UniqueFunction<void>&& command);
|
||||
void SendCommandSync(Common::UniqueFunction<void>&& command);
|
||||
// Retries blocked submissions now instead of after the 100 ms stall timeout.
|
||||
void Wake();
|
||||
|
||||
// Submitted command memory is borrowed and must remain valid until GPU execution completes.
|
||||
void Submit(std::span<const uint32_t> draw_commands,
|
||||
@@ -87,6 +89,7 @@ private:
|
||||
bool m_processing = false;
|
||||
bool m_accepting = true;
|
||||
bool m_stopping = false;
|
||||
bool m_wake_pending = false;
|
||||
bool m_shutdown_complete = false;
|
||||
// Completion callbacks can outlive GuestGpu during renderer shutdown.
|
||||
std::shared_ptr<std::binary_semaphore> m_suspend_point_ready =
|
||||
|
||||
@@ -43,6 +43,7 @@ struct GraphicContext {
|
||||
uint32_t min_subgroup_size = 0;
|
||||
uint32_t max_subgroup_size = 0;
|
||||
uint32_t max_push_descriptors = 0;
|
||||
vk::DescriptorSetLayout image_heap_layout = nullptr;
|
||||
vk::ShaderStageFlags required_subgroup_size_stages = {};
|
||||
Common::Mutex queue_mutex;
|
||||
uint32_t queue_family = static_cast<uint32_t>(-1);
|
||||
|
||||
+10
-1
@@ -229,6 +229,7 @@ void TextureCache::UnregisterImage(ImageId id) {
|
||||
if (!image.registered) {
|
||||
return;
|
||||
}
|
||||
image.HeapChanged();
|
||||
UntrackImage(id);
|
||||
ImagePageTable::PageRange pages {};
|
||||
if (!ImagePageTable::TryGetPageRange(image.info.data.address, image.info.data.size, pages)) {
|
||||
@@ -378,6 +379,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
|
||||
const auto address = Common::AlignDown(begin + TRACKER_PAGE_SIZE, TRACKER_PAGE_SIZE);
|
||||
const auto size = address - begin;
|
||||
image.track_addr = address;
|
||||
image.HeapChanged();
|
||||
if (image.track_addr == image.track_addr_end) {
|
||||
MarkAsMaybeDirty(id, image);
|
||||
}
|
||||
@@ -395,6 +397,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
|
||||
const auto address = Common::AlignDown(end, TRACKER_PAGE_SIZE);
|
||||
const auto size = end - address;
|
||||
image.track_addr_end = address;
|
||||
image.HeapChanged();
|
||||
if (image.track_addr == image.track_addr_end) {
|
||||
MarkAsMaybeDirty(id, image);
|
||||
}
|
||||
@@ -795,7 +798,8 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
|
||||
auto& cached = *owner;
|
||||
const auto current_tick = m_scheduler.CurrentTick();
|
||||
const bool safe_to_delete =
|
||||
current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval;
|
||||
current_tick - std::min(current_tick, cached.tick_accessed_last) > NumFramesBeforeRemoval &&
|
||||
cached.heap_refs.load(std::memory_order_relaxed) == 0u;
|
||||
|
||||
const uint32_t requested_block = requested.bytes_per_block * requested.samples;
|
||||
const uint32_t cached_block = cached.info.bytes_per_block * cached.info.samples;
|
||||
@@ -2015,6 +2019,11 @@ void TextureCache::RunGarbageCollector() {
|
||||
if (owner == nullptr || !owner->registered || owner->depth_id) {
|
||||
continue;
|
||||
}
|
||||
if (owner->heap_refs.load(std::memory_order_relaxed) != 0u) {
|
||||
// Resident heaps sample it without touching it.
|
||||
TouchImage(*owner);
|
||||
continue;
|
||||
}
|
||||
if (owner->IsGpuModified()) {
|
||||
const bool safe = owner->SafeToDownload();
|
||||
if (safe && owner->info.IsTiled()) {
|
||||
|
||||
@@ -191,6 +191,7 @@ private:
|
||||
friend struct TextureCacheTestAccess;
|
||||
friend class BufferCache;
|
||||
friend class RenderExecutor;
|
||||
friend class ImageHeap;
|
||||
friend bool BcAlias::Matches(const ImageInfo&, const ImageInfo&);
|
||||
friend bool BcAlias::Publish(TextureCache&, Image&);
|
||||
friend void BcAlias::Resolve(TextureCache&, ImageId, ImageId);
|
||||
|
||||
@@ -90,7 +90,8 @@ ImageCreateFlags(const GraphicContext& graphics, const ImageInfo& info, vk::Imag
|
||||
if (HasFormatFeature(properties, vk::FormatFeatureFlagBits::eColorAttachment)) {
|
||||
usage |= vk::ImageUsageFlagBits::eColorAttachment;
|
||||
}
|
||||
if (info.samples == 1) {
|
||||
if (graphics.GetPhysicalDeviceProperties().limits.storageImageSampleCounts &
|
||||
static_cast<vk::SampleCountFlagBits>(info.samples)) {
|
||||
usage |= vk::ImageUsageFlagBits::eStorage;
|
||||
}
|
||||
return usage;
|
||||
@@ -211,6 +212,13 @@ Image::Barriers Image::GetBarriers(vk::ImageLayout destinat
|
||||
barriers.push_back(barrier);
|
||||
}
|
||||
|
||||
// Resident heaps sample their images in the read state; only leaving it invalidates them.
|
||||
if (std::ranges::any_of(barriers, [&](const vk::ImageMemoryBarrier2& barrier) {
|
||||
return barrier.oldLayout == ReadLayout() &&
|
||||
barrier.srcAccessMask == vk::AccessFlagBits2::eShaderRead;
|
||||
})) {
|
||||
HeapChanged();
|
||||
}
|
||||
state = {destination_stage, destination_access, destination_layout};
|
||||
return barriers;
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/image/imageInfo.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <compare>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
@@ -71,6 +72,7 @@ public:
|
||||
void MarkCpuDirty() {
|
||||
m_cpu_dirty = true;
|
||||
m_maybe_cpu_dirty = false;
|
||||
HeapChanged();
|
||||
}
|
||||
|
||||
[[nodiscard]] bool IsCpuDirty() const { return m_cpu_dirty || m_maybe_cpu_dirty; }
|
||||
@@ -80,6 +82,7 @@ public:
|
||||
if (!IsCpuDirty()) {
|
||||
m_maybe_cpu_hash = HashGuestEdges();
|
||||
m_maybe_cpu_dirty = true;
|
||||
HeapChanged();
|
||||
}
|
||||
}
|
||||
[[nodiscard]] bool ResolveMaybeCpuHash(uint64_t hash) {
|
||||
@@ -104,7 +107,10 @@ public:
|
||||
void ClearGpuModified() noexcept { m_gpu_modified = false; }
|
||||
|
||||
[[nodiscard]] bool IsBufferModified() const noexcept { return m_buffer_modified; }
|
||||
void MarkBufferModified() noexcept { m_buffer_modified = true; }
|
||||
void MarkBufferModified() noexcept {
|
||||
m_buffer_modified = true;
|
||||
HeapChanged();
|
||||
}
|
||||
void ClearBufferModified() noexcept { m_buffer_modified = false; }
|
||||
|
||||
[[nodiscard]] bool Overlaps(uint64_t address, uint64_t size,
|
||||
@@ -120,6 +126,19 @@ public:
|
||||
return backing.image == nullptr ? 0 : Common::AlignUp(info.data.size, 1024);
|
||||
}
|
||||
[[nodiscard]] uint64_t HashGuestEdges() const;
|
||||
[[nodiscard]] vk::ImageLayout ReadLayout() const noexcept {
|
||||
return info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
|
||||
: vk::ImageLayout::eShaderReadOnlyOptimal;
|
||||
}
|
||||
// Resident image heaps sample an image without rediscovering it until one changes its
|
||||
// contents, tracking, identity or layout.
|
||||
void HeapChanged() const noexcept {
|
||||
if (heap_refs.load(std::memory_order_relaxed) != 0u) {
|
||||
heap_epoch.fetch_add(1u, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
static inline std::atomic<uint64_t> heap_epoch {0};
|
||||
|
||||
ImageInfo info;
|
||||
VulkanImage backing;
|
||||
@@ -137,6 +156,8 @@ public:
|
||||
ImageSubresourceRange stencil_subresources;
|
||||
uint64_t tick_accessed_last = 0;
|
||||
size_t lru_id = 0;
|
||||
// Resident image heap elements that select this image.
|
||||
std::atomic<uint32_t> heap_refs {0};
|
||||
|
||||
private:
|
||||
friend struct ImageTestAccess;
|
||||
|
||||
@@ -101,6 +101,7 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
|
||||
(resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1D ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim1DArray ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
|
||||
((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
|
||||
|
||||
@@ -254,6 +254,10 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
|
||||
const bool is_3d = resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D &&
|
||||
descriptor.Type() == Prospero::ImageType::kColor3D &&
|
||||
descriptor.BaseArray5() == 0;
|
||||
const bool is_2d_msaa =
|
||||
resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa &&
|
||||
descriptor.Type() == Prospero::ImageType::kColor2DMsaa && descriptor.Depth() == 0 &&
|
||||
descriptor.BaseArray5() == 0;
|
||||
TileTextureBlockLayout tile_layout {};
|
||||
bool supported_tile = false;
|
||||
switch (tile) {
|
||||
@@ -284,8 +288,8 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
|
||||
const bool supported_swizzle =
|
||||
IsValidImageSwizzle(swizzle) &&
|
||||
(swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
|
||||
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
|
||||
descriptor.BaseLevel() <= descriptor.LastLevel() &&
|
||||
return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d || is_2d_msaa) &&
|
||||
supported_tile && descriptor.BaseLevel() <= descriptor.LastLevel() &&
|
||||
descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
|
||||
!descriptor.MsaaDepth();
|
||||
}
|
||||
@@ -356,7 +360,7 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
|
||||
}
|
||||
view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
|
||||
view.mapping =
|
||||
storage || surface_format.conversion_format != Prospero::BufferFormat::kInvalid
|
||||
storage || resource.conversion_format != Prospero::BufferFormat::kInvalid
|
||||
? vk::ComponentMapping {}
|
||||
: TextureGetComponentMapping(descriptor.DstSelXYZW(), surface_format.host_to_storage);
|
||||
switch (resource.dimension) {
|
||||
@@ -515,7 +519,9 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
|
||||
}
|
||||
const auto samples = multisampled ? 1u << last_level : 1u;
|
||||
const auto depth = static_cast<uint32_t>(descriptor.Depth()) + 1u;
|
||||
const auto format = descriptor.Format();
|
||||
const auto format = ShaderRecompiler::IR::ReadsDepthBits(resource)
|
||||
? Prospero::BufferFormat::k16UNorm
|
||||
: descriptor.Format();
|
||||
const auto surface_format = TextureGetSurfaceFormatInfo(format);
|
||||
const bool shader_conversion =
|
||||
surface_format.conversion_format != Prospero::BufferFormat::kInvalid;
|
||||
@@ -579,6 +585,10 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
|
||||
const auto alignment = desc.info.DataAlignment();
|
||||
EXIT_NOT_IMPLEMENTED(desc.info.data.size == 0 || alignment == 0 ||
|
||||
(address & (static_cast<uint64_t>(alignment) - 1u)) != 0);
|
||||
// Image heaps keep the descriptors of unmapped textures, which PS5 never samples.
|
||||
if (resource.heap && !m_context.IsMapped(address, desc.info.data.size)) {
|
||||
return {.layout = vk::ImageLayout::eGeneral};
|
||||
}
|
||||
if (storage) {
|
||||
ValidateStorageTexture(resource, descriptor, desc.info.data.size);
|
||||
}
|
||||
@@ -684,6 +694,10 @@ void RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime,
|
||||
prepared.samplers.clear();
|
||||
prepared.shader_data.clear();
|
||||
for (uint32_t i = 0; i < program.info.images.size(); i++) {
|
||||
if (program.info.images[i].heap) {
|
||||
prepared.images[i] = {};
|
||||
continue;
|
||||
}
|
||||
auto binding = ResolveTexture(program.info.images[i], snapshot.images[i]);
|
||||
BindImage(binding.image_id, binding.desc.type == TextureCache::BindingType::Storage);
|
||||
binding.mip_views.swap(prepared.images[i].mip_views);
|
||||
@@ -846,14 +860,19 @@ void RenderExecutor::RebindImages(PreparedBindings& prepared) {
|
||||
|
||||
void RenderExecutor::PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
|
||||
std::span<RenderColorInfo> colors) {
|
||||
bool uses_dma = false;
|
||||
bool uses_dma = false;
|
||||
bool uses_image_heap = false;
|
||||
FindBuffers(stages);
|
||||
for (auto* stage: stages) {
|
||||
uses_dma |= stage->runtime->program->info.uses_dma;
|
||||
uses_image_heap |= stage->runtime->program->info.uses_image_heap;
|
||||
}
|
||||
if (uses_dma) {
|
||||
m_context.PrepareBda();
|
||||
}
|
||||
if (uses_image_heap) {
|
||||
m_context.GetImageHeap().Rebind(*this);
|
||||
}
|
||||
for (auto* stage: stages) {
|
||||
RebindImages(*stage);
|
||||
}
|
||||
@@ -953,6 +972,9 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pipeline.uses_image_heap) {
|
||||
m_context.GetImageHeap().Commit(buffer, pipeline_bind_point, pipeline.pipeline_layout);
|
||||
}
|
||||
m_descriptor_images.resize(image_descriptor_count);
|
||||
m_descriptor_writes.clear();
|
||||
m_descriptor_writes.reserve(write_count);
|
||||
@@ -1010,9 +1032,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
|
||||
vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
|
||||
range, vk_buffer);
|
||||
} else {
|
||||
image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
|
||||
: vk::ImageLayout::eShaderReadOnlyOptimal,
|
||||
vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
|
||||
image.Transit(image.ReadLayout(), vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
|
||||
}
|
||||
binding.layout = image.backing.state.layout;
|
||||
}
|
||||
@@ -1083,6 +1103,7 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
|
||||
for (uint32_t i = 0; i < program.info.images.size(); ++i) {
|
||||
const auto& resource = program.info.images[i];
|
||||
const auto& texture = descriptors.images[i];
|
||||
if (resource.heap) continue;
|
||||
EXIT_IF(texture.image_id &&
|
||||
resource.mip_count != (texture.mip_views.empty() ? 1u : texture.mip_views.size()));
|
||||
const auto kind = *ShaderRecompiler::IR::DescriptorBindingForImage(resource);
|
||||
|
||||
@@ -0,0 +1,544 @@
|
||||
#include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "common/assert.h"
|
||||
#include "common/profiler.h"
|
||||
#include "graphics/host_gpu/pageManager.h"
|
||||
#include "graphics/host_gpu/regionDefinitions.h"
|
||||
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
|
||||
#include "kernel/memory.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <xxhash.h>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
namespace {
|
||||
|
||||
namespace IR = ShaderRecompiler::IR;
|
||||
|
||||
// Smaller tables are cheap to materialize on each draw and are often transient.
|
||||
constexpr uint64_t MinTableBytes = 64ull * 1024ull;
|
||||
constexpr uint32_t ElementMask = (1u << IR::ImageHeapKindShift) - 1u;
|
||||
|
||||
template <typename Pred, typename Fn>
|
||||
void ForEachRun(size_t begin, size_t end, Pred&& pred, Fn&& fn) {
|
||||
while (begin < end) {
|
||||
if (!pred(begin)) {
|
||||
++begin;
|
||||
continue;
|
||||
}
|
||||
auto run = begin + 1u;
|
||||
while (run < end && pred(run)) ++run;
|
||||
fn(begin, run);
|
||||
begin = run;
|
||||
}
|
||||
}
|
||||
|
||||
// Grows [begin, end) to cover [first, last).
|
||||
void Widen(size_t& begin, size_t& end, size_t first, size_t last) {
|
||||
const bool empty = begin == end;
|
||||
begin = empty ? first : std::min(begin, first);
|
||||
end = empty ? last : std::max(end, last);
|
||||
}
|
||||
|
||||
// Guest bytes of a table that its tracker pages [begin, end) cover.
|
||||
std::pair<uint64_t, uint64_t> PageRange(uint64_t base, uint64_t size, size_t begin, size_t end) {
|
||||
const auto origin = Common::AlignDown(base, TRACKER_PAGE_SIZE);
|
||||
return {std::max(base, origin + begin * TRACKER_PAGE_SIZE),
|
||||
std::min(base + size, origin + end * TRACKER_PAGE_SIZE)};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
size_t ImageHeap::DescriptorHash::operator()(const DescriptorValue& value) const noexcept {
|
||||
return XXH3_64bits(value.dwords.data(), sizeof(value.dwords));
|
||||
}
|
||||
|
||||
ImageHeap::ImageHeap(RenderContext& context, PageManager& page_manager)
|
||||
: m_context(context), m_page_manager(page_manager) {
|
||||
auto& graphics = context.GetGraphics();
|
||||
vk::PhysicalDeviceVulkan12Properties limits {};
|
||||
vk::PhysicalDeviceProperties2 properties {};
|
||||
properties.pNext = &limits;
|
||||
graphics.physical_device.getProperties2(&properties);
|
||||
EXIT_NOT_IMPLEMENTED(std::min({limits.maxDescriptorSetUpdateAfterBindSampledImages,
|
||||
limits.maxPerStageDescriptorUpdateAfterBindSampledImages,
|
||||
limits.maxPerStageUpdateAfterBindResources}) <
|
||||
IR::ImageHeapCapacity);
|
||||
|
||||
const vk::DescriptorBindingFlags binding_flags =
|
||||
vk::DescriptorBindingFlagBits::ePartiallyBound |
|
||||
vk::DescriptorBindingFlagBits::eUpdateAfterBind |
|
||||
vk::DescriptorBindingFlagBits::eUpdateUnusedWhilePending;
|
||||
vk::DescriptorSetLayoutBindingFlagsCreateInfo flags_info {};
|
||||
flags_info.bindingCount = 1;
|
||||
flags_info.pBindingFlags = &binding_flags;
|
||||
const vk::DescriptorSetLayoutBinding binding {0, vk::DescriptorType::eSampledImage,
|
||||
IR::ImageHeapCapacity,
|
||||
vk::ShaderStageFlagBits::eAll, nullptr};
|
||||
vk::DescriptorSetLayoutCreateInfo layout_info {};
|
||||
layout_info.pNext = &flags_info;
|
||||
layout_info.flags = vk::DescriptorSetLayoutCreateFlagBits::eUpdateAfterBindPool;
|
||||
layout_info.bindingCount = 1;
|
||||
layout_info.pBindings = &binding;
|
||||
EXIT_IF(graphics.device.createDescriptorSetLayout(&layout_info, nullptr, &m_layout) !=
|
||||
vk::Result::eSuccess);
|
||||
const vk::DescriptorPoolSize pool_size {vk::DescriptorType::eSampledImage,
|
||||
IR::ImageHeapCapacity};
|
||||
vk::DescriptorPoolCreateInfo pool_info {};
|
||||
pool_info.flags = vk::DescriptorPoolCreateFlagBits::eUpdateAfterBind;
|
||||
pool_info.maxSets = 1;
|
||||
pool_info.poolSizeCount = 1;
|
||||
pool_info.pPoolSizes = &pool_size;
|
||||
EXIT_IF(graphics.device.createDescriptorPool(&pool_info, nullptr, &m_pool) !=
|
||||
vk::Result::eSuccess);
|
||||
vk::DescriptorSetAllocateInfo allocate {};
|
||||
allocate.descriptorPool = m_pool;
|
||||
allocate.descriptorSetCount = 1;
|
||||
allocate.pSetLayouts = &m_layout;
|
||||
EXIT_IF(graphics.device.allocateDescriptorSets(&allocate, &m_set) != vk::Result::eSuccess);
|
||||
graphics.image_heap_layout = m_layout;
|
||||
// Slot word 0 selects element 0, which stays a null image.
|
||||
m_elements.push_back({.binding = {.layout = vk::ImageLayout::eShaderReadOnlyOptimal}});
|
||||
m_writes.push_back(0u);
|
||||
WriteDescriptors();
|
||||
}
|
||||
|
||||
ImageHeap::~ImageHeap() {
|
||||
for (const auto& memory: m_memories) Unwatch(*memory);
|
||||
auto& graphics = m_context.GetGraphics();
|
||||
graphics.image_heap_layout = nullptr;
|
||||
graphics.device.destroyDescriptorPool(m_pool, nullptr);
|
||||
graphics.device.destroyDescriptorSetLayout(m_layout, nullptr);
|
||||
}
|
||||
|
||||
bool ImageHeap::Acquire(const IR::ImageHeapRequest& request, IR::ImageHeapView& result) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
const auto size = request.size & ~uint64_t {3};
|
||||
if (size < MinTableBytes || request.count == 0u || request.step == 0u ||
|
||||
((request.base | request.first | request.step) & 3u) != 0u) {
|
||||
return false;
|
||||
}
|
||||
auto memory = std::ranges::find_if(m_memories, [&](const auto& candidate) {
|
||||
return candidate->base == request.base && candidate->size == size;
|
||||
});
|
||||
if (memory == m_memories.end()) {
|
||||
if (!m_context.IsMapped(request.base, size)) return false;
|
||||
auto created = std::make_unique<Memory>();
|
||||
created->base = request.base;
|
||||
created->size = size;
|
||||
created->words.resize(size / 4u);
|
||||
created->dead.resize(size / 4u);
|
||||
created->dirty_pages.assign((Common::AlignUp(request.base + size, TRACKER_PAGE_SIZE) -
|
||||
Common::AlignDown(request.base, TRACKER_PAGE_SIZE)) /
|
||||
TRACKER_PAGE_SIZE,
|
||||
uint8_t {1});
|
||||
std::scoped_lock lock {m_mutex};
|
||||
memory = m_memories.insert(m_memories.end(), std::move(created));
|
||||
}
|
||||
// A draw refreshes a table and a view takes its changes once, so all programs of the draw see
|
||||
// one snapshot and one set of kinds.
|
||||
if ((*memory)->draw != m_draw) {
|
||||
if (!Refresh(**memory)) return false;
|
||||
(*memory)->draw = m_draw;
|
||||
}
|
||||
auto& views = (*memory)->views;
|
||||
const auto profile = IR::ImageHeapResource(*request.root, 0u);
|
||||
auto view = std::ranges::find_if(views, [&](const auto& candidate) {
|
||||
return candidate->step == request.step && candidate->first == request.first &&
|
||||
candidate->count == request.count && candidate->profile == profile;
|
||||
});
|
||||
if (view == views.end()) {
|
||||
const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
|
||||
auto created = std::make_unique<View>(View {.memory = **memory,
|
||||
.step = request.step,
|
||||
.first = request.first,
|
||||
.count = request.count,
|
||||
.profile = profile,
|
||||
.resolve_epoch = epoch,
|
||||
.layout_epoch = epoch});
|
||||
created->slots.resize(size / 4u + 1u);
|
||||
created->buffer = std::make_unique<Buffer>(
|
||||
m_context.GetGraphics(), m_context.GetCommandScheduler(), MemoryUsage::DeviceLocal, 0,
|
||||
vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer |
|
||||
vk::BufferUsageFlagBits::eShaderDeviceAddress,
|
||||
size + 4u);
|
||||
UpdateProbes(*created, 0u, size);
|
||||
// The first upload also clears the slots no probe selects.
|
||||
created->dirty_begin = 0;
|
||||
created->dirty_end = created->slots.size();
|
||||
view = views.insert(views.end(), std::move(created));
|
||||
} else {
|
||||
// Empty after the first acquire in a draw, since the table refreshes once per draw.
|
||||
UpdateProbes(**view, (*view)->changed_begin, (*view)->changed_end);
|
||||
(*view)->changed_begin = (*view)->changed_end = 0;
|
||||
}
|
||||
if (std::ranges::find(m_active, view->get()) == m_active.end()) m_active.push_back(view->get());
|
||||
const auto& selected = **view;
|
||||
result = {.slots = selected.buffer->BufferDeviceAddress(),
|
||||
.slot_count = static_cast<uint32_t>(selected.slots.size() - 1u),
|
||||
.kinds = selected.kinds};
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ImageHeap::Refresh(Memory& memory) {
|
||||
// GPU writes leave no fault; reading the table downloads them.
|
||||
const bool gpu_dirty = m_context.GetBufferCache().HasGpuDirtyBytes(memory.base, memory.size);
|
||||
const auto pages = memory.dirty_pages.size();
|
||||
m_runs.clear();
|
||||
{
|
||||
std::scoped_lock lock {m_mutex};
|
||||
// Watch before reading, so a write during the read faults the page again.
|
||||
ForEachRun(
|
||||
0, pages, [&](size_t page) { return memory.dirty_pages[page] != 0u; },
|
||||
[&](size_t begin, size_t end) {
|
||||
std::fill(memory.dirty_pages.begin() + begin, memory.dirty_pages.begin() + end,
|
||||
uint8_t {0});
|
||||
Watch<true>(memory, begin, end);
|
||||
m_runs.emplace_back(begin, end);
|
||||
});
|
||||
}
|
||||
if (gpu_dirty) m_runs.assign(1u, {0u, pages});
|
||||
for (size_t run = 0; run < m_runs.size(); ++run) {
|
||||
const auto [address, end] = PageRange(memory.base, memory.size, m_runs[run].first,
|
||||
m_runs[run].second);
|
||||
m_scratch.resize((end - address) / 4u);
|
||||
if (!LibKernel::Memory::TryReadBufferBacking(address, m_scratch.data(), end - address)) {
|
||||
std::scoped_lock lock {m_mutex};
|
||||
for (; run < m_runs.size(); ++run) {
|
||||
MarkDirty(memory, m_runs[run].first, m_runs[run].second);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const auto words = std::span(memory.words).subspan((address - memory.base) / 4u,
|
||||
m_scratch.size());
|
||||
size_t first = 0;
|
||||
size_t last = words.size();
|
||||
while (first < last && words[first] == m_scratch[first]) ++first;
|
||||
while (first < last && words[last - 1u] == m_scratch[last - 1u]) --last;
|
||||
if (first == last) continue;
|
||||
std::copy(m_scratch.begin() + first, m_scratch.begin() + last, words.begin() + first);
|
||||
const auto offset = address - memory.base;
|
||||
for (auto& view: memory.views) {
|
||||
Widen(view->changed_begin, view->changed_end, offset + first * 4u, offset + last * 4u);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void ImageHeap::MarkDirty(Memory& memory, size_t begin, size_t end) {
|
||||
ForEachRun(
|
||||
begin, end, [&](size_t page) { return memory.dirty_pages[page] == 0u; },
|
||||
[&](size_t first, size_t last) {
|
||||
std::fill(memory.dirty_pages.begin() + first, memory.dirty_pages.begin() + last,
|
||||
uint8_t {1});
|
||||
Watch<false>(memory, first, last);
|
||||
});
|
||||
}
|
||||
|
||||
template <bool track>
|
||||
void ImageHeap::Watch(const Memory& memory, size_t begin, size_t end) {
|
||||
const auto [address, limit] = PageRange(memory.base, memory.size, begin, end);
|
||||
m_page_manager.UpdatePageWatchers<track>(address, limit - address);
|
||||
}
|
||||
|
||||
void ImageHeap::Unwatch(const Memory& memory) {
|
||||
ForEachRun(
|
||||
0, memory.dirty_pages.size(), [&](size_t page) { return memory.dirty_pages[page] == 0u; },
|
||||
[&](size_t begin, size_t end) { Watch<false>(memory, begin, end); });
|
||||
}
|
||||
|
||||
void ImageHeap::InvalidateMemory(uint64_t address, uint64_t size) {
|
||||
std::scoped_lock lock {m_mutex};
|
||||
for (auto& memory: m_memories) {
|
||||
const auto begin = std::max(address, memory->base);
|
||||
const auto end = std::min(address + size, memory->base + memory->size);
|
||||
if (begin >= end) continue;
|
||||
const auto origin = Common::AlignDown(memory->base, TRACKER_PAGE_SIZE);
|
||||
MarkDirty(*memory, (begin - origin) / TRACKER_PAGE_SIZE,
|
||||
(end - 1u - origin) / TRACKER_PAGE_SIZE + 1u);
|
||||
}
|
||||
}
|
||||
|
||||
void ImageHeap::UpdateProbes(View& view, uint64_t begin, uint64_t end) {
|
||||
// A probe reads its descriptor from first + probe * step.
|
||||
const uint64_t bytes = view.profile.r128 ? 16u : 32u;
|
||||
auto probe = begin < view.first + bytes ? 0u : (begin - view.first - bytes) / view.step + 1u;
|
||||
for (; probe < view.count && view.first + probe * view.step < end; ++probe) {
|
||||
UpdateProbe(view, probe);
|
||||
}
|
||||
}
|
||||
|
||||
void ImageHeap::UpdateProbe(View& view, uint64_t probe) {
|
||||
const auto& words = view.memory.words;
|
||||
const auto offset = (view.first + probe * view.step) / 4u;
|
||||
DescriptorValue value {.dword_count = 8u};
|
||||
std::copy_n(words.begin() + offset,
|
||||
std::min<size_t>(view.profile.r128 ? 4u : 8u, words.size() - offset),
|
||||
value.dwords.begin());
|
||||
// A buried word stays null in every view until its descriptor changes.
|
||||
auto& dead = view.memory.dead[offset];
|
||||
if (dead != 0u && dead != DescriptorHash {}(value)) dead = 0u;
|
||||
const auto kind = dead != 0u ? 0u : IR::ImageHeapKind(view.profile, value);
|
||||
Assign(view, offset, kind == 0u ? 0u : Intern(view, kind, value));
|
||||
}
|
||||
|
||||
void ImageHeap::Assign(View& view, size_t slot, uint32_t entry) {
|
||||
const auto previous = std::exchange(view.slots[slot], entry);
|
||||
Release(view, previous);
|
||||
if (previous != entry) Widen(view.dirty_begin, view.dirty_end, slot, slot + 1u);
|
||||
}
|
||||
|
||||
void ImageHeap::Bury() {
|
||||
if (m_unmapped.empty()) return;
|
||||
for (const auto& memory: m_memories) {
|
||||
for (const auto& view: memory->views) {
|
||||
bool buried = false;
|
||||
for (const auto& [value, index]: view->elements) {
|
||||
const auto& binding = m_elements[index].binding;
|
||||
if (binding.image_id && std::ranges::any_of(m_unmapped, [&](GuestRange range) {
|
||||
return ImageRangeOverlaps(binding.desc.info.data, range);
|
||||
})) {
|
||||
m_elements[index].stale = buried = true;
|
||||
}
|
||||
}
|
||||
if (!buried) continue;
|
||||
for (uint64_t probe = 0; probe < view->count; ++probe) {
|
||||
const auto slot = (view->first + probe * view->step) / 4u;
|
||||
const auto& element = m_elements[view->slots[slot] & ElementMask];
|
||||
if (!element.stale) continue;
|
||||
memory->dead[slot] = DescriptorHash {}(element.value);
|
||||
Assign(*view, slot, 0u);
|
||||
}
|
||||
}
|
||||
}
|
||||
m_unmapped.clear();
|
||||
}
|
||||
|
||||
uint32_t ImageHeap::Intern(View& view, uint32_t kind, const DescriptorValue& value) {
|
||||
const auto arm = static_cast<uint32_t>(std::ranges::find(view.kind_keys, kind) -
|
||||
view.kind_keys.begin());
|
||||
if (arm == view.kind_keys.size()) {
|
||||
view.kind_keys.push_back(kind);
|
||||
view.kinds.push_back(value);
|
||||
}
|
||||
auto [found, inserted] = view.elements.try_emplace(value, 0u);
|
||||
if (inserted) {
|
||||
found->second = Allocate();
|
||||
auto& element = m_elements[found->second];
|
||||
element.value = value;
|
||||
element.kind = kind;
|
||||
view.pending.push_back(found->second);
|
||||
}
|
||||
++m_elements[found->second].refs;
|
||||
return arm << IR::ImageHeapKindShift | found->second;
|
||||
}
|
||||
|
||||
void ImageHeap::Release(View& view, uint32_t entry) {
|
||||
const auto index = entry & ElementMask;
|
||||
if (index == 0u || --m_elements[index].refs != 0u) return;
|
||||
auto& element = m_elements[index];
|
||||
if (const auto found = view.elements.find(element.value);
|
||||
found != view.elements.end() && found->second == index) {
|
||||
view.elements.erase(found);
|
||||
}
|
||||
Unreference(element);
|
||||
view.released.push_back(index);
|
||||
}
|
||||
|
||||
void ImageHeap::Unreference(Element& element) {
|
||||
if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(element.binding.image_id)) {
|
||||
image->heap_refs.fetch_sub(1u, std::memory_order_relaxed);
|
||||
image->tick_accessed_last = m_context.GetCommandScheduler().CurrentTick();
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t ImageHeap::Allocate() {
|
||||
auto& scheduler = m_context.GetCommandScheduler();
|
||||
while (!m_retired.empty() && scheduler.IsFree(m_retired.front().first)) {
|
||||
m_free.push_back(m_retired.front().second);
|
||||
m_retired.pop_front();
|
||||
}
|
||||
if (m_free.empty()) {
|
||||
if (m_elements.size() == IR::ImageHeapCapacity) {
|
||||
EXIT("image heaps exceed %u resident images\n", IR::ImageHeapCapacity);
|
||||
}
|
||||
m_elements.emplace_back();
|
||||
return static_cast<uint32_t>(m_elements.size() - 1u);
|
||||
}
|
||||
const auto index = m_free.back();
|
||||
m_free.pop_back();
|
||||
m_elements[index] = {};
|
||||
return index;
|
||||
}
|
||||
|
||||
void ImageHeap::Retire(View& view) {
|
||||
const auto tick = m_context.GetCommandScheduler().CurrentTick();
|
||||
for (const auto index: view.released) m_retired.emplace_back(tick, index);
|
||||
view.released.clear();
|
||||
}
|
||||
|
||||
void ImageHeap::Rebind(RenderExecutor& executor) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
auto& textures = m_context.GetTextureCache();
|
||||
const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
|
||||
for (auto* view: m_active) {
|
||||
bool stale = false;
|
||||
const auto mark = [&](uint32_t index) { m_elements[index].stale = stale = true; };
|
||||
if (view->resolve_epoch != epoch) {
|
||||
view->resolve_epoch = epoch;
|
||||
for (const auto& [value, index]: view->elements) {
|
||||
const auto& binding = m_elements[index].binding;
|
||||
if (!binding.image_id) continue;
|
||||
const auto* image = textures.m_slot_images.try_get(binding.image_id);
|
||||
if (image == nullptr || !image->registered || image->depth_id ||
|
||||
image->binding.needs_rebind) {
|
||||
mark(index);
|
||||
} else {
|
||||
(void)textures.FindTexture(binding.image_id, binding.desc);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Discovery applies the color metadata clears written since the last draw.
|
||||
std::erase_if(view->dcc, [&](uint32_t index) {
|
||||
const auto& element = m_elements[index];
|
||||
if (element.refs == 0u || element.stale) return true;
|
||||
if (executor.ResolveTexture(IR::ImageHeapResource(view->profile, element.kind),
|
||||
element.value).image_id == element.binding.image_id) {
|
||||
return false;
|
||||
}
|
||||
mark(index);
|
||||
return true;
|
||||
});
|
||||
if (stale) {
|
||||
// A changed image needs a new element; pending draws still read the old one. The new
|
||||
// element keeps the old value, since a view takes table changes only when acquired.
|
||||
std::erase_if(view->elements,
|
||||
[&](const auto& item) { return m_elements[item.second].stale; });
|
||||
for (uint64_t probe = 0; probe < view->count; ++probe) {
|
||||
const auto slot = (view->first + probe * view->step) / 4u;
|
||||
const auto& element = m_elements[view->slots[slot] & ElementMask];
|
||||
if (!element.stale) continue;
|
||||
const auto value = element.value; // Intern may grow m_elements.
|
||||
Assign(*view, slot, Intern(*view, element.kind, value));
|
||||
}
|
||||
}
|
||||
for (const auto index: view->pending) {
|
||||
if (m_elements[index].refs == 0u) continue;
|
||||
auto binding = executor.ResolveTexture(
|
||||
IR::ImageHeapResource(view->profile, m_elements[index].kind),
|
||||
m_elements[index].value);
|
||||
if (binding.image_id) {
|
||||
auto& image = textures.GetImage(binding.image_id);
|
||||
image.heap_refs.fetch_add(1u, std::memory_order_relaxed);
|
||||
binding.layout = image.ReadLayout();
|
||||
binding.image_view = textures.FindTexture(binding.image_id, binding.desc);
|
||||
if (binding.desc.info.metadata.kind == ImageMetadataKind::Dcc) {
|
||||
view->dcc.push_back(index);
|
||||
}
|
||||
view->away.push_back(index);
|
||||
}
|
||||
m_elements[index].binding = std::move(binding);
|
||||
m_writes.push_back(index);
|
||||
}
|
||||
view->pending.clear();
|
||||
}
|
||||
WriteDescriptors();
|
||||
}
|
||||
|
||||
void ImageHeap::WriteDescriptors() {
|
||||
if (m_writes.empty()) return;
|
||||
std::vector<vk::DescriptorImageInfo> infos;
|
||||
std::vector<vk::WriteDescriptorSet> writes(m_writes.size());
|
||||
infos.reserve(m_writes.size());
|
||||
for (size_t write = 0; write < m_writes.size(); ++write) {
|
||||
const auto& binding = m_elements[m_writes[write]].binding;
|
||||
infos.push_back({nullptr, binding.image_view, binding.layout});
|
||||
writes[write].dstSet = m_set;
|
||||
writes[write].dstArrayElement = m_writes[write];
|
||||
writes[write].descriptorCount = 1;
|
||||
writes[write].descriptorType = vk::DescriptorType::eSampledImage;
|
||||
writes[write].pImageInfo = &infos.back();
|
||||
}
|
||||
m_context.GetGraphics().device.updateDescriptorSets(static_cast<uint32_t>(writes.size()),
|
||||
writes.data(), 0, nullptr);
|
||||
m_writes.clear();
|
||||
}
|
||||
|
||||
void ImageHeap::Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point,
|
||||
vk::PipelineLayout layout) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
auto& textures = m_context.GetTextureCache();
|
||||
const auto epoch = Image::heap_epoch.load(std::memory_order_relaxed);
|
||||
for (auto* view: m_active) {
|
||||
if (view->dirty_begin != view->dirty_end) {
|
||||
const auto slots = std::span(view->slots).subspan(view->dirty_begin,
|
||||
view->dirty_end - view->dirty_begin);
|
||||
auto& stream = m_context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
|
||||
const auto offset = stream.Copy(slots.data(), slots.size_bytes(), sizeof(uint32_t));
|
||||
view->buffer->CopyFrom(buffer, stream, offset, view->dirty_begin * sizeof(uint32_t),
|
||||
slots.size_bytes());
|
||||
view->dirty_begin = view->dirty_end = 0;
|
||||
}
|
||||
Retire(*view);
|
||||
if (view->layout_epoch != epoch) {
|
||||
view->layout_epoch = epoch;
|
||||
view->away.clear();
|
||||
for (const auto& [value, index]: view->elements) view->away.push_back(index);
|
||||
}
|
||||
// This draw's own bindings own the layouts of its attachments and storage images. An image
|
||||
// this draw's discovery replaced lives until the tick ends, and its slots still select it.
|
||||
std::erase_if(view->away, [&](uint32_t index) {
|
||||
const auto& element = m_elements[index];
|
||||
auto* image = textures.m_slot_images.try_get(element.binding.image_id);
|
||||
if (element.refs == 0u || image == nullptr) return true;
|
||||
if (image->binding.is_target || image->binding.shader_write ||
|
||||
image->binding.force_general) {
|
||||
return false;
|
||||
}
|
||||
const auto& view_info = element.binding.desc.view_info;
|
||||
image->Transit(image->ReadLayout(), vk::AccessFlagBits2::eShaderRead,
|
||||
ImageSubresourceRange {view_info.base_level, view_info.level_count,
|
||||
view_info.base_layer, view_info.layer_count},
|
||||
buffer.Handle());
|
||||
return true;
|
||||
});
|
||||
}
|
||||
m_active.clear();
|
||||
buffer.Handle().bindDescriptorSets(bind_point, layout, IR::ImageHeapSet, 1, &m_set, 0,
|
||||
nullptr);
|
||||
}
|
||||
|
||||
void ImageHeap::UnmapMemory(uint64_t address, uint64_t size) {
|
||||
// Descriptors of textures in the range stay null even once it is mapped again.
|
||||
if (!m_unmapped.empty() && m_unmapped.back().End() == address) {
|
||||
m_unmapped.back().size += size;
|
||||
} else {
|
||||
m_unmapped.push_back({address, size});
|
||||
}
|
||||
auto& scheduler = m_context.GetCommandScheduler();
|
||||
std::scoped_lock lock {m_mutex};
|
||||
std::erase_if(m_memories, [&](const std::unique_ptr<Memory>& memory) {
|
||||
if (address >= memory->base + memory->size || memory->base >= address + size) {
|
||||
return false;
|
||||
}
|
||||
Unwatch(*memory);
|
||||
for (auto& view: memory->views) {
|
||||
std::erase(m_active, view.get());
|
||||
for (const auto& [value, index]: view->elements) {
|
||||
Unreference(m_elements[index]);
|
||||
view->released.push_back(index);
|
||||
}
|
||||
Retire(*view);
|
||||
if (scheduler.Active()) {
|
||||
scheduler.DeferOperation([buffer = std::move(view->buffer)] {});
|
||||
}
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
@@ -0,0 +1,138 @@
|
||||
#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
|
||||
#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
|
||||
|
||||
#include "common/common.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
class Buffer;
|
||||
class CommandBuffer;
|
||||
class PageManager;
|
||||
class RenderContext;
|
||||
class RenderExecutor;
|
||||
struct ImageHeapTestAccess;
|
||||
|
||||
// Guest image tables that shaders index by GPU-computed offsets, kept resident in one bindless
|
||||
// sampled-image array. Each view of a table owns device slot words, one per table dword, that
|
||||
// select an array element. CPU writes to a table fault in its snapshot, which each draw refreshes
|
||||
// once; a view takes the changes when a draw first acquires it. Changes to the images behind
|
||||
// elements bump Image::heap_epoch.
|
||||
class ImageHeap final: public ShaderRecompiler::IR::ImageHeapSource {
|
||||
public:
|
||||
ImageHeap(RenderContext& context, PageManager& page_manager);
|
||||
~ImageHeap();
|
||||
KYTY_CLASS_NO_COPY(ImageHeap);
|
||||
|
||||
// Starts a draw or dispatch, before its programs acquire views.
|
||||
void BeginDraw() {
|
||||
++m_draw;
|
||||
Bury();
|
||||
}
|
||||
bool Acquire(const ShaderRecompiler::IR::ImageHeapRequest& request,
|
||||
ShaderRecompiler::IR::ImageHeapView& result) override;
|
||||
// Image discovery: resolves new elements and rediscovers changed images.
|
||||
void Rebind(RenderExecutor& executor);
|
||||
// Recording: uploads slots, moves resident images to their read layout and binds the array.
|
||||
void Commit(CommandBuffer& buffer, vk::PipelineBindPoint bind_point, vk::PipelineLayout layout);
|
||||
// CPU write fault, on any thread.
|
||||
void InvalidateMemory(uint64_t address, uint64_t size);
|
||||
void UnmapMemory(uint64_t address, uint64_t size);
|
||||
|
||||
private:
|
||||
using DescriptorValue = ShaderRecompiler::IR::DescriptorValue;
|
||||
|
||||
struct DescriptorHash {
|
||||
size_t operator()(const DescriptorValue& value) const noexcept;
|
||||
};
|
||||
|
||||
struct Element {
|
||||
DescriptorValue value;
|
||||
TextureBinding binding;
|
||||
uint32_t kind = 0;
|
||||
uint32_t refs = 0; // Slots of the owning view that select it.
|
||||
bool stale = false;
|
||||
};
|
||||
|
||||
struct Memory;
|
||||
|
||||
struct View {
|
||||
Memory& memory;
|
||||
uint64_t step = 0;
|
||||
uint64_t first = 0;
|
||||
uint64_t count = 0;
|
||||
ShaderRecompiler::IR::ImageResource profile;
|
||||
std::vector<uint32_t> slots; // Plus a trailing null slot.
|
||||
std::vector<uint32_t> kind_keys;
|
||||
std::vector<DescriptorValue> kinds; // One representative per kind_keys entry.
|
||||
std::unordered_map<DescriptorValue, uint32_t, DescriptorHash> elements;
|
||||
std::vector<uint32_t> pending; // Not yet resolved.
|
||||
std::vector<uint32_t> released; // No slot selects them after the next upload.
|
||||
std::vector<uint32_t> away; // Not yet moved to the read layout.
|
||||
std::vector<uint32_t> dcc; // Color metadata clears apply on each discovery.
|
||||
std::unique_ptr<Buffer> buffer;
|
||||
size_t dirty_begin = 0; // Slots to upload.
|
||||
size_t dirty_end = 0;
|
||||
size_t changed_begin = 0; // Table bytes changed since it was probed.
|
||||
size_t changed_end = 0;
|
||||
uint64_t resolve_epoch = 0;
|
||||
uint64_t layout_epoch = 0;
|
||||
};
|
||||
|
||||
struct Memory {
|
||||
uint64_t base = 0;
|
||||
uint64_t size = 0;
|
||||
std::vector<uint32_t> words;
|
||||
std::vector<uint8_t> dirty_pages; // Unwatched, so unread since a write.
|
||||
std::vector<uint64_t> dead; // Per word, a buried descriptor's hash.
|
||||
uint64_t draw = 0; // Last draw that refreshed it.
|
||||
std::vector<std::unique_ptr<View>> views;
|
||||
};
|
||||
|
||||
[[nodiscard]] bool Refresh(Memory& memory);
|
||||
void MarkDirty(Memory& memory, size_t begin, size_t end);
|
||||
template <bool track>
|
||||
void Watch(const Memory& memory, size_t begin, size_t end);
|
||||
void Unwatch(const Memory& memory);
|
||||
void UpdateProbes(View& view, uint64_t begin, uint64_t end);
|
||||
void UpdateProbe(View& view, uint64_t probe);
|
||||
void Assign(View& view, size_t slot, uint32_t entry);
|
||||
void Bury();
|
||||
uint32_t Intern(View& view, uint32_t kind, const DescriptorValue& value);
|
||||
void Release(View& view, uint32_t entry);
|
||||
void Unreference(Element& element);
|
||||
uint32_t Allocate();
|
||||
void Retire(View& view);
|
||||
void WriteDescriptors();
|
||||
|
||||
RenderContext& m_context;
|
||||
PageManager& m_page_manager;
|
||||
std::mutex m_mutex; // Fault-visible Memory state and m_memories.
|
||||
std::vector<std::unique_ptr<Memory>> m_memories;
|
||||
std::vector<View*> m_active; // Acquired since the last commit.
|
||||
uint64_t m_draw = 1;
|
||||
std::vector<Element> m_elements;
|
||||
std::vector<uint32_t> m_free;
|
||||
std::deque<std::pair<uint64_t, uint32_t>> m_retired; // Tick, element.
|
||||
std::vector<std::pair<size_t, size_t>> m_runs;
|
||||
std::vector<uint32_t> m_scratch;
|
||||
std::vector<uint32_t> m_writes; // Elements whose descriptors changed.
|
||||
std::vector<GuestRange> m_unmapped; // Since the last Bury.
|
||||
vk::DescriptorSetLayout m_layout = nullptr;
|
||||
vk::DescriptorPool m_pool = nullptr;
|
||||
vk::DescriptorSet m_set = nullptr;
|
||||
|
||||
friend struct ImageHeapTestAccess;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics
|
||||
|
||||
#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGEHEAP_H_
|
||||
@@ -353,6 +353,7 @@ struct PipelineCache::ProgramCache {
|
||||
.user_data = user_data,
|
||||
.shader_base = params.Base(),
|
||||
.read_specialization_memory = ReadShaderGuestMemory,
|
||||
.image_heap = image_heap,
|
||||
};
|
||||
if constexpr (std::is_same_v<InputInfo, ShaderComputeInputInfo>) {
|
||||
runtime.workgroup_counts = input_info.workgroup_counts;
|
||||
@@ -480,8 +481,8 @@ struct PipelineCache::ProgramCache {
|
||||
return permutation.handle;
|
||||
}
|
||||
|
||||
explicit ProgramCache(GraphicContext& graphics)
|
||||
: graphics(graphics),
|
||||
ProgramCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap)
|
||||
: graphics(graphics), image_heap(image_heap),
|
||||
// AMD GPUs (vendor 0x1002) run NMin/NMax as v_min/v_max, which give the guest's NaN and
|
||||
// signed-zero results; GLSL.std.450 alone does not order -0 below +0.
|
||||
profile {.maintenance8 = graphics.maintenance8_enabled,
|
||||
@@ -501,12 +502,14 @@ struct PipelineCache::ProgramCache {
|
||||
std::unordered_map<ProgramKey, SourceEntry, ProgramKeyHash> programs;
|
||||
ProgramKey lookup_key;
|
||||
GraphicContext& graphics;
|
||||
ShaderRecompiler::IR::ImageHeapSource* image_heap;
|
||||
const ShaderRecompiler::Spirv::Profile profile;
|
||||
uint64_t next_shader_id = 0;
|
||||
};
|
||||
|
||||
PipelineCache::PipelineCache(GraphicContext& graphics)
|
||||
: m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics)),
|
||||
PipelineCache::PipelineCache(GraphicContext& graphics,
|
||||
ShaderRecompiler::IR::ImageHeapSource* image_heap)
|
||||
: m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics, image_heap)),
|
||||
m_next_driver_cache_save(std::chrono::steady_clock::now() + DriverCacheSaveInterval) {
|
||||
EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
|
||||
InitializeDriverCache();
|
||||
|
||||
@@ -30,6 +30,10 @@ class UserConfig;
|
||||
struct ComputeShaderInfo;
|
||||
} // namespace HW
|
||||
|
||||
namespace ShaderRecompiler::IR {
|
||||
class ImageHeapSource;
|
||||
} // namespace ShaderRecompiler::IR
|
||||
|
||||
#pragma pack(push, 1)
|
||||
|
||||
struct PipelineStaticParameters {
|
||||
@@ -104,7 +108,7 @@ struct ShaderProgram {
|
||||
// The owning renderer serializes access, including saves while the GPU is running.
|
||||
class PipelineCache {
|
||||
public:
|
||||
explicit PipelineCache(GraphicContext& graphics);
|
||||
PipelineCache(GraphicContext& graphics, ShaderRecompiler::IR::ImageHeapSource* image_heap);
|
||||
~PipelineCache();
|
||||
KYTY_CLASS_NO_COPY(PipelineCache);
|
||||
void Save();
|
||||
@@ -114,6 +118,7 @@ public:
|
||||
vk::Pipeline pipeline = nullptr;
|
||||
vk::DescriptorSetLayout descriptor_set_layout = nullptr;
|
||||
bool uses_push_descriptors = false;
|
||||
bool uses_image_heap = false;
|
||||
};
|
||||
|
||||
struct GraphicsPrograms {
|
||||
|
||||
@@ -473,20 +473,23 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
|
||||
for (const auto& stage: vertex_info) {
|
||||
const auto native_stage = NativeShaderStage(stage.logical_stage);
|
||||
AddLayoutBindings(descriptor_bindings, *stage.stage.program, native_stage);
|
||||
pipeline.uses_image_heap |= stage.stage.program->info.uses_image_heap;
|
||||
graphics_stages |= native_stage;
|
||||
}
|
||||
if (ps_active) {
|
||||
EXIT_IF(!ps_input_info->stage);
|
||||
AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
|
||||
vk::ShaderStageFlagBits::eFragment);
|
||||
pipeline.uses_image_heap |= ps_input_info->stage.program->info.uses_image_heap;
|
||||
}
|
||||
CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
|
||||
const vk::PushConstantRange push_constants {graphics_stages, 0,
|
||||
ShaderRecompiler::IR::NativePushConstantSize};
|
||||
const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout};
|
||||
|
||||
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
|
||||
pipeline_layout_info.setLayoutCount = 1;
|
||||
pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout;
|
||||
pipeline_layout_info.setLayoutCount = pipeline.uses_image_heap ? 2u : 1u;
|
||||
pipeline_layout_info.pSetLayouts = set_layouts.data();
|
||||
pipeline_layout_info.pushConstantRangeCount = 1;
|
||||
pipeline_layout_info.pPushConstantRanges = &push_constants;
|
||||
|
||||
@@ -615,13 +618,15 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
|
||||
std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
|
||||
AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
|
||||
vk::ShaderStageFlagBits::eCompute);
|
||||
pipeline.uses_image_heap = input_info.stage.program->info.uses_image_heap;
|
||||
CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
|
||||
const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
|
||||
ShaderRecompiler::IR::NativePushConstantSize};
|
||||
const std::array set_layouts {pipeline.descriptor_set_layout, graphics.image_heap_layout};
|
||||
|
||||
vk::PipelineLayoutCreateInfo pipeline_layout_info {};
|
||||
pipeline_layout_info.setLayoutCount = 1;
|
||||
pipeline_layout_info.pSetLayouts = &pipeline.descriptor_set_layout;
|
||||
pipeline_layout_info.setLayoutCount = pipeline.uses_image_heap ? 2u : 1u;
|
||||
pipeline_layout_info.pSetLayouts = set_layouts.data();
|
||||
pipeline_layout_info.pushConstantRangeCount = 1;
|
||||
pipeline_layout_info.pPushConstantRanges = &push_constants;
|
||||
|
||||
|
||||
@@ -171,6 +171,8 @@ public:
|
||||
void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
|
||||
const PipelineCache::Pipeline& pipeline,
|
||||
std::span<PreparedBindings* const> bindings);
|
||||
[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
const ShaderRecompiler::IR::DescriptorValue& value);
|
||||
|
||||
private:
|
||||
// False, with nothing drawn, when the draw's indirect records need CPU counts.
|
||||
@@ -182,8 +184,6 @@ private:
|
||||
std::optional<PreparedBindings> pixel;
|
||||
};
|
||||
|
||||
[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
const ShaderRecompiler::IR::DescriptorValue& value);
|
||||
void PrepareGraphicsBindings(std::span<PreparedBindings* const> stages,
|
||||
std::span<RenderColorInfo> colors);
|
||||
void ResolveRenderColorTarget(CommandBuffer& buffer, RenderColorInfo& target,
|
||||
|
||||
@@ -311,6 +311,7 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
input_info.workgroup_counts[axis] = threads / size + (threads % size != 0u);
|
||||
}
|
||||
}
|
||||
m_context.GetImageHeap().BeginDraw();
|
||||
const auto compute_program =
|
||||
m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
|
||||
if (!compute_program) return;
|
||||
@@ -363,6 +364,9 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
if (program.info.uses_dma) {
|
||||
m_context.PrepareBda();
|
||||
}
|
||||
if (program.info.uses_image_heap) {
|
||||
m_context.GetImageHeap().Rebind(*this);
|
||||
}
|
||||
RebindImages(bindings);
|
||||
BindSharedMemory(m_context, input_info, bindings);
|
||||
RebindBuffers(bindings);
|
||||
@@ -497,6 +501,7 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
ShaderComputeInputInfo input_info {};
|
||||
input_info.async_compute = async_compute;
|
||||
input_info.indirect_args_address = args_addr;
|
||||
m_context.GetImageHeap().BeginDraw();
|
||||
const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
|
||||
cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
|
||||
if (!compute_program) return;
|
||||
@@ -511,6 +516,9 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
m_context.PrepareBda();
|
||||
}
|
||||
BindSharedMemory(m_context, input_info, bindings, args_addr);
|
||||
if (program.info.uses_image_heap) {
|
||||
m_context.GetImageHeap().Rebind(*this);
|
||||
}
|
||||
RebindImages(bindings);
|
||||
// Acquiring arguments can merge cache buffers; finalize shader bindings afterward.
|
||||
const auto [args_buffer, args_offset] = m_context.GetBufferCache().ObtainBuffer(
|
||||
|
||||
@@ -13,9 +13,10 @@ namespace Libs::Graphics {
|
||||
RenderContext::RenderContext(GraphicContext& graphics)
|
||||
: m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
|
||||
m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
|
||||
m_pipeline_cache(graphics), m_sampler_cache(graphics),
|
||||
m_pipeline_cache(graphics, &m_image_heap), m_sampler_cache(graphics),
|
||||
m_buffer_cache(graphics, m_command_scheduler, m_page_manager, m_texture_cache),
|
||||
m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache) {
|
||||
m_texture_cache(graphics, m_command_scheduler, m_page_manager, m_buffer_cache),
|
||||
m_image_heap(*this, m_page_manager) {
|
||||
EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
|
||||
}
|
||||
|
||||
@@ -64,6 +65,7 @@ bool RenderContext::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) no
|
||||
if (access == PageFaultAccess::Write) {
|
||||
m_buffer_cache.InvalidateMemory(fault_vaddr, fault_size);
|
||||
m_texture_cache.InvalidateMemory(fault_vaddr, fault_size);
|
||||
m_image_heap.InvalidateMemory(fault_vaddr, fault_size);
|
||||
} else {
|
||||
m_buffer_cache.ReadMemory(fault_vaddr, fault_size);
|
||||
}
|
||||
@@ -76,6 +78,7 @@ bool RenderContext::InvalidateMemory(uint64_t vaddr, uint64_t size) {
|
||||
}
|
||||
m_buffer_cache.InvalidateMemory(vaddr, size);
|
||||
m_texture_cache.InvalidateMemory(vaddr, size);
|
||||
m_image_heap.InvalidateMemory(vaddr, size);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -112,6 +115,7 @@ void RenderContext::UnmapMemory(uint64_t vaddr, uint64_t size) {
|
||||
}
|
||||
m_buffer_cache.InvalidateMemory(vaddr, size);
|
||||
m_texture_cache.UnmapMemory(vaddr, size);
|
||||
m_image_heap.UnmapMemory(vaddr, size);
|
||||
std::lock_guard lock(m_mapped_ranges_mutex);
|
||||
m_mapped_ranges.Subtract(vaddr, size);
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "graphics/host_gpu/renderer/cache/textureCache.h"
|
||||
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/imageHeap.h"
|
||||
#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
|
||||
#include "kernel/eventQueue.h"
|
||||
|
||||
@@ -43,6 +44,7 @@ public:
|
||||
CommandScheduler& GetCommandScheduler() { return m_command_scheduler; }
|
||||
PipelineCache& GetPipelineCache() { return m_pipeline_cache; }
|
||||
DescriptorHeap& GetDescriptorHeap() { return m_descriptor_heap; }
|
||||
ImageHeap& GetImageHeap() { return m_image_heap; }
|
||||
SamplerCache& GetSamplerCache() { return m_sampler_cache; }
|
||||
BufferCache& GetBufferCache() { return m_buffer_cache; }
|
||||
TextureCache& GetTextureCache() { return m_texture_cache; }
|
||||
@@ -76,6 +78,7 @@ private:
|
||||
PageManager m_page_manager;
|
||||
BufferCache m_buffer_cache;
|
||||
TextureCache m_texture_cache;
|
||||
ImageHeap m_image_heap;
|
||||
mutable std::shared_mutex m_mapped_ranges_mutex;
|
||||
RangeSet m_mapped_ranges;
|
||||
std::unique_ptr<GuestGpu> m_gpu;
|
||||
|
||||
@@ -893,6 +893,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
|
||||
}
|
||||
}
|
||||
auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
|
||||
buffer.GetContext().GetImageHeap().BeginDraw();
|
||||
if (draw.IsIndexed()) {
|
||||
LogDrawPhase(draw.Name(), "GetGraphicsPrograms");
|
||||
}
|
||||
|
||||
@@ -221,7 +221,7 @@ public:
|
||||
void WaitForSubmitSlot(VideoOutConfig& cfg);
|
||||
bool Flip(uint32_t micros);
|
||||
void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out);
|
||||
void Wait(VideoOutConfig& cfg, int index);
|
||||
bool IsSafeForRendering(VideoOutConfig& cfg, int index);
|
||||
|
||||
private:
|
||||
enum class RequestState { Reserved, Recording, Ready, Presenting };
|
||||
@@ -240,6 +240,7 @@ private:
|
||||
Graphics::Presenter::Frame* frame;
|
||||
bool premultiplied_alpha;
|
||||
};
|
||||
void SignalDoneLocked();
|
||||
|
||||
Graphics::Presenter& m_presenter;
|
||||
Common::Mutex m_mutex;
|
||||
@@ -250,6 +251,7 @@ private:
|
||||
std::list<Request> m_cpu_requests;
|
||||
std::list<Request> m_cancelled_requests;
|
||||
bool m_processing = false;
|
||||
bool m_gpu_waiting = false;
|
||||
uint64_t m_next_request_id = 1;
|
||||
};
|
||||
|
||||
@@ -953,7 +955,7 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe
|
||||
}
|
||||
}
|
||||
}
|
||||
m_done_cond_var.SignalAll();
|
||||
SignalDoneLocked();
|
||||
}
|
||||
if (cfg.flip_status.flipPendingNum >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
|
||||
Common::DebugCounters::Add(Common::DebugCounters::Counter::FlipsRejected);
|
||||
@@ -1033,7 +1035,7 @@ void FlipQueue::Cancel(VideoOutConfig& cfg) {
|
||||
it = queue->erase(it);
|
||||
}
|
||||
}
|
||||
m_done_cond_var.SignalAll();
|
||||
SignalDoneLocked();
|
||||
m_submit_slot_cond_var.SignalAll();
|
||||
m_submit_cond_var.SignalAll();
|
||||
m_mutex.Unlock();
|
||||
@@ -1205,19 +1207,26 @@ void FlipQueue::WaitForSubmitSlot(VideoOutConfig& cfg) {
|
||||
}
|
||||
}
|
||||
|
||||
void FlipQueue::Wait(VideoOutConfig& cfg, int index) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
|
||||
auto has_request = [this, &cfg, index] {
|
||||
auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
|
||||
return std::any_of(m_requests.begin(), m_requests.end(), matches) ||
|
||||
std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
|
||||
};
|
||||
while (has_request()) {
|
||||
m_done_cond_var.Wait(&m_mutex);
|
||||
void FlipQueue::SignalDoneLocked() {
|
||||
m_done_cond_var.SignalAll();
|
||||
// A stalled GPU cannot drain and shut down without re-checking under m_mutex, so it is alive here.
|
||||
if (m_gpu_waiting) {
|
||||
m_gpu_waiting = false;
|
||||
m_presenter.Renderer().GetGpu().Wake();
|
||||
}
|
||||
}
|
||||
|
||||
// A display buffer is safe for rendering once it is neither displayed nor waiting to be.
|
||||
bool FlipQueue::IsSafeForRendering(VideoOutConfig& cfg, int index) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
auto matches = [&cfg, index](const auto& r) { return r.cfg == &cfg && r.index == index; };
|
||||
const bool safe = cfg.flip_status.currentBuffer != index &&
|
||||
std::none_of(m_requests.begin(), m_requests.end(), matches) &&
|
||||
std::none_of(m_cpu_requests.begin(), m_cpu_requests.end(), matches);
|
||||
m_gpu_waiting = !safe;
|
||||
return safe;
|
||||
}
|
||||
|
||||
bool FlipQueue::Flip(uint32_t micros) {
|
||||
KYTY_PROFILER_BLOCK("FlipQueue::Flip");
|
||||
|
||||
@@ -1311,7 +1320,7 @@ bool FlipQueue::Flip(uint32_t micros) {
|
||||
it = m_requests.erase(it);
|
||||
}
|
||||
m_processing = false;
|
||||
m_done_cond_var.SignalAll();
|
||||
SignalDoneLocked();
|
||||
m_submit_slot_cond_var.SignalAll();
|
||||
m_mutex.Unlock();
|
||||
for (size_t i = count; i != 0; i--) {
|
||||
@@ -1694,12 +1703,12 @@ void VideoOutDriver::WaitForSubmitSlot(int handle) {
|
||||
m_impl->GetFlipQueue().WaitForSubmitSlot(*cfg);
|
||||
}
|
||||
|
||||
void VideoOutDriver::WaitFlipDone(int handle, int index) {
|
||||
bool VideoOutDriver::IsSafeForRendering(int handle, int index) {
|
||||
auto* ctx = m_impl->Get(handle);
|
||||
EXIT_IF(ctx == nullptr);
|
||||
|
||||
EXIT_NOT_IMPLEMENTED(!IsValidBufferIndex(index));
|
||||
m_impl->GetFlipQueue().Wait(*ctx, index);
|
||||
return m_impl->GetFlipQueue().IsSafeForRendering(*ctx, index);
|
||||
}
|
||||
|
||||
KYTY_SYSV_ABI int VideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status) {
|
||||
|
||||
@@ -38,7 +38,7 @@ public:
|
||||
void CompleteFlip(uint64_t request_id);
|
||||
void SubmitFlipPreparation(uint64_t request_id);
|
||||
void WaitForSubmitSlot(int handle);
|
||||
void WaitFlipDone(int handle, int index);
|
||||
[[nodiscard]] bool IsSafeForRendering(int handle, int index);
|
||||
|
||||
[[nodiscard]] Impl& State() noexcept;
|
||||
|
||||
|
||||
@@ -65,6 +65,9 @@ vk::PhysicalDeviceVulkan12Features WindowContext::RequiredVulkan12Features() noe
|
||||
features.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
|
||||
features.shaderStorageImageArrayNonUniformIndexing = VK_TRUE;
|
||||
features.shaderStorageBufferArrayNonUniformIndexing = VK_TRUE;
|
||||
features.descriptorBindingSampledImageUpdateAfterBind = VK_TRUE;
|
||||
features.descriptorBindingUpdateUnusedWhilePending = VK_TRUE;
|
||||
features.descriptorBindingPartiallyBound = VK_TRUE;
|
||||
return features;
|
||||
}
|
||||
|
||||
@@ -295,6 +298,14 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
check_feature(features12.shaderStorageBufferArrayNonUniformIndexing,
|
||||
"shaderStorageBufferArrayNonUniformIndexing",
|
||||
required_features12.shaderStorageBufferArrayNonUniformIndexing);
|
||||
check_feature(features12.descriptorBindingSampledImageUpdateAfterBind,
|
||||
"descriptorBindingSampledImageUpdateAfterBind",
|
||||
required_features12.descriptorBindingSampledImageUpdateAfterBind);
|
||||
check_feature(features12.descriptorBindingUpdateUnusedWhilePending,
|
||||
"descriptorBindingUpdateUnusedWhilePending",
|
||||
required_features12.descriptorBindingUpdateUnusedWhilePending);
|
||||
check_feature(features12.descriptorBindingPartiallyBound, "descriptorBindingPartiallyBound",
|
||||
required_features12.descriptorBindingPartiallyBound);
|
||||
check_feature(features13.robustImageAccess, "robustImageAccess");
|
||||
check_feature(robustness2.nullDescriptor, "nullDescriptor");
|
||||
check_feature(features13.dynamicRendering, "dynamicRendering",
|
||||
@@ -609,6 +620,8 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
device_features.depthClamp = VK_TRUE;
|
||||
#endif
|
||||
device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
|
||||
device_features.shaderStorageImageMultisample =
|
||||
supported_features2.features.shaderStorageImageMultisample;
|
||||
device_features.shaderImageGatherExtended = VK_TRUE;
|
||||
device_features.independentBlend = VK_TRUE;
|
||||
device_features.dualSrcBlend = VK_TRUE;
|
||||
@@ -623,6 +636,11 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
device_features.multiViewport = VK_TRUE;
|
||||
device_features.fillModeNonSolid = VK_TRUE;
|
||||
device_features.vertexPipelineStoresAndAtomics = VK_TRUE;
|
||||
// Storage image sample counts beyond 1x require the feature.
|
||||
if (!device_features.shaderStorageImageMultisample) {
|
||||
graphics.physical_device_properties.limits.storageImageSampleCounts =
|
||||
vk::SampleCountFlagBits::e1;
|
||||
}
|
||||
graphics.sample_rate_shading_enabled = true;
|
||||
device_features.shaderInt64 = VK_TRUE;
|
||||
device_features.shaderFloat64 =
|
||||
|
||||
@@ -169,6 +169,9 @@ uint32_t Builder::AddString(const char* text) {
|
||||
}
|
||||
|
||||
void Builder::AddName(uint32_t target, const char* name) {
|
||||
if (!m_emit_names) {
|
||||
return;
|
||||
}
|
||||
std::vector<uint32_t> operands = {target};
|
||||
AppendString(operands, name);
|
||||
AppendInstruction(m_debug, spv::OpName, operands);
|
||||
|
||||
@@ -67,6 +67,7 @@ public:
|
||||
}
|
||||
|
||||
uint32_t AddString(const char* text);
|
||||
void EnableNames() { m_emit_names = true; }
|
||||
void AddName(uint32_t target, const char* name);
|
||||
template <typename... Args>
|
||||
void AddAnnotation(spv::Op opcode, const Args&... operands) {
|
||||
@@ -159,6 +160,7 @@ private:
|
||||
std::map<std::string, uint32_t> m_import_ids;
|
||||
std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
|
||||
size_t m_unpatched_phi_incomings = 0;
|
||||
bool m_emit_names = false;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Spirv
|
||||
|
||||
@@ -161,6 +161,7 @@ void ValidateNativeProgram(const IR::Program& program, bool lds_storage) {
|
||||
if (!kind.has_value()) {
|
||||
Fail(program, "native shader plan has an invalid image class");
|
||||
}
|
||||
if (image.heap) continue;
|
||||
if (image.mip_count == 0u ||
|
||||
(image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
|
||||
Fail(program, "native shader plan has an invalid image mip descriptor count");
|
||||
@@ -226,6 +227,7 @@ std::vector<uint32_t> EmitProgram(IR::Program& program, ShaderStageInputInfo inp
|
||||
std::snprintf(name, sizeof(name), "rdna2-%016" PRIx64 "-block-byte-pc-plus-one",
|
||||
program.shader_hash);
|
||||
state.debug_source = state.builder.AddString(name);
|
||||
state.builder.EnableNames();
|
||||
}
|
||||
const auto* workgroup = ShaderWorkgroupInput(program.stage, input_info);
|
||||
state.lane_count =
|
||||
|
||||
@@ -120,7 +120,14 @@ uint32_t ImageType(EmitterState& state, const IR::ImageResource& image) {
|
||||
} else {
|
||||
EXIT("invalid image resource class");
|
||||
}
|
||||
if (image.dimension == ImageDimension::Dim1D || image.dimension == ImageDimension::Dim1DArray) {
|
||||
state.builder.RequireCapability(sampled == 1 ? spv::CapabilitySampled1D
|
||||
: spv::CapabilityImage1D);
|
||||
}
|
||||
const auto& info = ImageDimensionInfoFor(image.dimension);
|
||||
if (sampled == 2 && info.multisampled != 0u) {
|
||||
state.builder.RequireCapability(spv::CapabilityStorageImageMultisample);
|
||||
}
|
||||
const auto scalar_type =
|
||||
image.atomic64 ? TypeU64(state) : ImageScalarType(state, image.numeric_class);
|
||||
return state.builder.Type(spv::OpTypeImage, scalar_type,
|
||||
@@ -144,14 +151,20 @@ void DecorateNonUniformDescriptor(EmitterState& state, uint32_t id, spv::Capabil
|
||||
state.builder.RequireCapability(capability);
|
||||
state.builder.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform);
|
||||
}
|
||||
|
||||
ImageDefinition& ImageDescriptorDefinition(EmitterState& state, const IR::ImageResource& image,
|
||||
IR::DescriptorBindingKind kind) {
|
||||
return (image.heap ? state.heap_images : state.images)[IR::ImageBindingIndex(kind)];
|
||||
}
|
||||
} // namespace
|
||||
|
||||
uint32_t LoadImageDescriptor(EmitterState& state, uint32_t resource, uint32_t mip,
|
||||
uint32_t array_index) {
|
||||
const auto pointer = ImageDescriptorPointer(state, resource, mip, array_index);
|
||||
const auto image = state.builder.AllocateId();
|
||||
const auto kind = *IR::DescriptorBindingForImage(state.program.info.images[resource]);
|
||||
state.builder.AddFunction(spv::OpLoad, state.images[IR::ImageBindingIndex(kind)].type,
|
||||
const auto& resource_info = state.program.info.images[resource];
|
||||
const auto kind = *IR::DescriptorBindingForImage(resource_info);
|
||||
state.builder.AddFunction(spv::OpLoad, ImageDescriptorDefinition(state, resource_info, kind).type,
|
||||
image, pointer);
|
||||
if (array_index != 0u) {
|
||||
DecorateNonUniformDescriptor(state, image,
|
||||
@@ -184,13 +197,13 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
|
||||
EXIT_IF(image_resource.resource_class != IR::ImageResourceClass::Sampled);
|
||||
const auto image = LoadImageDescriptor(state, resource, mip, array_index);
|
||||
const auto sampled_image = state.builder.AllocateId();
|
||||
const auto kind = *IR::DescriptorBindingForImage(image_resource);
|
||||
auto& sampled_type = state.images[IR::ImageBindingIndex(kind)].sampled_type;
|
||||
if (sampled_type == 0) {
|
||||
sampled_type = state.builder.Type(
|
||||
spv::OpTypeSampledImage, state.images[IR::ImageBindingIndex(kind)].type);
|
||||
auto& definition = ImageDescriptorDefinition(state, image_resource,
|
||||
*IR::DescriptorBindingForImage(image_resource));
|
||||
if (definition.sampled_type == 0) {
|
||||
definition.sampled_type = state.builder.Type(spv::OpTypeSampledImage, definition.type);
|
||||
}
|
||||
state.builder.AddFunction(spv::OpSampledImage, sampled_type, sampled_image, image, sampler_id);
|
||||
state.builder.AddFunction(spv::OpSampledImage, definition.sampled_type, sampled_image, image,
|
||||
sampler_id);
|
||||
if (array_index != 0u || sampler_dynamic) {
|
||||
DecorateNonUniformDescriptor(state, sampled_image, spv::CapabilitySampledImageArrayNonUniformIndexing);
|
||||
}
|
||||
@@ -202,26 +215,31 @@ uint32_t ImageDescriptorPointer(EmitterState& state, uint32_t resource, uint32_t
|
||||
const auto& image = state.program.info.images.at(resource);
|
||||
EXIT_IF(mip >= image.mip_count);
|
||||
const auto kind = IR::DescriptorBindingForImage(image);
|
||||
EXIT_IF(!kind.has_value());
|
||||
EXIT_IF(!kind.has_value() || (image.heap && array_index == 0u));
|
||||
if (array_index == 0u) {
|
||||
array_index = ConstantU32(state, image.descriptor_index + mip);
|
||||
}
|
||||
const auto& definition = state.images[IR::ImageBindingIndex(*kind)];
|
||||
const auto& definition = ImageDescriptorDefinition(state, image, *kind);
|
||||
return DescriptorElementPointer(state, definition.pointer_type, definition.variable,
|
||||
array_index, *kind, resource,
|
||||
"image descriptor array was not emitted");
|
||||
}
|
||||
|
||||
void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
|
||||
uint32_t texel, uint32_t array_index) {
|
||||
uint32_t texel, uint32_t array_index, uint32_t sample) {
|
||||
const auto& image = state.program.info.images.at(resource);
|
||||
EXIT_IF(image.resource_class != IR::ImageResourceClass::Storage);
|
||||
if (!image.atomic) {
|
||||
state.builder.RequireCapability(spv::CapabilityStorageImageWriteWithoutFormat);
|
||||
}
|
||||
const auto EmitWrite = [&](uint32_t mip) {
|
||||
state.builder.AddFunction(spv::OpImageWrite, LoadImageDescriptor(state, resource, mip, array_index),
|
||||
coord, texel);
|
||||
const auto descriptor = LoadImageDescriptor(state, resource, mip, array_index);
|
||||
if (sample == 0u) {
|
||||
state.builder.AddFunction(spv::OpImageWrite, descriptor, coord, texel);
|
||||
} else {
|
||||
state.builder.AddFunction(spv::OpImageWrite, descriptor, coord, texel,
|
||||
spv::ImageOperandsSampleMask, sample);
|
||||
}
|
||||
};
|
||||
if (image.mip_mode != IR::ImageMipMode::Dynamic) {
|
||||
EmitWrite(0);
|
||||
|
||||
@@ -389,6 +389,7 @@ Format::BufferFormatInfo ImageConversionFormat(const EmitterState& state,
|
||||
const auto format = image.conversion_format;
|
||||
if (format == Prospero::BufferFormat::kInvalid) return {};
|
||||
const auto info = Format::GetFormatInfo(format);
|
||||
if (IR::ReadsDepthBits(image)) return info;
|
||||
const bool raw_signed = IR::RawSignedStorageFormat(image, format) != Prospero::BufferFormat::kInvalid;
|
||||
EXIT_IF((Prospero::RemapTextureFormat(format) == format && !raw_signed) || info.component_count == 0u ||
|
||||
info.component_count > 4u);
|
||||
@@ -424,6 +425,7 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
|
||||
|
||||
const auto numeric_class = ctx.state.program.info.images[mem.resource].numeric_class;
|
||||
const auto scalar_type = ImageScalarType(ctx.state, numeric_class);
|
||||
const bool integer = info.packed_bitfield || info.type == Format::ComponentType::Uint;
|
||||
uint32_t packed = 0;
|
||||
if (info.packed_bitfield) {
|
||||
packed = ctx.state.builder.AllocateId();
|
||||
@@ -441,8 +443,16 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeExtract, scalar_type,
|
||||
components[component], texel, component);
|
||||
// UNorm backing preserves the guest's filtering; scaling precedes swizzle constants.
|
||||
const auto unorm_max = static_cast<float>((1u << info.component_bits[component]) - 1u);
|
||||
components[component] = Binary(ctx.state, spv::OpFMul, scalar_type, components[component],
|
||||
ConstantF32Value(ctx.state, 255.0f));
|
||||
ConstantF32Value(ctx.state, unorm_max));
|
||||
if (integer) {
|
||||
// Depth bits: round back to the stored UNorm code.
|
||||
components[component] =
|
||||
Unary(ctx.state, spv::OpConvertFToU, TypeU32(ctx.state),
|
||||
Binary(ctx.state, spv::OpFAdd, scalar_type, components[component],
|
||||
ConstantF32Value(ctx.state, 0.5f)));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (uint32_t component = info.component_count; component < 4u; component++) {
|
||||
@@ -454,20 +464,19 @@ uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint
|
||||
for (uint32_t component = 0; component < 4u; component++) {
|
||||
const auto selector = (swizzle >> (component * 3u)) & 7u;
|
||||
if (selector == 1u) {
|
||||
selected[component] = info.packed_bitfield
|
||||
selected[component] = integer
|
||||
? ConstantU32(ctx.state, Format::FormattedConstantBits(info, Format::FormattedSourceKind::One))
|
||||
: ConstantF32Value(ctx.state, 1.0f);
|
||||
} else if (selector >= 4u) {
|
||||
selected[component] = components[selector - 4u];
|
||||
} else {
|
||||
selected[component] = info.packed_bitfield ? ConstantU32(ctx.state, 0u)
|
||||
: ZeroF32(ctx.state);
|
||||
selected[component] = integer ? ConstantU32(ctx.state, 0u) : ZeroF32(ctx.state);
|
||||
}
|
||||
}
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeConstruct,
|
||||
ImageVectorType(ctx.state, numeric_class, 4), result, selected[0],
|
||||
selected[1], selected[2], selected[3]);
|
||||
integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
|
||||
result, selected[0], selected[1], selected[2], selected[3]);
|
||||
return info.type == Format::ComponentType::Unorm
|
||||
? Unary(ctx.state, spv::OpBitcast, TypeF32Vector(ctx.state, 4), result) : result;
|
||||
}
|
||||
@@ -691,6 +700,21 @@ auto EmitImageAccess(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t resul
|
||||
image.indirect_resources.size() < 2u) {
|
||||
ctx.Fail(inst, "has no indirect image runtime mapping");
|
||||
}
|
||||
if (image.heap) {
|
||||
const auto entry = EmitImageHeapEntry(state, key, image.indirect_mapping_offset);
|
||||
const auto kinds = static_cast<uint32_t>(image.indirect_resources.size()) - 1u;
|
||||
// One kind leaves every slot word a bare element.
|
||||
if (kinds == 1u) return emit(image.indirect_resources[1], entry);
|
||||
const auto element = Binary(state, spv::OpBitwiseAnd, TypeU32(state), entry,
|
||||
ConstantU32(state, (1u << IR::ImageHeapKindShift) - 1u));
|
||||
const auto EmitKind = [&](uint32_t kind) {
|
||||
return emit(image.indirect_resources[kind + 1u], element);
|
||||
};
|
||||
return EmitIndexSwitch(state,
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), entry,
|
||||
ConstantU32(state, IR::ImageHeapKindShift)),
|
||||
kinds, result_type, EmitKind);
|
||||
}
|
||||
const auto selected = EmitIndirectResourceIndex(
|
||||
state, key, image.indirect_mapping_offset, image.indirect_search_iterations, 0u);
|
||||
const auto SameType = [&](uint32_t lhs, uint32_t rhs) {
|
||||
@@ -870,14 +894,19 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
}
|
||||
if (op == IR::ValueOpcode::ImageQueryLod) {
|
||||
state.builder.RequireCapability(spv::CapabilityImageQuery);
|
||||
const auto dimension = image.dimension;
|
||||
const auto sampled = MakeSampledImage(state, mem.resource,
|
||||
LoadSamplerDescriptor(state, mem.sampler));
|
||||
const auto lod = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
spv::OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
|
||||
CoordF32(ctx, mem, *address, 0, ImageDimensionInfoFor(dimension).spatial_components,
|
||||
image.cube));
|
||||
const auto sampler = LoadSamplerDescriptor(state, mem.sampler);
|
||||
const auto lod = EmitImageAccess(ctx, inst, TypeF32Vector(state, 2),
|
||||
[&](uint32_t resource, uint32_t array_index) {
|
||||
const auto& candidate = state.program.info.images[resource];
|
||||
const auto query = state.builder.AllocateId();
|
||||
state.builder.AddFunction(
|
||||
spv::OpImageQueryLod, TypeF32Vector(state, 2), query,
|
||||
MakeSampledImage(state, resource, sampler, 0u, array_index),
|
||||
CoordF32(ctx, mem, *address, 0,
|
||||
ImageDimensionInfoFor(candidate.dimension).spatial_components,
|
||||
candidate.cube));
|
||||
return query;
|
||||
});
|
||||
uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
|
||||
ConstantU32(state, 0)};
|
||||
for (uint32_t index = 0; index < 2u; index++) {
|
||||
@@ -932,9 +961,12 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
const auto mip_lod = candidate.mip_mode == IR::ImageMipMode::Dynamic
|
||||
? LodU32(ctx, mem, *address, mem.image_dimension) : 0u;
|
||||
const auto coord = CoordU32(ctx, mem, *address, candidate.dimension);
|
||||
const auto& dimension_info = ImageDimensionInfoFor(candidate.dimension);
|
||||
const auto sample = dimension_info.multisampled != 0u
|
||||
? AddressU32(ctx, mem, *address, dimension_info.coordinate_components) : 0u;
|
||||
const auto texel = StoreTexel(ctx, selected, ctx.Arg(inst, 2),
|
||||
candidate.numeric_class == Prospero::TextureNumericClass::Uint);
|
||||
EmitStorageImageWrite(state, resource, mip_lod, coord, texel, array_index);
|
||||
EmitStorageImageWrite(state, resource, mip_lod, coord, texel, array_index, sample);
|
||||
});
|
||||
});
|
||||
return;
|
||||
|
||||
@@ -130,6 +130,7 @@ struct EmitterState {
|
||||
uint32_t lds_u64_variable = 0;
|
||||
std::array<uint32_t, 2> scratch_variable {};
|
||||
std::array<ImageDefinition, IR::ImageBindingCount> images {};
|
||||
std::array<ImageDefinition, IR::ImageBindingCount> heap_images {};
|
||||
uint32_t sampler_variable = 0;
|
||||
uint32_t sampler_type = 0;
|
||||
uint32_t sampler_pointer_type = 0;
|
||||
@@ -339,8 +340,9 @@ uint32_t MakeSampledImage(EmitterState& state, uint32_t resource, uint32_t sampl
|
||||
uint32_t mip = 0, uint32_t array_index = 0,
|
||||
bool sampler_dynamic = false);
|
||||
|
||||
// sample is 0 (no id) for single-sampled images.
|
||||
void EmitStorageImageWrite(EmitterState& state, uint32_t resource, uint32_t mip_lod, uint32_t coord,
|
||||
uint32_t texel, uint32_t array_index);
|
||||
uint32_t texel, uint32_t array_index, uint32_t sample);
|
||||
|
||||
spv::ExecutionModel ExecutionModelForStage(ShaderType stage);
|
||||
|
||||
@@ -430,6 +432,8 @@ struct MemoryResourceAccess {
|
||||
|
||||
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
|
||||
uint32_t search_iterations, uint32_t default_resource);
|
||||
// Slot word of an image heap byte offset; offsets past the heap select the null element.
|
||||
uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset);
|
||||
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem,
|
||||
uint32_t array_index = 0u);
|
||||
|
||||
|
||||
@@ -38,19 +38,39 @@ uint32_t EmitBinaryU32(EmitterState& state, spv::Op opcode, uint32_t lhs, uint32
|
||||
return ret;
|
||||
}
|
||||
|
||||
static uint32_t LoadFlattenedSrt(EmitterState& state, uint32_t index) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
|
||||
state.flattened_srt_variable, ConstantU32(state, 0), index);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t EmitImageHeapEntry(EmitterState& state, uint32_t key, uint32_t mapping_offset) {
|
||||
const auto Mapping = [&](uint32_t word) {
|
||||
return LoadFlattenedSrt(state, ConstantU32(state, mapping_offset + word));
|
||||
};
|
||||
// Keys past the table select its trailing null slot.
|
||||
const auto index = EmitUMin32(state, Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
|
||||
ConstantU32(state, 2u)),
|
||||
Mapping(0u));
|
||||
const auto offset = Unary(state, spv::OpUConvert, TypeU64(state),
|
||||
Binary(state, spv::OpShiftLeftLogical, TypeU32(state), index,
|
||||
ConstantU32(state, 2u)));
|
||||
const auto pointer = Unary(state, spv::OpConvertUToPtr, TypePhysicalU32Pointer(state),
|
||||
Binary(state, spv::OpIAdd, TypeU64(state),
|
||||
PackU64(state, Mapping(1u), Mapping(2u)), offset));
|
||||
const auto entry = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpLoad, TypeU32(state), entry, pointer,
|
||||
spv::MemoryAccessAlignedMask, 4u);
|
||||
return entry;
|
||||
}
|
||||
|
||||
uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t mapping_offset,
|
||||
uint32_t search_iterations, uint32_t default_resource) {
|
||||
const auto LoadMapping = [&](uint32_t index) {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state),
|
||||
pointer, state.flattened_srt_variable, ConstantU32(state, 0),
|
||||
index);
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpLoad, TypeU32(state), value, pointer);
|
||||
return value;
|
||||
};
|
||||
const auto mapping = ConstantU32(state, mapping_offset);
|
||||
const auto count = LoadMapping(mapping);
|
||||
const auto count = LoadFlattenedSrt(state, mapping);
|
||||
if (search_iterations == 0u) {
|
||||
const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), key,
|
||||
ConstantU32(state, 2u));
|
||||
@@ -58,7 +78,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
|
||||
const auto safe = Select(state, TypeU32(state), valid, index, ConstantU32(state, 0u));
|
||||
const auto entry = Binary(state, spv::OpIAdd, TypeU32(state), safe,
|
||||
ConstantU32(state, mapping_offset + 1u));
|
||||
return Select(state, TypeU32(state), valid, LoadMapping(entry),
|
||||
return Select(state, TypeU32(state), valid, LoadFlattenedSrt(state, entry),
|
||||
ConstantU32(state, default_resource));
|
||||
}
|
||||
const auto entry_at = [&](uint32_t index) {
|
||||
@@ -76,7 +96,7 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
|
||||
ConstantU32(state, 1u));
|
||||
const auto probe = Select(state, TypeU32(state), active, mid, ConstantU32(state, 0u));
|
||||
const auto less = Binary(state, spv::OpULessThan, TypeBool(state),
|
||||
LoadMapping(entry_at(probe)), key);
|
||||
LoadFlattenedSrt(state, entry_at(probe)), key);
|
||||
low = Select(state, TypeU32(state),
|
||||
Binary(state, spv::OpLogicalAnd, TypeBool(state), active, less),
|
||||
Binary(state, spv::OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low);
|
||||
@@ -87,9 +107,10 @@ uint32_t EmitIndirectResourceIndex(EmitterState& state, uint32_t key, uint32_t m
|
||||
const auto in_range = Binary(state, spv::OpULessThan, TypeBool(state), low, count);
|
||||
const auto entry = entry_at(Select(state, TypeU32(state), in_range, low, ConstantU32(state, 0u)));
|
||||
const auto match = Binary(state, spv::OpLogicalAnd, TypeBool(state), in_range,
|
||||
Binary(state, spv::OpIEqual, TypeBool(state), LoadMapping(entry), key));
|
||||
const auto resource = LoadMapping(Binary(state, spv::OpIAdd, TypeU32(state), entry,
|
||||
ConstantU32(state, 1u)));
|
||||
Binary(state, spv::OpIEqual, TypeBool(state),
|
||||
LoadFlattenedSrt(state, entry), key));
|
||||
const auto resource = LoadFlattenedSrt(
|
||||
state, Binary(state, spv::OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
|
||||
return Select(state, TypeU32(state), match, resource, ConstantU32(state, default_resource));
|
||||
}
|
||||
|
||||
|
||||
@@ -206,6 +206,17 @@ uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
|
||||
return state.builder.Type(spv::OpTypeArray, TypeF32(state), ConstantU32(state, count));
|
||||
}
|
||||
|
||||
void DefineImageArray(EmitterState& state, ImageDefinition& definition, const IR::ImageResource& image,
|
||||
uint32_t count, uint32_t set, uint32_t binding) {
|
||||
definition.type = ImageType(state, image);
|
||||
const auto array = state.builder.Type(spv::OpTypeArray, definition.type, ConstantU32(state, count));
|
||||
definition.variable = state.builder.DefineGlobalVariable(
|
||||
TypePointer(state, spv::StorageClassUniformConstant, array), spv::StorageClassUniformConstant);
|
||||
state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationDescriptorSet, set);
|
||||
state.builder.AddAnnotation(spv::OpDecorate, definition.variable, spv::DecorationBinding, binding);
|
||||
definition.pointer_type = TypePointer(state, spv::StorageClassUniformConstant, definition.type);
|
||||
}
|
||||
|
||||
void DefineDescriptors(EmitterState& state) {
|
||||
using Kind = IR::DescriptorBindingKind;
|
||||
auto& info = state.program.info;
|
||||
@@ -222,6 +233,14 @@ void DefineDescriptors(EmitterState& state) {
|
||||
(image.mip_mode != IR::ImageMipMode::Dynamic && image.mip_count != 1u)) {
|
||||
EXIT("shader image has an invalid descriptor type or mip count");
|
||||
}
|
||||
if (image.heap) {
|
||||
// Heap kinds alias one array; the root itself is never accessed.
|
||||
auto& definition = state.heap_images[IR::ImageBindingIndex(*kind)];
|
||||
if (image.indirect_resources.empty() && definition.variable == 0) {
|
||||
DefineImageArray(state, definition, image, IR::ImageHeapCapacity, IR::ImageHeapSet, 0u);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
auto& count = counts[static_cast<size_t>(*kind)];
|
||||
image.descriptor_index = count;
|
||||
if (count == 0) image_types[IR::ImageBindingIndex(*kind)] = ℑ
|
||||
@@ -331,21 +350,11 @@ void DefineDescriptors(EmitterState& state) {
|
||||
default: {
|
||||
EXIT_IF(IR::ImageBindingResourceClass(kind) ==
|
||||
IR::ImageResourceClass::None);
|
||||
const auto& image = *image_types[IR::ImageBindingIndex(kind)];
|
||||
const auto name = "image_" + std::to_string(static_cast<uint32_t>(kind));
|
||||
auto& definition = state.images[IR::ImageBindingIndex(kind)];
|
||||
definition.type = ImageType(state, image);
|
||||
definition.variable = Define(ArrayType(definition.type), name.c_str(),
|
||||
spv::StorageClassUniformConstant);
|
||||
definition.pointer_type = TypePointer(
|
||||
state, spv::StorageClassUniformConstant, definition.type);
|
||||
if (image.dimension == ImageDimension::Dim1D ||
|
||||
image.dimension == ImageDimension::Dim1DArray) {
|
||||
state.builder.RequireCapability(image.resource_class ==
|
||||
IR::ImageResourceClass::Sampled
|
||||
? spv::CapabilitySampled1D
|
||||
: spv::CapabilityImage1D);
|
||||
}
|
||||
DefineImageArray(state, definition, *image_types[IR::ImageBindingIndex(kind)],
|
||||
counts[index], 0u, IR::NativeBinding(state.program.stage, kind));
|
||||
state.builder.AddName(definition.variable,
|
||||
("image_" + std::to_string(static_cast<uint32_t>(kind))).c_str());
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -752,7 +761,7 @@ void DefineModule(EmitterState& state) {
|
||||
if (state.program.info.buffer_u16) {
|
||||
state.builder.RequireCapability(spv::CapabilityStorageBuffer16BitAccess);
|
||||
}
|
||||
if (state.program.info.uses_dma) {
|
||||
if (state.program.info.uses_dma || state.program.info.uses_image_heap) {
|
||||
state.builder.RequireCapability(spv::CapabilityInt64);
|
||||
state.builder.RequireCapability(spv::CapabilityPhysicalStorageBufferAddresses);
|
||||
state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
|
||||
@@ -820,7 +829,7 @@ void DefineModule(EmitterState& state) {
|
||||
state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
|
||||
}
|
||||
state.builder.RequireExtension("SPV_KHR_float_controls");
|
||||
state.builder.AddMemoryModel(state.program.info.uses_dma
|
||||
state.builder.AddMemoryModel(state.program.info.uses_dma || state.program.info.uses_image_heap
|
||||
? spv::AddressingModelPhysicalStorageBuffer64
|
||||
: spv::AddressingModelLogical,
|
||||
spv::MemoryModelGLSL450);
|
||||
|
||||
@@ -169,6 +169,7 @@ struct ImageResource {
|
||||
bool depth_compare = false;
|
||||
bool cube = false;
|
||||
bool r128 = false;
|
||||
bool heap = false; // Resident image heap root or kind.
|
||||
uint32_t indirect_root = NoIndirectImage;
|
||||
uint32_t indirect_mapping_offset = 0;
|
||||
uint32_t indirect_search_iterations = 0;
|
||||
@@ -188,6 +189,12 @@ RawSignedStorageFormat(const ImageResource& image, Prospero::BufferFormat format
|
||||
return Prospero::BufferFormat::kInvalid;
|
||||
}
|
||||
|
||||
// A raw load of a kDepth k16UInt T# reads a D16 surface's codes. Vulkan cannot view D16 as
|
||||
// R16Uint, so the host binds it as UNorm and the shader converts back.
|
||||
[[nodiscard]] constexpr bool ReadsDepthBits(const ImageResource& image) {
|
||||
return image.conversion_format == Prospero::BufferFormat::k16UInt;
|
||||
}
|
||||
|
||||
struct SamplerResource {
|
||||
static constexpr uint32_t NoIndirectSampler = UINT32_MAX;
|
||||
|
||||
@@ -337,7 +344,12 @@ struct StageOutput {
|
||||
inline constexpr uint32_t FirstImageBinding = 1u;
|
||||
inline constexpr uint32_t FirstComparisonImageBinding = 22u;
|
||||
inline constexpr uint32_t FirstStorageImageBinding = 29u;
|
||||
inline constexpr uint32_t ImageBindingCount = 48u;
|
||||
inline constexpr uint32_t ImageBindingCount = 50u;
|
||||
// Resident image heaps share one sampled-image array at binding 0 of this set. A heap slot
|
||||
// word is kind << ImageHeapKindShift | element; element 0 is the null descriptor.
|
||||
inline constexpr uint32_t ImageHeapSet = 1u;
|
||||
inline constexpr uint32_t ImageHeapCapacity = 65536u;
|
||||
inline constexpr uint32_t ImageHeapKindShift = 20u;
|
||||
|
||||
enum class DescriptorBindingKind : uint32_t {
|
||||
Buffers = 0u,
|
||||
@@ -351,8 +363,8 @@ enum class DescriptorBindingKind : uint32_t {
|
||||
Count,
|
||||
};
|
||||
|
||||
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u);
|
||||
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u);
|
||||
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 51u);
|
||||
static_assert(static_cast<uint32_t>(DescriptorBindingKind::Count) == 58u);
|
||||
|
||||
struct PushData {
|
||||
static constexpr uint32_t DwordCount = 32;
|
||||
@@ -413,8 +425,8 @@ DescriptorBindingForImage(const ImageResource& image) {
|
||||
constexpr uint32_t SampledUintBinding = 8u;
|
||||
constexpr uint32_t SampledSintBinding = 15u;
|
||||
constexpr uint32_t StorageFloatBinding = FirstStorageImageBinding;
|
||||
constexpr uint32_t StorageUintBinding = StorageFloatBinding + 5u;
|
||||
constexpr uint32_t AtomicUintBinding = StorageUintBinding + 5u;
|
||||
constexpr uint32_t StorageUintBinding = StorageFloatBinding + 6u;
|
||||
constexpr uint32_t AtomicUintBinding = StorageUintBinding + 6u;
|
||||
|
||||
uint32_t base = 0;
|
||||
bool sampled = false;
|
||||
@@ -461,10 +473,10 @@ DescriptorBindingForImage(const ImageResource& image) {
|
||||
case Decoder::ImageDimension::Dim2D: dimension = 2u; break;
|
||||
case Decoder::ImageDimension::Dim2DArray: dimension = 3u; break;
|
||||
case Decoder::ImageDimension::Dim2DMsaa:
|
||||
if (!sampled) {
|
||||
if (image.atomic) {
|
||||
return std::nullopt;
|
||||
}
|
||||
dimension = 4u;
|
||||
dimension = sampled ? 4u : 5u;
|
||||
break;
|
||||
case Decoder::ImageDimension::Dim2DMsaaArray:
|
||||
if (!sampled) {
|
||||
@@ -523,6 +535,7 @@ struct ShaderInfo {
|
||||
bool uses_lds = false;
|
||||
bool uses_gds = false;
|
||||
bool uses_flattened_srt = false;
|
||||
bool uses_image_heap = false;
|
||||
bool uses_dispatch_threads = false;
|
||||
bool bvh = false;
|
||||
bool subgroup_ballot = false;
|
||||
|
||||
@@ -681,6 +681,32 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
|
||||
const auto count = table_size >= first + 4u
|
||||
? std::min((table_size - first - 4u) / step + 1u, domain)
|
||||
: 0u;
|
||||
descriptors[resource_index] = {.dword_count = dword_count};
|
||||
if constexpr (std::is_same_v<Specialization, ResourceSpecialization::Image>) {
|
||||
const auto& image = program.info.images[resource_index];
|
||||
// A resident heap maps slot words instead of materializing its entries.
|
||||
ImageHeapView view;
|
||||
if (image_heap && runtime.image_heap != nullptr &&
|
||||
image.resource_class == ImageResourceClass::Sampled &&
|
||||
image.mip_mode != ImageMipMode::Dynamic &&
|
||||
runtime.image_heap->Acquire({table_base, table_size, step, first, count, &image},
|
||||
view)) {
|
||||
snapshot.specialization_reads.emplace_back(table_base, table_size);
|
||||
if (view.kinds.empty()) return true;
|
||||
auto root = root_resource;
|
||||
root.indirect_root = resource_index;
|
||||
root.indirect_mapping_offset = static_cast<uint32_t>(snapshot.flattened_srt.size());
|
||||
root.heap = true;
|
||||
specializations[resource_index] = root;
|
||||
snapshot.flattened_srt.insert(snapshot.flattened_srt.end(),
|
||||
{view.slot_count, static_cast<uint32_t>(view.slots), static_cast<uint32_t>(view.slots >> 32u)});
|
||||
for (const auto& kind: view.kinds) {
|
||||
descriptors.push_back(kind);
|
||||
specializations.push_back(root);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (child_count == 0u || (!image_heap && count * child_count > MaxIndirectDescriptorProbes))
|
||||
return failed("candidate count", fmt::format("rows={} limit={}", count, MaxIndirectDescriptorProbes));
|
||||
// Affine SMEM offsets wrap in U32 before adding the aligned immediate.
|
||||
@@ -692,7 +718,6 @@ bool MaterializeIndirectDescriptor(const ResourcePlan& p
|
||||
return read_failed("image heap read", 0u, 0u, table_base, 0u, heap_words.size() * sizeof(uint32_t));
|
||||
}
|
||||
const auto mapping_offset = snapshot.flattened_srt.size();
|
||||
descriptors[resource_index] = {.dword_count = dword_count};
|
||||
snapshot.flattened_srt.resize(mapping_offset + 2u, 0u);
|
||||
for (uint32_t word = 0; word < count; ++word) {
|
||||
const auto table_offset = affine ? first + word * step : uint64_t {word} * 4u;
|
||||
@@ -932,7 +957,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
return SpecializationFail("FMASK requires a direct image load");
|
||||
}
|
||||
}
|
||||
image.conversion_format = ImageConversionFormat(format);
|
||||
// See ReadsDepthBits; raw loads only, since filtering would blend the codes.
|
||||
const bool depth_bits =
|
||||
!storage && base.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt &&
|
||||
static_cast<Prospero::TileMode>((descriptor.dwords[3] >> 20u) & 0x1fu) ==
|
||||
Prospero::TileMode::kDepth;
|
||||
image.conversion_format = depth_bits ? format : ImageConversionFormat(format);
|
||||
if (storage || image.conversion_format != Prospero::BufferFormat::kInvalid) {
|
||||
image.shader_swizzle = DescriptorImageSwizzle(descriptor);
|
||||
}
|
||||
@@ -940,7 +970,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
const bool raw_sint_storage = raw_sint_format != Prospero::BufferFormat::kInvalid;
|
||||
if (raw_sint_format == Prospero::BufferFormat::k16UInt) image.conversion_format = format;
|
||||
image.numeric_class = Prospero::SampledTextureNumericClass(
|
||||
Prospero::RemapTextureFormat(format));
|
||||
depth_bits ? Prospero::BufferFormat::k16UNorm : Prospero::RemapTextureFormat(format));
|
||||
if (!storage && format == Prospero::BufferFormat::k11_11_10UNorm) {
|
||||
if (image.indirect_root != ImageResource::NoIndirectImage) {
|
||||
return SpecializationFail("packed UNorm sampling requires a direct image");
|
||||
@@ -1005,10 +1035,12 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
const auto key_count = root.indirect_mapping_offset < snapshot.flattened_srt.size()
|
||||
? snapshot.flattened_srt[root.indirect_mapping_offset]
|
||||
: 0u;
|
||||
// A heap maps its slot count and slot address; tables map each key.
|
||||
size_t mapping_words = 3u;
|
||||
if (!root.heap)
|
||||
mapping_words = 1u + size_t {key_count} * (root.indirect_search_iterations == 0u ? 1u : 2u);
|
||||
if (key_count == 0u ||
|
||||
static_cast<size_t>(root.indirect_mapping_offset) + 1u +
|
||||
static_cast<size_t>(key_count) * (root.indirect_search_iterations == 0u ? 1u : 2u) >
|
||||
snapshot.flattened_srt.size()) {
|
||||
root.indirect_mapping_offset + mapping_words > snapshot.flattened_srt.size()) {
|
||||
return SpecializationFail("indirect image specialization has an invalid key mapping");
|
||||
}
|
||||
uint32_t exemplar = ImageResource::NoIndirectImage;
|
||||
@@ -1744,6 +1776,43 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
return BuildResourceSpecialization(program, snapshot, specialization);
|
||||
}
|
||||
|
||||
uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value) {
|
||||
// Mirrors BuildResourceSpecialization for sampled roots: entries it would reject, and
|
||||
// entries that need shader-side conversion, read as null instead of failing the draw.
|
||||
const auto format = static_cast<Prospero::BufferFormat>((value.dwords[1] >> 20u) & 0x1ffu);
|
||||
const auto tile = static_cast<Prospero::TileMode>((value.dwords[3] >> 20u) & 0x1fu);
|
||||
const auto numeric_class =
|
||||
Prospero::SampledTextureNumericClass(Prospero::RemapTextureFormat(format));
|
||||
if (NullImageDescriptor(value) || !ValidImageDescriptor(value, root.r128) ||
|
||||
DescriptorDimension(value, root.dimension) != root.dimension ||
|
||||
Prospero::IsFmaskTextureFormat(format) || format == Prospero::BufferFormat::k11_11_10UNorm ||
|
||||
ImageConversionFormat(format) != Prospero::BufferFormat::kInvalid ||
|
||||
(root.unfiltered_access_only && format == Prospero::BufferFormat::k16UInt &&
|
||||
tile == Prospero::TileMode::kDepth) ||
|
||||
numeric_class == Prospero::TextureNumericClass::Unsupported ||
|
||||
(root.depth_compare && numeric_class != Prospero::TextureNumericClass::Float)) {
|
||||
value.dwords.fill(0u);
|
||||
return 0u;
|
||||
}
|
||||
return 1u + static_cast<uint32_t>(numeric_class) * 2u + (DescriptorIsCube(value) ? 1u : 0u);
|
||||
}
|
||||
|
||||
ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind) {
|
||||
ImageResource resource;
|
||||
resource.resource_class = ImageResourceClass::Sampled;
|
||||
resource.read = true;
|
||||
resource.heap = true;
|
||||
resource.dimension = root.dimension;
|
||||
resource.unfiltered_access_only = root.unfiltered_access_only;
|
||||
resource.depth_compare = root.depth_compare;
|
||||
resource.r128 = root.r128;
|
||||
if (kind != 0u) {
|
||||
resource.numeric_class = static_cast<Prospero::TextureNumericClass>((kind - 1u) / 2u);
|
||||
resource.cube = ((kind - 1u) & 1u) != 0u;
|
||||
}
|
||||
return resource;
|
||||
}
|
||||
|
||||
void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization) {
|
||||
EXIT_IF(!program.resource_tracking_complete || program.shader_info_complete);
|
||||
EXIT_IF(program.info.buffers.size() > specialization.buffers.size() ||
|
||||
@@ -1794,6 +1863,7 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
|
||||
image.indirect_mapping_offset = source.indirect_mapping_offset;
|
||||
image.indirect_search_iterations = source.indirect_search_iterations;
|
||||
image.cube = source.cube;
|
||||
image.heap = source.heap;
|
||||
image.indirect_resources.clear();
|
||||
}
|
||||
for (uint32_t index = 0; index < images.size(); index++) {
|
||||
|
||||
@@ -31,6 +31,7 @@ struct ResourceSpecialization {
|
||||
uint32_t indirect_search_iterations = 0;
|
||||
bool cube = false;
|
||||
bool fmask = false;
|
||||
bool heap = false;
|
||||
bool operator==(const Image&) const = default;
|
||||
};
|
||||
|
||||
@@ -59,6 +60,13 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
// Applies an already-derived specialization to native IR before layout and emission.
|
||||
void ApplyResourceSpecialization(Program& program, const ResourceSpecialization& specialization);
|
||||
|
||||
// Kind of an image heap entry as root sees it, or 0 (and a zeroed value) when root reads it as
|
||||
// null. Entries of one kind specialize identically.
|
||||
uint32_t ImageHeapKind(const ImageResource& root, DescriptorValue& value);
|
||||
// The sampled resource that root resolves entries of kind as; kind 0 gives the identity of
|
||||
// root's view of a heap.
|
||||
ImageResource ImageHeapResource(const ImageResource& root, uint32_t kind);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::IR
|
||||
|
||||
#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_RESOURCEMATERIALIZATION_H_ */
|
||||
@@ -594,6 +594,7 @@ void CollectShaderInfo(Program& program, ShaderStageInputInfo input_info) {
|
||||
};
|
||||
info.uses_flattened_srt |= std::ranges::any_of(info.buffers, uses_mapping) ||
|
||||
std::ranges::any_of(info.images, uses_mapping);
|
||||
info.uses_image_heap = std::ranges::any_of(info.images, &ImageResource::heap);
|
||||
switch (program.stage) {
|
||||
case ShaderType::Vertex:
|
||||
case ShaderType::Local: CollectVertexInputs(input_info.vertex, info, inputs.components); break;
|
||||
|
||||
@@ -12,6 +12,33 @@ class Value;
|
||||
|
||||
using SrtMemoryReader = bool (*)(void* userdata, uint64_t address, std::span<uint32_t> values);
|
||||
|
||||
// Image descriptors at base + first + k * step, k < count, within size bytes that a sampled
|
||||
// root selects by GPU-computed byte offset. The host keeps them resident instead of
|
||||
// materializing them.
|
||||
struct ImageHeapRequest {
|
||||
uint64_t base = 0;
|
||||
uint64_t size = 0;
|
||||
uint64_t step = 0;
|
||||
uint64_t first = 0;
|
||||
uint64_t count = 0;
|
||||
const ImageResource* root = nullptr;
|
||||
};
|
||||
|
||||
struct ImageHeapView {
|
||||
uint64_t slots = 0; // Device address of one slot per heap dword.
|
||||
uint32_t slot_count = 0; // Keys past it select a trailing null slot.
|
||||
std::span<const DescriptorValue> kinds; // One representative descriptor per kind.
|
||||
};
|
||||
|
||||
class ImageHeapSource {
|
||||
public:
|
||||
// False when the host does not keep the table resident.
|
||||
virtual bool Acquire(const ImageHeapRequest& request, ImageHeapView& view) = 0;
|
||||
|
||||
protected:
|
||||
~ImageHeapSource() = default;
|
||||
};
|
||||
|
||||
struct SrtRuntime {
|
||||
std::span<const uint32_t> user_data;
|
||||
uint64_t shader_base = 0;
|
||||
@@ -20,6 +47,7 @@ struct SrtRuntime {
|
||||
SrtMemoryReader read_specialization_memory = nullptr;
|
||||
std::span<const uint32_t> workgroup_counts;
|
||||
uint64_t indirect_args_address = 0;
|
||||
ImageHeapSource* image_heap = nullptr;
|
||||
};
|
||||
|
||||
class SrtReadCapture {
|
||||
|
||||
@@ -410,6 +410,11 @@ std::filesystem::path GetRealFilename(const std::string& mounted_file_name) {
|
||||
return g_mount_points->ResolvePath(mounted_file_name);
|
||||
}
|
||||
|
||||
bool IsOpenDescriptor(int d) {
|
||||
const auto* file = g_files->GetFile(d);
|
||||
return file != nullptr && file->opened;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
||||
int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
|
||||
PRINT_NAME();
|
||||
|
||||
@@ -56,6 +56,8 @@ void Mount(const std::filesystem::path& folder, const std::stri
|
||||
void Umount(const std::string& folder_or_point);
|
||||
// Returns an empty path when the guest path has no mounted filesystem.
|
||||
std::filesystem::path GetRealFilename(const std::string& mounted_file_name);
|
||||
// Returns whether d is an open file, directory or device descriptor.
|
||||
bool IsOpenDescriptor(int d);
|
||||
|
||||
int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode);
|
||||
int KYTY_SYSV_ABI KernelClose(int d);
|
||||
|
||||
+21
-1
@@ -30,6 +30,7 @@
|
||||
#include "common/emulatorConfig.h"
|
||||
#include "common/logging/log.h"
|
||||
#include "common/threads.h"
|
||||
#include "kernel/fileSystem.h"
|
||||
#include "kernel/pthread.h"
|
||||
#include "libs/errno.h"
|
||||
#include "libs/libs.h"
|
||||
@@ -465,7 +466,7 @@ Network::Id Network::HttpCreateTemplate(Id http_ctx_id, const char* user_agent,
|
||||
HttpTemplate tn {};
|
||||
tn.used = true;
|
||||
tn.http_ver = http_ver;
|
||||
tn.user_agent = std::string(user_agent);
|
||||
tn.user_agent = user_agent != nullptr ? user_agent : "";
|
||||
tn.is_auto_proxy_conf = is_auto_proxy_conf;
|
||||
tn.http_ctx_id = http_ctx_id.GetId();
|
||||
tn.nonblock = false;
|
||||
@@ -2577,6 +2578,7 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
|
||||
|
||||
std::vector<std::pair<int, NativeSocket>> descriptors;
|
||||
std::vector<std::shared_ptr<SocketTransport>> stream_reads;
|
||||
std::vector<std::pair<void*, int>> ready_files;
|
||||
int host_nfds = 0;
|
||||
for (int fd = 0; fd < nfds; fd++) {
|
||||
const bool read = GuestFdIsSet(readfds, fd);
|
||||
@@ -2585,6 +2587,16 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
|
||||
if (!read && !write && !except) {
|
||||
continue;
|
||||
}
|
||||
// FreeBSD polls files and devices ready to read and write at once.
|
||||
if (!IsSocket(fd) && LibKernel::FileSystem::IsOpenDescriptor(fd)) {
|
||||
if (read) {
|
||||
ready_files.emplace_back(readfds, fd);
|
||||
}
|
||||
if (write) {
|
||||
ready_files.emplace_back(writefds, fd);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
NativeSocket socket = INVALID_NATIVE_SOCKET;
|
||||
std::shared_ptr<SocketTransport> transport;
|
||||
#if !defined(_WIN32)
|
||||
@@ -2636,6 +2648,10 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
|
||||
host_timeout.tv_usec = static_cast<decltype(host_timeout.tv_usec)>(guest_timeout->tv_usec);
|
||||
host_timeout_ptr = &host_timeout;
|
||||
}
|
||||
if (!ready_files.empty()) {
|
||||
host_timeout = {};
|
||||
host_timeout_ptr = &host_timeout;
|
||||
}
|
||||
|
||||
int result = 0;
|
||||
#if defined(_WIN32)
|
||||
@@ -2672,6 +2688,10 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
|
||||
result++;
|
||||
}
|
||||
}
|
||||
for (const auto& [fds, fd]: ready_files) {
|
||||
GuestFdSet(fds, fd);
|
||||
result++;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "common/abi.h"
|
||||
#include "libs/network.h"
|
||||
#include "loader/symbolDatabase.h"
|
||||
|
||||
#include <array>
|
||||
@@ -162,6 +163,17 @@ void TestBuildHonoursOption(HttpUriParse parse, HttpUriBuild build) {
|
||||
CHECK(BuildWith(build, element, 0) == url);
|
||||
}
|
||||
|
||||
// Cyberpunk 2077 (PPSA03974) creates a template with a NULL user agent.
|
||||
void TestTemplateWithoutUserAgent() {
|
||||
using namespace Libs::Network;
|
||||
Initialize();
|
||||
const int pool = Net::NetPoolCreate("http", 0x10000, 0);
|
||||
const int ssl = Ssl::SslInit(0x10000);
|
||||
const int http = Http::HttpInit(pool, ssl, 0x10000);
|
||||
CHECK(Http::HttpCreateTemplate(http, nullptr, 2, 1) > 0);
|
||||
Shutdown();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
@@ -176,5 +188,6 @@ int main() {
|
||||
TestAbsentQuery(parse);
|
||||
TestEmptyUri(parse);
|
||||
TestPresentQuery(parse);
|
||||
TestTemplateWithoutUserAgent();
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -324,6 +324,7 @@ void CheckSaveRename(const std::filesystem::path &root,
|
||||
}
|
||||
|
||||
void TestRandomDevices() {
|
||||
namespace Net = Libs::Network::Net;
|
||||
for (const auto* path : {"/dev/urandom", "/dev/random"}) {
|
||||
const int fd = FileSystem::KernelOpen(path, 0, 0);
|
||||
FileSystem::FileStat stat {};
|
||||
@@ -333,7 +334,18 @@ void TestRandomDevices() {
|
||||
std::array<uint8_t, 32> entropy {};
|
||||
Check(FileSystem::KernelRead(fd, entropy.data(), entropy.size()) == entropy.size(),
|
||||
"character device supplies requested entropy");
|
||||
// OpenSSL RAND_poll selects the device before reading it; a null timeout must not block.
|
||||
std::array<uint64_t, 16> readable {};
|
||||
readable[fd / 64] = uint64_t {1} << (fd % 64);
|
||||
auto writable = readable;
|
||||
const auto requested = readable;
|
||||
Check(Net::Select(fd + 1, readable.data(), writable.data(), nullptr, nullptr) == 2 &&
|
||||
readable == requested && writable == requested,
|
||||
"select reports an entropy source ready at once");
|
||||
Check(FileSystem::KernelClose(fd) == OK, "close entropy source");
|
||||
Check(Net::Select(fd + 1, readable.data(), nullptr, nullptr, nullptr) == -1 &&
|
||||
*Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EBADF,
|
||||
"select rejects a closed descriptor");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2453,8 +2465,8 @@ int main(int argc, char** argv) {
|
||||
TestAioBatches();
|
||||
CheckSaveRename(temporary.Path(), "first-save");
|
||||
CheckSaveRename(temporary.Path(), "replacement-save");
|
||||
FileSystem::Shutdown();
|
||||
CheckSocketWakeup();
|
||||
FileSystem::Shutdown();
|
||||
CheckEtherAddressFormatting();
|
||||
#if defined(__linux__)
|
||||
TestNetCtlHostIpv4();
|
||||
|
||||
+142
-12
@@ -436,6 +436,128 @@ void TestInvariantIndirectImageMaterialization(bool nested) {
|
||||
"malformed image table tracking was not transactional");
|
||||
}
|
||||
|
||||
void TestResidentImageHeap() {
|
||||
struct FakeImageHeap final : ImageHeapSource {
|
||||
bool accept = true;
|
||||
std::vector<DescriptorValue> kinds;
|
||||
std::vector<ImageHeapRequest> requests;
|
||||
bool Acquire(const ImageHeapRequest &request, ImageHeapView &view) override {
|
||||
requests.push_back(request);
|
||||
if (!accept) return false;
|
||||
view = {.slots = 0x123456789abcull, .slot_count = 36u, .kinds = kinds};
|
||||
return true;
|
||||
}
|
||||
};
|
||||
auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u);
|
||||
fixture->PlanAndTrack();
|
||||
const auto plan = ExtractResourcePlan(fixture->program);
|
||||
std::array<uint32_t, 8> descriptor{};
|
||||
descriptor[0] = 0x20u;
|
||||
descriptor[1] = static_cast<uint32_t>(Libs::Graphics::Prospero::BufferFormat::k32_32_32_32Float) << 20u;
|
||||
descriptor[2] = 3u | (3u << 14u);
|
||||
descriptor[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
|
||||
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kColor2D) << 28u);
|
||||
auto cube = descriptor;
|
||||
cube[3] = Libs::Graphics::DstSel(4, 5, 6, 7) |
|
||||
(static_cast<uint32_t>(Libs::Graphics::Prospero::ImageType::kCube) << 28u);
|
||||
std::array<uint32_t, 16> data{0x3000u, 4u << 16u, 0x100000u, 0u,
|
||||
0x2000u, 48u << 16u, 3u, 0u};
|
||||
std::copy_n(data.begin(), 4u, data.begin() + 12u);
|
||||
LinearTestMemory memory;
|
||||
memory.fail_address = 0x3000u;
|
||||
memory.watched_address = 0x2000u;
|
||||
std::copy(descriptor.begin(), descriptor.end(), memory.words.begin() + 0x1010u / 4u);
|
||||
FakeImageHeap heap;
|
||||
heap.kinds = {{descriptor, 8u}, {cube, 8u}};
|
||||
SrtRuntime runtime{.user_data = data, .userdata = &memory,
|
||||
.read_specialization_memory = ReadLinearTestMemory, .image_heap = &heap};
|
||||
ResourceSnapshot snapshot;
|
||||
ResourceSpecialization specialization;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
heap.requests.size() == 1u && heap.requests[0].base == 0x2000u &&
|
||||
heap.requests[0].size == 144u && heap.requests[0].step == 16u &&
|
||||
heap.requests[0].first == 0u && heap.requests[0].count == 9u &&
|
||||
heap.requests[0].root == &plan.info.images[0] &&
|
||||
memory.watched_reads == 0u && snapshot.images.size() == 3u,
|
||||
"resident image heap was materialized entry by entry");
|
||||
const auto &root = specialization.images[0];
|
||||
const auto mapping = root.indirect_mapping_offset;
|
||||
Check(root.heap && root.indirect_root == 0u && root.indirect_search_iterations == 0u &&
|
||||
snapshot.flattened_srt.size() == mapping + 3u &&
|
||||
snapshot.flattened_srt[mapping] == 36u &&
|
||||
snapshot.flattened_srt[mapping + 1u] == 0x56789abcu &&
|
||||
snapshot.flattened_srt[mapping + 2u] == 0x1234u &&
|
||||
specialization.images[1].heap && specialization.images[1].indirect_root == 0u &&
|
||||
!specialization.images[1].cube && specialization.images[2].heap &&
|
||||
specialization.images[2].cube && snapshot.images[2].dwords == cube &&
|
||||
std::ranges::find(snapshot.specialization_reads,
|
||||
std::pair<uint64_t, uint64_t>{0x2000u, 144u}) !=
|
||||
snapshot.specialization_reads.end(),
|
||||
"resident image heap lost its slot mapping, typed kinds or read dependency");
|
||||
auto &program = fixture->program;
|
||||
ApplyResourceSpecialization(program, specialization);
|
||||
ShaderComputeInputInfo compute{};
|
||||
CollectShaderInfo(program, {.compute = &compute});
|
||||
Check(program.info.uses_image_heap && program.info.images.size() == 3u &&
|
||||
program.info.images[0].indirect_resources == std::vector<uint32_t>({0u, 1u, 2u}) &&
|
||||
std::ranges::all_of(program.info.images, &ImageResource::heap),
|
||||
"applied heap kinds lost their root or heap flag");
|
||||
heap.kinds.clear();
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
snapshot.images.size() == 1u && !specialization.images[0].heap &&
|
||||
specialization.images[0].indirect_root == ImageResource::NoIndirectImage,
|
||||
"a resident heap without kinds did not read as one null image");
|
||||
heap.accept = false;
|
||||
Check(MaterializeResources(plan, runtime, snapshot, specialization) &&
|
||||
heap.requests.size() == 3u && memory.watched_reads == 1u &&
|
||||
snapshot.images.size() == 2u && !specialization.images[0].heap &&
|
||||
snapshot.images[1].dwords == descriptor,
|
||||
"a declined heap was not materialized entry by entry");
|
||||
}
|
||||
|
||||
void TestImageHeapKinds() {
|
||||
using Format = Libs::Graphics::Prospero::BufferFormat;
|
||||
using Type = Libs::Graphics::Prospero::ImageType;
|
||||
ImageResource root;
|
||||
root.resource_class = ImageResourceClass::Sampled;
|
||||
root.dimension = Decoder::ImageDimension::Dim2DArray;
|
||||
root.source = 3u;
|
||||
const auto descriptor = [](Format format, Type type) {
|
||||
DescriptorValue value{.dword_count = 8u};
|
||||
value.dwords[0] = 0x20u;
|
||||
value.dwords[1] = static_cast<uint32_t>(format) << 20u;
|
||||
value.dwords[2] = 3u | (3u << 14u);
|
||||
value.dwords[3] = Libs::Graphics::DstSel(4, 5, 6, 7) | (static_cast<uint32_t>(type) << 28u);
|
||||
return value;
|
||||
};
|
||||
const auto kind_of = [&](Format format, Type type) {
|
||||
auto value = descriptor(format, type);
|
||||
return ImageHeapKind(root, value);
|
||||
};
|
||||
const auto float_kind = kind_of(Format::k32_32_32_32Float, Type::kColor2DArray);
|
||||
const auto uint_kind = kind_of(Format::k32UInt, Type::kColor2DArray);
|
||||
const auto cube_kind = kind_of(Format::k32_32_32_32Float, Type::kCube);
|
||||
auto volume = descriptor(Format::k32_32_32_32Float, Type::kColor3D);
|
||||
Check(float_kind != 0u && uint_kind != 0u && cube_kind != 0u && float_kind != uint_kind &&
|
||||
float_kind != cube_kind && kind_of(Format::kInvalid, Type::kColor2DArray) == 0u &&
|
||||
ImageHeapKind(root, volume) == 0u &&
|
||||
std::ranges::all_of(volume.dwords, [](uint32_t word) { return word == 0u; }),
|
||||
"heap kinds did not separate typed entries or null mismatched ones");
|
||||
const auto resource = ImageHeapResource(root, uint_kind);
|
||||
const auto cube_resource = ImageHeapResource(root, cube_kind);
|
||||
auto other = root;
|
||||
other.source = 9u;
|
||||
Check(resource.numeric_class == Libs::Graphics::Prospero::TextureNumericClass::Uint &&
|
||||
!resource.cube && cube_resource.cube &&
|
||||
cube_resource.numeric_class == Libs::Graphics::Prospero::TextureNumericClass::Float &&
|
||||
resource.resource_class == ImageResourceClass::Sampled && resource.read &&
|
||||
ImageHeapResource(root, 0u) == ImageHeapResource(other, 0u),
|
||||
"heap kind resources lost their type or kept root identity");
|
||||
root.depth_compare = true;
|
||||
Check(kind_of(Format::k32UInt, Type::kColor2DArray) == 0u,
|
||||
"depth comparison accepted an integer heap entry");
|
||||
}
|
||||
|
||||
void TestBoundedImageViewEligibility() {
|
||||
using Type = Libs::Graphics::Prospero::ImageType;
|
||||
auto fixture = MakeIndirectImageFixture(false, 48u, 0u, 16u);
|
||||
@@ -8547,16 +8669,16 @@ void TestShaderInfoCollection() {
|
||||
void TestImageBindingAbi() {
|
||||
using NumericClass = Libs::Graphics::Prospero::TextureNumericClass;
|
||||
|
||||
Check(ImageBindingCount == 48u &&
|
||||
Check(ImageBindingCount == 50u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Buffers) == 0u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 49u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Gds) == 50u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::BdaPagetable) == 51u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::FaultBuffer) == 52u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::FlattenedSrt) == 53u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::ShaderData) == 54u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::SharedMemory) == 55u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Count) == 56u,
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Samplers) == 51u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Gds) == 52u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::BdaPagetable) == 53u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::FaultBuffer) == 54u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::FlattenedSrt) == 55u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::ShaderData) == 56u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::SharedMemory) == 57u &&
|
||||
static_cast<uint32_t>(DescriptorBindingKind::Count) == 58u,
|
||||
"native descriptor binding anchors changed");
|
||||
|
||||
const std::array sampled_dimensions{
|
||||
@@ -8615,6 +8737,8 @@ void TestImageBindingAbi() {
|
||||
CheckBinding(ImageResourceClass::Storage, numeric_class, dimension,
|
||||
false);
|
||||
}
|
||||
CheckBinding(ImageResourceClass::Storage, numeric_class,
|
||||
Decoder::ImageDimension::Dim2DMsaa, false);
|
||||
}
|
||||
for (const auto dimension : storage_dimensions) {
|
||||
CheckBinding(ImageResourceClass::Storage, NumericClass::Uint, dimension,
|
||||
@@ -8658,11 +8782,15 @@ void TestImageBindingAbi() {
|
||||
image.numeric_class = NumericClass::Sint;
|
||||
Check(Invalid(image), "signed storage image received a descriptor binding");
|
||||
image.numeric_class = NumericClass::Float;
|
||||
image.dimension = Decoder::ImageDimension::Dim2DMsaa;
|
||||
image.dimension = Decoder::ImageDimension::Dim2DMsaaArray;
|
||||
Check(Invalid(image),
|
||||
"multisampled storage image received a descriptor binding");
|
||||
image.dimension = Decoder::ImageDimension::Dim2D;
|
||||
"multisampled storage array received a descriptor binding");
|
||||
image.dimension = Decoder::ImageDimension::Dim2DMsaa;
|
||||
image.numeric_class = NumericClass::Uint;
|
||||
image.atomic = true;
|
||||
Check(Invalid(image), "multisampled atomic image received a descriptor binding");
|
||||
image.numeric_class = NumericClass::Float;
|
||||
image.dimension = Decoder::ImageDimension::Dim2D;
|
||||
Check(Invalid(image), "float atomic image received a descriptor binding");
|
||||
}
|
||||
|
||||
@@ -8790,6 +8918,8 @@ int main() {
|
||||
Run("dynamic storage mips", TestDynamicStorageMipTracking);
|
||||
Run("invariant indirect images", [] { TestInvariantIndirectImageMaterialization(true); });
|
||||
Run("unbounded material image keys", [] { TestInvariantIndirectImageMaterialization(false); });
|
||||
Run("resident image heap", TestResidentImageHeap);
|
||||
Run("image heap kinds", TestImageHeapKinds);
|
||||
Run("bounded image view eligibility", TestBoundedImageViewEligibility);
|
||||
Run("exact unfiltered image candidates", TestExactUnfilteredImageCandidates);
|
||||
Run("waterfall image table", TestWaterfallImageTable);
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
#include "graphics/host_gpu/renderer/sync.h"
|
||||
#include "graphics/host_gpu/vulkanCommon.h"
|
||||
#include <vulkan/vulkan_format_traits.hpp>
|
||||
#include "graphics/presentation/videoOut.h"
|
||||
#include "graphics/presentation/window/windowInternal.h"
|
||||
#include "graphics/shader/recompiler/BufferFormat.h"
|
||||
#include "graphics/shader/recompiler/ShaderRecompiler.h"
|
||||
@@ -227,6 +228,18 @@ struct TileManagerTestAccess {
|
||||
}
|
||||
};
|
||||
|
||||
struct ImageHeapTestAccess {
|
||||
static size_t ElementCount(const ImageHeap &heap) { return heap.m_elements.size(); }
|
||||
static size_t MemoryCount(const ImageHeap &heap) { return heap.m_memories.size(); }
|
||||
static const auto &Views(const ImageHeap &heap) { return heap.m_memories.at(0)->views; }
|
||||
static const auto &Element(const ImageHeap &heap, uint32_t entry) {
|
||||
return heap.m_elements[entry & ((1u << ShaderRecompiler::IR::ImageHeapKindShift) - 1u)];
|
||||
}
|
||||
static ImageId ImageOf(const ImageHeap &heap, uint32_t entry) {
|
||||
return Element(heap, entry).binding.image_id;
|
||||
}
|
||||
};
|
||||
|
||||
struct TextureCacheTestAccess {
|
||||
static_assert(std::same_as<decltype(TextureCache::m_slot_images),
|
||||
Common::SlotVector<Image>>);
|
||||
@@ -1619,6 +1632,17 @@ void CheckRectListShaders() {
|
||||
ValidateSpirv(name, position_only.evaluation);
|
||||
}
|
||||
|
||||
// Words without OpName/OpString/OpLine, which only graphics debug dumps emit.
|
||||
size_t SpirvCodeWords(const std::vector<u32> &spirv) {
|
||||
size_t words = spirv.size();
|
||||
for (size_t offset = 5; offset < spirv.size(); offset += spirv[offset] >> 16u) {
|
||||
const auto opcode = static_cast<spv::Op>(spirv[offset] & 0xffffu);
|
||||
if (opcode == spv::OpName || opcode == spv::OpString || opcode == spv::OpLine)
|
||||
words -= spirv[offset] >> 16u;
|
||||
}
|
||||
return words;
|
||||
}
|
||||
|
||||
void CheckSpirvEmission(const TestCase &test, const std::vector<u32> &spirv) {
|
||||
std::vector<u32> result_types(spirv[3]);
|
||||
for (size_t offset = 5; offset < spirv.size();) {
|
||||
@@ -1679,6 +1703,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
|
||||
options.stage = ShaderType::Compute;
|
||||
options.profile = test.profile;
|
||||
options.dump_ir = true;
|
||||
options.emit_debug_info = true; // Checks find variables and functions by OpName.
|
||||
auto compute_info = test.compute_info;
|
||||
compute_info.host_subgroup_size = host_subgroup_size;
|
||||
options.input_info.compute = &compute_info;
|
||||
@@ -1750,7 +1775,7 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
|
||||
result.program.shader_info_complete = false;
|
||||
ShaderRecompiler::IR::CollectShaderInfo(result.program, options.input_info);
|
||||
result.spirv = ShaderRecompiler::Spirv::EmitProgram(
|
||||
result.program, options.input_info, 0u, *options.profile);
|
||||
result.program, options.input_info, 0u, *options.profile, true);
|
||||
const auto shader_data = result.program.bindings.descriptor_counts[
|
||||
static_cast<size_t>(ShaderRecompiler::IR::DescriptorBindingKind::ShaderData)];
|
||||
Require(test.name, "binding layout",
|
||||
@@ -1764,11 +1789,12 @@ CompiledShader CompileCase(const TestCase &test, u32 host_subgroup_size = 64) {
|
||||
ValidateSpirv(test.name, result.spirv);
|
||||
CheckSpirvEmission(test, result.spirv);
|
||||
if (test.max_spirv_words != 0u) {
|
||||
Require(test.name, "emitted shader size", result.spirv.size() <= test.max_spirv_words,
|
||||
"SPIR-V words=" + std::to_string(result.spirv.size()) +
|
||||
const auto words = SpirvCodeWords(result.spirv);
|
||||
Require(test.name, "emitted shader size", words <= test.max_spirv_words,
|
||||
"SPIR-V words=" + std::to_string(words) +
|
||||
", limit=" + std::to_string(test.max_spirv_words));
|
||||
std::printf("[compute] %s wave=%u SPIR-V words=%zu\n", test.name,
|
||||
result.program.wave_size, result.spirv.size());
|
||||
result.program.wave_size, words);
|
||||
}
|
||||
if (test.expected_buffer_resources) {
|
||||
std::vector<u32> resources;
|
||||
@@ -5136,6 +5162,289 @@ public:
|
||||
std::printf("[host] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckResidentImageHeap() {
|
||||
constexpr const char *name = "ResidentImageHeap";
|
||||
constexpr uintptr_t base = 0x0000000206800000ull;
|
||||
constexpr uint64_t allocation_size = 0x200000;
|
||||
constexpr uint64_t table = base + 0x100000;
|
||||
constexpr uint64_t idle_table = base + 0x180000;
|
||||
constexpr uint64_t table_size = 0x10000;
|
||||
namespace IR = ShaderRecompiler::IR;
|
||||
using Format = Prospero::BufferFormat;
|
||||
EnsureRuntimeContext();
|
||||
int64_t direct_offset = -1;
|
||||
Require(name, "direct allocation",
|
||||
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
|
||||
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
|
||||
allocation_size, allocation_size, 0, &direct_offset) == 0,
|
||||
"image heap allocation failed");
|
||||
void *mapped = reinterpret_cast<void *>(base);
|
||||
Require(name, "direct mapping",
|
||||
Libs::LibKernel::Memory::KernelMapDirectMemory(
|
||||
&mapped, allocation_size, 0x3, 0x10, direct_offset,
|
||||
allocation_size) == 0 && mapped == reinterpret_cast<void *>(base),
|
||||
"image heap mapping failed");
|
||||
std::memset(mapped, 0, allocation_size);
|
||||
// A 4x4 linear 2D texture per index.
|
||||
const auto descriptor = [](uint32_t texture, Format format) {
|
||||
const uint64_t address = base + 0x10000u + texture * 0x1000u;
|
||||
std::array<uint32_t, 8> words{};
|
||||
words[0] = static_cast<uint32_t>(address >> 8u);
|
||||
words[1] = static_cast<uint32_t>(address >> 40u) | (static_cast<uint32_t>(format) << 20u);
|
||||
words[2] = 3u | (3u << 14u);
|
||||
words[3] = DstSel(4, 5, 6, 7) |
|
||||
(static_cast<uint32_t>(Prospero::ImageType::kColor2D) << 28u);
|
||||
return words;
|
||||
};
|
||||
const auto put = [&](uint64_t probe, const std::array<uint32_t, 8> &words) {
|
||||
std::memcpy(reinterpret_cast<void *>(table + probe * 32u), words.data(), sizeof(words));
|
||||
};
|
||||
put(0, descriptor(0, Format::k8_8_8_8UNorm));
|
||||
put(1, descriptor(0, Format::k8_8_8_8UNorm));
|
||||
put(5, descriptor(1, Format::k8_8_8_8UNorm));
|
||||
{
|
||||
RenderContext context(m_runtime_context);
|
||||
auto &scheduler = context.GetCommandScheduler();
|
||||
HW::Context registers{};
|
||||
HW::UserConfig user_config{};
|
||||
HW::Shader shaders{};
|
||||
scheduler.Begin(registers, user_config, shaders);
|
||||
context.MapMemory(base, allocation_size);
|
||||
auto &heap = context.GetImageHeap();
|
||||
auto &executor = context.GetRenderExecutor();
|
||||
auto &textures = context.GetTextureCache();
|
||||
auto &device = context.GetGraphics().device;
|
||||
const vk::DescriptorSetLayoutCreateInfo empty_info{};
|
||||
vk::DescriptorSetLayout empty = nullptr;
|
||||
vk::PipelineLayout layout = nullptr;
|
||||
Require(name, "set 0 layout",
|
||||
device.createDescriptorSetLayout(&empty_info, nullptr, &empty) ==
|
||||
vk::Result::eSuccess,
|
||||
"could not create the set 0 layout");
|
||||
const std::array set_layouts{empty, context.GetGraphics().image_heap_layout};
|
||||
vk::PipelineLayoutCreateInfo layout_info{};
|
||||
layout_info.setLayoutCount = 2;
|
||||
layout_info.pSetLayouts = set_layouts.data();
|
||||
Require(name, "pipeline layout",
|
||||
device.createPipelineLayout(&layout_info, nullptr, &layout) == vk::Result::eSuccess,
|
||||
"could not create a pipeline layout with the heap set");
|
||||
|
||||
IR::ImageResource sampled{};
|
||||
sampled.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
|
||||
sampled.unfiltered_access_only = false;
|
||||
auto unfiltered = sampled;
|
||||
unfiltered.unfiltered_access_only = true;
|
||||
IR::ImageHeapView view{};
|
||||
const auto acquire = [&](const IR::ImageResource &root, uint64_t address = table,
|
||||
uint64_t count = table_size / 32u) {
|
||||
return heap.Acquire({address, table_size, 32u, 0u, count, &root}, view);
|
||||
};
|
||||
const auto draw = [&] {
|
||||
heap.Rebind(executor);
|
||||
heap.Commit(scheduler.Current(), vk::PipelineBindPoint::eCompute, layout);
|
||||
scheduler.Finish();
|
||||
heap.BeginDraw();
|
||||
};
|
||||
const auto write = [&](uint64_t probe, const std::array<uint32_t, 8> &words) {
|
||||
Libs::LibKernel::Memory::WriteBacking(table + probe * 32u, words.data(), sizeof(words));
|
||||
Require(name, "table write", context.InvalidateMemory(table + probe * 32u, sizeof(words)),
|
||||
"the table was not mapped");
|
||||
};
|
||||
|
||||
Require(name, "first view",
|
||||
acquire(sampled) && view.kinds.size() == 1u && view.slot_count == table_size / 4u,
|
||||
"a 64-KiB table was not kept resident");
|
||||
const auto &views = ImageHeapTestAccess::Views(heap);
|
||||
const auto &filtered = *views[0];
|
||||
Require(name, "slot words",
|
||||
filtered.slots[0] != 0u && filtered.slots[8] == filtered.slots[0] &&
|
||||
filtered.slots[40] != 0u && filtered.slots[40] != filtered.slots[0] &&
|
||||
filtered.slots[1] == 0u && filtered.slots.back() == 0u &&
|
||||
ImageHeapTestAccess::ElementCount(heap) == 3u &&
|
||||
ImageHeapTestAccess::Element(heap, 0u).binding.layout ==
|
||||
vk::ImageLayout::eShaderReadOnlyOptimal,
|
||||
"probes did not share equal descriptors, left a non-null gap or wrote the null "
|
||||
"element without a sampled layout");
|
||||
Require(name, "second view",
|
||||
acquire(unfiltered) && views.size() == 2u &&
|
||||
ImageHeapTestAccess::ElementCount(heap) == 5u,
|
||||
"a second root profile did not get its own view");
|
||||
const auto &idle = *views[1];
|
||||
draw();
|
||||
const auto resident = ImageHeapTestAccess::ImageOf(heap, filtered.slots[0]);
|
||||
Require(name, "resident images",
|
||||
resident && textures.GetImage(resident).heap_refs.load() == 2u,
|
||||
"resolved elements did not pin their image");
|
||||
|
||||
// Only acquired views take table changes; the idle one must not allocate.
|
||||
const auto idle_slot = idle.slots[40];
|
||||
for (uint32_t texture = 2; texture < 4u; ++texture) {
|
||||
write(5, descriptor(texture, Format::k8_8_8_8UNorm));
|
||||
Require(name, "changed table", acquire(sampled), "a changed table was declined");
|
||||
draw();
|
||||
}
|
||||
Require(name, "idle view",
|
||||
idle.slots[40] == idle_slot && ImageHeapTestAccess::ElementCount(heap) <= 6u,
|
||||
"an idle view took table changes or leaked released elements");
|
||||
Require(name, "reactivated view", acquire(unfiltered), "the idle view was declined");
|
||||
draw();
|
||||
Require(name, "deferred probes",
|
||||
ImageHeapTestAccess::ImageOf(heap, idle.slots[40]) ==
|
||||
ImageHeapTestAccess::ImageOf(heap, filtered.slots[40]) &&
|
||||
ImageHeapTestAccess::ElementCount(heap) <= 7u,
|
||||
"a reactivated view missed the changes made while it was idle");
|
||||
|
||||
// All programs of one draw see the table and kinds of its first acquire.
|
||||
Require(name, "draw kinds", acquire(sampled), "the view was declined");
|
||||
write(7, descriptor(12, Format::k8_8_8_8UInt));
|
||||
Require(name, "stable kinds",
|
||||
acquire(sampled) && view.kinds.size() == 1u && filtered.slots[56] == 0u,
|
||||
"a later program of a draw saw a kind the first one lacked");
|
||||
draw();
|
||||
Require(name, "next draw kinds",
|
||||
acquire(sampled) && view.kinds.size() == 2u &&
|
||||
filtered.slots[56] >> IR::ImageHeapKindShift == 1u,
|
||||
"the next draw did not take the new kind");
|
||||
draw();
|
||||
|
||||
Require(name, "bounded view",
|
||||
acquire(sampled, table, 4u) && views.size() == 3u && views[2]->slots[0] != 0u &&
|
||||
views[2]->slots[40] == 0u,
|
||||
"a view probed descriptors past the guest table bound");
|
||||
draw();
|
||||
|
||||
// Only leaving the read state invalidates resident images.
|
||||
auto &image = textures.GetImage(resident);
|
||||
const auto epoch = Libs::Graphics::Image::heap_epoch.load();
|
||||
for (uint32_t pass = 0; pass < 2u; ++pass) {
|
||||
image.Transit(vk::ImageLayout::eGeneral, vk::AccessFlagBits2::eShaderWrite, {},
|
||||
scheduler.Current().Handle());
|
||||
}
|
||||
Require(name, "write epoch", Libs::Graphics::Image::heap_epoch.load() == epoch + 1u,
|
||||
"repeated writes after leaving the read state bumped the heap epoch again");
|
||||
Require(name, "read again", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "read layout",
|
||||
image.backing.state.layout == vk::ImageLayout::eShaderReadOnlyOptimal &&
|
||||
Libs::Graphics::Image::heap_epoch.load() == epoch + 1u,
|
||||
"commit did not return a resident image to its read layout");
|
||||
|
||||
// An image freed during a draw's discovery lives until the tick ends, and the draw's
|
||||
// slots still select it.
|
||||
write(9, descriptor(13, Format::k8_8_8_8UNorm));
|
||||
Require(name, "freed image", acquire(sampled), "the view was declined");
|
||||
heap.Rebind(executor);
|
||||
const auto freed = ImageHeapTestAccess::ImageOf(heap, filtered.slots[72]);
|
||||
textures.UnmapMemory(base + 0x10000u + 13u * 0x1000u, 0x1000u);
|
||||
heap.Commit(scheduler.Current(), vk::PipelineBindPoint::eCompute, layout);
|
||||
Require(name, "freed image layout",
|
||||
!textures.GetImage(freed).registered &&
|
||||
textures.GetImage(freed).backing.state.layout ==
|
||||
vk::ImageLayout::eShaderReadOnlyOptimal,
|
||||
"commit skipped the read transition of a freed but sampled image");
|
||||
scheduler.Finish();
|
||||
heap.BeginDraw();
|
||||
|
||||
// A draw skipped after its programs acquired views still leaves the next draw a fresh
|
||||
// table.
|
||||
Require(name, "skipped draw", acquire(sampled), "the view was declined");
|
||||
heap.BeginDraw();
|
||||
write(11, descriptor(16, Format::k8_8_8_8UNorm));
|
||||
Require(name, "draw after a skipped one", acquire(sampled) && filtered.slots[88] != 0u,
|
||||
"a draw sampled the table of a skipped draw");
|
||||
// The skipped draw's view stays acquired. Another view refreshing the table must not let
|
||||
// a stale element take a newer kind from it.
|
||||
heap.BeginDraw();
|
||||
write(5, descriptor(15, Format::k8_8_8_8SInt));
|
||||
Require(name, "other view", acquire(unfiltered), "the view was declined");
|
||||
textures.UnmapMemory(base + 0x10000u + 3u * 0x1000u, 0x1000u);
|
||||
draw();
|
||||
Require(name, "stale rescan",
|
||||
filtered.kinds.size() == 2u &&
|
||||
ImageHeapTestAccess::Element(heap, filtered.slots[40]).value.dwords ==
|
||||
descriptor(3, Format::k8_8_8_8UNorm),
|
||||
"a stale element's rescan took a kind its view's programs lack");
|
||||
|
||||
// Tables keep the descriptors of unmapped textures, which PS5 never samples. Their words
|
||||
// stay null in every view, even when the memory is mapped again before the next draw,
|
||||
// until one of their own words changes.
|
||||
const auto texture = [](uint32_t index) { return base + 0x10000u + index * 0x1000u; };
|
||||
write(20, descriptor(20, Format::k8_8_8_8UNorm));
|
||||
Require(name, "live texture", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "live image",
|
||||
static_cast<bool>(ImageHeapTestAccess::ImageOf(heap, filtered.slots[160])),
|
||||
"a mapped texture was not resolved");
|
||||
context.UnmapMemory(texture(20), 0x1000u);
|
||||
context.MapMemory(texture(20), 0x1000u);
|
||||
heap.BeginDraw();
|
||||
Require(name, "dead in a new view",
|
||||
acquire(sampled, table, 64u) && views.back()->slots[160] == 0u,
|
||||
"a view created after the remap resolved a dead descriptor");
|
||||
Require(name, "remapped texture", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "dead descriptor",
|
||||
filtered.slots[160] == 0u && !textures.IsRegionRegistered(texture(20), 0x1000u),
|
||||
"the descriptor of an unmapped texture was resolved again after a remap");
|
||||
write(19, descriptor(19, Format::k8_8_8_8UNorm));
|
||||
write(21, descriptor(21, Format::k8_8_8_8UNorm));
|
||||
Require(name, "neighbour writes", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "dead reprobe", filtered.slots[160] == 0u,
|
||||
"rescanning changed neighbours revived a dead descriptor");
|
||||
write(24, descriptor(20, Format::k8_8_8_8UNorm));
|
||||
Require(name, "republished texture", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "republished descriptor",
|
||||
filtered.slots[160] == 0u && ImageHeapTestAccess::ImageOf(heap, filtered.slots[192]),
|
||||
"the same descriptor in another word was not resolved");
|
||||
write(20, descriptor(22, Format::k8_8_8_8UNorm));
|
||||
Require(name, "rewritten word", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "rewritten descriptor",
|
||||
static_cast<bool>(ImageHeapTestAccess::ImageOf(heap, filtered.slots[160])),
|
||||
"a new descriptor in a dead word was not resolved");
|
||||
// A view that a skipped draw left acquired buries before its rescan.
|
||||
Require(name, "skipped draw view", acquire(sampled), "the view was declined");
|
||||
context.UnmapMemory(texture(22), 0x1000u);
|
||||
context.MapMemory(texture(22), 0x1000u);
|
||||
heap.BeginDraw();
|
||||
draw();
|
||||
Require(name, "dead after a skipped draw", filtered.slots[160] == 0u,
|
||||
"the rescan of a view left acquired resolved a dead descriptor");
|
||||
// A new descriptor over unmapped memory is never resolved.
|
||||
context.UnmapMemory(texture(23), 0x1000u);
|
||||
write(23, descriptor(23, Format::k8_8_8_8UNorm));
|
||||
Require(name, "unmapped texture", acquire(sampled), "the view was declined");
|
||||
draw();
|
||||
Require(name, "unmapped descriptor", !textures.IsRegionRegistered(texture(23), 0x1000u),
|
||||
"a descriptor over unmapped memory was resolved");
|
||||
|
||||
heap.UnmapMemory(table, table_size);
|
||||
Require(name, "unmapped table",
|
||||
ImageHeapTestAccess::MemoryCount(heap) == 0u &&
|
||||
textures.GetImage(resident).heap_refs.load() == 0u,
|
||||
"unmapping the table kept its images pinned");
|
||||
// The idle table stays watched; destroying the heap must unwatch it before the page
|
||||
// manager fails on live watches.
|
||||
Require(name, "idle table",
|
||||
acquire(sampled, idle_table) && view.kinds.empty() &&
|
||||
ImageHeapTestAccess::MemoryCount(heap) == 1u,
|
||||
"an all-null table was not kept resident");
|
||||
draw();
|
||||
context.UnmapMemory(base, idle_table - base);
|
||||
scheduler.Finish();
|
||||
device.destroyPipelineLayout(layout, nullptr);
|
||||
device.destroyDescriptorSetLayout(empty, nullptr);
|
||||
}
|
||||
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
|
||||
"image heap unmap failed");
|
||||
Require(name, "release", Libs::LibKernel::Memory::KernelReleaseDirectMemory(
|
||||
direct_offset, allocation_size) == 0,
|
||||
"image heap release failed");
|
||||
std::printf("[host] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckComputeMetaClearClassification() {
|
||||
constexpr const char *name = "ComputeMetaClearClassification";
|
||||
constexpr uint64_t read_only_meta = 0x0000000204201f00ull;
|
||||
@@ -12464,6 +12773,212 @@ public:
|
||||
std::printf("[gpu] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
// Cyberpunk 2077 copies its HTile D16 depth target through a kDepth k16UInt image load.
|
||||
void CheckDepthBitsCopy() {
|
||||
constexpr const char *name = "DepthBitsCopy";
|
||||
constexpr uintptr_t base = 0x0000000205000000ull;
|
||||
constexpr uint64_t allocation_size = 0x30000;
|
||||
constexpr uint64_t htile_address = base + 0x10000;
|
||||
constexpr uint64_t copy_address = base + 0x20000;
|
||||
constexpr u32 depth_code = 0x2000u;
|
||||
std::vector<u32> copy{EncodeMimg0(0x00, 0x1), EncodeMimg1(4, 0, 2),
|
||||
EncodeMimg0(0x08, 0x1), EncodeMimg1(4, 0, 0)};
|
||||
AppendEnd(©);
|
||||
ShaderMapUserData(reinterpret_cast<uint64_t>(copy.data()),
|
||||
{.type = Prospero::ShaderBinaryType::kCs,
|
||||
.code_size_bytes = static_cast<u32>(copy.size() * sizeof(u32))});
|
||||
EnsureRuntimeContext();
|
||||
int64_t direct_offset = -1;
|
||||
Require(name, "direct allocation",
|
||||
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
|
||||
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
|
||||
allocation_size, 0x10000, 0, &direct_offset) == 0,
|
||||
"depth allocation failed");
|
||||
void *mapped = reinterpret_cast<void *>(base);
|
||||
Require(name, "direct mapping",
|
||||
Libs::LibKernel::Memory::KernelMapDirectMemory(
|
||||
&mapped, allocation_size, 0x3, 0x10, direct_offset, 0x10000) == 0 &&
|
||||
mapped == reinterpret_cast<void *>(base),
|
||||
"depth mapping failed");
|
||||
std::memset(mapped, 0, allocation_size);
|
||||
RenderContext context(m_runtime_context);
|
||||
context.InitializeGpu(nullptr);
|
||||
LibKernel::Memory::InstallGpuResources(&context);
|
||||
context.GetGpu().SendCommandSync([&] {
|
||||
CommandProcessor processor(context, 0);
|
||||
processor.BufferInit();
|
||||
auto &scheduler = context.GetCommandScheduler();
|
||||
auto &textures = context.GetTextureCache();
|
||||
auto &shaders = processor.GetShCtx();
|
||||
context.MapMemory(base, allocation_size);
|
||||
ImageDesc depth{};
|
||||
depth.type = BindingType::DepthTarget;
|
||||
depth.info.data = {base, 0x10000};
|
||||
depth.info.pixel_format = vk::Format::eD16Unorm;
|
||||
depth.info.guest_format = Prospero::BufferFormat::k16UNorm;
|
||||
depth.info.type = Prospero::ImageType::kColor2D;
|
||||
depth.info.extent = {256, 128, 1};
|
||||
depth.info.resources = {1, 1};
|
||||
depth.info.pitch = 256;
|
||||
depth.info.bytes_per_block = 2;
|
||||
depth.info.samples = 1;
|
||||
depth.info.tile_mode = Prospero::TileMode::kDepth;
|
||||
depth.info.UpdateSize();
|
||||
depth.info.metadata.kind = ImageMetadataKind::Htile;
|
||||
depth.info.metadata.range = {htile_address, 0x8000};
|
||||
depth.info.htile_initial_clear = false;
|
||||
depth.view_info.format = vk::Format::eD16Unorm;
|
||||
depth.view_info.type = vk::ImageViewType::e2D;
|
||||
depth.view_info.aspect = vk::ImageAspectFlagBits::eDepth;
|
||||
depth.view_info.usage = vk::ImageUsageFlagBits::eDepthStencilAttachment;
|
||||
const auto depth_id = textures.FindImage(depth);
|
||||
Require(name, "HTile depth target", textures.FindDepthTarget(depth_id, depth) != nullptr,
|
||||
"the D16 depth target was not created");
|
||||
vk::ClearValue clear{};
|
||||
clear.depthStencil.depth = static_cast<float>(depth_code) / 65535.0f;
|
||||
TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), depth_id,
|
||||
{vk::ImageAspectFlagBits::eDepth, 0, 1, 0, 1}, clear);
|
||||
|
||||
// The captured T# with this depth target's size and HTile; the copy writes 2x2 texels.
|
||||
const ShaderTextureResource depth_bits{{static_cast<u32>(base >> 8u),
|
||||
(static_cast<u32>(Prospero::BufferFormat::k16UInt) << 20u) | (3u << 30u),
|
||||
0x3fu | (127u << 14u),
|
||||
DstSel(4, 0, 0, 1) | (static_cast<u32>(Prospero::TileMode::kDepth) << 20u) |
|
||||
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
|
||||
0, 0x00700000u, 0x00280000u | (static_cast<u32>(htile_address >> 8u) << 24u),
|
||||
static_cast<u32>(htile_address >> 16u)}};
|
||||
const ShaderTextureResource target{{static_cast<u32>(copy_address >> 8u),
|
||||
(static_cast<u32>(Prospero::BufferFormat::k16UInt) << 20u) | (1u << 30u), 1u << 14u,
|
||||
DstSel(4, 5, 6, 7) | (static_cast<u32>(Prospero::TileMode::kLinear) << 20u) |
|
||||
(static_cast<u32>(Prospero::ImageType::kColor2D) << 28u),
|
||||
0, 0x00700000u, 0, 0}};
|
||||
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(copy.data()),
|
||||
.num_thread_x = 2, .num_thread_y = 2, .num_thread_z = 1,
|
||||
.wave_size = 64, .user_sgpr = 16, .tidig_comp_cnt = 1});
|
||||
for (u32 i = 0; i < 8; i++) {
|
||||
shaders.SetCsUserSgpr(i, target.fields[i], HW::UserSgprType::Unknown);
|
||||
shaders.SetCsUserSgpr(8u + i, depth_bits.fields[i], HW::UserSgprType::Unknown);
|
||||
}
|
||||
processor.DispatchDirect(1, 1, 1, 0x41u);
|
||||
|
||||
ShaderRecompiler::IR::DescriptorValue value{};
|
||||
value.dword_count = 8;
|
||||
std::copy_n(target.fields, 8, value.dwords.begin());
|
||||
ShaderRecompiler::IR::ImageResource resource{};
|
||||
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
|
||||
resource.numeric_class = Prospero::TextureNumericClass::Uint;
|
||||
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2D;
|
||||
resource.written = true;
|
||||
const auto binding =
|
||||
RenderExecutorTestAccess::ResolveTexture(context.GetRenderExecutor(), resource, value);
|
||||
Require(name, "raw depth bits",
|
||||
ReadCachedTexel(name, context, binding.image_id, {}, {2, 2, 1}) ==
|
||||
std::vector<u32>(2, depth_code | depth_code << 16u),
|
||||
"the copy did not read the stored D16 codes");
|
||||
Require(name, "depth target retained", TextureCacheTestAccess::Contains(textures, depth_id),
|
||||
"reading depth bits replaced the HTile depth target");
|
||||
context.UnmapMemory(base, allocation_size);
|
||||
scheduler.Finish();
|
||||
});
|
||||
context.ShutdownGpu();
|
||||
LibKernel::Memory::InstallGpuResources(nullptr);
|
||||
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
|
||||
"depth mapping release failed");
|
||||
Require(name, "release",
|
||||
Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0,
|
||||
"depth allocation release failed");
|
||||
std::printf("[gpu] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
// Cyberpunk 2077 clears a rect of an MSAA color target from compute, one sample per store.
|
||||
void CheckMsaaStorageClear() {
|
||||
constexpr const char *name = "MsaaStorageClear";
|
||||
constexpr uintptr_t base = 0x0000000205000000ull;
|
||||
constexpr uint64_t allocation_size = 0x10000;
|
||||
// Captured CS 0xad6c86aca9c0916e: s[8:11] color, s[12:13] origin, s14 width,
|
||||
// s15 height << 16 | samples.
|
||||
std::vector<u32> clear{0xbfa00001u, 0xd7460001u, 0x04050611u, 0xd7460000u, 0x04010610u,
|
||||
0x7da21ef9u, 0x85060001u, 0x7da8000eu, 0xbf88000fu, 0x4a08000cu,
|
||||
0x4a0a020du, 0x996b800fu, 0xbeea0380u, 0xbf0a6b6au, 0xbf840009u,
|
||||
0x7e0c026au, 0x7e000208u, 0x7e020209u, 0x7e04020au, 0x7e06020bu,
|
||||
0x816a816au, 0xf0200f30u, 0x00000004u, 0xbf82fff5u, 0xbf810000u};
|
||||
ShaderMapUserData(reinterpret_cast<uint64_t>(clear.data()),
|
||||
{.type = Prospero::ShaderBinaryType::kCs,
|
||||
.code_size_bytes = static_cast<u32>(clear.size() * sizeof(u32))});
|
||||
EnsureRuntimeContext();
|
||||
int64_t direct_offset = -1;
|
||||
Require(name, "direct allocation",
|
||||
Libs::LibKernel::Memory::KernelAllocateDirectMemory(
|
||||
0, Libs::LibKernel::Memory::KernelGetDirectMemorySize(),
|
||||
allocation_size, 0x10000, 0, &direct_offset) == 0,
|
||||
"MSAA allocation failed");
|
||||
void *mapped = reinterpret_cast<void *>(base);
|
||||
Require(name, "direct mapping",
|
||||
Libs::LibKernel::Memory::KernelMapDirectMemory(
|
||||
&mapped, allocation_size, 0x3, 0x10, direct_offset, 0x10000) == 0 &&
|
||||
mapped == reinterpret_cast<void *>(base),
|
||||
"MSAA mapping failed");
|
||||
RenderContext context(m_runtime_context);
|
||||
context.InitializeGpu(nullptr);
|
||||
LibKernel::Memory::InstallGpuResources(&context);
|
||||
context.GetGpu().SendCommandSync([&] {
|
||||
CommandProcessor processor(context, 0);
|
||||
processor.BufferInit();
|
||||
auto &scheduler = context.GetCommandScheduler();
|
||||
auto &textures = context.GetTextureCache();
|
||||
auto &shaders = processor.GetShCtx();
|
||||
context.MapMemory(base, allocation_size);
|
||||
// 8x8 4x MSAA RGBA8: LAST_LEVEL and MAX_MIP hold log2(samples).
|
||||
const ShaderTextureResource target{{static_cast<u32>(base >> 8u),
|
||||
(static_cast<u32>(Prospero::BufferFormat::k8_8_8_8UNorm) << 20u) | (3u << 30u),
|
||||
1u | (7u << 14u),
|
||||
DstSel(4, 5, 6, 7) | (2u << 16u) |
|
||||
(static_cast<u32>(Prospero::TileMode::kRenderTarget) << 20u) |
|
||||
(static_cast<u32>(Prospero::ImageType::kColor2DMsaa) << 28u),
|
||||
0, 0x00700020u, 0, 0}};
|
||||
ShaderRecompiler::IR::DescriptorValue value{};
|
||||
value.dword_count = 8;
|
||||
std::copy_n(target.fields, 8, value.dwords.begin());
|
||||
ShaderRecompiler::IR::ImageResource resource{};
|
||||
resource.resource_class = ShaderRecompiler::IR::ImageResourceClass::Storage;
|
||||
resource.numeric_class = Prospero::TextureNumericClass::Float;
|
||||
resource.dimension = ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa;
|
||||
resource.written = true;
|
||||
const auto binding =
|
||||
RenderExecutorTestAccess::ResolveTexture(context.GetRenderExecutor(), resource, value);
|
||||
TextureCacheTestAccess::ClearImage(textures, scheduler.Current(), binding.image_id,
|
||||
{vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}, {});
|
||||
shaders.SetCsShader({.data_addr = reinterpret_cast<uint64_t>(clear.data()),
|
||||
.num_thread_x = 8, .num_thread_y = 8, .num_thread_z = 1,
|
||||
.wave_size = 64, .user_sgpr = 16, .tgid_x_en = true,
|
||||
.tgid_y_en = true, .tidig_comp_cnt = 1});
|
||||
const std::array<u32, 8> args{std::bit_cast<u32>(1.0f), 0, 0, std::bit_cast<u32>(1.0f),
|
||||
1, 1, 2, (2u << 16u) | 4u};
|
||||
for (u32 i = 0; i < 8; i++) {
|
||||
shaders.SetCsUserSgpr(i, target.fields[i], HW::UserSgprType::Unknown);
|
||||
shaders.SetCsUserSgpr(8u + i, args[i], HW::UserSgprType::Unknown);
|
||||
}
|
||||
processor.DispatchDirect(1, 1, 1, 0x41u);
|
||||
for (const int row : {1, 2, 3}) {
|
||||
const u32 inside = row == 3 ? 0u : 0xff0000ffu;
|
||||
Require(name, "every sample in the rect",
|
||||
ReadCachedTexel(name, context, binding.image_id, {0, row, 0}, {4, 1, 1}) ==
|
||||
std::vector<u32>{0, inside, inside, 0},
|
||||
"the compute clear missed samples or wrote outside its rect");
|
||||
}
|
||||
context.UnmapMemory(base, allocation_size);
|
||||
scheduler.Finish();
|
||||
});
|
||||
context.ShutdownGpu();
|
||||
LibKernel::Memory::InstallGpuResources(nullptr);
|
||||
Require(name, "unmap", Libs::LibKernel::Memory::KernelMunmap(base, allocation_size) == 0,
|
||||
"MSAA mapping release failed");
|
||||
Require(name, "release",
|
||||
Libs::LibKernel::Memory::KernelReleaseDirectMemory(direct_offset, allocation_size) == 0,
|
||||
"MSAA allocation release failed");
|
||||
std::printf("[gpu] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckSampledDccClear() {
|
||||
constexpr const char *name = "SampledDccClear";
|
||||
constexpr uintptr_t base = 0x0000000205000000ull;
|
||||
@@ -20872,6 +21387,7 @@ private:
|
||||
device_info.pNext = &conditional_rendering;
|
||||
vk::PhysicalDeviceFeatures device_features{};
|
||||
device_features.shaderStorageImageWriteWithoutFormat = true;
|
||||
device_features.shaderStorageImageMultisample = available_features.shaderStorageImageMultisample;
|
||||
device_features.shaderImageGatherExtended = true;
|
||||
device_features.sampleRateShading = true;
|
||||
device_features.shaderInt64 = true;
|
||||
@@ -37759,6 +38275,39 @@ template <bool rg, bool snorm = false> TestCase ImageSamplePacked8() {
|
||||
return test;
|
||||
}
|
||||
|
||||
// A kDepth k16UInt image load returns the stored D16 codes and integer swizzle constants.
|
||||
TestCase ImageLoadDepthBits() {
|
||||
using O = ShaderOpcode;
|
||||
TestCase test;
|
||||
test.name = "ImageLoadDepthBits";
|
||||
constexpr std::array<u32, 4> depth{0x2000u, 0xc000u, 0xffffu, 0x0001u};
|
||||
for (u32 texel = 0; texel < depth.size(); ++texel) {
|
||||
AppendVMovU32(&test.code, 20, texel % 2u);
|
||||
AppendVMovU32(&test.code, 21, texel / 2u);
|
||||
test.code.push_back(EncodeMimg0(0x00, 0xf));
|
||||
test.code.push_back(EncodeMimg1(0, 20));
|
||||
for (u32 component = 0; component < 4; ++component) {
|
||||
AppendStoreVgpr(&test.code, component, texel * 4u + component);
|
||||
}
|
||||
test.expected.insert(test.expected.end(), {depth[texel], 0u, 0u, 1u});
|
||||
}
|
||||
AppendEnd(&test.code);
|
||||
test.image_width = test.image_height = 2;
|
||||
test.sampled_image_rgba = {depth[0] | depth[1] << 16u, depth[2] | depth[3] << 16u};
|
||||
test.sampled_image_format = vk::Format::eR16Unorm;
|
||||
test.sampled_image_dwords_per_pixel = 1;
|
||||
test.user_data = MakeSampledTextureData(Prospero::BufferFormat::k16UInt);
|
||||
test.user_data[1] |= 1u << 30u;
|
||||
test.user_data[2] = 1u << 14u;
|
||||
test.user_data[3] |= static_cast<u32>(Prospero::TileMode::kDepth) << 20u;
|
||||
test.user_data[50] = test.expected.size() * sizeof(u32);
|
||||
test.user_data[51] = 3u << 28u;
|
||||
test.has_user_data = true;
|
||||
test.image_descriptor_swizzle = DstSel(4, 0, 0, 1);
|
||||
test.opcodes = {O::V_MOV_B32, O::IMAGE_LOAD, O::BUFFER_STORE_DWORD, O::S_ENDPGM};
|
||||
return test;
|
||||
}
|
||||
|
||||
TestCase ImageLoadR128IgnoresAdjacentMaskSgprs() {
|
||||
using O = ShaderOpcode;
|
||||
|
||||
@@ -40580,7 +41129,7 @@ void CheckNestedPointerImageTable(VulkanHarness &vulkan) {
|
||||
compiled.program.info.samplers.size() == rows + 1u &&
|
||||
tools.Disassemble(compiled.spirv, &text) &&
|
||||
CountText(text, "OpImageSampleExplicitLod") == halves &&
|
||||
compiled.spirv.size() <= 8192u * halves,
|
||||
SpirvCodeWords(compiled.spirv) <= 8192u * halves,
|
||||
"59 sampler records hit the SGPR root limit or duplicated image sampling bodies");
|
||||
}
|
||||
if (large_samplers)
|
||||
@@ -41035,6 +41584,41 @@ void CheckIndirectImageKeySwitch(VulkanHarness &vulkan) {
|
||||
Require(name, "alternating image emitted size", spirv.size() <= 8192u,
|
||||
"alternating descriptors exceeded the 32 KiB SPIR-V regression budget");
|
||||
|
||||
// A resident heap selects its typed kind and element from a device slot word.
|
||||
auto heap_root = root;
|
||||
heap_root.heap = true;
|
||||
heap_root.indirect_search_iterations = 0;
|
||||
heap_root.indirect_resources = {0u, 1u, 2u};
|
||||
auto heap_kind = candidate;
|
||||
heap_kind.heap = true;
|
||||
auto heap_uint = heap_kind;
|
||||
heap_uint.numeric_class = Prospero::TextureNumericClass::Uint;
|
||||
for (const bool mixed : {true, false}) {
|
||||
program.info.images = {heap_root, heap_kind};
|
||||
if (mixed) program.info.images.push_back(heap_uint);
|
||||
program.info.images[0].indirect_resources.resize(program.info.images.size());
|
||||
// ApplyResourceSpecialization adds the integer sampler variant of a Uint kind.
|
||||
program.info.samplers.resize(1);
|
||||
if (mixed) {
|
||||
program.info.samplers.push_back(program.info.samplers[0]);
|
||||
program.info.samplers[1].integer_border = true;
|
||||
}
|
||||
program.shader_info_complete = false;
|
||||
CollectShaderInfo(program, {.compute = &compute});
|
||||
spirv = ShaderRecompiler::Spirv::EmitProgram(program, {.compute = &compute});
|
||||
ValidateSpirv("ResidentImageHeap", spirv);
|
||||
Require(name, "image heap disassembly", tools.Disassemble(spirv, &text),
|
||||
"failed to disassemble the image heap shader");
|
||||
Require(name, "image heap selection",
|
||||
program.info.uses_image_heap && CountText(text, "OpConvertUToPtr") == 1 &&
|
||||
CountText(text, "UMin") == 1 &&
|
||||
CountText(text, "OpSwitch") == (mixed ? 1 : 0) &&
|
||||
CountText(text, "OpImageSampleExplicitLod") == (mixed ? 2 : 1) &&
|
||||
CountText(text, "DescriptorSet 1") == (mixed ? 2 : 1) &&
|
||||
text.find("PhysicalStorageBuffer64") != std::string::npos,
|
||||
"image heap kinds did not sample the slot-selected heap element");
|
||||
}
|
||||
|
||||
program.memory_info[0].image_dimension =
|
||||
ShaderRecompiler::Decoder::ImageDimension::Dim2DArray;
|
||||
program.memory_info[0].image_address_components = 4;
|
||||
@@ -43665,6 +44249,7 @@ std::vector<TestCase> MakeCases() {
|
||||
AddCase(ImageSamplePacked8<false>);
|
||||
AddCase(ImageSamplePacked8<true>);
|
||||
AddCase(ImageSamplePacked8<false, true>);
|
||||
AddCase(ImageLoadDepthBits);
|
||||
AddCase(ImageLoadR128IgnoresAdjacentMaskSgprs);
|
||||
AddCase(ImageLoad1DUsesScalarCoordinate);
|
||||
AddCase(ImageGather2DInstructionWith1DDescriptor);
|
||||
@@ -43799,7 +44384,7 @@ void CheckComputeLdsLimit(VulkanHarness &vulkan) {
|
||||
regs.cs_regs.num_thread_y = 1;
|
||||
regs.cs_regs.num_thread_z = 1;
|
||||
regs.cs_regs.wave_size = 32;
|
||||
PipelineCache cache(graphics);
|
||||
PipelineCache cache(graphics, nullptr);
|
||||
ShaderProgram previous;
|
||||
uint16_t previous_units = 0;
|
||||
for (const auto units : {static_cast<uint16_t>(limit_units - 1u), limit_units,
|
||||
@@ -49095,6 +49680,94 @@ void CheckPm4IndirectControlFlow(RenderContext &renderer) {
|
||||
std::printf("[host] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
// waitUntilSafeForRendering stalls only its own command processor, and a display buffer
|
||||
// stays busy while its flip is pending or while it is on screen.
|
||||
void CheckWaitUntilSafeForRendering() {
|
||||
constexpr const char *name = "WaitUntilSafeForRendering";
|
||||
Require(name, "video", SDL_InitSubSystem(SDL_INIT_VIDEO), "SDL video init failed");
|
||||
WindowContext window;
|
||||
window.graphic_ctx.screen_width = 64;
|
||||
window.graphic_ctx.screen_height = 64;
|
||||
window.window = SDL_CreateWindow(name, 64, 64, SDL_WINDOW_VULKAN | SDL_WINDOW_HIDDEN);
|
||||
Require(name, "window", window.window != nullptr, "hidden Vulkan window failed");
|
||||
window.CreateVulkan();
|
||||
auto &video_out = VideoOut::VideoOutInit(64, 64, *window.presenter);
|
||||
auto &renderer = *window.render_context;
|
||||
renderer.InitializeGpu(&video_out);
|
||||
auto &gpu = renderer.GetGpu();
|
||||
|
||||
constexpr int user_system = 255;
|
||||
constexpr int bus_main = 0;
|
||||
constexpr int flip_mode_vsync = 1;
|
||||
// Special buffer indices flip without a registered surface.
|
||||
constexpr int index_blank = -1;
|
||||
constexpr int index_black = -2;
|
||||
const int handle = VideoOut::VideoOutOpen(user_system, bus_main, 0, nullptr);
|
||||
Require(name, "open", handle > 0, "video-out open failed");
|
||||
const auto wait_until = [](const auto &done) {
|
||||
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
|
||||
while (!done() && std::chrono::steady_clock::now() < deadline) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
}
|
||||
return done();
|
||||
};
|
||||
const auto address = [](const void *value) { return reinterpret_cast<uint64_t>(value); };
|
||||
|
||||
// Hold the graphics queue so the CPU flip of BLACK, prepared behind it, stays pending.
|
||||
std::atomic_uint32_t gate{0};
|
||||
std::array<uint32_t, 7> hold{};
|
||||
hold[0] = KYTY_PM4(7, Pm4::IT_WAIT_REG_MEM, 0);
|
||||
hold[1] = 0x10u | 3u;
|
||||
hold[2] = static_cast<uint32_t>(address(&gate));
|
||||
hold[3] = static_cast<uint32_t>(address(&gate) >> 32u);
|
||||
hold[4] = 1;
|
||||
hold[5] = UINT32_MAX;
|
||||
gpu.Submit(hold, {});
|
||||
Require(name, "flip black",
|
||||
VideoOut::VideoOutSubmitFlip(handle, index_black, flip_mode_vsync, 0) == 0,
|
||||
"CPU flip failed");
|
||||
|
||||
std::atomic_uint32_t rendered{0};
|
||||
std::array<uint32_t, 12> render{};
|
||||
render[0] = KYTY_PM4(7, Pm4::IT_NOP, Pm4::R_WAIT_FLIP_DONE);
|
||||
render[1] = static_cast<uint32_t>(handle);
|
||||
render[2] = static_cast<uint32_t>(index_black);
|
||||
render[7] = KYTY_PM4(5, Pm4::IT_WRITE_DATA, 0);
|
||||
render[9] = static_cast<uint32_t>(address(&rendered));
|
||||
render[10] = static_cast<uint32_t>(address(&rendered) >> 32u);
|
||||
render[11] = 1;
|
||||
gpu.SubmitCompute(0x20, render);
|
||||
|
||||
std::binary_semaphore lane{0};
|
||||
std::jthread command_lane([&] {
|
||||
gpu.SendCommandSync([] {});
|
||||
lane.release();
|
||||
});
|
||||
Require(name, "command lane", lane.try_acquire_for(std::chrono::seconds(2)),
|
||||
"a busy display buffer blocked the GPU thread");
|
||||
Require(name, "pending", rendered.load() == 0,
|
||||
"rendering started while its display buffer had a pending flip");
|
||||
|
||||
gate.store(1);
|
||||
Require(name, "flip shown", wait_until([handle] { return VideoOut::VideoOutIsFlipPending(handle) == 0; }),
|
||||
"the BLACK flip was not presented");
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
Require(name, "displayed", rendered.load() == 0,
|
||||
"rendering started while its display buffer was on screen");
|
||||
|
||||
Require(name, "flip blank",
|
||||
VideoOut::VideoOutSubmitFlip(handle, index_blank, flip_mode_vsync, 0) == 0,
|
||||
"CPU flip failed");
|
||||
Require(name, "resume", wait_until([&] { return rendered.load() == 1; }),
|
||||
"rendering did not resume once its display buffer left the screen");
|
||||
|
||||
gpu.WaitForIdle();
|
||||
Require(name, "close", VideoOut::VideoOutClose(handle) == 0, "video-out close failed");
|
||||
renderer.ShutdownGpu();
|
||||
VideoOut::VideoOutShutdown();
|
||||
std::printf("[host] %-32s ok\n", name);
|
||||
}
|
||||
|
||||
void CheckPm4WaitResume(RenderContext &renderer) {
|
||||
GraphicsInitJmpTables();
|
||||
CommandProcessor processor(renderer, 0);
|
||||
@@ -49617,6 +50290,17 @@ int main(int argc, char **argv) {
|
||||
RunCase(&vulkan, ImageSamplePacked8<false, true>());
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--msaa-storage-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
vulkan.CheckMsaaStorageClear();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--depth-bits-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
vulkan.CheckDepthBitsCopy();
|
||||
RunCase(&vulkan, ImageLoadDepthBits());
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--packed-texture-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
RunCase(&vulkan, ImageLoadSintUsesSignedSampledImage<false>());
|
||||
@@ -49974,6 +50658,10 @@ int main(int argc, char **argv) {
|
||||
vulkan.CheckScalarLeafCall();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--wait-until-safe-for-rendering-only") == 0) {
|
||||
CheckWaitUntilSafeForRendering();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::strcmp(argv[1], "--gpu-command-lane-only") == 0) {
|
||||
VulkanHarness vulkan;
|
||||
vulkan.CheckGpuCommandLane();
|
||||
@@ -50295,6 +50983,7 @@ int main(int argc, char **argv) {
|
||||
CheckNestedPointerImageTable(vulkan);
|
||||
CheckGuardedRecordImageTable(vulkan);
|
||||
CheckIndirectImageKeySwitch(vulkan);
|
||||
vulkan.CheckResidentImageHeap();
|
||||
CheckFiniteFirstLaneImageChoice(vulkan);
|
||||
CheckFiniteInlineSamplerPhi(vulkan);
|
||||
RunCase(&vulkan, ImageCubeGradientsPreserveDerivatives());
|
||||
@@ -50473,6 +51162,7 @@ int main(int argc, char **argv) {
|
||||
CheckNestedPointerImageTable(vulkan);
|
||||
CheckGuardedRecordImageTable(vulkan);
|
||||
CheckIndirectImageKeySwitch(vulkan);
|
||||
vulkan.CheckResidentImageHeap();
|
||||
CheckDynamicSampleOffsets(vulkan, 32u);
|
||||
CheckDynamicSampleOffsets(vulkan, 64u);
|
||||
CheckFiniteFirstLaneImageChoice(vulkan);
|
||||
@@ -50504,6 +51194,8 @@ int main(int argc, char **argv) {
|
||||
vulkan.CheckRenderExecutorColorVolumeDiscovery();
|
||||
vulkan.CheckRenderExecutorColorMetadataClear();
|
||||
vulkan.CheckSampledDccClear();
|
||||
vulkan.CheckDepthBitsCopy();
|
||||
vulkan.CheckMsaaStorageClear();
|
||||
vulkan.CheckRenderExecutorColorStandardTileDiscovery();
|
||||
vulkan.CheckRenderExecutorColorDepthTileDiscovery();
|
||||
vulkan.CheckRenderExecutorStencilBindingDiscovery();
|
||||
@@ -50527,6 +51219,8 @@ int main(int argc, char **argv) {
|
||||
vulkan.CheckGpuSuspendPoint();
|
||||
vulkan.CheckGpuCommandLane();
|
||||
vulkan.CheckScalarLeafCall();
|
||||
// Last: its window installs global GPU resources.
|
||||
CheckWaitUntilSafeForRendering();
|
||||
if (skipped_device_checks) {
|
||||
std::printf(
|
||||
"ShaderRecompilerComputeTests: device rasterization checks skipped, this "
|
||||
|
||||
@@ -99,6 +99,7 @@ ShaderRecompiler::CompileOptions MakeCompileOptions(ShaderType stage) {
|
||||
ShaderRecompiler::CompileOptions options;
|
||||
options.stage = stage;
|
||||
options.user_data = user_data;
|
||||
options.emit_debug_info = true; // Checks find variables and functions by OpName.
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex:
|
||||
options.input_info.vertex = &vertex;
|
||||
@@ -403,6 +404,11 @@ SpirvMetrics MeasureSpirv(const std::vector<uint32_t> &binary) {
|
||||
if (word_count == 0 || i + word_count > binary.size()) {
|
||||
break;
|
||||
}
|
||||
if (opcode == spv::OpName || opcode == spv::OpString || opcode == spv::OpLine) {
|
||||
metrics.words -= word_count; // Budgets measure the module without debug info.
|
||||
i += word_count;
|
||||
continue;
|
||||
}
|
||||
metrics.instructions++;
|
||||
switch (opcode) {
|
||||
case 11u: // OpExtInstImport
|
||||
@@ -10354,7 +10360,7 @@ void TestNewShaderRecompilerPlanningOnlyLoads() {
|
||||
ir.Emit(IR::ValueOpcode::ReferenceU32, {read});
|
||||
program.shader_info_complete = false;
|
||||
IR::CollectShaderInfo(program, options.input_info);
|
||||
const auto spirv = Spirv::EmitProgram(program, options.input_info);
|
||||
const auto spirv = Spirv::EmitProgram(program, options.input_info, 0, {}, true);
|
||||
CheckSpirvBinaryValidates(spirv);
|
||||
Check(spirv == result.spirv,
|
||||
"planning-only loads or references emitted GPU operands/instructions");
|
||||
@@ -11100,6 +11106,7 @@ void TestMeshExportStorage() {
|
||||
options.stage = ShaderType::Mesh;
|
||||
options.input_info.vertex = &input;
|
||||
options.user_data = user_data;
|
||||
options.emit_debug_info = true; // The checks find push data by its OpName.
|
||||
struct Case {
|
||||
uint32_t subgroup_size, push_data_start;
|
||||
bool split;
|
||||
@@ -14352,7 +14359,8 @@ void TestNewShaderRecompilerNativeBindingPlan() {
|
||||
const auto bindings = result.program.bindings;
|
||||
result.program.bindings = {};
|
||||
result.program.info.buffers[0].descriptor_index = UINT32_MAX;
|
||||
Check(ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info) == result.spirv &&
|
||||
Check(ShaderRecompiler::Spirv::EmitProgram(result.program, options.input_info, 0, {}, true) ==
|
||||
result.spirv &&
|
||||
result.program.bindings == bindings &&
|
||||
result.program.info.buffers[0].descriptor_index == 0,
|
||||
"repeated emission did not rebuild descriptor slots deterministically");
|
||||
@@ -16423,11 +16431,15 @@ void TestRepeatedExportsHaveOneInterface() {
|
||||
void CheckSpirvBlockSourceMapping(TestCompileResult &result) {
|
||||
using namespace ShaderRecompiler;
|
||||
const auto options = MakeCompileOptions(ShaderType::Compute);
|
||||
Check(SpirvInstructionOpcodeCount(result.spirv, spv::OpString) == 0 &&
|
||||
SpirvInstructionOpcodeCount(result.spirv, spv::OpLine) == 0,
|
||||
"default emission unexpectedly contains source mapping");
|
||||
const auto plain = Spirv::EmitProgram(result.program, options.input_info);
|
||||
Check(SpirvInstructionOpcodeCount(plain, spv::OpString) == 0 &&
|
||||
SpirvInstructionOpcodeCount(plain, spv::OpLine) == 0 &&
|
||||
SpirvInstructionOpcodeCount(plain, spv::OpName) == 0,
|
||||
"default emission unexpectedly contains debug info");
|
||||
const auto mapped =
|
||||
Spirv::EmitProgram(result.program, options.input_info, 0, {}, true);
|
||||
Check(SpirvInstructionOpcodeCount(mapped, spv::OpName) != 0,
|
||||
"debug emission omitted names");
|
||||
CheckSpirvBinaryValidates(mapped);
|
||||
uint32_t source = 0;
|
||||
uint32_t label = 0;
|
||||
@@ -16456,7 +16468,7 @@ void CheckSpirvBlockSourceMapping(TestCompileResult &result) {
|
||||
}
|
||||
Check(source != 0 && lines >= result.program.blocks.size(),
|
||||
"debug emission omitted native block source mapping");
|
||||
Check(Spirv::EmitProgram(result.program, options.input_info) == result.spirv,
|
||||
Check(Spirv::EmitProgram(result.program, options.input_info) == plain,
|
||||
"debug re-emission changed the default binary");
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user