mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-28 08:13:05 +00:00
Mac
491dd71b2a
The caches decided when to collect from the driver's process-wide usage estimate, which varied by about 4x between reads of the same moment on this setup and is not what the allocator is holding. Read VMA's own accounting instead, and let the collector stand down on the same figure it stood up on rather than a stale tally. Adds a small deferral guard so a completion callback cannot re-enter the resource caches while the caller still holds their state.
381 lines
15 KiB
C++
381 lines
15 KiB
C++
#include <cinttypes>
|
|
#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
|
|
|
|
#include "common/assert.h"
|
|
#include "common/logging/log.h"
|
|
#include "common/profiler.h"
|
|
#include "graphics/host_gpu/graphicContext.h"
|
|
#include "graphics/host_gpu/renderer/commandScheduler.h"
|
|
#include "graphics/host_gpu/vma.h"
|
|
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <limits>
|
|
#include <numeric>
|
|
#include <vk_mem_alloc.h>
|
|
|
|
namespace Libs::Graphics {
|
|
|
|
namespace {
|
|
|
|
constexpr size_t WATCHES_INITIAL_RESERVE = 0x4000;
|
|
constexpr size_t WATCHES_RESERVE_CHUNK = 0x1000;
|
|
|
|
[[nodiscard]] VmaAllocationCreateFlags AllocationFlags(MemoryUsage usage) {
|
|
switch (usage) {
|
|
case MemoryUsage::Upload:
|
|
case MemoryUsage::Stream:
|
|
return VMA_ALLOCATION_CREATE_MAPPED_BIT |
|
|
VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT;
|
|
case MemoryUsage::Download:
|
|
return VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT;
|
|
case MemoryUsage::DeviceLocal: return {};
|
|
}
|
|
return {};
|
|
}
|
|
|
|
[[nodiscard]] VmaMemoryUsage AllocationUsage(MemoryUsage usage) {
|
|
switch (usage) {
|
|
case MemoryUsage::DeviceLocal:
|
|
case MemoryUsage::Stream: return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
|
|
case MemoryUsage::Upload:
|
|
case MemoryUsage::Download: return VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
|
|
}
|
|
return VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
|
|
}
|
|
|
|
[[nodiscard]] bool AlignUp(uint64_t value, uint64_t alignment, uint64_t& result) {
|
|
if (alignment == 0) {
|
|
result = value;
|
|
return true;
|
|
}
|
|
const auto remainder = value % alignment;
|
|
if (remainder == 0) {
|
|
result = value;
|
|
return true;
|
|
}
|
|
const auto increment = alignment - remainder;
|
|
if (value > std::numeric_limits<uint64_t>::max() - increment) {
|
|
return false;
|
|
}
|
|
result = value + increment;
|
|
return true;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
|
|
uint64_t cpu_address, vk::BufferUsageFlags flags, uint64_t size)
|
|
: m_graphics(&graphics), m_scheduler(&scheduler), m_usage(usage), m_cpu_address(cpu_address),
|
|
m_size(size), m_buffer(std::make_unique<VulkanBuffer>()) {
|
|
KYTY_PROFILER_FUNCTION();
|
|
EXIT_IF(graphics.allocator == nullptr || size == 0);
|
|
|
|
vk::BufferCreateInfo buffer_info {};
|
|
buffer_info.size = size;
|
|
buffer_info.usage = flags;
|
|
buffer_info.sharingMode = vk::SharingMode::eExclusive;
|
|
|
|
VmaAllocationCreateInfo allocation_info {};
|
|
allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
|
|
allocation_info.usage = AllocationUsage(usage);
|
|
allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
|
|
? VkMemoryPropertyFlags {}
|
|
: VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
|
|
|
|
VmaAllocationInfo allocation_result {};
|
|
VkBuffer native_buffer = VK_NULL_HANDLE;
|
|
const auto result = static_cast<vk::Result>(vmaCreateBuffer(
|
|
graphics.allocator, static_cast<const VkBufferCreateInfo*>(buffer_info), &allocation_info,
|
|
&native_buffer, &m_buffer->memory.allocation, &allocation_result));
|
|
if (result != vk::Result::eSuccess) {
|
|
graphics.LogMemoryBudget();
|
|
}
|
|
EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
|
|
|
|
m_buffer->buffer = native_buffer;
|
|
m_buffer->usage = flags;
|
|
m_buffer->buffer_size = size;
|
|
m_buffer->memory.allocation_info = allocation_result;
|
|
m_buffer->memory.memory = allocation_result.deviceMemory;
|
|
m_buffer->memory.offset = allocation_result.offset;
|
|
m_buffer->memory.type = allocation_result.memoryType;
|
|
m_buffer->memory.unique_id = VulkanNextMemoryUniqueId();
|
|
graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
|
|
if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
|
|
vk::BufferDeviceAddressInfo address_info {};
|
|
address_info.sType = vk::StructureType::eBufferDeviceAddressInfo;
|
|
address_info.buffer = m_buffer->buffer;
|
|
m_device_address = graphics.device.getBufferAddress(address_info);
|
|
EXIT_IF(m_device_address == 0);
|
|
}
|
|
|
|
VkMemoryPropertyFlags properties = 0;
|
|
vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
|
|
m_buffer->memory.property = vk::MemoryPropertyFlags(properties);
|
|
m_is_coherent = (properties & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
|
|
if (allocation_result.pMappedData != nullptr) {
|
|
m_mapped = {static_cast<uint8_t*>(allocation_result.pMappedData),
|
|
static_cast<size_t>(size)};
|
|
}
|
|
VulkanTrackAllocation(m_buffer->memory);
|
|
}
|
|
|
|
Buffer::~Buffer() {
|
|
if (m_buffer->buffer != nullptr) {
|
|
VulkanUntrackAllocation(m_buffer->memory);
|
|
vmaDestroyBuffer(m_graphics->allocator, m_buffer->buffer, m_buffer->memory.allocation);
|
|
}
|
|
}
|
|
|
|
vk::Buffer Buffer::Handle() const noexcept {
|
|
return m_buffer->buffer;
|
|
}
|
|
|
|
vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
|
|
EXIT_IF(m_device_address == 0);
|
|
return m_device_address;
|
|
}
|
|
|
|
bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
|
|
return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
|
|
}
|
|
|
|
void Buffer::Write(uint64_t offset, const void* source, uint64_t size) {
|
|
EXIT_IF(source == nullptr || m_mapped.empty() || offset > m_size || size > m_size - offset);
|
|
std::memcpy(m_mapped.data() + offset, source, static_cast<size_t>(size));
|
|
Flush(offset, size);
|
|
}
|
|
|
|
void Buffer::Flush(uint64_t offset, uint64_t size) {
|
|
EXIT_IF(m_mapped.empty() || offset > m_size || size > m_size - offset);
|
|
if (!m_is_coherent && size != 0) {
|
|
const auto result =
|
|
vmaFlushAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
|
|
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
|
|
}
|
|
}
|
|
|
|
void Buffer::Invalidate(uint64_t offset, uint64_t size) {
|
|
EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
|
|
if (!m_is_coherent && size != 0) {
|
|
const auto result =
|
|
vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
|
|
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
|
|
}
|
|
}
|
|
|
|
vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
|
|
vk::AccessFlags destination) const {
|
|
if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
|
|
EXIT("Buffer: invalid DMA barrier, handle=%p offset=0x%016" PRIx64 " size=0x%016" PRIx64
|
|
" capacity=0x%016" PRIx64 "\n",
|
|
static_cast<const void*>(Handle()), offset, size, m_size);
|
|
}
|
|
vk::BufferMemoryBarrier barrier {};
|
|
barrier.sType = vk::StructureType::eBufferMemoryBarrier;
|
|
barrier.srcAccessMask = source;
|
|
barrier.dstAccessMask = destination;
|
|
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
barrier.buffer = Handle();
|
|
barrier.offset = offset;
|
|
barrier.size = size;
|
|
return barrier;
|
|
}
|
|
|
|
void Buffer::CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
|
|
uint64_t destination_offset, uint64_t size, vk::AccessFlags source_before,
|
|
vk::AccessFlags destination_before, vk::AccessFlags source_after,
|
|
vk::AccessFlags destination_after, std::source_location where) {
|
|
if (size == 0 || source_offset > source.m_size || size > source.m_size - source_offset ||
|
|
destination_offset > m_size || size > m_size - destination_offset) {
|
|
EXIT("Buffer: invalid copy range from %s:%u: src_size=0x%" PRIx64 " src_off=0x%" PRIx64
|
|
" dst_size=0x%" PRIx64 " dst_off=0x%" PRIx64 " size=0x%" PRIx64 "\n",
|
|
where.file_name(), where.line(), source.m_size, source_offset, m_size,
|
|
destination_offset, size);
|
|
}
|
|
if (source.Handle() == Handle() && source_offset < destination_offset + size &&
|
|
destination_offset < source_offset + size) {
|
|
EXIT("Buffer: overlapping self-copy\n");
|
|
}
|
|
command.EndRendering();
|
|
const vk::BufferMemoryBarrier before[] = {
|
|
source.Barrier(source_offset, size, source_before, vk::AccessFlagBits::eTransferRead),
|
|
Barrier(destination_offset, size, destination_before, vk::AccessFlagBits::eTransferWrite),
|
|
};
|
|
const auto host_access = vk::AccessFlagBits::eHostRead | vk::AccessFlagBits::eHostWrite;
|
|
auto before_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands};
|
|
if (static_cast<bool>((source_before | destination_before) & host_access)) {
|
|
before_stage |= vk::PipelineStageFlagBits::eHost;
|
|
}
|
|
const auto native = command.Handle();
|
|
native.pipelineBarrier(before_stage, vk::PipelineStageFlagBits::eTransfer,
|
|
vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, before, 0, nullptr);
|
|
const vk::BufferCopy copy {source_offset, destination_offset, size};
|
|
native.copyBuffer(source.Handle(), Handle(), 1, ©);
|
|
const vk::BufferMemoryBarrier after[] = {
|
|
source.Barrier(source_offset, size, vk::AccessFlagBits::eTransferRead, source_after),
|
|
Barrier(destination_offset, size, vk::AccessFlagBits::eTransferWrite, destination_after),
|
|
};
|
|
auto after_stage = vk::PipelineStageFlags {vk::PipelineStageFlagBits::eAllCommands};
|
|
if (static_cast<bool>((source_after | destination_after) & host_access)) {
|
|
after_stage |= vk::PipelineStageFlagBits::eHost;
|
|
}
|
|
native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, after_stage,
|
|
vk::DependencyFlagBits::eByRegion, 0, nullptr, 2, after, 0, nullptr);
|
|
}
|
|
|
|
void Buffer::Fill(uint64_t offset, uint64_t size, uint32_t value) {
|
|
if (((offset | size) & 3u) != 0) {
|
|
EXIT("Buffer: fill range must be dword aligned\n");
|
|
}
|
|
if (offset > m_size || size > m_size - offset) {
|
|
EXIT("Buffer: fill range outside the buffer, offset=0x%" PRIx64 " size=0x%" PRIx64
|
|
" capacity=0x%" PRIx64 "\n",
|
|
offset, size, m_size);
|
|
}
|
|
auto& command = Scheduler().Current();
|
|
command.EndRendering();
|
|
const auto before =
|
|
Barrier(offset, size, vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite,
|
|
vk::AccessFlagBits::eTransferWrite);
|
|
const auto native = command.Handle();
|
|
native.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands,
|
|
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlagBits::eByRegion,
|
|
0, nullptr, 1, &before, 0, nullptr);
|
|
native.fillBuffer(Handle(), offset, size, value);
|
|
const auto after = Barrier(offset, size, vk::AccessFlagBits::eTransferWrite,
|
|
vk::AccessFlagBits::eMemoryRead | vk::AccessFlagBits::eMemoryWrite);
|
|
native.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
|
|
vk::PipelineStageFlagBits::eAllCommands,
|
|
vk::DependencyFlagBits::eByRegion, 0, nullptr, 1, &after, 0, nullptr);
|
|
}
|
|
|
|
StreamBuffer::StreamBuffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsage usage,
|
|
uint64_t size)
|
|
: Buffer(graphics, scheduler, usage, 0, AllFlags, size) {
|
|
ReserveWatches(m_current_watches, WATCHES_INITIAL_RESERVE);
|
|
ReserveWatches(m_previous_watches, WATCHES_INITIAL_RESERVE);
|
|
}
|
|
|
|
bool StreamBuffer::NormalizeReservation(bool coherent, uint64_t atom, uint64_t& size,
|
|
uint64_t& alignment) {
|
|
if (coherent) {
|
|
return true;
|
|
}
|
|
if (!AlignUp(size, atom, size)) {
|
|
return false;
|
|
}
|
|
const auto divisor = std::gcd(alignment, atom);
|
|
if (alignment != 0 && alignment / divisor > UINT64_MAX / atom) {
|
|
return false;
|
|
}
|
|
alignment = alignment == 0 ? atom : alignment / divisor * atom;
|
|
return true;
|
|
}
|
|
|
|
std::pair<uint8_t*, uint64_t> StreamBuffer::Map(uint64_t size, uint64_t alignment,
|
|
bool allow_wait) {
|
|
if (Mapped().empty()) {
|
|
return {nullptr, 0};
|
|
}
|
|
uint64_t mapped_size = size;
|
|
const auto atom = Graphics().physical_device_properties.limits.nonCoherentAtomSize;
|
|
if (!NormalizeReservation(IsCoherent(), atom, mapped_size, alignment)) {
|
|
return {nullptr, 0};
|
|
}
|
|
if (mapped_size > Size()) {
|
|
return {nullptr, 0};
|
|
}
|
|
|
|
uint64_t aligned_offset = 0;
|
|
if (!AlignUp(m_offset, alignment, aligned_offset)) {
|
|
return {nullptr, 0};
|
|
}
|
|
|
|
const bool wrap = aligned_offset > Size() - mapped_size;
|
|
if (wrap) {
|
|
aligned_offset = 0;
|
|
}
|
|
|
|
auto wait_cursor = wrap ? size_t {0} : m_wait_cursor;
|
|
auto wait_bound = wrap ? uint64_t {0} : m_wait_bound;
|
|
auto invalidation_mark =
|
|
wrap ? std::optional<size_t> {m_current_watch_cursor} : m_invalidation_mark;
|
|
auto& pending_watches = wrap ? m_current_watches : m_previous_watches;
|
|
if (!WaitPendingOperations(pending_watches, invalidation_mark, aligned_offset + mapped_size,
|
|
allow_wait, wait_cursor, wait_bound)) {
|
|
return {nullptr, 0};
|
|
}
|
|
|
|
if (wrap) {
|
|
m_invalidation_mark = invalidation_mark;
|
|
m_current_watch_cursor = 0;
|
|
std::swap(m_previous_watches, m_current_watches);
|
|
}
|
|
m_wait_cursor = wait_cursor;
|
|
m_wait_bound = wait_bound;
|
|
m_offset = aligned_offset;
|
|
m_mapped_size = mapped_size;
|
|
return {Mapped().data() + m_offset, m_offset};
|
|
}
|
|
|
|
void StreamBuffer::Commit() {
|
|
if (!IsCoherent() && Usage() != MemoryUsage::Download && m_mapped_size != 0) {
|
|
const auto result = vmaFlushAllocation(
|
|
Graphics().allocator, NativeBuffer().memory.allocation, m_offset, m_mapped_size);
|
|
EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
|
|
}
|
|
|
|
m_offset += m_mapped_size;
|
|
const auto tick = Scheduler().CurrentTick();
|
|
if (m_current_watch_cursor != 0 && m_current_watches[m_current_watch_cursor - 1].tick == tick) {
|
|
m_current_watches[m_current_watch_cursor - 1].upper_bound = m_offset;
|
|
return;
|
|
}
|
|
if (m_current_watch_cursor + 1 >= m_current_watches.size()) {
|
|
ReserveWatches(m_current_watches, WATCHES_RESERVE_CHUNK);
|
|
}
|
|
auto& watch = m_current_watches[m_current_watch_cursor++];
|
|
watch.upper_bound = m_offset;
|
|
watch.tick = tick;
|
|
}
|
|
|
|
uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
|
|
EXIT_IF(source == nullptr);
|
|
const auto [data, offset] = Map(size, alignment);
|
|
EXIT_IF(data == nullptr);
|
|
std::memcpy(data, source, static_cast<size_t>(size));
|
|
Commit();
|
|
return offset;
|
|
}
|
|
|
|
void StreamBuffer::ReserveWatches(std::vector<Watch>& watches, size_t grow_size) {
|
|
watches.resize(watches.size() + grow_size);
|
|
}
|
|
|
|
bool StreamBuffer::WaitPendingOperations(const std::vector<Watch>& watches,
|
|
std::optional<size_t> invalidation_mark,
|
|
uint64_t requested_upper_bound, bool allow_wait,
|
|
size_t& wait_cursor, uint64_t& wait_bound) {
|
|
if (!invalidation_mark.has_value()) {
|
|
return true;
|
|
}
|
|
while (requested_upper_bound > wait_bound && wait_cursor < *invalidation_mark) {
|
|
const auto& watch = watches[wait_cursor];
|
|
if (!Scheduler().IsFree(watch.tick) && !allow_wait) {
|
|
return false;
|
|
}
|
|
Scheduler().Wait(watch.tick);
|
|
if (Usage() == MemoryUsage::Download) {
|
|
Scheduler().WaitPriorityOperations(watch.tick);
|
|
}
|
|
wait_bound = watch.upper_bound;
|
|
++wait_cursor;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
} // namespace Libs::Graphics
|