gremling/KytyPS5
1
0 Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-29 16:51:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

renderer: remove legacy code left from refactoring

Browse Source
This commit is contained in:
nmzik committed 2026-07-31 23:13:48 +02:00
1 parent d68a477276
commit f830d6b2e4
15 files changed
+339 -2084
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/graphics/host_gpu/memoryTracker.cpp
-90
View File
@@ -157,7 +157,6 @@ void MemoryTracker::UntrackMemoryLocked(uint64_t vaddr, uint64_t size) {
const auto changed = const auto changed =
manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes); manager->ChangeState<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, bytes);
manager->ApplyProtection(changed, false); manager->ApplyProtection(changed, false);
manager->Untrack(manager->GetCpuAddr() + offset, bytes);
}); });
locks.clear(); locks.clear();
} }
@@ -168,93 +167,4 @@ void MemoryTracker::UntrackMemory(uint64_t vaddr, uint64_t size) {
UntrackMemoryLocked(vaddr, size); UntrackMemoryLocked(vaddr, size);
} }
bool MemoryTracker::InvalidateRegion(uint64_t vaddr, uint64_t size, PageFaultPhase phase) noexcept {
switch (phase) {
case PageFaultPhase::Release: return true;
case PageFaultPhase::Invalidate: {
const auto action = BeginCpuFault(vaddr, size);
switch (action) {
case CpuFaultAction::Untracked: return false;
case CpuFaultAction::Continue: return true;
case CpuFaultAction::Download:
EXIT("generic region invalidation cannot download GPU-dirty memory\n");
}
}
case PageFaultPhase::Complete:
return CompleteCpuFault(vaddr, size, PageFaultAccess::Write, false);
}
EXIT("unsupported region invalidation phase\n");
}
bool MemoryTracker::InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
switch (phase) {
case PageFaultPhase::Release: return true;
case PageFaultPhase::Invalidate: {
const bool gpu_modified = Iterate<false>(
vaddr, size, [](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
return manager->IsModified<DirtySource::Gpu>(offset, bytes);
});
if (!gpu_modified) {
return false;
}
const auto action = BeginCpuFault(vaddr, size);
if (access != PageFaultAccess::Write || action != CpuFaultAction::Download) {
EXIT("virtual GPU write fault requires write access to GPU-dirty memory\n");
}
return true;
}
case PageFaultPhase::Complete: {
if (access != PageFaultAccess::Write) {
EXIT("virtual GPU write completion requires write access\n");
}
bool completed = false;
Iterate<false>(
vaddr, size, [&completed](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (completed) {
EXIT("virtual GPU write fault spans multiple tracked regions\n");
}
completed =
manager->CompleteVirtualGpuWrite(manager->GetCpuAddr() + offset, bytes);
});
return completed;
}
}
EXIT("unsupported virtual GPU write invalidation phase\n");
}
CpuFaultAction MemoryTracker::BeginCpuFault(uint64_t vaddr, uint64_t size,
PageFaultAccess access) noexcept {
CheckNotInUploadCallback();
CpuFaultAction action = CpuFaultAction::Untracked;
Iterate<false>(
vaddr, size, [&action, access](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (action != CpuFaultAction::Untracked) {
EXIT("CPU fault spans multiple tracked regions\n");
}
action = manager->BeginCpuFault(manager->GetCpuAddr() + offset, bytes, access);
});
return action;
}
bool MemoryTracker::CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
bool downloaded) noexcept {
CheckNotInUploadCallback();
bool found = false;
Iterate<false>(
vaddr, size,
[&found, access, downloaded](RegionManager* manager, uint64_t offset, uint64_t bytes) {
std::scoped_lock lock(manager->lock);
if (found) {
EXIT("CPU fault completion spans multiple tracked regions\n");
}
found = manager->CompleteCpuFault(manager->GetCpuAddr() + offset, bytes, access,
downloaded);
});
return found;
}
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/host_gpu/memoryTracker.h
-13
View File
@@ -30,13 +30,6 @@ public:
void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void MarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size); void UnmarkRegionAsGpuModified(uint64_t vaddr, uint64_t size);
void UntrackMemory(uint64_t vaddr, uint64_t size); void UntrackMemory(uint64_t vaddr, uint64_t size);
[[nodiscard]] CpuFaultAction
BeginCpuFault(uint64_t vaddr, uint64_t size,
PageFaultAccess access = PageFaultAccess::Write) noexcept;
[[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
bool downloaded) noexcept;
[[nodiscard]] bool InvalidateRegion(uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
template <typename Flush> template <typename Flush>
void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) { void InvalidateRegion(uint64_t vaddr, uint64_t size, Flush&& on_flush) {
static_assert(std::is_invocable_v<Flush&>); static_assert(std::is_invocable_v<Flush&>);
@@ -78,8 +71,6 @@ public:
EXIT("memory invalidation retained GPU-owned pages\n"); EXIT("memory invalidation retained GPU-owned pages\n");
} }
} }
[[nodiscard]] bool InvalidateVirtualGpuWrite(PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size, void ValidateGpuDirtyPages(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
const char* operation) const noexcept; const char* operation) const noexcept;
void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size, void ValidateGpuDirtyOwnership(const RangeSet& dirty, uint64_t vaddr, uint64_t size,
@@ -102,9 +93,6 @@ public:
} }
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
const auto address = manager->GetCpuAddr() + offset; const auto address = manager->GetCpuAddr() + offset;
if (manager->HasPendingFault(address, bytes)) {
EXIT("GPU download synchronization raced a pending CPU fault\n");
}
manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes, manager->template ForEachModifiedRange<DirtySource::Gpu, false>(address, bytes,
preflight); preflight);
}); });
@@ -141,7 +129,6 @@ public:
const auto* previous_upload_owner = std::exchange(s_upload_owner, this); const auto* previous_upload_owner = std::exchange(s_upload_owner, this);
Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) { Iterate<false>(vaddr, size, [&](RegionManager* manager, uint64_t offset, uint64_t bytes) {
manager->lock.lock(); manager->lock.lock();
manager->Track(manager->GetCpuAddr() + offset, bytes);
manager->ForEachModifiedRange<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset, manager->ForEachModifiedRange<DirtySource::Cpu, true>(manager->GetCpuAddr() + offset,
bytes, range_func); bytes, range_func);
if (!is_written) { if (!is_written) {
src/graphics/host_gpu/pageManager.cpp
+12 -200
View File
@@ -56,8 +56,6 @@ constexpr uint32_t READ_WRITE_PROTECTION = PAGE_READWRITE;
// Zero is the unknown protection sentinel. // Zero is the unknown protection sentinel.
constexpr uint32_t UNKNOWN_PROTECTION = 0; constexpr uint32_t UNKNOWN_PROTECTION = 0;
thread_local bool g_in_fault_resolution = false;
[[noreturn]] void FailFast(const char* reason = nullptr) noexcept { [[noreturn]] void FailFast(const char* reason = nullptr) noexcept {
std::fputs("PageManager fail-fast: ", stderr); std::fputs("PageManager fail-fast: ", stderr);
std::fputs(reason != nullptr ? reason : "invalid page state", stderr); std::fputs(reason != nullptr ? reason : "invalid page state", stderr);
@@ -164,9 +162,6 @@ struct PageManager::Impl {
// Shadow the protection applied through Protect(). // Shadow the protection applied through Protect().
uint32_t current_protection = UNKNOWN_PROTECTION; uint32_t current_protection = UNKNOWN_PROTECTION;
bool resolving = false; bool resolving = false;
bool resolving_read_write = false;
bool late_read_pending = false;
bool late_write_pending = false;
}; };
struct Region { struct Region {
@@ -193,10 +188,7 @@ struct PageManager::Impl {
std::span<PageState*> m_pages; std::span<PageState*> m_pages;
}; };
Impl(PageFaultHandler handler, void* context): fault_handler(handler), fault_context(context) { Impl() {
if (fault_handler == nullptr) {
Fatal("null page-manager fault callback");
}
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
SYSTEM_INFO info {}; SYSTEM_INFO info {};
GetSystemInfo(&info); GetSystemInfo(&info);
@@ -268,17 +260,6 @@ struct PageManager::Impl {
return page.original_protection; return page.original_protection;
} }
static void PublishDelayedFaults(PageState& page, uint32_t old_protection,
uint32_t new_protection) {
if (old_protection == NO_ACCESS_PROTECTION && new_protection != NO_ACCESS_PROTECTION) {
page.late_read_pending = true;
}
if ((old_protection == NO_ACCESS_PROTECTION || old_protection == READ_ONLY_PROTECTION) &&
new_protection == READ_WRITE_PROTECTION) {
page.late_write_pending = true;
}
}
static void InitializeProtection(std::span<PageState*> pages) { static void InitializeProtection(std::span<PageState*> pages) {
for (auto* page: pages) { for (auto* page: pages) {
page->original_protection = READ_WRITE_PROTECTION; page->original_protection = READ_WRITE_PROTECTION;
@@ -286,19 +267,8 @@ struct PageManager::Impl {
} }
} }
static bool AllowsAccess(const PageState& page, [[maybe_unused]] uint64_t vaddr,
PageFaultAccess access) noexcept {
switch (access) {
case PageFaultAccess::Read:
return page.current_protection == READ_ONLY_PROTECTION ||
page.current_protection == READ_WRITE_PROTECTION;
case PageFaultAccess::Write: return page.current_protection == READ_WRITE_PROTECTION;
default: return false;
}
}
void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection, void ProtectRange(std::span<PageState*> pages, uint64_t vaddr, uint32_t protection,
std::span<const uint32_t> expected_old, bool fault_path) noexcept { std::span<const uint32_t> expected_old) noexcept {
const auto size = pages.size() * PAGE_SIZE; const auto size = pages.size() * PAGE_SIZE;
if (pages.size() != expected_old.size()) { if (pages.size() != expected_old.size()) {
FailFast("protection range state size mismatch"); FailFast("protection range state size mismatch");
@@ -306,9 +276,6 @@ struct PageManager::Impl {
for (size_t i = 0; i < pages.size(); i++) { for (size_t i = 0; i < pages.size(); i++) {
const auto actual = pages[i]->current_protection; const auto actual = pages[i]->current_protection;
if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) { if (actual != UNKNOWN_PROTECTION && actual != expected_old[i]) {
if (fault_path) {
FailFast("mprotect fault transition did not match expected protection");
}
Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32 Fatal("invalid protection transition at 0x%016" PRIx64 ", old=0x%08" PRIx32
", expected=0x%08" PRIx32 ", new=0x%08" PRIx32, ", expected=0x%08" PRIx32 ", new=0x%08" PRIx32,
vaddr + i * PAGE_SIZE, actual, expected_old[i], protection); vaddr + i * PAGE_SIZE, actual, expected_old[i], protection);
@@ -316,9 +283,6 @@ struct PageManager::Impl {
} }
if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size, if (!Libs::LibKernel::Memory::ProtectGuestHostMemory(vaddr, size,
ToMemoryMode(protection))) { ToMemoryMode(protection))) {
if (fault_path) {
FailFast("address-space fault protection transition failed");
}
Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr, Fatal("address-space protection failed at 0x%016" PRIx64 ", new=0x%08" PRIx32, vaddr,
protection); protection);
} }
@@ -327,47 +291,28 @@ struct PageManager::Impl {
} }
} }
void Protect(PageState& page, uint64_t vaddr, uint32_t protection, uint32_t expected_old, void Protect(PageState& page, uint64_t vaddr, uint32_t protection,
bool fault_path) noexcept { uint32_t expected_old) noexcept {
PageState* pages[] = {&page}; PageState* pages[] = {&page};
uint32_t expected[] = {expected_old}; uint32_t expected[] = {expected_old};
ProtectRange(pages, vaddr, protection, expected, fault_path); ProtectRange(pages, vaddr, protection, expected);
} }
std::unique_ptr<std::atomic<Region*>[]> regions; std::unique_ptr<std::atomic<Region*>[]> regions;
std::vector<std::unique_ptr<Region>> region_storage; std::vector<std::unique_ptr<Region>> region_storage;
std::mutex region_mutex; std::mutex region_mutex;
PageFaultHandler fault_handler = nullptr;
void* fault_context = nullptr;
}; };
static_assert(std::atomic<void*>::is_always_lock_free); static_assert(std::atomic<void*>::is_always_lock_free);
PageManager::PageManager(PageFaultHandler fault_handler, void* fault_context) PageManager::PageManager(): m_impl(std::make_unique<Impl>()) {}
: m_impl(std::make_unique<Impl>(fault_handler, fault_context)) {}
PageManager::~PageManager() = default; PageManager::~PageManager() = default;
uint64_t PageManager::GetPageSize() const { uint64_t PageManager::GetPageSize() const {
if (g_in_fault_resolution) {
FailFast("nested page fault while resolving a watched page");
}
return PAGE_SIZE; return PAGE_SIZE;
} }
bool PageManager::IsTracked(uint64_t vaddr) const noexcept {
if (g_in_fault_resolution) {
FailFast("IsTracked called during fault resolution");
}
auto* region = m_impl->FindRegion(vaddr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, vaddr);
SpinGuard lock(page.lock);
return page.write_watchers != 0 || page.access_watchers != 0;
}
void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode) { PageWatchMode mode) {
if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) { if (mode != PageWatchMode::Write && mode != PageWatchMode::ReadWrite) {
@@ -473,27 +418,14 @@ void PageManager::UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
} }
m_impl->ProtectRange(std::span {pages}.subspan(first, last - first), m_impl->ProtectRange(std::span {pages}.subspan(first, last - first),
chunk_begin + first * PAGE_SIZE, protection, chunk_begin + first * PAGE_SIZE, protection,
std::span {old_protections}.subspan(first, last - first), false); std::span {old_protections}.subspan(first, last - first));
first = current; first = current;
} }
for (size_t i = 0; i < page_count; i++) { for (auto* page: pages) {
auto& page = *pages[i]; if (!track && page->backing_writer == 0 && page->write_watchers == 0 &&
const auto protection = new_protections[i]; page->access_watchers == 0) {
if (track) { page->original_protection = 0;
switch (protection) {
case NO_ACCESS_PROTECTION:
page.late_read_pending = false;
page.late_write_pending = false;
break;
case READ_ONLY_PROTECTION: page.late_write_pending = false; break;
default: break;
}
} else if (page.backing_writer == 0) {
Impl::PublishDelayedFaults(page, old_protections[i], protection);
if (page.write_watchers == 0 && page.access_watchers == 0) {
page.original_protection = 0;
}
} }
} }
chunk_begin = chunk_end; chunk_begin = chunk_end;
@@ -546,9 +478,6 @@ PageManager::ReserveBackingWrites(std::span<const RangeSet::Range> ranges) {
} }
void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept { void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
if (g_in_fault_resolution) {
FailFast("backing write began during fault resolution");
}
const auto end = PageEnd(vaddr, size); const auto end = PageEnd(vaddr, size);
const auto writer = CurrentThread(); const auto writer = CurrentThread();
for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
@@ -562,15 +491,11 @@ void PageManager::BeginBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
Fatal("backing write races page resolution at 0x%016" PRIx64, address); Fatal("backing write races page resolution at 0x%016" PRIx64, address);
} }
page.resolving = true; page.resolving = true;
page.resolving_read_write = true;
page.backing_writer = writer; page.backing_writer = writer;
} }
} }
void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept { void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
if (g_in_fault_resolution) {
FailFast("backing write ended during fault resolution");
}
const auto end = PageEnd(vaddr, size); const auto end = PageEnd(vaddr, size);
const auto writer = CurrentThread(); const auto writer = CurrentThread();
for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) { for (auto address = PageStart(vaddr); address < end; address += PAGE_SIZE) {
@@ -586,127 +511,14 @@ void PageManager::EndBackingWrite(uint64_t vaddr, uint64_t size) noexcept {
const auto old_protection = NO_ACCESS_PROTECTION; const auto old_protection = NO_ACCESS_PROTECTION;
const auto new_protection = Impl::WatcherProtection(page); const auto new_protection = Impl::WatcherProtection(page);
if (new_protection != old_protection) { if (new_protection != old_protection) {
m_impl->Protect(page, address, new_protection, old_protection, false); m_impl->Protect(page, address, new_protection, old_protection);
} }
Impl::PublishDelayedFaults(page, old_protection, new_protection);
if (page.write_watchers == 0 && page.access_watchers == 0) { if (page.write_watchers == 0 && page.access_watchers == 0) {
page.original_protection = 0; page.original_protection = 0;
} }
page.backing_writer = 0; page.backing_writer = 0;
page.resolving = false; page.resolving = false;
page.resolving_read_write = false;
} }
} }
bool PageManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
if (g_in_fault_resolution) {
FailFast("nested HandleFault call");
}
auto* region = m_impl->FindRegion(fault_vaddr);
if (region == nullptr) {
return false;
}
auto& page = m_impl->GetPage(*region, fault_vaddr);
bool waited = false;
while (true) {
SpinGuard lock(page.lock);
if (access == PageFaultAccess::Read && page.late_read_pending &&
Impl::AllowsAccess(page, fault_vaddr, access)) {
page.late_read_pending = false;
return true;
}
if (access == PageFaultAccess::Write && page.late_write_pending &&
Impl::AllowsAccess(page, fault_vaddr, access)) {
page.late_write_pending = false;
return true;
}
if (page.resolving) {
if (page.backing_writer == CurrentThread()) {
FailFast("backing writer faulted on its own reserved page");
}
if ((!page.resolving_read_write && access != PageFaultAccess::Write) ||
(page.resolving_read_write && access != PageFaultAccess::Read &&
access != PageFaultAccess::Write)) {
FailFast("fault access is incompatible with the active resolver");
}
waited = true;
continue;
}
if (page.write_watchers == 0 && page.access_watchers == 0) {
if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) {
return false;
}
bool& pending = (access == PageFaultAccess::Read ? page.late_read_pending
: page.late_write_pending);
const bool allowed = Impl::AllowsAccess(page, fault_vaddr, access);
pending = false;
if (waited && !allowed) {
FailFast("page remained inaccessible after waiting for its resolver");
}
// More than one CPU can fault before a protection transition becomes visible. The first
// delayed fault consumes the hint bit; later faults must also resume once the mapped
// page already permits the requested access. A genuinely read-only/no-access page still
// falls through to the guest exception path.
return allowed;
}
if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) ||
(access == PageFaultAccess::Read && page.access_watchers == 0)) {
FailFast("fault access is incompatible with active page watchers");
}
page.resolving = true;
page.resolving_read_write = page.access_watchers != 0;
break;
}
g_in_fault_resolution = true;
const bool handled = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
PageFaultPhase::Invalidate);
g_in_fault_resolution = false;
{
SpinGuard lock(page.lock);
if (!handled || !page.resolving) {
FailFast("fault invalidation did not preserve the resolving state");
}
}
g_in_fault_resolution = true;
const bool completed = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
PageFaultPhase::Complete);
g_in_fault_resolution = false;
{
SpinGuard lock(page.lock);
if (!completed || !page.resolving) {
FailFast("fault completion did not preserve the resolving state");
}
if (page.write_watchers != 0 || page.access_watchers != 0) {
const auto old_protection = Impl::WatcherProtection(page);
const bool read_only_fault = access == PageFaultAccess::Read;
if (read_only_fault && page.access_watchers == 0) {
FailFast("read fault completed without a read/write watcher");
}
page.access_watchers = 0;
if (!read_only_fault) {
page.write_watchers = 0;
}
const auto restored_protection = Impl::WatcherProtection(page);
m_impl->Protect(page, PageStart(fault_vaddr), restored_protection, old_protection,
true);
if (page.write_watchers == 0) {
page.original_protection = 0;
}
Impl::PublishDelayedFaults(page, old_protection, restored_protection);
} else if (!Impl::AllowsAccess(page, fault_vaddr, access)) {
FailFast("fault completion left the page inaccessible");
}
page.resolving = false;
page.resolving_read_write = false;
}
g_in_fault_resolution = true;
const bool released = m_impl->fault_handler(m_impl->fault_context, access, fault_vaddr, 1,
PageFaultPhase::Release);
g_in_fault_resolution = false;
if (!released) {
FailFast("fault release callback failed");
}
return true;
}
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/host_gpu/pageManager.h
+1 -6
View File
@@ -11,11 +11,8 @@
namespace Libs::Graphics { namespace Libs::Graphics {
enum class PageFaultAccess { Read, Write, Execute, Unknown }; enum class PageFaultAccess { Read, Write, Execute, Unknown };
enum class PageFaultPhase { Invalidate, Complete, Release };
enum class PageWatchMode { Write, ReadWrite }; enum class PageWatchMode { Write, ReadWrite };
using PageFaultHandler = bool (*)(void* context, PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept;
class PageManager final { class PageManager final {
public: public:
class BackingWrite final { class BackingWrite final {
@@ -30,21 +27,19 @@ public:
uint64_t m_size = 0; uint64_t m_size = 0;
}; };
PageManager(PageFaultHandler fault_handler, void* fault_context); PageManager();
// The owner must stop all PageManager callers before destruction. // The owner must stop all PageManager callers before destruction.
~PageManager(); ~PageManager();
KYTY_CLASS_NO_COPY(PageManager); KYTY_CLASS_NO_COPY(PageManager);
[[nodiscard]] uint64_t GetPageSize() const; [[nodiscard]] uint64_t GetPageSize() const;
[[nodiscard]] bool IsTracked(uint64_t vaddr) const noexcept;
void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size, void UpdatePageWatchers(bool track, uint64_t vaddr, uint64_t size,
PageWatchMode mode = PageWatchMode::Write); PageWatchMode mode = PageWatchMode::Write);
void OnGpuMap(uint64_t vaddr, uint64_t size); void OnGpuMap(uint64_t vaddr, uint64_t size);
void OnGpuUnmap(uint64_t vaddr, uint64_t size); void OnGpuUnmap(uint64_t vaddr, uint64_t size);
[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
[[nodiscard]] std::vector<std::unique_ptr<BackingWrite>> [[nodiscard]] std::vector<std::unique_ptr<BackingWrite>>
ReserveBackingWrites(std::span<const RangeSet::Range> ranges); ReserveBackingWrites(std::span<const RangeSet::Range> ranges);
src/graphics/host_gpu/regionManager.h
+4 -120
View File
@@ -25,8 +25,6 @@
namespace Libs::Graphics { namespace Libs::Graphics {
enum class CpuFaultAction { Untracked, Continue, Download };
class TrackingSpinLock final { class TrackingSpinLock final {
public: public:
void lock() noexcept { void lock() noexcept {
@@ -85,18 +83,6 @@ public:
KYTY_CLASS_NO_COPY(RegionManager); KYTY_CLASS_NO_COPY(RegionManager);
[[nodiscard]] uint64_t GetCpuAddr() const { return m_cpu_addr; } [[nodiscard]] uint64_t GetCpuAddr() const { return m_cpu_addr; }
void Track(uint64_t vaddr, uint64_t size) {
const auto [start, end] = GetPageRange(vaddr, size);
for (auto page = start; page < end; page++) {
m_tracked.set(page);
}
}
void Untrack(uint64_t vaddr, uint64_t size) {
const auto [start, end] = GetPageRange(vaddr, size);
for (auto page = start; page < end; page++) {
m_tracked.reset(page);
}
}
template <DirtySource source> template <DirtySource source>
[[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const { [[nodiscard]] bool IsModified(uint64_t offset, uint64_t size) const {
const auto [start, end] = GetPageRange(m_cpu_addr + offset, size); const auto [start, end] = GetPageRange(m_cpu_addr + offset, size);
@@ -126,15 +112,15 @@ public:
const auto [start, end] = GetPageRange(vaddr, size); const auto [start, end] = GetPageRange(vaddr, size);
if constexpr (source == DirtySource::Cpu && enable) { if constexpr (source == DirtySource::Cpu && enable) {
for (auto page = start; page < end; page++) { for (auto page = start; page < end; page++) {
if (m_gpu_dirty.test(page) || m_fault_pending.test(page)) { if (m_gpu_dirty.test(page)) {
EXIT("CPU dirty state conflicts with GPU dirty or pending fault state\n"); EXIT("CPU dirty state conflicts with GPU dirty state\n");
} }
} }
} }
if constexpr (source == DirtySource::Gpu && enable) { if constexpr (source == DirtySource::Gpu && enable) {
for (auto page = start; page < end; page++) { for (auto page = start; page < end; page++) {
if (m_cpu_dirty.test(page) || m_fault_pending.test(page)) { if (m_cpu_dirty.test(page)) {
EXIT("GPU dirty state conflicts with CPU dirty or pending fault state\n"); EXIT("GPU dirty state conflicts with CPU dirty state\n");
} }
} }
} }
@@ -151,110 +137,10 @@ public:
return changed; return changed;
} }
[[nodiscard]] CpuFaultAction BeginCpuFault(uint64_t vaddr, uint64_t size,
PageFaultAccess access = PageFaultAccess::Write) {
if (access != PageFaultAccess::Read && access != PageFaultAccess::Write) {
EXIT("unsupported CPU fault access while beginning ownership transfer\n");
}
const auto [start, end] = GetPageRange(vaddr, size);
const bool tracked = m_tracked.test(start);
for (auto page = start; page < end; page++) {
if (m_tracked.test(page) != tracked) {
EXIT("CPU fault spans mixed tracked and untracked pages\n");
}
if (m_fault_pending.test(page)) {
return CpuFaultAction::Untracked;
}
if (m_cpu_dirty.test(page) != m_writable.test(page) ||
(m_gpu_dirty.test(page) && (m_cpu_dirty.test(page) || m_writable.test(page)))) {
EXIT("inconsistent CPU fault page state\n");
}
}
if (!tracked) {
return CpuFaultAction::Untracked;
}
bool gpu_dirty = m_gpu_dirty.test(start);
bool writable = m_writable.test(start);
for (auto page = start + 1; page < end; page++) {
if (m_gpu_dirty.test(page) != gpu_dirty || m_writable.test(page) != writable) {
EXIT("CPU fault spans pages with incompatible dirty or writable state\n");
}
}
for (auto page = start; page < end; page++) {
if (!gpu_dirty && access == PageFaultAccess::Write) {
m_cpu_dirty.set(page);
m_writable.set(page);
}
m_fault_pending.set(page);
}
return gpu_dirty ? CpuFaultAction::Download : CpuFaultAction::Continue;
}
[[nodiscard]] bool CompleteCpuFault(uint64_t vaddr, uint64_t size, PageFaultAccess access,
bool downloaded) {
const auto [start, end] = GetPageRange(vaddr, size);
for (auto page = start; page < end; page++) {
if (!m_fault_pending.test(page)) {
return false;
}
}
for (auto page = start; page < end; page++) {
const bool gpu_dirty = m_gpu_dirty.test(page);
if (gpu_dirty != downloaded) {
EXIT("CPU fault download result disagrees with GPU dirty state\n");
}
if (gpu_dirty) {
m_gpu_dirty.reset(page);
switch (access) {
case PageFaultAccess::Read: break;
case PageFaultAccess::Write:
m_cpu_dirty.set(page);
m_writable.set(page);
break;
default: EXIT("unsupported CPU fault access after GPU download\n");
}
}
m_fault_pending.reset(page);
}
return true;
}
[[nodiscard]] bool HasPendingFault(uint64_t vaddr, uint64_t size) const {
const auto [start, end] = GetPageRange(vaddr, size);
for (auto page = start; page < end; page++) {
if (m_fault_pending.test(page)) {
return true;
}
}
return false;
}
[[nodiscard]] bool CompleteVirtualGpuWrite(uint64_t vaddr, uint64_t size) {
const auto [start, end] = GetPageRange(vaddr, size);
for (auto page = start; page < end; page++) {
if (!m_fault_pending.test(page)) {
return false;
}
if (!m_gpu_dirty.test(page)) {
EXIT("virtual GPU write completion found a non-GPU-dirty page\n");
}
}
for (auto page = start; page < end; page++) {
m_gpu_dirty.reset(page);
m_cpu_dirty.set(page);
m_writable.set(page);
m_fault_pending.reset(page);
}
return true;
}
template <DirtySource source, bool clear, typename Func> template <DirtySource source, bool clear, typename Func>
RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) { RegionBits ForEachModifiedRange(uint64_t vaddr, uint64_t size, Func&& func) {
const auto [start, end] = GetPageRange(vaddr, size); const auto [start, end] = GetPageRange(vaddr, size);
auto mask = GetBits<source>(); auto mask = GetBits<source>();
if constexpr (source == DirtySource::Cpu) {
mask &= ~m_fault_pending;
}
for (auto page = 0u; page < start; page++) { for (auto page = 0u; page < start; page++) {
mask.reset(page); mask.reset(page);
} }
@@ -351,8 +237,6 @@ private:
RegionBits m_cpu_dirty; RegionBits m_cpu_dirty;
RegionBits m_gpu_dirty; RegionBits m_gpu_dirty;
RegionBits m_writable; RegionBits m_writable;
RegionBits m_fault_pending;
RegionBits m_tracked;
}; };
} // namespace Libs::Graphics } // namespace Libs::Graphics
src/graphics/host_gpu/renderer/cache/bufferCache.cpp
Vendored
+2 -110
View File
@@ -123,24 +123,6 @@ struct BufferCache::RetiredBuffer {
std::shared_ptr<Buffer> owner; std::shared_ptr<Buffer> owner;
}; };
struct BufferCache::FaultReadback {
PageFaultAccess access = PageFaultAccess::Unknown;
uint64_t vaddr = 0;
uint64_t size = 0;
std::vector<DownloadRange> ranges;
bool installed = false;
[[nodiscard]] bool Active() const noexcept { return !ranges.empty(); }
void Reset() {
access = PageFaultAccess::Unknown;
vaddr = 0;
size = 0;
installed = false;
ranges.clear();
}
};
struct BufferCache::PendingBackingPublication { struct BufferCache::PendingBackingPublication {
uint64_t address = 0; uint64_t address = 0;
uint64_t size = 0; uint64_t size = 0;
@@ -253,7 +235,7 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
ResourceMutex& resource_mutex) ResourceMutex& resource_mutex)
: m_graphics(graphics), m_scheduler(scheduler), : m_graphics(graphics), m_scheduler(scheduler),
m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize), m_gds_buffer(graphics, scheduler, MemoryUsage::Stream, 0, AllFlags, GdsBufferSize),
m_fault_readback(std::make_unique<FaultReadback>()), m_memory_tracker(page_manager), m_memory_tracker(page_manager),
m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB), m_staging_buffer(graphics, scheduler, MemoryUsage::Upload, 512 * MiB),
m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB), m_stream_buffer(graphics, scheduler, MemoryUsage::Stream, 64 * MiB),
m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB), m_download_buffer(graphics, scheduler, MemoryUsage::Download, 32 * MiB),
@@ -277,9 +259,6 @@ BufferCache::BufferCache(GraphicContext& graphics, CommandScheduler& scheduler,
} }
BufferCache::~BufferCache() { BufferCache::~BufferCache() {
if (m_fault_readback->Active()) {
EXIT("BufferCache: destroyed with an active fault readback\n");
}
if (!m_gpu_modified_ranges.Empty()) { if (!m_gpu_modified_ranges.Empty()) {
EXIT("BufferCache: destroyed with pending GPU-modified ranges\n"); EXIT("BufferCache: destroyed with pending GPU-modified ranges\n");
} }
@@ -447,93 +426,6 @@ void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size) {
} }
} }
bool BufferCache::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
const auto page = vaddr & ~(TRACKER_PAGE_SIZE - 1);
if (size == 0 || size > page + TRACKER_PAGE_SIZE - vaddr) {
EXIT("BufferCache: invalid page-fault range\n");
}
if (phase == PageFaultPhase::Complete) {
FaultSafeCacheLock lock(this, m_mutex);
auto& fault = *m_fault_readback;
if (!fault.Active()) {
return m_memory_tracker.CompleteCpuFault(vaddr, size, access, false);
}
if (fault.access != access || fault.vaddr != vaddr || fault.size != size ||
fault.installed) {
EXIT("BufferCache: mismatched fault readback completion\n");
}
PublishDownloads(fault.ranges);
if (!m_memory_tracker.CompleteCpuFault(vaddr, size, access, true)) {
EXIT("BufferCache: failed to complete downloaded CPU fault\n");
}
fault.installed = true;
return true;
}
if (phase == PageFaultPhase::Release) {
FaultSafeCacheLock lock(this, m_mutex);
auto& fault = *m_fault_readback;
if (fault.Active()) {
if (fault.access != access || fault.vaddr != vaddr || fault.size != size ||
!fault.installed) {
EXIT("BufferCache: mismatched fault readback release\n");
}
for (const auto& range: fault.ranges) {
m_gpu_modified_ranges.Subtract(range.address, range.size);
}
fault.Reset();
}
return true;
}
if (phase != PageFaultPhase::Invalidate) {
EXIT("BufferCache: unsupported page-fault phase\n");
}
const auto action = m_memory_tracker.BeginCpuFault(vaddr, size, access);
if (action != CpuFaultAction::Download) {
return action == CpuFaultAction::Continue;
}
auto& fault = *m_fault_readback;
std::vector<DownloadCopy> copies;
{
FaultSafeCacheLock lock(this, m_mutex);
if (fault.Active()) {
EXIT("BufferCache: nested fault readback\n");
}
fault.access = access;
fault.vaddr = vaddr;
fault.size = size;
m_gpu_modified_ranges.ForEachIntersection(
page, TRACKER_PAGE_SIZE, [&](RangeSet::Range range) {
auto owner = m_buffers.upper_bound(range.address);
if (owner == m_buffers.begin()) {
EXIT("BufferCache: fault readback has no buffer owner\n");
}
--owner;
auto& cached = *owner->second;
if (!cached.buffer->IsInBounds(range.address, range.size)) {
EXIT("BufferCache: fault readback is outside its buffer owner\n");
}
copies.push_back({cached.buffer, cached.buffer->Offset(range.address),
range.address, range.size});
});
if (copies.empty()) {
EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n");
}
}
fault.ranges = RecordDownloads(copies);
if (!fault.Active()) {
EXIT("BufferCache: GPU-dirty fault page has no dirty byte ranges\n");
}
m_scheduler.FinishCurrent();
return true;
}
void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) { void BufferCache::UnmapMemory(uint64_t vaddr, uint64_t size) {
if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) { if (vaddr == 0 || size == 0 || size > UINT64_MAX - vaddr) {
EXIT("BufferCache: invalid unmap range\n"); EXIT("BufferCache: invalid unmap range\n");
@@ -1201,7 +1093,7 @@ void BufferCache::RunGarbageCollector() {
if (m_graphics.CanReportMemoryUsage()) { if (m_graphics.CanReportMemoryUsage()) {
m_total_used_memory = m_graphics.GetDeviceMemoryUsage(); m_total_used_memory = m_graphics.GetDeviceMemoryUsage();
} }
if (m_total_used_memory < m_trigger_gc_memory || m_fault_readback->Active()) { if (m_total_used_memory < m_trigger_gc_memory) {
return; return;
} }
src/graphics/host_gpu/renderer/cache/bufferCache.h
Vendored
-4
View File
@@ -47,8 +47,6 @@ public:
~BufferCache(); ~BufferCache();
KYTY_CLASS_NO_COPY(BufferCache); KYTY_CLASS_NO_COPY(BufferCache);
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
void InvalidateMemory(uint64_t vaddr, uint64_t size); void InvalidateMemory(uint64_t vaddr, uint64_t size);
void ReadMemory(uint64_t vaddr, uint64_t size); void ReadMemory(uint64_t vaddr, uint64_t size);
void UnmapMemory(uint64_t vaddr, uint64_t size); void UnmapMemory(uint64_t vaddr, uint64_t size);
@@ -90,7 +88,6 @@ private:
struct DownloadCopy; struct DownloadCopy;
struct DownloadRange; struct DownloadRange;
struct RetiredBuffer; struct RetiredBuffer;
struct FaultReadback;
struct PendingBackingPublication; struct PendingBackingPublication;
static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64; static constexpr uint64_t DOWNLOAD_ALIGNMENT = 64;
[[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept; [[nodiscard]] static uint64_t AlignDown(uint64_t value) noexcept;
@@ -120,7 +117,6 @@ private:
Common::Mutex m_mutex; Common::Mutex m_mutex;
std::shared_ptr<Buffer> m_null_buffer; std::shared_ptr<Buffer> m_null_buffer;
std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers; std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
std::unique_ptr<FaultReadback> m_fault_readback;
RangeSet m_gpu_modified_ranges; RangeSet m_gpu_modified_ranges;
RangeSet m_image_invalidated_ranges; RangeSet m_image_invalidated_ranges;
std::mutex m_publication_mutex; std::mutex m_publication_mutex;
src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp
Vendored
+1 -23
View File
@@ -7,33 +7,11 @@
namespace Libs::Graphics { namespace Libs::Graphics {
GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler) GpuResourceManager::GpuResourceManager(GraphicContext& graphics, CommandScheduler& scheduler)
: m_page_manager(FaultThunk, this), : m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
m_buffer_cache(graphics, scheduler, m_page_manager, m_texture_cache, m_resource_mutex),
m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {} m_texture_cache(graphics, scheduler, m_page_manager, m_buffer_cache, m_resource_mutex) {}
GpuResourceManager::~GpuResourceManager() = default; GpuResourceManager::~GpuResourceManager() = default;
bool GpuResourceManager::FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr,
uint64_t size, PageFaultPhase phase) noexcept {
return static_cast<GpuResourceManager*>(context)->InvalidateMemory(access, vaddr, size, phase);
}
bool GpuResourceManager::InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
// Let the authoritative image materialize first. A clean overlapping buffer marks a write
// fault CPU-dirty when it begins ownership transfer; doing that before image preflight would
// make the image appear to race a real CPU write. Completion and release retain buffer-first
// ordering so its pending fault is gone before TextureCache publishes the downloaded backing.
if (phase == PageFaultPhase::Invalidate) {
const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
return buffer_handled || image_handled;
}
const bool buffer_handled = m_buffer_cache.InvalidateMemory(access, vaddr, size, phase);
const bool image_handled = m_texture_cache.InvalidateMemory(access, vaddr, size, phase);
return buffer_handled || image_handled;
}
bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept { bool GpuResourceManager::HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept {
constexpr uint64_t fault_size = 8; constexpr uint64_t fault_size = 8;
if (!IsMapped(fault_vaddr, fault_size)) { if (!IsMapped(fault_vaddr, fault_size)) {
src/graphics/host_gpu/renderer/cache/gpuResourceManager.h
Vendored
-5
View File
@@ -34,11 +34,6 @@ public:
void RunGarbageCollector(); void RunGarbageCollector();
private: private:
static bool FaultThunk(void* context, PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept;
PageManager m_page_manager; PageManager m_page_manager;
ResourceMutex m_resource_mutex; ResourceMutex m_resource_mutex;
BufferCache m_buffer_cache; BufferCache m_buffer_cache;
src/graphics/host_gpu/renderer/cache/textureCache.cpp
Vendored
-31
View File
@@ -1826,37 +1826,6 @@ bool TextureCache::TouchMeta(uint64_t address, uint32_t slice, bool is_clear) {
return true; return true;
} }
bool TextureCache::InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size,
PageFaultPhase phase) noexcept {
if ((access != PageFaultAccess::Read && access != PageFaultAccess::Write) ||
!GuestRange {address, size}.Valid()) {
return false;
}
if (access == PageFaultAccess::Read) {
return false;
}
if (phase == PageFaultPhase::Invalidate) {
CacheLock lock(*this, m_lock);
const bool tracked =
std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
const auto owner = ResolveOwner(id);
return owner != nullptr && !owner->depth_id && owner->IsTracked();
});
if (tracked) {
InvalidateCpuAliases(address, size);
}
return tracked;
}
if (phase != PageFaultPhase::Complete && phase != PageFaultPhase::Release) {
return false;
}
CacheLock lock(*this, m_lock);
return std::ranges::any_of(FindImagesInRegion(address, size, true), [&](ImageId id) {
const auto owner = ResolveOwner(id);
return owner != nullptr && !owner->depth_id;
});
}
void TextureCache::UnmapMemory(uint64_t address, uint64_t size) { void TextureCache::UnmapMemory(uint64_t address, uint64_t size) {
if (!GuestRange {address, size}.Valid()) { if (!GuestRange {address, size}.Valid()) {
EXIT("TextureCache: invalid unmap range\n"); EXIT("TextureCache: invalid unmap range\n");
src/graphics/host_gpu/renderer/cache/textureCache.h
Vendored
-2
View File
@@ -76,8 +76,6 @@ public:
[[nodiscard]] bool ClearMeta(uint64_t address); [[nodiscard]] bool ClearMeta(uint64_t address);
[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear); [[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
[[nodiscard]] bool InvalidateMemory(PageFaultAccess access, uint64_t address, uint64_t size,
PageFaultPhase phase) noexcept;
void UnmapMemory(uint64_t address, uint64_t size); void UnmapMemory(uint64_t address, uint64_t size);
void ProcessDownloadImages(); void ProcessDownloadImages();
void RunGarbageCollector(); void RunGarbageCollector();
src/kernel/memory.cpp
+16 -20
View File
@@ -18,6 +18,7 @@
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <map> #include <map>
#include <memory>
#include <mutex> #include <mutex>
#include <vector> #include <vector>
@@ -805,11 +806,11 @@ private:
Common::Mutex m_mutex; Common::Mutex m_mutex;
}; };
static PhysicalMemory* g_physical_memory = nullptr; static std::unique_ptr<PhysicalMemory> g_physical_memory;
static FlexibleMemory* g_flexible_memory = nullptr; static std::unique_ptr<FlexibleMemory> g_flexible_memory;
static PooledMemory* g_pooled_memory = nullptr; static std::unique_ptr<PooledMemory> g_pooled_memory;
static VirtualRanges* g_virtual_ranges = nullptr; static std::unique_ptr<VirtualRanges> g_virtual_ranges;
static GuestAddressSpace* g_guest_address_space = nullptr; static std::unique_ptr<GuestAddressSpace> g_guest_address_space;
static callback_func_t g_alloc_callback = nullptr; static callback_func_t g_alloc_callback = nullptr;
static callback_func_t g_free_callback = nullptr; static callback_func_t g_free_callback = nullptr;
static std::atomic<uint64_t> g_memory_pool_committed = 0; static std::atomic<uint64_t> g_memory_pool_committed = 0;
@@ -970,11 +971,11 @@ static bool ReplaceFixedRangeWithReserved(uint64_t start, uint64_t size);
KYTY_SUBSYSTEM_INIT(Memory) { KYTY_SUBSYSTEM_INIT(Memory) {
g_flexible_memory_size_frozen = true; g_flexible_memory_size_frozen = true;
VirtualMemory::Init(); VirtualMemory::Init();
g_guest_address_space = new GuestAddressSpace(PhysicalMemory::TotalSize()); g_guest_address_space = std::make_unique<GuestAddressSpace>(PhysicalMemory::TotalSize());
g_physical_memory = new PhysicalMemory; g_physical_memory = std::make_unique<PhysicalMemory>();
g_flexible_memory = new FlexibleMemory; g_flexible_memory = std::make_unique<FlexibleMemory>();
g_pooled_memory = new PooledMemory; g_pooled_memory = std::make_unique<PooledMemory>();
g_virtual_ranges = new VirtualRanges; g_virtual_ranges = std::make_unique<VirtualRanges>();
EXIT_IF(!g_guest_address_space->SelfTest()); EXIT_IF(!g_guest_address_space->SelfTest());
EXIT_IF(!SelfTestSub64SharedPlaceholderAlias()); EXIT_IF(!SelfTestSub64SharedPlaceholderAlias());
} }
@@ -982,16 +983,11 @@ KYTY_SUBSYSTEM_INIT(Memory) {
KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Memory) {} KYTY_SUBSYSTEM_UNEXPECTED_SHUTDOWN(Memory) {}
KYTY_SUBSYSTEM_DESTROY(Memory) { KYTY_SUBSYSTEM_DESTROY(Memory) {
delete g_pooled_memory; g_pooled_memory.reset();
g_pooled_memory = nullptr; g_flexible_memory.reset();
delete g_flexible_memory; g_physical_memory.reset();
g_flexible_memory = nullptr; g_virtual_ranges.reset();
delete g_physical_memory; g_guest_address_space.reset();
g_physical_memory = nullptr;
delete g_virtual_ranges;
g_virtual_ranges = nullptr;
delete g_guest_address_space;
g_guest_address_space = nullptr;
} }
struct AlignedPos { struct AlignedPos {
tests/MemoryTrackerTests.cpp
+174 -927
View File
File diff suppressed because it is too large. Load diff
tests/PageManagerTests.cpp
+127 -517
View File
@@ -1,14 +1,12 @@
#include "common/virtualMemory.h" #include "common/virtualMemory.h"
#include "graphics/host_gpu/pageManager.h" #include "graphics/host_gpu/pageManager.h"
#include <atomic>
#include <cstdint> #include <cstdint>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <memory> #include <memory>
#include <string> #include <string>
#include <thread>
#include <vector> #include <vector>
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -19,20 +17,18 @@
#undef min #undef min
#undef max #undef max
#else #else
#include <csignal>
#include <map> #include <map>
#include <sys/mman.h> #include <sys/mman.h>
#include <sys/wait.h> #include <sys/wait.h>
#include <ucontext.h>
#include <unistd.h> #include <unistd.h>
#endif #endif
namespace { namespace {
using Libs::Graphics::PageFaultAccess;
using Libs::Graphics::PageManager; using Libs::Graphics::PageManager;
using Libs::Graphics::PageWatchMode;
void Check(bool value, const char* text) { void Check(bool value, const char *text) {
if (!value) { if (!value) {
std::fprintf(stderr, "PageManagerTests: failed: %s\n", text); std::fprintf(stderr, "PageManagerTests: failed: %s\n", text);
std::abort(); std::abort();
@@ -40,7 +36,6 @@ void Check(bool value, const char* text) {
} }
#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
// POSIX shims for the shared test body.
using DWORD = uint32_t; using DWORD = uint32_t;
constexpr uint32_t PAGE_NOACCESS = 1; constexpr uint32_t PAGE_NOACCESS = 1;
constexpr uint32_t PAGE_READONLY = 2; constexpr uint32_t PAGE_READONLY = 2;
@@ -49,22 +44,25 @@ constexpr uint32_t MEM_RELEASE = 0;
int ToHostProt(uint32_t protection) { int ToHostProt(uint32_t protection) {
switch (protection) { switch (protection) {
case PAGE_NOACCESS: return PROT_NONE; case PAGE_NOACCESS:
case PAGE_READONLY: return PROT_READ; return PROT_NONE;
default: return PROT_READ | PROT_WRITE; case PAGE_READONLY:
return PROT_READ;
default:
return PROT_READ | PROT_WRITE;
} }
} }
uint32_t Protection(const void* address) { uint32_t Protection(const void *address) {
const auto addr = reinterpret_cast<uintptr_t>(address); const auto addr = reinterpret_cast<uintptr_t>(address);
std::FILE* maps = std::fopen("/proc/self/maps", "r"); std::FILE *maps = std::fopen("/proc/self/maps", "r");
Check(maps != nullptr, "open /proc/self/maps failed"); Check(maps != nullptr, "open /proc/self/maps failed");
char line[512]; char line[512];
uint32_t result = 0; // 0 => not mapped at all uint32_t result = 0;
while (std::fgets(line, sizeof(line), maps) != nullptr) { while (std::fgets(line, sizeof(line), maps) != nullptr) {
unsigned long start = 0; unsigned long start = 0;
unsigned long end = 0; unsigned long end = 0;
char perms[8] {}; char perms[8]{};
if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) { if (std::sscanf(line, "%lx-%lx %7s", &start, &end, perms) != 3) {
continue; continue;
} }
@@ -79,18 +77,13 @@ uint32_t Protection(const void* address) {
return result; return result;
} }
bool IsWritable(const void* address) { std::map<void *, size_t> &AllocationSizes() {
return Protection(address) == PAGE_READWRITE; static std::map<void *, size_t> sizes;
}
// munmap needs the length that VirtualFree's callers pass as 0, so sizes are remembered here.
std::map<void*, size_t>& AllocationSizes() {
static std::map<void*, size_t> sizes;
return sizes; return sizes;
} }
int VirtualFree(void* address, size_t /*size*/, DWORD /*type*/) { int VirtualFree(void *address, size_t, DWORD) {
auto& sizes = AllocationSizes(); auto &sizes = AllocationSizes();
auto it = sizes.find(address); auto it = sizes.find(address);
if (it == sizes.end()) { if (it == sizes.end()) {
return 0; return 0;
@@ -100,31 +93,29 @@ int VirtualFree(void* address, size_t /*size*/, DWORD /*type*/) {
return ok; return ok;
} }
int VirtualProtect(void* address, size_t size, uint32_t protection, DWORD* old_protection) { int VirtualProtect(void *address, size_t size, uint32_t protection,
DWORD *old_protection) {
if (old_protection != nullptr) { if (old_protection != nullptr) {
*old_protection = Protection(address); *old_protection = Protection(address);
} }
return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0; return ::mprotect(address, size, ToHostProt(protection)) == 0 ? 1 : 0;
} }
#endif #else
uint32_t Protection(const void *address) {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS MEMORY_BASIC_INFORMATION info{};
bool IsWritable(const void* address) {
MEMORY_BASIC_INFORMATION info {};
Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed");
return info.Protect == PAGE_READWRITE;
}
uint32_t Protection(const void* address) {
MEMORY_BASIC_INFORMATION info {};
Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed"); Check(VirtualQuery(address, &info, sizeof(info)) != 0, "VirtualQuery failed");
return info.Protect; return info.Protect;
} }
#endif #endif
std::atomic_uint64_t g_protection_calls {0}; bool IsWritable(const void *address) {
return Protection(address) == PAGE_READWRITE;
}
bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { uint64_t g_protection_calls = 0;
bool ProtectAddressSpace(uint64_t vaddr, uint64_t size,
Common::VirtualMemory::Mode mode) {
uint32_t protection = PAGE_NOACCESS; uint32_t protection = PAGE_NOACCESS;
if (mode == Common::VirtualMemory::Mode::Read) { if (mode == Common::VirtualMemory::Mode::Read) {
protection = PAGE_READONLY; protection = PAGE_READONLY;
@@ -132,397 +123,125 @@ bool ProtectAddressSpace(uint64_t vaddr, uint64_t size, Common::VirtualMemory::M
protection = PAGE_READWRITE; protection = PAGE_READWRITE;
} }
DWORD old_protection = 0; DWORD old_protection = 0;
g_protection_calls.fetch_add(1, std::memory_order_relaxed); g_protection_calls++;
return VirtualProtect(reinterpret_cast<void*>(vaddr), size, protection, &old_protection) != 0; return VirtualProtect(reinterpret_cast<void *>(vaddr), size, protection,
&old_protection) != 0;
} }
#if 1 uint8_t *Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
struct FaultContext {
PageManager* manager = nullptr;
bool result = true;
bool reenter = false;
uint64_t reenter_address = 0;
bool block = false;
std::atomic_uint32_t calls {0};
std::atomic_bool entered {false};
std::atomic_bool release {false};
};
std::atomic<PageManager*> g_native_fault_manager {nullptr};
std::atomic_bool g_delay_native_fault {false};
std::atomic_bool g_native_fault_entered {false};
std::atomic_bool g_release_native_fault {false};
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
LONG CALLBACK NativeFaultHandler(EXCEPTION_POINTERS* exception) {
if (exception == nullptr || exception->ExceptionRecord == nullptr ||
exception->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) {
return EXCEPTION_CONTINUE_SEARCH;
}
const auto operation = exception->ExceptionRecord->ExceptionInformation[0];
const auto access = operation == 0 ? PageFaultAccess::Read
: operation == 1 ? PageFaultAccess::Write
: operation == 8 ? PageFaultAccess::Execute
: PageFaultAccess::Unknown;
auto* manager = g_native_fault_manager.load(std::memory_order_acquire);
if (manager == nullptr) {
return EXCEPTION_CONTINUE_SEARCH;
}
if (g_delay_native_fault.load(std::memory_order_acquire)) {
g_native_fault_entered.store(true, std::memory_order_release);
while (!g_release_native_fault.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
}
return manager->HandleFault(access, exception->ExceptionRecord->ExceptionInformation[1])
? EXCEPTION_CONTINUE_EXECUTION
: EXCEPTION_CONTINUE_SEARCH;
}
#else
// SIGSEGV stands in for the vectored exception handler.
void NativeFaultHandler(int signal_number, siginfo_t* info, void* native_context) {
auto* context = static_cast<ucontext_t*>(native_context);
const auto error_code = static_cast<uint64_t>(context->uc_mcontext.gregs[REG_ERR]);
const auto access = (error_code & 0x10u) != 0 ? PageFaultAccess::Execute
: (error_code & 0x02u) != 0 ? PageFaultAccess::Write
: PageFaultAccess::Read;
auto* manager = g_native_fault_manager.load(std::memory_order_acquire);
if (manager != nullptr) {
if (g_delay_native_fault.load(std::memory_order_acquire)) {
g_native_fault_entered.store(true, std::memory_order_release);
while (!g_release_native_fault.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
}
if (manager->HandleFault(access, reinterpret_cast<uint64_t>(info->si_addr))) {
return;
}
}
struct sigaction restore {};
restore.sa_handler = SIG_DFL;
sigemptyset(&restore.sa_mask);
::sigaction(signal_number, &restore, nullptr);
}
struct sigaction g_saved_segv_action {};
void* AddVectoredExceptionHandler(unsigned long /*first*/,
void (*handler)(int, siginfo_t*, void*)) {
struct sigaction action {};
action.sa_sigaction = handler;
sigemptyset(&action.sa_mask);
action.sa_flags = SA_SIGINFO;
if (::sigaction(SIGSEGV, &action, &g_saved_segv_action) != 0) {
return nullptr;
}
return reinterpret_cast<void*>(handler);
}
int RemoveVectoredExceptionHandler(void* /*token*/) {
return ::sigaction(SIGSEGV, &g_saved_segv_action, nullptr) == 0 ? 1 : 0;
}
#endif
bool InvalidateFault(void* context, Libs::Graphics::PageFaultAccess, uint64_t vaddr, uint64_t size,
Libs::Graphics::PageFaultPhase phase) noexcept {
auto* fault = static_cast<FaultContext*>(context);
Check(fault != nullptr && fault->manager != nullptr, "invalid fault context");
if (phase != Libs::Graphics::PageFaultPhase::Invalidate) {
return true;
}
fault->calls.fetch_add(1, std::memory_order_relaxed);
if (fault->reenter) {
const auto address = fault->reenter_address != 0 ? fault->reenter_address : vaddr;
(void)fault->manager->HandleFault(PageFaultAccess::Write, address);
}
if (fault->block) {
fault->entered.store(true, std::memory_order_release);
while (!fault->release.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
}
(void)size;
return fault->result;
}
uint8_t* Allocate(uint64_t size, uint32_t protection = PAGE_READWRITE) {
constexpr uintptr_t test_address = 0x0000000200010000ull; constexpr uintptr_t test_address = 0x0000000200010000ull;
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
auto* memory = static_cast<uint8_t*>(VirtualAlloc(reinterpret_cast<void*>(test_address), size, auto *memory = static_cast<uint8_t *>(
VirtualAlloc(reinterpret_cast<void *>(test_address), size,
MEM_RESERVE | MEM_COMMIT, protection)); MEM_RESERVE | MEM_COMMIT, protection));
Check(memory == reinterpret_cast<void*>(test_address), "fixed low VirtualAlloc failed"); Check(memory == reinterpret_cast<void *>(test_address),
"fixed low VirtualAlloc failed");
#else #else
// Do not overwrite a leaked mapping from an earlier case. void *raw = ::mmap(reinterpret_cast<void *>(test_address), size,
void* raw = ::mmap(reinterpret_cast<void*>(test_address), size, ToHostProt(protection), ToHostProt(protection),
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
Check(raw == reinterpret_cast<void*>(test_address), "fixed low mmap failed"); Check(raw == reinterpret_cast<void *>(test_address), "fixed low mmap failed");
auto* memory = static_cast<uint8_t*>(raw); auto *memory = static_cast<uint8_t *>(raw);
AllocationSizes()[raw] = static_cast<size_t>(size); AllocationSizes()[raw] = static_cast<size_t>(size);
#endif #endif
return memory; return memory;
} }
void TestWatchFaultAndUnwatch() { void TestWatchAndUnwatch() {
g_protection_calls.store(0, std::memory_order_relaxed); g_protection_calls = 0;
FaultContext context; PageManager manager;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize(); const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size * 2); auto *memory = Allocate(page_size * 2);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(reinterpret_cast<uint64_t>(memory), page_size * 2); manager.OnGpuMap(address, page_size * 2);
manager.UpdatePageWatchers(true, reinterpret_cast<uint64_t>(memory), page_size); manager.UpdatePageWatchers(true, address, page_size);
Check(manager.IsTracked(reinterpret_cast<uint64_t>(memory)) && !IsWritable(memory), Check(Protection(memory) == PAGE_READONLY && IsWritable(memory + page_size),
"watch did not protect the page"); "write watch installed incorrect protections");
Check(g_protection_calls.load(std::memory_order_relaxed) != 0, Check(g_protection_calls != 0,
"watch protection bypassed the address-space owner callback"); "watch protection bypassed the address-space owner callback");
Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast<uint64_t>(memory + 32)), manager.UpdatePageWatchers(false, address, page_size);
"tracked write fault was not handled"); Check(IsWritable(memory), "write unwatch did not restore access");
Check(!manager.IsTracked(reinterpret_cast<uint64_t>(memory)) && IsWritable(memory), manager.OnGpuUnmap(address, page_size * 2);
"fault invalidation did not remove the watcher");
Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast<uint64_t>(memory)),
"single delayed write fault was not coalesced");
Check(manager.HandleFault(PageFaultAccess::Write, reinterpret_cast<uint64_t>(memory)),
"second delayed write fault was not coalesced");
manager.OnGpuUnmap(reinterpret_cast<uint64_t>(memory), page_size * 2);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
} }
void TestSharedWatcherFault() { void TestSharedWatcherCounts() {
FaultContext context; PageManager manager;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize(); const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size); auto *memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory); const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size); manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address + 8, 32); manager.UpdatePageWatchers(true, address + 8, 32);
manager.UpdatePageWatchers(true, address + 128, 64); manager.UpdatePageWatchers(true, address + 128, 64);
Check(manager.HandleFault(PageFaultAccess::Write, address + 16), manager.UpdatePageWatchers(false, address + 8, 32);
"shared-watcher fault was not handled");
Check(!manager.IsTracked(address) && IsWritable(memory),
"fault callback did not clear every shared watcher");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestReadWriteWatcherFault() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::Write);
manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite);
manager.UpdatePageWatchers(false, address, page_size, Libs::Graphics::PageWatchMode::Write);
Check(Protection(memory) == PAGE_NOACCESS,
"read/write watcher did not install no-access protection");
Check(manager.HandleFault(PageFaultAccess::Read, address + 8),
"tracked read fault was not handled");
Check(Protection(memory) == PAGE_READWRITE && !manager.IsTracked(address),
"read fault did not release every watcher");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestPermittedMappedLateFaultsResume() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size);
manager.UpdatePageWatchers(false, address, page_size);
Check(manager.HandleFault(PageFaultAccess::Write, address),
"first delayed mapped write was not accepted");
Check(manager.HandleFault(PageFaultAccess::Write, address),
"second delayed mapped write was not accepted");
Check(manager.HandleFault(PageFaultAccess::Read, address),
"delayed mapped read was not accepted on readable backing");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestPartialMappingUnmapPreservesTokens() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.OnGpuMap(address + 8, 16);
manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite);
manager.UpdatePageWatchers(false, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite);
manager.OnGpuUnmap(address + 8, 16);
Check(manager.HandleFault(PageFaultAccess::Read, address),
"partial mapping unmap erased delayed read ownership");
Check(manager.HandleFault(PageFaultAccess::Write, address),
"partial mapping unmap erased delayed write ownership");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestNativeDelayedReadAfterModeDowngrade() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
memory[0] = 0x6d;
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size);
manager.UpdatePageWatchers(true, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite);
manager.UpdatePageWatchers(false, address, page_size);
Check(Protection(memory) == PAGE_NOACCESS,
"read/write ownership did not install no-access protection");
void* handler = AddVectoredExceptionHandler(1, NativeFaultHandler);
Check(handler != nullptr, "AddVectoredExceptionHandler failed");
Check(g_native_fault_manager.exchange(&manager, std::memory_order_acq_rel) == nullptr,
"native fault manager already installed");
g_native_fault_entered.store(false, std::memory_order_release);
g_release_native_fault.store(false, std::memory_order_release);
g_delay_native_fault.store(true, std::memory_order_release);
uint8_t value = 0;
std::thread reader([&] { value = *static_cast<volatile uint8_t*>(memory); });
while (!g_native_fault_entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
manager.UpdatePageWatchers(true, address, page_size);
manager.UpdatePageWatchers(false, address, page_size, Libs::Graphics::PageWatchMode::ReadWrite);
Check(Protection(memory) == PAGE_READONLY, Check(Protection(memory) == PAGE_READONLY,
"mode downgrade did not restore readable protection"); "first unwatch released a shared watcher");
manager.UpdatePageWatchers(true, address, page_size); manager.UpdatePageWatchers(false, address + 128, 64);
Check(IsWritable(memory), "last unwatch did not restore access");
context.block = true;
bool write_handled = false;
std::thread writer(
[&] { write_handled = manager.HandleFault(PageFaultAccess::Write, address); });
while (!context.entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
g_release_native_fault.store(true, std::memory_order_release);
reader.join();
context.release.store(true, std::memory_order_release);
writer.join();
g_delay_native_fault.store(false, std::memory_order_release);
Check(g_native_fault_manager.exchange(nullptr, std::memory_order_acq_rel) == &manager,
"native fault manager publication changed");
Check(RemoveVectoredExceptionHandler(handler) != 0, "RemoveVectoredExceptionHandler failed");
Check(value == 0x6d && write_handled && context.calls.load(std::memory_order_relaxed) == 1,
"delayed read was not coalesced across write ownership/resolution");
manager.OnGpuUnmap(address, page_size); manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
} }
void TestDelayedFaultAfterExplicitUnwatch() { void TestMixedWatcherModes() {
FaultContext context; PageManager manager;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize(); const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size); auto *memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory); const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size); manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size); manager.UpdatePageWatchers(true, address, page_size, PageWatchMode::Write);
manager.UpdatePageWatchers(true, address, page_size,
// Models a store that already raised an AV before another thread published PageWatchMode::ReadWrite);
// explicit CPU dirtiness and removed the watcher. Check(Protection(memory) == PAGE_NOACCESS,
manager.UpdatePageWatchers(false, address, page_size); "read/write watcher did not deny access");
Check(manager.HandleFault(PageFaultAccess::Write, address), manager.UpdatePageWatchers(false, address, page_size, PageWatchMode::Write);
"delayed watched write was not accepted after explicit unwatch"); Check(Protection(memory) == PAGE_NOACCESS,
Check(manager.HandleFault(PageFaultAccess::Write, address) && "write unwatch released a read/write watcher");
context.calls.load(std::memory_order_relaxed) == 0, manager.UpdatePageWatchers(false, address, page_size,
"second explicit-unwatch fault was not coalesced or dispatched invalidation"); PageWatchMode::ReadWrite);
Check(IsWritable(memory), "read/write unwatch did not restore access");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestNativeAccessViolation() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size);
void* handler = AddVectoredExceptionHandler(1, NativeFaultHandler);
Check(handler != nullptr, "AddVectoredExceptionHandler failed");
Check(g_native_fault_manager.exchange(&manager, std::memory_order_acq_rel) == nullptr,
"native fault manager already installed");
*static_cast<volatile uint8_t*>(memory) = 0x5a;
Check(g_native_fault_manager.exchange(nullptr, std::memory_order_acq_rel) == &manager,
"native fault manager publication changed");
Check(RemoveVectoredExceptionHandler(handler) != 0, "RemoveVectoredExceptionHandler failed");
Check(memory[0] == 0x5a && !manager.IsTracked(address) &&
context.calls.load(std::memory_order_relaxed) == 1,
"native access violation did not invalidate and resume the store");
manager.OnGpuUnmap(address, page_size); manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
} }
void TestCrossRegionRange() { void TestCrossRegionRange() {
FaultContext context; PageManager manager;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize(); const auto page_size = manager.GetPageSize();
constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; constexpr uint64_t region_size = 4ull * 1024ull * 1024ull;
auto* memory = Allocate(region_size * 2); auto *memory = Allocate(region_size * 2);
const auto base = reinterpret_cast<uint64_t>(memory); const auto base = reinterpret_cast<uint64_t>(memory);
const auto boundary = (base + region_size - 1) & ~(region_size - 1); const auto boundary = (base + region_size - 1) & ~(region_size - 1);
Check(boundary >= base + page_size && boundary + page_size <= base + region_size * 2, Check(boundary >= base + page_size &&
boundary + page_size <= base + region_size * 2,
"test allocation does not contain a region boundary"); "test allocation does not contain a region boundary");
manager.OnGpuMap(base, region_size * 2); manager.OnGpuMap(base, region_size * 2);
manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2); manager.UpdatePageWatchers(true, boundary - page_size, page_size * 2);
Check(!IsWritable(reinterpret_cast<void*>(boundary - page_size)) && Check(!IsWritable(reinterpret_cast<void *>(boundary - page_size)) &&
!IsWritable(reinterpret_cast<void*>(boundary)), !IsWritable(reinterpret_cast<void *>(boundary)),
"cross-region watch did not protect both pages"); "cross-region watch did not protect both pages");
manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2); manager.UpdatePageWatchers(false, boundary - page_size, page_size * 2);
Check(IsWritable(reinterpret_cast<void*>(boundary - page_size)) && Check(IsWritable(reinterpret_cast<void *>(boundary - page_size)) &&
IsWritable(reinterpret_cast<void*>(boundary)), IsWritable(reinterpret_cast<void *>(boundary)),
"cross-region unwatch did not restore both pages"); "cross-region unwatch did not restore both pages");
manager.OnGpuUnmap(base, region_size * 2); manager.OnGpuUnmap(base, region_size * 2);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
} }
void TestBatchedWatcherRanges() { void TestBatchedWatcherRanges() {
FaultContext context; PageManager manager;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
const auto page_size = manager.GetPageSize(); const auto page_size = manager.GetPageSize();
constexpr uint64_t region_size = 4ull * 1024ull * 1024ull; constexpr uint64_t region_size = 4ull * 1024ull * 1024ull;
constexpr uint64_t allocation_size = region_size * 3; constexpr uint64_t allocation_size = region_size * 3;
auto* memory = Allocate(allocation_size); auto *memory = Allocate(allocation_size);
const auto address = reinterpret_cast<uint64_t>(memory); const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, allocation_size); manager.OnGpuMap(address, allocation_size);
manager.UpdatePageWatchers(true, address + page_size, page_size); manager.UpdatePageWatchers(true, address + page_size, page_size);
manager.UpdatePageWatchers(true, address + page_size * 3, page_size); manager.UpdatePageWatchers(true, address + page_size * 3, page_size);
manager.UpdatePageWatchers(true, address, page_size * 5); manager.UpdatePageWatchers(true, address, page_size * 5);
for (uint64_t page = 0; page < 5; page++) {
Check(Protection(memory + page * page_size) == PAGE_READONLY,
"fragmented watch did not coalesce to read-only");
}
manager.UpdatePageWatchers(false, address, page_size * 5); manager.UpdatePageWatchers(false, address, page_size * 5);
Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_READONLY && Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_READONLY &&
IsWritable(memory + page_size * 2) && IsWritable(memory + page_size * 2) &&
@@ -532,103 +251,60 @@ void TestBatchedWatcherRanges() {
manager.UpdatePageWatchers(false, address + page_size, page_size); manager.UpdatePageWatchers(false, address + page_size, page_size);
manager.UpdatePageWatchers(false, address + page_size * 3, page_size); manager.UpdatePageWatchers(false, address + page_size * 3, page_size);
g_protection_calls = 0;
manager.UpdatePageWatchers(true, address, allocation_size); manager.UpdatePageWatchers(true, address, allocation_size);
Check(!IsWritable(memory) && !IsWritable(memory + region_size) && Check(g_protection_calls == 4 && Protection(memory) == PAGE_READONLY &&
!IsWritable(memory + region_size * 2) && Protection(memory + region_size) == PAGE_READONLY &&
!IsWritable(memory + allocation_size - page_size), Protection(memory + region_size * 2) == PAGE_READONLY &&
"large cross-region watch did not protect the full range"); Protection(memory + allocation_size - page_size) == PAGE_READONLY,
"large watch was not batched and protected by tracking region");
manager.UpdatePageWatchers(false, address, allocation_size); manager.UpdatePageWatchers(false, address, allocation_size);
Check(IsWritable(memory) && IsWritable(memory + region_size) && Check(IsWritable(memory) && IsWritable(memory + region_size) &&
IsWritable(memory + region_size * 2) && IsWritable(memory + region_size * 2) &&
IsWritable(memory + allocation_size - page_size), IsWritable(memory + allocation_size - page_size),
"large cross-region unwatch did not restore the full range"); "large cross-region unwatch did not restore the full range");
manager.UpdatePageWatchers(true, address, page_size * 5);
manager.UpdatePageWatchers(true, address + page_size, page_size * 3,
Libs::Graphics::PageWatchMode::ReadWrite);
Check(Protection(memory) == PAGE_READONLY && Protection(memory + page_size) == PAGE_NOACCESS &&
Protection(memory + page_size * 2) == PAGE_NOACCESS &&
Protection(memory + page_size * 3) == PAGE_NOACCESS &&
Protection(memory + page_size * 4) == PAGE_READONLY,
"mixed watcher modes installed incorrect protections");
manager.UpdatePageWatchers(false, address, page_size * 5);
Check(IsWritable(memory) && Protection(memory + page_size) == PAGE_NOACCESS &&
Protection(memory + page_size * 2) == PAGE_NOACCESS &&
Protection(memory + page_size * 3) == PAGE_NOACCESS &&
IsWritable(memory + page_size * 4),
"write unwatch incorrectly released read/write watchers");
manager.UpdatePageWatchers(false, address + page_size, page_size * 3,
Libs::Graphics::PageWatchMode::ReadWrite);
Check(IsWritable(memory + page_size) && IsWritable(memory + page_size * 2) &&
IsWritable(memory + page_size * 3),
"read/write unwatch did not restore writable protection");
manager.OnGpuUnmap(address, allocation_size); manager.OnGpuUnmap(address, allocation_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed"); Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
} }
[[noreturn]] void RunDeathCase(const char* name) { [[noreturn]] void RunDeathCase(const char *name) {
FaultContext context; PageManager manager;
auto manager = std::make_unique<PageManager>(InvalidateFault, &context); const auto page_size = manager.GetPageSize();
context.manager = manager.get();
const auto page_size = manager->GetPageSize();
if (std::strcmp(name, "invalid-range") == 0) { if (std::strcmp(name, "invalid-range") == 0) {
manager->UpdatePageWatchers(true, (1ull << 40u) - 1, 2); manager.UpdatePageWatchers(true, (1ull << 40u) - 1, 2);
} else if (std::strcmp(name, "unknown-untrack") == 0) { } else if (std::strcmp(name, "unknown-untrack") == 0) {
manager->UpdatePageWatchers(false, 0x1000, page_size); manager.UpdatePageWatchers(false, 0x1000, page_size);
} else { } else if (std::strcmp(name, "destructor-watch") == 0) {
const bool two_pages = std::strcmp(name, "cross-reentrant") == 0; auto doomed = std::make_unique<PageManager>();
auto* memory = Allocate(two_pages ? page_size * 2 : page_size); auto *memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory); const auto address = reinterpret_cast<uint64_t>(memory);
manager->OnGpuMap(address, two_pages ? page_size * 2 : page_size); doomed->OnGpuMap(address, page_size);
manager->UpdatePageWatchers(true, address, page_size); doomed->UpdatePageWatchers(true, address, page_size);
if (two_pages) { doomed.reset();
manager->UpdatePageWatchers(true, address + page_size, page_size);
}
if (std::strcmp(name, "destructor-watch") == 0) {
manager.reset();
} else if (std::strcmp(name, "non-write") == 0) {
(void)manager->HandleFault(PageFaultAccess::Read, address);
} else if (std::strcmp(name, "callback-false") == 0) {
context.result = false;
(void)manager->HandleFault(PageFaultAccess::Write, address);
} else if (std::strcmp(name, "reentrant") == 0) {
context.reenter = true;
(void)manager->HandleFault(PageFaultAccess::Write, address);
} else if (std::strcmp(name, "cross-reentrant") == 0) {
context.reenter = true;
context.reenter_address = address + page_size;
(void)manager->HandleFault(PageFaultAccess::Write, address);
} else if (std::strcmp(name, "concurrent-non-write") == 0) {
context.block = true;
std::thread first([&] { (void)manager->HandleFault(PageFaultAccess::Write, address); });
while (!context.entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
(void)manager->HandleFault(PageFaultAccess::Read, address);
first.join();
} else {
std::_Exit(0x7f);
}
} }
std::_Exit(0x7f); std::_Exit(0x7f);
} }
void CheckDeathCase(const char* name) { void CheckDeathCase(const char *name) {
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
char path[MAX_PATH] {}; char path[MAX_PATH]{};
Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0, "GetModuleFileName failed"); Check(GetModuleFileNameA(nullptr, path, MAX_PATH) != 0,
"GetModuleFileName failed");
std::string command = std::string("\"") + path + "\" --death " + name; std::string command = std::string("\"") + path + "\" --death " + name;
std::vector<char> mutable_command(command.begin(), command.end()); std::vector<char> mutable_command(command.begin(), command.end());
mutable_command.push_back('\0'); mutable_command.push_back('\0');
STARTUPINFOA startup {sizeof(startup)}; STARTUPINFOA startup{sizeof(startup)};
PROCESS_INFORMATION process {}; PROCESS_INFORMATION process{};
Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE, CREATE_NO_WINDOW, Check(CreateProcessA(nullptr, mutable_command.data(), nullptr, nullptr, FALSE,
nullptr, nullptr, &startup, &process) != 0, CREATE_NO_WINDOW, nullptr, nullptr, &startup,
&process) != 0,
"CreateProcess failed"); "CreateProcess failed");
Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0, "death test timed out"); Check(WaitForSingleObject(process.hProcess, 10000) == WAIT_OBJECT_0,
"death test timed out");
DWORD exit_code = 0; DWORD exit_code = 0;
Check(GetExitCodeProcess(process.hProcess, &exit_code) != 0 && Check(
GetExitCodeProcess(process.hProcess, &exit_code) != 0 &&
(exit_code == 322 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION), (exit_code == 322 || exit_code == EXCEPTION_NONCONTINUABLE_EXCEPTION),
"death case did not use the PageManager fatal exit"); "death case did not use the PageManager fatal exit");
CloseHandle(process.hThread); CloseHandle(process.hThread);
@@ -642,108 +318,42 @@ void CheckDeathCase(const char* name) {
} }
int status = 0; int status = 0;
Check(::waitpid(pid, &status, 0) == pid, "waitpid failed"); Check(::waitpid(pid, &status, 0) == pid, "waitpid failed");
// Exit status carries only the low 8 bits. const bool fatal_exit =
const bool fatal_exit = WIFEXITED(status) && WEXITSTATUS(status) == (322 & 0xff); WIFEXITED(status) && WEXITSTATUS(status) == (322 & 0xff);
const bool fatal_signal = WIFSIGNALED(status); const bool fatal_signal = WIFSIGNALED(status);
Check(fatal_exit || fatal_signal, "death case did not use the PageManager fatal exit"); Check(fatal_exit || fatal_signal,
"death case did not use the PageManager fatal exit");
#endif #endif
} }
void TestFatalPaths() { void TestFatalPaths() {
for (const char* name: for (const char *name :
{"invalid-range", "unknown-untrack", "destructor-watch", "non-write", "callback-false", {"invalid-range", "unknown-untrack", "destructor-watch"}) {
"reentrant", "cross-reentrant", "concurrent-non-write"}) {
CheckDeathCase(name); CheckDeathCase(name);
} }
} }
void TestConcurrentFault() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
context.block = true;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size);
bool first_result = false;
bool second_result = false;
std::thread first([&] { first_result = manager.HandleFault(PageFaultAccess::Write, address); });
while (!context.entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
std::thread second(
[&] { second_result = manager.HandleFault(PageFaultAccess::Write, address); });
context.release.store(true, std::memory_order_release);
first.join();
second.join();
Check(first_result && second_result && context.calls.load(std::memory_order_relaxed) == 1,
"concurrent faults dispatched invalidation more than once");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
void TestExternalDirtyTransferDuringResolution() {
FaultContext context;
PageManager manager(InvalidateFault, &context);
context.manager = &manager;
context.block = true;
const auto page_size = manager.GetPageSize();
auto* memory = Allocate(page_size);
const auto address = reinterpret_cast<uint64_t>(memory);
manager.OnGpuMap(address, page_size);
manager.UpdatePageWatchers(true, address, page_size);
bool handled = false;
std::thread fault([&] { handled = manager.HandleFault(PageFaultAccess::Write, address); });
while (!context.entered.load(std::memory_order_acquire)) {
std::this_thread::yield();
}
manager.UpdatePageWatchers(false, address, page_size);
context.release.store(true, std::memory_order_release);
fault.join();
Check(handled && !manager.IsTracked(address) && IsWritable(memory),
"external dirty transfer did not satisfy active resolution");
manager.OnGpuUnmap(address, page_size);
Check(VirtualFree(memory, 0, MEM_RELEASE) != 0, "VirtualFree failed");
}
#endif
} // namespace } // namespace
namespace Libs::LibKernel::Memory { namespace Libs::LibKernel::Memory {
bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size, Common::VirtualMemory::Mode mode) { bool ProtectGuestHostMemory(uint64_t vaddr, uint64_t size,
Common::VirtualMemory::Mode mode) {
return ProtectAddressSpace(vaddr, size, mode); return ProtectAddressSpace(vaddr, size, mode);
} }
} // namespace Libs::LibKernel::Memory } // namespace Libs::LibKernel::Memory
int main(int argc, char** argv) { int main(int argc, char **argv) {
#if 1
if (argc == 3 && std::strcmp(argv[1], "--death") == 0) { if (argc == 3 && std::strcmp(argv[1], "--death") == 0) {
RunDeathCase(argv[2]); RunDeathCase(argv[2]);
} }
TestWatchFaultAndUnwatch(); TestWatchAndUnwatch();
TestSharedWatcherFault(); TestSharedWatcherCounts();
TestReadWriteWatcherFault(); TestMixedWatcherModes();
TestPermittedMappedLateFaultsResume();
TestPartialMappingUnmapPreservesTokens();
TestNativeDelayedReadAfterModeDowngrade();
TestDelayedFaultAfterExplicitUnwatch();
TestNativeAccessViolation();
TestCrossRegionRange(); TestCrossRegionRange();
TestBatchedWatcherRanges(); TestBatchedWatcherRanges();
TestConcurrentFault();
TestExternalDirtyTransferDuringResolution();
TestFatalPaths(); TestFatalPaths();
std::puts("PageManagerTests: all cases passed"); std::puts("PageManagerTests: all cases passed");
return 0; return 0;
#else
(void)argc;
(void)argv;
std::fputs("PageManagerTests: unsupported platform\n", stderr);
return 1;
#endif
} }
tests/ShaderRecompilerComputeTests.cpp
+2 -16
View File
@@ -3135,10 +3135,7 @@ public:
Require(name, "partial image unmap tracking", Require(name, "partial image unmap tracking",
partial_unmap_image_id && partial_unmap_image_id &&
!texture_cache.FindImageFromRange(partial_unmap_image.info.data.address, !texture_cache.FindImageFromRange(partial_unmap_image.info.data.address,
0x2000, false) && 0x2000, false),
!texture_cache.InvalidateMemory(
PageFaultAccess::Write, partial_unmap_image.info.data.address + 0x1000,
0x1000, PageFaultPhase::Invalidate),
"partial unmap left the deleted image's mapped tail tracked"); "partial unmap left the deleted image's mapped tail tracked");
constexpr uint64_t unformatted_alias_offset = 0x2500000; constexpr uint64_t unformatted_alias_offset = 0x2500000;
@@ -14459,17 +14456,6 @@ void CheckEmbeddedFetchVertexOffset() {
std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset"); std::printf("[host] %-32s ok\n", "EmbeddedFetchVertexOffset");
} }
struct CacheFaultContext {
TextureCache* texture = nullptr;
};
bool CacheFault(void* opaque, PageFaultAccess access, uint64_t vaddr, uint64_t size,
PageFaultPhase phase) noexcept {
auto* context = static_cast<CacheFaultContext*>(opaque);
return context != nullptr && context->texture != nullptr &&
context->texture->InvalidateMemory(access, vaddr, size, phase);
}
[[noreturn]] void RunReverseRenderTargetDeathCase() { [[noreturn]] void RunReverseRenderTargetDeathCase() {
(void)TextureGetRenderTargetFormat(12u, 7u, 3u); (void)TextureGetRenderTargetFormat(12u, 7u, 3u);
std::_Exit(0x7f); std::_Exit(0x7f);
@@ -16110,7 +16096,7 @@ void CheckStorageTextureGpuOwnedRebindState() {
auto* memory = reinterpret_cast<uint8_t*>(guest_memory); auto* memory = reinterpret_cast<uint8_t*>(guest_memory);
Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base, Require("StorageTextureGpuOwnedRebind", "allocation", guest_memory == base,
"fixed guest-owner allocation failed"); "fixed guest-owner allocation failed");
PageManager page_manager(CacheFault, nullptr); PageManager page_manager;
MemoryTracker tracker(page_manager); MemoryTracker tracker(page_manager);
page_manager.OnGpuMap(base, size); page_manager.OnGpuMap(base, size);
tracker.ForEachUploadRange( tracker.ForEachUploadRange(