gremling/KytyPS5
1
0 Fork 0
mirror of https://github.com/KytyPS5/KytyPS5.git synced 2026-10-29 16:51:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

ampr: schedule address waits without blocking other priorities

Browse Source 
Co-authored-by: Mac <me@itsmemac.com>
This commit is contained in:
nmzikandMac committed 2026-09-29 21:56:38 +04:00
1 parent 5bf9218b32
commit c045f08904
2 files changed
+366 -251
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/libs/libAmpr.cpp
+241 -251
View File
@@ -12,11 +12,16 @@
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdint> #include <cstdint>
#include <cstring> #include <cstring>
#include <deque>
#include <limits> #include <limits>
#include <mutex> #include <mutex>
#include <string> #include <string>
#include <thread>
#include <unordered_map> #include <unordered_map>
#include <variant> #include <variant>
#include <vector> #include <vector>
@@ -37,13 +42,6 @@ struct ResultBuffer {
uint32_t errorOffset = 0; uint32_t errorOffset = 0;
}; };
struct SubmissionState {
uint64_t command_buffer = 0;
uint64_t result = 0;
int32_t execution_result = 0;
uint32_t error_offset = 0;
};
struct ResolvedPathInfo { struct ResolvedPathInfo {
int result = OK; int result = OK;
uint32_t file_id = 0xffffffffu; uint32_t file_id = 0xffffffffu;
@@ -54,7 +52,8 @@ struct ResolvedPathInfo {
static std::mutex g_mutex; static std::mutex g_mutex;
static uint32_t g_next_submission_id = 1; static uint32_t g_next_submission_id = 1;
static std::unordered_map<uint32_t, SubmissionState> g_submissions; static std::unordered_map<uint32_t, bool> g_submissions;
static std::condition_variable g_submission_cv;
static std::unordered_map<uint32_t, std::string> g_files; static std::unordered_map<uint32_t, std::string> g_files;
static std::unordered_map<uint32_t, uint64_t> g_file_sizes; static std::unordered_map<uint32_t, uint64_t> g_file_sizes;
static std::unordered_map<std::string, ResolvedPathInfo> g_resolved_paths; static std::unordered_map<std::string, ResolvedPathInfo> g_resolved_paths;
@@ -340,38 +339,35 @@ static int ResolvePathsCommon(const char* const* path_list, uint32_t count, uint
return results != nullptr ? static_cast<int>(success_count) : OK; return results != nullptr ? static_cast<int>(success_count) : OK;
} }
static uint32_t AllocateSubmissionId(uint64_t command_buffer, uint64_t result) { static uint32_t AllocateSubmissionId() {
std::scoped_lock lock(g_mutex); std::scoped_lock lock(g_mutex);
auto id = g_next_submission_id++; auto id = g_next_submission_id++;
if (id == 0) { if (id == 0) {
id = g_next_submission_id++; id = g_next_submission_id++;
} }
g_submissions[id] = SubmissionState {command_buffer, result}; g_submissions.emplace(id, false);
return id; return id;
} }
static void SetSubmissionResult(uint32_t submission_id, int32_t execution_result, static void SetSubmissionComplete(uint32_t submission_id) {
uint32_t error_offset) { if (submission_id == 0) {
std::scoped_lock lock(g_mutex); return;
auto it = g_submissions.find(submission_id);
if (it != g_submissions.end()) {
it->second.execution_result = execution_result;
it->second.error_offset = error_offset;
} }
std::scoped_lock lock(g_mutex);
g_submissions.at(submission_id) = true;
g_submission_cv.notify_all();
} }
static bool CompleteSubmission(uint32_t submission_id, SubmissionState* state) { static bool CompleteSubmission(uint32_t submission_id) {
std::scoped_lock lock(g_mutex); std::unique_lock lock(g_mutex);
auto it = g_submissions.find(submission_id); if (!g_submissions.contains(submission_id)) {
if (it == g_submissions.end()) {
return false; return false;
} }
if (state != nullptr) { g_submission_cv.wait(lock, [submission_id] {
*state = it->second; const auto it = g_submissions.find(submission_id);
} return it == g_submissions.end() || it->second;
g_submissions.erase(it); });
return true; return g_submissions.erase(submission_id) != 0;
} }
static int WriteResult(void* result, int32_t execution_result = 0, uint32_t error_offset = 0) { static int WriteResult(void* result, int32_t execution_result = 0, uint32_t error_offset = 0) {
@@ -390,8 +386,9 @@ static int WriteResult(void* result, int32_t execution_result = 0, uint32_t erro
} // namespace AprShared } // namespace AprShared
namespace LibAmpr::Ampr { namespace LibAmpr::Ampr {
static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result, enum class Engine { Apr, Amm };
uint32_t* error_offset); static int QueueCommandBuffer(Engine engine, uint64_t command_buffer, uint32_t bytes,
uint32_t priority, void* result, uint32_t* out_submission_id);
} }
namespace LibKernelApr { namespace LibKernelApr {
@@ -537,89 +534,37 @@ static int KYTY_SYSV_ABI ResolveFilepathsWithPrefixToIdsAndFileSizesForEach(
AprShared::ResolvePathsCommon(path_list, count, ids, sizes, nullptr, prefix, results)); AprShared::ResolvePathsCommon(path_list, count, ids, sizes, nullptr, prefix, results));
} }
static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint64_t, static int KYTY_SYSV_ABI SubmitCommandBufferAndGetResult(void* command_buffer, uint32_t priority,
void* result, void* result,
uint32_t* out_submission_id) { uint32_t* out_submission_id) {
PRINT_NAME(); PRINT_NAME();
return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer(
if (command_buffer == nullptr) { LibAmpr::Ampr::Engine::Apr, reinterpret_cast<uint64_t>(command_buffer), 0, priority, result,
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); out_submission_id));
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer);
const auto id =
AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast<uint64_t>(result));
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr,
&execution_result, &error_offset);
if (submit_result != OK) {
return KernelSyscallResult(submit_result);
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (out_submission_id != nullptr) {
if (!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT);
}
}
return KernelSyscallResult(AprShared::WriteResult(result, execution_result, error_offset));
} }
static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint64_t) { static int KYTY_SYSV_ABI SubmitCommandBuffer(void* command_buffer, uint32_t priority) {
PRINT_NAME(); PRINT_NAME();
return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer(
if (command_buffer == nullptr) { LibAmpr::Ampr::Engine::Apr, reinterpret_cast<uint64_t>(command_buffer), 0, priority,
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); nullptr, nullptr));
}
int32_t execution_result = OK;
uint32_t error_offset = 0;
return KernelSyscallResult(LibAmpr::Ampr::ExecuteAprCommandBuffer(
reinterpret_cast<uint64_t>(command_buffer), &execution_result, &error_offset));
} }
static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint64_t, static int KYTY_SYSV_ABI SubmitCommandBufferAndGetId(void* command_buffer, uint32_t priority,
uint32_t* out_submission_id) { uint32_t* out_submission_id) {
PRINT_NAME(); PRINT_NAME();
if (out_submission_id == nullptr) {
if (command_buffer == nullptr || out_submission_id == nullptr) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL); return KernelSyscallResult(LibKernel::KERNEL_ERROR_EINVAL);
} }
return KernelSyscallResult(LibAmpr::Ampr::QueueCommandBuffer(
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer); LibAmpr::Ampr::Engine::Apr, reinterpret_cast<uint64_t>(command_buffer), 0, priority,
const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0); nullptr, out_submission_id));
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result = LibAmpr::Ampr::ExecuteAprCommandBuffer(command_buffer_addr,
&execution_result, &error_offset);
if (submit_result != OK) {
return KernelSyscallResult(submit_result);
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_EFAULT);
}
return OK;
} }
static int KYTY_SYSV_ABI WaitCommandBuffer(uint32_t submission_id) { static int KYTY_SYSV_ABI WaitCommandBuffer(uint32_t submission_id) {
PRINT_NAME(); PRINT_NAME();
return KernelSyscallResult(
AprShared::SubmissionState state {}; AprShared::CompleteSubmission(submission_id) ? OK : LibKernel::KERNEL_ERROR_ESRCH);
if (!AprShared::CompleteSubmission(submission_id, &state)) {
return KernelSyscallResult(LibKernel::KERNEL_ERROR_ESRCH);
}
if (state.result != 0) {
return KernelSyscallResult(AprShared::WriteResult(
reinterpret_cast<void*>(state.result), state.execution_result, state.error_offset));
}
return OK;
} }
} // namespace Apr } // namespace Apr
@@ -720,6 +665,11 @@ struct CommandBufferState {
uint64_t address = 0; uint64_t address = 0;
uint64_t value = 0; uint64_t value = 0;
}; };
struct WaitAddressCommand {
uint64_t address;
uint64_t value;
uint8_t compare;
};
struct AmmMapCommand { struct AmmMapCommand {
AmmCommandKind kind = AmmCommandKind::MapAuto; AmmCommandKind kind = AmmCommandKind::MapAuto;
uint64_t va = 0; uint64_t va = 0;
@@ -729,9 +679,11 @@ struct CommandBufferState {
int32_t prot = 0; int32_t prot = 0;
uint8_t gpu_mask_id = 0; uint8_t gpu_mask_id = 0;
}; };
using CommandData = std::variant<ReadFileCommand, KernelEventCommand, WriteAddressCommand,
AmmMapCommand, WaitAddressCommand>;
struct Command { struct Command {
uint64_t record_offset; uint64_t record_offset;
std::variant<ReadFileCommand, KernelEventCommand, WriteAddressCommand, AmmMapCommand> data; CommandData data;
}; };
std::vector<Command> commands; std::vector<Command> commands;
bool gather_scatter_valid = false; bool gather_scatter_valid = false;
@@ -1141,27 +1093,6 @@ static bool AppendKernelEventRecord(uint64_t command_buffer, uint64_t eq, int32_
return CommitCommandBufferRecord(command_buffer, &state, KERNEL_EVENT_RECORD_SIZE); return CommitCommandBufferRecord(command_buffer, &state, KERNEL_EVENT_RECORD_SIZE);
} }
static bool AppendWriteAddressRecord(uint64_t command_buffer, uint64_t address, uint64_t value,
uint64_t record_size = 0x20) {
std::scoped_lock lock(g_command_buffer_mutex);
CommandBufferIterator it;
if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) {
return false;
}
auto& state = it->second;
if (!EnsureCommandBufferRecordSpace(command_buffer, &state, record_size)) {
return false;
}
const auto record_offset = state.write_offset;
std::memset(reinterpret_cast<void*>(state.buffer + record_offset), 0,
static_cast<size_t>(record_size));
state.commands.push_back(
{record_offset, CommandBufferState::WriteAddressCommand {address, value}});
return CommitCommandBufferRecord(command_buffer, &state, record_size);
}
static bool ValidateAmmMapArgs(uint64_t va, uint64_t size) { static bool ValidateAmmMapArgs(uint64_t va, uint64_t size) {
return va != 0 && size != 0 && (va & (AMM_PAGE_SIZE - 1u)) == 0 && return va != 0 && size != 0 && (va & (AMM_PAGE_SIZE - 1u)) == 0 &&
(size & (AMM_PAGE_SIZE - 1u)) == 0 && va + size >= va; (size & (AMM_PAGE_SIZE - 1u)) == 0 && va + size >= va;
@@ -1219,8 +1150,8 @@ static int ExecuteAmmMapCommand(const CommandBufferState::AmmMapCommand& command
static_cast<int64_t>(command.dmem_offset), AMM_PAGE_SIZE); static_cast<int64_t>(command.dmem_offset), AMM_PAGE_SIZE);
} }
static bool AppendAmmMapRecord(uint64_t command_buffer, static bool AppendCommandRecord(uint64_t command_buffer, CommandBufferState::CommandData command,
const CommandBufferState::AmmMapCommand& cmd, uint64_t record_size) { uint64_t record_size) {
std::scoped_lock lock(g_command_buffer_mutex); std::scoped_lock lock(g_command_buffer_mutex);
CommandBufferIterator it; CommandBufferIterator it;
if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) { if (!GetOrCreateCommandBufferStateLocked(command_buffer, &it)) {
@@ -1234,51 +1165,29 @@ static bool AppendAmmMapRecord(uint64_t command_
auto* record_addr = reinterpret_cast<void*>(state.buffer + state.write_offset); auto* record_addr = reinterpret_cast<void*>(state.buffer + state.write_offset);
std::memset(record_addr, 0, static_cast<size_t>(record_size)); std::memset(record_addr, 0, static_cast<size_t>(record_size));
state.commands.push_back({state.write_offset, cmd}); state.commands.push_back({state.write_offset, std::move(command)});
return CommitCommandBufferRecord(command_buffer, &state, record_size); return CommitCommandBufferRecord(command_buffer, &state, record_size);
} }
static int ReadHostFileToGuest(const std::string& host_path, uint64_t file_offset, static int ReadHostFileToGuest(const std::string& host_path, uint64_t file_offset,
uint64_t destination, uint64_t size, uint64_t* bytes_read); uint64_t destination, uint64_t size, uint64_t* bytes_read);
static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_result, static int ExecuteCommand(const CommandBufferState::Command& entry) {
uint32_t* error_offset) {
if (execution_result == nullptr || error_offset == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
*execution_result = OK;
*error_offset = 0;
CommandBufferState state {};
if (!TryGetCommandBufferState(command_buffer, &state)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
for (const auto& entry: state.commands) {
if (entry.record_offset >= state.write_offset) {
break;
}
if (const auto* payload = std::get_if<CommandBufferState::ReadFileCommand>(&entry.data)) { if (const auto* payload = std::get_if<CommandBufferState::ReadFileCommand>(&entry.data)) {
const auto& command = *payload; const auto& command = *payload;
std::string host_path; std::string host_path;
if (!AprShared::TryGetHostPath(command.file_id, &host_path)) { if (!AprShared::TryGetHostPath(command.file_id, &host_path)) {
LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", command.file_id); LOGF("\tAPR submit failed for unknown file id: 0x%08" PRIx32 "\n", command.file_id);
*execution_result = LibKernel::KERNEL_ERROR_ENOENT; return LibKernel::KERNEL_ERROR_ENOENT;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
} }
uint64_t bytes_read = 0; uint64_t bytes_read = 0;
auto result = ReadHostFileToGuest(host_path, command.file_offset, command.destination, auto result = ReadHostFileToGuest(host_path, command.file_offset, command.destination,
command.size, &bytes_read); command.size, &bytes_read);
if (result != OK) { if (result != OK) {
LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 LOGF("\tAPR submit read failed: id=0x%08" PRIx32 ", result=0x%08" PRIx32 ", path=%s\n",
", path=%s\n",
command.file_id, static_cast<uint32_t>(result), host_path.c_str()); command.file_id, static_cast<uint32_t>(result), host_path.c_str());
*execution_result = result; return result;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
} }
} else if (const auto* payload = } else if (const auto* payload =
std::get_if<CommandBufferState::KernelEventCommand>(&entry.data)) { std::get_if<CommandBufferState::KernelEventCommand>(&entry.data)) {
@@ -1290,23 +1199,14 @@ static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_r
LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32 LOGF("\tAPR submit event failed: eq=0x%016" PRIx64 ", id=%" PRId32
", result=0x%08" PRIx32 "\n", ", result=0x%08" PRIx32 "\n",
command.eq, command.id, static_cast<uint32_t>(result)); command.eq, command.id, static_cast<uint32_t>(result));
*execution_result = result; return result;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
} }
} else if (const auto* payload = } else if (const auto* payload =
std::get_if<CommandBufferState::WriteAddressCommand>(&entry.data)) { std::get_if<CommandBufferState::WriteAddressCommand>(&entry.data)) {
const auto& command = *payload; const auto& command = *payload;
if (!AprShared::WriteGuest(command.address, command.value)) { std::atomic_ref(*reinterpret_cast<uint64_t*>(command.address))
LOGF("\tAMPR submit write-address failed: address=0x%016" PRIx64 .store(command.value, std::memory_order_release);
" value=0x%016" PRIx64 "\n", } else if (const auto* payload = std::get_if<CommandBufferState::AmmMapCommand>(&entry.data)) {
command.address, command.value);
*execution_result = LibKernel::KERNEL_ERROR_EFAULT;
*error_offset = static_cast<uint32_t>(entry.record_offset);
return OK;
}
} else if (const auto* payload =
std::get_if<CommandBufferState::AmmMapCommand>(&entry.data)) {
const auto& command = *payload; const auto& command = *payload;
int result = OK; int result = OK;
if (command.kind == AmmCommandKind::Unmap) { if (command.kind == AmmCommandKind::Unmap) {
@@ -1317,18 +1217,150 @@ static int ExecuteAprCommandBuffer(uint64_t command_buffer, int32_t* execution_r
if (result != OK) { if (result != OK) {
LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 " dmem=0x%016" PRIx64 LOGF("\tAMM submit command failed: kind=%u va=0x%016" PRIx64 " dmem=0x%016" PRIx64
" size=0x%016" PRIx64 " type=%" PRId32 " prot=0x%08" PRIx32 " size=0x%016" PRIx64 " type=%" PRId32 " prot=0x%08" PRIx32 " result=0x%08" PRIx32
" result=0x%08" PRIx32 "\n", "\n",
static_cast<uint32_t>(command.kind), command.va, command.dmem_offset, static_cast<uint32_t>(command.kind), command.va, command.dmem_offset, command.size,
command.size, command.type, static_cast<uint32_t>(command.prot), command.type, static_cast<uint32_t>(command.prot), static_cast<uint32_t>(result));
static_cast<uint32_t>(result)); return result;
*execution_result = result; }
*error_offset = static_cast<uint32_t>(entry.record_offset); }
return OK; return OK;
}
struct Submission {
std::vector<CommandBufferState::Command> commands;
size_t cursor = 0;
void* result = nullptr;
uint32_t id = 0;
};
static bool IsWaitSatisfied(const CommandBufferState::WaitAddressCommand& wait) {
const uint64_t value =
std::atomic_ref(*reinterpret_cast<uint64_t*>(wait.address)).load(std::memory_order_acquire);
switch (wait.compare) {
case 0: return value == wait.value;
case 1: return value > wait.value;
case 2: return value < wait.value;
case 3: return value != wait.value;
case 4: return static_cast<int64_t>(value - wait.value) >= 0;
case 5: return static_cast<int64_t>(value) > static_cast<int64_t>(wait.value);
default: return static_cast<int64_t>(value) < static_cast<int64_t>(wait.value);
}
}
class CommandEngine {
public:
explicit CommandEngine(size_t priorities)
: m_queues(priorities), m_worker([this](std::stop_token stop) { Run(stop); }) {}
~CommandEngine() {
std::scoped_lock lock(m_mutex);
m_worker.request_stop();
m_cv.notify_all();
}
void Submit(uint32_t priority, Submission submission) {
std::scoped_lock lock(m_mutex);
m_queues[priority].push_back(std::move(submission));
m_cv.notify_one();
}
private:
void Run(std::stop_token stop) {
std::unique_lock lock(m_mutex);
while (!stop.stop_requested()) {
std::deque<Submission>* ready = nullptr;
bool pending = false;
for (auto& queue: m_queues) {
if (queue.empty()) {
continue;
}
pending = true;
auto& submission = queue.front();
if (submission.cursor < submission.commands.size()) {
const auto* wait = std::get_if<CommandBufferState::WaitAddressCommand>(
&submission.commands[submission.cursor].data);
if (wait != nullptr && !IsWaitSatisfied(*wait)) {
continue;
}
}
ready = &queue;
break;
}
if (ready == nullptr) {
if (pending) {
// CPU and GPU fence writes do not notify the submission queue.
m_cv.wait_for(lock, std::chrono::microseconds(100));
} else {
m_cv.wait(lock);
}
continue;
}
auto& submission = ready->front();
lock.unlock();
int result = OK;
uint32_t error_offset = 0;
if (submission.cursor < submission.commands.size()) {
const auto& command = submission.commands[submission.cursor++];
result = ExecuteCommand(command);
if (result != OK) {
error_offset = static_cast<uint32_t>(command.record_offset);
}
}
const bool complete = result != OK || submission.cursor == submission.commands.size();
if (complete) {
AprShared::WriteResult(submission.result, result, error_offset);
AprShared::SetSubmissionComplete(submission.id);
}
lock.lock();
if (complete) {
ready->pop_front();
} }
} }
} }
std::mutex m_mutex;
std::condition_variable m_cv;
std::vector<std::deque<Submission>> m_queues;
std::jthread m_worker;
};
static int QueueCommandBuffer(Engine engine, uint64_t command_buffer, uint32_t bytes,
uint32_t priority, void* result, uint32_t* out_submission_id) {
const bool amm = engine == Engine::Amm;
if (command_buffer == 0 || priority > (amm ? 2u : 6u)) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
if ((result != nullptr &&
!AprShared::IsValidGuestRange(reinterpret_cast<uint64_t>(result),
sizeof(AprShared::ResultBuffer), true)) ||
(out_submission_id != nullptr &&
!AprShared::IsValidGuestRange(reinterpret_cast<uint64_t>(out_submission_id),
sizeof(uint32_t), true))) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
CommandBufferState state {};
if (!TryGetCommandBufferState(command_buffer, &state)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
const uint64_t end = amm ? bytes : state.write_offset;
if (end == 0 || end > state.write_offset) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
state.commands.erase(std::lower_bound(state.commands.begin(), state.commands.end(), end,
[](const auto& command, uint64_t offset) {
return command.record_offset < offset;
}),
state.commands.end());
const uint32_t id = out_submission_id != nullptr ? AprShared::AllocateSubmissionId() : 0;
if (out_submission_id != nullptr) {
AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id);
}
static CommandEngine apr_engine(7);
static CommandEngine amm_engine(3);
(amm ? amm_engine : apr_engine)
.Submit(priority, {std::move(state.commands), 0, result, id});
return OK; return OK;
} }
@@ -1882,11 +1914,20 @@ static int KYTY_SYSV_ABI CommandBufferPopMarker(void* command_buffer) {
return AppendNoOpCommand(command_buffer, sizeof(uint32_t)); return AppendNoOpCommand(command_buffer, sizeof(uint32_t));
} }
static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer, volatile uint64_t*, static int KYTY_SYSV_ABI CommandBufferWaitOnAddress(void* command_buffer,
uint64_t, uint8_t, uint8_t) { volatile uint64_t* address, uint64_t value,
uint8_t compare, uint8_t) {
PRINT_NAME(); PRINT_NAME();
if (command_buffer == nullptr || address == nullptr ||
return AppendNoOpCommand(command_buffer, 0x20); (reinterpret_cast<uint64_t>(address) & 7) != 0 || compare > 6) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer),
CommandBufferState::WaitAddressCommand {
reinterpret_cast<uint64_t>(address), value, compare},
0x20)
? OK
: LibKernel::KERNEL_ERROR_EBUSY;
} }
static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_t, uint8_t, static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_t, uint8_t,
@@ -1898,11 +1939,14 @@ static int KYTY_SYSV_ABI CommandBufferWaitOnCounter(void* command_buffer, uint8_
static int AppendWriteAddressCommand(void* command_buffer, volatile uint64_t* address, static int AppendWriteAddressCommand(void* command_buffer, volatile uint64_t* address,
uint64_t value) { uint64_t value) {
if (command_buffer == nullptr || address == nullptr) { if (command_buffer == nullptr || address == nullptr ||
(reinterpret_cast<uint64_t>(address) & 7) != 0) {
return LibKernel::KERNEL_ERROR_EINVAL; return LibKernel::KERNEL_ERROR_EINVAL;
} }
return AppendWriteAddressRecord(reinterpret_cast<uint64_t>(command_buffer), return AppendCommandRecord(
reinterpret_cast<uint64_t>(address), value) reinterpret_cast<uint64_t>(command_buffer),
CommandBufferState::WriteAddressCommand {reinterpret_cast<uint64_t>(address), value},
0x20)
? OK ? OK
: LibKernel::KERNEL_ERROR_EBUSY; : LibKernel::KERNEL_ERROR_EBUSY;
} }
@@ -2160,7 +2204,7 @@ static int KYTY_SYSV_ABI AmmCommandBufferMap(void* command_buffer, uint64_t va,
command.type = type; command.type = type;
command.prot = prot; command.prot = prot;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command, return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_RECORD_SIZE) AMM_MAP_RECORD_SIZE)
? OK ? OK
: LibKernel::KERNEL_ERROR_EBUSY; : LibKernel::KERNEL_ERROR_EBUSY;
@@ -2183,7 +2227,7 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapWithGpuMaskId(void* command_buffer,
command.prot = prot; command.prot = prot;
command.gpu_mask_id = gpu_mask_id; command.gpu_mask_id = gpu_mask_id;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command, return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_RECORD_SIZE) AMM_MAP_RECORD_SIZE)
? OK ? OK
: LibKernel::KERNEL_ERROR_EBUSY; : LibKernel::KERNEL_ERROR_EBUSY;
@@ -2207,7 +2251,7 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapDirect(void* command_buffer, uint64_
command.type = type; command.type = type;
command.prot = prot; command.prot = prot;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command, return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_DIRECT_RECORD_SIZE) AMM_MAP_DIRECT_RECORD_SIZE)
? OK ? OK
: LibKernel::KERNEL_ERROR_EBUSY; : LibKernel::KERNEL_ERROR_EBUSY;
@@ -2233,7 +2277,7 @@ static int KYTY_SYSV_ABI AmmCommandBufferMapDirectWithGpuMaskId(void* command_bu
command.prot = prot; command.prot = prot;
command.gpu_mask_id = gpu_mask_id; command.gpu_mask_id = gpu_mask_id;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command, return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_MAP_DIRECT_RECORD_SIZE) AMM_MAP_DIRECT_RECORD_SIZE)
? OK ? OK
: LibKernel::KERNEL_ERROR_EBUSY; : LibKernel::KERNEL_ERROR_EBUSY;
@@ -2251,7 +2295,7 @@ static int KYTY_SYSV_ABI AmmCommandBufferUnmap(void* command_buffer, uint64_t va
command.va = va; command.va = va;
command.size = size; command.size = size;
return AppendAmmMapRecord(reinterpret_cast<uint64_t>(command_buffer), command, return AppendCommandRecord(reinterpret_cast<uint64_t>(command_buffer), command,
AMM_UNMAP_RECORD_SIZE) AMM_UNMAP_RECORD_SIZE)
? OK ? OK
: LibKernel::KERNEL_ERROR_EBUSY; : LibKernel::KERNEL_ERROR_EBUSY;
@@ -2316,90 +2360,36 @@ static int KYTY_SYSV_ABI AmmSetPageTablePoolOccupancyNotificationThreshold(uint3
return OK; return OK;
} }
static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t, uint32_t) { static int KYTY_SYSV_ABI AmmSubmitCommandBuffer(void* command_buffer_base, uint32_t bytes,
uint32_t priority) {
PRINT_NAME(); PRINT_NAME();
return QueueCommandBuffer(Engine::Amm, reinterpret_cast<uint64_t>(command_buffer_base), bytes,
if (command_buffer_base == nullptr) { priority, nullptr, nullptr);
return LibKernel::KERNEL_ERROR_EINVAL;
}
int32_t execution_result = OK;
uint32_t error_offset = 0;
return ExecuteAprCommandBuffer(reinterpret_cast<uint64_t>(command_buffer_base),
&execution_result, &error_offset);
} }
static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t, static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetId(void* command_buffer_base, uint32_t bytes,
uint32_t, uint32_t* out_submission_id) { uint32_t priority,
PRINT_NAME();
if (command_buffer_base == nullptr || out_submission_id == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL;
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer_base);
const auto id = AprShared::AllocateSubmissionId(command_buffer_addr, 0);
int32_t execution_result = OK;
uint32_t error_offset = 0;
auto submit_result =
ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset);
if (submit_result != OK) {
return submit_result;
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
return OK;
}
static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base, uint32_t,
uint32_t, void* result,
uint32_t* out_submission_id) { uint32_t* out_submission_id) {
PRINT_NAME(); PRINT_NAME();
if (out_submission_id == nullptr) {
if (command_buffer_base == nullptr) {
return LibKernel::KERNEL_ERROR_EINVAL; return LibKernel::KERNEL_ERROR_EINVAL;
} }
return QueueCommandBuffer(Engine::Amm, reinterpret_cast<uint64_t>(command_buffer_base), bytes,
priority, nullptr, out_submission_id);
}
const auto command_buffer_addr = reinterpret_cast<uint64_t>(command_buffer_base); static int KYTY_SYSV_ABI AmmSubmitCommandBufferAndGetResult(void* command_buffer_base,
const auto id = uint32_t bytes, uint32_t priority,
AprShared::AllocateSubmissionId(command_buffer_addr, reinterpret_cast<uint64_t>(result)); void* result,
uint32_t* out_submission_id) {
int32_t execution_result = OK; PRINT_NAME();
uint32_t error_offset = 0; return QueueCommandBuffer(Engine::Amm, reinterpret_cast<uint64_t>(command_buffer_base), bytes,
auto submit_result = priority, result, out_submission_id);
ExecuteAprCommandBuffer(command_buffer_addr, &execution_result, &error_offset);
if (submit_result != OK) {
return submit_result;
}
AprShared::SetSubmissionResult(id, execution_result, error_offset);
if (out_submission_id != nullptr &&
!AprShared::WriteGuest(reinterpret_cast<uint64_t>(out_submission_id), id)) {
return LibKernel::KERNEL_ERROR_EFAULT;
}
return AprShared::WriteResult(result, execution_result, error_offset);
} }
static int KYTY_SYSV_ABI AmmWaitCommandBufferCompletion(uint32_t submission_id) { static int KYTY_SYSV_ABI AmmWaitCommandBufferCompletion(uint32_t submission_id) {
PRINT_NAME(); PRINT_NAME();
return AprShared::CompleteSubmission(submission_id) ? OK : LibKernel::KERNEL_ERROR_ESRCH;
AprShared::SubmissionState state {};
if (!AprShared::CompleteSubmission(submission_id, &state)) {
return LibKernel::KERNEL_ERROR_ESRCH;
}
if (state.result != 0) {
return AprShared::WriteResult(reinterpret_cast<void*>(state.result), state.execution_result,
state.error_offset);
}
return OK;
} }
} // namespace Ampr } // namespace Ampr
tests/KernelFileSystemTests.cpp
+125
View File
@@ -17,6 +17,7 @@
#include <algorithm> #include <algorithm>
#include <array> #include <array>
#include <atomic>
#include <chrono> #include <chrono>
#include <csignal> #include <csignal>
@@ -34,6 +35,10 @@ namespace Libs::LibKernelApr {
void InitLibKernel_1_Apr(Loader::SymbolDatabase *symbols); void InitLibKernel_1_Apr(Loader::SymbolDatabase *symbols);
} }
namespace Libs::LibAmpr {
void InitAmpr_1(Loader::SymbolDatabase *symbols);
}
namespace Libs::LibNet { namespace Libs::LibNet {
void InitNet_1_Net(Loader::SymbolDatabase *symbols); void InitNet_1_Net(Loader::SymbolDatabase *symbols);
} }
@@ -496,6 +501,125 @@ void CheckDirectoryStream(const std::filesystem::path &root) {
FileSystem::Umount("/app0"); FileSystem::Umount("/app0");
} }
void CheckAmprOrdering(Loader::SymbolDatabase &symbols, uint32_t file_id) {
Libs::LibAmpr::InitAmpr_1(&symbols);
const auto find = [&](const char *nid) {
const auto *symbol = symbols.FindByNid(nid, Loader::SymbolType::Func);
Check(symbol != nullptr, "AMPR command and submission exports resolve");
return symbol->vaddr;
};
using Unary = void (KYTY_SYSV_ABI *)(void *);
using AprConstructor = void (KYTY_SYSV_ABI *)(void *, void *, void *);
using SetBuffer = int (KYTY_SYSV_ABI *)(void *, void *, uint32_t);
using Offset = uint64_t (KYTY_SYSV_ABI *)(void *);
using WaitAddress = int (KYTY_SYSV_ABI *)(void *, volatile uint64_t *, uint64_t,
uint8_t, uint8_t);
using WriteAddress = int (KYTY_SYSV_ABI *)(void *, volatile uint64_t *, uint64_t);
using ReadFile = int (KYTY_SYSV_ABI *)(void *, uint64_t, uint64_t, uint32_t,
void *, uint64_t, uint64_t);
struct Result { int32_t result; uint32_t error_offset; };
using SubmitApr = int (KYTY_SYSV_ABI *)(void *, uint32_t, Result *, uint32_t *);
using SubmitAmm = int (KYTY_SYSV_ABI *)(void *, uint32_t, uint32_t, uint32_t *);
using WaitSubmission = int (KYTY_SYSV_ABI *)(uint32_t);
const auto construct = reinterpret_cast<Unary>(find("8aI7R7WaOlc"));
const auto construct_apr = reinterpret_cast<AprConstructor>(find("a8uLzYY--tM"));
const auto construct_amm = reinterpret_cast<Unary>(find("EDq5bqCqYpA"));
const auto destroy = reinterpret_cast<Unary>(find("GuchCTefuZw"));
const auto set_buffer = reinterpret_cast<SetBuffer>(find("N-FSPA4S3nI"));
const auto offset = reinterpret_cast<Offset>(find("GnxKOHEawhk"));
const auto wait_address = reinterpret_cast<WaitAddress>(find("DLfoNxTFNVk"));
const auto write_address = reinterpret_cast<WriteAddress>(find("sJXyWHjP-F8"));
const auto read_file = reinterpret_cast<ReadFile>(find("mQ16-QdKv7k"));
const auto submit_apr = reinterpret_cast<SubmitApr>(find("ASoW5WE-UPo"));
const auto submit_amm = reinterpret_cast<SubmitAmm>(find("NnKhlMJtIsI"));
const auto wait_apr = reinterpret_cast<WaitSubmission>(find("rqwFKI4PAiM"));
const auto wait_amm = reinterpret_cast<WaitSubmission>(find("HXymib4T8gc"));
struct Buffer {
std::array<uint64_t, 5> header {};
std::array<uint32_t, 256> data {};
};
std::array<Buffer, 4> buffers;
const auto reset = [&](size_t apr_count) {
for (size_t i = 0; i < buffers.size(); ++i) {
auto &buffer = buffers[i];
construct(buffer.header.data());
if (i < apr_count) {
construct_apr(buffer.header.data(), &buffer.header[3], &buffer.header[4]);
} else {
construct_amm(buffer.header.data());
}
Check(set_buffer(buffer.header.data(), buffer.data.data(), sizeof(buffer.data)) == OK,
"initialize AMPR command buffer");
}
};
// SDK WaitCompare order: ==, unsigned >/<, !=, wrapped >=, signed >/<.
struct Comparison { uint8_t compare; uint64_t blocked, reference, released; };
constexpr std::array comparisons {
Comparison{0, 1, 2, 2}, Comparison{1, 0x40000000000019c3, 0x40000000000019c3,
0x40000000000019c4},
Comparison{1, 0, INT64_MAX, uint64_t{1} << 63},
Comparison{2, UINT64_MAX, uint64_t{1} << 63, INT64_MAX}, Comparison{3, 2, 2, 3},
Comparison{4, UINT64_MAX - 1, UINT64_MAX, 0},
Comparison{5, UINT64_MAX, 0, 1}, Comparison{6, 0, 0, UINT64_MAX}};
for (const auto &comparison : comparisons) {
reset(3);
uint64_t fence = comparison.blocked;
uint64_t blocked_done = 0, read_done = 0, lower_done = 0;
std::array<char, 3> output {};
std::array<Result, 3> results {{{1234, 5678}, {1234, 5678}, {1234, 5678}}};
std::array<uint32_t, 4> ids {};
auto *blocked = buffers[0].header.data();
auto *reader = buffers[1].header.data();
auto *lower = buffers[2].header.data();
auto *producer = buffers[3].header.data();
Check(wait_address(blocked, &fence, comparison.reference, comparison.compare, 0) == OK &&
write_address(blocked, &blocked_done, 1) == OK &&
read_file(reader, reinterpret_cast<uint64_t>(&buffers[1].header[3]),
reinterpret_cast<uint64_t>(&buffers[1].header[4]), file_id,
output.data(), output.size(), 0) == OK &&
write_address(reader, &read_done, 2) == OK &&
write_address(lower, &lower_done, 3) == OK &&
write_address(producer, &fence, comparison.released) == OK,
"build dependent APR read and later AMM fence producer");
Check(submit_apr(blocked, 3, &results[0], &ids[0]) == OK &&
submit_apr(reader, 3, &results[1], &ids[1]) == OK &&
submit_apr(lower, 4, &results[2], &ids[2]) == OK && wait_apr(ids[2]) == OK,
"APR submit returns while blocked and a lower priority completes");
Check(lower_done == 3 && blocked_done == 0 && read_done == 0 &&
output == std::array<char, 3>{} && results[0].result == 1234 &&
results[1].result == 1234,
"unsatisfied wait blocks later buffers at its priority without publishing completion");
Check(submit_amm(buffers[3].data.data(), static_cast<uint32_t>(offset(producer)), 1,
&ids[3]) == OK && wait_amm(ids[3]) == OK &&
wait_apr(ids[0]) == OK && wait_apr(ids[1]) == OK,
"later AMM submission releases the APR dependency");
Check(blocked_done == 1 && read_done == 2 && std::memcmp(output.data(), "APR", 3) == 0,
"APR reads real file bytes only after its dependency completes");
for (const auto &result : results) {
Check(result.result == OK,
"submission wait observes the completed result");
}
}
reset(0);
uint64_t cpu_fence = 0, amm_done = 0, lower_done = 0;
std::array<uint32_t, 2> ids {};
Check(wait_address(buffers[0].header.data(), &cpu_fence, 0, 1, 0) == OK &&
write_address(buffers[0].header.data(), &amm_done, 1) == OK &&
write_address(buffers[1].header.data(), &lower_done, 1) == OK &&
submit_amm(buffers[0].data.data(), static_cast<uint32_t>(offset(buffers[0].header.data())),
0, &ids[0]) == OK &&
submit_amm(buffers[1].data.data(), static_cast<uint32_t>(offset(buffers[1].header.data())),
1, &ids[1]) == OK && wait_amm(ids[1]) == OK,
"AMM lower priority progresses while its high priority waits");
Check(lower_done == 1 && amm_done == 0, "AMM wait preserves its dependency");
std::atomic_ref(cpu_fence).store(1, std::memory_order_release);
Check(wait_amm(ids[0]) == OK && amm_done == 1,
"AMM observes an external CPU fence store without a submission notification");
for (auto &buffer : buffers) {
destroy(buffer.header.data());
}
}
void CheckAprPaths(const std::filesystem::path &root) { void CheckAprPaths(const std::filesystem::path &root) {
Loader::SymbolDatabase symbols; Loader::SymbolDatabase symbols;
Libs::LibKernelApr::InitLibKernel_1_Apr(&symbols); Libs::LibKernelApr::InitLibKernel_1_Apr(&symbols);
@@ -562,6 +686,7 @@ void CheckAprPaths(const std::filesystem::path &root) {
Check(resolve(unterminated.data(), paths, 1, ids, sizes, &error_index) == -1 && Check(resolve(unterminated.data(), paths, 1, ids, sizes, &error_index) == -1 &&
*Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_ENAMETOOLONG, *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_ENAMETOOLONG,
"APR rejects an unterminated prefix"); "APR rejects an unterminated prefix");
CheckAmprOrdering(symbols, expected_id);
FileSystem::Umount("/app0"); FileSystem::Umount("/app0");
} }