mirror of
https://github.com/KytyPS5/KytyPS5.git
synced 2026-10-29 16:51:20 +00:00
Compare commits
83
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62a0222e24 | ||
|
|
6f241bef49 | ||
|
|
21d7e8e1f0 | ||
|
|
49db3ea54c | ||
|
|
4b0de1b672 | ||
|
|
8e61798bfd | ||
|
|
6ee1bf9db0 | ||
|
|
0471c9991c | ||
|
|
b10bd53dd3 | ||
|
|
4cf456fb77 | ||
|
|
22ff4693c5 | ||
|
|
a107d1e333 | ||
|
|
05f92b6422 | ||
|
|
a892097ff0 | ||
|
|
c8baa7b909 | ||
|
|
34a3eb6b58 | ||
|
|
a20b64d3e5 | ||
|
|
2016f8e2fe | ||
|
|
c91fe80d64 | ||
|
|
ab2c1c4c03 | ||
|
|
0a5774be7b | ||
|
|
0791f921f6 | ||
|
|
421684e75f | ||
|
|
025b39c018 | ||
|
|
7c8583d204 | ||
|
|
9229c04ebf | ||
|
|
1b6af301b5 | ||
|
|
e51f252b75 | ||
|
|
0f1ff37f23 | ||
|
|
da33a1c4a3 | ||
|
|
f53f0ca483 | ||
|
|
7b6bfdec64 | ||
|
|
a88181b897 | ||
|
|
041874f663 | ||
|
|
496432bbb4 | ||
|
|
bec67b7863 | ||
|
|
a22a9d9348 | ||
|
|
81f312638f | ||
|
|
f1f930e06c | ||
|
|
e0b682e4b2 | ||
|
|
16b83a034c | ||
|
|
cf268af839 | ||
|
|
fd2e15ee73 | ||
|
|
4e8d8c849d | ||
|
|
1e438136fa | ||
|
|
313b60e922 | ||
|
|
cda236dbb3 | ||
|
|
1216fa36cf | ||
|
|
497166009c | ||
|
|
0fb3ac5605 | ||
|
|
e056d55dd5 | ||
|
|
1371219f5c | ||
|
|
cf423df624 | ||
|
|
5ce4f08364 | ||
|
|
7e1c2d1532 | ||
|
|
0ec43a8d3d | ||
|
|
bfdbba3aa4 | ||
|
|
f7ae6e0a83 | ||
|
|
3585cd8f03 | ||
|
|
ff54608569 | ||
|
|
6bb68a3fd9 | ||
|
|
93185953f1 | ||
|
|
7855c980cb | ||
|
|
5a705dd15f | ||
|
|
e73324c2e0 | ||
|
|
a20fb7f9f8 | ||
|
|
1cc0ac7ed8 | ||
|
|
83628a436e | ||
|
|
2165c710c3 | ||
|
|
b9a0bd2791 | ||
|
|
96874909ab | ||
|
|
d646fd92d1 | ||
|
|
d72a79af2d | ||
|
|
dc9280d75d | ||
|
|
6a9ae4ae43 | ||
|
|
392f39e2f6 | ||
|
|
cce0cd950c | ||
|
|
e43c242683 | ||
|
|
732057b072 | ||
|
|
2aaf0c9ba3 | ||
|
|
db7745ef41 | ||
|
|
02578f3c6b | ||
|
|
fcf0f3b67e |
No files matched your search
@@ -112,12 +112,16 @@ jobs:
|
||||
- name: Build
|
||||
shell: cmd
|
||||
run: |
|
||||
cmake --build _Build/windows --target launcher audio_out2_port_tests ime_dialog_tests virtual_memory_allocation_tests --parallel
|
||||
cmake --build _Build/windows --target launcher ^
|
||||
audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests ^
|
||||
shader_recompiler_compute_tests ^
|
||||
resource_tracking_tests scalar_provenance_tests ^
|
||||
resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests --parallel
|
||||
|
||||
- name: Test
|
||||
shell: cmd
|
||||
run: |
|
||||
ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|ime_dialog|virtual_memory_allocation)$"
|
||||
ctest --test-dir _Build/windows --output-on-failure -R "^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$"
|
||||
|
||||
- name: Install
|
||||
shell: cmd
|
||||
@@ -198,14 +202,17 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
cmake --build _Build/macos \
|
||||
--target launcher audio_out2_port_tests ime_dialog_tests virtual_memory_allocation_tests \
|
||||
--target launcher audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
|
||||
memory_tracker_tests page_manager_tests shader_recompiler_compute_tests \
|
||||
resource_tracking_tests scalar_provenance_tests \
|
||||
resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
|
||||
--parallel
|
||||
|
||||
- name: Test
|
||||
shell: bash
|
||||
run: |
|
||||
ctest --test-dir _Build/macos --output-on-failure \
|
||||
-R '^(audio_out2_port|ime_dialog|virtual_memory_allocation)$'
|
||||
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
|
||||
|
||||
- name: Install
|
||||
shell: bash
|
||||
@@ -379,14 +386,16 @@ jobs:
|
||||
run: |
|
||||
cmake --build _Build/linux \
|
||||
--target launcher page_manager_tests memory_tracker_tests \
|
||||
audio_out2_port_tests ime_dialog_tests virtual_memory_allocation_tests \
|
||||
audio_out2_port_tests pad_haptics_tests ime_dialog_tests lru_cache_tests virtual_memory_allocation_tests \
|
||||
resource_tracking_tests scalar_provenance_tests \
|
||||
resource_materialization_tests shader_vertex_metadata_tests archive_file_tests avplayer_file_tests \
|
||||
--parallel
|
||||
|
||||
- name: Test
|
||||
shell: bash
|
||||
run: |
|
||||
ctest --test-dir _Build/linux --output-on-failure \
|
||||
-R '^(audio_out2_port|ime_dialog|page_manager|memory_tracker|virtual_memory_allocation)$'
|
||||
-R '^(audio_out2_port|pad_haptics|ime_dialog|lru_cache|memory_tracker|page_manager|resource_tracking|scalar_provenance|resource_materialization|shader_vertex_metadata|archive_file|avplayer_file|virtual_memory_allocation)$'
|
||||
|
||||
- name: Install
|
||||
shell: bash
|
||||
|
||||
Vendored
+47
@@ -28,7 +28,54 @@ FetchContent_Declare(
|
||||
|
||||
FetchContent_MakeAvailable(xbyak zydis)
|
||||
|
||||
# ZArchive provides random-access reads for compressed game dumps. Build only
|
||||
# the reader/writer library and its zstd dependency; the command-line tool is
|
||||
# not part of KytyPS5's runtime distribution.
|
||||
set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE)
|
||||
set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE)
|
||||
set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE)
|
||||
set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
||||
set(ZSTD_LEGACY_SUPPORT OFF CACHE BOOL "" FORCE)
|
||||
set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "" FORCE)
|
||||
FetchContent_Declare(
|
||||
zstd
|
||||
GIT_REPOSITORY https://github.com/facebook/zstd.git
|
||||
GIT_TAG f8745da6ff1ad1e7bab384bd1f9d742439278e99
|
||||
GIT_SHALLOW FALSE
|
||||
SOURCE_SUBDIR build/cmake
|
||||
)
|
||||
FetchContent_Declare(
|
||||
zarchive_source
|
||||
GIT_REPOSITORY https://github.com/Exzap/ZArchive.git
|
||||
GIT_TAG d2c717730092c7bf8cbb033b12fd4001b7c4d932
|
||||
GIT_SHALLOW FALSE
|
||||
PATCH_COMMAND "${GIT_EXECUTABLE}" apply --ignore-whitespace
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/patches/zarchive-reader.patch"
|
||||
SOURCE_SUBDIR kyty-library-only
|
||||
)
|
||||
FetchContent_MakeAvailable(zstd zarchive_source)
|
||||
|
||||
add_library(zarchive STATIC
|
||||
"${zarchive_source_SOURCE_DIR}/src/zarchivereader.cpp"
|
||||
"${zarchive_source_SOURCE_DIR}/src/zarchivewriter.cpp"
|
||||
"${zarchive_source_SOURCE_DIR}/src/sha_256.c"
|
||||
)
|
||||
add_library(ZArchive::zarchive ALIAS zarchive)
|
||||
target_include_directories(zarchive SYSTEM PUBLIC "${zarchive_source_SOURCE_DIR}/include")
|
||||
target_include_directories(zarchive PRIVATE "${zstd_SOURCE_DIR}/lib")
|
||||
target_link_libraries(zarchive PRIVATE libzstd_static)
|
||||
target_compile_features(zarchive PRIVATE cxx_std_20)
|
||||
if(MSVC)
|
||||
target_compile_options(zarchive PRIVATE /W0)
|
||||
else()
|
||||
target_compile_options(zarchive PRIVATE -w)
|
||||
endif()
|
||||
|
||||
set(SDL_SHARED OFF CACHE BOOL "" FORCE)
|
||||
if(APPLE)
|
||||
# SDL's built-in iconv fallback does not support CP932.
|
||||
set(SDL_SYSTEM_ICONV ON CACHE BOOL "Use system iconv for CP932 conversion" FORCE)
|
||||
endif()
|
||||
add_subdirectory(SDL3)
|
||||
|
||||
add_library(Vulkan-Headers INTERFACE)
|
||||
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
--- a/include/zarchive/zarchivecommon.h
|
||||
+++ b/include/zarchive/zarchivecommon.h
|
||||
@@ -256,7 +256,7 @@
|
||||
|
||||
bool IsWithinValidRange(uint64_t fileSize) const
|
||||
{
|
||||
- return (offset + size) <= fileSize;
|
||||
+ return offset <= fileSize && size <= fileSize - offset;
|
||||
}
|
||||
};
|
||||
|
||||
--- a/src/zarchivereader.cpp
|
||||
+++ b/src/zarchivereader.cpp
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
static bool _ifstream_readBytes(std::ifstream& file, uint64_t offset, void* buffer, uint32_t size)
|
||||
{
|
||||
+ file.clear();
|
||||
file.seekg(offset, std::ios_base::beg);
|
||||
file.read((char*)buffer, size);
|
||||
return file.gcount() == size;
|
||||
@@ -80,9 +81,27 @@
|
||||
// verify file tree
|
||||
if (fileTree[0].IsFile())
|
||||
return nullptr; // first entry must be root directory
|
||||
- auto rootName = GetName(nameTable, fileTree[0].GetNameOffset());
|
||||
- if (!rootName.empty())
|
||||
+ if (fileTree[0].GetNameOffset() != 0x7FFFFFFF)
|
||||
return nullptr; // root node must not have a name
|
||||
+ const uint64_t dataCapacity = (uint64_t)offsetRecords.size() * _ZARCHIVE::ENTRIES_PER_OFFSETRECORD * _ZARCHIVE::COMPRESSED_BLOCK_SIZE;
|
||||
+ for (size_t i = 0; i < fileTree.size(); ++i)
|
||||
+ {
|
||||
+ auto& entry = fileTree[i];
|
||||
+ if (i != 0)
|
||||
+ {
|
||||
+ auto name = GetName(nameTable, entry.GetNameOffset());
|
||||
+ if (name.empty() || name == "." || name == ".." || name.find_first_of("/\\") != std::string_view::npos || name.find('\0') != std::string_view::npos)
|
||||
+ return nullptr;
|
||||
+ }
|
||||
+ if (entry.IsFile())
|
||||
+ {
|
||||
+ if (entry.GetFileOffset() > dataCapacity || entry.GetFileSize() > dataCapacity - entry.GetFileOffset())
|
||||
+ return nullptr;
|
||||
+ }
|
||||
+ else if (entry.directoryRecord.nodeStartIndex > fileTree.size() ||
|
||||
+ entry.directoryRecord.count > fileTree.size() - entry.directoryRecord.nodeStartIndex)
|
||||
+ return nullptr;
|
||||
+ }
|
||||
// read meta data
|
||||
// todo
|
||||
|
||||
@@ -238,7 +257,7 @@
|
||||
uint32_t stepSize = std::min<uint32_t>(remainingBytes, _ZARCHIVE::COMPRESSED_BLOCK_SIZE - blockOffset);
|
||||
CacheBlock* block = GetCachedBlock(blockIdx);
|
||||
if (!block)
|
||||
- return 0;
|
||||
+ return bytesToRead - remainingBytes;
|
||||
std::memcpy(bufferU8, block->data + blockOffset, stepSize);
|
||||
rawReadOffset += stepSize;
|
||||
remainingBytes -= stepSize;
|
||||
@@ -325,14 +344,18 @@
|
||||
// determine offset and size of compressed block
|
||||
auto& record = m_offsetRecords[recordIndex];
|
||||
uint64_t offset = record.baseOffset;
|
||||
+ if (offset > m_compressedDataSize)
|
||||
+ return false;
|
||||
for (uint32_t i = 0; i < recordSubIndex; i++)
|
||||
{
|
||||
- offset += (uint64_t)record.size[i];
|
||||
- offset++;
|
||||
+ const uint64_t size = (uint64_t)record.size[i] + 1;
|
||||
+ if (size > m_compressedDataSize - offset)
|
||||
+ return false;
|
||||
+ offset += size;
|
||||
}
|
||||
uint32_t compressedSize = (uint32_t)record.size[recordSubIndex] + 1;
|
||||
// load file data
|
||||
- if ((offset + compressedSize) > m_compressedDataSize)
|
||||
+ if (compressedSize > m_compressedDataSize - offset)
|
||||
return false;
|
||||
offset += m_compressedDataOffset;
|
||||
if (compressedSize == _ZARCHIVE::COMPRESSED_BLOCK_SIZE)
|
||||
@@ -350,7 +373,7 @@
|
||||
// returns empty view on failure
|
||||
std::string_view ZArchiveReader::GetName(const std::vector<uint8_t>& nameTable, uint32_t nameOffset)
|
||||
{
|
||||
- if (nameOffset == 0x7FFFFFFF || nameOffset > nameTable.size())
|
||||
+ if (nameOffset == 0x7FFFFFFF || nameOffset >= nameTable.size())
|
||||
return "";
|
||||
// parse header
|
||||
uint16_t nameLength = nameTable[nameOffset] & 0x7F;
|
||||
@@ -359,7 +382,7 @@
|
||||
// extended 2-byte length
|
||||
if (nameOffset + 1 >= nameTable.size())
|
||||
return "";
|
||||
- nameLength |= ((uint16_t)nameTable[nameOffset] << 7);
|
||||
+ nameLength |= ((uint16_t)nameTable[nameOffset + 1] << 7);
|
||||
nameOffset += 2;
|
||||
}
|
||||
else
|
||||
+68
-2
@@ -1,4 +1,4 @@
|
||||
cmake_minimum_required(VERSION 3.16...4.3)
|
||||
cmake_minimum_required(VERSION 3.18...4.3)
|
||||
|
||||
if(CMAKE_VERSION VERSION_GREATER_EQUAL 4.0)
|
||||
set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum policy version for bundled dependencies")
|
||||
@@ -301,6 +301,7 @@ set(gpu_blit_generated_dir "${PROJECT_BINARY_DIR}/gpu_blit_shaders")
|
||||
set(gpu_blit_shader_sources
|
||||
"${gpu_tiler_shader_dir}/gpu_blit_fs_triangle.vert"
|
||||
"${gpu_tiler_shader_dir}/gpu_blit_color_to_ms_depth.frag"
|
||||
"${gpu_tiler_shader_dir}/gpu_video_out_overlay.frag"
|
||||
)
|
||||
foreach(shader_source IN LISTS gpu_blit_shader_sources)
|
||||
get_filename_component(shader_name "${shader_source}" NAME_WE)
|
||||
@@ -406,11 +407,26 @@ add_executable(page_manager_tests EXCLUDE_FROM_ALL
|
||||
target_link_libraries(page_manager_tests common)
|
||||
target_include_directories(page_manager_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(archive_file_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/ArchiveFileTests.cpp"
|
||||
)
|
||||
target_link_libraries(archive_file_tests common ZArchive::zarchive)
|
||||
target_include_directories(archive_file_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(avplayer_file_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/AvPlayerFileTests.cpp")
|
||||
target_link_libraries(avplayer_file_tests common FFmpeg::ffmpeg ZArchive::zarchive)
|
||||
target_include_directories(avplayer_file_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(bit_array_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/BitArrayTests.cpp"
|
||||
)
|
||||
target_include_directories(bit_array_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(lru_cache_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/LruCacheTests.cpp"
|
||||
)
|
||||
target_include_directories(lru_cache_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(ime_dialog_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/ImeDialogTests.cpp"
|
||||
"${KYTY_SOURCE_DIR}/libs/imeCommon.cpp"
|
||||
@@ -465,12 +481,39 @@ add_executable(audio_out2_port_tests EXCLUDE_FROM_ALL
|
||||
target_link_libraries(audio_out2_port_tests common fmt::fmt)
|
||||
target_include_directories(audio_out2_port_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(ngs2_sampler_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/Ngs2SamplerTests.cpp"
|
||||
"${KYTY_SOURCE_DIR}/loader/timer.cpp"
|
||||
)
|
||||
target_link_libraries(ngs2_sampler_tests common fmt::fmt LibAtrac9)
|
||||
target_include_directories(ngs2_sampler_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(pad_haptics_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/PadHapticsTests.cpp"
|
||||
)
|
||||
target_link_libraries(pad_haptics_tests common fmt::fmt SDL3::SDL3-static)
|
||||
target_include_directories(pad_haptics_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(save_data_memory_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/SaveDataMemoryTests.cpp"
|
||||
)
|
||||
target_link_libraries(save_data_memory_tests common fmt::fmt)
|
||||
target_include_directories(save_data_memory_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(ces_tests EXCLUDE_FROM_ALL "${KYTY_TESTS_DIR}/CesTests.cpp")
|
||||
target_link_libraries(ces_tests common fmt::fmt SDL3::SDL3-static)
|
||||
target_include_directories(ces_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_executable(save_data_dialog_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/SaveDataDialogTests.cpp"
|
||||
"${KYTY_SOURCE_DIR}/libs/dialog.cpp"
|
||||
)
|
||||
target_link_libraries(save_data_dialog_tests common fmt::fmt)
|
||||
target_include_directories(save_data_dialog_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_kyty_full_emulator_test(http_uri_parse_tests
|
||||
"${KYTY_TESTS_DIR}/HttpUriParseTests.cpp")
|
||||
|
||||
add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/EventQueueLifetimeTests.cpp"
|
||||
"${KYTY_SOURCE_DIR}/kernel/eventQueue.cpp"
|
||||
@@ -481,6 +524,7 @@ target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
|
||||
|
||||
add_kyty_full_emulator_test(kernel_file_system_tests
|
||||
"${KYTY_TESTS_DIR}/KernelFileSystemTests.cpp")
|
||||
target_link_libraries(kernel_file_system_tests ZArchive::zarchive)
|
||||
|
||||
add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
|
||||
"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
|
||||
@@ -547,7 +591,10 @@ if(BUILD_TESTING)
|
||||
add_test(NAME image_page_table COMMAND $<TARGET_FILE:image_page_table_tests>)
|
||||
add_test(NAME memory_tracker COMMAND $<TARGET_FILE:memory_tracker_tests>)
|
||||
add_test(NAME page_manager COMMAND $<TARGET_FILE:page_manager_tests>)
|
||||
add_test(NAME archive_file COMMAND $<TARGET_FILE:archive_file_tests>)
|
||||
add_test(NAME avplayer_file COMMAND $<TARGET_FILE:avplayer_file_tests>)
|
||||
add_test(NAME bit_array COMMAND $<TARGET_FILE:bit_array_tests>)
|
||||
add_test(NAME lru_cache COMMAND $<TARGET_FILE:lru_cache_tests>)
|
||||
add_test(NAME shader_vertex_metadata COMMAND $<TARGET_FILE:shader_vertex_metadata_tests>)
|
||||
add_test(NAME resource_materialization COMMAND $<TARGET_FILE:resource_materialization_tests>)
|
||||
add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
|
||||
@@ -555,7 +602,12 @@ if(BUILD_TESTING)
|
||||
add_test(NAME kernel_file_system COMMAND $<TARGET_FILE:kernel_file_system_tests>)
|
||||
add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
|
||||
add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
|
||||
add_test(NAME ngs2_sampler COMMAND $<TARGET_FILE:ngs2_sampler_tests>)
|
||||
add_test(NAME pad_haptics COMMAND $<TARGET_FILE:pad_haptics_tests>)
|
||||
add_test(NAME save_data_memory COMMAND $<TARGET_FILE:save_data_memory_tests>)
|
||||
add_test(NAME ces COMMAND $<TARGET_FILE:ces_tests>)
|
||||
add_test(NAME save_data_dialog COMMAND $<TARGET_FILE:save_data_dialog_tests>)
|
||||
add_test(NAME http_uri_parse COMMAND $<TARGET_FILE:http_uri_parse_tests>)
|
||||
add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
|
||||
add_test(NAME shader_recompiler_alignbyte
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
|
||||
@@ -592,6 +644,8 @@ if(BUILD_TESTING)
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --compute-meta-clear-only)
|
||||
add_test(NAME buffer_cache_dirty_gc
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --buffer-cache-gc-only)
|
||||
add_test(NAME color_render_target_1d
|
||||
COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --rt1d-only)
|
||||
|
||||
if(WIN32)
|
||||
# These tests still depend on the Windows multisample-depth path.
|
||||
@@ -610,7 +664,10 @@ if(BUILD_TESTING)
|
||||
image_page_table_tests
|
||||
memory_tracker_tests
|
||||
page_manager_tests
|
||||
archive_file_tests
|
||||
avplayer_file_tests
|
||||
bit_array_tests
|
||||
lru_cache_tests
|
||||
shader_vertex_metadata_tests
|
||||
resource_materialization_tests
|
||||
resource_tracking_tests
|
||||
@@ -620,6 +677,12 @@ if(BUILD_TESTING)
|
||||
shader_recompiler_compute_tests
|
||||
virtual_memory_allocation_tests
|
||||
audio_out2_port_tests
|
||||
ngs2_sampler_tests
|
||||
pad_haptics_tests
|
||||
save_data_memory_tests
|
||||
ces_tests
|
||||
save_data_dialog_tests
|
||||
http_uri_parse_tests
|
||||
)
|
||||
endif()
|
||||
|
||||
@@ -677,7 +740,10 @@ if(KYTY_CLANG_CL)
|
||||
)
|
||||
add_custom_command(TARGET kyty_emulator POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${KYTY_THIRD_PARTY_DIR}/winpthread/bin/libwinpthread-1.dll" $<TARGET_FILE_DIR:kyty_emulator>/libwinpthread-1.dll)
|
||||
elseif(WIN32 OR LINUX)
|
||||
set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS} -Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
|
||||
set_target_properties(kyty_emulator PROPERTIES LINK_FLAGS "${KYTY_LD_OPTIONS}")
|
||||
# LINK_FLAGS is a space-delimited string: a -Map path containing spaces would
|
||||
# be split into bogus arguments, so pass it as a discrete link option instead.
|
||||
target_link_options(kyty_emulator PRIVATE "-Wl,-Map=${KYTY_EMULATOR_MAP_LINK_PATH}")
|
||||
endif()
|
||||
|
||||
if(LINUX)
|
||||
|
||||
@@ -4,13 +4,17 @@
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
|
||||
[](https://github.com/KytyPS5/KytyPS5/actions/workflows/build.yml)
|
||||
[](#system-requirements)
|
||||
[](#current-status)
|
||||
[](#current-status)
|
||||
[](LICENSE)
|
||||
|
||||
**[Weekly updates](https://github.com/KytyPS5/KytyPS5/discussions/862)** — game progress, recent fixes and ongoing development.
|
||||
|
||||
**[Development on Discord](https://discord.gg/UNrkMqGaBg)** — KytyPS5 development.
|
||||
|
||||
KytyPS5 is a free and open-source PlayStation 5 emulator written in C++ for Windows and Linux,
|
||||
with experimental macOS support. It is based on a heavily modified version of
|
||||
[Kyty](https://github.com/InoriRus/Kyty). The project is in an early stage of development, so
|
||||
compatibility is limited and behavior may change significantly between builds.
|
||||
[Kyty](https://github.com/InoriRus/Kyty). The project is in active development, and behavior
|
||||
can change significantly between builds.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> KytyPS5 is not affiliated with Sony Interactive Entertainment or PlayStation. The project does
|
||||
@@ -20,12 +24,12 @@ compatibility is limited and behavior may change significantly between builds.
|
||||
## Current Status
|
||||
|
||||
KytyPS5 can boot 2D games and a selection of 3D games, including titles built with Unreal Engine
|
||||
4/5, Unity, and custom engines. No external low-level emulation modules are currently required.
|
||||
4/5, Unity, and custom engines. External low-level emulation modules are neither required nor
|
||||
planned.
|
||||
|
||||
Development is focused on compatibility and boot reliability.
|
||||
Development is currently focused on expanding game compatibility and improving boot reliability.
|
||||
|
||||
Windows is the primary platform and receives the most testing. Linux builds and runs; see
|
||||
[Building on Linux](#building-on-linux).
|
||||
Windows and Linux are the primary platforms and receive the most testing.
|
||||
|
||||
macOS support is experimental. The emulator is built for x86-64 and runs on Apple Silicon under
|
||||
Rosetta 2, with Vulkan provided by MoltenVK. A small number of titles have been verified in-game
|
||||
@@ -36,16 +40,16 @@ Community game test results are available in the
|
||||
|
||||
## Bugs and Issues
|
||||
|
||||
The project is in an early stage, so please be mindful when opening new issues. Expect crashes,
|
||||
graphical glitches, low compatibility, and poor performance.
|
||||
Compatibility, stability, and performance can vary between versions. You may encounter crashes
|
||||
or graphical glitches, so please include the version you tested when reporting an issue.
|
||||
|
||||
## Screenshots
|
||||
|
||||
<table align="center">
|
||||
<tr>
|
||||
<td align="center">
|
||||
<strong>Disgaea 6</strong><br>
|
||||
<img src="docs/screenshots/ps5-01.png" width="300" alt="Disgaea 6 running in KytyPS5">
|
||||
<strong>Astro Bot</strong><br>
|
||||
<img src="docs/screenshots/ps5-01.png" width="300" alt="Astro Bot running in KytyPS5">
|
||||
</td>
|
||||
<td align="center">
|
||||
<strong>Dreaming Sarah</strong><br>
|
||||
@@ -64,12 +68,12 @@ graphical glitches, low compatibility, and poor performance.
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center">
|
||||
<strong>Hellboy</strong><br>
|
||||
<img src="docs/screenshots/ps5-02.png" width="300" alt="Hellboy running in KytyPS5">
|
||||
<strong>Demon's Souls</strong><br>
|
||||
<img src="docs/screenshots/ps5-02.png" width="300" alt="Demon's Souls running in KytyPS5">
|
||||
</td>
|
||||
<td align="center">
|
||||
<strong>Paleo Pines</strong><br>
|
||||
<img src="docs/screenshots/ps5-06.png" width="300" alt="Paleo Pines running in KytyPS5">
|
||||
<strong>UFC 5</strong><br>
|
||||
<img src="docs/screenshots/ps5-06.png" width="300" alt="UFC 5 running in KytyPS5">
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
@@ -79,7 +83,7 @@ graphical glitches, low compatibility, and poor performance.
|
||||
## Contributing
|
||||
|
||||
Testing games and submitting detailed bug reports are useful ways to contribute. Search existing
|
||||
issues first, then use the **Game Emulation Bug Report** template and attach the complete log file.
|
||||
issues first, then use the **Game Emulation Status Report** template and attach the complete log file.
|
||||
|
||||
Code contributions should be focused, build successfully on the platforms they touch, and include
|
||||
relevant tests where practical. Windows is the primary target, so a change that alters shared code
|
||||
@@ -132,7 +136,7 @@ the Vulkan/SPIR-V validation rules.
|
||||
### Build requirements (Windows)
|
||||
|
||||
- Git
|
||||
- CMake 3.12 or newer
|
||||
- CMake 3.22.1 or newer
|
||||
- Ninja
|
||||
- Visual Studio 2022 or Build Tools 2022 with the **Desktop development with C++** workload and
|
||||
**C++ Clang tools for Windows** component
|
||||
@@ -178,7 +182,7 @@ sudo apt-get install --no-install-recommends \
|
||||
libasound2-dev libpulse-dev libudev-dev libdbus-1-dev libwayland-dev wayland-protocols
|
||||
```
|
||||
|
||||
Qt 6 (Concurrent, Network, Widgets) is also required — either the distribution packages
|
||||
Qt 6 (Concurrent, Network, Widgets) is required for the launcher — either the distribution packages
|
||||
(`qt6-base-dev`) or an official Qt installation.
|
||||
|
||||
```bash
|
||||
@@ -197,6 +201,18 @@ The install step copies the Qt libraries and plugins next to the binaries, so
|
||||
from the pinned [KytyPS5 FFmpeg core](https://github.com/KytyPS5/ext-ffmpeg-core)
|
||||
release, including VP9 and WebM support. System FFmpeg packages are not required.
|
||||
|
||||
To build `kyty_emulator` and the `kyty_tests` target without Qt, use a separate build directory:
|
||||
|
||||
```bash
|
||||
git submodule update --init --recursive
|
||||
|
||||
cmake -S . -B _Build/linux-no-qt -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
|
||||
-DKYTY_BUILD_LAUNCHER=OFF
|
||||
|
||||
cmake --build _Build/linux-no-qt --target kyty_emulator kyty_tests --parallel
|
||||
```
|
||||
|
||||
As on Windows, the MSVC compiler is not used; Clang is required. `cl.exe` is rejected at configure
|
||||
time.
|
||||
|
||||
@@ -220,9 +236,10 @@ cmake --build _Build/linux --target launcher --parallel
|
||||
cmake --install _Build/linux --prefix _Build/linux/install
|
||||
```
|
||||
|
||||
The configure step downloads the FFmpeg prebuilts and the `xbyak`/`zydis` sources, so it needs
|
||||
network access; a fully sandboxed `nix build` would require vendoring those inputs. A Vulkan 1.3
|
||||
driver must be available at runtime (on NixOS, `hardware.graphics.enable = true`).
|
||||
The configure step downloads the FFmpeg prebuilts and the `xbyak`, `zydis`, `zstd`, and ZArchive
|
||||
sources, so it needs network access; a fully sandboxed `nix build` would require vendoring those
|
||||
inputs. A Vulkan 1.3 driver must be available at runtime (on NixOS,
|
||||
`hardware.graphics.enable = true`).
|
||||
|
||||
### Building on macOS
|
||||
|
||||
@@ -316,17 +333,21 @@ open _Build/macos/install/KytyPS5.app # or double-click in Finder
|
||||
```
|
||||
|
||||
On first launch, add one or more game folders in the global settings. The launcher searches those
|
||||
folders recursively for game directories containing `eboot.bin`. Select a detected game and run it
|
||||
from the game list.
|
||||
folders recursively for game directories containing `eboot.bin` and ZArchive (`.zar`) game dumps
|
||||
whose archive root contains `eboot.bin`. Select a detected game and run it from the game list.
|
||||
ZArchive dumps are mounted read-only and streamed directly; they do not need to be extracted first.
|
||||
|
||||
The emulator can also be started directly with a legally obtained game directory or ELF file:
|
||||
The emulator can also be started directly with a legally obtained game directory, ELF file, or
|
||||
ZArchive dump:
|
||||
|
||||
```powershell
|
||||
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame"
|
||||
.\_Build\windows\install\kyty_emulator.exe --game "D:\Games\ExampleGame.zar"
|
||||
```
|
||||
|
||||
```bash
|
||||
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame"
|
||||
./_Build/linux/install/kyty_emulator --game "/games/ExampleGame.zar"
|
||||
```
|
||||
|
||||
On macOS, the adjacent flat or app-bundled `libMoltenVK.dylib` is found automatically; no
|
||||
@@ -370,5 +391,6 @@ licenses included with those components.
|
||||
|
||||
- [InoriRus/Kyty](https://github.com/InoriRus/Kyty) — KytyPS5 is based on a heavily modified version
|
||||
of the original Kyty project.
|
||||
- [shadps4-emu/shadPS4](https://github.com/shadps4-emu/shadPS4) — reference for memory-model
|
||||
understanding and the AVPlayer implementation.
|
||||
- [shadps4-emu/shadPS4](https://github.com/shadps4-emu/shadPS4) — reference for understanding PS4
|
||||
memory behavior, GPU resource aliasing and cache coherency,
|
||||
and the AVPlayer implementation.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 MiB After Width: | Height: | Size: 1.9 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 MiB After Width: | Height: | Size: 1.8 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 2.4 MiB After Width: | Height: | Size: 1.7 MiB |
@@ -26,6 +26,7 @@ target_link_libraries(common
|
||||
cpuinfo
|
||||
kyty_git_version
|
||||
spdlog::spdlog
|
||||
ZArchive::zarchive
|
||||
)
|
||||
|
||||
get_property(inc_headers TARGET common PROPERTY INCLUDE_DIRECTORIES)
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
#include "common/archive.h"
|
||||
|
||||
#include "common/stringUtils.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <condition_variable>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <zarchive/zarchivereader.h>
|
||||
|
||||
// POSIX hosts and MinGW's pthread-backed libstdc++ have cancellable condition waits.
|
||||
// MSVC/ClangCL and MinGW's native Win32 threading backend do not use pthread waits.
|
||||
#if !defined(_WIN32) || defined(_GLIBCXX_GCC_GTHR_POSIX_H)
|
||||
#include <pthread.h>
|
||||
#define KYTY_ARCHIVE_PTHREAD_CANCELLATION
|
||||
#endif
|
||||
|
||||
namespace Common {
|
||||
|
||||
// Backends expose immutable metadata and offset-based reads. Cursor management, path routing,
|
||||
// reader lifetime and the host-stack worker are shared by every archive format.
|
||||
class ArchiveReader {
|
||||
public:
|
||||
struct Entry {
|
||||
uint64_t id;
|
||||
uint64_t size;
|
||||
bool is_file;
|
||||
};
|
||||
|
||||
virtual ~ArchiveReader() = default;
|
||||
virtual std::optional<Entry> Find(std::string_view member) = 0;
|
||||
virtual std::vector<File::DirEntry> List(std::string_view member) = 0;
|
||||
virtual uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) = 0;
|
||||
};
|
||||
|
||||
namespace {
|
||||
|
||||
class ZArchiveReaderBackend final: public ArchiveReader {
|
||||
public:
|
||||
explicit ZArchiveReaderBackend(std::unique_ptr<ZArchiveReader> reader)
|
||||
: m_reader(std::move(reader)) {}
|
||||
|
||||
std::optional<Entry> Find(std::string_view member) override {
|
||||
const auto node = m_reader->LookUp(member);
|
||||
if (node == ZARCHIVE_INVALID_NODE) {
|
||||
return {};
|
||||
}
|
||||
return Entry {node, m_reader->GetFileSize(node), m_reader->IsFile(node)};
|
||||
}
|
||||
|
||||
std::vector<File::DirEntry> List(std::string_view member) override {
|
||||
const auto node = m_reader->LookUp(member);
|
||||
if (!m_reader->IsDirectory(node)) {
|
||||
return {};
|
||||
}
|
||||
const auto count = m_reader->GetDirEntryCount(node);
|
||||
if (count > (1u << 20u)) {
|
||||
return {};
|
||||
}
|
||||
std::vector<File::DirEntry> result;
|
||||
result.reserve(count);
|
||||
for (uint32_t index = 0; index < count; ++index) {
|
||||
ZArchiveReader::DirEntry entry {};
|
||||
if (m_reader->GetDirEntry(node, index, entry)) {
|
||||
result.push_back({std::string(entry.name), entry.isFile});
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
uint64_t Read(uint64_t id, uint64_t offset, uint32_t size, void* data) override {
|
||||
// ZArchiveReader already serializes access to its stream and decompression cache.
|
||||
return m_reader->ReadFromFile(static_cast<ZArchiveNodeHandle>(id), offset, size, data);
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<ZArchiveReader> m_reader;
|
||||
};
|
||||
|
||||
std::shared_ptr<ArchiveReader> OpenZArchive(const std::filesystem::path& path) {
|
||||
auto reader = std::unique_ptr<ZArchiveReader>(ZArchiveReader::OpenFromFile(path));
|
||||
return reader ? std::make_shared<ZArchiveReaderBackend>(std::move(reader)) : nullptr;
|
||||
}
|
||||
|
||||
struct ArchiveFormat {
|
||||
std::string_view extension;
|
||||
std::shared_ptr<ArchiveReader> (*open)(const std::filesystem::path&);
|
||||
};
|
||||
|
||||
constexpr ArchiveFormat Formats[] = {{".zar", OpenZArchive}};
|
||||
using NativeView = std::basic_string_view<std::filesystem::path::value_type>;
|
||||
|
||||
const ArchiveFormat* FindFormat(NativeView path) {
|
||||
for (const auto& format: Formats) {
|
||||
if (path.size() < format.extension.size()) {
|
||||
continue;
|
||||
}
|
||||
const auto extension = path.substr(path.size() - format.extension.size());
|
||||
if (std::equal(
|
||||
extension.begin(), extension.end(), format.extension.begin(),
|
||||
[](auto a, char b) { return (a >= 'A' && a <= 'Z' ? a + ('a' - 'A') : a) == b; })) {
|
||||
return &format;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
size_t FindMarker(NativeView path) {
|
||||
// Native files use this check on every operation. The usual path has no marker and allocates
|
||||
// nothing.
|
||||
for (auto marker = path.find('!'); marker != NativeView::npos;
|
||||
marker = path.find('!', marker + 1)) {
|
||||
if ((marker + 1 == path.size() || path[marker + 1] == '/' || path[marker + 1] == '\\') &&
|
||||
FindFormat(path.substr(0, marker)) != nullptr) {
|
||||
return marker;
|
||||
}
|
||||
}
|
||||
return NativeView::npos;
|
||||
}
|
||||
|
||||
struct ParsedPath {
|
||||
std::filesystem::path archive;
|
||||
std::string member;
|
||||
};
|
||||
|
||||
std::optional<ParsedPath> ParsePath(const std::filesystem::path& path) {
|
||||
const auto& native = path.native();
|
||||
const auto marker = FindMarker(native);
|
||||
if (marker == NativeView::npos) {
|
||||
return {};
|
||||
}
|
||||
ParsedPath parsed {.archive = std::filesystem::path(native.substr(0, marker))};
|
||||
auto member = PathToGenericString(std::filesystem::path(native.substr(marker + 1)));
|
||||
std::replace(member.begin(), member.end(), '\\', '/');
|
||||
std::string_view remaining = member;
|
||||
while (!remaining.empty()) {
|
||||
const auto separator = remaining.find('/');
|
||||
const auto component = remaining.substr(0, separator);
|
||||
remaining.remove_prefix(separator == std::string_view::npos ? remaining.size() : separator + 1);
|
||||
if (component.empty() || component == ".") {
|
||||
continue;
|
||||
}
|
||||
if (component == "..") {
|
||||
if (parsed.member.empty()) {
|
||||
return {};
|
||||
}
|
||||
const auto separator = parsed.member.rfind('/');
|
||||
parsed.member.resize(separator == std::string::npos ? 0 : separator);
|
||||
} else {
|
||||
if (!parsed.member.empty()) {
|
||||
parsed.member += '/';
|
||||
}
|
||||
parsed.member += component;
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
std::mutex g_readers_mutex;
|
||||
std::unordered_map<std::filesystem::path, std::weak_ptr<ArchiveReader>> g_readers;
|
||||
|
||||
// Guest stacks can be only 64 KiB. Decompress on an ordinary host stack, then copy into guest
|
||||
// memory on the caller so GPU-tracked writes use its fault handler. A single worker bounds host
|
||||
// threads and avoids duplicating the archive's own block cache. Requests live on waiting callers.
|
||||
class ArchiveIoThread {
|
||||
public:
|
||||
static ArchiveIoThread& Instance() {
|
||||
// Guest exit can run static destructors while other guest threads are still reading.
|
||||
// Keep the worker alive until process termination, as with the guest threads themselves.
|
||||
static auto* instance = new ArchiveIoThread;
|
||||
return *instance;
|
||||
}
|
||||
|
||||
uint64_t Read(ArchiveReader& reader, uint64_t id, uint64_t offset, uint32_t size, void* data) {
|
||||
Request request {reader, id, offset, size, data};
|
||||
std::unique_lock lock(m_mutex);
|
||||
if (m_last != nullptr) {
|
||||
m_last->next = &request;
|
||||
} else {
|
||||
m_first = &request;
|
||||
}
|
||||
m_last = &request;
|
||||
m_wake.notify_one();
|
||||
request.ready.wait(lock, [&] { return request.done; });
|
||||
return request.result;
|
||||
}
|
||||
|
||||
KYTY_CLASS_NO_COPY(ArchiveIoThread);
|
||||
|
||||
private:
|
||||
struct Request {
|
||||
ArchiveReader& reader;
|
||||
uint64_t id;
|
||||
uint64_t offset;
|
||||
uint32_t size;
|
||||
void* data;
|
||||
Request* next = nullptr;
|
||||
uint64_t result = 0;
|
||||
bool done = false;
|
||||
std::condition_variable ready {};
|
||||
};
|
||||
|
||||
ArchiveIoThread() {
|
||||
std::thread([this] { Loop(); }).detach();
|
||||
}
|
||||
|
||||
void Loop() {
|
||||
for (;;) {
|
||||
Request* request;
|
||||
{
|
||||
std::unique_lock lock(m_mutex);
|
||||
m_wake.wait(lock, [this] { return m_first != nullptr; });
|
||||
request = m_first;
|
||||
m_first = request->next;
|
||||
if (m_first == nullptr) {
|
||||
m_last = nullptr;
|
||||
}
|
||||
}
|
||||
const auto result =
|
||||
request->reader.Read(request->id, request->offset, request->size, request->data);
|
||||
{
|
||||
std::lock_guard lock(m_mutex);
|
||||
request->result = result;
|
||||
request->done = true;
|
||||
// Notify before releasing the mutex: the waiting caller owns the condition
|
||||
// variable.
|
||||
request->ready.notify_one();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::mutex m_mutex;
|
||||
std::condition_variable m_wake;
|
||||
Request* m_first = nullptr;
|
||||
Request* m_last = nullptr;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
struct ArchiveFile::Private {
|
||||
std::shared_ptr<ArchiveReader> reader;
|
||||
ArchiveReader::Entry entry;
|
||||
uint64_t position = 0;
|
||||
};
|
||||
|
||||
ArchiveFile::ArchiveFile(std::unique_ptr<Private> p): m_p(std::move(p)) {}
|
||||
ArchiveFile::~ArchiveFile() = default;
|
||||
|
||||
uint64_t ArchiveFile::Size() const {
|
||||
return m_p->entry.size;
|
||||
}
|
||||
uint64_t ArchiveFile::Tell() const {
|
||||
return m_p->position;
|
||||
}
|
||||
bool ArchiveFile::Seek(uint64_t offset) {
|
||||
m_p->position = offset;
|
||||
return true;
|
||||
}
|
||||
|
||||
void ArchiveFile::Read(void* data, uint32_t size, uint32_t* bytes_read) {
|
||||
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
|
||||
// A pending cancellation must not destroy the waiting request or its TLS buffer while the
|
||||
// worker still uses them. Restore the caller's state only after the copy is complete.
|
||||
int cancellation_state = PTHREAD_CANCEL_ENABLE;
|
||||
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancellation_state);
|
||||
#endif
|
||||
constexpr uint32_t chunk_size = 1u << 20u;
|
||||
thread_local std::vector<uint8_t> chunk;
|
||||
uint32_t read = 0;
|
||||
if (data != nullptr && size != 0 && m_p->position < Size()) {
|
||||
const auto requested =
|
||||
static_cast<uint32_t>(std::min<uint64_t>(size, Size() - m_p->position));
|
||||
if (chunk.size() < std::min(chunk_size, requested)) {
|
||||
chunk.resize(std::min(chunk_size, requested));
|
||||
}
|
||||
while (read < requested) {
|
||||
const auto step = std::min(chunk_size, requested - read);
|
||||
const auto got = ArchiveIoThread::Instance().Read(*m_p->reader, m_p->entry.id,
|
||||
m_p->position, step, chunk.data());
|
||||
if (got == 0 || got > step) {
|
||||
break;
|
||||
}
|
||||
std::memcpy(static_cast<uint8_t*>(data) + read, chunk.data(), got);
|
||||
read += static_cast<uint32_t>(got);
|
||||
m_p->position += got;
|
||||
if (got < step) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bytes_read != nullptr) {
|
||||
*bytes_read = read;
|
||||
}
|
||||
#if defined(KYTY_ARCHIVE_PTHREAD_CANCELLATION)
|
||||
pthread_setcancelstate(cancellation_state, nullptr);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
|
||||
const std::filesystem::path& member) {
|
||||
auto root = archive;
|
||||
root += "!";
|
||||
return member.empty() ? root : root / member.relative_path();
|
||||
}
|
||||
|
||||
bool IsSupportedArchive(const std::filesystem::path& path) {
|
||||
return FindFormat(path.native()) != nullptr;
|
||||
}
|
||||
|
||||
bool IsArchivePath(const std::filesystem::path& path) {
|
||||
return FindMarker(path.native()) != NativeView::npos;
|
||||
}
|
||||
|
||||
std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path) {
|
||||
const auto marker = FindMarker(path.native());
|
||||
return marker == NativeView::npos ? std::filesystem::path {}
|
||||
: std::filesystem::path(path.native().substr(0, marker));
|
||||
}
|
||||
|
||||
std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path) {
|
||||
const auto archive = IsArchivePath(path) ? GetArchiveHostPath(path) : path;
|
||||
const auto* format = FindFormat(archive.native());
|
||||
if (format == nullptr) {
|
||||
return {};
|
||||
}
|
||||
std::error_code error;
|
||||
auto key = std::filesystem::absolute(archive, error);
|
||||
if (error) {
|
||||
return {};
|
||||
}
|
||||
{
|
||||
std::lock_guard lock(g_readers_mutex);
|
||||
if (auto it = g_readers.find(key); it != g_readers.end()) {
|
||||
if (auto reader = it->second.lock()) {
|
||||
return reader;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Do not hold the cache mutex while opening an unrelated archive from disk.
|
||||
auto reader = format->open(archive);
|
||||
if (reader == nullptr) {
|
||||
return {};
|
||||
}
|
||||
std::lock_guard lock(g_readers_mutex);
|
||||
std::erase_if(g_readers, [](const auto& entry) { return entry.second.expired(); });
|
||||
auto& cached = g_readers[key];
|
||||
if (auto existing = cached.lock()) {
|
||||
return existing;
|
||||
}
|
||||
cached = reader;
|
||||
return reader;
|
||||
}
|
||||
|
||||
std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path) {
|
||||
const auto parsed = ParsePath(path);
|
||||
if (parsed) {
|
||||
if (auto reader = OpenArchive(parsed->archive)) {
|
||||
if (auto entry = reader->Find(parsed->member)) {
|
||||
return File::Info {entry->is_file, entry->size};
|
||||
}
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path) {
|
||||
const auto parsed = ParsePath(path);
|
||||
if (parsed) {
|
||||
if (auto reader = OpenArchive(parsed->archive)) {
|
||||
return reader->List(parsed->member);
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path) {
|
||||
auto parsed = ParsePath(path);
|
||||
if (!parsed) {
|
||||
return {};
|
||||
}
|
||||
auto reader = OpenArchive(parsed->archive);
|
||||
const auto entry = reader ? reader->Find(parsed->member) : std::nullopt;
|
||||
if (!entry || !entry->is_file) {
|
||||
return {};
|
||||
}
|
||||
auto p = std::make_unique<ArchiveFile::Private>();
|
||||
p->reader = std::move(reader);
|
||||
p->entry = *entry;
|
||||
return std::unique_ptr<ArchiveFile>(new ArchiveFile(std::move(p)));
|
||||
}
|
||||
|
||||
} // namespace Common
|
||||
@@ -0,0 +1,50 @@
|
||||
#ifndef KYTY_COMMON_ARCHIVE_H_
|
||||
#define KYTY_COMMON_ARCHIVE_H_
|
||||
|
||||
#include "common/file.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace Common {
|
||||
|
||||
class ArchiveReader;
|
||||
|
||||
class ArchiveFile {
|
||||
public:
|
||||
~ArchiveFile();
|
||||
|
||||
[[nodiscard]] uint64_t Size() const;
|
||||
[[nodiscard]] uint64_t Tell() const;
|
||||
bool Seek(uint64_t offset);
|
||||
void Read(void* data, uint32_t size, uint32_t* bytes_read);
|
||||
|
||||
KYTY_CLASS_NO_COPY(ArchiveFile);
|
||||
|
||||
private:
|
||||
struct Private;
|
||||
explicit ArchiveFile(std::unique_ptr<Private> p);
|
||||
std::unique_ptr<Private> m_p;
|
||||
|
||||
friend std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
|
||||
};
|
||||
|
||||
// Virtual paths use "archive.ext!/member" and retain native host-path encoding.
|
||||
[[nodiscard]] std::filesystem::path MakeArchivePath(const std::filesystem::path& archive,
|
||||
const std::filesystem::path& member = {});
|
||||
[[nodiscard]] bool IsSupportedArchive(const std::filesystem::path& path);
|
||||
[[nodiscard]] bool IsArchivePath(const std::filesystem::path& path);
|
||||
[[nodiscard]] std::optional<File::Info> GetArchiveInfo(const std::filesystem::path& path);
|
||||
[[nodiscard]] std::vector<File::DirEntry> GetArchiveDirEntries(const std::filesystem::path& path);
|
||||
[[nodiscard]] std::filesystem::path GetArchiveHostPath(const std::filesystem::path& path);
|
||||
[[nodiscard]] std::unique_ptr<ArchiveFile> OpenArchiveFile(const std::filesystem::path& path);
|
||||
|
||||
// Retain this handle while a mount or metadata scan is active to reuse the index and block cache.
|
||||
// Accepts either a host archive filename or a virtual path inside the archive.
|
||||
[[nodiscard]] std::shared_ptr<ArchiveReader> OpenArchive(const std::filesystem::path& path);
|
||||
|
||||
} // namespace Common
|
||||
|
||||
#endif /* KYTY_COMMON_ARCHIVE_H_ */
|
||||
+140
-17
@@ -1,5 +1,6 @@
|
||||
#include "common/file.h"
|
||||
|
||||
#include "common/archive.h"
|
||||
#include "common/assert.h"
|
||||
#include "common/common.h"
|
||||
#include "common/platform/sysFileIO.h"
|
||||
@@ -61,7 +62,8 @@ SysFileTimeStruct DateTimeToFileTimeUtc(const DateTime& date_time) {
|
||||
} // namespace
|
||||
|
||||
struct File::FilePrivate {
|
||||
sys_file_t* f;
|
||||
sys_file_t* f;
|
||||
std::unique_ptr<ArchiveFile> archive;
|
||||
};
|
||||
|
||||
File::File(): m_p(std::make_unique<FilePrivate>()) {
|
||||
@@ -85,11 +87,15 @@ File::File(const std::filesystem::path& name, Mode mode): m_p(std::make_unique<F
|
||||
}
|
||||
|
||||
bool File::IsInvalid() const {
|
||||
return m_p->f == nullptr;
|
||||
return m_p->f == nullptr && m_p->archive == nullptr;
|
||||
}
|
||||
|
||||
bool File::Create(const std::filesystem::path& name) {
|
||||
EXIT_IF(m_p->f != nullptr);
|
||||
EXIT_IF(!IsInvalid());
|
||||
|
||||
if (IsArchivePath(name)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
m_file_name = name;
|
||||
|
||||
@@ -104,9 +110,16 @@ bool File::Create(const std::filesystem::path& name) {
|
||||
}
|
||||
|
||||
bool File::Open(const std::filesystem::path& name, Mode mode) {
|
||||
EXIT_IF(m_p->f != nullptr);
|
||||
EXIT_IF(!IsInvalid());
|
||||
|
||||
m_file_name = name;
|
||||
if (IsArchivePath(name)) {
|
||||
if (mode != Mode::Read) {
|
||||
return false;
|
||||
}
|
||||
m_p->archive = OpenArchiveFile(name);
|
||||
return m_p->archive != nullptr;
|
||||
}
|
||||
|
||||
switch (mode) {
|
||||
case Mode::Read: m_p->f = SysFileOpenR(name); break;
|
||||
@@ -124,7 +137,7 @@ bool File::Open(const std::filesystem::path& name, Mode mode) {
|
||||
}
|
||||
|
||||
bool File::OpenInMem(void* buf, uint32_t buf_size) {
|
||||
EXIT_IF(m_p->f != nullptr);
|
||||
EXIT_IF(!IsInvalid());
|
||||
|
||||
m_p->f = SysFileOpen(static_cast<uint8_t*>(buf), buf_size);
|
||||
|
||||
@@ -137,7 +150,7 @@ bool File::OpenInMem(void* buf, uint32_t buf_size) {
|
||||
}
|
||||
|
||||
bool File::CreateInMem() {
|
||||
EXIT_IF(m_p->f != nullptr);
|
||||
EXIT_IF(!IsInvalid());
|
||||
|
||||
m_p->f = SysFileCreate();
|
||||
|
||||
@@ -150,6 +163,7 @@ bool File::CreateInMem() {
|
||||
}
|
||||
|
||||
void File::Close() {
|
||||
m_p->archive.reset();
|
||||
if (m_p->f != nullptr) {
|
||||
SysFileClose(m_p->f);
|
||||
m_p->f = nullptr;
|
||||
@@ -157,7 +171,11 @@ void File::Close() {
|
||||
}
|
||||
|
||||
uint64_t File::Size() const {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
return m_p->archive->Size();
|
||||
}
|
||||
|
||||
if (m_p->f == nullptr) {
|
||||
return 0;
|
||||
@@ -167,41 +185,76 @@ uint64_t File::Size() const {
|
||||
}
|
||||
|
||||
uint64_t File::Remaining() const {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
return Size() - Tell();
|
||||
const auto size = Size();
|
||||
const auto position = Tell();
|
||||
return position < size ? size - position : 0;
|
||||
}
|
||||
|
||||
uint64_t File::Size(const std::filesystem::path& name) {
|
||||
if (IsArchivePath(name)) {
|
||||
const auto info = GetArchiveInfo(name);
|
||||
return info ? info->size : 0;
|
||||
}
|
||||
return SysFileSize(name);
|
||||
}
|
||||
|
||||
std::optional<File::Info> File::GetInfo(const std::filesystem::path& name) {
|
||||
if (IsArchivePath(name)) {
|
||||
return GetArchiveInfo(name);
|
||||
}
|
||||
Info info {};
|
||||
return SysFileGetInfo(name, &info.is_file, &info.size) ? std::optional(info) : std::nullopt;
|
||||
}
|
||||
|
||||
bool File::Seek(uint64_t offset) {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
return m_p->archive->Seek(offset);
|
||||
}
|
||||
|
||||
return SysFileSeek(*m_p->f, offset);
|
||||
}
|
||||
|
||||
bool File::Truncate(uint64_t size) {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return SysFileTruncate(*m_p->f, size);
|
||||
}
|
||||
|
||||
bool File::Unlink() {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return SysFileUnlink(*m_p->f, m_file_name);
|
||||
}
|
||||
|
||||
uint64_t File::Tell() const {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
return m_p->archive->Tell();
|
||||
}
|
||||
|
||||
return SysFileTell(*m_p->f);
|
||||
}
|
||||
|
||||
void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
m_p->archive->Read(data, size, bytes_read);
|
||||
return;
|
||||
}
|
||||
|
||||
if (m_p->f != nullptr) {
|
||||
SysFileRead(data, size, *m_p->f, bytes_read);
|
||||
@@ -209,7 +262,14 @@ void File::Read(void* data, uint32_t size, uint32_t* bytes_read) {
|
||||
}
|
||||
|
||||
void File::Write(const void* data, uint32_t size, uint32_t* bytes_written) {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
if (bytes_written != nullptr) {
|
||||
*bytes_written = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
SysFileWrite(data, size, *m_p->f, bytes_written);
|
||||
}
|
||||
@@ -257,24 +317,41 @@ static std::filesystem::path WithoutTrailingSeparator(
|
||||
|
||||
|
||||
bool File::IsDirectoryExisting(const std::filesystem::path& path) {
|
||||
if (IsArchivePath(path)) {
|
||||
const auto info = GetArchiveInfo(path);
|
||||
return info && !info->is_file;
|
||||
}
|
||||
return SysFileIsDirectoryExisting(WithoutTrailingSeparator(path));
|
||||
}
|
||||
|
||||
bool File::IsFileExisting(const std::filesystem::path& name) {
|
||||
if (IsArchivePath(name)) {
|
||||
const auto info = GetArchiveInfo(name);
|
||||
return info && info->is_file;
|
||||
}
|
||||
return SysFileIsFileExisting(name);
|
||||
}
|
||||
|
||||
bool File::CreateDirectory(const std::filesystem::path& path) {
|
||||
if (IsArchivePath(path)) {
|
||||
return false;
|
||||
}
|
||||
return SysFileCreateDirectory(
|
||||
WithoutTrailingSeparator(path));
|
||||
}
|
||||
|
||||
bool File::DeleteDirectory(const std::filesystem::path& path) {
|
||||
if (IsArchivePath(path)) {
|
||||
return false;
|
||||
}
|
||||
return SysFileDeleteDirectory(
|
||||
WithoutTrailingSeparator(path));
|
||||
}
|
||||
|
||||
bool File::CreateDirectories(const std::filesystem::path& path) {
|
||||
if (IsArchivePath(path)) {
|
||||
return false;
|
||||
}
|
||||
const auto normalized_path =
|
||||
WithoutTrailingSeparator(path);
|
||||
|
||||
@@ -297,6 +374,9 @@ bool File::CreateDirectories(const std::filesystem::path& path) {
|
||||
}
|
||||
|
||||
bool File::DeleteDirectories(const std::filesystem::path& path) {
|
||||
if (IsArchivePath(path)) {
|
||||
return false;
|
||||
}
|
||||
const auto normalized_path =
|
||||
WithoutTrailingSeparator(path);
|
||||
|
||||
@@ -322,11 +402,18 @@ bool File::DeleteDirectories(const std::filesystem::path& path) {
|
||||
bool File::DeleteFile(
|
||||
const std::filesystem::path& name) // @suppress("Member declaration not found")
|
||||
{
|
||||
if (IsArchivePath(name)) {
|
||||
return false;
|
||||
}
|
||||
return SysFileDeleteFile(name);
|
||||
}
|
||||
|
||||
bool File::Flush() {
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return SysFileFlush(*m_p->f);
|
||||
}
|
||||
@@ -348,6 +435,9 @@ std::vector<std::byte> File::ReadWholeBuffer() {
|
||||
}
|
||||
|
||||
DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
|
||||
if (IsArchivePath(name)) {
|
||||
return GetLastAccessTimeUTC(GetArchiveHostPath(name));
|
||||
}
|
||||
SysTimeStruct t {};
|
||||
SysFileToSystemTimeUtc(SysFileGetLastAccessTimeUtc(name), t);
|
||||
|
||||
@@ -359,6 +449,9 @@ DateTime File::GetLastAccessTimeUTC(const std::filesystem::path& name) {
|
||||
}
|
||||
|
||||
DateTime File::GetLastWriteTimeUTC(const std::filesystem::path& name) {
|
||||
if (IsArchivePath(name)) {
|
||||
return GetLastWriteTimeUTC(GetArchiveHostPath(name));
|
||||
}
|
||||
SysTimeStruct t {};
|
||||
SysFileToSystemTimeUtc(SysFileGetLastWriteTimeUtc(name), t);
|
||||
|
||||
@@ -373,6 +466,10 @@ void File::GetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, DateT
|
||||
DateTime* write) {
|
||||
EXIT_IF(access == nullptr);
|
||||
EXIT_IF(write == nullptr);
|
||||
if (IsArchivePath(name)) {
|
||||
GetLastAccessAndWriteTimeUTC(GetArchiveHostPath(name), access, write);
|
||||
return;
|
||||
}
|
||||
|
||||
SysTimeStruct at {};
|
||||
SysTimeStruct wt {};
|
||||
@@ -399,7 +496,12 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
|
||||
EXIT_IF(access == nullptr);
|
||||
EXIT_IF(write == nullptr);
|
||||
|
||||
EXIT_IF(m_p->f == nullptr);
|
||||
EXIT_IF(IsInvalid());
|
||||
|
||||
if (m_p->archive != nullptr) {
|
||||
GetLastAccessAndWriteTimeUTC(m_file_name, access, write);
|
||||
return;
|
||||
}
|
||||
|
||||
if (m_p->f == nullptr) {
|
||||
*access = DateTime();
|
||||
@@ -429,23 +531,35 @@ void File::GetLastAccessAndWriteTimeUTC(DateTime* access, DateTime* write) {
|
||||
}
|
||||
|
||||
bool File::SetLastAccessTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
|
||||
if (IsArchivePath(name)) {
|
||||
return false;
|
||||
}
|
||||
auto f = DateTimeToFileTimeUtc(dt);
|
||||
return SysFileSetLastAccessTimeUtc(name, f);
|
||||
}
|
||||
|
||||
bool File::SetLastWriteTimeUTC(const std::filesystem::path& name, const DateTime& dt) {
|
||||
if (IsArchivePath(name)) {
|
||||
return false;
|
||||
}
|
||||
auto f = DateTimeToFileTimeUtc(dt);
|
||||
return SysFileSetLastWriteTimeUtc(name, f);
|
||||
}
|
||||
|
||||
bool File::SetLastAccessAndWriteTimeUTC(const std::filesystem::path& name, const DateTime& access,
|
||||
const DateTime& write) {
|
||||
if (IsArchivePath(name)) {
|
||||
return false;
|
||||
}
|
||||
auto af = DateTimeToFileTimeUtc(access);
|
||||
auto wf = DateTimeToFileTimeUtc(write);
|
||||
return SysFileSetLastAccessAndWriteTimeUtc(name, af, wf);
|
||||
}
|
||||
|
||||
std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& path) {
|
||||
if (IsArchivePath(path)) {
|
||||
return GetArchiveDirEntries(path);
|
||||
}
|
||||
std::vector<sys_dir_entry_t> files;
|
||||
|
||||
SysFileGetDents(path, files);
|
||||
@@ -463,12 +577,18 @@ std::vector<File::DirEntry> File::GetDirEntries(const std::filesystem::path& pat
|
||||
bool File::CopyFile(const std::filesystem::path& src,
|
||||
const std::filesystem::path& dst) // @suppress("Member declaration not found")
|
||||
{
|
||||
if (IsArchivePath(src) || IsArchivePath(dst)) {
|
||||
return false;
|
||||
}
|
||||
return SysFileCopyFile(src, dst);
|
||||
}
|
||||
|
||||
bool File::RenameFile(const std::filesystem::path& src,
|
||||
const std::filesystem::path& dst) // @suppress("Member declaration not found")
|
||||
{
|
||||
if (IsArchivePath(src) || IsArchivePath(dst)) {
|
||||
return false;
|
||||
}
|
||||
if (IsFileExisting(dst)) {
|
||||
DeleteFile(dst); // @suppress("Invalid arguments")
|
||||
}
|
||||
@@ -477,6 +597,9 @@ bool File::RenameFile(const std::filesystem::path& src,
|
||||
}
|
||||
|
||||
void File::RemoveReadonly(const std::filesystem::path& name) {
|
||||
if (IsArchivePath(name)) {
|
||||
return;
|
||||
}
|
||||
SysFileRemoveReadonly(name);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <cstddef>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -34,6 +35,10 @@ public:
|
||||
std::string name;
|
||||
bool is_file;
|
||||
};
|
||||
struct Info {
|
||||
bool is_file;
|
||||
uint64_t size;
|
||||
};
|
||||
|
||||
File();
|
||||
explicit File(const std::filesystem::path& name);
|
||||
@@ -72,6 +77,7 @@ public:
|
||||
std::vector<std::byte> ReadWholeBuffer();
|
||||
|
||||
static uint64_t Size(const std::filesystem::path& name);
|
||||
static std::optional<Info> GetInfo(const std::filesystem::path& name);
|
||||
|
||||
static bool IsDirectoryExisting(const std::filesystem::path& path);
|
||||
static bool IsFileExisting(const std::filesystem::path& name);
|
||||
|
||||
@@ -45,6 +45,7 @@ bool SysFileTruncate(sys_file_t& f, uint64_t size); // NOLINT(google-runt
|
||||
bool SysFileUnlink(sys_file_t& f,
|
||||
const std::filesystem::path& name); // NOLINT(google-runtime-references)
|
||||
uint64_t SysFileSize(const std::filesystem::path& file_name);
|
||||
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size);
|
||||
bool SysFileIsError(sys_file_t& f); // NOLINT(google-runtime-references)
|
||||
bool SysFileIsDirectoryExisting(const std::filesystem::path& path);
|
||||
bool SysFileIsFileExisting(const std::filesystem::path& name);
|
||||
|
||||
@@ -54,7 +54,7 @@ struct sys_file_t {
|
||||
#endif
|
||||
|
||||
static std::filesystem::path get_internal_name(const std::filesystem::path& name) {
|
||||
return name.is_absolute() ? name : (std::filesystem::path(".") / name);
|
||||
return name.empty() || name.is_absolute() ? name : (std::filesystem::path(".") / name);
|
||||
}
|
||||
|
||||
// Pass access-pattern hints to the host.
|
||||
@@ -297,10 +297,21 @@ uint64_t SysFileSize(sys_file_t& f) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
|
||||
const auto path = get_internal_name(name);
|
||||
struct stat info {};
|
||||
if (stat(path.c_str(), &info) != 0) {
|
||||
return false;
|
||||
}
|
||||
*is_file = !S_ISDIR(info.st_mode);
|
||||
*size = *is_file ? static_cast<uint64_t>(info.st_size) : 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t SysFileSize(const std::filesystem::path& file_name) {
|
||||
sys_file_t* f = SysFileOpenR(file_name);
|
||||
uint64_t size = SysFileSize(*f);
|
||||
SysFileClose(f);
|
||||
bool is_file;
|
||||
uint64_t size = 0;
|
||||
SysFileGetInfo(file_name, &is_file, &size);
|
||||
return size;
|
||||
}
|
||||
|
||||
@@ -353,29 +364,15 @@ bool SysFileIsError(sys_file_t& f) {
|
||||
}
|
||||
|
||||
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
|
||||
auto real_name = get_internal_name(path);
|
||||
auto real_name_str = real_name.string();
|
||||
|
||||
struct stat s {};
|
||||
|
||||
if (0 != stat(real_name_str.c_str(), &s)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return S_ISDIR(s.st_mode); // NOLINT
|
||||
bool is_file;
|
||||
uint64_t size;
|
||||
return SysFileGetInfo(path, &is_file, &size) && !is_file;
|
||||
}
|
||||
|
||||
bool SysFileIsFileExisting(const std::filesystem::path& name) {
|
||||
auto real_name = get_internal_name(name);
|
||||
auto real_name_str = real_name.string();
|
||||
|
||||
struct stat s {};
|
||||
|
||||
if (0 != stat(real_name_str.c_str(), &s)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !S_ISDIR(s.st_mode); // NOLINT
|
||||
bool is_file;
|
||||
uint64_t size;
|
||||
return SysFileGetInfo(name, &is_file, &size) && is_file;
|
||||
}
|
||||
|
||||
bool SysFileCreateDirectory(const std::filesystem::path& path) {
|
||||
|
||||
@@ -266,19 +266,27 @@ uint64_t SysFileSize(sys_file_t& f) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t SysFileSize(const std::filesystem::path& file_name) {
|
||||
LARGE_INTEGER s;
|
||||
WIN32_FILE_ATTRIBUTE_DATA a;
|
||||
|
||||
auto wide = file_name.wstring();
|
||||
if (GetFileAttributesExW(wide.c_str(), GetFileExInfoStandard, &a) == 0) {
|
||||
return 0;
|
||||
bool SysFileGetInfo(const std::filesystem::path& name, bool* is_file, uint64_t* size) {
|
||||
const bool directory_path = !name.empty() && !name.has_filename() && name != name.root_path();
|
||||
WIN32_FILE_ATTRIBUTE_DATA info {};
|
||||
if (GetFileAttributesExW(directory_path ? name.parent_path().c_str() : name.c_str(),
|
||||
GetFileExInfoStandard, &info) == 0) {
|
||||
return false;
|
||||
}
|
||||
const bool file = (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
|
||||
if (directory_path && file) {
|
||||
return false;
|
||||
}
|
||||
*is_file = file;
|
||||
*size = *is_file ? ((static_cast<uint64_t>(info.nFileSizeHigh) << 32u) | info.nFileSizeLow) : 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
s.HighPart = static_cast<LONG>(a.nFileSizeHigh);
|
||||
s.LowPart = a.nFileSizeLow;
|
||||
|
||||
return s.QuadPart;
|
||||
uint64_t SysFileSize(const std::filesystem::path& file_name) {
|
||||
bool is_file;
|
||||
uint64_t size = 0;
|
||||
SysFileGetInfo(file_name, &is_file, &size);
|
||||
return size;
|
||||
}
|
||||
|
||||
bool SysFileTruncate(sys_file_t& f, uint64_t size) {
|
||||
@@ -346,17 +354,15 @@ bool SysFileIsError(sys_file_t& f) {
|
||||
}
|
||||
|
||||
bool SysFileIsDirectoryExisting(const std::filesystem::path& path) {
|
||||
auto wide = path.wstring();
|
||||
DWORD a = GetFileAttributesW(wide.c_str());
|
||||
return a != INVALID_FILE_ATTRIBUTES &&
|
||||
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) != 0u);
|
||||
bool is_file;
|
||||
uint64_t size;
|
||||
return SysFileGetInfo(path, &is_file, &size) && !is_file;
|
||||
}
|
||||
|
||||
bool SysFileIsFileExisting(const std::filesystem::path& name) {
|
||||
auto wide = name.wstring();
|
||||
DWORD a = GetFileAttributesW(wide.c_str());
|
||||
return a != INVALID_FILE_ATTRIBUTES &&
|
||||
((a & static_cast<DWORD>(FILE_ATTRIBUTE_DIRECTORY)) == 0u);
|
||||
bool is_file;
|
||||
uint64_t size;
|
||||
return SysFileGetInfo(name, &is_file, &size) && is_file;
|
||||
}
|
||||
|
||||
bool SysFileCreateDirectory(const std::filesystem::path& path) {
|
||||
|
||||
+1
-10
@@ -38,16 +38,7 @@ bool Timer::IsPaused() const {
|
||||
|
||||
// return time in milliseconds
|
||||
double Timer::GetTimeMs() const {
|
||||
if (m_is_paused) {
|
||||
return 1000.0 * (static_cast<double>(m_PauseTime - m_StartTime)) /
|
||||
static_cast<double>(m_Frequency);
|
||||
}
|
||||
|
||||
uint64_t current_time = 0;
|
||||
SysQueryPerformanceCounter(¤t_time);
|
||||
|
||||
return 1000.0 * (static_cast<double>(current_time - m_StartTime)) /
|
||||
static_cast<double>(m_Frequency);
|
||||
return GetTimeS() * 1000.0;
|
||||
}
|
||||
|
||||
// return time in seconds
|
||||
|
||||
@@ -976,6 +976,16 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
|
||||
const auto args_size = indexed ? sizeof(DrawIndexedIndirectArgs) : sizeof(DrawIndirectArgs);
|
||||
EXIT_NOT_IMPLEMENTED(stride_in_bytes < args_size);
|
||||
|
||||
uint64_t index_size = 0;
|
||||
if (indexed) {
|
||||
switch (m_index_type_and_size) {
|
||||
case 0: index_size = 2; break;
|
||||
case 1: index_size = 4; break;
|
||||
case 2: index_size = 1; break;
|
||||
default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < draw_count; i++) {
|
||||
const auto args_addr = m_draw_indirect_args_base_addr + data_offset +
|
||||
static_cast<uint64_t>(i) * stride_in_bytes;
|
||||
@@ -993,14 +1003,6 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
|
||||
|
||||
auto* args = reinterpret_cast<const DrawIndexedIndirectArgs*>(args_addr);
|
||||
|
||||
uint64_t index_size = 0;
|
||||
switch (m_index_type_and_size) {
|
||||
case 0: index_size = 2; break;
|
||||
case 1: index_size = 4; break;
|
||||
case 2: index_size = 1; break;
|
||||
default: EXIT("unknown index_type_and_size: %u\n", m_index_type_and_size);
|
||||
}
|
||||
|
||||
auto* index_addr = reinterpret_cast<const void*>(
|
||||
m_index_base_addr + static_cast<uint64_t>(args->start_index_location) * index_size);
|
||||
|
||||
@@ -1203,8 +1205,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
|
||||
}
|
||||
break;
|
||||
case 0x02:
|
||||
case 0x04:
|
||||
if constexpr (sizeof(T) == sizeof(uint32_t)) {
|
||||
if (eop_event_type == 0x2f && event_index == 0x06) {
|
||||
if (event_write_source == 0x02 && eop_event_type == 0x2f && event_index == 0x06) {
|
||||
switch (cache_action) {
|
||||
case 0x00: write32(false); return;
|
||||
case 0x38: write32(true); return;
|
||||
@@ -1212,6 +1215,9 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (event_write_source == 0x04) {
|
||||
value = Sync::ReadReferenceClock();
|
||||
}
|
||||
auto write64 = [&](bool with_writeback) {
|
||||
auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
|
||||
std::memcpy(dst, &value, sizeof(value));
|
||||
@@ -1268,7 +1274,7 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
|
||||
case 0x14:
|
||||
case 0x28:
|
||||
if (((eop_event_type == 0x04 || eop_event_type == 0x28) &&
|
||||
event_index == 0x05 && !with_interrupt) ||
|
||||
event_index == 0x05) ||
|
||||
(event_index == 0x00)) {
|
||||
write64(true);
|
||||
return;
|
||||
@@ -1300,45 +1306,6 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 0x04:
|
||||
if constexpr (sizeof(T) == sizeof(uint64_t)) {
|
||||
const auto clock = Sync::ReadReferenceClock();
|
||||
auto* dst = static_cast<uint64_t*>(dst_gpu_addr);
|
||||
std::memcpy(dst, &clock, sizeof(clock));
|
||||
switch (cache_action) {
|
||||
case 0x00:
|
||||
if ((eop_event_type == 0x04 && event_index == 0x05) ||
|
||||
(eop_event_type == 0x28 && event_index == 0x00)) {
|
||||
if (with_interrupt) {
|
||||
Sync::WriteAtEndOfPipeWithInterrupt64(
|
||||
m_submit_id, command, dst, clock, m_interrupt_event_id,
|
||||
interrupt_context_id);
|
||||
} else {
|
||||
Sync::WriteAtEndOfPipeClockCounter(m_submit_id, command,
|
||||
dst, clock);
|
||||
}
|
||||
return;
|
||||
}
|
||||
break;
|
||||
case 0x38:
|
||||
if ((eop_event_type == 0x04 &&
|
||||
(event_index == 0x00 || event_index == 0x05)) ||
|
||||
(eop_event_type == 0x28 && event_index == 0x00)) {
|
||||
if (with_interrupt) {
|
||||
Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
|
||||
m_submit_id, command, dst, clock, m_interrupt_event_id,
|
||||
interrupt_context_id);
|
||||
} else {
|
||||
Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
|
||||
m_submit_id, command, dst, clock);
|
||||
}
|
||||
return;
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
|
||||
@@ -1535,8 +1502,9 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
|
||||
ProcessorScope processor_scope(*this);
|
||||
|
||||
m_renderer.GetVideoOut().PrepareFlip(request_id, command);
|
||||
GetScheduler().DeferPriorityOperation(
|
||||
[this, request_id] { m_renderer.GetVideoOut().CompleteFlip(request_id); });
|
||||
GetScheduler().Flush();
|
||||
m_renderer.GetVideoOut().CompleteFlip(request_id);
|
||||
}
|
||||
|
||||
void CommandProcessor::SynchronizeGpu() {
|
||||
|
||||
@@ -1213,20 +1213,12 @@ bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch,
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TileGetDccSize(uint32_t width, uint32_t height, uint32_t slices,
|
||||
uint32_t bytes_per_element, uint32_t levels, Prospero::TileMode tile,
|
||||
TileSizeAlign& total_size, uint32_t num_fragments_log2) {
|
||||
total_size = {};
|
||||
if (width == 0 || height == 0 || slices == 0 || levels != 1 || num_fragments_log2 != 0 ||
|
||||
!std::has_single_bit(bytes_per_element) || bytes_per_element > 16 ||
|
||||
(tile != Prospero::TileMode::kRenderTarget && tile != Prospero::TileMode::kDepth)) {
|
||||
static bool TileGetColorMetadataSize(uint32_t width, uint32_t height, uint32_t slices,
|
||||
uint32_t block_width, uint32_t block_height,
|
||||
TileSizeAlign& total_size) {
|
||||
if (width == 0 || height == 0 || slices == 0) {
|
||||
return false;
|
||||
}
|
||||
// Gen5 color metadata uses 4 KiB blocks, with one byte per 256 bytes of color data.
|
||||
// Metadata stays thin for volume surfaces, so every depth slice has its own block raster.
|
||||
const uint32_t coverage_bits = 20u - std::countr_zero(bytes_per_element);
|
||||
const uint32_t block_width = 1u << ((coverage_bits + 1u) / 2u);
|
||||
const uint32_t block_height = 1u << (coverage_bits / 2u);
|
||||
const uint64_t blocks_x = (static_cast<uint64_t>(width) + block_width - 1u) / block_width;
|
||||
const uint64_t blocks_y = (static_cast<uint64_t>(height) + block_height - 1u) / block_height;
|
||||
const uint64_t blocks = blocks_x * blocks_y;
|
||||
@@ -1238,36 +1230,27 @@ bool TileGetDccSize(uint32_t width, uint32_t height, uint32_t slices,
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TileGetRenderTargetMipLayout(uint32_t width, uint32_t height, uint32_t pitch,
|
||||
uint32_t bytes_per_element, uint32_t levels,
|
||||
TileSizeAlign& total_size, TileSizeOffset* level_sizes,
|
||||
TilePaddedSize* padded_size) {
|
||||
bool TileGetDccSize(uint32_t width, uint32_t height, uint32_t slices,
|
||||
uint32_t bytes_per_element, uint32_t levels, Prospero::TileMode tile,
|
||||
TileSizeAlign& total_size, uint32_t num_fragments_log2) {
|
||||
total_size = {};
|
||||
if (width == 0 || height == 0 || levels == 0 || levels > 16 ||
|
||||
pitch != TileGetRenderTargetPitch(width, bytes_per_element)) {
|
||||
if (levels != 1 || num_fragments_log2 != 0 ||
|
||||
!std::has_single_bit(bytes_per_element) || bytes_per_element > 16 ||
|
||||
(tile != Prospero::TileMode::kRenderTarget && tile != Prospero::TileMode::kDepth)) {
|
||||
return false;
|
||||
}
|
||||
uint32_t max_levels = 1;
|
||||
uint32_t max_dimension = std::max(width, height);
|
||||
while (max_dimension > 1) {
|
||||
max_dimension >>= 1u;
|
||||
max_levels++;
|
||||
}
|
||||
if (levels > max_levels) {
|
||||
return false;
|
||||
}
|
||||
auto format = Prospero::BufferFormat::kInvalid;
|
||||
switch (bytes_per_element) {
|
||||
case 1: format = Prospero::BufferFormat::k8UNorm; break;
|
||||
case 2: format = Prospero::BufferFormat::k16UNorm; break;
|
||||
case 4: format = Prospero::BufferFormat::k32Float; break;
|
||||
case 8: format = Prospero::BufferFormat::k16_16_16_16Float; break;
|
||||
case 16: format = Prospero::BufferFormat::k32_32_32_32Float; break;
|
||||
default: return false;
|
||||
}
|
||||
TileGetTextureSize(format, width, height, levels, Prospero::TileMode::kRenderTarget,
|
||||
&total_size, level_sizes, padded_size);
|
||||
return total_size.size != 0 && total_size.align == 65536;
|
||||
// Gen5 DCC uses 4 KiB blocks, with one byte per 256 bytes of color data.
|
||||
// Metadata stays thin for volume surfaces, so every depth slice has its own block raster.
|
||||
const uint32_t coverage_bits = 20u - std::countr_zero(bytes_per_element);
|
||||
return TileGetColorMetadataSize(width, height, slices, 1u << ((coverage_bits + 1u) / 2u),
|
||||
1u << (coverage_bits / 2u), total_size);
|
||||
}
|
||||
|
||||
bool TileGetCmaskSize(uint32_t width, uint32_t height, uint32_t slices, uint32_t levels,
|
||||
TileSizeAlign& total_size) {
|
||||
total_size = {};
|
||||
// Gen5 CMASK stores four bits per 8x8 tile in 4 KiB blocks covering 1024x512 pixels.
|
||||
return levels == 1 && TileGetColorMetadataSize(width, height, slices, 1024, 512, total_size);
|
||||
}
|
||||
|
||||
void TileGetTextureSize(Prospero::BufferFormat format, uint32_t width, uint32_t height,
|
||||
|
||||
@@ -126,10 +126,8 @@ bool TileGetRenderTargetSize(uint32_t width, uint32_t height, uint32_t pitch
|
||||
bool TileGetDccSize(uint32_t width, uint32_t height, uint32_t slices,
|
||||
uint32_t bytes_per_element, uint32_t levels, Prospero::TileMode tile,
|
||||
TileSizeAlign& total_size, uint32_t num_fragments_log2 = 0);
|
||||
bool TileGetRenderTargetMipLayout(uint32_t width, uint32_t height, uint32_t pitch,
|
||||
uint32_t bytes_per_element, uint32_t levels,
|
||||
TileSizeAlign& total_size, TileSizeOffset* level_sizes,
|
||||
TilePaddedSize* padded_size);
|
||||
bool TileGetCmaskSize(uint32_t width, uint32_t height, uint32_t slices, uint32_t levels,
|
||||
TileSizeAlign& total_size);
|
||||
void TileGetTextureSize(Prospero::BufferFormat format, uint32_t width, uint32_t height,
|
||||
uint32_t levels, Prospero::TileMode tile, TileSizeAlign* total_size,
|
||||
TileSizeOffset* level_sizes, TilePaddedSize* padded_size);
|
||||
|
||||
@@ -29,6 +29,7 @@ struct GraphicContext {
|
||||
bool memory_budget_ext_enabled = false;
|
||||
bool compute_subgroup_size_control_enabled = false;
|
||||
bool sample_rate_shading_enabled = false;
|
||||
// bool fp64_denorm_preserve = false; // Temporarily disabled.
|
||||
bool attachment_feedback_loop_enabled = false;
|
||||
bool provoking_vertex_last_enabled = false;
|
||||
bool supports_block_texel_view = false;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "common/bitArray.h"
|
||||
#include "common/common.h"
|
||||
#include "kernel/memory.h"
|
||||
|
||||
#include <compare>
|
||||
|
||||
@@ -10,7 +11,8 @@ namespace Libs::Graphics {
|
||||
|
||||
constexpr uint64_t TRACKER_PAGE_SIZE = 4ull * 1024ull;
|
||||
constexpr uint64_t TRACKER_REGION_SIZE = 4ull * 1024ull * 1024ull;
|
||||
constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 40u;
|
||||
constexpr uint64_t TRACKER_ADDRESS_SIZE = 1ull << 44u;
|
||||
constexpr uint64_t LOWER_ADDRESS_SIZE = 1ull << 40u;
|
||||
constexpr size_t TRACKER_REGION_PAGES = TRACKER_REGION_SIZE / TRACKER_PAGE_SIZE;
|
||||
|
||||
struct GuestRange {
|
||||
@@ -19,8 +21,11 @@ struct GuestRange {
|
||||
|
||||
[[nodiscard]] constexpr bool Empty() const noexcept { return address == 0 && size == 0; }
|
||||
[[nodiscard]] constexpr bool Valid() const noexcept {
|
||||
return address != 0 && size != 0 && address < TRACKER_ADDRESS_SIZE &&
|
||||
size <= TRACKER_ADDRESS_SIZE - address;
|
||||
if (address == 0 || size == 0) return false;
|
||||
if (address < LOWER_ADDRESS_SIZE) return size <= LOWER_ADDRESS_SIZE - address;
|
||||
constexpr auto base = LibKernel::Memory::kExtendedMemoryBase;
|
||||
constexpr auto length = LibKernel::Memory::kExtendedMemorySize;
|
||||
return address >= base && address - base < length && size <= length - (address - base);
|
||||
}
|
||||
[[nodiscard]] constexpr bool ValidOrEmpty() const noexcept { return Empty() || Valid(); }
|
||||
[[nodiscard]] constexpr uint64_t End() const noexcept { return address + size; }
|
||||
|
||||
+11
-5
@@ -55,7 +55,8 @@ template <bool insert>
|
||||
void BufferCache::ChangeRegister(BufferId id) {
|
||||
auto& buffer = m_slot_buffers[id];
|
||||
PageTable::PageRange pages {};
|
||||
EXIT_IF(!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
|
||||
EXIT_IF(!(GuestRange {buffer.CpuAddress(), buffer.Size()}.Valid()) ||
|
||||
!PageTable::TryGetPageRange(buffer.CpuAddress(), buffer.Size(), pages));
|
||||
for (size_t page = pages.first; page < pages.last_exclusive; ++page) {
|
||||
if constexpr (insert) {
|
||||
m_page_table[page] = id;
|
||||
@@ -64,6 +65,7 @@ void BufferCache::ChangeRegister(BufferId id) {
|
||||
}
|
||||
}
|
||||
const auto size_pages = pages.last_exclusive - pages.first;
|
||||
const auto table_offset = PageIndex(buffer.CpuAddress()) * sizeof(vk::DeviceAddress);
|
||||
if constexpr (insert) {
|
||||
const auto [it, inserted] = m_buffers.emplace(buffer.CpuAddress(), id);
|
||||
(void)it;
|
||||
@@ -75,7 +77,7 @@ void BufferCache::ChangeRegister(BufferId id) {
|
||||
for (uint64_t i = 0; i < size_pages; ++i) {
|
||||
addresses.push_back(buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS));
|
||||
}
|
||||
WriteDataBuffer(m_bda_pagetable_buffer, pages.first * sizeof(vk::DeviceAddress),
|
||||
WriteDataBuffer(m_bda_pagetable_buffer, table_offset,
|
||||
addresses.data(), addresses.size() * sizeof(vk::DeviceAddress));
|
||||
} else {
|
||||
const auto found = m_buffers.find(buffer.CpuAddress());
|
||||
@@ -84,7 +86,7 @@ void BufferCache::ChangeRegister(BufferId id) {
|
||||
EXIT_IF(buffer.Size() > m_total_used_memory);
|
||||
m_total_used_memory -= buffer.Size();
|
||||
m_lru_cache.Free(buffer.lru_id);
|
||||
m_bda_pagetable_buffer.Fill(pages.first * sizeof(vk::DeviceAddress),
|
||||
m_bda_pagetable_buffer.Fill(table_offset,
|
||||
size_pages * sizeof(vk::DeviceAddress), 0);
|
||||
buffer.is_deleted = true;
|
||||
}
|
||||
@@ -312,10 +314,14 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(uint64_t vaddr, uint64_t
|
||||
// Reserve space in the incoming stream's direction of growth.
|
||||
// The old buffer extending left of the request predicts growth to the right, and vice versa.
|
||||
if (expands_left) {
|
||||
end += std::min(StreamLeapSize, PageTable::kAddressSpaceSize - end);
|
||||
end += std::min(StreamLeapSize, (vaddr < LOWER_ADDRESS_SIZE ? LOWER_ADDRESS_SIZE
|
||||
: LibKernel::Memory::kExtendedMemoryBase +
|
||||
LibKernel::Memory::kExtendedMemorySize) - end);
|
||||
}
|
||||
if (expands_right) {
|
||||
const auto minimum = CACHING_PAGESIZE * 2;
|
||||
const auto minimum = vaddr < LOWER_ADDRESS_SIZE
|
||||
? CACHING_PAGESIZE * 2
|
||||
: LibKernel::Memory::kExtendedMemoryBase;
|
||||
if (begin > minimum) {
|
||||
begin -= std::min(StreamLeapSize, begin - minimum);
|
||||
}
|
||||
|
||||
+14
-2
@@ -29,10 +29,22 @@ class BufferCache {
|
||||
public:
|
||||
static constexpr uint32_t CACHING_PAGEBITS = 14;
|
||||
static constexpr uint64_t CACHING_PAGESIZE = uint64_t {1} << CACHING_PAGEBITS;
|
||||
static constexpr uint64_t CACHING_NUMPAGES = uint64_t {1} << (40 - CACHING_PAGEBITS);
|
||||
static constexpr uint64_t CACHING_NUMPAGES = (LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemorySize) >> CACHING_PAGEBITS;
|
||||
static constexpr uint64_t BDA_PAGETABLE_SIZE =
|
||||
CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
|
||||
|
||||
static constexpr uint64_t PageIndex(uint64_t address) {
|
||||
return (address < LOWER_ADDRESS_SIZE
|
||||
? address
|
||||
: address - LibKernel::Memory::kExtendedMemoryBase + LOWER_ADDRESS_SIZE) >>
|
||||
CACHING_PAGEBITS;
|
||||
}
|
||||
static constexpr uint64_t GuestAddress(uint64_t offset) {
|
||||
return offset < LOWER_ADDRESS_SIZE
|
||||
? offset
|
||||
: offset - LOWER_ADDRESS_SIZE + LibKernel::Memory::kExtendedMemoryBase;
|
||||
}
|
||||
|
||||
BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
|
||||
TextureCache& texture_cache);
|
||||
~BufferCache();
|
||||
@@ -88,7 +100,7 @@ private:
|
||||
bool has_stream_leap;
|
||||
};
|
||||
|
||||
using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
|
||||
using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 44, 20>;
|
||||
static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
|
||||
void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
|
||||
void TouchBuffer(const Buffer& buffer);
|
||||
|
||||
+3
-2
@@ -134,8 +134,9 @@ void FaultManager::ProcessFaultBuffer() {
|
||||
const auto* faults = std::bit_cast<const uint64_t*>(mapped);
|
||||
const auto count = static_cast<uint32_t>(faults[0]);
|
||||
for (uint32_t index = 1; index <= count; ++index) {
|
||||
fault_ranges.Add(faults[index], BufferCache::CACHING_PAGESIZE);
|
||||
LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
|
||||
const auto address = BufferCache::GuestAddress(faults[index]);
|
||||
fault_ranges.Add(address, BufferCache::CACHING_PAGESIZE);
|
||||
LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", address);
|
||||
}
|
||||
fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
|
||||
EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());
|
||||
|
||||
+12
-14
@@ -14,10 +14,10 @@ SamplerCache::~SamplerCache() {
|
||||
}
|
||||
}
|
||||
|
||||
vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
|
||||
vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r, bool integer_border) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
|
||||
const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3]};
|
||||
const SamplerKey key {r.fields[0], r.fields[1], r.fields[2], r.fields[3], integer_border};
|
||||
if (auto iter = m_samplers.find(key); iter != m_samplers.end()) {
|
||||
return iter->second;
|
||||
}
|
||||
@@ -55,11 +55,9 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
|
||||
case Prospero::SamplerAnisoRatio::kTwo: aniso_ratio = 2.0f; break;
|
||||
case Prospero::SamplerAnisoRatio::kFour: aniso_ratio = 4.0f; break;
|
||||
case Prospero::SamplerAnisoRatio::kEight: aniso_ratio = 8.0f; break;
|
||||
case Prospero::SamplerAnisoRatio::kSixteen: aniso_ratio = 16.0f; break;
|
||||
default:
|
||||
EXIT("unknown ratio: %d dwords=%08x,%08x,%08x,%08x\n",
|
||||
static_cast<int>(r.MaxAnisoRatio()), r.fields[0], r.fields[1], r.fields[2],
|
||||
r.fields[3]);
|
||||
case Prospero::SamplerAnisoRatio::kSixteen:
|
||||
// Reserved encodings 5-7 fall back to the maximum ratio.
|
||||
default: aniso_ratio = 16.0f; break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,23 +93,23 @@ vk::Sampler SamplerCache::GetSampler(const ShaderSamplerResource& r) {
|
||||
return vk::SamplerAddressMode::eClampToBorder;
|
||||
};
|
||||
|
||||
vk::BorderColor border = vk::BorderColor::eIntTransparentBlack;
|
||||
vk::BorderColor border = integer_border ? vk::BorderColor::eIntTransparentBlack
|
||||
: vk::BorderColor::eFloatTransparentBlack;
|
||||
switch (static_cast<Prospero::SamplerBorderColor>(r.BorderColorType())) {
|
||||
case Prospero::SamplerBorderColor::kTransBlack:
|
||||
border = vk::BorderColor::eIntTransparentBlack;
|
||||
break;
|
||||
case Prospero::SamplerBorderColor::kTransBlack: break;
|
||||
case Prospero::SamplerBorderColor::kOpaqueBlack:
|
||||
border = vk::BorderColor::eIntOpaqueBlack;
|
||||
border = integer_border ? vk::BorderColor::eIntOpaqueBlack
|
||||
: vk::BorderColor::eFloatOpaqueBlack;
|
||||
break;
|
||||
case Prospero::SamplerBorderColor::kOpaqueWhite:
|
||||
border = vk::BorderColor::eIntOpaqueWhite;
|
||||
border = integer_border ? vk::BorderColor::eIntOpaqueWhite
|
||||
: vk::BorderColor::eFloatOpaqueWhite;
|
||||
break;
|
||||
case Prospero::SamplerBorderColor::kFromTable:
|
||||
LOGF(
|
||||
"temporary: approximating table border color as transparent black, index = %" PRIu16
|
||||
"\n",
|
||||
r.BorderColorPtr());
|
||||
border = vk::BorderColor::eIntTransparentBlack;
|
||||
break;
|
||||
default: EXIT("unknown border color: %d", static_cast<int>(r.BorderColorType()));
|
||||
}
|
||||
|
||||
+2
-2
@@ -24,10 +24,10 @@ public:
|
||||
~SamplerCache();
|
||||
KYTY_CLASS_NO_COPY(SamplerCache);
|
||||
|
||||
vk::Sampler GetSampler(const ShaderSamplerResource& r);
|
||||
vk::Sampler GetSampler(const ShaderSamplerResource& r, bool integer_border);
|
||||
|
||||
private:
|
||||
using SamplerKey = std::array<uint32_t, 4>;
|
||||
using SamplerKey = std::array<uint32_t, 5>;
|
||||
|
||||
struct SamplerKeyHash {
|
||||
std::size_t operator()(const SamplerKey& key) const {
|
||||
|
||||
+55
-85
@@ -33,24 +33,27 @@ namespace {
|
||||
|
||||
constexpr uint64_t NumFramesBeforeRemoval = 32;
|
||||
|
||||
[[nodiscard]] bool DecodeDccClear(const TextureCache::ImageDesc& desc, uint8_t code,
|
||||
[[nodiscard]] bool DecodeColorClear(const TextureCache::ImageDesc& desc, uint8_t code,
|
||||
vk::ClearColorValue& clear) {
|
||||
const auto& metadata = desc.info.metadata;
|
||||
const auto format = desc.view_info.format;
|
||||
const bool cmask = metadata.kind == ImageMetadataKind::Cmask;
|
||||
if (cmask ? code == 0 : code == 0x20) {
|
||||
// Register clears belong to the color buffer; the texture pipe cannot decode them.
|
||||
return desc.type == TextureCache::BindingType::RenderTarget &&
|
||||
metadata.clear_register_valid &&
|
||||
DecodePackedColorClear(format, metadata.clear_word, clear);
|
||||
}
|
||||
if (cmask) {
|
||||
return false;
|
||||
}
|
||||
switch (code) {
|
||||
case 0x00:
|
||||
case 0x20:
|
||||
case 0x40:
|
||||
case 0x80:
|
||||
case 0xc0: break;
|
||||
default: return false;
|
||||
}
|
||||
const auto& metadata = desc.info.metadata;
|
||||
const auto format = desc.view_info.format;
|
||||
if (code == 0x20) {
|
||||
// Clear-to-register is a color-buffer operation; the texture pipe cannot decode it.
|
||||
return desc.type == TextureCache::BindingType::RenderTarget &&
|
||||
metadata.dcc_clear_register_valid &&
|
||||
DecodePackedColorClear(format, metadata.dcc_clear_word, clear);
|
||||
}
|
||||
clear = {};
|
||||
if (code == 0x00) {
|
||||
return true;
|
||||
@@ -107,36 +110,6 @@ constexpr uint64_t NumFramesBeforeRemoval = 32;
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] const char* BindingTypeName(TextureCache::BindingType type) {
|
||||
switch (type) {
|
||||
case TextureCache::BindingType::Texture: return "Texture";
|
||||
case TextureCache::BindingType::Storage: return "StorageTexture";
|
||||
case TextureCache::BindingType::RenderTarget: return "ColorTarget";
|
||||
case TextureCache::BindingType::DepthTarget: return "DepthTarget";
|
||||
case TextureCache::BindingType::VideoOut: return "VideoOut";
|
||||
}
|
||||
return "Image";
|
||||
}
|
||||
|
||||
void NameImageBinding(GraphicContext& graphics, Image& image, vk::ImageView view,
|
||||
TextureCache::BindingType type, const ImageViewInfo& view_info) {
|
||||
const auto* role = BindingTypeName(type);
|
||||
SetVulkanObjectNameF(
|
||||
graphics.device, image.backing.image,
|
||||
"Kyty.{}.Image[guest=0x{:016x} size=0x{:x} extent={}x{}x{} format={} mips={} layers={} "
|
||||
"samples={}]",
|
||||
role, image.info.data.address, image.info.data.size, image.info.extent.width,
|
||||
image.info.extent.height, image.info.extent.depth,
|
||||
static_cast<uint32_t>(image.info.pixel_format), image.info.resources.levels,
|
||||
image.info.resources.layers, image.info.samples);
|
||||
SetVulkanObjectNameF(
|
||||
graphics.device, view,
|
||||
"Kyty.{}.View[guest=0x{:016x} format={} aspect=0x{:x} mip={}+{} layer={}+{}]", role,
|
||||
image.info.data.address, static_cast<uint32_t>(view_info.format),
|
||||
static_cast<vk::ImageAspectFlags::MaskType>(view_info.aspect), view_info.base_level,
|
||||
view_info.level_count, view_info.base_layer, view_info.layer_count);
|
||||
}
|
||||
|
||||
[[nodiscard]] std::vector<vk::BufferImageCopy> BuildDepthCopies(const ImageInfo& info,
|
||||
uint64_t slice_stride,
|
||||
vk::ImageAspectFlags aspect) {
|
||||
@@ -223,7 +196,7 @@ bool TextureCache::SameBacking(const ImageInfo& cached, const ImageInfo& request
|
||||
return false;
|
||||
}
|
||||
if (!ImageViewOps::FormatsCompatible(cached.pixel_format, requested.pixel_format) ||
|
||||
(cached.type != requested.type && requested.extent != vk::Extent3D {1, 1, 1})) {
|
||||
cached.type != requested.type) {
|
||||
return false;
|
||||
}
|
||||
if (exact_format && cached.pixel_format != requested.pixel_format) {
|
||||
@@ -249,14 +222,6 @@ TextureCache::BindingType TextureCache::UploadBinding(const Image& image) {
|
||||
return image.usage.storage ? BindingType::Storage : BindingType::Texture;
|
||||
}
|
||||
|
||||
bool TextureCache::SafeToDownload(const Image& image) {
|
||||
if (!image.SafeToDownload()) {
|
||||
return false;
|
||||
}
|
||||
const auto range = image.info.data;
|
||||
return !m_buffer_cache.HasGpuDirtyBytes(range.address, range.size);
|
||||
}
|
||||
|
||||
ImageId TextureCache::InsertImage(const ImageInfo& info) {
|
||||
const auto id = m_slot_images.insert(m_graphics, m_scheduler, info);
|
||||
if (!info.data.Empty()) {
|
||||
@@ -669,9 +634,12 @@ void TextureCache::CopyImage(ImageId destination_id, ImageId source_id) {
|
||||
const bool source_depth = source.info.IsDepth();
|
||||
const bool dest_depth = destination.info.IsDepth();
|
||||
const bool direct_copy =
|
||||
source.backing.format == destination.backing.format ||
|
||||
(!source_depth && !dest_depth &&
|
||||
vk::blockSize(source.backing.format) == vk::blockSize(destination.backing.format));
|
||||
(source.backing.image_type == destination.backing.image_type ||
|
||||
(source.backing.image_type != vk::ImageType::e1D &&
|
||||
destination.backing.image_type != vk::ImageType::e1D)) &&
|
||||
(source.backing.format == destination.backing.format ||
|
||||
(!source_depth && !dest_depth &&
|
||||
vk::blockSize(source.backing.format) == vk::blockSize(destination.backing.format)));
|
||||
if (direct_copy) {
|
||||
destination.CopyImage(source);
|
||||
} else if (!CopyD16(destination, source)) {
|
||||
@@ -705,7 +673,8 @@ void TextureCache::CopyImageMip(ImageId destination_id, ImageId source_id, uint3
|
||||
ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingType binding,
|
||||
ImageId cached_id) {
|
||||
auto& cached = m_slot_images[cached_id];
|
||||
if (!cached.info.IsDepth() && !requested.IsDepth()) {
|
||||
if ((!cached.info.IsDepth() && !requested.IsDepth()) ||
|
||||
cached.info.tile_mode != requested.tile_mode) {
|
||||
return {};
|
||||
}
|
||||
const bool stencil_match = requested.HasStencil() == cached.info.HasStencil();
|
||||
@@ -719,8 +688,8 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
|
||||
requested.samples == 1 && requested.data == cached.info.data &&
|
||||
requested.extent == cached.info.extent && requested.resources == cached.info.resources &&
|
||||
requested.type == cached.info.type && requested.pitch == cached.info.pitch &&
|
||||
requested.tile_mode == cached.info.tile_mode && !requested.HasStencil() &&
|
||||
!cached.info.HasStencil() && !requested.HasMetadata() && !cached.info.HasMetadata();
|
||||
!requested.HasStencil() && !cached.info.HasStencil() && !requested.HasMetadata() &&
|
||||
!cached.info.HasMetadata();
|
||||
// PPSA04264
|
||||
const bool retain_cached_layout =
|
||||
requested.samples == 1 && cached.info.samples == 1 && cached.backing.samples == 1 &&
|
||||
@@ -731,7 +700,7 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
|
||||
requested.resources.layers != 0 && cached.info.resources.layers != 0 &&
|
||||
requested.resources.layers < cached.info.resources.layers &&
|
||||
requested.type == cached.info.type && requested.pitch == cached.info.pitch &&
|
||||
requested.tile_mode == cached.info.tile_mode && requested.mip_layout[0].offset == 0 &&
|
||||
requested.mip_layout[0].offset == 0 &&
|
||||
cached.info.mip_layout[0].offset == 0 &&
|
||||
requested.mip_layout[0].size == requested.data.size &&
|
||||
cached.info.mip_layout[0].size == cached.info.data.size &&
|
||||
@@ -840,12 +809,15 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
|
||||
return {merged_id};
|
||||
}
|
||||
// PPSA08394
|
||||
// A view cannot change the native image type or grow its extent.
|
||||
if (requested.data.size == cached.info.data.size &&
|
||||
requested.resources == cached.info.resources && requested.type == cached.info.type &&
|
||||
requested.extent.width > cached.info.extent.width &&
|
||||
requested.extent.height >= cached.info.extent.height &&
|
||||
requested.extent.depth >= cached.info.extent.depth &&
|
||||
ImageViewOps::FormatsCompatible(cached.info.pixel_format, requested.pixel_format)) {
|
||||
requested.resources == cached.info.resources &&
|
||||
ImageViewOps::FormatsCompatible(cached.info.pixel_format, requested.pixel_format) &&
|
||||
(requested.type != cached.info.type
|
||||
? requested.extent == cached.info.extent
|
||||
: requested.extent.width > cached.info.extent.width &&
|
||||
requested.extent.height >= cached.info.extent.height &&
|
||||
requested.extent.depth >= cached.info.extent.depth)) {
|
||||
return {ExpandImage(requested, cached_id)};
|
||||
}
|
||||
// PS5 mip tails can expose more levels without increasing the guest allocation.
|
||||
@@ -1148,9 +1120,10 @@ void TextureCache::InitializeImage(ImageId id) {
|
||||
}
|
||||
}
|
||||
|
||||
void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
|
||||
void TextureCache::MaterializeColorClear(ImageId id, const ImageDesc& desc,
|
||||
uint32_t metadata_base_layer) {
|
||||
if (desc.info.metadata.kind != ImageMetadataKind::Dcc) {
|
||||
if (desc.info.metadata.kind != ImageMetadataKind::Dcc &&
|
||||
desc.info.metadata.kind != ImageMetadataKind::Cmask) {
|
||||
return;
|
||||
}
|
||||
const auto range = desc.info.metadata.range;
|
||||
@@ -1158,18 +1131,18 @@ void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
|
||||
std::scoped_lock lock {m_lock};
|
||||
auto& image = m_slot_images[id];
|
||||
image.info.metadata = desc.info.metadata;
|
||||
// A native DCC allocation must not retain a reused HTile/CMask/FMask interpretation.
|
||||
// Native color metadata must not retain a reused HTile/CMask/FMask clear flag.
|
||||
m_surface_metas.erase(range.address);
|
||||
if (range.size == 0 || desc.info.resources.levels != 1 || image.info.resources.levels != 1) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const auto layers = desc.info.TransferLayers();
|
||||
// These one-mip surfaces use complete 4 KiB DCC metadata blocks.
|
||||
// These one-mip surfaces use complete 4 KiB color metadata blocks.
|
||||
constexpr uint64_t MetadataBlockSize = 0x1000;
|
||||
if (!range.Valid() || range.address % MetadataBlockSize != 0 || layers == 0 ||
|
||||
range.size % layers != 0 || (range.size / layers) % MetadataBlockSize != 0) {
|
||||
EXIT("TextureCache: DCC slices must contain aligned 4 KiB blocks\n");
|
||||
EXIT("TextureCache: color metadata slices must contain aligned 4 KiB blocks\n");
|
||||
}
|
||||
const auto& view = desc.view_info;
|
||||
const bool volume_texture = desc.info.IsVolume() && view.type == vk::ImageViewType::e3D;
|
||||
@@ -1177,7 +1150,7 @@ void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
|
||||
const auto image_first = volume_texture ? 0u : view.base_layer;
|
||||
const auto count = volume_texture ? desc.info.extent.depth : view.layer_count;
|
||||
if (first >= layers || count > layers - first) {
|
||||
EXIT("TextureCache: DCC view exceeds its native metadata slices\n");
|
||||
EXIT("TextureCache: color view exceeds its native metadata slices\n");
|
||||
}
|
||||
// Finish native metadata writes before reading backing bytes. This can submit the scheduler,
|
||||
// so discovery runs before final draw uploads and never holds the texture lock across it.
|
||||
@@ -1189,15 +1162,15 @@ void TextureCache::MaterializeDccClear(ImageId id, const ImageDesc& desc,
|
||||
const auto address = range.address + slice_size * (first + slice);
|
||||
uint8_t code = 0;
|
||||
if (!LibKernel::Memory::TryReadBacking(address, &code, sizeof(code))) {
|
||||
EXIT("TextureCache: failed to read DCC metadata backing\n");
|
||||
EXIT("TextureCache: failed to read color metadata backing\n");
|
||||
}
|
||||
vk::ClearValue clear {};
|
||||
if (!DecodeDccClear(desc, code, clear.color)) {
|
||||
if (!DecodeColorClear(desc, code, clear.color)) {
|
||||
continue;
|
||||
}
|
||||
std::vector<uint8_t> bytes(slice_size);
|
||||
if (!LibKernel::Memory::TryReadBacking(address, bytes.data(), bytes.size())) {
|
||||
EXIT("TextureCache: failed to read DCC metadata slice\n");
|
||||
EXIT("TextureCache: failed to read color metadata slice\n");
|
||||
}
|
||||
if (!std::all_of(bytes.begin(), bytes.end(), [code](uint8_t byte) { return byte == code; })) {
|
||||
continue;
|
||||
@@ -1342,7 +1315,7 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
|
||||
image.tick_accessed_last = m_scheduler.CurrentTick();
|
||||
TouchImage(image);
|
||||
}
|
||||
MaterializeDccClear(result, desc, metadata_base_layer);
|
||||
MaterializeColorClear(result, desc, metadata_base_layer);
|
||||
if (desc.type == BindingType::VideoOut &&
|
||||
desc.info.metadata.compression != VideoOutCompression::Uncompressed) {
|
||||
std::scoped_lock lock {m_lock};
|
||||
@@ -1379,7 +1352,7 @@ ImageId TextureCache::FindImageFromRange(uint64_t address, uint64_t size, bool e
|
||||
if (ensure_valid && owner->depth_id) {
|
||||
owner = m_slot_images.try_get(owner->depth_id);
|
||||
}
|
||||
if (owner == nullptr || (ensure_valid && !SafeToDownload(*owner))) {
|
||||
if (owner == nullptr || (ensure_valid && !owner->SafeToDownload())) {
|
||||
continue;
|
||||
}
|
||||
matches.push_back(id);
|
||||
@@ -1443,9 +1416,7 @@ vk::ImageView TextureCache::FindTexture(ImageId id, const ImageDesc& desc) {
|
||||
break;
|
||||
default: EXIT("TextureCache: invalid texture binding\n");
|
||||
}
|
||||
const auto view = image.FindView(desc.view_info);
|
||||
NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
|
||||
return view;
|
||||
return image.FindView(desc.view_info);
|
||||
}
|
||||
|
||||
vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc) {
|
||||
@@ -1463,9 +1434,7 @@ vk::ImageView TextureCache::FindRenderTarget(ImageId id, const ImageDesc& desc)
|
||||
RefreshImage(id);
|
||||
CommitGpuWrite(image);
|
||||
TrackImageDownload(id, image);
|
||||
const auto view = image.FindView(desc.view_info);
|
||||
NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
|
||||
return view;
|
||||
return image.FindView(desc.view_info);
|
||||
}
|
||||
|
||||
vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
|
||||
@@ -1492,9 +1461,7 @@ vk::ImageView TextureCache::FindDepthTarget(ImageId id, const ImageDesc& desc) {
|
||||
if (desc.info.HasStencil()) {
|
||||
RefreshImage(AssociateStencil(id, desc.info.stencil));
|
||||
}
|
||||
const auto view = image.FindView(desc.view_info);
|
||||
NameImageBinding(m_graphics, image, view, desc.type, desc.view_info);
|
||||
return view;
|
||||
return image.FindView(desc.view_info);
|
||||
}
|
||||
|
||||
void TextureCache::MarkGpuWritten(ImageId id) {
|
||||
@@ -1776,7 +1743,7 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uin
|
||||
std::scoped_lock lock {m_texture_cache.m_lock};
|
||||
auto& image = m_texture_cache.m_slot_images[selected];
|
||||
// The GPU thread owns image retirement; CPU invalidation can dirty this image after lookup.
|
||||
if (!m_texture_cache.SafeToDownload(image)) {
|
||||
if (!image.SafeToDownload()) {
|
||||
return false;
|
||||
}
|
||||
if (!buffer.IsInBounds(image.info.data.address, 1)) {
|
||||
@@ -1839,7 +1806,7 @@ bool TextureCache::DownloadImageMemory(ImageId id) {
|
||||
return false;
|
||||
}
|
||||
auto transfer = BuildDownload(image);
|
||||
if (!transfer.valid || !SafeToDownload(image)) {
|
||||
if (!transfer.valid || !image.SafeToDownload()) {
|
||||
return false;
|
||||
}
|
||||
const auto range = image.info.data;
|
||||
@@ -1900,7 +1867,10 @@ bool TextureCache::IsRegionGpuModified(uint64_t address, uint64_t size) {
|
||||
std::scoped_lock lock {m_lock};
|
||||
for (const auto id: FindImagesInRegion(address, size, false)) {
|
||||
const auto& image = m_slot_images[id];
|
||||
if (!image.depth_id && image.IsGpuModified()) {
|
||||
// PPSA17168: S_LOAD_DWORD reads shader data at an address overlapping an old
|
||||
// render target whose memory the CPU has reused. The cached image still retains
|
||||
// its earlier GPU-modified flag.
|
||||
if (!image.depth_id && image.IsGpuModified() && !image.IsDefinitelyCpuDirty()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -2026,7 +1996,7 @@ void TextureCache::RunGarbageCollector() {
|
||||
continue;
|
||||
}
|
||||
if (owner->IsGpuModified()) {
|
||||
const bool safe = SafeToDownload(*owner);
|
||||
const bool safe = owner->SafeToDownload();
|
||||
if (safe && owner->info.IsTiled()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
+3
-4
@@ -91,9 +91,9 @@ private:
|
||||
};
|
||||
|
||||
using ImageIds = InlinePageOwnerList<ImageId, 16>;
|
||||
using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
|
||||
using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 44, 14>;
|
||||
|
||||
// Callers have validated the nonempty 40-bit range with TryGetPageRange.
|
||||
// Callers have validated the nonempty 44-bit range with TryGetPageRange.
|
||||
template <typename Func>
|
||||
static void ForEachPage(uint64_t address, size_t size, Func&& func) {
|
||||
using FuncReturn = typename std::invoke_result<Func, uint64_t>::type;
|
||||
@@ -128,7 +128,6 @@ private:
|
||||
[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
|
||||
bool exact_format);
|
||||
[[nodiscard]] static BindingType UploadBinding(const Image& image);
|
||||
[[nodiscard]] bool SafeToDownload(const Image& image);
|
||||
|
||||
// Caller holds m_lock; it also serializes the per-image query epoch.
|
||||
[[nodiscard]] ImageIds FindImagesInRegion(uint64_t address, uint64_t size,
|
||||
@@ -139,7 +138,7 @@ private:
|
||||
ImageId cached);
|
||||
[[nodiscard]] ImageId ExpandImage(const ImageInfo& info, ImageId source);
|
||||
void RefreshImage(ImageId id);
|
||||
void MaterializeDccClear(ImageId id, const ImageDesc& desc,
|
||||
void MaterializeColorClear(ImageId id, const ImageDesc& desc,
|
||||
uint32_t metadata_base_layer);
|
||||
void InitializeImage(ImageId id);
|
||||
[[nodiscard]] TextureTransfer
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <bit>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
@@ -138,8 +139,7 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
|
||||
// Color-control state selects the color-buffer operation and logical blend operation.
|
||||
// The normal copy operation is a regular color write, not an attachment clear.
|
||||
// Nonlinear clear values are still stored as normalized components.
|
||||
// Fast color clears are metadata driven and must be handled explicitly when
|
||||
// that metadata path is implemented; render-pass load must preserve contents.
|
||||
// Metadata clears are materialized during image discovery; render-pass loads preserve contents.
|
||||
uint32_t width = 0;
|
||||
uint32_t height = 0;
|
||||
uint32_t pitch = 0;
|
||||
@@ -160,9 +160,6 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
|
||||
break;
|
||||
default: EXIT("unknown tile mode: %u\n", static_cast<uint32_t>(rt.attrib3.tile_mode));
|
||||
}
|
||||
if (!tile && levels > 1) {
|
||||
EXIT("linear mipmapped render targets are unsupported\n");
|
||||
}
|
||||
if (samples > 1 && (!tile || levels != 1)) {
|
||||
EXIT("multisampled render targets require a single-mip tiled surface\n");
|
||||
}
|
||||
@@ -203,8 +200,7 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
|
||||
" layer=%u/%u\n",
|
||||
rt.attrib3.dimension, rt.attrib3.depth, view.base_layer, view.image_layers);
|
||||
}
|
||||
// PPSA28068: the linear EULA target and its sampled view must share the padded pitch.
|
||||
if (!tile || volume || texture_tile) {
|
||||
if (samples == 1) {
|
||||
pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
|
||||
} else {
|
||||
pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
|
||||
@@ -232,37 +228,26 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
|
||||
}
|
||||
size = volume_layout.block_slice_size;
|
||||
backing_size = volume_layout.total_size;
|
||||
} else if (tile) {
|
||||
} else {
|
||||
TileSizeAlign layout {};
|
||||
bool valid_layout = false;
|
||||
if (texture_tile) {
|
||||
if (samples == 1) {
|
||||
TileGetTextureSize(transfer_format, width, height, levels, rt.attrib3.tile_mode,
|
||||
&layout, mip_sizes, mip_padded);
|
||||
valid_layout = layout.size != 0 && layout.align == texture_tile_layout.block.block_size;
|
||||
valid_layout = layout.size != 0 && layout.align != 0 &&
|
||||
(rt.attrib3.tile_mode != Prospero::TileMode::kRenderTarget ||
|
||||
levels <= std::bit_width(std::max(width, height)));
|
||||
} else {
|
||||
valid_layout =
|
||||
levels == 1 ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element,
|
||||
layout, rt.attrib.num_fragments)
|
||||
: TileGetRenderTargetMipLayout(width, height, pitch, bytes_per_element,
|
||||
levels, layout, mip_sizes, mip_padded);
|
||||
valid_layout = TileGetRenderTargetSize(width, height, pitch, bytes_per_element,
|
||||
layout, rt.attrib.num_fragments);
|
||||
mip_sizes[0] = {layout.size, 0, 0, 0, 0, 0};
|
||||
mip_padded[0] = {pitch, height};
|
||||
}
|
||||
if (!valid_layout) {
|
||||
EXIT("unsupported render-target layout: %ux%u pitch=%u bytes=%u levels=%u\n", width,
|
||||
height, pitch, bytes_per_element, levels);
|
||||
}
|
||||
size = layout.size;
|
||||
EXIT_IF(size > UINT32_MAX);
|
||||
if (levels == 1) {
|
||||
mip_sizes[0] = {static_cast<uint32_t>(size), 0, 0, 0, 0, 0};
|
||||
mip_padded[0] = {pitch, height};
|
||||
}
|
||||
} else {
|
||||
size = static_cast<uint64_t>(pitch) * height * bytes_per_element * samples;
|
||||
if (size > UINT32_MAX) {
|
||||
EXIT("linear render-target slice exceeds the supported layout size\n");
|
||||
}
|
||||
mip_sizes[0] = {static_cast<uint32_t>(size), 0, 0, 0, 0, 0};
|
||||
mip_padded[0] = {pitch, height};
|
||||
}
|
||||
if (size == 0 || (!volume && size > UINT64_MAX / view.image_layers)) {
|
||||
EXIT("render-target memory footprint is invalid\n");
|
||||
@@ -304,15 +289,25 @@ void RenderExecutor::ResolveRenderColorTarget(CommandBuffer& buffer, RenderColor
|
||||
desc.info.samples = samples;
|
||||
desc.info.tile_mode = rt.attrib3.tile_mode;
|
||||
const bool has_dcc = rt.info.dcc_compression_enable && rt.dcc_addr.addr != 0;
|
||||
if (has_dcc) {
|
||||
const bool has_cmask = !rt.info.dcc_compression_enable && rt.info.cmask_fast_clear_enable &&
|
||||
rt.cmask.addr != 0 && samples == 1 &&
|
||||
!rt.info.fmask_compression_enable &&
|
||||
!rt.attrib3.write_vrs_rate_hint_to_cmask;
|
||||
if (has_dcc || has_cmask) {
|
||||
TileSizeAlign metadata_size {};
|
||||
(void)TileGetDccSize(width, height, volume ? depth : view.image_layers, bytes_per_element,
|
||||
levels, rt.attrib3.tile_mode, metadata_size, rt.attrib.num_fragments);
|
||||
desc.info.metadata.kind = ImageMetadataKind::Dcc;
|
||||
desc.info.metadata.range = {rt.dcc_addr.addr, metadata_size.size};
|
||||
desc.info.metadata.dcc_clear_word = rt.clear_word0.word0;
|
||||
desc.info.metadata.dcc_clear_register_valid = true;
|
||||
desc.info.metadata.dcc_alpha_msb = DccAlphaOnMsb(rt.info);
|
||||
const auto layers = volume ? depth : view.image_layers;
|
||||
if (has_dcc) {
|
||||
(void)TileGetDccSize(width, height, layers, bytes_per_element, levels,
|
||||
rt.attrib3.tile_mode, metadata_size, rt.attrib.num_fragments);
|
||||
desc.info.metadata.dcc_alpha_msb = DccAlphaOnMsb(rt.info);
|
||||
} else {
|
||||
(void)TileGetCmaskSize(width, height, layers, levels, metadata_size);
|
||||
}
|
||||
// DCC owns the clear when both planes are enabled; single-sample CMASK stays expanded.
|
||||
desc.info.metadata.kind = has_dcc ? ImageMetadataKind::Dcc : ImageMetadataKind::Cmask;
|
||||
desc.info.metadata.range = {has_dcc ? rt.dcc_addr.addr : rt.cmask.addr, metadata_size.size};
|
||||
desc.info.metadata.clear_word = rt.clear_word0.word0;
|
||||
desc.info.metadata.clear_register_valid = true;
|
||||
}
|
||||
for (uint32_t level = 0; level < levels; level++) {
|
||||
if (volume) {
|
||||
|
||||
@@ -219,17 +219,6 @@ static void RtCheck(const HW::RenderTarget& rt) {
|
||||
}
|
||||
}
|
||||
|
||||
if (rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable) {
|
||||
static bool logged = false;
|
||||
if (!logged) {
|
||||
LOGF("RenderTarget: temporary: ignoring PS5 color metadata fast_clear=%s dcc=%s "
|
||||
"cmask=0x%016" PRIx64 " dcc_addr=0x%016" PRIx64 "\n",
|
||||
rt.info.cmask_fast_clear_enable ? "true" : "false",
|
||||
rt.info.dcc_compression_enable ? "true" : "false", rt.cmask.addr,
|
||||
rt.dcc_addr.addr);
|
||||
logged = true;
|
||||
}
|
||||
}
|
||||
if (rt.info.blend_bypass) {
|
||||
static bool logged = false;
|
||||
if (!logged) {
|
||||
|
||||
@@ -627,12 +627,11 @@ void Validate(const ImageInfo& info) {
|
||||
}
|
||||
break;
|
||||
case ImageMetadataKind::Dcc:
|
||||
if (info.metadata.range.address == 0 ||
|
||||
info.metadata.range.address >= TRACKER_ADDRESS_SIZE ||
|
||||
(info.metadata.range.size != 0 &&
|
||||
info.metadata.range.size > TRACKER_ADDRESS_SIZE - info.metadata.range.address) ||
|
||||
case ImageMetadataKind::Cmask:
|
||||
if (!GuestRange {info.metadata.range.address,
|
||||
std::max<uint64_t>(info.metadata.range.size, 1)}.Valid() ||
|
||||
info.metadata.compression == VideoOutCompression::Unsupported) {
|
||||
EXIT("invalid DCC metadata\n");
|
||||
EXIT("invalid color metadata\n");
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -690,6 +689,12 @@ Image::Image(GraphicContext& graphics, CommandScheduler& scheduler, const ImageI
|
||||
create.extent.width, create.extent.height, create.extent.depth,
|
||||
static_cast<int>(create.format), create.arrayLayers, create.mipLevels);
|
||||
}
|
||||
SetVulkanObjectNameF(
|
||||
graphics.device, backing.image,
|
||||
"Kyty.Image[guest=0x{:016x} size=0x{:x} extent={}x{}x{} format={} mips={} layers={} samples={}]",
|
||||
info.data.address, info.data.size, info.extent.width, info.extent.height, info.extent.depth,
|
||||
static_cast<uint32_t>(info.pixel_format), info.resources.levels, info.resources.layers,
|
||||
info.samples);
|
||||
}
|
||||
|
||||
uint64_t Image::HashGuestEdges() const {
|
||||
|
||||
@@ -17,17 +17,17 @@ namespace Libs::Graphics {
|
||||
|
||||
enum class VideoOutCompression : uint8_t { Uncompressed, Dcc256_256_0, Dcc256_64_64, Unsupported };
|
||||
|
||||
enum class ImageMetadataKind : uint8_t { None, Htile, Dcc };
|
||||
enum class ImageMetadataKind : uint8_t { None, Htile, Dcc, Cmask };
|
||||
|
||||
struct ImageMetadataInfo {
|
||||
GuestRange range;
|
||||
ImageMetadataKind kind = ImageMetadataKind::None;
|
||||
uint32_t control = 0;
|
||||
uint32_t dcc_clear_word = 0;
|
||||
VideoOutCompression compression = VideoOutCompression::Uncompressed;
|
||||
bool stencil_compressed = false;
|
||||
bool dcc_clear_register_valid = false;
|
||||
bool dcc_alpha_msb = true;
|
||||
ImageMetadataKind kind = ImageMetadataKind::None;
|
||||
uint32_t control = 0;
|
||||
uint32_t clear_word = 0;
|
||||
VideoOutCompression compression = VideoOutCompression::Uncompressed;
|
||||
bool stencil_compressed = false;
|
||||
bool clear_register_valid = false;
|
||||
bool dcc_alpha_msb = true;
|
||||
};
|
||||
|
||||
struct ImageSubresources {
|
||||
@@ -329,7 +329,9 @@ struct VideoOutFormatPolicy {
|
||||
VideoOutPixelFormatInfo info;
|
||||
};
|
||||
|
||||
inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {{
|
||||
inline constexpr uint64_t VIDEO_OUT_PIXEL_FORMAT_R10_G10_B10_A2_BT2100_PQ = 0x8100070422000000ull;
|
||||
|
||||
inline constexpr std::array<VideoOutFormatPolicy, 7> VIDEO_OUT_FORMAT_POLICIES {{
|
||||
{0x8000000022000000ull,
|
||||
{vk::Format::eR8G8B8A8Srgb, Prospero::BufferFormat::k8_8_8_8Srgb, 4, false}},
|
||||
{0x8000000000000000ull,
|
||||
@@ -338,6 +340,8 @@ inline constexpr std::array<VideoOutFormatPolicy, 6> VIDEO_OUT_FORMAT_POLICIES {
|
||||
{vk::Format::eA2B10G10R10UnormPack32, Prospero::BufferFormat::k10_10_10_2UNorm, 4, false}},
|
||||
{0x8100000000000000ull,
|
||||
{vk::Format::eA2R10G10B10UnormPack32, Prospero::BufferFormat::k10_10_10_2UNorm, 4, false}},
|
||||
{VIDEO_OUT_PIXEL_FORMAT_R10_G10_B10_A2_BT2100_PQ,
|
||||
{vk::Format::eA2B10G10R10UnormPack32, Prospero::BufferFormat::k10_10_10_2UNorm, 4, false}},
|
||||
{0xc001000622000000ull,
|
||||
{vk::Format::eR16G16B16A16Sfloat, Prospero::BufferFormat::k16_16_16_16Float, 8, false}},
|
||||
{0xc001000600000000ull,
|
||||
|
||||
@@ -391,6 +391,12 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
|
||||
view_info.level_count, view_info.base_layer, view_info.layer_count,
|
||||
static_cast<vk::ImageUsageFlags::MaskType>(view_info.usage));
|
||||
}
|
||||
SetVulkanObjectNameF(
|
||||
m_graphics.device, view,
|
||||
"Kyty.ImageView[guest=0x{:016x} format={} aspect=0x{:x} mip={}+{} layer={}+{}]",
|
||||
info.data.address, static_cast<uint32_t>(normalized.format),
|
||||
static_cast<vk::ImageAspectFlags::MaskType>(normalized.aspect), normalized.base_level,
|
||||
normalized.level_count, normalized.base_layer, normalized.layer_count);
|
||||
views.push_back({normalized, view});
|
||||
return view;
|
||||
}
|
||||
|
||||
@@ -91,19 +91,6 @@ vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t el
|
||||
return {nullptr, view, texture.layout};
|
||||
}
|
||||
|
||||
static const char* ShaderStageResourceName(ShaderType stage) {
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex: return "Vertex";
|
||||
case ShaderType::Mesh: return "Mesh";
|
||||
case ShaderType::Local: return "Local";
|
||||
case ShaderType::TessellationControl: return "Hull";
|
||||
case ShaderType::TessellationEvaluation: return "Domain";
|
||||
case ShaderType::Pixel: return "Pixel";
|
||||
case ShaderType::Compute: return "Compute";
|
||||
default: return "Unknown";
|
||||
}
|
||||
}
|
||||
|
||||
static Prospero::ImageType TextureType(const ShaderTextureResource& descriptor) {
|
||||
const auto type = descriptor.Type();
|
||||
return type == Prospero::ImageType::kCube ? Prospero::ImageType::kColor2DArray : type;
|
||||
@@ -126,8 +113,7 @@ static bool IsMultisampledTexture(Prospero::ImageType type) {
|
||||
|
||||
static vk::DescriptorBufferInfo
|
||||
NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource& source,
|
||||
const ShaderRecompiler::IR::BufferResource& resource, ShaderType stage,
|
||||
uint32_t slot, uint32_t& buffer_offset) {
|
||||
const ShaderRecompiler::IR::BufferResource& resource, uint32_t& buffer_offset) {
|
||||
buffer_offset = 0;
|
||||
|
||||
const auto& [address, size, id] = source;
|
||||
@@ -152,16 +138,6 @@ NativeStorageBuffer(RenderContext& context, const PreparedBindings::BufferSource
|
||||
if (resource.written) {
|
||||
context.GetTextureCache().InvalidateMemoryFromGPU(address, size);
|
||||
}
|
||||
const char* access = "Read";
|
||||
if (resource.written && resource.read) {
|
||||
access = "ReadWrite";
|
||||
} else if (resource.written) {
|
||||
access = "Write";
|
||||
}
|
||||
SetVulkanObjectNameF(
|
||||
graphics.device, result.buffer,
|
||||
"Kyty.{}.StorageBuffer[slot={} guest=0x{:016x} size=0x{:x} access={} formatted={}]",
|
||||
ShaderStageResourceName(stage), slot, address, size, access, resource.formatted);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -196,8 +172,8 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, bo
|
||||
constexpr uint32_t htile_control = 0x00280000u;
|
||||
const uint32_t expected_control = htile_control | (descriptor.MsaaDepth() ? (1u << 10u) : 0u);
|
||||
const auto metadata_addr = descriptor.MetaAddr() << 8u;
|
||||
return metadata_control == expected_control && metadata_addr != 0 &&
|
||||
metadata_addr < TRACKER_ADDRESS_SIZE && (metadata_addr & 0x7fffu) == 0 &&
|
||||
return metadata_control == expected_control && GuestRange {metadata_addr, 1}.Valid() &&
|
||||
(metadata_addr & 0x7fffu) == 0 &&
|
||||
descriptor.TileMode() == Prospero::TileMode::kDepth;
|
||||
}
|
||||
|
||||
@@ -335,8 +311,10 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
const bool raw_sint_storage = format == Prospero::BufferFormat::k32SInt && uint_resource &&
|
||||
resource.written && !resource.read && !resource.atomic;
|
||||
const auto numeric_class = Prospero::SampledTextureNumericClass(format);
|
||||
const bool raw_float_atomic = format == Prospero::BufferFormat::k32Float && uint_resource &&
|
||||
resource.atomic;
|
||||
const bool format_ok =
|
||||
raw_sint_storage ||
|
||||
raw_sint_storage || raw_float_atomic ||
|
||||
(numeric_class != Prospero::TextureNumericClass::Unsupported &&
|
||||
numeric_class != Prospero::TextureNumericClass::Sint &&
|
||||
uint_resource == (numeric_class == Prospero::TextureNumericClass::Uint) &&
|
||||
@@ -512,19 +490,50 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
|
||||
return view;
|
||||
}
|
||||
|
||||
static bool TextureViewPreservesMipLayout(const TileSurfaceDescription& description,
|
||||
uint32_t view_levels) {
|
||||
static bool ResolveTextureMipView(const TileSurfaceDescription& description, bool metadata,
|
||||
uint32_t view_levels, uint32_t& levels, uint32_t& base_level) {
|
||||
TileSurfaceLayout physical {};
|
||||
TileSurfaceLayout view {};
|
||||
auto view_description = description;
|
||||
view_description.levels = view_levels;
|
||||
return TileGetTiledTextureLayout(description, physical) &&
|
||||
TileGetTiledTextureLayout(view_description, view) &&
|
||||
physical.first_tail_level == view.first_tail_level &&
|
||||
physical.block_slice_size == view.block_slice_size &&
|
||||
physical.total_size == view.total_size &&
|
||||
std::equal(std::begin(physical.mips), std::begin(physical.mips) + description.levels,
|
||||
std::begin(view.mips));
|
||||
view_description.levels = levels;
|
||||
if (!TileGetTiledTextureLayout(description, physical) ||
|
||||
!TileGetTiledTextureLayout(view_description, view)) {
|
||||
return false;
|
||||
}
|
||||
if (physical.first_tail_level == view.first_tail_level &&
|
||||
physical.block_slice_size == view.block_slice_size &&
|
||||
physical.total_size == view.total_size &&
|
||||
std::equal(std::begin(physical.mips), std::begin(physical.mips) + description.levels,
|
||||
std::begin(view.mips))) {
|
||||
return true;
|
||||
}
|
||||
if (metadata || ((description.layers > 1 || description.depth > 1) &&
|
||||
physical.block_slice_size != view.block_slice_size)) {
|
||||
return false;
|
||||
}
|
||||
// T# addresses the last mip. A view can select the same stored subresources
|
||||
// with different mip indices; inaccessible mips need no host representation.
|
||||
for (uint32_t base = 0; base + view_levels <= description.levels; ++base) {
|
||||
bool matches = true;
|
||||
for (uint32_t i = 0; i < view_levels; ++i) {
|
||||
const auto source = base_level + i;
|
||||
const auto target = base + i;
|
||||
if (physical.mips[target] != view.mips[source] ||
|
||||
(target >= physical.first_tail_level) != (source >= view.first_tail_level) ||
|
||||
std::max(description.width >> target, 1u) != std::max(description.width >> source, 1u) ||
|
||||
std::max(description.height >> target, 1u) != std::max(description.height >> source, 1u) ||
|
||||
std::max(description.depth >> target, 1u) != std::max(description.depth >> source, 1u)) {
|
||||
matches = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matches) {
|
||||
levels = description.levels;
|
||||
base_level = base;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
|
||||
@@ -558,7 +567,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
|
||||
const auto view_levels = multisampled || single_storage_mip
|
||||
? 1u
|
||||
: static_cast<uint32_t>(last_level - base_level) + 1u;
|
||||
const auto levels =
|
||||
auto levels =
|
||||
multisampled ? 1u : std::max(physical_levels, base_level + view_levels);
|
||||
const auto tile = descriptor.TileMode();
|
||||
const bool depth_tile = tile == Prospero::TileMode::kDepth;
|
||||
@@ -600,12 +609,13 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
|
||||
type == Prospero::ImageType::kColor2DArray ||
|
||||
type == Prospero::ImageType::kColor2DMsaaArray;
|
||||
const auto image_layers = layered ? depth : 1u;
|
||||
auto view_base = static_cast<uint32_t>(base_level);
|
||||
if (levels > physical_levels) {
|
||||
const TileSurfaceDescription physical {
|
||||
format, tile, volume ? TileSurfaceDimension::Dim3D : TileSurfaceDimension::Dim2D,
|
||||
width, height, volume ? depth : 1u, physical_levels, image_layers};
|
||||
// Texture mip views take precedence over the resource count, but must keep its storage layout.
|
||||
if (!TextureViewPreservesMipLayout(physical, levels)) {
|
||||
if (!ResolveTextureMipView(physical, !resource.r128 && descriptor.MetaCompress(),
|
||||
view_levels, levels, view_base)) {
|
||||
EXIT("unsupported texture mip view changes physical layout: base=%u last=%u max=%u "
|
||||
"extent=%ux%ux%u tile=%u\n",
|
||||
base_level, last_level, max_mip, width, height, depth,
|
||||
@@ -640,7 +650,8 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
|
||||
pixel_format = depth_format->depth_attachment_format;
|
||||
}
|
||||
}
|
||||
const auto storage_view_format = storage && format == Prospero::BufferFormat::k32SInt
|
||||
const auto storage_view_format = storage && (resource.atomic ||
|
||||
format == Prospero::BufferFormat::k32SInt)
|
||||
? vk::Format::eR32Uint
|
||||
: SrgbStorageViewFormat(pixel_format);
|
||||
const auto view_format = storage && storage_view_format != vk::Format::eUndefined
|
||||
@@ -676,6 +687,7 @@ TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageR
|
||||
}
|
||||
desc.view_info = TextureViewInfo(resource, descriptor, view_format, surface_format, storage,
|
||||
view_levels, desc.info.resources.layers);
|
||||
desc.view_info.base_level = view_base;
|
||||
desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
|
||||
|
||||
auto id = texture_cache.FindImage(desc, shader_conversion);
|
||||
@@ -702,14 +714,15 @@ static vk::Sampler NativeSampler(RenderContext& context,
|
||||
const ShaderRecompiler::IR::CompiledShaderInfo& program,
|
||||
uint32_t index,
|
||||
const ShaderRecompiler::IR::DescriptorValue& value) {
|
||||
auto descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
|
||||
if (!program.info.samplers[index].depth_compare) {
|
||||
auto descriptor = DecodeNativeDescriptor<ShaderSamplerResource>(value);
|
||||
const auto& sampler = program.info.samplers[index];
|
||||
if (!sampler.depth_compare) {
|
||||
descriptor.fields[0] &= ~(0x7u << 12u);
|
||||
}
|
||||
if (program.info.samplers[index].force_point_filtering) {
|
||||
if (sampler.force_point_filtering) {
|
||||
descriptor.SetPointFiltering();
|
||||
}
|
||||
return context.GetSamplerCache().GetSampler(descriptor);
|
||||
return context.GetSamplerCache().GetSampler(descriptor, sampler.integer_border);
|
||||
}
|
||||
|
||||
static vk::DescriptorBufferInfo NativeUpload(RenderContext& context,
|
||||
@@ -795,9 +808,14 @@ void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
|
||||
auto& cache = m_context.GetBufferCache();
|
||||
|
||||
prepared.buffer_sources.clear();
|
||||
prepared.buffer_sources.reserve(program.info.buffers.size());
|
||||
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
|
||||
auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[i]);
|
||||
const auto& layout = program.bindings;
|
||||
if (layout.memory_offset_count == 0) {
|
||||
return;
|
||||
}
|
||||
const auto& resources = layout.descriptors.front().resources;
|
||||
prepared.buffer_sources.reserve(resources.size());
|
||||
for (const auto resource: resources) {
|
||||
auto descriptor = DecodeNativeDescriptor<ShaderBufferResource>(snapshot.buffers[resource]);
|
||||
const auto address = descriptor.Base48();
|
||||
const auto requested_size = descriptor.GetSize();
|
||||
if (address == 0 || requested_size == 0) {
|
||||
@@ -815,10 +833,10 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
|
||||
const auto& program = *prepared.runtime->program;
|
||||
const auto& snapshot = *prepared.runtime->resources;
|
||||
const auto& layout = program.bindings;
|
||||
EXIT_IF(prepared.buffer_sources.size() != program.info.buffers.size());
|
||||
EXIT_IF(prepared.buffer_sources.size() != layout.memory_offset_count);
|
||||
|
||||
prepared.buffers.clear();
|
||||
prepared.buffers.reserve(program.info.buffers.size());
|
||||
prepared.buffers.reserve(layout.memory_offset_count);
|
||||
EXIT_IF(prepared.shader_data.size() != layout.ShaderDataDwords());
|
||||
std::fill(prepared.shader_data.begin() + layout.memory_offset_dword,
|
||||
prepared.shader_data.end(), 0);
|
||||
@@ -827,10 +845,11 @@ void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
|
||||
const auto shift = (index % 4u) * 8u;
|
||||
prepared.shader_data[dword] |= offset << shift;
|
||||
};
|
||||
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
|
||||
for (uint32_t i = 0; i < layout.memory_offset_count; i++) {
|
||||
const auto resource = layout.descriptors.front().resources[i];
|
||||
uint32_t buffer_offset = 0;
|
||||
prepared.buffers.push_back(NativeStorageBuffer(m_context, prepared.buffer_sources[i],
|
||||
program.info.buffers[i], program.stage, i,
|
||||
program.info.buffers[resource],
|
||||
buffer_offset));
|
||||
pack_memory_offset(i, buffer_offset);
|
||||
}
|
||||
@@ -1042,8 +1061,8 @@ void RenderExecutor::CommitBindings(CommandBuffer& buffer,
|
||||
} else {
|
||||
switch (binding.kind) {
|
||||
case BindingKind::Buffers:
|
||||
for (const auto resource: binding.resources) {
|
||||
const auto& view = descriptors.buffers.at(resource);
|
||||
EXIT_IF(descriptors.buffers.size() != binding.resources.size());
|
||||
for (const auto& view: descriptors.buffers) {
|
||||
EXIT_IF(view.buffer == nullptr);
|
||||
m_descriptor_buffers.push_back(view);
|
||||
}
|
||||
|
||||
@@ -29,7 +29,6 @@
|
||||
#include <span>
|
||||
#include <spirv-tools/libspirv.hpp>
|
||||
#include <string_view>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <xxhash.h>
|
||||
@@ -116,33 +115,22 @@ void DumpShaderSpirv(const char* stage_name, uint64_t shader_hash,
|
||||
}
|
||||
|
||||
void DumpShaderOriginal(const char* stage_name, uint64_t shader_hash,
|
||||
std::span<const uint32_t> code, const std::string& decoded_dump) {
|
||||
std::span<const uint32_t> code) {
|
||||
if (!Config::GraphicsDebugDumpEnabled()) {
|
||||
return;
|
||||
}
|
||||
EXIT_IF(code.empty());
|
||||
static std::atomic_int id = 0;
|
||||
const auto base = Config::GetShaderLogFolder() / "original" /
|
||||
fmt::format("{:04d}_new_shader_{}_{:016x}", id++, stage_name, shader_hash);
|
||||
Common::File::CreateDirectories(base.parent_path());
|
||||
for (const auto& [suffix, data, size]: {
|
||||
std::tuple {".bin", static_cast<const void*>(code.data()), code.size_bytes()},
|
||||
std::tuple {".rdna2", static_cast<const void*>(decoded_dump.data()),
|
||||
decoded_dump.size()},
|
||||
}) {
|
||||
if (size == 0) {
|
||||
continue;
|
||||
}
|
||||
auto path = base;
|
||||
path += suffix;
|
||||
Common::File file(path);
|
||||
if (file.IsInvalid()) {
|
||||
const auto path_text = Common::PathToString(path);
|
||||
LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
|
||||
} else {
|
||||
file.Write(data, size);
|
||||
}
|
||||
const auto path = Config::GetShaderLogFolder() / "original" /
|
||||
fmt::format("{:04d}_new_shader_{}_{:016x}.bin", id++, stage_name, shader_hash);
|
||||
Common::File::CreateDirectories(path.parent_path());
|
||||
Common::File file(path);
|
||||
if (file.IsInvalid()) {
|
||||
const auto path_text = Common::PathToString(path);
|
||||
LOGF_COLOR(Log::Color::BrightRed, "Can't create file: %s\n", path_text.c_str());
|
||||
return;
|
||||
}
|
||||
file.Write(code.data(), code.size_bytes());
|
||||
}
|
||||
|
||||
bool ValidateShaderSpirv(const char* label, uint64_t shader_hash,
|
||||
@@ -204,6 +192,7 @@ struct PipelineCache::ProgramCache {
|
||||
ShaderRecompiler::IR::ResourceSnapshot resources;
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization;
|
||||
std::vector<Permutation> permutations;
|
||||
bool skip_dispatch = false;
|
||||
};
|
||||
|
||||
struct ProgramKeyHash {
|
||||
@@ -224,25 +213,13 @@ struct PipelineCache::ProgramCache {
|
||||
|
||||
static constexpr std::size_t MaxStaticKeyWords = 13 + ShaderVertexInputInfo::RES_MAX * 13;
|
||||
|
||||
Permutation CompilePermutation(const ShaderParams& params,
|
||||
Permutation CompilePermutation(const char* stage_name,
|
||||
const ShaderRecompiler::CompileOptions& options,
|
||||
ShaderRecompiler::TranslateResult translated,
|
||||
ShaderRecompiler::IR::ResourceSpecialization specialization,
|
||||
uint32_t push_data_start_dword) {
|
||||
const char* stage_name = nullptr;
|
||||
switch (options.stage) {
|
||||
case ShaderType::Vertex: stage_name = "vs"; break;
|
||||
case ShaderType::Mesh: stage_name = "ms"; break;
|
||||
case ShaderType::Local: stage_name = "ls"; break;
|
||||
case ShaderType::TessellationControl: stage_name = "hs"; break;
|
||||
case ShaderType::TessellationEvaluation: stage_name = "ds"; break;
|
||||
case ShaderType::Pixel: stage_name = "ps"; break;
|
||||
case ShaderType::Compute: stage_name = "cs"; break;
|
||||
default: EXIT("invalid pipeline shader stage\n");
|
||||
}
|
||||
auto result = ShaderRecompiler::CompileProgram(std::move(translated), options,
|
||||
specialization, push_data_start_dword);
|
||||
DumpShaderOriginal(stage_name, options.shader_hash, params.code, result.decoded_dump);
|
||||
if (!ValidateShaderSpirv(options.dump_label, options.shader_hash, result.spirv)) {
|
||||
DumpShaderSpirv(stage_name, options.shader_hash, result.spirv);
|
||||
EXIT("%s failed hash=0x%016" PRIx64 ": SPIR-V validation failed\n", options.dump_label,
|
||||
@@ -283,6 +260,9 @@ struct PipelineCache::ProgramCache {
|
||||
lookup_key.code_size = static_cast<uint32_t>(params.code.size());
|
||||
BuildStageStaticKey(input_info, lookup_key.static_state);
|
||||
auto entry = programs.find(lookup_key);
|
||||
if (entry != programs.end() && entry->second.skip_dispatch) {
|
||||
return {};
|
||||
}
|
||||
const ShaderRecompiler::IR::SrtRuntime runtime {
|
||||
.user_data = user_data,
|
||||
.shader_base = params.Base(),
|
||||
@@ -317,14 +297,15 @@ struct PipelineCache::ProgramCache {
|
||||
stage_input.compute = &input_info;
|
||||
}
|
||||
const char* label = nullptr;
|
||||
const char* stage_name = nullptr;
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex: label = "ShaderRecompiler VS"; break;
|
||||
case ShaderType::Mesh: label = "ShaderRecompiler MS"; break;
|
||||
case ShaderType::Local: label = "ShaderRecompiler LS"; break;
|
||||
case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; break;
|
||||
case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; break;
|
||||
case ShaderType::Pixel: label = "ShaderRecompiler PS"; break;
|
||||
case ShaderType::Compute: label = "ShaderRecompiler CS"; break;
|
||||
case ShaderType::Vertex: label = "ShaderRecompiler VS"; stage_name = "vs"; break;
|
||||
case ShaderType::Mesh: label = "ShaderRecompiler MS"; stage_name = "ms"; break;
|
||||
case ShaderType::Local: label = "ShaderRecompiler LS"; stage_name = "ls"; break;
|
||||
case ShaderType::TessellationControl: label = "ShaderRecompiler HS"; stage_name = "hs"; break;
|
||||
case ShaderType::TessellationEvaluation: label = "ShaderRecompiler DS"; stage_name = "ds"; break;
|
||||
case ShaderType::Pixel: label = "ShaderRecompiler PS"; stage_name = "ps"; break;
|
||||
case ShaderType::Compute: label = "ShaderRecompiler CS"; stage_name = "cs"; break;
|
||||
default: EXIT("invalid pipeline shader stage\n");
|
||||
}
|
||||
ShaderRecompiler::CompileOptions options;
|
||||
@@ -347,7 +328,13 @@ struct PipelineCache::ProgramCache {
|
||||
} else {
|
||||
options.wave_size = input_info.wave_size;
|
||||
}
|
||||
DumpShaderOriginal(stage_name, options.shader_hash, params.code);
|
||||
auto translated = ShaderRecompiler::TranslateProgram(params.code, options);
|
||||
if (translated.skip_dispatch) {
|
||||
entry = programs.try_emplace(lookup_key, ShaderRecompiler::IR::ResourcePlan {}).first;
|
||||
entry->second.skip_dispatch = true;
|
||||
return {};
|
||||
}
|
||||
if (entry == programs.end()) {
|
||||
entry = programs.try_emplace(lookup_key,
|
||||
ShaderRecompiler::IR::ExtractResourcePlan(translated.program)).first;
|
||||
@@ -356,7 +343,7 @@ struct PipelineCache::ProgramCache {
|
||||
entry->second.specialization));
|
||||
}
|
||||
entry->second.permutations.push_back(CompilePermutation(
|
||||
params, options, std::move(translated), entry->second.specialization, push_data_cursor));
|
||||
stage_name, options, std::move(translated), entry->second.specialization, push_data_cursor));
|
||||
const auto& permutation = entry->second.permutations.back();
|
||||
input_info.stage = {.program = &permutation.program, .resources = &entry->second.resources};
|
||||
permutation.program.bindings.AdvancePushData(push_data_cursor);
|
||||
|
||||
@@ -368,6 +368,8 @@ void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::Pipeline& p
|
||||
// MoltenVK lacks VK_EXT_depth_clip_enable; omit the depth-clip struct on macOS and accept
|
||||
// Vulkan's default depth clipping (enabled) instead of the PS5's clamp behavior.
|
||||
#if !defined(__APPLE__)
|
||||
// The DB clamps depth to the viewport range after polygon offset is applied.
|
||||
rasterizer.depthClampEnable = VK_TRUE;
|
||||
rasterizer.pNext = &clip_ext;
|
||||
#endif
|
||||
vk::PipelineRasterizationProvokingVertexStateCreateInfoEXT provoking_vertex {};
|
||||
|
||||
@@ -256,6 +256,10 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
input_info.dispatch_thread_dimensions = use_thread_dimensions;
|
||||
const auto compute_program =
|
||||
m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
|
||||
if (!compute_program) {
|
||||
// Temporary until RT is implemented.
|
||||
return;
|
||||
}
|
||||
if (use_thread_dimensions) {
|
||||
input_info.dispatch_threads_num[0] = thread_group_x;
|
||||
input_info.dispatch_threads_num[1] = thread_group_y;
|
||||
@@ -413,6 +417,10 @@ void RenderExecutor::DispatchIndirect(uint64_t submit_id, CommandBuffer& buffer,
|
||||
ShaderComputeInputInfo input_info {};
|
||||
const auto compute_program = m_context.GetPipelineCache().GetComputeProgram(
|
||||
cs_regs, buffer.GetRegisters().GetShaderRegisters(), input_info);
|
||||
if (!compute_program) {
|
||||
// Temporary until RT is implemented.
|
||||
return;
|
||||
}
|
||||
buffer.EndRendering();
|
||||
auto& pipeline = m_context.GetPipelineCache().GetComputePipeline(input_info, compute_program);
|
||||
auto& bindings = m_compute_bindings;
|
||||
|
||||
@@ -473,15 +473,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
}
|
||||
const auto image_view = cache.FindRenderTarget(target.image_id, target.desc);
|
||||
auto& image = cache.GetImage(target.image_id);
|
||||
SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
|
||||
"Kyty.MRT{}.Image[guest=0x{:016x} size=0x{:x} format={}]",
|
||||
target.target_slot, image.info.data.address, image.info.data.size,
|
||||
static_cast<uint32_t>(image.info.pixel_format));
|
||||
SetVulkanObjectNameF(m_context.GetGraphics().device, image_view,
|
||||
"Kyty.MRT{}.View[guest=0x{:016x} mip={} layer={}+{}]",
|
||||
target.target_slot, image.info.data.address,
|
||||
target.desc.view_info.base_level, target.desc.view_info.base_layer,
|
||||
target.desc.view_info.layer_count);
|
||||
EXIT_IF(image.backing.samples != target.desc.info.samples || image_view == nullptr);
|
||||
const auto& view = target.desc.view_info;
|
||||
const auto layout = image.binding.is_bound ? vk::ImageLayout::eGeneral
|
||||
@@ -522,14 +513,6 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
EXIT("failed to consume HTile clear state\n");
|
||||
}
|
||||
auto& image = cache.GetImage(depth.image_id);
|
||||
SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
|
||||
"Kyty.DepthTarget.Image[guest=0x{:016x} size=0x{:x} format={}]",
|
||||
image.info.data.address, image.info.data.size,
|
||||
static_cast<uint32_t>(image.info.pixel_format));
|
||||
SetVulkanObjectNameF(m_context.GetGraphics().device, image_view,
|
||||
"Kyty.DepthTarget.View[guest=0x{:016x} layer={}+{}]",
|
||||
image.info.data.address, depth.desc.view_info.base_layer,
|
||||
depth.desc.view_info.layer_count);
|
||||
EXIT_IF(image_view == nullptr || image.backing.samples != depth.desc.info.samples);
|
||||
const auto draw_writes = depth.AttachmentWriteAspects();
|
||||
vk::ImageAspectFlags sampled_aspects;
|
||||
@@ -592,12 +575,17 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
|
||||
return state;
|
||||
}
|
||||
|
||||
static bool DrawHasActivePixelShader(const CommandBuffer& buffer) {
|
||||
const auto& ctx = buffer.GetRegisters();
|
||||
const auto& sh_regs = ctx.GetShaderRegisters();
|
||||
const bool has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
|
||||
return buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
|
||||
(has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
|
||||
static uint32_t DrawColorOutputMask(const HW::Context& ctx) {
|
||||
const auto& sh_regs = ctx.GetShaderRegisters();
|
||||
const auto write_mask = ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask;
|
||||
uint32_t output_mask = 0;
|
||||
for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
|
||||
if (sh_regs.target_output_mode[slot] != 0 &&
|
||||
render_target_mask_slot(write_mask, slot) != 0) {
|
||||
output_mask |= 1u << slot;
|
||||
}
|
||||
}
|
||||
return output_mask;
|
||||
}
|
||||
|
||||
enum class CbColorMode : uint8_t {
|
||||
@@ -686,11 +674,13 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer& b
|
||||
vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX);
|
||||
|
||||
// Collect the non-empty guest vertex ranges.
|
||||
std::array<uint64_t, ShaderVertexInputInfo::RES_MAX> sizes {};
|
||||
std::array<VertexBufferRange, ShaderVertexInputInfo::RES_MAX> ranges {};
|
||||
uint32_t range_count = 0;
|
||||
for (int i = 0; i < vs_input_info.buffers_num; i++) {
|
||||
const auto& vertex = vs_input_info.buffers[i];
|
||||
const auto size = VertexBufferDescriptorSize(vertex, vs_input_info);
|
||||
sizes[i] = size;
|
||||
if (size == 0) {
|
||||
continue;
|
||||
}
|
||||
@@ -728,9 +718,6 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer& b
|
||||
Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
|
||||
range.acquired_end = range.base_address + size;
|
||||
range.binding = cache.ObtainBuffer(range.base_address, size, false);
|
||||
SetVulkanObjectNameF(
|
||||
buffer.GetContext().GetGraphics().device, range.binding.first->Handle(),
|
||||
"Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", range.base_address, size);
|
||||
}
|
||||
|
||||
// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
|
||||
@@ -739,7 +726,7 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer& b
|
||||
vk::Buffer null_buffer = nullptr;
|
||||
for (int i = 0; i < vs_input_info.buffers_num; i++) {
|
||||
const auto& vertex = vs_input_info.buffers[i];
|
||||
const auto size = VertexBufferDescriptorSize(vertex, vs_input_info);
|
||||
const auto size = sizes[i];
|
||||
if (size == 0) {
|
||||
if (null_buffer == nullptr) {
|
||||
null_buffer = cache.GetBuffer(NULL_BUFFER_ID).Handle();
|
||||
@@ -762,10 +749,6 @@ static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer& b
|
||||
prepared.buffers[i] = range->binding.first->Handle();
|
||||
prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address;
|
||||
prepared.sizes[i] = std::min(size, range->acquired_end - vertex.addr);
|
||||
SetVulkanObjectNameF(
|
||||
buffer.GetContext().GetGraphics().device, prepared.buffers[i],
|
||||
"Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i,
|
||||
vertex.addr, size, vertex.stride, vertex.num_records);
|
||||
}
|
||||
|
||||
return prepared;
|
||||
@@ -862,7 +845,7 @@ static bool ResolvePrimitiveRestart(const CommandBuffer& buffer,
|
||||
}
|
||||
|
||||
static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
|
||||
DrawRenderState& state) {
|
||||
uint32_t color_output_mask, DrawRenderState& state) {
|
||||
auto& ctx = buffer.GetRegisters();
|
||||
auto& sh_ctx = buffer.GetShaders();
|
||||
|
||||
@@ -876,7 +859,7 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
|
||||
target_export_mapping {};
|
||||
for (uint32_t slot = 0; slot < RENDER_COLOR_ATTACHMENTS_MAX; slot++) {
|
||||
const auto& rt = ctx.GetRenderTarget(slot);
|
||||
if (rt.base.addr != 0 && render_target_mask_slot(ctx.GetRenderTargetMask(), slot) != 0) {
|
||||
if ((color_output_mask & (1u << slot)) != 0 && rt.base.addr != 0) {
|
||||
target_export_mapping[slot] =
|
||||
TextureGetRenderTargetFormat(rt.info.format, rt.info.channel_type,
|
||||
rt.info.channel_order)
|
||||
@@ -895,8 +878,12 @@ static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw,
|
||||
bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCallInfo& draw,
|
||||
uint32_t render_target_slice_offset,
|
||||
DrawRenderState& state) {
|
||||
state.ps_active = DrawHasActivePixelShader(buffer);
|
||||
RefreshShaders(buffer, draw, state);
|
||||
const auto& shader_regs = buffer.GetRegisters().GetShaderRegisters();
|
||||
const auto color_output_mask = DrawColorOutputMask(buffer.GetRegisters());
|
||||
state.ps_active = buffer.GetShaders().GetPs().ps_regs.data_addr != 0 &&
|
||||
(color_output_mask != 0 ||
|
||||
PixelShaderHasDepthOrCoverageSideEffects(shader_regs));
|
||||
RefreshShaders(buffer, draw, color_output_mask, state);
|
||||
uint32_t mrt_mask = 0;
|
||||
if (state.ps_active) {
|
||||
for (const auto& output: state.ps_input_info.stage.program->info.outputs) {
|
||||
@@ -905,6 +892,7 @@ bool RenderExecutor::PrepareDrawRenderState(CommandBuffer& buffer, const DrawCal
|
||||
}
|
||||
}
|
||||
}
|
||||
mrt_mask &= color_output_mask;
|
||||
if (draw.IsIndexed()) {
|
||||
LogDrawPhase(draw.Name(), "ResolveRenderColorTarget");
|
||||
}
|
||||
@@ -942,17 +930,11 @@ static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer& buffe
|
||||
auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
|
||||
prepared.offset = stream.Copy(source.host_data, source.size, 16);
|
||||
prepared.buffer = stream.Handle();
|
||||
SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
|
||||
"Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
|
||||
static_cast<uint32_t>(source.type));
|
||||
} else {
|
||||
auto [buffer_ptr, offset] =
|
||||
buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
|
||||
prepared.buffer = buffer_ptr->Handle();
|
||||
prepared.offset = offset;
|
||||
SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
|
||||
"Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
|
||||
source.address, source.size, static_cast<uint32_t>(source.type));
|
||||
}
|
||||
return prepared;
|
||||
}
|
||||
|
||||
@@ -112,26 +112,6 @@ void WriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst
|
||||
EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Write);
|
||||
}
|
||||
|
||||
void WriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst_gpu_addr,
|
||||
uint64_t value) {
|
||||
RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), 0,
|
||||
EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Write);
|
||||
|
||||
LOGF_COLOR(Log::Color::BrightGreen,
|
||||
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
|
||||
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
|
||||
}
|
||||
|
||||
void WriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, CommandBuffer& buffer,
|
||||
uint64_t* dst_gpu_addr, uint64_t value) {
|
||||
RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), 0,
|
||||
EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::WriteBack);
|
||||
|
||||
LOGF_COLOR(Log::Color::BrightGreen,
|
||||
"EndOfPipe Signal!!! [0x%016" PRIx64 "] <- Clock: 0x%016" PRIx64 "\n",
|
||||
reinterpret_cast<uint64_t>(dst_gpu_addr), value);
|
||||
}
|
||||
|
||||
void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
|
||||
uint64_t* dst_gpu_addr, uint64_t value) {
|
||||
RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
|
||||
@@ -192,7 +172,7 @@ uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int f
|
||||
"\n",
|
||||
result, handle, index, flip_mode, flip_arg);
|
||||
}
|
||||
video_out.WaitForSubmitSlot();
|
||||
video_out.WaitForSubmitSlot(handle);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,10 +25,6 @@ void WriteAtEndOfPipe64(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst
|
||||
uint64_t value);
|
||||
void WriteAtEndOfPipeGds32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
|
||||
uint32_t dw_offset, uint32_t dw_num);
|
||||
void WriteAtEndOfPipeClockCounter(uint64_t submit_id, CommandBuffer& buffer, uint64_t* dst_gpu_addr,
|
||||
uint64_t value);
|
||||
void WriteAtEndOfPipeClockCounterWithWriteBack(uint64_t submit_id, CommandBuffer& buffer,
|
||||
uint64_t* dst_gpu_addr, uint64_t value);
|
||||
void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
|
||||
uint32_t* dst_gpu_addr, uint32_t value);
|
||||
void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
#version 450
|
||||
|
||||
layout(location = 0) in vec2 uv;
|
||||
layout(location = 0) out vec4 color;
|
||||
layout(binding = 0) uniform sampler2D source;
|
||||
layout(push_constant) uniform AlphaControl {
|
||||
uint premultiplied;
|
||||
};
|
||||
|
||||
void main() {
|
||||
color = texture(source, uv);
|
||||
if (premultiplied == 0u) {
|
||||
color.rgb *= color.a;
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "common/common.h"
|
||||
|
||||
#include <memory>
|
||||
#include <span>
|
||||
|
||||
namespace Libs::Graphics {
|
||||
|
||||
@@ -15,6 +16,11 @@ struct WindowContext;
|
||||
class Presenter final {
|
||||
public:
|
||||
struct Frame;
|
||||
struct Layer {
|
||||
Frame* frame;
|
||||
int bus;
|
||||
bool premultiplied_alpha;
|
||||
};
|
||||
|
||||
explicit Presenter(WindowContext& window);
|
||||
~Presenter();
|
||||
@@ -23,11 +29,13 @@ public:
|
||||
[[nodiscard]] Frame& PrepareFrame(CommandBuffer& command, const ImageInfo& info);
|
||||
[[nodiscard]] Frame& PrepareBlankFrame(uint32_t width, uint32_t height, bool opaque,
|
||||
CommandBuffer* producer = nullptr);
|
||||
[[nodiscard]] Frame* PrepareLastFrame();
|
||||
[[nodiscard]] bool PresentLastFrame();
|
||||
[[nodiscard]] bool IsGuestPaused() const noexcept;
|
||||
[[nodiscard]] bool NeedsSystemOverlayRefresh() const noexcept;
|
||||
[[nodiscard]] RenderContext& Renderer() const noexcept;
|
||||
void Present(Frame& frame, bool reuse = false);
|
||||
void Present(Frame& frame);
|
||||
void Present(std::span<const Layer> layers);
|
||||
void ClearLayer(int bus);
|
||||
void Discard(Frame& frame);
|
||||
|
||||
private:
|
||||
|
||||
@@ -24,6 +24,8 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <fmt/format.h>
|
||||
#include <list>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
@@ -49,6 +51,8 @@ constexpr int VIDEO_OUT_BUS_TYPE_OVERLAY = 1;
|
||||
constexpr int VIDEO_OUT_BUS_TYPE_SUB = 2;
|
||||
constexpr int VIDEO_OUT_FLIP_MODE_VSYNC = 1;
|
||||
constexpr int VIDEO_OUT_FLIP_MODE_VSYNC_MULTI = 4;
|
||||
constexpr int VIDEO_OUT_FLIP_MODE_SLAVE = 8;
|
||||
constexpr int VIDEO_OUT_FLIP_MODE_MASTER = 9;
|
||||
constexpr int VIDEO_OUT_BUFFER_INDEX_BLACK = -2;
|
||||
constexpr int VIDEO_OUT_BUFFER_INDEX_BLANK = -1;
|
||||
constexpr int VIDEO_OUT_BUFFER_NUM_MAX = 16;
|
||||
@@ -61,6 +65,7 @@ constexpr uint64_t VIDEO_OUT_REFRESH_RATE_119_88HZ = 13;
|
||||
constexpr int VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_UNCOMPRESSED = 0;
|
||||
constexpr int VIDEO_OUT_BUFFER_ATTRIBUTE_CATEGORY_COMPRESSED = 1;
|
||||
constexpr uint64_t VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY = 8;
|
||||
constexpr uint64_t VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_ALPHA_PREMULTIPLIED = 32;
|
||||
|
||||
enum class VideoOutEventKind : uintptr_t {
|
||||
Flip = VIDEO_OUT_EVENT_FLIP,
|
||||
@@ -181,6 +186,9 @@ struct VideoOutConfig {
|
||||
uint64_t generation = 0;
|
||||
bool opened = false;
|
||||
bool closing = false;
|
||||
int bus = 0;
|
||||
VideoOutConfig* master = nullptr;
|
||||
uint32_t slave_count = 0;
|
||||
int flip_rate = 0;
|
||||
uint64_t output_mode = VIDEO_OUT_OUTPUT_MODE_DEFAULT;
|
||||
float gamma = 1.0f;
|
||||
@@ -204,7 +212,7 @@ public:
|
||||
void Cancel(VideoOutConfig& cfg);
|
||||
void Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer);
|
||||
void Complete(uint64_t request_id);
|
||||
void WaitForSubmitSlot();
|
||||
void WaitForSubmitSlot(VideoOutConfig& cfg);
|
||||
bool Flip(uint32_t micros);
|
||||
void GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out);
|
||||
void Wait(VideoOutConfig& cfg, int index);
|
||||
@@ -214,7 +222,9 @@ private:
|
||||
|
||||
struct Request {
|
||||
uint64_t id;
|
||||
uint64_t group;
|
||||
VideoOutConfig* cfg;
|
||||
VideoOutConfig* master;
|
||||
uint64_t generation;
|
||||
int index;
|
||||
int64_t flip_arg;
|
||||
@@ -222,6 +232,7 @@ private:
|
||||
FlipRequestSource source;
|
||||
RequestState state;
|
||||
Graphics::Presenter::Frame* frame;
|
||||
bool premultiplied_alpha;
|
||||
};
|
||||
|
||||
Graphics::Presenter& m_presenter;
|
||||
@@ -256,6 +267,7 @@ public:
|
||||
VideoOutConfig* Get(int handle);
|
||||
VideoOutConfig* Get(int handle, uint64_t& generation);
|
||||
bool IsOpened(int handle);
|
||||
int SetFlipMaster(int slave_handle, int master_handle);
|
||||
|
||||
void Init(uint32_t width, uint32_t height);
|
||||
FlipQueue& GetFlipQueue() { return m_flip_queue; }
|
||||
@@ -496,7 +508,8 @@ static bool IsSpecialBufferIndex(int index) {
|
||||
}
|
||||
|
||||
static bool IsValidFlipMode(int mode) {
|
||||
return mode >= VIDEO_OUT_FLIP_MODE_VSYNC && mode <= VIDEO_OUT_FLIP_MODE_VSYNC_MULTI;
|
||||
return (mode >= VIDEO_OUT_FLIP_MODE_VSYNC && mode <= VIDEO_OUT_FLIP_MODE_VSYNC_MULTI) ||
|
||||
mode == VIDEO_OUT_FLIP_MODE_SLAVE || mode == VIDEO_OUT_FLIP_MODE_MASTER;
|
||||
}
|
||||
|
||||
static int ReserveFlipRequest(VideoOutDriver::Impl& driver, int handle, int index, int flip_mode,
|
||||
@@ -506,13 +519,17 @@ static int ReserveFlipRequest(VideoOutDriver::Impl& driver, int handle, int inde
|
||||
return VIDEO_OUT_ERROR_INVALID_HANDLE;
|
||||
}
|
||||
if (!IsValidFlipMode(flip_mode)) {
|
||||
return VIDEO_OUT_ERROR_INVALID_VALUE;
|
||||
return VIDEO_OUT_ERROR_INVALID_FLIP_MODE;
|
||||
}
|
||||
if (!IsValidBufferIndex(index)) {
|
||||
return VIDEO_OUT_ERROR_INVALID_INDEX;
|
||||
}
|
||||
|
||||
Common::LockGuard lock(video_out->mutex);
|
||||
if ((video_out->master != nullptr) != (flip_mode == VIDEO_OUT_FLIP_MODE_SLAVE) ||
|
||||
(video_out->slave_count != 0) != (flip_mode == VIDEO_OUT_FLIP_MODE_MASTER)) {
|
||||
return VIDEO_OUT_ERROR_INVALID_FLIP_MODE;
|
||||
}
|
||||
if (video_out->closing ||
|
||||
(!IsSpecialBufferIndex(index) && !video_out->buffers[index].Occupied())) {
|
||||
return VIDEO_OUT_ERROR_INVALID_INDEX;
|
||||
@@ -530,8 +547,8 @@ Graphics::ImageInfo BufferAttributeGroup::ImageInfo(const VideoOutBuffer& buffer
|
||||
if (attribute.reserved0 != 0 || attribute.aspect_ratio != 0 || attribute.width == 0 ||
|
||||
attribute.height == 0 || attribute.width > 16384 || attribute.height > 16384 ||
|
||||
attribute.pitch_in_pixel != 0 ||
|
||||
(attribute.option != 0 &&
|
||||
attribute.option != VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY) ||
|
||||
(attribute.option & ~(VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY |
|
||||
VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_ALPHA_PREMULTIPLIED)) != 0 ||
|
||||
attribute.tiling_mode != 0 || attribute.pad0 != 0 || attribute.reserved1[0] != 0 ||
|
||||
attribute.reserved1[1] != 0 || attribute.reserved1[2] != 0 || buffer.data_address == 0 ||
|
||||
compression == Graphics::VideoOutCompression::Unsupported) {
|
||||
@@ -541,6 +558,14 @@ Graphics::ImageInfo BufferAttributeGroup::ImageInfo(const VideoOutBuffer& buffer
|
||||
if (!Graphics::DecodeVideoOutPixelFormat(attribute.pixel_format, pixel_format)) {
|
||||
EXIT("unsupported video-out pixel format: 0x%016" PRIx64 "\n", attribute.pixel_format);
|
||||
}
|
||||
if (attribute.pixel_format == Graphics::VIDEO_OUT_PIXEL_FORMAT_R10_G10_B10_A2_BT2100_PQ) {
|
||||
static std::atomic_flag warned = ATOMIC_FLAG_INIT;
|
||||
if (!warned.test_and_set(std::memory_order_relaxed)) {
|
||||
Log::WriteToConsoleAndLog(fmt::format(
|
||||
"Warning: HDR-to-SDR conversion is not implemented; displaying PQ video-out "
|
||||
"pixels unchanged (format=0x{:016x}).\n", attribute.pixel_format));
|
||||
}
|
||||
}
|
||||
const auto tile_mode = Graphics::Prospero::TileMode::kRenderTarget;
|
||||
const auto pitch =
|
||||
Graphics::TileGetTexturePitch(pixel_format.guest_format, attribute.width, tile_mode);
|
||||
@@ -606,6 +631,11 @@ VideoOutDriver::Impl::~Impl() {
|
||||
m_present_thread.request_stop();
|
||||
m_present_thread.join();
|
||||
}
|
||||
for (int handle = 1; handle < VIDEO_OUT_NUM_MAX; handle++) {
|
||||
if (m_video_out_ctx[handle].master != nullptr) {
|
||||
(void)Close(handle);
|
||||
}
|
||||
}
|
||||
for (int handle = 1; handle < VIDEO_OUT_NUM_MAX; handle++) {
|
||||
(void)Close(handle);
|
||||
}
|
||||
@@ -650,6 +680,7 @@ int VideoOutDriver::Impl::Open(int bus_type, int index) {
|
||||
|
||||
config.closing = false;
|
||||
config.opened = true;
|
||||
config.bus = bus_type;
|
||||
if (++config.generation == 0) {
|
||||
EXIT("video-out port generation wrapped\n");
|
||||
}
|
||||
@@ -676,11 +707,13 @@ bool VideoOutDriver::Impl::Close(int handle) {
|
||||
VideoOutEventQueues pre_vblank_events;
|
||||
VideoOutEventQueues vblank_events;
|
||||
VideoOutEventQueues output_mode_events;
|
||||
VideoOutConfig* master = nullptr;
|
||||
{
|
||||
Common::LockGuard config_lock(config.mutex);
|
||||
if (config.closing) {
|
||||
return false;
|
||||
}
|
||||
EXIT_IF(config.slave_count != 0);
|
||||
config.opened = false;
|
||||
config.closing = true;
|
||||
if (++config.generation == 0) {
|
||||
@@ -693,7 +726,9 @@ bool VideoOutDriver::Impl::Close(int handle) {
|
||||
vblank_events = std::move(config.events->vblank);
|
||||
output_mode_events = std::move(config.events->output_mode);
|
||||
}
|
||||
config.flip_rate = 0;
|
||||
config.flip_rate = 0;
|
||||
master = config.master;
|
||||
config.master = nullptr;
|
||||
|
||||
for (const auto& buffer: config.buffers) {
|
||||
if (buffer.Occupied() &&
|
||||
@@ -710,8 +745,12 @@ bool VideoOutDriver::Impl::Close(int handle) {
|
||||
}
|
||||
config.vblank_cond.SignalAll();
|
||||
}
|
||||
|
||||
if (master != nullptr) {
|
||||
Common::LockGuard master_lock(master->mutex);
|
||||
master->slave_count--;
|
||||
}
|
||||
m_flip_queue.Cancel(config);
|
||||
m_presenter.ClearLayer(config.bus);
|
||||
DeleteVideoOutEvents(flip_events, VideoOutEventKind::Flip);
|
||||
DeleteVideoOutEvents(pre_vblank_events, VideoOutEventKind::PreVblankStart);
|
||||
DeleteVideoOutEvents(vblank_events, VideoOutEventKind::Vblank);
|
||||
@@ -749,6 +788,31 @@ bool VideoOutDriver::Impl::IsOpened(int handle) {
|
||||
return handle > 0 && handle < VIDEO_OUT_NUM_MAX && m_video_out_ctx[handle].opened;
|
||||
}
|
||||
|
||||
int VideoOutDriver::Impl::SetFlipMaster(int slave_handle, int master_handle) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
if (slave_handle <= 0 || slave_handle >= VIDEO_OUT_NUM_MAX ||
|
||||
master_handle <= 0 || master_handle >= VIDEO_OUT_NUM_MAX ||
|
||||
!m_video_out_ctx[slave_handle].opened || !m_video_out_ctx[master_handle].opened) {
|
||||
return VIDEO_OUT_ERROR_INVALID_HANDLE;
|
||||
}
|
||||
auto& slave = m_video_out_ctx[slave_handle];
|
||||
auto& master = m_video_out_ctx[master_handle];
|
||||
if (&slave == &master) {
|
||||
return VIDEO_OUT_ERROR_UNSUPPORTED_OPERATION;
|
||||
}
|
||||
Common::LockGuard first_lock(m_video_out_ctx[std::min(slave_handle, master_handle)].mutex);
|
||||
Common::LockGuard second_lock(m_video_out_ctx[std::max(slave_handle, master_handle)].mutex);
|
||||
if (slave.master != nullptr || slave.slave_count != 0 || master.master != nullptr) {
|
||||
return VIDEO_OUT_ERROR_UNSUPPORTED_OPERATION;
|
||||
}
|
||||
if (slave.flip_status.flipPendingNum != 0 || master.flip_status.flipPendingNum != 0) {
|
||||
return VIDEO_OUT_ERROR_RESOURCE_BUSY;
|
||||
}
|
||||
slave.master = &master;
|
||||
master.slave_count++;
|
||||
return OK;
|
||||
}
|
||||
|
||||
void VideoOutDriver::Impl::VblankBegin() {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
|
||||
@@ -811,9 +875,7 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
|
||||
const auto period = std::max(frequency / refresh, uint64_t {1});
|
||||
|
||||
if (m_presenter.IsGuestPaused()) {
|
||||
if (auto* frame = m_presenter.PrepareLastFrame(); frame != nullptr) {
|
||||
m_presenter.Present(*frame, true);
|
||||
}
|
||||
(void)m_presenter.PresentLastFrame();
|
||||
const auto frame_end = Common::Timer::QueryPerformanceCounter();
|
||||
total_wait +=
|
||||
static_cast<int64_t>(period) - static_cast<int64_t>(frame_end - frame_begin);
|
||||
@@ -823,8 +885,7 @@ void VideoOutDriver::Impl::PresentThread(std::stop_token token) {
|
||||
VblankBegin();
|
||||
bool presented = m_flip_queue.Flip(0);
|
||||
if (!presented && m_presenter.NeedsSystemOverlayRefresh()) {
|
||||
if (auto* frame = m_presenter.PrepareLastFrame(); frame != nullptr) {
|
||||
m_presenter.Present(*frame, true);
|
||||
if (m_presenter.PresentLastFrame()) {
|
||||
presented = true;
|
||||
} else {
|
||||
uint32_t width = 0;
|
||||
@@ -867,25 +928,57 @@ bool FlipQueue::Reserve(VideoOutConfig& cfg, int index, int64_t flip_arg, FlipRe
|
||||
uint64_t& request_id) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
|
||||
if (m_requests.size() + m_cpu_requests.size() >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
|
||||
// A newer slave request replaces the one not yet captured by a master flip.
|
||||
if (cfg.master != nullptr) {
|
||||
for (auto* queue: {&m_requests, &m_cpu_requests}) {
|
||||
for (auto it = queue->begin(); it != queue->end();) {
|
||||
if (it->cfg != &cfg || it->group != 0) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
cfg.flip_status.flipPendingNum--;
|
||||
cfg.flip_status.gcQueueNum -= it->source == FlipRequestSource::GpuEop;
|
||||
if (it->state == RequestState::Ready) {
|
||||
m_presenter.Discard(*it->frame);
|
||||
it = queue->erase(it);
|
||||
} else {
|
||||
auto cancelled = it++;
|
||||
m_cancelled_requests.splice(m_cancelled_requests.end(), *queue, cancelled);
|
||||
}
|
||||
}
|
||||
}
|
||||
m_done_cond_var.SignalAll();
|
||||
}
|
||||
if (cfg.flip_status.flipPendingNum >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
|
||||
return false;
|
||||
}
|
||||
auto& pending = source == FlipRequestSource::GpuEop ? m_requests : m_cpu_requests;
|
||||
|
||||
Request r {};
|
||||
r.id = m_next_request_id++;
|
||||
r.group = cfg.master == nullptr ? r.id : 0;
|
||||
r.cfg = &cfg;
|
||||
r.master = cfg.master;
|
||||
r.generation = cfg.generation;
|
||||
r.index = index;
|
||||
r.flip_arg = flip_arg;
|
||||
r.submit_ptc = LibKernel::KernelGetProcessTimeCounter();
|
||||
r.source = source;
|
||||
r.state = RequestState::Reserved;
|
||||
if (cfg.slave_count != 0) {
|
||||
for (auto* queue: {&m_requests, &m_cpu_requests}) {
|
||||
for (auto& slave: *queue) {
|
||||
if (slave.master == &cfg && slave.group == 0) {
|
||||
slave.group = r.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pending.push_back(r);
|
||||
request_id = r.id;
|
||||
|
||||
cfg.flip_status.flipPendingNum = static_cast<int>(m_requests.size() + m_cpu_requests.size());
|
||||
cfg.flip_status.flipPendingNum++;
|
||||
cfg.flip_status.submitProcessTimeCounter = r.submit_ptc;
|
||||
if (source == FlipRequestSource::GpuEop) {
|
||||
cfg.flip_status.gcQueueNum++;
|
||||
@@ -907,7 +1000,9 @@ FlipQueue::~FlipQueue() {
|
||||
void FlipQueue::Cancel(VideoOutConfig& cfg) {
|
||||
std::vector<Graphics::Presenter::Frame*> frames;
|
||||
m_mutex.Lock();
|
||||
while (m_processing && !m_requests.empty() && m_requests.front().cfg == &cfg) {
|
||||
while (m_processing && std::any_of(m_requests.begin(), m_requests.end(), [&cfg](const auto& r) {
|
||||
return r.cfg == &cfg && r.state == RequestState::Presenting;
|
||||
})) {
|
||||
m_done_cond_var.Wait(&m_mutex);
|
||||
}
|
||||
for (auto* queue: {&m_requests, &m_cpu_requests}) {
|
||||
@@ -981,6 +1076,7 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) {
|
||||
uint32_t width = 0;
|
||||
uint32_t height = 0;
|
||||
bool current = false;
|
||||
bool premultiplied_alpha = false;
|
||||
{
|
||||
Common::LockGuard lock(cfg->mutex);
|
||||
current = cfg->opened && !cfg->closing && cfg->generation == generation;
|
||||
@@ -1002,6 +1098,9 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) {
|
||||
request_id, index, surface.group_index);
|
||||
}
|
||||
source_info = cfg->groups[surface.group_index].ImageInfo(surface);
|
||||
premultiplied_alpha =
|
||||
(cfg->groups[surface.group_index].attribute.option &
|
||||
VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_ALPHA_PREMULTIPLIED) != 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1041,6 +1140,7 @@ void FlipQueue::Prepare(uint64_t request_id, Graphics::CommandBuffer& buffer) {
|
||||
EXIT("video-out request changed while recording, id=%" PRIu64 "\n", request_id);
|
||||
}
|
||||
prepared->frame = frame;
|
||||
prepared->premultiplied_alpha = premultiplied_alpha;
|
||||
}
|
||||
|
||||
void FlipQueue::Complete(uint64_t request_id) {
|
||||
@@ -1072,9 +1172,12 @@ void FlipQueue::Complete(uint64_t request_id) {
|
||||
}
|
||||
}
|
||||
|
||||
void FlipQueue::WaitForSubmitSlot() {
|
||||
void FlipQueue::WaitForSubmitSlot(VideoOutConfig& cfg) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
while (m_requests.size() + m_cpu_requests.size() >= VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
|
||||
auto belongs_to_port = [&cfg](const auto& request) { return request.cfg == &cfg; };
|
||||
while (std::count_if(m_requests.begin(), m_requests.end(), belongs_to_port) +
|
||||
std::count_if(m_cpu_requests.begin(), m_cpu_requests.end(), belongs_to_port) >=
|
||||
VIDEO_OUT_FLIP_QUEUE_CAPACITY) {
|
||||
if (m_requests.empty()) {
|
||||
EXIT("video-out queue is saturated by CPU flips queued behind the current EOP\n");
|
||||
}
|
||||
@@ -1101,75 +1204,90 @@ bool FlipQueue::Flip(uint32_t micros) {
|
||||
m_mutex.Lock();
|
||||
if (m_requests.empty()) {
|
||||
m_submit_cond_var.WaitFor(&m_mutex, micros);
|
||||
|
||||
if (m_requests.empty()) {
|
||||
m_mutex.Unlock();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (m_processing) {
|
||||
EXIT("video-out flip queue processing is already active\n");
|
||||
}
|
||||
if (m_requests.front().state != RequestState::Ready) {
|
||||
EXIT_IF(m_processing);
|
||||
const auto owner = std::find_if(m_requests.begin(), m_requests.end(),
|
||||
[](const auto& r) { return r.group == r.id; });
|
||||
if (owner == m_requests.end()) {
|
||||
m_mutex.Unlock();
|
||||
return false;
|
||||
}
|
||||
m_processing = true;
|
||||
auto r = m_requests.front();
|
||||
m_mutex.Unlock();
|
||||
|
||||
r.cfg->mutex.Lock();
|
||||
if (!IsFlipDueLocked(*r.cfg, r.generation)) {
|
||||
r.cfg->mutex.Unlock();
|
||||
Common::LockGuard queue_lock(m_mutex);
|
||||
m_processing = false;
|
||||
m_done_cond_var.SignalAll();
|
||||
const uint64_t group = owner->id;
|
||||
if (std::any_of(m_cpu_requests.begin(), m_cpu_requests.end(),
|
||||
[group](const auto& r) { return r.group == group; }) ||
|
||||
std::any_of(m_requests.begin(), m_requests.end(), [group](const auto& r) {
|
||||
return r.group == group && r.state != RequestState::Ready;
|
||||
})) {
|
||||
m_mutex.Unlock();
|
||||
return false;
|
||||
}
|
||||
|
||||
m_mutex.Lock();
|
||||
if (m_requests.empty() || m_requests.front().id != r.id ||
|
||||
m_requests.front().state != RequestState::Ready || !m_processing) {
|
||||
EXIT("video-out request changed before presentation, id=%" PRIu64 "\n", r.id);
|
||||
std::array<Request, VideoOutDriver::Impl::VIDEO_OUT_BUSES.size()> requests {};
|
||||
size_t count = 0;
|
||||
for (auto& request: m_requests) {
|
||||
if (request.group == group) {
|
||||
EXIT_IF(count == requests.size());
|
||||
request.state = RequestState::Presenting;
|
||||
requests[count++] = request;
|
||||
}
|
||||
}
|
||||
m_requests.front().state = RequestState::Presenting;
|
||||
m_processing = true;
|
||||
m_mutex.Unlock();
|
||||
|
||||
m_presenter.Present(*r.frame);
|
||||
// Lock each port in bus order, then present and complete the whole group at one Vblank.
|
||||
std::sort(requests.begin(), requests.begin() + count,
|
||||
[](const auto& a, const auto& b) { return a.cfg->bus < b.cfg->bus; });
|
||||
bool due = true;
|
||||
std::array<Graphics::Presenter::Layer, VideoOutDriver::Impl::VIDEO_OUT_BUSES.size()> layers {};
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
auto& r = requests[i];
|
||||
r.cfg->mutex.Lock();
|
||||
due &= r.cfg->opened && !r.cfg->closing && r.cfg->generation == r.generation;
|
||||
if (r.id == group) {
|
||||
due &= IsFlipDueLocked(*r.cfg, r.generation);
|
||||
}
|
||||
layers[i] = {r.frame, r.cfg->bus, r.premultiplied_alpha};
|
||||
}
|
||||
if (due) {
|
||||
m_presenter.Present(std::span(layers.data(), count));
|
||||
}
|
||||
|
||||
m_mutex.Lock();
|
||||
if (m_requests.empty() || m_requests.front().id != r.id ||
|
||||
m_requests.front().state != RequestState::Presenting) {
|
||||
EXIT("video-out flip queue changed while processing its front request\n");
|
||||
for (auto it = m_requests.begin(); it != m_requests.end();) {
|
||||
if (it->group != group) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
if (!due) {
|
||||
it++->state = RequestState::Ready;
|
||||
continue;
|
||||
}
|
||||
auto& r = *it;
|
||||
r.cfg->flip_status.count++;
|
||||
r.cfg->flip_status.processTime = LibKernel::KernelGetProcessTime();
|
||||
r.cfg->flip_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter();
|
||||
r.cfg->flip_status.submitProcessTimeCounter = r.submit_ptc;
|
||||
r.cfg->flip_status.flipArg = r.flip_arg;
|
||||
r.cfg->flip_status.currentBuffer = r.index;
|
||||
r.cfg->flip_status.flipPendingNum--;
|
||||
r.cfg->flip_status.gcQueueNum -= r.source == FlipRequestSource::GpuEop;
|
||||
TriggerVideoOutEvents(*r.cfg, VideoOutEventKind::Flip, reinterpret_cast<void*>(r.flip_arg));
|
||||
it = m_requests.erase(it);
|
||||
}
|
||||
m_requests.pop_front();
|
||||
|
||||
r.cfg->flip_status.count++;
|
||||
r.cfg->flip_status.processTime = LibKernel::KernelGetProcessTime();
|
||||
r.cfg->flip_status.processTimeCounter = LibKernel::KernelGetProcessTimeCounter();
|
||||
r.cfg->flip_status.submitProcessTimeCounter = r.submit_ptc;
|
||||
r.cfg->flip_status.flipArg = r.flip_arg;
|
||||
r.cfg->flip_status.currentBuffer = r.index;
|
||||
r.cfg->flip_status.flipPendingNum = static_cast<int>(m_requests.size() + m_cpu_requests.size());
|
||||
if (r.source == FlipRequestSource::GpuEop && r.cfg->flip_status.gcQueueNum > 0) {
|
||||
r.cfg->flip_status.gcQueueNum--;
|
||||
}
|
||||
TriggerVideoOutEvents(*r.cfg, VideoOutEventKind::Flip, reinterpret_cast<void*>(r.flip_arg));
|
||||
|
||||
m_processing = false;
|
||||
m_done_cond_var.SignalAll();
|
||||
m_submit_slot_cond_var.Signal();
|
||||
m_submit_slot_cond_var.SignalAll();
|
||||
m_mutex.Unlock();
|
||||
r.cfg->mutex.Unlock();
|
||||
|
||||
Graphics::RenderDocOnGuestFlip(m_presenter.Renderer());
|
||||
|
||||
if (Config::GraphicsDebugDumpEnabled() &&
|
||||
Config::GetPrintfDirection() != Config::LogDirection::Silent) {
|
||||
LOGF("Flip done: %d\n", r.index);
|
||||
for (size_t i = count; i != 0; i--) {
|
||||
requests[i - 1].cfg->mutex.Unlock();
|
||||
}
|
||||
|
||||
return true;
|
||||
if (due) {
|
||||
Graphics::RenderDocOnGuestFlip(m_presenter.Renderer());
|
||||
if (Config::GraphicsDebugDumpEnabled() &&
|
||||
Config::GetPrintfDirection() != Config::LogDirection::Silent) {
|
||||
LOGF("Flip done: %d\n", requests[0].index);
|
||||
}
|
||||
}
|
||||
return due;
|
||||
}
|
||||
|
||||
void FlipQueue::GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out) {
|
||||
@@ -1178,6 +1296,18 @@ void FlipQueue::GetFlipStatus(VideoOutConfig& cfg, VideoOutFlipStatus& out) {
|
||||
out = cfg.flip_status;
|
||||
}
|
||||
|
||||
KYTY_SYSV_ABI int VideoOutSetFlipMaster(int slave_handle, int master_handle) {
|
||||
PRINT_NAME();
|
||||
return DriverState().SetFlipMaster(slave_handle, master_handle);
|
||||
}
|
||||
|
||||
KYTY_SYSV_ABI void VideoOutAddBufferAttributeOption(VideoOutBufferAttribute2* attribute,
|
||||
uint64_t option) {
|
||||
PRINT_NAME();
|
||||
EXIT_IF(attribute == nullptr);
|
||||
attribute->option |= option;
|
||||
}
|
||||
|
||||
KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* param) {
|
||||
PRINT_NAME();
|
||||
|
||||
@@ -1457,7 +1587,7 @@ KYTY_SYSV_ABI int VideoOutSubmitFlip(int handle, int index, int flip_mode, int64
|
||||
uint64_t request_id = 0;
|
||||
const int result = ReserveFlipRequest(DriverState(), handle, index, flip_mode, flip_arg,
|
||||
FlipRequestSource::Cpu, request_id);
|
||||
if (result == VIDEO_OUT_ERROR_INVALID_VALUE) {
|
||||
if (result == VIDEO_OUT_ERROR_INVALID_FLIP_MODE) {
|
||||
LOGF("\t unsupported flip_mode = %d\n", flip_mode);
|
||||
}
|
||||
if (result != OK) {
|
||||
@@ -1494,8 +1624,10 @@ void VideoOutDriver::CompleteFlip(uint64_t request_id) {
|
||||
m_impl->GetFlipQueue().Complete(request_id);
|
||||
}
|
||||
|
||||
void VideoOutDriver::WaitForSubmitSlot() {
|
||||
m_impl->GetFlipQueue().WaitForSubmitSlot();
|
||||
void VideoOutDriver::WaitForSubmitSlot(int handle) {
|
||||
auto* cfg = m_impl->Get(handle);
|
||||
EXIT_IF(cfg == nullptr);
|
||||
m_impl->GetFlipQueue().WaitForSubmitSlot(*cfg);
|
||||
}
|
||||
|
||||
void VideoOutDriver::WaitFlipDone(int handle, int index) {
|
||||
@@ -1546,7 +1678,9 @@ KYTY_SYSV_ABI int VideoOutIsFlipPending(int handle) {
|
||||
VideoOutFlipStatus status {};
|
||||
DriverState().GetFlipQueue().GetFlipStatus(*ctx, status);
|
||||
|
||||
LOGF("\t flipPendingNum = %d\n", status.flipPendingNum);
|
||||
if (Config::GraphicsDebugDumpEnabled()) {
|
||||
LOGF("\t flipPendingNum = %d\n", status.flipPendingNum);
|
||||
}
|
||||
|
||||
return status.flipPendingNum;
|
||||
}
|
||||
@@ -1680,21 +1814,6 @@ KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* stat
|
||||
return OK;
|
||||
}
|
||||
|
||||
KYTY_SYSV_ABI int VideoOutGetVrrStatus(int handle, int32_t* status) {
|
||||
PRINT_NAME();
|
||||
|
||||
if (status == nullptr) {
|
||||
return VIDEO_OUT_ERROR_INVALID_ADDRESS;
|
||||
}
|
||||
if (DriverState().Get(handle) == nullptr) {
|
||||
return VIDEO_OUT_ERROR_INVALID_HANDLE;
|
||||
}
|
||||
|
||||
// Kyty currently presents at a fixed refresh rate and does not negotiate VRR.
|
||||
*status = 0;
|
||||
return OK;
|
||||
}
|
||||
|
||||
static int ValidateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options,
|
||||
void* reserved_ptr, uint64_t reserved) {
|
||||
if (!DriverState().IsOpened(handle)) {
|
||||
|
||||
@@ -37,7 +37,7 @@ public:
|
||||
void PrepareFlip(uint64_t request_id, Graphics::CommandBuffer& buffer);
|
||||
void CompleteFlip(uint64_t request_id);
|
||||
void SubmitFlipPreparation(uint64_t request_id);
|
||||
void WaitForSubmitSlot();
|
||||
void WaitForSubmitSlot(int handle);
|
||||
void WaitFlipDone(int handle, int index);
|
||||
|
||||
[[nodiscard]] Impl& State() noexcept;
|
||||
@@ -52,6 +52,9 @@ void VideoOutShutdown();
|
||||
|
||||
KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void* param);
|
||||
KYTY_SYSV_ABI int VideoOutClose(int handle);
|
||||
KYTY_SYSV_ABI int VideoOutSetFlipMaster(int slave_handle, int master_handle);
|
||||
KYTY_SYSV_ABI void VideoOutAddBufferAttributeOption(VideoOutBufferAttribute2* attribute,
|
||||
uint64_t option);
|
||||
KYTY_SYSV_ABI void VideoOutSetBufferAttribute2(VideoOutBufferAttribute2* attribute,
|
||||
uint64_t pixel_format, uint32_t tiling_mode,
|
||||
uint32_t width, uint32_t height, uint64_t option,
|
||||
@@ -87,7 +90,6 @@ KYTY_SYSV_ABI int VideoOutGetEventData(const LibKernel::EventQueue::KernelEvent*
|
||||
KYTY_SYSV_ABI int VideoOutGetEventCount(const LibKernel::EventQueue::KernelEvent* ev);
|
||||
KYTY_SYSV_ABI int VideoOutWaitVblank(int handle);
|
||||
KYTY_SYSV_ABI int VideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status);
|
||||
KYTY_SYSV_ABI int VideoOutGetVrrStatus(int handle, int32_t* status);
|
||||
KYTY_SYSV_ABI int VideoOutInitializeOutputOptions(VideoOutOutputOptions* options);
|
||||
KYTY_SYSV_ABI int VideoOutIsOutputSupported(int handle, uint64_t mode,
|
||||
const VideoOutOutputOptions* options,
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include "common/logging/log.h"
|
||||
#include "common/profiler.h"
|
||||
#include "common/threads.h"
|
||||
#include "gpu_blit_shaders/gpu_blit_fs_triangle_spv.h"
|
||||
#include "gpu_blit_shaders/gpu_video_out_overlay_spv.h"
|
||||
#include "graphics/host_gpu/graphicContext.h"
|
||||
#include "graphics/host_gpu/renderer/render.h"
|
||||
#include "graphics/host_gpu/renderer/renderContext.h"
|
||||
@@ -14,7 +16,7 @@
|
||||
#include "graphics/presentation/window/windowInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
@@ -25,10 +27,10 @@
|
||||
namespace Libs::Graphics {
|
||||
|
||||
struct Presenter::Frame {
|
||||
VulkanImage image;
|
||||
uint64_t present_tick = 0;
|
||||
bool busy = false;
|
||||
bool reusing_last = false;
|
||||
VulkanImage image;
|
||||
vk::ImageView view = nullptr;
|
||||
uint64_t present_tick = 0;
|
||||
bool busy = false;
|
||||
|
||||
void Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format);
|
||||
void Transit(vk::CommandBuffer command, vk::ImageLayout layout, vk::AccessFlags2 access);
|
||||
@@ -43,6 +45,7 @@ public:
|
||||
~FramePool() {
|
||||
m_scheduler.Wait(m_scheduler.CurrentTick() - 1);
|
||||
for (auto& frame: m_frames) {
|
||||
m_window.graphic_ctx.device.destroyImageView(frame->view, nullptr);
|
||||
if (frame->image.image != nullptr) {
|
||||
m_window.graphic_ctx.DeleteImage(frame->image);
|
||||
}
|
||||
@@ -60,6 +63,7 @@ public:
|
||||
}
|
||||
m_format = format;
|
||||
m_frames.reserve(count);
|
||||
m_free.reserve(count);
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
auto frame = std::make_unique<Presenter::Frame>();
|
||||
m_free.push_back(frame.get());
|
||||
@@ -83,60 +87,45 @@ public:
|
||||
return m_format;
|
||||
}
|
||||
|
||||
Presenter::Frame* Acquire() {
|
||||
Presenter::Frame* Acquire(vk::Extent2D extent, vk::Format format) {
|
||||
m_mutex.Lock();
|
||||
if (m_frames.empty()) {
|
||||
EXIT("prepared-frame pool was used before swapchain initialization\n");
|
||||
}
|
||||
while (m_free.empty()) {
|
||||
m_available.Wait(&m_mutex);
|
||||
// A synchronized flip may need more frames than the swapchain has images.
|
||||
// Queue capacity bounds growth; waiting here can prevent its master from recording.
|
||||
if (m_free.empty()) {
|
||||
auto frame = std::make_unique<Presenter::Frame>();
|
||||
m_free.push_back(frame.get());
|
||||
m_frames.push_back(std::move(frame));
|
||||
}
|
||||
auto* frame = m_free.front();
|
||||
m_free.pop_front();
|
||||
const auto compatible = std::ranges::find_if(m_free, [&](const auto* frame) {
|
||||
return frame->image.extent.width == extent.width &&
|
||||
frame->image.extent.height == extent.height && frame->image.format == format;
|
||||
});
|
||||
auto* frame = compatible == m_free.end() ? m_free.back() : *compatible;
|
||||
if (compatible != m_free.end()) {
|
||||
*compatible = m_free.back();
|
||||
}
|
||||
m_free.pop_back();
|
||||
if (frame->busy) {
|
||||
EXIT("prepared-frame pool returned an invalid frame\n");
|
||||
}
|
||||
if (m_last_frame == frame) {
|
||||
m_last_frame = nullptr;
|
||||
}
|
||||
frame->busy = true;
|
||||
frame->reusing_last = false;
|
||||
frame->busy = true;
|
||||
m_mutex.Unlock();
|
||||
|
||||
WaitForFrame(*frame);
|
||||
m_scheduler.Wait(frame->present_tick);
|
||||
return frame;
|
||||
}
|
||||
|
||||
Presenter::Frame* AcquireLast() {
|
||||
m_mutex.Lock();
|
||||
auto* frame = m_last_frame;
|
||||
if (frame == nullptr) {
|
||||
m_mutex.Unlock();
|
||||
return nullptr;
|
||||
}
|
||||
auto free = std::find(m_free.begin(), m_free.end(), frame);
|
||||
if (free == m_free.end() || frame->busy) {
|
||||
m_mutex.Unlock();
|
||||
EXIT("last submitted frame is not available for reuse\n");
|
||||
}
|
||||
m_free.erase(free);
|
||||
m_last_frame = nullptr;
|
||||
frame->busy = true;
|
||||
frame->reusing_last = true;
|
||||
m_mutex.Unlock();
|
||||
|
||||
WaitForFrame(*frame);
|
||||
return frame;
|
||||
}
|
||||
|
||||
void ValidateForPresent(Presenter::Frame* frame, bool reuse) {
|
||||
void ValidateForPresent(Presenter::Frame* frame) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
if (frame == nullptr || !frame->busy || frame->reusing_last != reuse) {
|
||||
if (frame == nullptr || !frame->busy) {
|
||||
EXIT("prepared frame has invalid presentation ownership\n");
|
||||
}
|
||||
}
|
||||
|
||||
void Release(Presenter::Frame* frame, bool make_last = false) {
|
||||
void Release(Presenter::Frame* frame) {
|
||||
if (frame == nullptr) {
|
||||
EXIT("cannot release a null prepared frame\n");
|
||||
}
|
||||
@@ -144,26 +133,17 @@ public:
|
||||
if (!frame->busy) {
|
||||
EXIT("prepared frame was released twice\n");
|
||||
}
|
||||
frame->busy = false;
|
||||
frame->reusing_last = false;
|
||||
if (make_last) {
|
||||
m_last_frame = frame;
|
||||
}
|
||||
frame->busy = false;
|
||||
m_free.push_back(frame);
|
||||
m_available.Signal();
|
||||
}
|
||||
|
||||
private:
|
||||
void WaitForFrame(Presenter::Frame& frame) { m_scheduler.Wait(frame.present_tick); }
|
||||
|
||||
WindowContext& m_window;
|
||||
CommandScheduler& m_scheduler;
|
||||
Common::Mutex m_mutex;
|
||||
Common::CondVar m_available;
|
||||
std::vector<std::unique_ptr<Presenter::Frame>> m_frames;
|
||||
std::deque<Presenter::Frame*> m_free;
|
||||
Presenter::Frame* m_last_frame = nullptr;
|
||||
vk::Format m_format = vk::Format::eUndefined;
|
||||
std::vector<Presenter::Frame*> m_free;
|
||||
vk::Format m_format = vk::Format::eUndefined;
|
||||
};
|
||||
|
||||
void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format) {
|
||||
@@ -181,13 +161,16 @@ void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
|
||||
const auto features = graphics.GetFormatProperties(format).optimalTilingFeatures;
|
||||
const auto required =
|
||||
vk::FormatFeatureFlagBits::eBlitSrc | vk::FormatFeatureFlagBits::eSampledImageFilterLinear |
|
||||
vk::FormatFeatureFlagBits::eTransferSrc | vk::FormatFeatureFlagBits::eTransferDst;
|
||||
vk::FormatFeatureFlagBits::eSampledImage | vk::FormatFeatureFlagBits::eTransferSrc |
|
||||
vk::FormatFeatureFlagBits::eTransferDst;
|
||||
if ((features & required) != required) {
|
||||
EXIT("prepared presentation format lacks optimal blit support: format=%d features=0x%x\n",
|
||||
static_cast<int>(format), static_cast<vk::FormatFeatureFlags::MaskType>(features));
|
||||
}
|
||||
|
||||
if (dst.image != nullptr) {
|
||||
graphics.device.destroyImageView(view, nullptr);
|
||||
view = nullptr;
|
||||
graphics.DeleteImage(dst);
|
||||
}
|
||||
|
||||
@@ -200,7 +183,8 @@ void Presenter::Frame::Configure(GraphicContext& graphics, vk::Extent2D extent,
|
||||
create.format = format;
|
||||
create.tiling = vk::ImageTiling::eOptimal;
|
||||
create.initialLayout = vk::ImageLayout::eUndefined;
|
||||
create.usage = vk::ImageUsageFlagBits::eTransferSrc | vk::ImageUsageFlagBits::eTransferDst;
|
||||
create.usage = vk::ImageUsageFlagBits::eTransferSrc | vk::ImageUsageFlagBits::eTransferDst |
|
||||
vk::ImageUsageFlagBits::eSampled;
|
||||
create.sharingMode = vk::SharingMode::eExclusive;
|
||||
create.samples = vk::SampleCountFlagBits::e1;
|
||||
if (!graphics.CreateImage(create, dst)) {
|
||||
@@ -258,7 +242,6 @@ void Presenter::Frame::CopyFrom(CommandBuffer& command_buffer, Image& source) {
|
||||
EXIT_IF(copy.srcSubresource.layerCount != copy.dstSubresource.layerCount);
|
||||
command.copyImage(source.backing.image, vk::ImageLayout::eTransferSrcOptimal, image.image,
|
||||
vk::ImageLayout::eTransferDstOptimal, copy);
|
||||
Transit(command, vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead);
|
||||
}
|
||||
|
||||
void Presenter::Frame::Clear(CommandBuffer& command_buffer, const vk::ClearColorValue& color) {
|
||||
@@ -267,7 +250,6 @@ void Presenter::Frame::Clear(CommandBuffer& command_buffer, const vk::ClearColor
|
||||
Transit(command, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits2::eTransferWrite);
|
||||
const vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
|
||||
command.clearColorImage(image.image, vk::ImageLayout::eTransferDstOptimal, &color, 1, &range);
|
||||
Transit(command, vk::ImageLayout::eTransferSrcOptimal, vk::AccessFlagBits2::eTransferRead);
|
||||
}
|
||||
|
||||
class Swapchain final {
|
||||
@@ -283,9 +265,9 @@ public:
|
||||
[[nodiscard]] bool NeedsResize() const;
|
||||
[[nodiscard]] Status AcquireNextImage();
|
||||
[[nodiscard]] bool PrepareSystemOverlay();
|
||||
void RecordPresentCommands(CommandBuffer& command, VulkanImage& source,
|
||||
bool draw_system_overlay);
|
||||
uint64_t Submit(CommandScheduler& scheduler);
|
||||
void RecordPresentCommands(CommandBuffer& command, Presenter::Frame* source,
|
||||
const Presenter::Layer& overlay, bool draw_system_overlay);
|
||||
uint64_t Submit(CommandScheduler& scheduler);
|
||||
[[nodiscard]] Status Present();
|
||||
|
||||
[[nodiscard]] uint32_t ImageCount() const noexcept {
|
||||
@@ -295,20 +277,25 @@ public:
|
||||
|
||||
private:
|
||||
void Destroy();
|
||||
void DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer);
|
||||
|
||||
WindowContext& m_window;
|
||||
vk::SwapchainKHR m_handle = nullptr;
|
||||
vk::Format m_format = vk::Format::eUndefined;
|
||||
vk::Extent2D m_extent {};
|
||||
WindowContext& m_window;
|
||||
vk::SwapchainKHR m_handle = nullptr;
|
||||
vk::Format m_format = vk::Format::eUndefined;
|
||||
vk::Extent2D m_extent {};
|
||||
// Drawable pixel size observed when this swapchain was created.
|
||||
vk::Extent2D m_window_extent {};
|
||||
std::vector<vk::Image> m_images;
|
||||
std::vector<vk::ImageView> m_image_views;
|
||||
std::vector<vk::Semaphore> m_image_acquired;
|
||||
std::vector<vk::Semaphore> m_render_complete;
|
||||
vk::Extent2D m_window_extent {};
|
||||
std::vector<vk::Image> m_images;
|
||||
std::vector<vk::ImageView> m_image_views;
|
||||
std::vector<vk::Semaphore> m_image_acquired;
|
||||
std::vector<vk::Semaphore> m_render_complete;
|
||||
std::unique_ptr<SystemOverlay> m_system_overlay;
|
||||
uint32_t m_image_index = static_cast<uint32_t>(-1);
|
||||
uint32_t m_frame_index = 0;
|
||||
vk::DescriptorSetLayout m_overlay_descriptors = nullptr;
|
||||
vk::PipelineLayout m_overlay_layout = nullptr;
|
||||
vk::Pipeline m_overlay_pipeline = nullptr;
|
||||
vk::Sampler m_overlay_sampler = nullptr;
|
||||
uint32_t m_image_index = static_cast<uint32_t>(-1);
|
||||
uint32_t m_frame_index = 0;
|
||||
};
|
||||
|
||||
struct Presenter::Impl {
|
||||
@@ -347,12 +334,15 @@ struct Presenter::Impl {
|
||||
cache.UpdateImage(image_id);
|
||||
return image;
|
||||
}
|
||||
void Present();
|
||||
|
||||
RenderContext& renderer;
|
||||
WindowContext& window;
|
||||
Swapchain swapchain;
|
||||
CommandScheduler present_scheduler;
|
||||
FramePool frames;
|
||||
Common::Mutex present_mutex;
|
||||
std::array<Layer, 2> layers {};
|
||||
std::atomic<uint64_t> presented_overlay_revision {0};
|
||||
};
|
||||
|
||||
@@ -370,7 +360,7 @@ void Swapchain::Create() {
|
||||
EXIT_IF(m_window_extent.width == 0);
|
||||
EXIT_IF(m_window_extent.height == 0);
|
||||
m_window.RefreshSurfaceCapabilities();
|
||||
const auto& surface = m_window.surface_capabilities;
|
||||
const auto& surface = m_window.surface_capabilities;
|
||||
EXIT_NOT_IMPLEMENTED(surface.formats.empty());
|
||||
|
||||
m_extent = surface.capabilities.currentExtent;
|
||||
@@ -442,7 +432,7 @@ void Swapchain::Create() {
|
||||
LOGF("warning: requested present mode is unavailable; falling back to Fifo\n");
|
||||
create_info.presentMode = vk::PresentModeKHR::eFifo;
|
||||
}
|
||||
create_info.clipped = VK_TRUE;
|
||||
create_info.clipped = VK_TRUE;
|
||||
RequireVulkanSuccess(graphics.device.createSwapchainKHR(&create_info, nullptr, &m_handle),
|
||||
"vkCreateSwapchainKHR");
|
||||
EXIT_IF(m_handle == nullptr);
|
||||
@@ -483,8 +473,8 @@ void Swapchain::Create() {
|
||||
graphics.device.createSemaphore(&semaphore_info, nullptr, &m_render_complete[i]),
|
||||
"create swapchain render-complete semaphore");
|
||||
}
|
||||
m_image_index = static_cast<uint32_t>(-1);
|
||||
m_frame_index = 0;
|
||||
m_image_index = static_cast<uint32_t>(-1);
|
||||
m_frame_index = 0;
|
||||
}
|
||||
|
||||
Swapchain::~Swapchain() {
|
||||
@@ -505,6 +495,14 @@ void Swapchain::Destroy() {
|
||||
if (m_system_overlay != nullptr) {
|
||||
m_system_overlay->ReleaseVulkan();
|
||||
}
|
||||
graphics.device.destroyPipeline(m_overlay_pipeline, nullptr);
|
||||
graphics.device.destroyPipelineLayout(m_overlay_layout, nullptr);
|
||||
graphics.device.destroyDescriptorSetLayout(m_overlay_descriptors, nullptr);
|
||||
graphics.device.destroySampler(m_overlay_sampler, nullptr);
|
||||
m_overlay_pipeline = nullptr;
|
||||
m_overlay_layout = nullptr;
|
||||
m_overlay_descriptors = nullptr;
|
||||
m_overlay_sampler = nullptr;
|
||||
|
||||
for (const auto semaphore: m_image_acquired) {
|
||||
if (semaphore != nullptr) {
|
||||
@@ -544,8 +542,8 @@ void Swapchain::Recreate(bool surface_lost) {
|
||||
// Surface recreation goes through SDL_Vulkan_CreateSurface, which touches the
|
||||
// window's view/layer and must run on the main thread on macOS.
|
||||
EXIT_IF(!SDL_RunOnMainThread(
|
||||
[](void* window) { static_cast<WindowContext*>(window)->RecreateSurface(); },
|
||||
&m_window, true));
|
||||
[](void* window) { static_cast<WindowContext*>(window)->RecreateSurface(); }, &m_window,
|
||||
true));
|
||||
#else
|
||||
m_window.RecreateSurface();
|
||||
#endif
|
||||
@@ -592,14 +590,147 @@ bool Swapchain::PrepareSystemOverlay() {
|
||||
return m_system_overlay->PrepareFrame(m_extent, m_format, ImageCount());
|
||||
}
|
||||
|
||||
void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& source,
|
||||
bool draw_system_overlay) {
|
||||
if (source.state.layout != vk::ImageLayout::eTransferSrcOptimal) {
|
||||
EXIT("invalid prepared presentation image, vk_image=%p layout=%d\n",
|
||||
static_cast<void*>(source.image), static_cast<int>(source.state.layout));
|
||||
void Swapchain::DrawOverlay(vk::CommandBuffer command, const Presenter::Layer& layer) {
|
||||
auto device = m_window.graphic_ctx.device;
|
||||
if (m_overlay_pipeline == nullptr) {
|
||||
vk::SamplerCreateInfo sampler {};
|
||||
sampler.magFilter = vk::Filter::eLinear;
|
||||
sampler.minFilter = vk::Filter::eLinear;
|
||||
sampler.addressModeU = vk::SamplerAddressMode::eClampToEdge;
|
||||
sampler.addressModeV = vk::SamplerAddressMode::eClampToEdge;
|
||||
sampler.addressModeW = vk::SamplerAddressMode::eClampToEdge;
|
||||
RequireVulkanSuccess(device.createSampler(&sampler, nullptr, &m_overlay_sampler),
|
||||
"create video-out overlay sampler");
|
||||
const vk::DescriptorSetLayoutBinding binding {0, vk::DescriptorType::eCombinedImageSampler,
|
||||
1, vk::ShaderStageFlagBits::eFragment};
|
||||
vk::DescriptorSetLayoutCreateInfo descriptors {};
|
||||
descriptors.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
|
||||
descriptors.bindingCount = 1;
|
||||
descriptors.pBindings = &binding;
|
||||
RequireVulkanSuccess(
|
||||
device.createDescriptorSetLayout(&descriptors, nullptr, &m_overlay_descriptors),
|
||||
"create video-out overlay descriptor layout");
|
||||
const vk::PushConstantRange alpha {vk::ShaderStageFlagBits::eFragment, 0, sizeof(uint32_t)};
|
||||
vk::PipelineLayoutCreateInfo layout {};
|
||||
layout.setLayoutCount = 1;
|
||||
layout.pSetLayouts = &m_overlay_descriptors;
|
||||
layout.pushConstantRangeCount = 1;
|
||||
layout.pPushConstantRanges = α
|
||||
RequireVulkanSuccess(device.createPipelineLayout(&layout, nullptr, &m_overlay_layout),
|
||||
"create video-out overlay pipeline layout");
|
||||
const auto vertex = CompileSPV(GPU_BLIT_FS_TRIANGLE_SPV, device);
|
||||
const auto fragment = CompileSPV(GPU_VIDEO_OUT_OVERLAY_SPV, device);
|
||||
std::array<vk::PipelineShaderStageCreateInfo, 2> stages {};
|
||||
stages[0].stage = vk::ShaderStageFlagBits::eVertex;
|
||||
stages[0].module = vertex;
|
||||
stages[0].pName = "main";
|
||||
stages[1].stage = vk::ShaderStageFlagBits::eFragment;
|
||||
stages[1].module = fragment;
|
||||
stages[1].pName = "main";
|
||||
vk::PipelineVertexInputStateCreateInfo vertex_input {};
|
||||
vk::PipelineInputAssemblyStateCreateInfo assembly {};
|
||||
assembly.topology = vk::PrimitiveTopology::eTriangleList;
|
||||
vk::PipelineViewportStateCreateInfo viewport {};
|
||||
viewport.viewportCount = 1;
|
||||
viewport.scissorCount = 1;
|
||||
vk::PipelineRasterizationStateCreateInfo rasterization {};
|
||||
rasterization.lineWidth = 1.0f;
|
||||
vk::PipelineMultisampleStateCreateInfo multisample {};
|
||||
multisample.rasterizationSamples = vk::SampleCountFlagBits::e1;
|
||||
vk::PipelineColorBlendAttachmentState attachment {};
|
||||
attachment.blendEnable = VK_TRUE;
|
||||
attachment.srcColorBlendFactor = vk::BlendFactor::eOne;
|
||||
attachment.dstColorBlendFactor = vk::BlendFactor::eOneMinusSrcAlpha;
|
||||
attachment.srcAlphaBlendFactor = vk::BlendFactor::eOne;
|
||||
attachment.dstAlphaBlendFactor = vk::BlendFactor::eOneMinusSrcAlpha;
|
||||
attachment.colorWriteMask = vk::ColorComponentFlagBits::eR |
|
||||
vk::ColorComponentFlagBits::eG |
|
||||
vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA;
|
||||
vk::PipelineColorBlendStateCreateInfo blend {};
|
||||
blend.attachmentCount = 1;
|
||||
blend.pAttachments = &attachment;
|
||||
const std::array dynamic_states {vk::DynamicState::eViewport, vk::DynamicState::eScissor};
|
||||
vk::PipelineDynamicStateCreateInfo dynamic {};
|
||||
dynamic.dynamicStateCount = static_cast<uint32_t>(dynamic_states.size());
|
||||
dynamic.pDynamicStates = dynamic_states.data();
|
||||
vk::PipelineRenderingCreateInfo rendering {};
|
||||
rendering.colorAttachmentCount = 1;
|
||||
rendering.pColorAttachmentFormats = &m_format;
|
||||
vk::GraphicsPipelineCreateInfo create {};
|
||||
create.pNext = &rendering;
|
||||
create.stageCount = static_cast<uint32_t>(stages.size());
|
||||
create.pStages = stages.data();
|
||||
create.pVertexInputState = &vertex_input;
|
||||
create.pInputAssemblyState = &assembly;
|
||||
create.pViewportState = &viewport;
|
||||
create.pRasterizationState = &rasterization;
|
||||
create.pMultisampleState = &multisample;
|
||||
create.pColorBlendState = &blend;
|
||||
create.pDynamicState = &dynamic;
|
||||
create.layout = m_overlay_layout;
|
||||
RequireVulkanSuccess(
|
||||
device.createGraphicsPipelines(nullptr, 1, &create, nullptr, &m_overlay_pipeline),
|
||||
"create video-out overlay pipeline");
|
||||
device.destroyShaderModule(fragment, nullptr);
|
||||
device.destroyShaderModule(vertex, nullptr);
|
||||
}
|
||||
auto& frame = *layer.frame;
|
||||
if (frame.view == nullptr) {
|
||||
vk::ImageViewCreateInfo view {};
|
||||
view.image = frame.image.image;
|
||||
view.viewType = vk::ImageViewType::e2D;
|
||||
view.format = frame.image.format;
|
||||
view.subresourceRange = {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1};
|
||||
RequireVulkanSuccess(device.createImageView(&view, nullptr, &frame.view),
|
||||
"create video-out overlay image view");
|
||||
}
|
||||
const vk::DescriptorImageInfo image {m_overlay_sampler, frame.view,
|
||||
vk::ImageLayout::eShaderReadOnlyOptimal};
|
||||
vk::WriteDescriptorSet write {};
|
||||
write.dstBinding = 0;
|
||||
write.descriptorCount = 1;
|
||||
write.descriptorType = vk::DescriptorType::eCombinedImageSampler;
|
||||
write.pImageInfo = ℑ
|
||||
command.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, m_overlay_layout, 0, 1, &write);
|
||||
const uint32_t premultiplied = layer.premultiplied_alpha;
|
||||
command.pushConstants(m_overlay_layout, vk::ShaderStageFlagBits::eFragment, 0,
|
||||
sizeof(premultiplied), &premultiplied);
|
||||
command.bindPipeline(vk::PipelineBindPoint::eGraphics, m_overlay_pipeline);
|
||||
vk::RenderingAttachmentInfo attachment {};
|
||||
attachment.imageView = m_image_views[m_image_index];
|
||||
attachment.imageLayout = vk::ImageLayout::eColorAttachmentOptimal;
|
||||
attachment.loadOp = vk::AttachmentLoadOp::eLoad;
|
||||
attachment.storeOp = vk::AttachmentStoreOp::eStore;
|
||||
vk::RenderingInfo rendering {};
|
||||
rendering.renderArea.extent = m_extent;
|
||||
rendering.layerCount = 1;
|
||||
rendering.colorAttachmentCount = 1;
|
||||
rendering.pColorAttachments = &attachment;
|
||||
command.beginRendering(&rendering);
|
||||
const vk::Viewport viewport {
|
||||
0.0f, 0.0f, static_cast<float>(m_extent.width), static_cast<float>(m_extent.height),
|
||||
0.0f, 1.0f};
|
||||
const vk::Rect2D scissor {{0, 0}, m_extent};
|
||||
command.setViewport(0, 1, &viewport);
|
||||
command.setScissor(0, 1, &scissor);
|
||||
command.draw(3, 1, 0, 0);
|
||||
command.endRendering();
|
||||
}
|
||||
|
||||
void Swapchain::RecordPresentCommands(CommandBuffer& command, Presenter::Frame* source,
|
||||
const Presenter::Layer& overlay, bool draw_system_overlay) {
|
||||
EXIT_IF(m_image_index >= m_images.size());
|
||||
auto vk_command = command.Handle();
|
||||
auto vk_command = command.Handle();
|
||||
const bool draw_overlay = overlay.frame != nullptr;
|
||||
const bool draw_attachment = draw_overlay || draw_system_overlay;
|
||||
if (source != nullptr) {
|
||||
source->Transit(vk_command, vk::ImageLayout::eTransferSrcOptimal,
|
||||
vk::AccessFlagBits2::eTransferRead);
|
||||
}
|
||||
if (draw_overlay) {
|
||||
overlay.frame->Transit(vk_command, vk::ImageLayout::eShaderReadOnlyOptimal,
|
||||
vk::AccessFlagBits2::eShaderRead);
|
||||
}
|
||||
|
||||
vk::ImageMemoryBarrier to_transfer {};
|
||||
to_transfer.sType = vk::StructureType::eImageMemoryBarrier;
|
||||
@@ -619,49 +750,68 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
|
||||
vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags {}, 0,
|
||||
nullptr, 0, nullptr, 1, &to_transfer);
|
||||
|
||||
vk::ImageBlit region {};
|
||||
region.srcSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
|
||||
region.srcSubresource.mipLevel = 0;
|
||||
region.srcSubresource.baseArrayLayer = 0;
|
||||
region.srcSubresource.layerCount = 1;
|
||||
region.srcOffsets[1].x = static_cast<int>(source.extent.width);
|
||||
region.srcOffsets[1].y = static_cast<int>(source.extent.height);
|
||||
region.srcOffsets[1].z = 1;
|
||||
region.dstSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
|
||||
region.dstSubresource.mipLevel = 0;
|
||||
region.dstSubresource.baseArrayLayer = 0;
|
||||
region.dstSubresource.layerCount = 1;
|
||||
region.dstOffsets[1].x = static_cast<int>(m_extent.width);
|
||||
region.dstOffsets[1].y = static_cast<int>(m_extent.height);
|
||||
region.dstOffsets[1].z = 1;
|
||||
vk_command.blitImage(source.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal, 1, ®ion,
|
||||
vk::Filter::eLinear);
|
||||
if (source != nullptr) {
|
||||
vk::ImageBlit region {};
|
||||
region.srcSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
|
||||
region.srcSubresource.mipLevel = 0;
|
||||
region.srcSubresource.baseArrayLayer = 0;
|
||||
region.srcSubresource.layerCount = 1;
|
||||
region.srcOffsets[1].x = static_cast<int>(source->image.extent.width);
|
||||
region.srcOffsets[1].y = static_cast<int>(source->image.extent.height);
|
||||
region.srcOffsets[1].z = 1;
|
||||
region.dstSubresource.aspectMask = vk::ImageAspectFlagBits::eColor;
|
||||
region.dstSubresource.mipLevel = 0;
|
||||
region.dstSubresource.baseArrayLayer = 0;
|
||||
region.dstSubresource.layerCount = 1;
|
||||
region.dstOffsets[1].x = static_cast<int>(m_extent.width);
|
||||
region.dstOffsets[1].y = static_cast<int>(m_extent.height);
|
||||
region.dstOffsets[1].z = 1;
|
||||
vk_command.blitImage(source->image.image, vk::ImageLayout::eTransferSrcOptimal,
|
||||
m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal, 1,
|
||||
®ion, vk::Filter::eLinear);
|
||||
} else {
|
||||
const vk::ClearColorValue black {};
|
||||
vk_command.clearColorImage(m_images[m_image_index], vk::ImageLayout::eTransferDstOptimal,
|
||||
&black, 1, &to_transfer.subresourceRange);
|
||||
}
|
||||
|
||||
vk::ImageMemoryBarrier to_present {};
|
||||
to_present.sType = vk::StructureType::eImageMemoryBarrier;
|
||||
to_present.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
|
||||
to_present.dstAccessMask = draw_system_overlay ? vk::AccessFlagBits::eColorAttachmentRead |
|
||||
vk::AccessFlagBits::eColorAttachmentWrite
|
||||
: vk::AccessFlagBits::eMemoryRead;
|
||||
to_present.oldLayout = vk::ImageLayout::eTransferDstOptimal;
|
||||
to_present.newLayout = draw_system_overlay ? vk::ImageLayout::eColorAttachmentOptimal
|
||||
: vk::ImageLayout::ePresentSrcKHR;
|
||||
to_present.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
to_present.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
to_present.image = m_images[m_image_index];
|
||||
to_present.sType = vk::StructureType::eImageMemoryBarrier;
|
||||
to_present.srcAccessMask = vk::AccessFlagBits::eTransferWrite;
|
||||
to_present.dstAccessMask = draw_attachment ? vk::AccessFlagBits::eColorAttachmentRead |
|
||||
vk::AccessFlagBits::eColorAttachmentWrite
|
||||
: vk::AccessFlagBits::eMemoryRead;
|
||||
to_present.oldLayout = vk::ImageLayout::eTransferDstOptimal;
|
||||
to_present.newLayout = draw_attachment ? vk::ImageLayout::eColorAttachmentOptimal
|
||||
: vk::ImageLayout::ePresentSrcKHR;
|
||||
to_present.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
to_present.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
to_present.image = m_images[m_image_index];
|
||||
to_present.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor;
|
||||
to_present.subresourceRange.baseMipLevel = 0;
|
||||
to_present.subresourceRange.levelCount = 1;
|
||||
to_present.subresourceRange.baseArrayLayer = 0;
|
||||
to_present.subresourceRange.layerCount = 1;
|
||||
vk_command.pipelineBarrier(
|
||||
vk::PipelineStageFlagBits::eTransfer,
|
||||
draw_system_overlay ? vk::PipelineStageFlagBits::eColorAttachmentOutput
|
||||
: vk::PipelineStageFlagBits::eAllCommands,
|
||||
vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1, &to_present);
|
||||
if (draw_system_overlay) {
|
||||
m_system_overlay->Record(vk_command, m_image_views[m_image_index]);
|
||||
vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer,
|
||||
draw_attachment ? vk::PipelineStageFlagBits::eColorAttachmentOutput
|
||||
: vk::PipelineStageFlagBits::eAllCommands,
|
||||
vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1,
|
||||
&to_present);
|
||||
if (draw_attachment) {
|
||||
if (draw_overlay) {
|
||||
DrawOverlay(vk_command, overlay);
|
||||
if (draw_system_overlay) {
|
||||
to_present.srcAccessMask = vk::AccessFlagBits::eColorAttachmentWrite;
|
||||
to_present.oldLayout = vk::ImageLayout::eColorAttachmentOptimal;
|
||||
vk_command.pipelineBarrier(vk::PipelineStageFlagBits::eColorAttachmentOutput,
|
||||
vk::PipelineStageFlagBits::eColorAttachmentOutput,
|
||||
vk::DependencyFlagBits::eByRegion, 0, nullptr, 0,
|
||||
nullptr, 1, &to_present);
|
||||
}
|
||||
}
|
||||
if (draw_system_overlay) {
|
||||
m_system_overlay->Record(vk_command, m_image_views[m_image_index]);
|
||||
}
|
||||
to_present.srcAccessMask = vk::AccessFlagBits::eColorAttachmentWrite;
|
||||
to_present.dstAccessMask = vk::AccessFlagBits::eMemoryRead;
|
||||
to_present.oldLayout = vk::ImageLayout::eColorAttachmentOptimal;
|
||||
@@ -721,19 +871,18 @@ Presenter::~Presenter() = default;
|
||||
Presenter::Frame& Presenter::PrepareFrame(CommandBuffer& buffer, const ImageInfo& info) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
EXIT_IF(buffer.IsInvalid());
|
||||
auto* frame = m_impl->frames.Acquire();
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
auto& image = m_impl->ResolveSurface(info);
|
||||
if (image.backing.format == vk::Format::eUndefined) {
|
||||
EXIT("unsupported presentation source, image=%p\n", static_cast<const void*>(&image));
|
||||
}
|
||||
|
||||
auto frame_format = info.pixel_format;
|
||||
switch (frame_format) {
|
||||
case vk::Format::eR8G8B8A8Srgb: frame_format = vk::Format::eR8G8B8A8Unorm; break;
|
||||
case vk::Format::eB8G8R8A8Srgb: frame_format = vk::Format::eB8G8R8A8Unorm; break;
|
||||
default: break;
|
||||
}
|
||||
auto* frame = m_impl->frames.Acquire({info.extent.width, info.extent.height}, frame_format);
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
auto& image = m_impl->ResolveSurface(info);
|
||||
if (image.backing.format == vk::Format::eUndefined) {
|
||||
EXIT("unsupported presentation source, image=%p\n", static_cast<const void*>(&image));
|
||||
}
|
||||
frame->Configure(m_impl->window.graphic_ctx,
|
||||
{image.backing.extent.width, image.backing.extent.height}, frame_format);
|
||||
frame->CopyFrom(buffer, image);
|
||||
@@ -744,7 +893,7 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
|
||||
CommandBuffer* producer) {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
auto format = m_impl->frames.GetFormat();
|
||||
auto* frame = m_impl->frames.Acquire();
|
||||
auto* frame = m_impl->frames.Acquire({width, height}, format);
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
frame->Configure(m_impl->window.graphic_ctx, {width, height}, format);
|
||||
vk::ClearColorValue clear {};
|
||||
@@ -760,8 +909,13 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
|
||||
return *frame;
|
||||
}
|
||||
|
||||
Presenter::Frame* Presenter::PrepareLastFrame() {
|
||||
return m_impl->frames.AcquireLast();
|
||||
bool Presenter::PresentLastFrame() {
|
||||
Common::LockGuard lock(m_impl->present_mutex);
|
||||
if (m_impl->layers[0].frame == nullptr && m_impl->layers[1].frame == nullptr) {
|
||||
return false;
|
||||
}
|
||||
m_impl->Present();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Presenter::IsGuestPaused() const noexcept {
|
||||
@@ -778,44 +932,76 @@ RenderContext& Presenter::Renderer() const noexcept {
|
||||
return m_impl->renderer;
|
||||
}
|
||||
|
||||
void Presenter::Present(Frame& frame, bool reuse) {
|
||||
void Presenter::Present(Frame& frame) {
|
||||
const Layer layer {&frame, 0, false};
|
||||
Present(std::span(&layer, 1));
|
||||
}
|
||||
|
||||
void Presenter::Present(std::span<const Layer> layers) {
|
||||
Common::LockGuard lock(m_impl->present_mutex);
|
||||
for (const auto& layer: layers) {
|
||||
EXIT_IF(layer.bus < 0 || layer.bus >= static_cast<int>(m_impl->layers.size()));
|
||||
m_impl->frames.ValidateForPresent(layer.frame);
|
||||
auto& previous = m_impl->layers[layer.bus];
|
||||
if (previous.frame != nullptr) {
|
||||
m_impl->frames.Release(previous.frame);
|
||||
}
|
||||
previous = layer;
|
||||
}
|
||||
m_impl->Present();
|
||||
}
|
||||
|
||||
void Presenter::ClearLayer(int bus) {
|
||||
if (static_cast<size_t>(bus) >= m_impl->layers.size()) {
|
||||
return;
|
||||
}
|
||||
Common::LockGuard lock(m_impl->present_mutex);
|
||||
auto& layer = m_impl->layers[bus];
|
||||
if (layer.frame != nullptr) {
|
||||
m_impl->frames.Release(layer.frame);
|
||||
layer = {};
|
||||
}
|
||||
}
|
||||
|
||||
void Presenter::Impl::Present() {
|
||||
KYTY_PROFILER_FUNCTION();
|
||||
m_impl->frames.ValidateForPresent(&frame, reuse);
|
||||
|
||||
const auto overlay_visual = GetSystemOverlayVisualState();
|
||||
auto& swapchain = m_impl->swapchain;
|
||||
// Some window systems keep presenting an old swapchain after a resize.
|
||||
if (swapchain.NeedsResize()) {
|
||||
m_impl->RecoverSwapchain(Swapchain::Status::Recreate);
|
||||
RecoverSwapchain(Swapchain::Status::Recreate);
|
||||
}
|
||||
for (uint32_t attempt = 0; attempt < 2; attempt++) {
|
||||
auto status = swapchain.AcquireNextImage();
|
||||
if (status != Swapchain::Status::Success) {
|
||||
m_impl->RecoverSwapchain(status);
|
||||
RecoverSwapchain(status);
|
||||
continue;
|
||||
}
|
||||
{
|
||||
Common::LockGuard render_lock(m_impl->renderer.GetMutex());
|
||||
auto& command = m_impl->present_scheduler.BeginCommand();
|
||||
Common::LockGuard render_lock(renderer.GetMutex());
|
||||
auto& command = present_scheduler.BeginCommand();
|
||||
const bool draw_system_overlay =
|
||||
overlay_visual.active && swapchain.PrepareSystemOverlay();
|
||||
swapchain.RecordPresentCommands(command, frame.image, draw_system_overlay);
|
||||
frame.present_tick = swapchain.Submit(m_impl->present_scheduler);
|
||||
swapchain.RecordPresentCommands(command, layers[0].frame, layers[1],
|
||||
draw_system_overlay);
|
||||
const auto tick = swapchain.Submit(present_scheduler);
|
||||
for (const auto& layer: layers) {
|
||||
if (layer.frame != nullptr) {
|
||||
layer.frame->present_tick = tick;
|
||||
}
|
||||
}
|
||||
}
|
||||
status = swapchain.Present();
|
||||
if (status != Swapchain::Status::Success) {
|
||||
m_impl->RecoverSwapchain(status);
|
||||
RecoverSwapchain(status);
|
||||
continue;
|
||||
}
|
||||
|
||||
m_impl->presented_overlay_revision.store(overlay_visual.revision,
|
||||
std::memory_order_release);
|
||||
m_impl->window.UpdateTitle();
|
||||
m_impl->frames.Release(&frame, true);
|
||||
presented_overlay_revision.store(overlay_visual.revision, std::memory_order_release);
|
||||
window.UpdateTitle();
|
||||
return;
|
||||
}
|
||||
LOGF("Vulkan presentation retry exhausted; dropping frame\n");
|
||||
m_impl->frames.Release(&frame, reuse);
|
||||
}
|
||||
|
||||
void Presenter::Discard(Frame& frame) {
|
||||
|
||||
@@ -238,6 +238,10 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
|
||||
#endif
|
||||
}
|
||||
#if !defined(__APPLE__)
|
||||
if (device_features2.features.depthClamp != VK_TRUE) {
|
||||
LOGF("depthClamp is not supported\n");
|
||||
skip_device = true;
|
||||
}
|
||||
if (fragment_barycentric.fragmentShaderBarycentric != VK_TRUE) {
|
||||
LOGF("fragmentShaderBarycentric is not supported\n");
|
||||
skip_device = true;
|
||||
@@ -560,8 +564,10 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
vk::PhysicalDeviceVulkan11Properties properties11 {};
|
||||
properties11.pNext = &subgroup_size_control;
|
||||
|
||||
vk::PhysicalDeviceFloatControlsProperties float_controls {};
|
||||
float_controls.pNext = &properties11;
|
||||
vk::PhysicalDeviceProperties2 properties2 {};
|
||||
properties2.pNext = &properties11;
|
||||
properties2.pNext = &float_controls;
|
||||
|
||||
if (graphics.mesh_shader_enabled) {
|
||||
subgroup_size_control.pNext = &graphics.mesh_shader_properties;
|
||||
@@ -607,6 +613,7 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
device_features.robustBufferAccess = VK_TRUE;
|
||||
#if !defined(__APPLE__)
|
||||
device_features.depthBounds = VK_TRUE; // unsupported by MoltenVK
|
||||
device_features.depthClamp = VK_TRUE;
|
||||
#endif
|
||||
device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
|
||||
device_features.shaderImageGatherExtended = VK_TRUE;
|
||||
@@ -623,6 +630,15 @@ static vk::Device VulkanCreateDevice(GraphicContext& graphics,
|
||||
device_features.vertexPipelineStoresAndAtomics = VK_TRUE;
|
||||
graphics.sample_rate_shading_enabled = true;
|
||||
device_features.shaderInt64 = VK_TRUE;
|
||||
device_features.shaderFloat64 =
|
||||
supported_features2.features.shaderFloat64 &&
|
||||
float_controls.shaderSignedZeroInfNanPreserveFloat64 &&
|
||||
float_controls.shaderRoundingModeRTEFloat32;
|
||||
// if (device_features.shaderFloat64 && !float_controls.shaderDenormPreserveFloat64) {
|
||||
// Log::WriteToConsoleAndLog(
|
||||
// "WARNING: Vulkan device does not guarantee FP64 denormal preservation; "
|
||||
// "continuing with native FP64 arithmetic. Very small values may be flushed to zero.\n");
|
||||
// }
|
||||
|
||||
vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
|
||||
#if defined(__APPLE__)
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <fmt/format.h>
|
||||
#include <map>
|
||||
@@ -523,6 +523,20 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
|
||||
static_cast<uint64_t>(decoded.instructions.size()), phase_ms());
|
||||
|
||||
// Temporary workaround for games that compile ray-tracing shaders before
|
||||
// the player can select a mode without ray tracing.
|
||||
if (options.stage == ShaderType::Compute && decoded.has_bvh) {
|
||||
static std::atomic_flag warned = ATOMIC_FLAG_INIT;
|
||||
if (!warned.test_and_set(std::memory_order_relaxed)) {
|
||||
const auto& bvh = decoded.instructions.back();
|
||||
Log::WriteToConsoleAndLog(fmt::format(
|
||||
"Warning: ray tracing is not implemented; skipping compute dispatches containing "
|
||||
"BVH intersection instructions (shader=0x{:016x}, pc=0x{:08x}, opcode=0x{:02x}).\n",
|
||||
options.shader_hash, bvh.pc, bvh.opcode_id));
|
||||
}
|
||||
return {.skip_dispatch = true};
|
||||
}
|
||||
|
||||
std::string decoded_dump;
|
||||
if (options.dump_ir) {
|
||||
decoded_dump = Decoder::ProgramToString(decoded);
|
||||
@@ -533,30 +547,40 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
||||
|
||||
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph\n", GetDumpLabel(options),
|
||||
StageName(options.stage), options.shader_hash);
|
||||
auto cfg = CFG::BuildGraph(decoded);
|
||||
auto native_cfg = CFG::BuildGraph(decoded);
|
||||
CFG::Graph structured_cfg;
|
||||
auto* selected_cfg = &native_cfg;
|
||||
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG BuildGraph blocks=%" PRIu64
|
||||
" loops=%" PRIu64 " back_edges=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
|
||||
static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
|
||||
static_cast<uint64_t>(cfg.back_edges.size()), phase_ms());
|
||||
if (cfg.irreducible) {
|
||||
LogDispatcherFallback(options, cfg, "build");
|
||||
static_cast<uint64_t>(native_cfg.blocks.size()),
|
||||
static_cast<uint64_t>(native_cfg.natural_loops.size()),
|
||||
static_cast<uint64_t>(native_cfg.back_edges.size()), phase_ms());
|
||||
if (native_cfg.irreducible) {
|
||||
LogDispatcherFallback(options, native_cfg, "build");
|
||||
} else {
|
||||
LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash);
|
||||
if (!CFG::Structurize(cfg)) {
|
||||
LogDispatcherFallback(options, cfg, "structurize");
|
||||
structured_cfg = CFG::Structurize(native_cfg);
|
||||
if (structured_cfg.unsupported) {
|
||||
native_cfg.unsupported = true;
|
||||
native_cfg.failure_kind = structured_cfg.failure_kind;
|
||||
native_cfg.failure_block = structured_cfg.failure_block;
|
||||
native_cfg.unsupported_reason = structured_cfg.unsupported_reason;
|
||||
LogDispatcherFallback(options, native_cfg, "structurize");
|
||||
} else {
|
||||
selected_cfg = &structured_cfg;
|
||||
LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
|
||||
static_cast<uint64_t>(cfg.blocks.size()));
|
||||
static_cast<uint64_t>(selected_cfg->blocks.size()));
|
||||
}
|
||||
LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " CFG Structurize blocks=%" PRIu64
|
||||
" loops=%" PRIu64 " elapsed_ms=%" PRIu64 "\n",
|
||||
GetDumpLabel(options), StageName(options.stage), options.shader_hash,
|
||||
static_cast<uint64_t>(cfg.blocks.size()),
|
||||
static_cast<uint64_t>(cfg.natural_loops.size()), phase_ms());
|
||||
static_cast<uint64_t>(selected_cfg->blocks.size()),
|
||||
static_cast<uint64_t>(selected_cfg->natural_loops.size()), phase_ms());
|
||||
}
|
||||
|
||||
const auto& cfg = *selected_cfg;
|
||||
Frontend::EmbeddedFetchPlan embedded_fetch;
|
||||
if ((options.stage == ShaderType::Vertex || options.stage == ShaderType::Local) &&
|
||||
options.input_info.vertex != nullptr && options.input_info.vertex->fetch_embedded) {
|
||||
@@ -599,15 +623,21 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
||||
IR::EliminateDeadCode(ir.blocks);
|
||||
}
|
||||
LowerTessellationMemory(ir, options);
|
||||
IR::BuildSrtPlan(ir);
|
||||
IR::EliminateDeadCode(ir.blocks);
|
||||
IR::TrackResources(ir);
|
||||
std::string cfg_dump;
|
||||
if (options.dump_ir) {
|
||||
cfg_dump = CFG::GraphToString(cfg);
|
||||
if (options.early_dump) {
|
||||
LOGF("%s native IR before resource tracking:\n%s", GetDumpLabel(options),
|
||||
MakeIrDump(cfg_dump, ir).c_str());
|
||||
}
|
||||
}
|
||||
IR::TrackResources(ir, decoded, native_cfg);
|
||||
IR::EliminateDeadCode(ir.blocks);
|
||||
TranslateResult result;
|
||||
result.program = std::move(ir);
|
||||
if (options.dump_ir) {
|
||||
result.decoded_dump = std::move(decoded_dump);
|
||||
result.cfg_dump = CFG::GraphToString(cfg);
|
||||
result.cfg_dump = std::move(cfg_dump);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -615,9 +645,39 @@ TranslateResult TranslateProgram(std::span<const uint32_t> code, const CompileOp
|
||||
CompileResult CompileProgram(TranslateResult translated, const CompileOptions& options,
|
||||
const IR::ResourceSpecialization& specialization,
|
||||
uint32_t push_data_start_dword) {
|
||||
EXIT_IF(translated.skip_dispatch);
|
||||
const auto emit_begin = std::chrono::steady_clock::now();
|
||||
auto& ir = translated.program;
|
||||
IR::ApplyResourceSpecialization(ir, specialization);
|
||||
// The resource plan owns host descriptor evaluation now. Keep only dependencies consumed
|
||||
// by GPU memory operations; bound descriptor dwords must not retain shader instructions.
|
||||
for (auto& inst: ir.value_storage) {
|
||||
inst.Invalidate();
|
||||
}
|
||||
for (auto* block: ir.blocks) {
|
||||
for (auto& inst: *block) {
|
||||
const auto op = inst.GetOpcode();
|
||||
uint32_t first = 0;
|
||||
if (op == IR::ValueOpcode::GetBufferResource) {
|
||||
if (std::ranges::any_of(inst.Uses(), [&](const IR::Use& use) {
|
||||
return ir.memory_info[use.user->Flags<IR::MemoryFlags>().index].kind ==
|
||||
IR::ResourceKind::IndirectBuffer;
|
||||
})) {
|
||||
continue;
|
||||
}
|
||||
} else if (op == IR::ValueOpcode::GetImageResource) {
|
||||
const auto resource = inst.Flags<uint32_t>();
|
||||
first = resource < ir.info.images.size() &&
|
||||
ir.info.images[resource].indirect_root == resource ? 1u : 0u;
|
||||
} else if (op != IR::ValueOpcode::GetSamplerResource) {
|
||||
continue;
|
||||
}
|
||||
for (size_t index = first; index < inst.NumArgs(); index++) {
|
||||
inst.SetArg(index, IR::Value(0u));
|
||||
}
|
||||
}
|
||||
}
|
||||
ir.value_storage.clear();
|
||||
IR::RemoveIdentities(ir.blocks);
|
||||
IR::EliminateDeadCode(ir.blocks);
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ struct TranslateResult {
|
||||
IR::Program program;
|
||||
std::string decoded_dump;
|
||||
std::string cfg_dump;
|
||||
bool skip_dispatch = false;
|
||||
};
|
||||
|
||||
struct CompileResult {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "common/assert.h"
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
@@ -34,9 +35,6 @@ void ValidateNativeProgram(const IR::Program& program) {
|
||||
present[index] = true;
|
||||
expected[index] = std::move(resources);
|
||||
};
|
||||
if (!program.info.buffers.empty()) {
|
||||
Expect(Kind::Buffers, Dense(program.info.buffers.size()));
|
||||
}
|
||||
for (uint32_t i = 0; i < program.info.images.size(); i++) {
|
||||
const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
|
||||
if (!kind.has_value()) {
|
||||
@@ -54,23 +52,9 @@ void ValidateNativeProgram(const IR::Program& program) {
|
||||
if (!program.info.samplers.empty()) {
|
||||
Expect(Kind::Samplers, Dense(program.info.samplers.size()));
|
||||
}
|
||||
bool uses_gds = false;
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
|
||||
continue;
|
||||
}
|
||||
const auto index = inst.Flags<IR::MemoryFlags>().index;
|
||||
if (index >= program.memory_info.size()) {
|
||||
Fail(program, "shared operation has invalid memory metadata");
|
||||
}
|
||||
const auto kind = program.memory_info[index].kind;
|
||||
if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
|
||||
Fail(program, "shared operation has invalid resource kind");
|
||||
}
|
||||
uses_gds |= kind == IR::ResourceKind::Gds;
|
||||
}
|
||||
}
|
||||
auto& buffers = expected[static_cast<size_t>(Kind::Buffers)];
|
||||
const bool uses_gds = IR::CollectMemoryResources(program, buffers);
|
||||
present[static_cast<size_t>(Kind::Buffers)] = !buffers.empty();
|
||||
if (uses_gds) {
|
||||
Expect(Kind::Gds);
|
||||
}
|
||||
@@ -78,12 +62,7 @@ void ValidateNativeProgram(const IR::Program& program) {
|
||||
Expect(Kind::BdaPagetable);
|
||||
Expect(Kind::FaultBuffer);
|
||||
}
|
||||
const bool uses_flattened_runtime =
|
||||
!program.srt_reads.empty() ||
|
||||
std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
|
||||
return image.indirect_search_iterations != 0u;
|
||||
});
|
||||
if (uses_flattened_runtime) {
|
||||
if (IR::UsesFlattenedSrt(program)) {
|
||||
Expect(Kind::FlattenedSrt);
|
||||
}
|
||||
if (program.bindings.ShaderDataDwords() != 0 && !program.bindings.UsesPushData()) {
|
||||
@@ -109,7 +88,7 @@ void ValidateNativeProgram(const IR::Program& program) {
|
||||
if ((program.bindings.UsesPushData() &&
|
||||
!IR::PushData::CanFit(program.bindings.push_data_start_dword, shader_data_dwords)) ||
|
||||
program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
|
||||
program.bindings.memory_offset_count != program.info.buffers.size() ||
|
||||
program.bindings.memory_offset_count != buffers.size() ||
|
||||
has_shader_data_storage != (shader_data_dwords != 0 && !program.bindings.UsesPushData()) ||
|
||||
!std::is_sorted(program.bindings.user_data_registers.begin(),
|
||||
program.bindings.user_data_registers.end()) ||
|
||||
@@ -198,6 +177,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
|
||||
SpirvRequirements requirements {};
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
requirements.float64 |= inst.GetType() == IR::Type::F64;
|
||||
if (IR::BufferAccessOf(inst.GetOpcode()) == IR::BufferAccess::Atomic &&
|
||||
inst.GetType() == IR::Type::U64) {
|
||||
requirements.buffer_int64_atomics = true;
|
||||
@@ -227,6 +207,7 @@ Emitter::SpirvRequirements Emitter::AnalyzeProgramRequirements(const IR::Program
|
||||
requirements.subgroup_local_invocation_id = true;
|
||||
}
|
||||
if (memory.kind == IR::ResourceKind::Buffer) {
|
||||
requirements.coherent_buffers |= memory.coherent;
|
||||
if (memory.resource >= program.info.buffers.size()) {
|
||||
Fail(program, "buffer operation has invalid resource metadata");
|
||||
}
|
||||
|
||||
@@ -291,9 +291,25 @@ uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0) {
|
||||
return EmitF32ToU32(state, arg0, false);
|
||||
}
|
||||
|
||||
uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0) {
|
||||
const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0);
|
||||
return EmitNative<spv::OpConvertSToF, IR::Type::F32>(state, signed_value);
|
||||
uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0) {
|
||||
const auto converted = Unary(state, spv::OpFConvert, TypeF32(state), arg0);
|
||||
const auto source = EmitNative<spv::OpCompositeExtract, IR::Type::U32>(
|
||||
state, Unary(state, spv::OpBitcast, TypeU64(state), arg0), 1u);
|
||||
const auto exponent = EmitAndConstant(state, source, 0x7ff00000u);
|
||||
const auto overflow =
|
||||
Binary(state, spv::OpLogicalAnd, TypeBool(state),
|
||||
EmitCompareU32Constant(state, spv::OpUGreaterThan, exponent, 0x47e00000u),
|
||||
EmitCompareU32Constant(state, spv::OpULessThan, exponent, 0x7ff00000u));
|
||||
const auto clamped = EmitOrU32(state, EmitAndConstant(state, source, 0x80000000u),
|
||||
ConstantU32(state, 0x7f7fffffu));
|
||||
return EmitFlushF32DenormToSignedZero(
|
||||
state, Select(state, TypeF32(state), overflow,
|
||||
Unary(state, spv::OpBitcast, TypeF32(state), clamped), converted));
|
||||
}
|
||||
|
||||
uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0) {
|
||||
return EmitNative<spv::OpFConvert, IR::Type::F64>(state,
|
||||
EmitFlushF32DenormToSignedZero(state, arg0));
|
||||
}
|
||||
|
||||
uint32_t EmitCompositeExtractU64(EmitterState& state, uint32_t arg0, IR::Value arg1) {
|
||||
@@ -453,6 +469,11 @@ uint32_t EmitFPRecip32(EmitterState& state, uint32_t arg0) {
|
||||
return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), source);
|
||||
}
|
||||
|
||||
uint32_t EmitFPRecip64(EmitterState& state, uint32_t arg0) {
|
||||
const auto one = state.builder.Constant(spv::OpConstant, TypeF64(state), 0u, 0x3ff00000u);
|
||||
return EmitNative<spv::OpFDiv, IR::Type::F64>(state, one, arg0);
|
||||
}
|
||||
|
||||
uint32_t EmitFPRecipIFlag32(EmitterState& state, uint32_t arg0) {
|
||||
// Integer-to-float inputs used by IFLAG cannot be denormal.
|
||||
return Binary(state, spv::OpFDiv, TypeF32(state), ConstantF32(state, 0x3f800000u), arg0);
|
||||
|
||||
@@ -203,9 +203,7 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
|
||||
const auto bits = state.builder.AllocateId();
|
||||
const auto abs_bits = state.builder.AllocateId();
|
||||
const auto sign_bits = state.builder.AllocateId();
|
||||
const auto non_zero = state.builder.AllocateId();
|
||||
const auto subnormal = state.builder.AllocateId();
|
||||
const auto flush = state.builder.AllocateId();
|
||||
const auto selected = state.builder.AllocateId();
|
||||
const auto ret = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpBitcast, TypeU32(state), bits, value);
|
||||
@@ -213,12 +211,9 @@ uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
|
||||
ConstantU32(state, 0x7fffffffu));
|
||||
state.builder.AddFunction(spv::OpBitwiseAnd, TypeU32(state), sign_bits, bits,
|
||||
ConstantU32(state, 0x80000000u));
|
||||
state.builder.AddFunction(spv::OpINotEqual, TypeBool(state), non_zero, abs_bits,
|
||||
ConstantU32(state, 0));
|
||||
state.builder.AddFunction(spv::OpULessThan, TypeBool(state), subnormal, abs_bits,
|
||||
ConstantU32(state, 0x00800000u));
|
||||
state.builder.AddFunction(spv::OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal);
|
||||
state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, flush, sign_bits, bits);
|
||||
state.builder.AddFunction(spv::OpSelect, TypeU32(state), selected, subnormal, sign_bits, bits);
|
||||
state.builder.AddFunction(spv::OpBitcast, TypeF32(state), ret, selected);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -636,6 +636,29 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
ctx.Arg(inst, 1));
|
||||
}
|
||||
|
||||
uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
if (ctx.other_half == nullptr) return ctx.Arg(inst, 0);
|
||||
// A native scalar branch makes one decision for both emulated wave halves.
|
||||
if (ctx.half != 0) return ctx.other_half->Def(IR::Value(&inst));
|
||||
const auto kind = inst.Flags<CFG::BranchCondition>();
|
||||
if (kind == CFG::BranchCondition::ScalarInstruction) return ctx.Arg(inst, 0);
|
||||
const auto ballot = ctx.Ballot(inst.Arg(0));
|
||||
const auto low = ctx.state.builder.AllocateId();
|
||||
const auto high = ctx.state.builder.AllocateId();
|
||||
const auto combined = ctx.state.builder.AllocateId();
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
|
||||
const bool zero = kind == CFG::BranchCondition::ExecZero ||
|
||||
kind == CFG::BranchCondition::VccZero || kind == CFG::BranchCondition::SccZero;
|
||||
ctx.state.builder.AddFunction(zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr,
|
||||
TypeU32(ctx.state), combined, low, high);
|
||||
ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual,
|
||||
TypeBool(ctx.state), result, combined,
|
||||
ConstantU32(ctx.state, zero ? ~0u : 0u));
|
||||
return result;
|
||||
}
|
||||
|
||||
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate) {
|
||||
return ctx.Ballot(predicate);
|
||||
}
|
||||
|
||||
@@ -877,7 +877,7 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
return;
|
||||
}
|
||||
const auto atomic_opcode = ImageAtomicOpcode(op);
|
||||
if (atomic_opcode != spv::OpNop) {
|
||||
if (image_info.access == IR::ImageAccess::Atomic) {
|
||||
const auto dimension = image.dimension;
|
||||
ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
|
||||
const auto pointer = state.builder.AllocateId();
|
||||
@@ -887,6 +887,15 @@ void EmitImage(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
StorageImageDescriptorPointer(state, mem.resource),
|
||||
CoordU32(ctx, mem, *address, dimension),
|
||||
ConstantU32(state, 0));
|
||||
if (op == IR::ValueOpcode::ImageAtomicFMin32 ||
|
||||
op == IR::ValueOpcode::ImageAtomicFMax32) {
|
||||
return AtomicUpdate(state, pointer, IR::ResourceKind::Image,
|
||||
[&](uint32_t old) {
|
||||
return EmitFloatAtomicReplacement(
|
||||
state, old, ctx.Arg(inst, 2),
|
||||
op == IR::ValueOpcode::ImageAtomicFMax32);
|
||||
});
|
||||
}
|
||||
const auto old = state.builder.AllocateId();
|
||||
state.builder.AddFunction(atomic_opcode, TypeU32(state), old, pointer,
|
||||
ConstantU32(state, spv::ScopeDevice),
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
|
||||
|
||||
#include <bit>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
|
||||
// Operand signatures distinguish SPIR-V IDs (uint32_t), raw literals/values
|
||||
// (IR::Value), and instructions whose metadata or operand loads must stay lazy.
|
||||
@@ -13,14 +15,25 @@ inline constexpr auto EmitConvertU32U16 = EmitBitCastU16F16;
|
||||
inline constexpr auto EmitConvertU32U8 = EmitBitCastU16F16;
|
||||
inline constexpr auto EmitBitCastU32F32 = EmitBitCastU16F16;
|
||||
EMIT_NATIVE(BitCastF32U32, OpBitcast, F32, uint32_t)
|
||||
EMIT_NATIVE(BitCastU64F64, OpBitcast, U64, uint32_t)
|
||||
EMIT_NATIVE(BitCastF64U64, OpBitcast, F64, uint32_t)
|
||||
uint32_t EmitConvertU16U32(EmitterState& state, uint32_t arg0);
|
||||
uint32_t EmitConvertU8U32(EmitterState& state, uint32_t arg0);
|
||||
uint32_t EmitConvertF16F32(EmitterState& state, uint32_t arg0);
|
||||
inline constexpr auto EmitConvertF32F16 = EmitF16BitsToF32;
|
||||
uint32_t EmitConvertS32F32(EmitterState& state, uint32_t arg0);
|
||||
uint32_t EmitConvertU32F32(EmitterState& state, uint32_t arg0);
|
||||
uint32_t EmitConvertF32S32(EmitterState& state, uint32_t arg0);
|
||||
template <IR::Type type>
|
||||
uint32_t EmitConvertSigned32ToFloat(EmitterState& state, uint32_t arg0) {
|
||||
const auto signed_value = Unary(state, spv::OpBitcast, TypeI32(state), arg0);
|
||||
return EmitNative<spv::OpConvertSToF, type>(state, signed_value);
|
||||
}
|
||||
inline constexpr auto EmitConvertF32S32 = EmitConvertSigned32ToFloat<IR::Type::F32>;
|
||||
inline constexpr auto EmitConvertF64S32 = EmitConvertSigned32ToFloat<IR::Type::F64>;
|
||||
uint32_t EmitConvertF32F64(EmitterState& state, uint32_t arg0);
|
||||
uint32_t EmitConvertF64F32(EmitterState& state, uint32_t arg0);
|
||||
EMIT_NATIVE(ConvertF32U32, OpConvertUToF, F32, uint32_t)
|
||||
EMIT_NATIVE(ConvertF64U32, OpConvertUToF, F64, uint32_t)
|
||||
EMIT_NATIVE(CompositeConstructU64, OpCompositeConstruct, U64, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(CompositeConstructU32x2, OpCompositeConstruct, U32x2, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(CompositeConstructU32x3, OpCompositeConstruct, U32x3, uint32_t, uint32_t, uint32_t)
|
||||
@@ -103,23 +116,43 @@ EMIT_NATIVE(LogicalOr, OpLogicalOr, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(LogicalAnd, OpLogicalAnd, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(LogicalXor, OpLogicalNotEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(LogicalNot, OpLogicalNot, U1, uint32_t)
|
||||
EMIT_NATIVE(FPOrdEqual32, OpFOrdEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPUnordEqual32, OpFUnordEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPOrdNotEqual32, OpFOrdNotEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPUnordNotEqual32, OpFUnordNotEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPOrdLessThan32, OpFOrdLessThan, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPUnordLessThan32, OpFUnordLessThan, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPOrdGreaterThan32, OpFOrdGreaterThan, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPUnordGreaterThan32, OpFUnordGreaterThan, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPOrdLessThanEqual32, OpFOrdLessThanEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPUnordLessThanEqual32, OpFUnordLessThanEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPOrdGreaterThanEqual32, OpFOrdGreaterThanEqual, U1, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPUnordGreaterThanEqual32, OpFUnordGreaterThanEqual, U1, uint32_t, uint32_t)
|
||||
template <spv::Op opcode>
|
||||
uint32_t EmitFloatCompare32(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
// SPIR-V's DenormFlushToZero mode does not require comparison operands to flush.
|
||||
const bool flush = inst.Flags<IR::FPCompareFlags>().flush_input_denorms;
|
||||
const auto operand = [&](size_t index) {
|
||||
const auto value = inst.Arg(index);
|
||||
if (flush && value.IsImmediate()) {
|
||||
const auto bits = std::bit_cast<uint32_t>(value.F32Value());
|
||||
return ConstantF32(ctx.state, (bits & 0x7fffffffu) < 0x00800000u
|
||||
? bits & 0x80000000u : bits);
|
||||
}
|
||||
const auto id = ctx.Arg(inst, index);
|
||||
return flush ? EmitFlushF32DenormToSignedZero(ctx.state, id) : id;
|
||||
};
|
||||
return EmitNative<opcode, IR::Type::U1>(ctx.state, operand(0), operand(1));
|
||||
}
|
||||
inline constexpr auto EmitFPOrdEqual32 = EmitFloatCompare32<spv::OpFOrdEqual>;
|
||||
inline constexpr auto EmitFPUnordEqual32 = EmitFloatCompare32<spv::OpFUnordEqual>;
|
||||
inline constexpr auto EmitFPOrdNotEqual32 = EmitFloatCompare32<spv::OpFOrdNotEqual>;
|
||||
inline constexpr auto EmitFPUnordNotEqual32 = EmitFloatCompare32<spv::OpFUnordNotEqual>;
|
||||
inline constexpr auto EmitFPOrdLessThan32 = EmitFloatCompare32<spv::OpFOrdLessThan>;
|
||||
inline constexpr auto EmitFPUnordLessThan32 = EmitFloatCompare32<spv::OpFUnordLessThan>;
|
||||
inline constexpr auto EmitFPOrdGreaterThan32 = EmitFloatCompare32<spv::OpFOrdGreaterThan>;
|
||||
inline constexpr auto EmitFPUnordGreaterThan32 = EmitFloatCompare32<spv::OpFUnordGreaterThan>;
|
||||
inline constexpr auto EmitFPOrdLessThanEqual32 = EmitFloatCompare32<spv::OpFOrdLessThanEqual>;
|
||||
inline constexpr auto EmitFPUnordLessThanEqual32 = EmitFloatCompare32<spv::OpFUnordLessThanEqual>;
|
||||
inline constexpr auto EmitFPOrdGreaterThanEqual32 = EmitFloatCompare32<spv::OpFOrdGreaterThanEqual>;
|
||||
inline constexpr auto EmitFPUnordGreaterThanEqual32 = EmitFloatCompare32<spv::OpFUnordGreaterThanEqual>;
|
||||
uint32_t EmitFPIsNan32(EmitterState& state, uint32_t arg0);
|
||||
inline constexpr auto EmitFPCmpClass32 = EmitClassMaskF32;
|
||||
EMIT_NATIVE(FPAdd32, OpFAdd, F32, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPSub32, OpFSub, F32, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPMul32, OpFMul, F32, uint32_t, uint32_t)
|
||||
EMIT_NATIVE(FPMul64, OpFMul, F64, uint32_t, uint32_t)
|
||||
inline constexpr auto EmitFPFma64 =
|
||||
EmitGlsl<GLSLstd450Fma, IR::Type::F64, uint32_t, uint32_t, uint32_t>;
|
||||
uint32_t EmitFPRecip64(EmitterState& state, uint32_t arg0);
|
||||
inline constexpr auto EmitFPFma32 =
|
||||
EmitGlsl<GLSLstd450Fma, IR::Type::F32, uint32_t, uint32_t, uint32_t>;
|
||||
uint32_t EmitFPMin32(EmitterState& state, uint32_t arg0, uint32_t arg1);
|
||||
@@ -169,6 +202,7 @@ uint32_t EmitDppUpdateU32(ValueEmitContext& ctx, const IR::Inst& in
|
||||
uint32_t EmitWqmU64(EmitterState& state, uint32_t value);
|
||||
uint32_t EmitLaneId(EmitterState& state);
|
||||
uint32_t EmitBallot(ValueEmitContext& ctx, IR::Value predicate);
|
||||
uint32_t EmitConditionRef(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
uint32_t EmitReadFirstLane(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
uint32_t EmitReadLane(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
uint32_t EmitWriteLane(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
@@ -271,6 +305,8 @@ inline constexpr auto EmitImageAtomicUMax32 = EmitImage;
|
||||
inline constexpr auto EmitImageAtomicAnd32 = EmitImage;
|
||||
inline constexpr auto EmitImageAtomicOr32 = EmitImage;
|
||||
inline constexpr auto EmitImageAtomicXor32 = EmitImage;
|
||||
inline constexpr auto EmitImageAtomicFMin32 = EmitImage;
|
||||
inline constexpr auto EmitImageAtomicFMax32 = EmitImage;
|
||||
void EmitUnreachable(ValueEmitContext& ctx, const IR::Inst& inst);
|
||||
inline constexpr auto EmitPhi = EmitUnreachable;
|
||||
inline constexpr auto EmitTessellationBase = EmitUnreachable;
|
||||
|
||||
@@ -67,6 +67,8 @@ struct SpirvRequirements {
|
||||
bool function_scratch = false;
|
||||
bool pixel_valid_mask = false;
|
||||
bool buffer_int64_atomics = false;
|
||||
bool coherent_buffers = false;
|
||||
bool float64 = false;
|
||||
};
|
||||
|
||||
SpirvRequirements AnalyzeProgramRequirements(const IR::Program& program);
|
||||
@@ -138,6 +140,7 @@ uint32_t TypeU32Pair(EmitterState& state);
|
||||
uint32_t TypeI32(EmitterState& state);
|
||||
uint32_t TypeI32Pair(EmitterState& state);
|
||||
uint32_t TypeF32(EmitterState& state);
|
||||
uint32_t TypeF64(EmitterState& state);
|
||||
uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
|
||||
|
||||
uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
|
||||
@@ -166,6 +169,10 @@ template <spv::Op opcode, IR::Type type, typename... Args>
|
||||
uint32_t EmitNative(EmitterState& state, Args... args) {
|
||||
const auto result = state.builder.AllocateId();
|
||||
state.builder.AddFunction(opcode, TypeId(state, type), result, args...);
|
||||
if constexpr (type == IR::Type::F64 &&
|
||||
(opcode == spv::OpFMul || opcode == spv::OpFDiv || opcode == spv::OpExtInst)) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, result, spv::DecorationNoContraction);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -371,12 +378,13 @@ void EmitMemoryOffsets(EmitterState& state);
|
||||
uint32_t LdsDwordCount(const EmitterState& state);
|
||||
|
||||
struct MemoryResourceAccess {
|
||||
IR::ResourceKind kind = IR::ResourceKind::None;
|
||||
uint32_t object_pointer = 0;
|
||||
uint32_t length = 0;
|
||||
uint32_t index_offset = 0;
|
||||
uint32_t byte_offset = 0;
|
||||
bool add_index_offset = false;
|
||||
IR::ResourceKind kind = IR::ResourceKind::None;
|
||||
uint32_t object_pointer = 0;
|
||||
uint32_t length = 0;
|
||||
uint32_t index_offset = 0;
|
||||
uint32_t byte_offset = 0;
|
||||
bool add_index_offset = false;
|
||||
spv::MemoryAccessMask memory_access = spv::MemoryAccessMaskNone;
|
||||
};
|
||||
|
||||
MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
|
||||
|
||||
@@ -146,7 +146,12 @@ uint32_t GuestAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Mem
|
||||
ctx.Fail(inst, "has no address base pair");
|
||||
return ConstantDeviceAddress(state, 0);
|
||||
}
|
||||
const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1));
|
||||
auto base_low = ctx.Arg(*handle, 0);
|
||||
if (mem.kind == IR::ResourceKind::ScalarAddress) {
|
||||
base_low = Binary(state, spv::OpBitwiseAnd, TypeU32(state), base_low,
|
||||
ConstantU32(state, ~3u));
|
||||
}
|
||||
const auto base = DeviceAddressFromWords(state, base_low, ctx.Arg(*handle, 1));
|
||||
address = Binary(state, spv::OpIAdd, TypeScalarU64(state), base,
|
||||
Unary(state, spv::OpUConvert, TypeScalarU64(state), low));
|
||||
}
|
||||
@@ -313,7 +318,8 @@ uint32_t LoadWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& res
|
||||
uint32_t index) {
|
||||
const auto value = ctx.state.builder.AllocateId();
|
||||
const auto pointer = EmitMemoryElementPointer(ctx.state, resource, index);
|
||||
ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, pointer);
|
||||
ctx.state.builder.AddFunction(spv::OpLoad, TypeU32(ctx.state), value, pointer,
|
||||
resource.memory_access);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -498,21 +504,21 @@ void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo me
|
||||
});
|
||||
}
|
||||
|
||||
void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index,
|
||||
uint32_t data) {
|
||||
ctx.state.builder.AddFunction(spv::OpStore,
|
||||
EmitMemoryElementPointer(ctx.state, resource, index), data,
|
||||
resource.memory_access);
|
||||
}
|
||||
|
||||
void StoreWordPrepared(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
|
||||
const MemoryResourceAccess& resource, uint32_t data) {
|
||||
const auto index = EmitMemoryElementIndex(ctx.state, resource, DwordIndex(ctx, inst, mem));
|
||||
EmitIfCondition(ctx.state, EmitMemoryElementInBounds(ctx.state, resource, index), [&]() {
|
||||
ctx.state.builder.AddFunction(spv::OpStore,
|
||||
EmitMemoryElementPointer(ctx.state, resource, index), data);
|
||||
StoreWordInBounds(ctx, resource, index, data);
|
||||
});
|
||||
}
|
||||
|
||||
void StoreWordInBounds(ValueEmitContext& ctx, const MemoryResourceAccess& resource, uint32_t index,
|
||||
uint32_t data) {
|
||||
ctx.state.builder.AddFunction(spv::OpStore,
|
||||
EmitMemoryElementPointer(ctx.state, resource, index), data);
|
||||
}
|
||||
|
||||
void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem) {
|
||||
EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
|
||||
const auto resource = PrepareMemoryResourceAccess(ctx.state, mem);
|
||||
@@ -705,26 +711,66 @@ uint32_t FormattedOutOfBoundsValue(ValueEmitContext& ctx, const IR::MemoryInfo&
|
||||
return ConstructU32Composite(ctx.state, components, values);
|
||||
}
|
||||
|
||||
void StoreFormattedInBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem,
|
||||
const PreparedFormattedMemory& plan, uint32_t component,
|
||||
uint32_t data) {
|
||||
if (component >= plan.info.component_count) return;
|
||||
const auto bits = plan.info.component_bits[component];
|
||||
if (bits == 16u && (plan.info.type == Format::ComponentType::Snorm ||
|
||||
plan.info.type == Format::ComponentType::Float)) {
|
||||
uint32_t EncodeFormattedStoreComponent(ValueEmitContext& ctx,
|
||||
const Format::BufferFormatInfo& info,
|
||||
uint32_t component, uint32_t data) {
|
||||
const auto bits = info.component_bits[component];
|
||||
if (info.type == Format::ComponentType::Unorm) {
|
||||
const auto value = EmitBitCastF32U32(ctx.state, data);
|
||||
const auto clamped = EmitFPMin32(
|
||||
ctx.state, EmitFPMax32(ctx.state, value, ConstantF32Value(ctx.state, 0.0f)),
|
||||
ConstantF32Value(ctx.state, 1.0f));
|
||||
const auto scaled = EmitFPMul32(
|
||||
ctx.state, clamped, ConstantF32Value(ctx.state, float((1u << bits) - 1u)));
|
||||
return Unary(ctx.state, spv::OpConvertFToU, TypeU32(ctx.state),
|
||||
EmitFPRoundEven32(ctx.state, scaled));
|
||||
}
|
||||
if (bits == 16u && (info.type == Format::ComponentType::Snorm ||
|
||||
info.type == Format::ComponentType::Float)) {
|
||||
const auto value = EmitBitCastF32U32(ctx.state, data);
|
||||
const auto pair = EmitCompositeConstructF32x2(ctx.state, value,
|
||||
ConstantF32Value(ctx.state, 0.0f));
|
||||
data = plan.info.type == Format::ComponentType::Float
|
||||
? EmitPackHalf2x16(ctx.state, pair)
|
||||
: EmitPackSnorm2x16(ctx.state, pair);
|
||||
}
|
||||
if (bits == 8u || bits == 16u) {
|
||||
StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses[component],
|
||||
plan.indices[component], bits, data);
|
||||
} else {
|
||||
StoreWordInBounds(ctx, plan.resource, plan.indices[component], data);
|
||||
return info.type == Format::ComponentType::Float ? EmitPackHalf2x16(ctx.state, pair)
|
||||
: EmitPackSnorm2x16(ctx.state, pair);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
void StoreFormattedPrepared(ValueEmitContext& ctx, const IR::Inst& inst,
|
||||
const IR::MemoryInfo& mem, const MemoryResourceAccess& resource,
|
||||
const Format::BufferFormatInfo& info, uint32_t data,
|
||||
uint32_t components) {
|
||||
// RDNA2 formatted stores transfer the entire format, zero-filling missing VGPRs.
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, info.component_count,
|
||||
FormattedAccess::Store);
|
||||
EmitIfCondition(ctx.state, plan.in_bounds, [&]() {
|
||||
auto packed = ConstantU32(ctx.state, 0);
|
||||
for (uint32_t component = 0; component < info.component_count; component++) {
|
||||
auto value = ConstantU32(ctx.state, 0);
|
||||
if (component < components) {
|
||||
value = data;
|
||||
if (components != 1u) {
|
||||
value = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state),
|
||||
value, data, component);
|
||||
}
|
||||
}
|
||||
value = EncodeFormattedStoreComponent(ctx, info, component, value);
|
||||
const auto bits = info.component_bits[component];
|
||||
if (info.packed_bitfield) {
|
||||
packed = EmitBitFieldInsert(
|
||||
ctx.state, packed, value,
|
||||
ConstantU32(ctx.state, info.component_bit_offset[component]),
|
||||
ConstantU32(ctx.state, bits));
|
||||
} else if (bits == 8u || bits == 16u) {
|
||||
StoreSubwordInBounds(ctx, mem, resource, plan.addresses[component],
|
||||
plan.indices[component], bits, value);
|
||||
} else {
|
||||
StoreWordInBounds(ctx, resource, plan.indices[component], value);
|
||||
}
|
||||
}
|
||||
if (info.packed_bitfield) StoreWordInBounds(ctx, resource, plan.indices[0], packed);
|
||||
});
|
||||
}
|
||||
|
||||
void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
|
||||
@@ -736,11 +782,7 @@ void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::Memor
|
||||
StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data);
|
||||
return;
|
||||
}
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, 1u,
|
||||
FormattedAccess::Store);
|
||||
EmitIfCondition(ctx.state, plan.in_bounds, [&]() {
|
||||
StoreFormattedInBounds(ctx, mem, plan, 0u, data);
|
||||
});
|
||||
StoreFormattedPrepared(ctx, inst, mem, resource, info, data, 1u);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -858,16 +900,7 @@ void StoreWideBuffer(ValueEmitContext& ctx, const IR::Inst& inst, uint32_t compo
|
||||
const auto info = Format::GetFormatInfo(
|
||||
mem.formatted ? BufferFormat(ctx, mem) : Prospero::BufferFormat::kInvalid);
|
||||
if (info.type != Format::ComponentType::Unknown) {
|
||||
const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components,
|
||||
FormattedAccess::Store);
|
||||
EmitIfCondition(state, plan.in_bounds, [&]() {
|
||||
for (uint32_t component = 0; component < components; component++) {
|
||||
const auto data = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(state), data,
|
||||
composite, component);
|
||||
StoreFormattedInBounds(ctx, mem, plan, component, data);
|
||||
}
|
||||
});
|
||||
StoreFormattedPrepared(ctx, inst, mem, resource, info, composite, components);
|
||||
return;
|
||||
}
|
||||
for (uint32_t component = 0; component < components; component++) {
|
||||
@@ -942,7 +975,14 @@ void DefineGetBdaPointer(EmitterState& state) {
|
||||
state.builder.AddFunction(spv::OpFunctionParameter, type, address);
|
||||
EmitLabel(state, entry_label);
|
||||
|
||||
const auto page64 = Binary(state, spv::OpShiftRightLogical, type, address,
|
||||
const auto extended = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address,
|
||||
ConstantDeviceAddress(state, LibKernel::Memory::kExtendedMemoryBase));
|
||||
const auto packed = Select(state, type, extended,
|
||||
Binary(state, spv::OpISub, type, address,
|
||||
ConstantDeviceAddress(state,
|
||||
LibKernel::Memory::kExtendedMemoryBase - LOWER_ADDRESS_SIZE)),
|
||||
address);
|
||||
const auto page64 = Binary(state, spv::OpShiftRightLogical, type, packed,
|
||||
ConstantDeviceAddress(state, BufferCache::CACHING_PAGEBITS));
|
||||
const auto page = Unary(state, spv::OpUConvert, TypeU32(state), page64);
|
||||
const auto entry_pointer = state.builder.AllocateId();
|
||||
@@ -1071,16 +1111,27 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
if (ctx.half == 1) {
|
||||
return ctx.other_half->Def(IR::Value(const_cast<IR::Inst*>(&inst)));
|
||||
}
|
||||
const auto m0 = ctx.Arg(inst, 0);
|
||||
const auto base =
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
|
||||
const auto size =
|
||||
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
|
||||
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), base,
|
||||
ConstantU32(state, ctx.Memory(inst).offset));
|
||||
const auto mem = ctx.Memory(inst);
|
||||
const auto offset = mem.offset & 0xfffcu;
|
||||
auto address = ConstantU32(state, offset);
|
||||
uint32_t region_bounds = 0;
|
||||
if (mem.kind == IR::ResourceKind::Gds) {
|
||||
const auto m0 = ctx.Arg(inst, 0);
|
||||
const auto base =
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16));
|
||||
const auto size =
|
||||
Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu));
|
||||
address = Binary(state, spv::OpIAdd, TypeU32(state), base, address);
|
||||
// The entire M0 region must fit the PS5's 48 KiB GDS partition.
|
||||
region_bounds = AndCondition(
|
||||
state, Binary(state, spv::OpULessThan, TypeBool(state),
|
||||
ConstantU32(state, offset + 3u), size),
|
||||
Binary(state, spv::OpULessThanEqual, TypeBool(state),
|
||||
Binary(state, spv::OpIAdd, TypeU32(state), base, size),
|
||||
ConstantU32(state, 0xc000u)));
|
||||
}
|
||||
const auto raw_index =
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
|
||||
const auto mem = ctx.Memory(inst);
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto index = EmitMemoryElementIndex(state, access, raw_index);
|
||||
const auto exec = ctx.Arg(inst, 1);
|
||||
@@ -1099,19 +1150,17 @@ uint32_t EmitAppendConsume(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
: first;
|
||||
const auto is_first = Binary(state, spv::OpIEqual, TypeBool(state),
|
||||
EmitSubgroupLocalInvocationId(state), source_lane);
|
||||
const auto storage_bounds = EmitMemoryElementInBounds(state, access, index);
|
||||
const auto m0_bounds =
|
||||
mem.kind == IR::ResourceKind::Gds
|
||||
? Binary(state, spv::OpINotEqual, TypeBool(state), size, ConstantU32(state, 0))
|
||||
: Binary(state, spv::OpULessThan, TypeBool(state),
|
||||
ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
|
||||
auto bounds = EmitMemoryElementInBounds(state, access, index);
|
||||
if (mem.kind == IR::ResourceKind::Gds) {
|
||||
bounds = AndCondition(state, bounds, region_bounds);
|
||||
}
|
||||
const auto condition = AndCondition(
|
||||
state, is_first,
|
||||
AndCondition(state,
|
||||
state.lane_count == 2 ? Binary(state, spv::OpINotEqual, TypeBool(state), count,
|
||||
ConstantU32(state, 0))
|
||||
: exec,
|
||||
AndCondition(state, storage_bounds, m0_bounds)));
|
||||
bounds));
|
||||
const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
|
||||
const auto value = state.builder.AllocateId();
|
||||
state.builder.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state),
|
||||
@@ -1145,10 +1194,11 @@ void EmitReadConstBuffer(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
if (mem.planning_only) return;
|
||||
auto& state = ctx.state;
|
||||
mem.kind = IR::ResourceKind::ScalarBuffer;
|
||||
const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1),
|
||||
ConstantU32(state, mem.offset));
|
||||
const auto index =
|
||||
Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2));
|
||||
auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ctx.Arg(inst, 1),
|
||||
ConstantU32(state, 2));
|
||||
if (mem.offset >= 4u) {
|
||||
index = Binary(state, spv::OpIAdd, TypeU32(state), index, ConstantU32(state, mem.offset >> 2u));
|
||||
}
|
||||
const auto access = PrepareMemoryResourceAccess(state, mem);
|
||||
const auto element = EmitMemoryElementIndex(state, access, index);
|
||||
const auto condition = EmitMemoryElementInBounds(state, access, element);
|
||||
@@ -1170,6 +1220,8 @@ void EmitLoadMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
|
||||
value = LoadWideBuffer(ctx, inst, buffer_components);
|
||||
else if (shared_components > 1u)
|
||||
value = LoadWideShared(ctx, inst, shared_components);
|
||||
else if (mem.kind == IR::ResourceKind::ScalarAddress)
|
||||
value = LoadBdaDword(ctx, GuestAddress(ctx, inst, mem));
|
||||
else if (address_info.access == IR::AddressAccess::Read &&
|
||||
mem.kind != IR::ResourceKind::Scratch)
|
||||
value = LoadBda(ctx, GuestAddress(ctx, inst, mem), ctx.Arg(inst, inst.NumArgs() - 1),
|
||||
|
||||
@@ -86,7 +86,9 @@ MemoryResourceAccess PrepareStorageBufferResourceAccess(EmitterState& state,
|
||||
}
|
||||
const auto array_index =
|
||||
ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
|
||||
MemoryResourceAccess access {.kind = mem.kind};
|
||||
MemoryResourceAccess access {
|
||||
.kind = mem.kind,
|
||||
.memory_access = mem.coherent ? spv::MemoryAccessVolatileMask : spv::MemoryAccessMaskNone};
|
||||
access.object_pointer = state.builder.AllocateId();
|
||||
state.builder.AddFunction(spv::OpAccessChain, pointer_type, access.object_pointer, variable,
|
||||
ConstantU32(state, array_index));
|
||||
|
||||
@@ -47,6 +47,10 @@ uint32_t TypeF32(EmitterState& state) {
|
||||
return state.builder.Type(spv::OpTypeFloat, 32);
|
||||
}
|
||||
|
||||
uint32_t TypeF64(EmitterState& state) {
|
||||
return state.builder.Type(spv::OpTypeFloat, 64);
|
||||
}
|
||||
|
||||
uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
|
||||
return state.builder.Type(spv::OpTypeVector, TypeU32(state), components);
|
||||
}
|
||||
@@ -201,6 +205,16 @@ void DefineDescriptors(EmitterState& state) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
|
||||
spv::DecorationAliased);
|
||||
}
|
||||
if (state.requirements.coherent_buffers) {
|
||||
// RDNA2 stores publish to L2 even without GLC; every alias of the buffer
|
||||
// must participate in visibility for cache-bypassing polling loads.
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_variable,
|
||||
spv::DecorationCoherent);
|
||||
if (state.storage_buffer_u64_variable != 0) {
|
||||
state.builder.AddAnnotation(spv::OpDecorate, state.storage_buffer_u64_variable,
|
||||
spv::DecorationCoherent);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case IR::DescriptorBindingKind::BdaPagetable:
|
||||
state.bda_pagetable_variable = Define(StorageBufferU64Type(state), "bda_pagetable");
|
||||
@@ -669,6 +683,19 @@ void DefineModule(EmitterState& state) {
|
||||
// contract prevents host compilers from treating synthesized IEEE values as finite.
|
||||
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
|
||||
32u);
|
||||
if (state.requirements.float64) {
|
||||
EXIT_NOT_IMPLEMENTED(state.program.stage == ShaderType::Compute &&
|
||||
state.input_info.compute->float_mode != 0xc0);
|
||||
// MODE=0xc0 uses round-to-nearest-even and preserves FP64 input/output denormals.
|
||||
state.builder.RequireCapability(spv::CapabilityFloat64);
|
||||
state.builder.RequireCapability(spv::CapabilityRoundingModeRTE);
|
||||
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeSignedZeroInfNanPreserve,
|
||||
64u);
|
||||
// FP64 denormal preservation is temporarily disabled.
|
||||
// state.builder.RequireCapability(spv::CapabilityDenormPreserve);
|
||||
// state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeDenormPreserve, 64u);
|
||||
state.builder.AddExecutionMode(state.main_func, spv::ExecutionModeRoundingModeRTE, 32u);
|
||||
}
|
||||
if (const auto* cs = ShaderWorkgroupInput(state.program.stage, state.input_info)) {
|
||||
uint32_t local_x = state.requirements.compute_derivatives ? 2u : 1u;
|
||||
uint32_t local_y = state.requirements.compute_derivatives ? 2u : 1u;
|
||||
|
||||
@@ -97,30 +97,6 @@ void EmitReturn(ValueEmitContext& ctx) {
|
||||
ctx.state.builder.AddFunction(spv::OpReturn);
|
||||
}
|
||||
|
||||
uint32_t BranchCondition(ValueEmitContext& ctx, const IR::BlockInfo& info) {
|
||||
// Scalar-instruction conditions already test the full wave's raw register values.
|
||||
if (ctx.other_half == nullptr ||
|
||||
info.terminator.condition == CFG::BranchCondition::ScalarInstruction ||
|
||||
info.terminator.condition == CFG::BranchCondition::GotoVariable) {
|
||||
return ctx.Def(info.condition);
|
||||
}
|
||||
const auto ballot = ctx.Ballot(info.condition);
|
||||
const auto low = ctx.state.builder.AllocateId();
|
||||
const auto high = ctx.state.builder.AllocateId();
|
||||
const auto result = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), low, ballot, 0);
|
||||
ctx.state.builder.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), high, ballot, 1);
|
||||
const auto kind = info.terminator.condition;
|
||||
const bool zero = kind == CFG::BranchCondition::ExecZero ||
|
||||
kind == CFG::BranchCondition::VccZero ||
|
||||
kind == CFG::BranchCondition::SccZero;
|
||||
const auto combined =
|
||||
EmitBinaryU32(ctx.state, zero ? spv::OpBitwiseAnd : spv::OpBitwiseOr, low, high);
|
||||
ctx.state.builder.AddFunction(zero ? spv::OpIEqual : spv::OpINotEqual, TypeBool(ctx.state),
|
||||
result, combined, ConstantU32(ctx.state, zero ? ~0u : 0u));
|
||||
return result;
|
||||
}
|
||||
|
||||
void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
|
||||
const IR::BlockInfo& info) {
|
||||
const auto& program = ctx.state.program;
|
||||
@@ -160,7 +136,7 @@ void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
|
||||
EmitReturn(ctx);
|
||||
return;
|
||||
}
|
||||
const auto condition = BranchCondition(ctx, info);
|
||||
const auto condition = ctx.Def(info.condition);
|
||||
emit_merge();
|
||||
ctx.state.builder.AddFunction(spv::OpBranchConditional, condition,
|
||||
ctx.Label(true_block), ctx.Label(false_block));
|
||||
@@ -193,7 +169,7 @@ uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionSta
|
||||
EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
|
||||
const auto selected = ctx.state.builder.AllocateId();
|
||||
ctx.state.builder.AddFunction(
|
||||
spv::OpSelect, TypeU32(ctx.state), selected, BranchCondition(ctx, info),
|
||||
spv::OpSelect, TypeU32(ctx.state), selected, ctx.Def(info.condition),
|
||||
ConstantU32(ctx.state, term.true_block), ConstantU32(ctx.state, term.false_block));
|
||||
return selected;
|
||||
}
|
||||
@@ -440,6 +416,7 @@ uint32_t TypeId(EmitterState& state, IR::Type type) {
|
||||
case IR::Type::U64: return TypeU64(state);
|
||||
case IR::Type::U32x2: return TypeU32Pair(state);
|
||||
case IR::Type::F32: return TypeF32(state);
|
||||
case IR::Type::F64: return TypeF64(state);
|
||||
case IR::Type::U32x3: return TypeU32Vector(state, 3);
|
||||
case IR::Type::U32x4: return TypeU32Vector(state, 4);
|
||||
case IR::Type::F32x2: return TypeF32Vector(state, 2);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -18,10 +18,17 @@ enum class BranchCondition {
|
||||
ExecZero,
|
||||
ExecNonZero,
|
||||
ScalarInstruction,
|
||||
GotoVariable,
|
||||
Expression,
|
||||
Unknown
|
||||
};
|
||||
|
||||
struct ConditionExpression {
|
||||
enum class Op { Constant, Variable, Native, Not, Or };
|
||||
Op op;
|
||||
uint32_t lhs;
|
||||
uint32_t rhs = UINT32_MAX;
|
||||
};
|
||||
|
||||
enum class TerminatorKind { Branch, ConditionalBranch, IndirectBranch, Return, Unsupported };
|
||||
|
||||
enum class FailureKind {
|
||||
@@ -48,8 +55,7 @@ struct Terminator {
|
||||
std::vector<uint32_t> indirect_targets;
|
||||
std::vector<uint32_t> indirect_selector_values;
|
||||
std::vector<uint32_t> indirect_selector_targets;
|
||||
uint32_t goto_variable = UINT32_MAX;
|
||||
int32_t goto_value = -1;
|
||||
uint32_t expression = UINT32_MAX;
|
||||
bool loop_header = false;
|
||||
};
|
||||
|
||||
@@ -59,10 +65,11 @@ struct BasicBlock {
|
||||
uint32_t end_pc = 0;
|
||||
uint32_t inst_begin = 0;
|
||||
uint32_t inst_end = 0;
|
||||
struct Assignment { uint32_t variable; uint32_t expression; };
|
||||
std::vector<Assignment> assignments;
|
||||
std::vector<uint32_t> predecessors;
|
||||
std::vector<uint32_t> successors;
|
||||
std::vector<uint32_t> dominators;
|
||||
std::vector<uint32_t> post_dominators;
|
||||
Terminator terminator;
|
||||
};
|
||||
|
||||
@@ -73,12 +80,9 @@ struct BackEdge {
|
||||
};
|
||||
|
||||
struct NaturalLoop {
|
||||
uint32_t header = UINT32_MAX;
|
||||
uint32_t latch = UINT32_MAX;
|
||||
uint32_t merge = UINT32_MAX;
|
||||
uint32_t continue_block = UINT32_MAX;
|
||||
uint32_t header = UINT32_MAX;
|
||||
uint32_t latch = UINT32_MAX;
|
||||
std::vector<uint32_t> body_blocks;
|
||||
std::vector<uint32_t> exit_blocks;
|
||||
};
|
||||
|
||||
struct StronglyConnectedComponent {
|
||||
@@ -89,6 +93,7 @@ struct StronglyConnectedComponent {
|
||||
|
||||
struct Graph {
|
||||
std::vector<BasicBlock> blocks;
|
||||
std::vector<ConditionExpression> expressions;
|
||||
std::vector<BackEdge> back_edges;
|
||||
std::vector<NaturalLoop> natural_loops;
|
||||
std::vector<StronglyConnectedComponent> components;
|
||||
@@ -105,14 +110,12 @@ struct Graph {
|
||||
const BasicBlock* FindBlockByPc(uint32_t pc) const;
|
||||
BasicBlock* FindBlockByPc(uint32_t pc);
|
||||
bool Dominates(uint32_t dominator, uint32_t block) const;
|
||||
bool PostDominates(uint32_t post_dominator, uint32_t block) const;
|
||||
uint32_t FindNearestCommonPostDominator(uint32_t block_a, uint32_t block_b) const;
|
||||
};
|
||||
|
||||
Graph BuildGraph(const Decoder::Program& program);
|
||||
// Commits structured control flow on success; preserves the original graph with
|
||||
// failure diagnostics on failure. failure_block is an original block ID or UINT32_MAX.
|
||||
bool Structurize(Graph& graph);
|
||||
// Returns structured control flow or failure diagnostics without changing the native graph.
|
||||
// On failure, failure_block is an original block ID or UINT32_MAX.
|
||||
Graph Structurize(const Graph& graph);
|
||||
std::string BranchConditionToString(BranchCondition condition);
|
||||
std::string FailureKindToString(FailureKind kind);
|
||||
std::string GraphToString(const Graph& graph);
|
||||
|
||||
@@ -198,6 +198,8 @@ constexpr Detail::OpcodeMap MIMG_ATOMIC_OPCODE_LIST[] = {
|
||||
{0x18u, Opcode::IMAGE_ATOMIC_AND},
|
||||
{0x19u, Opcode::IMAGE_ATOMIC_OR},
|
||||
{0x1au, Opcode::IMAGE_ATOMIC_XOR},
|
||||
{0x1eu, Opcode::IMAGE_ATOMIC_FMIN},
|
||||
{0x1fu, Opcode::IMAGE_ATOMIC_FMAX},
|
||||
};
|
||||
|
||||
constexpr auto MIMG_SAMPLE_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_SAMPLE_OPCODE_LIST);
|
||||
|
||||
@@ -21,6 +21,7 @@ constexpr MemoryOpcodeInfo SMEM_OPCODE_LIST[] = {
|
||||
{0x04u, Opcode::S_LOAD_DWORDX16, 16, 32}, {0x08u, Opcode::S_BUFFER_LOAD_DWORD, 1, 32},
|
||||
{0x09u, Opcode::S_BUFFER_LOAD_DWORDX2, 2, 32}, {0x0au, Opcode::S_BUFFER_LOAD_DWORDX4, 4, 32},
|
||||
{0x0bu, Opcode::S_BUFFER_LOAD_DWORDX8, 8, 32}, {0x0cu, Opcode::S_BUFFER_LOAD_DWORDX16, 16, 32},
|
||||
{0x25u, Opcode::S_MEMREALTIME, 2, 32},
|
||||
};
|
||||
|
||||
constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
|
||||
@@ -245,6 +246,10 @@ void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
|
||||
}
|
||||
|
||||
DecodeScalarDestination(sdst, pc, inst.dst);
|
||||
if (inst.opcode == Opcode::S_MEMREALTIME) {
|
||||
inst.src_count = 0;
|
||||
return;
|
||||
}
|
||||
// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
|
||||
// pair index; their descriptor operand consumes four SGPRs from that base.
|
||||
DecodeScalarSource(sbase * 2u, pc, inst.src0);
|
||||
@@ -268,6 +273,7 @@ void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
|
||||
inst.idxen = ((word0 >> 13u) & 1u) != 0;
|
||||
inst.offen = ((word0 >> 12u) & 1u) != 0;
|
||||
inst.glc = ((word0 >> 14u) & 1u) != 0;
|
||||
inst.dlc = ((word0 >> 15u) & 1u) != 0;
|
||||
inst.slc = ((word1 >> 22u) & 1u) != 0;
|
||||
inst.family = Family::MUBUF;
|
||||
inst.opcode_id = opcode;
|
||||
@@ -303,6 +309,7 @@ void DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_inde
|
||||
inst.idxen = ((word0 >> 13u) & 1u) != 0;
|
||||
inst.offen = ((word0 >> 12u) & 1u) != 0;
|
||||
inst.glc = ((word0 >> 14u) & 1u) != 0;
|
||||
inst.dlc = ((word0 >> 15u) & 1u) != 0;
|
||||
inst.slc = ((word1 >> 22u) & 1u) != 0;
|
||||
inst.family = Family::MTBUF;
|
||||
inst.opcode_id = opcode;
|
||||
|
||||
@@ -73,11 +73,11 @@ std::string FormatMemory(const Instruction& inst) {
|
||||
text += OperandToString(*sources[i]);
|
||||
}
|
||||
text += fmt::format(" ; offset={} offset2={} dwords={} bits={} dfmt={} nfmt={} signed={} "
|
||||
"typed={} formatted={} segment={} glc={} slc={} idxen={} offen={}",
|
||||
"typed={} formatted={} segment={} glc={} dlc={} slc={} idxen={} offen={}",
|
||||
inst.offset, inst.secondary_offset, inst.data_dwords, inst.data_bits,
|
||||
inst.data_format, inst.number_format, inst.data_signed ? 1u : 0u,
|
||||
inst.typed ? 1u : 0u, inst.formatted ? 1u : 0u, inst.memory_segment,
|
||||
inst.glc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u,
|
||||
inst.glc ? 1u : 0u, inst.dlc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u,
|
||||
inst.offen ? 1u : 0u);
|
||||
return text;
|
||||
}
|
||||
@@ -403,6 +403,7 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
|
||||
program.instructions.clear();
|
||||
program.instructions.reserve(code.size());
|
||||
program.code = code;
|
||||
program.has_bvh = false;
|
||||
|
||||
std::vector<bool> branch_targets;
|
||||
for (uint32_t word_index = 0; word_index < code.size();) {
|
||||
@@ -411,6 +412,10 @@ void DecodeProgram(std::span<const uint32_t> code, Program& program) {
|
||||
|
||||
const auto& inst = program.instructions.back();
|
||||
word_index += inst.word_count;
|
||||
if (inst.family == Family::MIMG && (inst.opcode_id == 0xe6u || inst.opcode_id == 0xe7u)) {
|
||||
program.has_bvh = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (IsDirectBranch(inst.opcode)) {
|
||||
const auto target_index = inst.branch_target / sizeof(uint32_t);
|
||||
@@ -574,6 +579,8 @@ std::string InstructionToString(const Instruction& inst) {
|
||||
case Opcode::IMAGE_ATOMIC_AND:
|
||||
case Opcode::IMAGE_ATOMIC_OR:
|
||||
case Opcode::IMAGE_ATOMIC_XOR:
|
||||
case Opcode::IMAGE_ATOMIC_FMIN:
|
||||
case Opcode::IMAGE_ATOMIC_FMAX:
|
||||
case Opcode::IMAGE_LOAD:
|
||||
case Opcode::IMAGE_LOAD_MIP:
|
||||
case Opcode::IMAGE_GET_RESINFO:
|
||||
|
||||
@@ -154,6 +154,10 @@ enum class Opcode {
|
||||
V_CUBEMA_F32,
|
||||
V_CNDMASK_B32,
|
||||
V_DOT2C_F32_F16,
|
||||
V_CVT_F64_I32,
|
||||
V_CVT_F32_F64,
|
||||
V_CVT_F64_F32,
|
||||
V_CVT_F64_U32,
|
||||
V_CVT_F32_I32,
|
||||
V_CVT_F32_U32,
|
||||
V_CVT_U32_F32,
|
||||
@@ -173,6 +177,7 @@ enum class Opcode {
|
||||
V_CVT_F32_UBYTE1,
|
||||
V_CVT_F32_UBYTE2,
|
||||
V_CVT_F32_UBYTE3,
|
||||
V_RCP_F64,
|
||||
V_RCP_F32,
|
||||
V_RCP_IFLAG_F32,
|
||||
V_FRACT_F32,
|
||||
@@ -234,6 +239,8 @@ enum class Opcode {
|
||||
V_MAD_I32_I24,
|
||||
V_MAD_U32_U24,
|
||||
V_MAD_U64_U32,
|
||||
V_FMA_F64,
|
||||
V_MUL_F64,
|
||||
V_FMA_F32,
|
||||
V_FMA_F16,
|
||||
V_PACK_B32_F16,
|
||||
@@ -354,6 +361,7 @@ enum class Opcode {
|
||||
V_CMPX_GT_F32,
|
||||
V_CMPX_LG_F32,
|
||||
V_CMPX_GE_F32,
|
||||
V_CMPX_O_F32,
|
||||
V_CMPX_NGE_F32,
|
||||
V_CMPX_NLG_F32,
|
||||
V_CMPX_NGT_F32,
|
||||
@@ -404,6 +412,7 @@ enum class Opcode {
|
||||
V_CMP_LE_U16,
|
||||
V_CMP_GT_U16,
|
||||
V_CMPX_LT_U16,
|
||||
V_CMPX_EQ_U16,
|
||||
V_CMPX_GT_U16,
|
||||
V_CMP_NE_U16,
|
||||
V_CMP_GE_U16,
|
||||
@@ -439,6 +448,7 @@ enum class Opcode {
|
||||
S_BUFFER_LOAD_DWORDX4,
|
||||
S_BUFFER_LOAD_DWORDX8,
|
||||
S_BUFFER_LOAD_DWORDX16,
|
||||
S_MEMREALTIME,
|
||||
BUFFER_LOAD_FORMAT_X,
|
||||
BUFFER_LOAD_FORMAT_XY,
|
||||
BUFFER_LOAD_FORMAT_XYZ,
|
||||
@@ -568,6 +578,8 @@ enum class Opcode {
|
||||
IMAGE_ATOMIC_AND,
|
||||
IMAGE_ATOMIC_OR,
|
||||
IMAGE_ATOMIC_XOR,
|
||||
IMAGE_ATOMIC_FMIN,
|
||||
IMAGE_ATOMIC_FMAX,
|
||||
IMAGE_SAMPLE,
|
||||
IMAGE_GATHER4_L,
|
||||
IMAGE_GATHER4_LZ,
|
||||
@@ -711,6 +723,7 @@ struct Instruction {
|
||||
bool formatted = false;
|
||||
bool gds = false;
|
||||
bool glc = false;
|
||||
bool dlc = false;
|
||||
bool slc = false;
|
||||
bool idxen = false;
|
||||
bool offen = false;
|
||||
@@ -729,6 +742,7 @@ struct Instruction {
|
||||
struct Program {
|
||||
std::span<const uint32_t> code;
|
||||
std::vector<Instruction> instructions;
|
||||
bool has_bvh = false;
|
||||
};
|
||||
|
||||
// Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.
|
||||
|
||||
@@ -40,7 +40,7 @@ constexpr Vop2OpcodeInfo VOP2_OPCODE_LIST[] = {
|
||||
{0x0fu, Opcode::V_MIN_F32},
|
||||
{0x10u, Opcode::V_MAX_F32},
|
||||
{0x11u, Opcode::V_MIN_I32},
|
||||
{0x12u, Opcode::V_MAX_I32},
|
||||
{0x12u, Opcode::V_MAX_I32, Vop2SdwaProfile::IntegerFullDestination},
|
||||
{0x13u, Opcode::V_MIN_U32, Vop2SdwaProfile::IntegerPartialDestination},
|
||||
{0x14u, Opcode::V_MAX_U32, Vop2SdwaProfile::IntegerFullDestination},
|
||||
{0x15u, Opcode::V_LSHR_B32},
|
||||
@@ -87,6 +87,11 @@ constexpr OpcodeMap VOP1_OPCODE_LIST[] = {
|
||||
{0x00u, Opcode::V_NOP},
|
||||
{0x01u, Opcode::V_MOV_B32},
|
||||
{0x02u, Opcode::V_READFIRSTLANE_B32},
|
||||
{0x04u, Opcode::V_CVT_F64_I32},
|
||||
{0x0fu, Opcode::V_CVT_F32_F64},
|
||||
{0x10u, Opcode::V_CVT_F64_F32},
|
||||
{0x16u, Opcode::V_CVT_F64_U32},
|
||||
{0x2fu, Opcode::V_RCP_F64},
|
||||
{0x05u, Opcode::V_CVT_F32_I32},
|
||||
{0x06u, Opcode::V_CVT_F32_U32},
|
||||
{0x07u, Opcode::V_CVT_U32_F32},
|
||||
@@ -146,6 +151,11 @@ constexpr OpcodeMap VOP3_ENCODED_VOP1_OPCODE_LIST[] = {
|
||||
{0x00u, Opcode::V_NOP},
|
||||
{0x01u, Opcode::V_MOV_B32},
|
||||
{0x02u, Opcode::V_READFIRSTLANE_B32},
|
||||
{0x04u, Opcode::V_CVT_F64_I32},
|
||||
{0x0fu, Opcode::V_CVT_F32_F64},
|
||||
{0x10u, Opcode::V_CVT_F64_F32},
|
||||
{0x16u, Opcode::V_CVT_F64_U32},
|
||||
{0x2fu, Opcode::V_RCP_F64},
|
||||
{0x05u, Opcode::V_CVT_F32_I32},
|
||||
{0x06u, Opcode::V_CVT_F32_U32},
|
||||
{0x07u, Opcode::V_CVT_U32_F32},
|
||||
@@ -214,6 +224,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
|
||||
{0x11u, Opcode::V_CMPX_LT_F32}, {0x12u, Opcode::V_CMPX_EQ_F32},
|
||||
{0x13u, Opcode::V_CMPX_LE_F32}, {0x14u, Opcode::V_CMPX_GT_F32},
|
||||
{0x15u, Opcode::V_CMPX_LG_F32}, {0x16u, Opcode::V_CMPX_GE_F32},
|
||||
{0x17u, Opcode::V_CMPX_O_F32},
|
||||
{0x19u, Opcode::V_CMPX_NGE_F32}, {0x1au, Opcode::V_CMPX_NLG_F32},
|
||||
{0x1bu, Opcode::V_CMPX_NGT_F32}, {0x1cu, Opcode::V_CMPX_NLE_F32},
|
||||
{0x1du, Opcode::V_CMPX_NEQ_F32}, {0x1eu, Opcode::V_CMPX_NLT_F32},
|
||||
@@ -231,7 +242,7 @@ constexpr VopcOpcodeInfo VOPC_OPCODE_LIST[] = {
|
||||
{0xa9u, Opcode::V_CMP_LT_U16}, {0xaau, Opcode::V_CMP_EQ_U16},
|
||||
{0xabu, Opcode::V_CMP_LE_U16}, {0xacu, Opcode::V_CMP_GT_U16},
|
||||
{0xadu, Opcode::V_CMP_NE_U16}, {0xaeu, Opcode::V_CMP_GE_U16},
|
||||
{0xb9u, Opcode::V_CMPX_LT_U16, false},
|
||||
{0xb9u, Opcode::V_CMPX_LT_U16, false}, {0xbau, Opcode::V_CMPX_EQ_U16, false},
|
||||
{0xbcu, Opcode::V_CMPX_GT_U16}, {0xc0u, Opcode::V_CMP_F_U32},
|
||||
{0xc1u, Opcode::V_CMP_LT_U32}, {0xc2u, Opcode::V_CMP_EQ_U32},
|
||||
{0xc3u, Opcode::V_CMP_LE_U32}, {0xc4u, Opcode::V_CMP_GT_U32},
|
||||
@@ -266,6 +277,8 @@ constexpr OpcodeMap VOP3_OPCODE_LIST[] = {
|
||||
{0x146u, Opcode::V_CUBETC_F32},
|
||||
{0x147u, Opcode::V_CUBEMA_F32},
|
||||
{0x14bu, Opcode::V_FMA_F32},
|
||||
{0x14cu, Opcode::V_FMA_F64},
|
||||
{0x165u, Opcode::V_MUL_F64},
|
||||
{0x148u, Opcode::V_BFE_U32},
|
||||
{0x149u, Opcode::V_BFE_I32},
|
||||
{0x14au, Opcode::V_BFI_B32},
|
||||
@@ -451,6 +464,9 @@ bool IsVopcCompareExec(Opcode opcode);
|
||||
|
||||
bool IsVop1FloatSourceOpcode(Opcode opcode) {
|
||||
switch (opcode) {
|
||||
case Opcode::V_CVT_F32_F64:
|
||||
case Opcode::V_CVT_F64_F32:
|
||||
case Opcode::V_RCP_F64:
|
||||
case Opcode::V_MOV_B32:
|
||||
case Opcode::V_CVT_F32_F16:
|
||||
case Opcode::V_CVT_U32_F32:
|
||||
@@ -521,9 +537,12 @@ constexpr Vop1SdwaRule VOP1_SDWA_RULES[] = {
|
||||
{Opcode::V_CVT_F32_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
|
||||
{Opcode::V_CVT_F32_I32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
|
||||
{Opcode::V_CVT_F32_UBYTE0, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
|
||||
// The source field extract and the destination insert are independent (Translate.cpp), so a
|
||||
// partial destination may combine with any source selector.
|
||||
{Opcode::V_NOT_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelBytes() | SdwaSelWords(),
|
||||
SdwaSelFull(), false},
|
||||
SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), false},
|
||||
{Opcode::V_FFBL_B32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
|
||||
{Opcode::V_FFBH_U32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), 0, 0, false},
|
||||
{Opcode::V_CVT_F32_F16, SdwaSelWords() | SdwaSelFull(), 0, 0, true},
|
||||
{Opcode::V_CVT_F16_F32, SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), SdwaSelWords(),
|
||||
SdwaSelBytes() | SdwaSelWords() | SdwaSelFull(), true},
|
||||
@@ -706,6 +725,12 @@ void ApplyDppModifier(Operand& operand, uint32_t modifier, uint32_t encoding) {
|
||||
|
||||
void DecodeVop1Dpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
|
||||
uint32_t opcode, uint32_t vdst, Instruction& inst) {
|
||||
if (inst.opcode == Opcode::V_CVT_F64_I32 || inst.opcode == Opcode::V_CVT_F32_F64 ||
|
||||
inst.opcode == Opcode::V_CVT_F64_F32 || inst.opcode == Opcode::V_CVT_F64_U32 ||
|
||||
inst.opcode == Opcode::V_RCP_F64) {
|
||||
SetUnsupported(inst, Family::VOP1, opcode, "FP64 instructions do not support DPP");
|
||||
return;
|
||||
}
|
||||
const auto modifier = code[word_index + 1u];
|
||||
const auto src0 = modifier & 0xffu;
|
||||
SetRawWords(inst, code, word_index, 2);
|
||||
@@ -755,6 +780,7 @@ bool IsVop2FloatOpcode(Opcode opcode) {
|
||||
|
||||
bool IsVop1FloatResultOpcode(Opcode opcode) {
|
||||
switch (opcode) {
|
||||
case Opcode::V_CVT_F32_F64:
|
||||
case Opcode::V_CVT_F32_I32:
|
||||
case Opcode::V_CVT_F32_U32:
|
||||
case Opcode::V_CVT_F32_F16:
|
||||
@@ -820,6 +846,7 @@ bool IsVopcFloatCompareOpcode(Opcode opcode) {
|
||||
case Opcode::V_CMPX_GT_F32:
|
||||
case Opcode::V_CMPX_LG_F32:
|
||||
case Opcode::V_CMPX_GE_F32:
|
||||
case Opcode::V_CMPX_O_F32:
|
||||
case Opcode::V_CMPX_NGE_F32:
|
||||
case Opcode::V_CMPX_NLG_F32:
|
||||
case Opcode::V_CMPX_NGT_F32:
|
||||
@@ -1174,6 +1201,7 @@ void DecodeVopcDpp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_in
|
||||
|
||||
uint32_t NativeVop3SourceCount(Opcode opcode) {
|
||||
switch (opcode) {
|
||||
case Opcode::V_MUL_F64:
|
||||
case Opcode::V_MUL_LO_U32:
|
||||
case Opcode::V_MUL_HI_U32:
|
||||
case Opcode::V_MUL_LO_I32:
|
||||
@@ -1324,6 +1352,8 @@ bool SupportsNativeVop3SourceModifiers(Opcode opcode) {
|
||||
case Opcode::V_MAX_F32:
|
||||
case Opcode::V_MAC_F32:
|
||||
case Opcode::V_MAD_F32:
|
||||
case Opcode::V_MUL_F64:
|
||||
case Opcode::V_FMA_F64:
|
||||
case Opcode::V_FMA_F32:
|
||||
case Opcode::V_PACK_B32_F16:
|
||||
case Opcode::V_CUBEID_F32:
|
||||
@@ -1356,6 +1386,7 @@ bool SupportsNativeVop3ResultModifiers(Opcode opcode) {
|
||||
case Opcode::V_MAD_F32:
|
||||
case Opcode::V_FMA_F32:
|
||||
case Opcode::V_FMA_F16:
|
||||
case Opcode::V_CUBEID_F32:
|
||||
case Opcode::V_CVT_PKRTZ_F16_F32:
|
||||
case Opcode::V_LDEXP_F32:
|
||||
case Opcode::V_MIN3_F32:
|
||||
@@ -1432,6 +1463,7 @@ bool IsVopcCompareExec(Opcode opcode) {
|
||||
case Opcode::V_CMPX_GT_F32:
|
||||
case Opcode::V_CMPX_LG_F32:
|
||||
case Opcode::V_CMPX_GE_F32:
|
||||
case Opcode::V_CMPX_O_F32:
|
||||
case Opcode::V_CMPX_NGE_F32:
|
||||
case Opcode::V_CMPX_NLG_F32:
|
||||
case Opcode::V_CMPX_NGT_F32:
|
||||
@@ -1454,6 +1486,7 @@ bool IsVopcCompareExec(Opcode opcode) {
|
||||
case Opcode::V_CMPX_NE_I64:
|
||||
case Opcode::V_CMPX_NE_U64:
|
||||
case Opcode::V_CMPX_LT_U16:
|
||||
case Opcode::V_CMPX_EQ_U16:
|
||||
case Opcode::V_CMPX_GT_U16:
|
||||
case Opcode::V_CMPX_LT_F16:
|
||||
case Opcode::V_CMPX_EQ_F16:
|
||||
|
||||
@@ -106,12 +106,12 @@ void Translator::EXP(const Decoder::Instruction& inst) {
|
||||
ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
|
||||
}
|
||||
|
||||
bool Translator::EmitInterpolation(const Decoder::Instruction& inst) {
|
||||
void Translator::EmitInterpolation(const Decoder::Instruction& inst) {
|
||||
switch (inst.opcode) {
|
||||
case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return true;
|
||||
case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return true;
|
||||
case Decoder::Opcode::V_INTERP_MOV_F32: V_INTERP_MOV_F32(inst); return true;
|
||||
default: return false;
|
||||
case Decoder::Opcode::V_INTERP_P1_F32: V_INTERP_P1_F32(); return;
|
||||
case Decoder::Opcode::V_INTERP_P2_F32: V_INTERP_P2_F32(inst); return;
|
||||
case Decoder::Opcode::V_INTERP_MOV_F32: V_INTERP_MOV_F32(inst); return;
|
||||
default: return FailMissingTranslation(inst);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,15 +44,16 @@ void Translator::EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpc
|
||||
half ? IR::Value(ReadF16AsF32(inst.src0)) : ReadOperand(inst.src0, IR::Type::F32);
|
||||
const auto rhs =
|
||||
half ? IR::Value(ReadF16AsF32(inst.src1)) : ReadOperand(inst.src1, IR::Type::F32);
|
||||
EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs})), false, cmpx);
|
||||
const IR::FPCompareFlags flags{.flush_input_denorms = !half && flush_f32_inputs};
|
||||
EmitCompareResult(inst, IR::U1(ir.Emit(opcode, {lhs, rhs}, flags)), false, cmpx);
|
||||
}
|
||||
|
||||
void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered) {
|
||||
void Translator::EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx) {
|
||||
const auto lhs = IR::F32(ReadOperand(inst.src0, IR::Type::F32));
|
||||
const auto rhs = IR::F32(ReadOperand(inst.src1, IR::Type::F32));
|
||||
const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})),
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs})));
|
||||
EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, false);
|
||||
EmitCompareResult(inst, ordered ? ir.LogicalNot(unordered) : unordered, false, cmpx);
|
||||
}
|
||||
|
||||
void Translator::EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx) {
|
||||
|
||||
@@ -302,23 +302,95 @@ void Translator::S_MOV_B64(const Decoder::Instruction& inst) {
|
||||
}
|
||||
|
||||
void Translator::S_WQM(const Decoder::Instruction& inst, bool wide) {
|
||||
const auto source = wide ? ReadU64(inst.src0)
|
||||
: ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
|
||||
const bool source_is_mask = inst.src0.kind == Decoder::OperandKind::Sgpr ||
|
||||
inst.src0.kind == Decoder::OperandKind::ExecLo ||
|
||||
inst.src0.kind == Decoder::OperandKind::VccLo ||
|
||||
(!wide && IsExecOrVcc(inst.src0));
|
||||
const auto source_valid = ReadMaskValid(inst.src0);
|
||||
auto source_live = IR::U1(IR::Value(false));
|
||||
if (source_is_mask) {
|
||||
if (inst.src0.kind == Decoder::OperandKind::VccLo ||
|
||||
inst.src0.kind == Decoder::OperandKind::VccHi) {
|
||||
source_live = ir.GetVcc();
|
||||
} else {
|
||||
source_live = ReadMask(inst.src0);
|
||||
}
|
||||
}
|
||||
const auto retained_live = ir.LogicalAnd(source_valid, source_live);
|
||||
|
||||
IR::U64 source;
|
||||
if (wide) {
|
||||
source = ReadU64(inst.src0);
|
||||
} else {
|
||||
source = ir.ConstructU64(ReadU32(inst.src0), IR::U32(IR::Value(0u)));
|
||||
}
|
||||
const auto result = IR::U64(ir.Emit(IR::ValueOpcode::WqmU64, {source}));
|
||||
if (!wide) {
|
||||
const auto low = ir.CompositeExtract(result, 0);
|
||||
const bool destination_is_mask = IsExecOrVcc(inst.dst);
|
||||
const bool destination_is_exec = inst.dst.kind == Decoder::OperandKind::ExecLo ||
|
||||
inst.dst.kind == Decoder::OperandKind::ExecHi;
|
||||
auto old_live = IR::U1(IR::Value(false));
|
||||
if (destination_is_mask) {
|
||||
if (destination_is_exec) {
|
||||
old_live = ir.GetExec();
|
||||
} else {
|
||||
old_live = ir.GetVcc();
|
||||
}
|
||||
}
|
||||
|
||||
// A 32-bit write preserves the other EXEC/VCC half and invalidates any
|
||||
// overlapping SGPR-pair mask provenance through the ordinary scalar path.
|
||||
WriteRawU32(inst.dst, low);
|
||||
if (destination_is_mask) {
|
||||
const auto is_high_half = [](const Decoder::Operand& operand) {
|
||||
return operand.kind == Decoder::OperandKind::ExecHi ||
|
||||
operand.kind == Decoder::OperandKind::VccHi;
|
||||
};
|
||||
const bool destination_is_high = is_high_half(inst.dst);
|
||||
auto affected_lanes = IR::U1(IR::Value(!destination_is_high));
|
||||
if (program.wave_size == 64u) {
|
||||
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
|
||||
affected_lanes = ir.ULessThan(lane, IR::U32(IR::Value(32u)));
|
||||
if (destination_is_high) {
|
||||
affected_lanes = ir.LogicalNot(affected_lanes);
|
||||
}
|
||||
}
|
||||
|
||||
IR::U1 live;
|
||||
if (destination_is_exec) {
|
||||
live = ir.GetExec();
|
||||
} else {
|
||||
live = ir.GetVcc();
|
||||
}
|
||||
// A cross-half copy refers to different lanes, so only scalar bits apply.
|
||||
if (is_high_half(inst.src0) == destination_is_high) {
|
||||
live = ir.LogicalOr(retained_live, live);
|
||||
}
|
||||
live = IR::U1(ir.Emit(IR::ValueOpcode::SelectU1, {affected_lanes, live, old_live}));
|
||||
if (destination_is_exec) {
|
||||
ir.SetExec(live);
|
||||
} else {
|
||||
ir.SetVcc(live);
|
||||
}
|
||||
}
|
||||
ir.SetScc(ir.INotEqual(low, IR::U32(IR::Value(0u))));
|
||||
return;
|
||||
}
|
||||
const auto mask_valid = ReadMaskValid(inst.src0);
|
||||
|
||||
// WQM adds live lanes but never removes a source lane. Keep that fact separate
|
||||
// from the scalar mask: reconstructing an entry-true predicate from a ballot
|
||||
// otherwise introduces a spurious dependency into every vector write.
|
||||
const auto live = ir.LogicalOr(retained_live, ThreadBit(ExtractU64(result)));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
if (inst.dst.kind == Decoder::OperandKind::ExecLo) {
|
||||
ir.SetExec(live);
|
||||
} else if (inst.dst.kind == Decoder::OperandKind::VccLo) {
|
||||
ir.SetVcc(live);
|
||||
} else if (inst.dst.kind == Decoder::OperandKind::Sgpr) {
|
||||
const auto dst = static_cast<IR::ScalarReg>(inst.dst.reg);
|
||||
ir.SetThreadBitScalarReg(dst, ThreadBit(ExtractU64(result)));
|
||||
ir.SetScalarMaskTag(dst, mask_valid);
|
||||
ir.SetThreadBitScalarReg(dst, live);
|
||||
ir.SetScalarMaskTag(dst, source_valid);
|
||||
}
|
||||
ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
|
||||
}
|
||||
|
||||
@@ -3,6 +3,11 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
void Translator::FailMissingTranslation(const Decoder::Instruction& inst) {
|
||||
EXIT("opcode %s at pc 0x%08x has no IR translation",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc);
|
||||
}
|
||||
|
||||
void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
|
||||
current_opcode = inst.opcode;
|
||||
current_pc = inst.pc;
|
||||
@@ -16,35 +21,26 @@ void Translator::TranslateInstruction(const Decoder::Instruction& inst) {
|
||||
default: break;
|
||||
}
|
||||
|
||||
bool translated = false;
|
||||
switch (inst.family) {
|
||||
case Decoder::Family::SOP1:
|
||||
case Decoder::Family::SOP2:
|
||||
case Decoder::Family::SOPK:
|
||||
case Decoder::Family::SOPC:
|
||||
case Decoder::Family::SOPP: translated = EmitScalar(inst); break;
|
||||
case Decoder::Family::SOPP: return EmitScalar(inst);
|
||||
case Decoder::Family::VOP1:
|
||||
case Decoder::Family::VOP2:
|
||||
case Decoder::Family::VOP3:
|
||||
case Decoder::Family::VOP3P:
|
||||
case Decoder::Family::VOPC: translated = EmitVector(inst); break;
|
||||
case Decoder::Family::VOPC: return EmitVector(inst);
|
||||
case Decoder::Family::SMEM:
|
||||
case Decoder::Family::MUBUF:
|
||||
case Decoder::Family::MTBUF:
|
||||
case Decoder::Family::FLAT:
|
||||
case Decoder::Family::DS:
|
||||
case Decoder::Family::MIMG: translated = EmitMemory(inst); break;
|
||||
case Decoder::Family::VINTRP: translated = EmitInterpolation(inst); break;
|
||||
case Decoder::Family::EXP:
|
||||
EXP(inst);
|
||||
translated = true;
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
|
||||
if (!translated) {
|
||||
EXIT("opcode %s at pc 0x%08x has no IR translation",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc);
|
||||
case Decoder::Family::MIMG: return EmitMemory(inst);
|
||||
case Decoder::Family::VINTRP: return EmitInterpolation(inst);
|
||||
case Decoder::Family::EXP: return EXP(inst);
|
||||
default: return FailMissingTranslation(inst);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
bool Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool accumulator, bool quiet_snan) {
|
||||
const auto translate_lane = [&](bool high) {
|
||||
const auto lhs = ReadF16LaneAsF32(inst.src0, high, true);
|
||||
@@ -43,16 +43,15 @@ bool Translator::PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode
|
||||
result = PackU16Lanes(override_lane(false), override_lane(true));
|
||||
}
|
||||
WriteOperand(raw, result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool invalid_negative) {
|
||||
const auto argument = ReadF16AsF32(inst.src0);
|
||||
auto result = IR::F32(ir.Emit(opcode, {argument}));
|
||||
if (!invalid_negative) {
|
||||
WriteF16(DestinationOperand(inst), result);
|
||||
return true;
|
||||
return;
|
||||
}
|
||||
const auto negative =
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {argument, IR::Value::F32(0.0f)}));
|
||||
@@ -60,10 +59,9 @@ bool Translator::Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode
|
||||
const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
|
||||
const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
|
||||
Write16Bits(DestinationOperand(inst), ir.Select(negative, invalid, bits));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto argument = ReadF16AsF32(inst.src0);
|
||||
const auto magnitude =
|
||||
ir.BitwiseAnd(ir.BitCastU32(argument), IR::U32(IR::Value(0x7fffffffu)));
|
||||
@@ -96,18 +94,16 @@ bool Translator::Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode o
|
||||
const auto bits = PackHalf2x16(result, IR::F32(IR::Value::F32(0.0f)));
|
||||
const auto invalid = IR::U32(IR::Value(inst.dst.clamp ? 0u : 0xfe00u));
|
||||
Write16Bits(DestinationOperand(inst), ir.Select(infinite, invalid, bits));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool reverse) {
|
||||
const auto lhs = ReadF16AsF32(reverse ? inst.src1 : inst.src0);
|
||||
const auto rhs = ReadF16AsF32(reverse ? inst.src0 : inst.src1);
|
||||
WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {lhs, rhs})));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool accumulator, bool mix) {
|
||||
std::array<IR::Value, 3> args;
|
||||
for (uint32_t index = 0; index < args.size(); index++) {
|
||||
@@ -115,16 +111,14 @@ bool Translator::Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcod
|
||||
args[index] = mix ? IR::Value(ReadMixF32(operand)) : IR::Value(ReadF16AsF32(operand));
|
||||
}
|
||||
WriteF16(DestinationOperand(inst), IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto type = IR::ArgTypeOf(opcode, 0);
|
||||
WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {ReadOperand(inst.src0, type)}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool reverse) {
|
||||
std::array<IR::Value, 2> args;
|
||||
for (uint32_t index = 0; index < args.size(); index++) {
|
||||
@@ -132,10 +126,9 @@ bool Translator::FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode o
|
||||
args[index] = ReadOperand(operand, IR::ArgTypeOf(opcode, index));
|
||||
}
|
||||
WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1]}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool accumulator, bool mix) {
|
||||
std::array<IR::Value, 3> args;
|
||||
for (uint32_t index = 0; index < args.size(); index++) {
|
||||
@@ -145,10 +138,9 @@ bool Translator::FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode
|
||||
: ReadOperand(operand, type);
|
||||
}
|
||||
WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {args[0], args[1], args[2]}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
|
||||
const auto bits = ReadU32(inst.src0);
|
||||
const auto exponent =
|
||||
IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
|
||||
@@ -166,10 +158,9 @@ bool Translator::V_FREXP_MANT_F32(const Decoder::Instruction& inst) {
|
||||
ir.Select(zero, bits, subnormal));
|
||||
const auto result = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite);
|
||||
WriteOperand(DestinationOperand(inst), ir.BitCastF32(result));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
|
||||
void Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
|
||||
auto a = inst.src0;
|
||||
a.op_sel = false;
|
||||
a.op_sel_hi = true;
|
||||
@@ -183,26 +174,25 @@ bool Translator::V_DOT2C_F32_F16(const Decoder::Instruction& inst) {
|
||||
const auto acc = ReadMixF32(inst.dst);
|
||||
const auto lo = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc}));
|
||||
WriteOperand(DestinationOperand(inst), ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_CUBEID_F32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_CUBEID_F32(const Decoder::Instruction& inst) {
|
||||
return FloatCube(inst, 0u);
|
||||
}
|
||||
|
||||
bool Translator::V_CUBESC_F32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_CUBESC_F32(const Decoder::Instruction& inst) {
|
||||
return FloatCube(inst, 1u);
|
||||
}
|
||||
|
||||
bool Translator::V_CUBETC_F32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_CUBETC_F32(const Decoder::Instruction& inst) {
|
||||
return FloatCube(inst, 2u);
|
||||
}
|
||||
|
||||
bool Translator::V_CUBEMA_F32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_CUBEMA_F32(const Decoder::Instruction& inst) {
|
||||
return FloatCube(inst, 3u);
|
||||
}
|
||||
|
||||
bool Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kind) {
|
||||
void Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kind) {
|
||||
const auto x = ReadMixF32(inst.src0);
|
||||
const auto y = ReadMixF32(inst.src1);
|
||||
const auto z = ReadMixF32(inst.src2);
|
||||
@@ -242,7 +232,6 @@ bool Translator::FloatCube(const Decoder::Instruction& inst, uint32_t result_kin
|
||||
default: EXIT("invalid cube result kind");
|
||||
}
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Frontend
|
||||
@@ -4,25 +4,23 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
bool Translator::Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool arithmetic) {
|
||||
const auto value = ReadU16AsU32(inst.src1, arithmetic);
|
||||
const auto count = ir.BitwiseAnd(ReadU16AsU32(inst.src0, false), IR::U32(IR::Value(15u)));
|
||||
const auto result = IR::U32(ir.Emit(opcode, {value, count}));
|
||||
Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool sign) {
|
||||
const auto lhs = ReadU16AsU32(inst.src0, sign);
|
||||
const auto rhs = ReadU16AsU32(inst.src1, sign);
|
||||
const auto result = IR::U32(ir.Emit(opcode, {lhs, rhs}));
|
||||
Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_MAD_I16(const Decoder::Instruction& inst) {
|
||||
void Translator::V_MAD_I16(const Decoder::Instruction& inst) {
|
||||
const auto lhs = ReadU16AsU32(inst.src0, true);
|
||||
const auto rhs = ReadU16AsU32(inst.src1, true);
|
||||
auto result = ir.IAdd(ir.IMul(lhs, rhs), ReadU16AsU32(inst.src2, true));
|
||||
@@ -31,18 +29,16 @@ bool Translator::V_MAD_I16(const Decoder::Instruction& inst) {
|
||||
result = IR::U32(ir.Emit(IR::ValueOpcode::SMin32, {result, IR::Value(0x7fffu)}));
|
||||
}
|
||||
Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_MED3_I16(const Decoder::Instruction& inst) {
|
||||
void Translator::V_MED3_I16(const Decoder::Instruction& inst) {
|
||||
const auto result = IR::U32(ir.Emit(
|
||||
IR::ValueOpcode::SMedTri32, {ReadU16AsU32(inst.src0, true), ReadU16AsU32(inst.src1, true),
|
||||
ReadU16AsU32(inst.src2, true)}));
|
||||
Write16Bits(DestinationOperand(inst), ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool arithmetic) {
|
||||
const auto translate_lane = [&](bool high) {
|
||||
const auto count =
|
||||
@@ -52,10 +48,9 @@ bool Translator::PackedInteger16Shift(const Decoder::Instruction& inst, IR::Valu
|
||||
};
|
||||
WriteOperand(DestinationOperand(inst),
|
||||
PackU16Lanes(translate_lane(false), translate_lane(true)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto translate_lane = [&](bool high) {
|
||||
const auto lhs = ReadU16LaneAsU32(inst.src0, high, false);
|
||||
const auto rhs = ReadU16LaneAsU32(inst.src1, high, false);
|
||||
@@ -63,10 +58,9 @@ bool Translator::PackedInteger16Binary(const Decoder::Instruction& inst, IR::Val
|
||||
};
|
||||
WriteOperand(DestinationOperand(inst),
|
||||
PackU16Lanes(translate_lane(false), translate_lane(true)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign) {
|
||||
void Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto translate_lane = [&](bool high) {
|
||||
const auto lhs = ReadU16LaneAsU32(inst.src0, high, false);
|
||||
const auto rhs = ReadU16LaneAsU32(inst.src1, high, false);
|
||||
@@ -74,10 +68,9 @@ bool Translator::PackedInteger16Mad(const Decoder::Instruction& inst, bool sign)
|
||||
};
|
||||
WriteOperand(DestinationOperand(inst),
|
||||
PackU16Lanes(translate_lane(false), translate_lane(true)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool sign) {
|
||||
const auto translate_lane = [&](bool high) {
|
||||
const auto lhs = ReadU16LaneAsU32(inst.src0, high, sign);
|
||||
@@ -86,7 +79,6 @@ bool Translator::PackedInteger16MinMax(const Decoder::Instruction& inst, IR::Val
|
||||
};
|
||||
WriteOperand(DestinationOperand(inst),
|
||||
PackU16Lanes(translate_lane(false), translate_lane(true)));
|
||||
return true;
|
||||
}
|
||||
|
||||
IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
@@ -100,7 +92,7 @@ IR::U1 Translator::U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpco
|
||||
return negate_result ? ir.LogicalNot(result) : result;
|
||||
}
|
||||
|
||||
bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
|
||||
void Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
|
||||
IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result,
|
||||
bool unary) {
|
||||
const auto invocation_result =
|
||||
@@ -119,7 +111,7 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
|
||||
(inst.src_count > 1u && is_exec_or_vcc(inst.src1))) {
|
||||
const auto mask = WriteMask(inst.dst, invocation_result, true);
|
||||
ir.SetScc(ir.INotEqual(ir.BitwiseOr(mask[0], mask[1]), IR::U32(IR::Value(0u))));
|
||||
return true;
|
||||
return;
|
||||
}
|
||||
|
||||
const auto lhs = ReadU32Pair(inst.src0);
|
||||
@@ -146,10 +138,9 @@ bool Translator::S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode lo
|
||||
ir.SetScalarMaskTag(dst, mask_valid);
|
||||
}
|
||||
ir.SetScc(ir.INotEqual(ir.BitwiseOr(result[0], result[1]), IR::U32(IR::Value(0u))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
IR::Type type, bool reverse, bool mask_shift_count,
|
||||
bool update_scc) {
|
||||
std::array<IR::Value, 3> args;
|
||||
@@ -177,10 +168,9 @@ bool Translator::SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode
|
||||
ir.SetScc(ir.INotEqual(IR::U32(result), IR::U32(IR::Value(0u))));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_ASHR_I64(const Decoder::Instruction& inst) {
|
||||
void Translator::S_ASHR_I64(const Decoder::Instruction& inst) {
|
||||
// Signed 64-bit sources sign-extend literals; generic B64 operands zero-extend them.
|
||||
const auto source =
|
||||
inst.src0.kind == Decoder::OperandKind::LiteralConstant
|
||||
@@ -192,10 +182,9 @@ bool Translator::S_ASHR_I64(const Decoder::Instruction& inst) {
|
||||
ir.Emit(IR::ValueOpcode::ShiftRightArithmetic64, {source, ReadU32(inst.src1)});
|
||||
WriteOperand(inst.dst, result);
|
||||
ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool negate_rhs, bool negate_result, bool update_scc) {
|
||||
const auto lhs = ReadU32(inst.src0);
|
||||
auto rhs = ReadU32(inst.src1);
|
||||
@@ -210,31 +199,27 @@ bool Translator::ComposedIntegerBinary(const Decoder::Instruction& inst, IR::Val
|
||||
if (update_scc) {
|
||||
ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_AND_OR_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_AND_OR_B32(const Decoder::Instruction& inst) {
|
||||
const auto result =
|
||||
ir.BitwiseOr(ir.BitwiseAnd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_OR3_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_OR3_B32(const Decoder::Instruction& inst) {
|
||||
const auto result =
|
||||
ir.BitwiseOr(ir.BitwiseOr(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_XOR3_B32(const Decoder::Instruction& inst) {
|
||||
const auto result =
|
||||
ir.BitwiseXor(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
|
||||
void Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
|
||||
const auto source = ExtractU64(ReadU64(inst.src0));
|
||||
const auto low_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[0]}));
|
||||
const auto high_lsb = IR::U32(ir.Emit(IR::ValueOpcode::FindILsb32, {source[1]}));
|
||||
@@ -243,10 +228,9 @@ bool Translator::S_FF1_I32_B64(const Decoder::Instruction& inst) {
|
||||
ir.Select(ir.INotEqual(source[1], IR::U32(IR::Value(0u))),
|
||||
high_position, IR::U32(IR::Value(0xffffffffu))));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
|
||||
void Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto source = ReadU32(inst.src0);
|
||||
const auto value = sign ? ir.BitwiseXor(
|
||||
source, ir.ShiftRightArithmetic(source, IR::U32(IR::Value(31u))))
|
||||
@@ -256,10 +240,9 @@ bool Translator::V_FFBH_32(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto result = ir.Select(ir.INotEqual(value, IR::U32(IR::Value(0u))), position,
|
||||
IR::U32(IR::Value(0xffffffffu)));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
|
||||
void Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
|
||||
const auto source = ReadU64(inst.src0);
|
||||
const auto msb = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb64, {source}));
|
||||
const auto position = ir.ISub(IR::U32(IR::Value(63u)), msb);
|
||||
@@ -268,10 +251,9 @@ bool Translator::S_FLBIT_I32_B64(const Decoder::Instruction& inst) {
|
||||
{source, ir.ConstructU64(IR::U32(IR::Value(0u)), IR::U32(IR::Value(0u)))}));
|
||||
const auto result = ir.Select(nonzero, position, IR::U32(IR::Value(0xffffffffu)));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
|
||||
void Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool addend) {
|
||||
const auto extract24 = [&](IR::U32 value) {
|
||||
return IR::U32(
|
||||
ir.Emit(sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract,
|
||||
@@ -284,10 +266,9 @@ bool Translator::Integer24(const Decoder::Instruction& inst, bool sign, bool add
|
||||
result = ir.IAdd(result, ReadU32(inst.src2));
|
||||
}
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
|
||||
const auto lhs = ReadU32(inst.src0);
|
||||
const auto rhs = ReadU32(inst.src1);
|
||||
const auto add = ExtractU64(ReadU64(inst.src2));
|
||||
@@ -305,36 +286,32 @@ bool Translator::V_MAD_U64_U32(const Decoder::Instruction& inst) {
|
||||
inst.dst2.kind != Decoder::OperandKind::Unknown) {
|
||||
WriteMask(inst.dst2, ir.LogicalOr(carry0, carry1));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_SAD_U32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_SAD_U32(const Decoder::Instruction& inst) {
|
||||
const auto lhs = ReadU32(inst.src0);
|
||||
const auto rhs = ReadU32(inst.src1);
|
||||
const auto lo = IR::U32(ir.Emit(IR::ValueOpcode::UMin32, {lhs, rhs}));
|
||||
const auto hi = IR::U32(ir.Emit(IR::ValueOpcode::UMax32, {lhs, rhs}));
|
||||
const auto result = ir.IAdd(ir.ISub(hi, lo), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_ADD3_U32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_ADD3_U32(const Decoder::Instruction& inst) {
|
||||
const auto result =
|
||||
ir.IAdd(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BITSET_B32(const Decoder::Instruction& inst, bool set) {
|
||||
void Translator::S_BITSET_B32(const Decoder::Instruction& inst, bool set) {
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u)));
|
||||
const auto bit = ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), offset);
|
||||
const auto old = ReadU32(inst.dst);
|
||||
const auto result = set ? ir.BitwiseOr(old, bit) : ir.BitwiseAnd(old, ir.BitwiseNot(bit));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BITSET_B64(const Decoder::Instruction& inst, bool set) {
|
||||
void Translator::S_BITSET_B64(const Decoder::Instruction& inst, bool set) {
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
|
||||
const auto word_bit = ir.BitwiseAnd(offset, IR::U32(IR::Value(31u)));
|
||||
const auto bit = ir.ShiftLeftLogical(IR::U32(IR::Value(1u)), word_bit);
|
||||
@@ -347,16 +324,14 @@ bool Translator::S_BITSET_B64(const Decoder::Instruction& inst, bool set) {
|
||||
{offset, IR::Value(32u)}));
|
||||
WriteU32Pair(inst.dst,
|
||||
{ir.Select(high, old[0], low_value), ir.Select(high, high_value, old[1])});
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_BCNT_U32_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_BCNT_U32_B32(const Decoder::Instruction& inst) {
|
||||
const auto count = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {ReadU32(inst.src0)}));
|
||||
WriteOperand(DestinationOperand(inst), ir.IAdd(count, ReadU32(inst.src1)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
|
||||
void Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
|
||||
const auto lane = IR::U32(ir.Emit(IR::ValueOpcode::LaneId));
|
||||
const auto local = ir.BitwiseAnd(lane, IR::U32(IR::Value(31u)));
|
||||
const auto below =
|
||||
@@ -368,10 +343,9 @@ bool Translator::V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low) {
|
||||
const auto active = ir.BitwiseAnd(ReadU32(inst.src0), thread_mask);
|
||||
const auto count = IR::U32(ir.Emit(IR::ValueOpcode::BitCount32, {active}));
|
||||
WriteOperand(DestinationOperand(inst), ir.IAdd(count, ReadU32(inst.src1)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
|
||||
const auto replicate = [&](IR::U32 value) {
|
||||
auto bits = ir.BitwiseOr(value, ir.ShiftLeftLogical(value, IR::U32(IR::Value(8u))));
|
||||
bits = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x00ff00ffu)));
|
||||
@@ -387,10 +361,9 @@ bool Translator::S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst) {
|
||||
const auto low = ir.BitwiseAnd(source, IR::U32(IR::Value(0xffffu)));
|
||||
const auto high = ir.ShiftRightLogical(source, IR::U32(IR::Value(16u)));
|
||||
WriteOperand(DestinationOperand(inst), ir.ConstructU64(replicate(low), replicate(high)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
|
||||
void Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
|
||||
const auto compact = [&](IR::U32 value) {
|
||||
auto bits = ir.BitwiseOr(value, ir.ShiftRightLogical(value, IR::U32(IR::Value(1u))));
|
||||
bits = ir.BitwiseOr(bits, ir.ShiftRightLogical(bits, IR::U32(IR::Value(2u))));
|
||||
@@ -409,16 +382,14 @@ bool Translator::S_QUADMASK_B64(const Decoder::Instruction& inst) {
|
||||
const auto result = ir.ConstructU64(quads, IR::U32(IR::Value(0u)));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::BFM_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::BFM_B32(const Decoder::Instruction& inst) {
|
||||
const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(31u)));
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
|
||||
const auto result = ir.Emit(IR::ValueOpcode::BitFieldInsert,
|
||||
{IR::Value(0u), IR::Value(0xffffffffu), offset, count});
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
IR::U32 Translator::RightMask32(IR::U32 count) {
|
||||
@@ -435,15 +406,14 @@ IR::U64 Translator::RightMask64(IR::U32 count) {
|
||||
return ir.ConstructU64(RightMask32(low_count), RightMask32(high_count));
|
||||
}
|
||||
|
||||
bool Translator::S_BFM_B64(const Decoder::Instruction& inst) {
|
||||
void Translator::S_BFM_B64(const Decoder::Instruction& inst) {
|
||||
const auto count = ir.BitwiseAnd(ReadU32(inst.src0), IR::U32(IR::Value(63u)));
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
|
||||
const auto result = ir.Emit(IR::ValueOpcode::ShiftLeftLogical64, {RightMask64(count), offset});
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) {
|
||||
void Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto source = ReadU32(inst.src0);
|
||||
const auto field = ReadU32(inst.src1);
|
||||
const auto offset =
|
||||
@@ -457,10 +427,9 @@ bool Translator::S_BFE_U32(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto result = IR::U32(ir.Emit(opcode, {source, offset, count}));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
ir.SetScc(ir.INotEqual(result, IR::U32(IR::Value(0u))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BFE_U64(const Decoder::Instruction& inst) {
|
||||
void Translator::S_BFE_U64(const Decoder::Instruction& inst) {
|
||||
const auto source = ReadU64(inst.src0);
|
||||
const auto field = ReadU32(inst.src1);
|
||||
const auto offset =
|
||||
@@ -473,10 +442,9 @@ bool Translator::S_BFE_U64(const Decoder::Instruction& inst) {
|
||||
const auto result = ir.Emit(IR::ValueOpcode::BitwiseAnd64, {shifted, RightMask64(count)});
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
ir.SetScc(IR::U1(ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) {
|
||||
void Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto source = ReadU32(inst.src0);
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
|
||||
const auto raw_count = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
|
||||
@@ -485,29 +453,26 @@ bool Translator::V_BFE_U32(const Decoder::Instruction& inst, bool sign) {
|
||||
const auto opcode =
|
||||
sign ? IR::ValueOpcode::BitFieldSExtract : IR::ValueOpcode::BitFieldUExtract;
|
||||
WriteOperand(DestinationOperand(inst), ir.Emit(opcode, {source, offset, count}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_BFI_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_BFI_B32(const Decoder::Instruction& inst) {
|
||||
const auto bits = ReadU32(inst.src0);
|
||||
const auto insert = ReadU32(inst.src1);
|
||||
const auto base = ReadU32(inst.src2);
|
||||
const auto result =
|
||||
ir.BitwiseOr(ir.BitwiseAnd(bits, insert), ir.BitwiseAnd(ir.BitwiseNot(bits), base));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) {
|
||||
void Translator::S_BITCMP_B32(const Decoder::Instruction& inst, bool expected) {
|
||||
const auto value = ReadU32(inst.src0);
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
|
||||
const auto bit =
|
||||
IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {value, offset, IR::Value(1u)}));
|
||||
WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
|
||||
void Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
|
||||
const auto value = ReadU32Pair(inst.src0);
|
||||
const auto offset = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(63u)));
|
||||
const auto word_bit = ir.BitwiseAnd(offset, IR::U32(IR::Value(31u)));
|
||||
@@ -516,10 +481,9 @@ bool Translator::S_BITCMP_B64(const Decoder::Instruction& inst, bool expected) {
|
||||
const auto bit =
|
||||
IR::U32(ir.Emit(IR::ValueOpcode::BitFieldUExtract, {word, word_bit, IR::Value(1u)}));
|
||||
WriteCompareResult(inst.dst, ir.IEqual(bit, IR::U32(IR::Value(expected ? 1u : 0u))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
|
||||
const auto hi = ReadU32(inst.src0);
|
||||
const auto lo = ReadU32(inst.src1);
|
||||
const auto shift = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
|
||||
@@ -530,10 +494,9 @@ bool Translator::V_ALIGNBIT_B32(const Decoder::Instruction& inst) {
|
||||
const auto hi_part =
|
||||
ir.Select(ir.INotEqual(shift, IR::U32(IR::Value(0u))), hi_part_raw, IR::U32(IR::Value(0u)));
|
||||
WriteOperand(DestinationOperand(inst), ir.BitwiseOr(lo_part, hi_part));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
|
||||
const auto hi = ReadU32(inst.src0);
|
||||
const auto lo = ReadU32(inst.src1);
|
||||
const auto byte_offset = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
|
||||
@@ -546,39 +509,34 @@ bool Translator::V_ALIGNBYTE_B32(const Decoder::Instruction& inst) {
|
||||
IR::U1(ir.Emit(IR::ValueOpcode::ULessThan32, {byte_offset, IR::Value(8u)}));
|
||||
WriteOperand(DestinationOperand(inst),
|
||||
ir.Select(in_range, ExtractU64(shifted)[0], IR::U32(IR::Value(0u))));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_LSHL_ADD_U32(const Decoder::Instruction& inst) {
|
||||
const auto shift = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
|
||||
const auto result = ir.IAdd(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_ADD_LSHL_U32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_ADD_LSHL_U32(const Decoder::Instruction& inst) {
|
||||
const auto shift = ir.BitwiseAnd(ReadU32(inst.src2), IR::U32(IR::Value(31u)));
|
||||
const auto result = ir.ShiftLeftLogical(ir.IAdd(ReadU32(inst.src0), ReadU32(inst.src1)), shift);
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_XAD_U32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_XAD_U32(const Decoder::Instruction& inst) {
|
||||
const auto result =
|
||||
ir.IAdd(ir.BitwiseXor(ReadU32(inst.src0), ReadU32(inst.src1)), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_LSHL_OR_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_LSHL_OR_B32(const Decoder::Instruction& inst) {
|
||||
const auto shift = ir.BitwiseAnd(ReadU32(inst.src1), IR::U32(IR::Value(31u)));
|
||||
const auto result =
|
||||
ir.BitwiseOr(ir.ShiftLeftLogical(ReadU32(inst.src0), shift), ReadU32(inst.src2));
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) {
|
||||
Decoder::Operand mask_operand;
|
||||
mask_operand.kind = Decoder::OperandKind::VccLo;
|
||||
if (inst.src_count >= 3u) {
|
||||
@@ -594,17 +552,15 @@ bool Translator::V_CNDMASK_B32(const Decoder::Instruction& inst) {
|
||||
result = ir.Select(condition, ReadU32(inst.src1), ReadU32(inst.src0));
|
||||
}
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::PackB16(const Decoder::Instruction& inst, bool high0, bool high1) {
|
||||
void Translator::PackB16(const Decoder::Instruction& inst, bool high0, bool high1) {
|
||||
const auto lo = high0 ? ir.ShiftRightLogical(ReadU32(inst.src0), IR::U32(IR::Value(16u)))
|
||||
: ReadU32(inst.src0);
|
||||
const auto hi = high1 ? ir.ShiftRightLogical(ReadU32(inst.src1), IR::U32(IR::Value(16u)))
|
||||
: ReadU32(inst.src1);
|
||||
const auto result = PackU16Lanes(lo, hi);
|
||||
WriteOperand(DestinationOperand(inst), result);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Frontend
|
||||
@@ -1,8 +1,11 @@
|
||||
#include "common/assert.h"
|
||||
#include "common/logging/log.h"
|
||||
#include "graphics/shader/recompiler/frontend/translate/Translator.h"
|
||||
#include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
@@ -123,6 +126,11 @@ IR::MemoryInfo MemoryInfoFromDecoded(const Decoder::Instruction& decoded) {
|
||||
memory.image_r128 = decoded.image_r128;
|
||||
memory.idxen = decoded.idxen;
|
||||
memory.offen = decoded.offen;
|
||||
// Vector loads use GLC/DLC to bypass L0/GL1; atomics use GLC only to return data.
|
||||
const bool buffer_atomic = decoded.opcode >= Decoder::Opcode::BUFFER_ATOMIC_SWAP &&
|
||||
decoded.opcode <= Decoder::Opcode::BUFFER_ATOMIC_FMAX;
|
||||
memory.coherent = memory.kind == ResourceKind::Buffer && !buffer_atomic &&
|
||||
(decoded.glc || decoded.dlc);
|
||||
memory.resource = ResourceIndexFromOperand(decoded.src1);
|
||||
memory.sampler = ResourceIndexFromOperand(decoded.src2);
|
||||
if (memory.kind == ResourceKind::ScalarBuffer) {
|
||||
@@ -215,7 +223,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
|
||||
const bool store_or_atomic = decoded.opcode == Decoder::Opcode::IMAGE_STORE ||
|
||||
decoded.opcode == Decoder::Opcode::IMAGE_STORE_MIP ||
|
||||
(decoded.opcode >= Decoder::Opcode::IMAGE_ATOMIC_SWAP &&
|
||||
decoded.opcode <= Decoder::Opcode::IMAGE_ATOMIC_XOR);
|
||||
decoded.opcode <= Decoder::Opcode::IMAGE_ATOMIC_FMAX);
|
||||
if (store_or_atomic) {
|
||||
return index == 0u ? decoded.dst : decoded.src0;
|
||||
}
|
||||
@@ -227,6 +235,7 @@ Decoder::Operand MemorySourceAt(const Decoder::Instruction& decoded, uint32_t in
|
||||
return index == 0u ? decoded.src0 : MakeImmediate(decoded.offset & 0xffffu);
|
||||
case Decoder::Opcode::DS_CONSUME:
|
||||
case Decoder::Opcode::DS_APPEND:
|
||||
return decoded.gds ? MakeM0Operand() : MakeImmediate(0);
|
||||
case Decoder::Opcode::DS_READ_ADDTID_B32: return MakeM0Operand();
|
||||
case Decoder::Opcode::DS_WRITE_ADDTID_B32:
|
||||
return index == 0u ? decoded.src1 : MakeM0Operand();
|
||||
@@ -397,7 +406,7 @@ IR::Value Translator::NarrowSubdword(IR::U32 value, uint32_t bits) {
|
||||
: ir.Emit(IR::ValueOpcode::ConvertU16U32, {value});
|
||||
}
|
||||
|
||||
bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
|
||||
void Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource =
|
||||
raw ? GetScalarAddressResource(RawScalarLoadBase(inst.src0)) : GetBufferResource(memory);
|
||||
@@ -420,10 +429,9 @@ bool Translator::S_LOAD(const Decoder::Instruction& inst, bool raw) {
|
||||
for (uint32_t component = 0; component < memory.data_dwords; component++) {
|
||||
WriteOperand(ScalarDestinationOperand(inst.dst, component), loaded[component]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
|
||||
void Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
IR::ValueOpcode opcode;
|
||||
const auto bits = memory.data_bits;
|
||||
@@ -437,10 +445,15 @@ bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
|
||||
case 2u: opcode = IR::ValueOpcode::LoadBufferU32x2; break;
|
||||
case 3u: opcode = IR::ValueOpcode::LoadBufferU32x3; break;
|
||||
case 4u: opcode = IR::ValueOpcode::LoadBufferU32x4; break;
|
||||
default: return false;
|
||||
default:
|
||||
EXIT("opcode %s at pc 0x%08x has unsupported buffer load dword count %u",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc,
|
||||
memory.data_dwords);
|
||||
}
|
||||
break;
|
||||
default: return false;
|
||||
default:
|
||||
EXIT("opcode %s at pc 0x%08x has unsupported buffer load width %u",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc, bits);
|
||||
}
|
||||
const auto resource = GetBufferResource(memory);
|
||||
const auto address = ReadBufferAddress(inst, 0);
|
||||
@@ -457,10 +470,9 @@ bool Translator::BUFFER_LOAD(const Decoder::Instruction& inst) {
|
||||
ir.CompositeExtract(loaded, component));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
|
||||
void Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetBufferResource(memory);
|
||||
const auto address = ReadBufferAddress(inst, 1);
|
||||
@@ -501,17 +513,21 @@ bool Translator::BUFFER_STORE(const Decoder::Instruction& inst) {
|
||||
ReadU32(OffsetOperand(data_src, 2u)),
|
||||
ReadU32(OffsetOperand(data_src, 3u))});
|
||||
break;
|
||||
default: return false;
|
||||
default:
|
||||
EXIT("opcode %s at pc 0x%08x has unsupported buffer store dword count %u",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc,
|
||||
memory.data_dwords);
|
||||
}
|
||||
break;
|
||||
default: return false;
|
||||
default:
|
||||
EXIT("opcode %s at pc 0x%08x has unsupported buffer store width %u",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc, memory.data_bits);
|
||||
}
|
||||
ir.Emit(opcode, {resource, address.index, address.offset, address.soffset, value, ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetBufferResource(memory);
|
||||
const auto address = ReadBufferAddress(inst, 1);
|
||||
@@ -536,10 +552,9 @@ bool Translator::BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
|
||||
if (inst.glc) {
|
||||
WriteOperand(inst.dst, result);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1));
|
||||
@@ -549,10 +564,9 @@ bool Translator::IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode
|
||||
if (inst.glc) {
|
||||
WriteOperand(inst.dst, result);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool returns_value) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto address = ReadU32(MemorySourceAt(inst, 1));
|
||||
@@ -561,10 +575,9 @@ bool Translator::DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opc
|
||||
if (returns_value) {
|
||||
WriteOperand(inst.dst, result);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
|
||||
void Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
IR::ValueOpcode opcode;
|
||||
const auto bits = memory.data_bits;
|
||||
@@ -573,7 +586,9 @@ bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
|
||||
case 8u: opcode = IR::ValueOpcode::LoadAddressU8; break;
|
||||
case 16u: opcode = IR::ValueOpcode::LoadAddressU16; break;
|
||||
case 32u: opcode = IR::ValueOpcode::LoadAddressU32; break;
|
||||
default: return false;
|
||||
default:
|
||||
EXIT("opcode %s at pc 0x%08x has unsupported flat load width %u",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc, bits);
|
||||
}
|
||||
const auto address = ReadAddressOperands(inst, 0);
|
||||
const auto active = ir.GetExec();
|
||||
@@ -588,10 +603,9 @@ bool Translator::FLAT_LOAD(const Decoder::Instruction& inst) {
|
||||
WriteOperand(OffsetOperand(inst.dst, index),
|
||||
bits == 32u ? loaded : WidenSubdword(loaded, bits, sign));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::FLAT_STORE(const Decoder::Instruction& inst) {
|
||||
void Translator::FLAT_STORE(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto data_op = MemorySourceAt(inst, 0);
|
||||
const auto address = ReadAddressOperands(inst, 1);
|
||||
@@ -600,7 +614,9 @@ bool Translator::FLAT_STORE(const Decoder::Instruction& inst) {
|
||||
case 8u: opcode = IR::ValueOpcode::StoreAddressU8; break;
|
||||
case 16u: opcode = IR::ValueOpcode::StoreAddressU16; break;
|
||||
case 32u: opcode = IR::ValueOpcode::StoreAddressU32; break;
|
||||
default: return false;
|
||||
default:
|
||||
EXIT("opcode %s at pc 0x%08x has unsupported flat store width %u",
|
||||
Decoder::InstructionToString(inst).c_str(), inst.pc, memory.data_bits);
|
||||
}
|
||||
const auto count = memory.data_bits == 32u ? memory.data_dwords : 1u;
|
||||
for (uint32_t index = 0; index < count; index++) {
|
||||
@@ -615,20 +631,18 @@ bool Translator::FLAT_STORE(const Decoder::Instruction& inst) {
|
||||
ir.Emit(opcode, {address.resource, address.low, address.high, value, ir.GetExec()},
|
||||
AddMemoryInfo(component, inst.pc));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_GET_RESINFO(const Decoder::Instruction& inst) {
|
||||
void Translator::IMAGE_GET_RESINFO(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0));
|
||||
const auto result = ir.Emit(IR::ValueOpcode::ImageQueryDimensions, {resource, address},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
WriteImageComponents(inst.dst, result, memory, 4u);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) {
|
||||
void Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto sampler = GetSamplerResource(memory);
|
||||
@@ -636,30 +650,27 @@ bool Translator::IMAGE_GET_LOD(const Decoder::Instruction& inst) {
|
||||
const auto result = ir.Emit(IR::ValueOpcode::ImageQueryLod, {resource, sampler, address},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
WriteImageComponents(inst.dst, result, memory, 2u);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_LOAD(const Decoder::Instruction& inst) {
|
||||
void Translator::IMAGE_LOAD(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 0));
|
||||
const auto result = ir.Emit(IR::ValueOpcode::ImageRead, {resource, address, ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
WriteImageComponents(inst.dst, result, memory, 4u);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_STORE(const Decoder::Instruction& inst) {
|
||||
void Translator::IMAGE_STORE(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto address = MakeImageAddress(inst, MemorySourceAt(inst, 1));
|
||||
const auto data = ConstructU32x4(MemorySourceAt(inst, 0), memory.data_dwords);
|
||||
ir.Emit(IR::ValueOpcode::ImageWrite, {resource, address, data, ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) {
|
||||
void Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto sampler = GetSamplerResource(memory);
|
||||
@@ -676,10 +687,9 @@ bool Translator::IMAGE_SAMPLE(const Decoder::Instruction& inst) {
|
||||
} else {
|
||||
WriteImageComponents(inst.dst, result, memory, 4u);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
|
||||
void Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto resource = GetImageResource(memory);
|
||||
const auto sampler = GetSamplerResource(memory);
|
||||
@@ -690,7 +700,6 @@ bool Translator::IMAGE_GATHER(const Decoder::Instruction& inst) {
|
||||
WriteOperand(OffsetOperand(inst.dst, index),
|
||||
ir.Emit(IR::ValueOpcode::CompositeExtractU32x4, {result, IR::Value(index)}));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
IR::Value Translator::LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory,
|
||||
@@ -746,7 +755,7 @@ void Translator::WriteSharedU32(uint32_t width, IR::U32 address,
|
||||
}
|
||||
}
|
||||
|
||||
bool Translator::DS_READ(const Decoder::Instruction& inst) {
|
||||
void Translator::DS_READ(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto address = ReadU32(MemorySourceAt(inst, 0));
|
||||
if (memory.data_bits == 32u) {
|
||||
@@ -755,16 +764,15 @@ bool Translator::DS_READ(const Decoder::Instruction& inst) {
|
||||
for (uint32_t index = 0; index < width; index++) {
|
||||
WriteOperand(OffsetOperand(inst.dst, index), ExtractSharedU32(loaded, width, index));
|
||||
}
|
||||
return true;
|
||||
return;
|
||||
}
|
||||
const auto opcode =
|
||||
memory.data_bits == 8u ? IR::ValueOpcode::LoadSharedU8 : IR::ValueOpcode::LoadSharedU16;
|
||||
const auto loaded = ir.Emit(opcode, {address, ir.GetExec()}, AddMemoryInfo(memory, inst.pc));
|
||||
WriteOperand(inst.dst, WidenSubdword(loaded, memory.data_bits, memory.data_signed));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_READ2(const Decoder::Instruction& inst) {
|
||||
void Translator::DS_READ2(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto width = memory.data_dwords / 2u;
|
||||
const auto address = ReadU32(MemorySourceAt(inst, 0));
|
||||
@@ -783,10 +791,9 @@ bool Translator::DS_READ2(const Decoder::Instruction& inst) {
|
||||
WriteOperand(OffsetOperand(inst.dst, width + index),
|
||||
ExtractSharedU32(second_value, width, index));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_WRITE(const Decoder::Instruction& inst) {
|
||||
void Translator::DS_WRITE(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto width = memory.data_dwords;
|
||||
const auto data_operand = MemorySourceAt(inst, 0);
|
||||
@@ -797,16 +804,15 @@ bool Translator::DS_WRITE(const Decoder::Instruction& inst) {
|
||||
const auto address = ReadU32(MemorySourceAt(inst, 1));
|
||||
if (memory.data_bits == 32u) {
|
||||
WriteSharedU32(width, address, values, memory, inst.pc);
|
||||
return true;
|
||||
return;
|
||||
}
|
||||
const auto opcode =
|
||||
memory.data_bits == 8u ? IR::ValueOpcode::WriteSharedU8 : IR::ValueOpcode::WriteSharedU16;
|
||||
ir.Emit(opcode, {address, NarrowSubdword(IR::U32(values[0]), memory.data_bits), ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_WRITE2(const Decoder::Instruction& inst) {
|
||||
void Translator::DS_WRITE2(const Decoder::Instruction& inst) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto width = memory.data_dwords / 2u;
|
||||
const auto address = ReadU32(MemorySourceAt(inst, 1));
|
||||
@@ -827,28 +833,24 @@ bool Translator::DS_WRITE2(const Decoder::Instruction& inst) {
|
||||
second.offset = memory.secondary_offset;
|
||||
WriteSharedU32(width, address, second_values, second, inst.pc);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
ir.Emit(opcode,
|
||||
{ReadU32(MemorySourceAt(inst, 1)), ReadU32(MemorySourceAt(inst, 0)),
|
||||
ReadU32(MemorySourceAt(inst, 2)), ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
void Translator::DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
WriteOperand(inst.dst, ir.Emit(opcode,
|
||||
{ReadU32(MemorySourceAt(inst, 0)), ir.GetExec(), ir.GetExecLo(),
|
||||
ir.GetExecHi()},
|
||||
{ReadU32(MemorySourceAt(inst, 0)), ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc)));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
|
||||
void Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
|
||||
const auto memory = MemoryInfoFromDecoded(inst);
|
||||
const auto base =
|
||||
ir.BitwiseAnd(ReadU32(MemorySourceAt(inst, write ? 1u : 0u)), IR::U32(IR::Value(0xffffu)));
|
||||
@@ -862,24 +864,21 @@ bool Translator::DS_ADDTID(const Decoder::Instruction& inst, bool write) {
|
||||
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::LoadSharedU32, {address, ir.GetExec()},
|
||||
AddMemoryInfo(memory, inst.pc)));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::DS_SWIZZLE_B32(const Decoder::Instruction& inst) {
|
||||
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::SwizzleU32,
|
||||
{ReadU32(MemorySourceAt(inst, 0)),
|
||||
ReadU32(MemorySourceAt(inst, 1)), ir.GetExec()}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
|
||||
void Translator::DS_BPERMUTE_B32(const Decoder::Instruction& inst) {
|
||||
const auto address = ir.IAdd(ReadU32(inst.src0), IR::U32(IR::Value(inst.offset)));
|
||||
WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::BpermuteU32,
|
||||
{ReadU32(inst.src1), address, ir.GetExec()}));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
void Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
switch (inst.opcode) {
|
||||
case Decoder::Opcode::S_LOAD_DWORD:
|
||||
case Decoder::Opcode::S_LOAD_DWORDX2:
|
||||
@@ -891,6 +890,17 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
case Decoder::Opcode::S_BUFFER_LOAD_DWORDX4:
|
||||
case Decoder::Opcode::S_BUFFER_LOAD_DWORDX8:
|
||||
case Decoder::Opcode::S_BUFFER_LOAD_DWORDX16: return S_LOAD(inst, false);
|
||||
case Decoder::Opcode::S_MEMREALTIME: {
|
||||
static std::atomic_flag warned = ATOMIC_FLAG_INIT;
|
||||
if (!warned.test_and_set(std::memory_order_relaxed)) {
|
||||
Log::WriteToConsoleAndLog(
|
||||
"Warning: S_MEMREALTIME uses placeholder UINT64_MAX; real-time clock not implemented.\n");
|
||||
}
|
||||
for (uint32_t component = 0; component < 2; component++) {
|
||||
WriteOperand(ScalarDestinationOperand(inst.dst, component), IR::Value(UINT32_MAX));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case Decoder::Opcode::BUFFER_LOAD_UBYTE:
|
||||
case Decoder::Opcode::BUFFER_LOAD_SBYTE:
|
||||
@@ -1016,6 +1026,10 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicOr32);
|
||||
case Decoder::Opcode::IMAGE_ATOMIC_XOR:
|
||||
return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicXor32);
|
||||
case Decoder::Opcode::IMAGE_ATOMIC_FMIN:
|
||||
return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicFMin32);
|
||||
case Decoder::Opcode::IMAGE_ATOMIC_FMAX:
|
||||
return IMAGE_ATOMIC(inst, IR::ValueOpcode::ImageAtomicFMax32);
|
||||
|
||||
case Decoder::Opcode::FLAT_LOAD_UBYTE:
|
||||
case Decoder::Opcode::FLAT_LOAD_SBYTE:
|
||||
@@ -1087,7 +1101,7 @@ bool Translator::EmitMemory(const Decoder::Instruction& inst) {
|
||||
case Decoder::Opcode::DS_WRITE_B64:
|
||||
case Decoder::Opcode::DS_WRITE_B96:
|
||||
case Decoder::Opcode::DS_WRITE_B128: return DS_WRITE(inst);
|
||||
default: return false;
|
||||
default: return FailMissingTranslation(inst);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,106 +2,106 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
bool Translator::EmitScalar(const Decoder::Instruction& inst) {
|
||||
void Translator::EmitScalar(const Decoder::Instruction& inst) {
|
||||
using O = Decoder::Opcode;
|
||||
switch (inst.opcode) {
|
||||
case O::S_MOV_B32:
|
||||
case O::S_MOVK_I32: MOV_B32(inst, false); return true;
|
||||
case O::S_MOV_B64: S_MOV_B64(inst); return true;
|
||||
case O::S_WQM_B32: S_WQM(inst, false); return true;
|
||||
case O::S_WQM_B64: S_WQM(inst, true); return true;
|
||||
case O::S_GETPC_B64: S_GETPC_B64(inst); return true;
|
||||
case O::S_SETPC_B64: return true;
|
||||
case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return true;
|
||||
case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return true;
|
||||
case O::S_CSELECT_B32: S_CSELECT_B32(inst); return true;
|
||||
case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return true;
|
||||
case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return true;
|
||||
case O::S_SETREG_B32: EmitControlNop(); return true;
|
||||
case O::S_WAITCNT: EmitWaitcnt(); return true;
|
||||
case O::S_MOVK_I32: MOV_B32(inst, false); return;
|
||||
case O::S_MOV_B64: S_MOV_B64(inst); return;
|
||||
case O::S_WQM_B32: S_WQM(inst, false); return;
|
||||
case O::S_WQM_B64: S_WQM(inst, true); return;
|
||||
case O::S_GETPC_B64: S_GETPC_B64(inst); return;
|
||||
case O::S_SETPC_B64: return;
|
||||
case O::S_SUBVECTOR_LOOP_BEGIN: S_SUBVECTOR_LOOP(inst, true); return;
|
||||
case O::S_SUBVECTOR_LOOP_END: S_SUBVECTOR_LOOP(inst, false); return;
|
||||
case O::S_CSELECT_B32: S_CSELECT_B32(inst); return;
|
||||
case O::S_CSELECT_B64: ScalarSelect64(inst, inst.src1); return;
|
||||
case O::S_CMOV_B64: ScalarSelect64(inst, inst.dst); return;
|
||||
case O::S_SETREG_B32: EmitControlNop(); return;
|
||||
case O::S_WAITCNT: EmitWaitcnt(); return;
|
||||
|
||||
case O::S_AND_SAVEEXEC_B32:
|
||||
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_ANDN1_SAVEEXEC_B32:
|
||||
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_ORN2_SAVEEXEC_B32:
|
||||
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_AND_SAVEEXEC_B64:
|
||||
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::S_ANDN1_SAVEEXEC_B64:
|
||||
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalAnd, false, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::S_ORN2_SAVEEXEC_B64:
|
||||
S_SAVEEXEC(inst, IR::ValueOpcode::LogicalOr, true, false, true);
|
||||
return true;
|
||||
case O::S_ADD_U32: ADD_U32(inst, false, false); return true;
|
||||
case O::S_ADDC_U32: ADD_U32(inst, false, true); return true;
|
||||
case O::S_SUB_U32: SUB_U32(inst, false, false); return true;
|
||||
case O::S_SUBB_U32: SUBB_U32(inst, false, false); return true;
|
||||
case O::S_ABSDIFF_I32: S_ABSDIFF_I32(inst); return true;
|
||||
case O::S_ADD_I32: S_ADD_SUB_I32(inst, false); return true;
|
||||
case O::S_SUB_I32: S_ADD_SUB_I32(inst, true); return true;
|
||||
case O::S_LSHL1_ADD_U32: S_LSHL_ADD_U32(inst, 1u); return true;
|
||||
case O::S_LSHL2_ADD_U32: S_LSHL_ADD_U32(inst, 2u); return true;
|
||||
case O::S_LSHL3_ADD_U32: S_LSHL_ADD_U32(inst, 3u); return true;
|
||||
case O::S_LSHL4_ADD_U32: S_LSHL_ADD_U32(inst, 4u); return true;
|
||||
return;
|
||||
case O::S_ADD_U32: ADD_U32(inst, false, false); return;
|
||||
case O::S_ADDC_U32: ADD_U32(inst, false, true); return;
|
||||
case O::S_SUB_U32: SUB_U32(inst, false, false); return;
|
||||
case O::S_SUBB_U32: SUBB_U32(inst, false, false); return;
|
||||
case O::S_ABSDIFF_I32: S_ABSDIFF_I32(inst); return;
|
||||
case O::S_ADD_I32: S_ADD_SUB_I32(inst, false); return;
|
||||
case O::S_SUB_I32: S_ADD_SUB_I32(inst, true); return;
|
||||
case O::S_LSHL1_ADD_U32: S_LSHL_ADD_U32(inst, 1u); return;
|
||||
case O::S_LSHL2_ADD_U32: S_LSHL_ADD_U32(inst, 2u); return;
|
||||
case O::S_LSHL3_ADD_U32: S_LSHL_ADD_U32(inst, 3u); return;
|
||||
case O::S_LSHL4_ADD_U32: S_LSHL_ADD_U32(inst, 4u); return;
|
||||
case O::S_MIN_I32:
|
||||
ScalarMinMax32(inst, IR::ValueOpcode::SMin32, IR::ValueOpcode::SLessThan32);
|
||||
return true;
|
||||
return;
|
||||
case O::S_MAX_I32:
|
||||
ScalarMinMax32(inst, IR::ValueOpcode::SMax32, IR::ValueOpcode::SGreaterThan32);
|
||||
return true;
|
||||
return;
|
||||
case O::S_MIN_U32:
|
||||
ScalarMinMax32(inst, IR::ValueOpcode::UMin32, IR::ValueOpcode::ULessThan32);
|
||||
return true;
|
||||
return;
|
||||
case O::S_MAX_U32:
|
||||
ScalarMinMax32(inst, IR::ValueOpcode::UMax32, IR::ValueOpcode::UGreaterThan32);
|
||||
return true;
|
||||
return;
|
||||
|
||||
case O::S_CMP_EQ_U32:
|
||||
case O::S_CMP_EQ_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_LG_U32:
|
||||
case O::S_CMP_LG_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_GT_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_GE_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, true,
|
||||
false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_LT_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_LE_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_GT_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_GE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, true,
|
||||
false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_LT_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_LE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_EQ_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::S_CMP_LG_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, true, false);
|
||||
return true;
|
||||
return;
|
||||
|
||||
case O::S_AND_B64:
|
||||
return S_U64_MASK(inst, IR::ValueOpcode::LogicalAnd, IR::ValueOpcode::BitwiseAnd32,
|
||||
@@ -217,12 +217,12 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
|
||||
case O::S_NOP:
|
||||
case O::S_SLEEP:
|
||||
case O::S_SETPRIO:
|
||||
case O::S_TRAP: EmitControlNop(); return true;
|
||||
case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return true;
|
||||
case O::S_BARRIER: S_BARRIER(); return true;
|
||||
case O::S_SENDMSG: S_SENDMSG(inst); return true;
|
||||
case O::S_TTRACEDATA: S_TTRACEDATA(); return true;
|
||||
case O::S_INST_PREFETCH: S_INST_PREFETCH(); return true;
|
||||
case O::S_TRAP: EmitControlNop(); return;
|
||||
case O::S_WAITCNT_DEPCTR: EmitWaitcnt(); return;
|
||||
case O::S_BARRIER: S_BARRIER(); return;
|
||||
case O::S_SENDMSG: S_SENDMSG(inst); return;
|
||||
case O::S_TTRACEDATA: S_TTRACEDATA(); return;
|
||||
case O::S_INST_PREFETCH: S_INST_PREFETCH(); return;
|
||||
case O::S_BRANCH:
|
||||
case O::S_CBRANCH_SCC0:
|
||||
case O::S_CBRANCH_SCC1:
|
||||
@@ -231,8 +231,8 @@ bool Translator::EmitScalar(const Decoder::Instruction& inst) {
|
||||
case O::S_CBRANCH_EXECZ:
|
||||
case O::S_CBRANCH_EXECNZ:
|
||||
case O::S_CBRANCH_CDBGSYS:
|
||||
case O::S_ENDPGM: return true;
|
||||
default: return false;
|
||||
case O::S_ENDPGM: return;
|
||||
default: return FailMissingTranslation(inst);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
|
||||
@@ -225,9 +226,28 @@ IR::Value Translator::ReadOperand(const Decoder::Operand& operand, IR::Type type
|
||||
}
|
||||
return ir.INotEqual(ReadRawU32(operand), IR::U32(IR::Value(0u)));
|
||||
}
|
||||
if (type == IR::Type::U64) {
|
||||
const auto pair = ReadU32Pair(operand);
|
||||
return ir.ConstructU64(pair[0], pair[1]);
|
||||
if (type == IR::Type::U64 || type == IR::Type::F64) {
|
||||
auto pair = ReadU32Pair(operand);
|
||||
if (type == IR::Type::F64) {
|
||||
if (operand.kind == Decoder::OperandKind::LiteralConstant) {
|
||||
pair = {IR::U32(IR::Value(0u)), IR::U32(IR::Value(operand.value))};
|
||||
} else if (operand.kind == Decoder::OperandKind::FloatInlineConstant) {
|
||||
const auto bits = operand.value == 0x3e22f983u
|
||||
? 0x3fc45f306dc9c882ull
|
||||
: std::bit_cast<uint64_t>(static_cast<double>(
|
||||
std::bit_cast<float>(operand.value)));
|
||||
pair = {IR::U32(IR::Value(static_cast<uint32_t>(bits))),
|
||||
IR::U32(IR::Value(static_cast<uint32_t>(bits >> 32u)))};
|
||||
}
|
||||
if (operand.absolute) {
|
||||
pair[1] = ir.BitwiseAnd(pair[1], IR::U32(IR::Value(0x7fffffffu)));
|
||||
}
|
||||
if (operand.negate) {
|
||||
pair[1] = ir.BitwiseXor(pair[1], IR::U32(IR::Value(0x80000000u)));
|
||||
}
|
||||
}
|
||||
const auto bits = ir.ConstructU64(pair[0], pair[1]);
|
||||
return type == IR::Type::F64 ? ir.Emit(IR::ValueOpcode::BitCastF64U64, {bits}) : IR::Value(bits);
|
||||
}
|
||||
auto bits = ApplyBitSourceModifiers(operand, ReadRawU32(operand));
|
||||
if (TypesOverlap(type, IR::Type::F32) && !TypesOverlap(type, IR::Type::U32)) {
|
||||
@@ -297,6 +317,7 @@ void Translator::WriteRawU32(const Decoder::Operand& operand, IR::U32 value) {
|
||||
case Decoder::OperandKind::Sgpr: {
|
||||
const auto reg = static_cast<IR::ScalarReg>(operand.reg);
|
||||
ir.SetScalarReg(reg, value);
|
||||
program.scalar_writes.push_back({current_pc, reg});
|
||||
ir.SetScalarMaskTag(reg, IR::U1(IR::Value(false)));
|
||||
if (IR::RegIndex(reg) > 0u) {
|
||||
ir.SetScalarMaskTag(static_cast<IR::ScalarReg>(IR::RegIndex(reg) - 1u),
|
||||
@@ -403,6 +424,10 @@ void Translator::WriteOperand(const Decoder::Operand& operand, IR::Value value)
|
||||
Write16Bits(operand, IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32U16, {bits})));
|
||||
return;
|
||||
}
|
||||
if (type == IR::Type::F64) {
|
||||
value = ir.Emit(IR::ValueOpcode::BitCastU64F64, {value});
|
||||
type = IR::Type::U64;
|
||||
}
|
||||
if (type == IR::Type::U64) {
|
||||
WriteU32Pair(operand, {ir.CompositeExtract(value, 0), ir.CompositeExtract(value, 1)});
|
||||
return;
|
||||
@@ -675,11 +700,13 @@ std::array<IR::U32, 2> Translator::WriteMask(const Decoder::Operand& operand, IR
|
||||
IR::U1(IR::Value(false)));
|
||||
}
|
||||
ir.SetScalarReg(reg, mask[0]);
|
||||
program.scalar_writes.push_back({current_pc, reg});
|
||||
// A wave32 VALU mask destination must not overwrite the neighboring SGPR.
|
||||
if ((write_64 || program.wave_size == 64u) &&
|
||||
IR::RegIndex(reg) + 1u < IR::NumScalarRegs) {
|
||||
const auto high = static_cast<IR::ScalarReg>(IR::RegIndex(reg) + 1u);
|
||||
ir.SetScalarReg(high, mask[1]);
|
||||
program.scalar_writes.push_back({current_pc, high});
|
||||
ir.SetThreadBitScalarReg(high, IR::U1(IR::Value(false)));
|
||||
ir.SetScalarMaskTag(high, IR::U1(IR::Value(false)));
|
||||
}
|
||||
@@ -722,55 +749,61 @@ void Translator::WriteCompareResult(const Decoder::Operand& operand, IR::U1 valu
|
||||
WriteMask(operand, ir.LogicalAnd(ir.GetExec(), value));
|
||||
}
|
||||
|
||||
void Translator::AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info) {
|
||||
if (source.terminator.goto_value >= 0) {
|
||||
if (source.terminator.goto_variable == UINT32_MAX) {
|
||||
EXIT("block %u sets an invalid goto variable", source.id);
|
||||
void Translator::AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source,
|
||||
IR::BlockInfo& info) {
|
||||
const auto native_condition = [&](CFG::BranchCondition kind) -> IR::U1 {
|
||||
IR::U1 condition;
|
||||
switch (kind) {
|
||||
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
|
||||
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
|
||||
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
|
||||
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
|
||||
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
|
||||
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
|
||||
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
|
||||
case CFG::BranchCondition::ScalarInstruction:
|
||||
EXIT_IF(instruction_branch_condition.IsEmpty());
|
||||
condition = instruction_branch_condition;
|
||||
break;
|
||||
default: EXIT("block %u has an invalid native branch condition", source.id);
|
||||
}
|
||||
ir.SetGotoVariable(source.terminator.goto_variable,
|
||||
IR::U1(IR::Value(source.terminator.goto_value != 0)));
|
||||
return IR::U1(ir.Emit(IR::ValueOpcode::ConditionRef, {condition}, kind));
|
||||
};
|
||||
const auto expression = [&](auto&& self, uint32_t index) -> IR::U1 {
|
||||
const auto& value = graph.expressions.at(index);
|
||||
switch (value.op) {
|
||||
case CFG::ConditionExpression::Op::Constant: return IR::U1(IR::Value(value.lhs != 0));
|
||||
case CFG::ConditionExpression::Op::Variable: return ir.GetGotoVariable(value.lhs);
|
||||
case CFG::ConditionExpression::Op::Native:
|
||||
return native_condition(static_cast<CFG::BranchCondition>(value.lhs));
|
||||
case CFG::ConditionExpression::Op::Not: return ir.LogicalNot(self(self, value.lhs));
|
||||
case CFG::ConditionExpression::Op::Or:
|
||||
return ir.LogicalOr(self(self, value.lhs), self(self, value.rhs));
|
||||
}
|
||||
EXIT("invalid CFG condition expression");
|
||||
};
|
||||
for (const auto& assignment: source.assignments) {
|
||||
ir.SetGotoVariable(assignment.variable, expression(expression, assignment.expression));
|
||||
}
|
||||
if (source.terminator.kind == CFG::TerminatorKind::IndirectBranch) {
|
||||
if (source.terminator.indirect_selector_code != UINT32_MAX) {
|
||||
info.indirect_target = ReadScalarCode(source.terminator.indirect_selector_code);
|
||||
} else if (source.terminator.indirect_pc_sgpr != UINT32_MAX) {
|
||||
info.indirect_target =
|
||||
ir.GetScalarReg(static_cast<IR::ScalarReg>(source.terminator.indirect_pc_sgpr));
|
||||
const auto& term = source.terminator;
|
||||
if (term.kind == CFG::TerminatorKind::IndirectBranch) {
|
||||
if (term.indirect_selector_code != UINT32_MAX) {
|
||||
info.indirect_target = ReadScalarCode(term.indirect_selector_code);
|
||||
} else if (term.indirect_pc_sgpr != UINT32_MAX) {
|
||||
info.indirect_target = ir.GetScalarReg(static_cast<IR::ScalarReg>(term.indirect_pc_sgpr));
|
||||
} else {
|
||||
EXIT("block %u has no indirect branch selector", source.id);
|
||||
}
|
||||
ir.Emit(IR::ValueOpcode::ReferenceU32, {info.indirect_target});
|
||||
return;
|
||||
}
|
||||
if (source.terminator.kind != CFG::TerminatorKind::ConditionalBranch) {
|
||||
return;
|
||||
if (term.kind == CFG::TerminatorKind::ConditionalBranch) {
|
||||
const auto condition = term.expression != UINT32_MAX
|
||||
? expression(expression, term.expression)
|
||||
: native_condition(term.condition);
|
||||
info.condition = condition;
|
||||
ir.Emit(IR::ValueOpcode::Reference, {condition});
|
||||
}
|
||||
// EXEC and VCC are invocation-local Boolean masks. Branching on that Boolean lets inactive
|
||||
// invocations leave the region without reconstructing a host-subgroup mask.
|
||||
IR::U1 condition;
|
||||
switch (source.terminator.condition) {
|
||||
case CFG::BranchCondition::Always: condition = IR::U1(IR::Value(true)); break;
|
||||
case CFG::BranchCondition::SccZero: condition = ir.LogicalNot(ir.GetScc()); break;
|
||||
case CFG::BranchCondition::SccNonZero: condition = ir.GetScc(); break;
|
||||
case CFG::BranchCondition::VccZero: condition = ir.LogicalNot(ir.GetVcc()); break;
|
||||
case CFG::BranchCondition::VccNonZero: condition = ir.GetVcc(); break;
|
||||
case CFG::BranchCondition::ExecZero: condition = ir.LogicalNot(ir.GetExec()); break;
|
||||
case CFG::BranchCondition::ExecNonZero: condition = ir.GetExec(); break;
|
||||
case CFG::BranchCondition::ScalarInstruction:
|
||||
EXIT_IF(instruction_branch_condition.IsEmpty());
|
||||
condition = instruction_branch_condition;
|
||||
break;
|
||||
case CFG::BranchCondition::GotoVariable:
|
||||
if (source.terminator.goto_variable == UINT32_MAX) {
|
||||
EXIT("block %u reads an invalid goto variable", source.id);
|
||||
}
|
||||
condition = ir.GetGotoVariable(source.terminator.goto_variable);
|
||||
break;
|
||||
case CFG::BranchCondition::Unknown:
|
||||
EXIT("block %u has an unknown branch condition", source.id);
|
||||
}
|
||||
info.condition = condition;
|
||||
ir.Emit(IR::ValueOpcode::Reference, {condition});
|
||||
}
|
||||
|
||||
namespace {
|
||||
@@ -842,6 +875,16 @@ void IncludeInstructionVectorRegisters(const Decoder::Instruction& inst, uint32_
|
||||
include_vector(inst.src1);
|
||||
include_vector(inst.src2);
|
||||
include_vector(inst.src3);
|
||||
switch (inst.opcode) {
|
||||
case Decoder::Opcode::V_CVT_F64_I32:
|
||||
case Decoder::Opcode::V_CVT_F64_F32:
|
||||
case Decoder::Opcode::V_CVT_F64_U32: include_vector(inst.dst, 2u); break;
|
||||
case Decoder::Opcode::V_FMA_F64: include_vector(inst.src2, 2u); [[fallthrough]];
|
||||
case Decoder::Opcode::V_MUL_F64: include_vector(inst.src1, 2u); [[fallthrough]];
|
||||
case Decoder::Opcode::V_RCP_F64: include_vector(inst.dst, 2u); [[fallthrough]];
|
||||
case Decoder::Opcode::V_CVT_F32_F64: include_vector(inst.src0, 2u); break;
|
||||
default: break;
|
||||
}
|
||||
if (inst.family == Decoder::Family::DS) {
|
||||
switch (inst.opcode) {
|
||||
case Decoder::Opcode::DS_WRITE_B64:
|
||||
@@ -1240,9 +1283,11 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
|
||||
builtin(IR::StageInputKind::InstanceIndex));
|
||||
}
|
||||
}
|
||||
const bool flush_f32_inputs = options.stage == ShaderType::Compute &&
|
||||
(options.input_info.compute->float_mode & 0x10u) == 0;
|
||||
for (const auto& cfg_block: cfg.blocks) {
|
||||
const auto typed_index = block_indices.at(cfg_block.id);
|
||||
Translator translator(result, result.blocks[typed_index], vector_limit);
|
||||
Translator translator(result, result.blocks[typed_index], vector_limit, flush_f32_inputs);
|
||||
for (uint32_t index = cfg_block.inst_begin; index < cfg_block.inst_end; index++) {
|
||||
const auto& instruction = decoded.instructions[index];
|
||||
if (IsCodeTableLoad(cfg, instruction.pc)) {
|
||||
@@ -1265,7 +1310,7 @@ IR::Program TranslateProgram(const Decoder::Program& decoded, const CFG::Graph&
|
||||
}
|
||||
translator.TranslateInstruction(instruction);
|
||||
}
|
||||
translator.AddBranchCondition(cfg_block, result.block_info[typed_index]);
|
||||
translator.AddBranchCondition(cfg, cfg_block, result.block_info[typed_index]);
|
||||
}
|
||||
IR::ValidateProgram(result, false);
|
||||
return result;
|
||||
|
||||
@@ -9,13 +9,15 @@ namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
class Translator {
|
||||
public:
|
||||
Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit)
|
||||
: program(program), ir(block), current_vector_limit(vector_limit) {}
|
||||
Translator(IR::Program& program, IR::Block* block, uint32_t vector_limit,
|
||||
bool flush_f32_inputs)
|
||||
: program(program), ir(block), current_vector_limit(vector_limit),
|
||||
flush_f32_inputs(flush_f32_inputs) {}
|
||||
|
||||
void TranslateInstruction(const Decoder::Instruction& inst);
|
||||
void TranslateEmbeddedFetch(const Decoder::Instruction& inst, uint32_t attribute,
|
||||
uint32_t component_count, const ShaderBufferResource& resource);
|
||||
void AddBranchCondition(const CFG::BasicBlock& source, IR::BlockInfo& info);
|
||||
void AddBranchCondition(const CFG::Graph& graph, const CFG::BasicBlock& source, IR::BlockInfo& info);
|
||||
|
||||
private:
|
||||
const Decoder::Operand& SourceAt(const Decoder::Instruction& inst, uint32_t index);
|
||||
@@ -80,34 +82,34 @@ private:
|
||||
BufferAddress ReadBufferAddress(const Decoder::Instruction& inst, uint32_t source_offset);
|
||||
IR::U32 WidenSubdword(IR::Value value, uint32_t bits, bool sign);
|
||||
IR::Value NarrowSubdword(IR::U32 value, uint32_t bits);
|
||||
bool S_LOAD(const Decoder::Instruction& inst, bool raw);
|
||||
bool BUFFER_LOAD(const Decoder::Instruction& inst);
|
||||
bool BUFFER_STORE(const Decoder::Instruction& inst);
|
||||
bool BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
|
||||
bool FLAT_LOAD(const Decoder::Instruction& inst);
|
||||
bool FLAT_STORE(const Decoder::Instruction& inst);
|
||||
bool IMAGE_GET_RESINFO(const Decoder::Instruction& inst);
|
||||
bool IMAGE_GET_LOD(const Decoder::Instruction& inst);
|
||||
bool IMAGE_LOAD(const Decoder::Instruction& inst);
|
||||
bool IMAGE_STORE(const Decoder::Instruction& inst);
|
||||
bool IMAGE_SAMPLE(const Decoder::Instruction& inst);
|
||||
bool IMAGE_GATHER(const Decoder::Instruction& inst);
|
||||
void S_LOAD(const Decoder::Instruction& inst, bool raw);
|
||||
void BUFFER_LOAD(const Decoder::Instruction& inst);
|
||||
void BUFFER_STORE(const Decoder::Instruction& inst);
|
||||
void BUFFER_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void IMAGE_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void DS_ATOMIC(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool returns_value);
|
||||
void FLAT_LOAD(const Decoder::Instruction& inst);
|
||||
void FLAT_STORE(const Decoder::Instruction& inst);
|
||||
void IMAGE_GET_RESINFO(const Decoder::Instruction& inst);
|
||||
void IMAGE_GET_LOD(const Decoder::Instruction& inst);
|
||||
void IMAGE_LOAD(const Decoder::Instruction& inst);
|
||||
void IMAGE_STORE(const Decoder::Instruction& inst);
|
||||
void IMAGE_SAMPLE(const Decoder::Instruction& inst);
|
||||
void IMAGE_GATHER(const Decoder::Instruction& inst);
|
||||
IR::Value LoadSharedU32(uint32_t width, IR::U32 address, const IR::MemoryInfo& memory,
|
||||
uint32_t pc);
|
||||
IR::Value ExtractSharedU32(IR::Value value, uint32_t width, uint32_t index);
|
||||
void WriteSharedU32(uint32_t width, IR::U32 address, const std::array<IR::Value, 4>& values,
|
||||
const IR::MemoryInfo& memory, uint32_t pc);
|
||||
bool DS_READ(const Decoder::Instruction& inst);
|
||||
bool DS_READ2(const Decoder::Instruction& inst);
|
||||
bool DS_WRITE(const Decoder::Instruction& inst);
|
||||
bool DS_WRITE2(const Decoder::Instruction& inst);
|
||||
bool DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool DS_ADDTID(const Decoder::Instruction& inst, bool write);
|
||||
bool DS_SWIZZLE_B32(const Decoder::Instruction& inst);
|
||||
bool DS_BPERMUTE_B32(const Decoder::Instruction& inst);
|
||||
void DS_READ(const Decoder::Instruction& inst);
|
||||
void DS_READ2(const Decoder::Instruction& inst);
|
||||
void DS_WRITE(const Decoder::Instruction& inst);
|
||||
void DS_WRITE2(const Decoder::Instruction& inst);
|
||||
void DS_MINMAX_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void DS_APPEND_CONSUME(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void DS_ADDTID(const Decoder::Instruction& inst, bool write);
|
||||
void DS_SWIZZLE_B32(const Decoder::Instruction& inst);
|
||||
void DS_BPERMUTE_B32(const Decoder::Instruction& inst);
|
||||
|
||||
IR::F32 SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value);
|
||||
IR::U32 ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
|
||||
@@ -124,7 +126,7 @@ private:
|
||||
bool signed_value, bool cmpx);
|
||||
void EmitFloatCompare(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool half,
|
||||
bool cmpx);
|
||||
void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered);
|
||||
void EmitFloatOrderedCompare(const Decoder::Instruction& inst, bool ordered, bool cmpx);
|
||||
void EmitFloatClassCompare(const Decoder::Instruction& inst, bool cmpx);
|
||||
void V_CVT_F32_UBYTE(const Decoder::Instruction& inst, uint32_t byte_index);
|
||||
void V_CVT_F32_U32(const Decoder::Instruction& inst);
|
||||
@@ -143,77 +145,77 @@ private:
|
||||
void V_CVT_PKNORM_F32(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void V_CVT_PK_U8_F32(const Decoder::Instruction& inst);
|
||||
void V_PACK_B32_F16(const Decoder::Instruction& inst);
|
||||
bool PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
|
||||
void PackedFloat16(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
|
||||
bool quiet_snan);
|
||||
bool Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void Float16Unary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool invalid_negative);
|
||||
bool Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
|
||||
bool Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
|
||||
void Float16Trig(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void Float16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
|
||||
void Float16Ternary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
|
||||
bool mix);
|
||||
bool FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
|
||||
bool FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
|
||||
void FloatUnary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void FloatBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool reverse);
|
||||
void FloatTernary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool accumulator,
|
||||
bool mix);
|
||||
bool V_FREXP_MANT_F32(const Decoder::Instruction& inst);
|
||||
bool V_DOT2C_F32_F16(const Decoder::Instruction& inst);
|
||||
bool V_CUBEID_F32(const Decoder::Instruction& inst);
|
||||
bool V_CUBESC_F32(const Decoder::Instruction& inst);
|
||||
bool V_CUBETC_F32(const Decoder::Instruction& inst);
|
||||
bool V_CUBEMA_F32(const Decoder::Instruction& inst);
|
||||
bool FloatCube(const Decoder::Instruction& inst, uint32_t result_kind);
|
||||
bool Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
|
||||
bool Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
|
||||
bool V_MAD_I16(const Decoder::Instruction& inst);
|
||||
bool V_MED3_I16(const Decoder::Instruction& inst);
|
||||
bool PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void V_FREXP_MANT_F32(const Decoder::Instruction& inst);
|
||||
void V_DOT2C_F32_F16(const Decoder::Instruction& inst);
|
||||
void V_CUBEID_F32(const Decoder::Instruction& inst);
|
||||
void V_CUBESC_F32(const Decoder::Instruction& inst);
|
||||
void V_CUBETC_F32(const Decoder::Instruction& inst);
|
||||
void V_CUBEMA_F32(const Decoder::Instruction& inst);
|
||||
void FloatCube(const Decoder::Instruction& inst, uint32_t result_kind);
|
||||
void Integer16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool arithmetic);
|
||||
void Integer16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
|
||||
void V_MAD_I16(const Decoder::Instruction& inst);
|
||||
void V_MED3_I16(const Decoder::Instruction& inst);
|
||||
void PackedInteger16Shift(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool arithmetic);
|
||||
bool PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
bool PackedInteger16Mad(const Decoder::Instruction& inst, bool sign);
|
||||
bool PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
|
||||
bool S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
|
||||
void PackedInteger16Binary(const Decoder::Instruction& inst, IR::ValueOpcode opcode);
|
||||
void PackedInteger16Mad(const Decoder::Instruction& inst, bool sign);
|
||||
void PackedInteger16MinMax(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool sign);
|
||||
void S_U64_MASK(const Decoder::Instruction& inst, IR::ValueOpcode logical_opcode,
|
||||
IR::ValueOpcode bit_opcode, bool negate_rhs, bool negate_result, bool unary);
|
||||
IR::U1 U64MaskBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode, bool negate_rhs,
|
||||
bool negate_result);
|
||||
bool SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
|
||||
void SimpleInteger(const Decoder::Instruction& inst, IR::ValueOpcode opcode, IR::Type type,
|
||||
bool reverse, bool mask_shift_count, bool update_scc);
|
||||
bool S_ASHR_I64(const Decoder::Instruction& inst);
|
||||
bool ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
void S_ASHR_I64(const Decoder::Instruction& inst);
|
||||
void ComposedIntegerBinary(const Decoder::Instruction& inst, IR::ValueOpcode opcode,
|
||||
bool negate_rhs, bool negate_result, bool update_scc);
|
||||
bool V_AND_OR_B32(const Decoder::Instruction& inst);
|
||||
bool V_OR3_B32(const Decoder::Instruction& inst);
|
||||
bool V_XOR3_B32(const Decoder::Instruction& inst);
|
||||
bool S_FF1_I32_B64(const Decoder::Instruction& inst);
|
||||
bool V_FFBH_32(const Decoder::Instruction& inst, bool sign);
|
||||
bool S_FLBIT_I32_B64(const Decoder::Instruction& inst);
|
||||
bool Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
|
||||
bool V_MAD_U64_U32(const Decoder::Instruction& inst);
|
||||
bool V_SAD_U32(const Decoder::Instruction& inst);
|
||||
bool V_ADD3_U32(const Decoder::Instruction& inst);
|
||||
bool S_BITSET_B32(const Decoder::Instruction& inst, bool set);
|
||||
bool S_BITSET_B64(const Decoder::Instruction& inst, bool set);
|
||||
bool V_BCNT_U32_B32(const Decoder::Instruction& inst);
|
||||
bool V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
|
||||
bool S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
|
||||
bool S_QUADMASK_B64(const Decoder::Instruction& inst);
|
||||
bool BFM_B32(const Decoder::Instruction& inst);
|
||||
void V_AND_OR_B32(const Decoder::Instruction& inst);
|
||||
void V_OR3_B32(const Decoder::Instruction& inst);
|
||||
void V_XOR3_B32(const Decoder::Instruction& inst);
|
||||
void S_FF1_I32_B64(const Decoder::Instruction& inst);
|
||||
void V_FFBH_32(const Decoder::Instruction& inst, bool sign);
|
||||
void S_FLBIT_I32_B64(const Decoder::Instruction& inst);
|
||||
void Integer24(const Decoder::Instruction& inst, bool sign, bool addend);
|
||||
void V_MAD_U64_U32(const Decoder::Instruction& inst);
|
||||
void V_SAD_U32(const Decoder::Instruction& inst);
|
||||
void V_ADD3_U32(const Decoder::Instruction& inst);
|
||||
void S_BITSET_B32(const Decoder::Instruction& inst, bool set);
|
||||
void S_BITSET_B64(const Decoder::Instruction& inst, bool set);
|
||||
void V_BCNT_U32_B32(const Decoder::Instruction& inst);
|
||||
void V_MBCNT_U32_B32(const Decoder::Instruction& inst, bool low);
|
||||
void S_BITREPLICATE_B64_B32(const Decoder::Instruction& inst);
|
||||
void S_QUADMASK_B64(const Decoder::Instruction& inst);
|
||||
void BFM_B32(const Decoder::Instruction& inst);
|
||||
IR::U32 RightMask32(IR::U32 count);
|
||||
IR::U64 RightMask64(IR::U32 count);
|
||||
bool S_BFM_B64(const Decoder::Instruction& inst);
|
||||
bool S_BFE_U32(const Decoder::Instruction& inst, bool sign);
|
||||
bool S_BFE_U64(const Decoder::Instruction& inst);
|
||||
bool V_BFE_U32(const Decoder::Instruction& inst, bool sign);
|
||||
bool V_BFI_B32(const Decoder::Instruction& inst);
|
||||
bool S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
|
||||
bool S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
|
||||
bool V_ALIGNBIT_B32(const Decoder::Instruction& inst);
|
||||
bool V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
|
||||
bool V_LSHL_ADD_U32(const Decoder::Instruction& inst);
|
||||
bool V_ADD_LSHL_U32(const Decoder::Instruction& inst);
|
||||
bool V_XAD_U32(const Decoder::Instruction& inst);
|
||||
bool V_LSHL_OR_B32(const Decoder::Instruction& inst);
|
||||
bool V_CNDMASK_B32(const Decoder::Instruction& inst);
|
||||
bool PackB16(const Decoder::Instruction& inst, bool high0, bool high1);
|
||||
void S_BFM_B64(const Decoder::Instruction& inst);
|
||||
void S_BFE_U32(const Decoder::Instruction& inst, bool sign);
|
||||
void S_BFE_U64(const Decoder::Instruction& inst);
|
||||
void V_BFE_U32(const Decoder::Instruction& inst, bool sign);
|
||||
void V_BFI_B32(const Decoder::Instruction& inst);
|
||||
void S_BITCMP_B32(const Decoder::Instruction& inst, bool expected);
|
||||
void S_BITCMP_B64(const Decoder::Instruction& inst, bool expected);
|
||||
void V_ALIGNBIT_B32(const Decoder::Instruction& inst);
|
||||
void V_ALIGNBYTE_B32(const Decoder::Instruction& inst);
|
||||
void V_LSHL_ADD_U32(const Decoder::Instruction& inst);
|
||||
void V_ADD_LSHL_U32(const Decoder::Instruction& inst);
|
||||
void V_XAD_U32(const Decoder::Instruction& inst);
|
||||
void V_LSHL_OR_B32(const Decoder::Instruction& inst);
|
||||
void V_CNDMASK_B32(const Decoder::Instruction& inst);
|
||||
void PackB16(const Decoder::Instruction& inst, bool high0, bool high1);
|
||||
|
||||
void S_SUBVECTOR_LOOP(const Decoder::Instruction& inst, bool begin);
|
||||
void S_SAVEEXEC(const Decoder::Instruction& inst, IR::ValueOpcode operation, bool negate_exec,
|
||||
@@ -249,10 +251,11 @@ private:
|
||||
void V_INTERP_MOV_F32(const Decoder::Instruction& inst);
|
||||
void EXP(const Decoder::Instruction& inst);
|
||||
|
||||
bool EmitScalar(const Decoder::Instruction& inst);
|
||||
bool EmitVector(const Decoder::Instruction& inst);
|
||||
bool EmitInterpolation(const Decoder::Instruction& inst);
|
||||
bool EmitMemory(const Decoder::Instruction& inst);
|
||||
void EmitScalar(const Decoder::Instruction& inst);
|
||||
void EmitVector(const Decoder::Instruction& inst);
|
||||
void EmitInterpolation(const Decoder::Instruction& inst);
|
||||
void EmitMemory(const Decoder::Instruction& inst);
|
||||
static void FailMissingTranslation(const Decoder::Instruction& inst);
|
||||
|
||||
IR::Program& program;
|
||||
IR::IREmitter ir;
|
||||
@@ -260,6 +263,7 @@ private:
|
||||
Decoder::Opcode current_opcode = Decoder::Opcode::UNKNOWN;
|
||||
uint32_t current_pc = 0;
|
||||
uint32_t current_vector_limit = 1;
|
||||
bool flush_f32_inputs;
|
||||
};
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::Frontend
|
||||
@@ -2,318 +2,326 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::Frontend {
|
||||
|
||||
bool Translator::EmitVector(const Decoder::Instruction& inst) {
|
||||
void Translator::EmitVector(const Decoder::Instruction& inst) {
|
||||
using O = Decoder::Opcode;
|
||||
switch (inst.opcode) {
|
||||
case O::V_NOP: return true;
|
||||
case O::V_ADD_I32: ADD_U32(inst, true, false); return true;
|
||||
case O::V_ADDC_U32: ADD_U32(inst, true, true); return true;
|
||||
case O::V_SUB_I32: SUB_U32(inst, true, false); return true;
|
||||
case O::V_SUBREV_I32: SUB_U32(inst, true, true); return true;
|
||||
case O::V_SUB_CO_CI_U32: SUBB_U32(inst, true, false); return true;
|
||||
case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true, true); return true;
|
||||
case O::V_NOP: return;
|
||||
case O::V_ADD_I32: ADD_U32(inst, true, false); return;
|
||||
case O::V_ADDC_U32: ADD_U32(inst, true, true); return;
|
||||
case O::V_SUB_I32: SUB_U32(inst, true, false); return;
|
||||
case O::V_SUBREV_I32: SUB_U32(inst, true, true); return;
|
||||
case O::V_SUB_CO_CI_U32: SUBB_U32(inst, true, false); return;
|
||||
case O::V_SUBREV_CO_CI_U32: SUBB_U32(inst, true, true); return;
|
||||
|
||||
case O::V_MOV_B32: MOV_B32(inst, true); return true;
|
||||
case O::V_MOVRELS_B32: V_MOVRELS_B32(inst); return true;
|
||||
case O::V_MOVRELD_B32: V_MOVRELD_B32(inst); return true;
|
||||
case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return true;
|
||||
case O::V_READLANE_B32: V_READLANE_B32(inst); return true;
|
||||
case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return true;
|
||||
case O::V_PERMLANE16_B32: V_PERMLANE16_B32(inst, false); return true;
|
||||
case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return true;
|
||||
case O::V_MOV_B32: MOV_B32(inst, true); return;
|
||||
case O::V_MOVRELS_B32: V_MOVRELS_B32(inst); return;
|
||||
case O::V_MOVRELD_B32: V_MOVRELD_B32(inst); return;
|
||||
case O::V_READFIRSTLANE_B32: V_READFIRSTLANE_B32(inst); return;
|
||||
case O::V_READLANE_B32: V_READLANE_B32(inst); return;
|
||||
case O::V_WRITELANE_B32: V_WRITELANE_B32(inst); return;
|
||||
case O::V_PERMLANE16_B32: V_PERMLANE16_B32(inst, false); return;
|
||||
case O::V_PERMLANEX16_B32: V_PERMLANE16_B32(inst, true); return;
|
||||
|
||||
case O::V_CMP_F_I32:
|
||||
case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return true;
|
||||
case O::V_CMP_F_U32: EmitCompareConstant(inst, false, false, false); return;
|
||||
case O::V_CMP_T_I32:
|
||||
case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return true;
|
||||
case O::V_CMP_T_U32: EmitCompareConstant(inst, true, false, false); return;
|
||||
case O::V_CMP_EQ_U32:
|
||||
case O::V_CMP_EQ_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_EQ_U32:
|
||||
case O::V_CMPX_EQ_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::IEqual32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NE_U32:
|
||||
case O::V_CMP_NE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NE_U32:
|
||||
case O::V_CMPX_NE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GT_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GE_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
|
||||
false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GE_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThanEqual32, IR::Type::U32, false,
|
||||
true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LT_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LE_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false,
|
||||
false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LE_U32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThanEqual32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GT_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThan32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
|
||||
false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SGreaterThanEqual32, IR::Type::U32, false,
|
||||
true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LT_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SLessThan32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false,
|
||||
false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LE_I32:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::SLessThanEqual32, IR::Type::U32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_EQ_I64:
|
||||
case O::V_CMP_EQ_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::IEqual64, IR::Type::U64, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::ULessThan64, IR::Type::U64, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::UGreaterThan64, IR::Type::U64, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NE_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NE_I64:
|
||||
case O::V_CMPX_NE_U64:
|
||||
EmitIntegerCompare(inst, IR::ValueOpcode::INotEqual64, IR::Type::U64, false, true);
|
||||
return true;
|
||||
return;
|
||||
|
||||
case O::V_CMP_EQ_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_EQ_I16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NE_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NE_I16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::INotEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GT_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThan32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GE_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::UGreaterThanEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LT_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::ULessThan32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_EQ_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::IEqual32, false, true);
|
||||
return;
|
||||
case O::V_CMP_LE_U16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::ULessThanEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_I16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThan32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GE_I16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::SGreaterThanEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_I16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::SLessThan32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LE_I16:
|
||||
EmitInteger16Compare(inst, IR::ValueOpcode::SLessThanEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
|
||||
case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return true;
|
||||
case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return true;
|
||||
case O::V_CMP_F_F32: EmitCompareConstant(inst, false, false, false); return;
|
||||
case O::V_CMP_TRU_F32: EmitCompareConstant(inst, true, false, false); return;
|
||||
case O::V_CMP_EQ_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_EQ_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LG_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LG_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NLG_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NLG_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NEQ_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NEQ_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NLE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NLE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThan32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NLT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NLT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NGE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NGE_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThan32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NGT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NGT_F32:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, false, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_EQ_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_EQ_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdEqual32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LG_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdNotEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThan32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_GE_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_GE_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdGreaterThanEqual32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThan32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_LE_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_LE_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPOrdLessThanEqual32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NGT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NGT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordLessThanEqual32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NEQ_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NEQ_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordNotEqual32, true, true);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMP_NLT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, false);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CMPX_NLT_F16:
|
||||
EmitFloatCompare(inst, IR::ValueOpcode::FPUnordGreaterThanEqual32, true, true);
|
||||
return true;
|
||||
case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true); return true;
|
||||
case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false); return true;
|
||||
case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return true;
|
||||
case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return true;
|
||||
return;
|
||||
case O::V_CMP_O_F32: EmitFloatOrderedCompare(inst, true, false); return;
|
||||
case O::V_CMP_U_F32: EmitFloatOrderedCompare(inst, false, false); return;
|
||||
case O::V_CMPX_O_F32: EmitFloatOrderedCompare(inst, true, true); return;
|
||||
case O::V_CMP_CLASS_F32: EmitFloatClassCompare(inst, false); return;
|
||||
case O::V_CMPX_CLASS_F32: EmitFloatClassCompare(inst, true); return;
|
||||
|
||||
case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return true;
|
||||
case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return true;
|
||||
case O::V_CVT_F32_UBYTE2: V_CVT_F32_UBYTE(inst, 2); return true;
|
||||
case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return true;
|
||||
case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return true;
|
||||
case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return true;
|
||||
case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return true;
|
||||
case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return true;
|
||||
case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return true;
|
||||
case O::V_CVT_F32_F16: V_CVT_F32_F16(inst); return true;
|
||||
case O::V_CVT_F16_U16: V_CVT_F16_16(inst, false); return true;
|
||||
case O::V_CVT_F16_I16: V_CVT_F16_16(inst, true); return true;
|
||||
case O::V_CVT_U16_F16: V_CVT_16_F16(inst, false); return true;
|
||||
case O::V_CVT_I16_F16: V_CVT_16_F16(inst, true); return true;
|
||||
case O::V_CVT_RPI_I32_F32: V_CVT_RPI_I32_F32(inst); return true;
|
||||
case O::V_CVT_FLR_I32_F32: V_CVT_FLR_I32_F32(inst); return true;
|
||||
case O::V_FREXP_EXP_I32_F32: V_FREXP_EXP_I32_F32(inst); return true;
|
||||
case O::V_CVT_OFF_F32_I4: V_CVT_OFF_F32_I4(inst); return true;
|
||||
case O::V_CVT_PKRTZ_F16_F32: V_CVT_PKRTZ_F16_F32(inst); return true;
|
||||
case O::V_CVT_F32_UBYTE0: V_CVT_F32_UBYTE(inst, 0); return;
|
||||
case O::V_CVT_F32_UBYTE1: V_CVT_F32_UBYTE(inst, 1); return;
|
||||
case O::V_CVT_F32_UBYTE2: V_CVT_F32_UBYTE(inst, 2); return;
|
||||
case O::V_CVT_F32_UBYTE3: V_CVT_F32_UBYTE(inst, 3); return;
|
||||
case O::V_CVT_F32_U32: V_CVT_F32_U32(inst); return;
|
||||
case O::V_CVT_F32_I32: V_CVT_F32_I32(inst); return;
|
||||
case O::V_CVT_F64_I32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64S32);
|
||||
case O::V_CVT_F32_F64: return FloatUnary(inst, IR::ValueOpcode::ConvertF32F64);
|
||||
case O::V_CVT_F64_F32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64F32);
|
||||
case O::V_CVT_F64_U32: return FloatUnary(inst, IR::ValueOpcode::ConvertF64U32);
|
||||
case O::V_CVT_U32_F32: V_CVT_U32_F32(inst); return;
|
||||
case O::V_CVT_I32_F32: V_CVT_I32_F32(inst); return;
|
||||
case O::V_CVT_F16_F32: V_CVT_F16_F32(inst); return;
|
||||
case O::V_CVT_F32_F16: V_CVT_F32_F16(inst); return;
|
||||
case O::V_CVT_F16_U16: V_CVT_F16_16(inst, false); return;
|
||||
case O::V_CVT_F16_I16: V_CVT_F16_16(inst, true); return;
|
||||
case O::V_CVT_U16_F16: V_CVT_16_F16(inst, false); return;
|
||||
case O::V_CVT_I16_F16: V_CVT_16_F16(inst, true); return;
|
||||
case O::V_CVT_RPI_I32_F32: V_CVT_RPI_I32_F32(inst); return;
|
||||
case O::V_CVT_FLR_I32_F32: V_CVT_FLR_I32_F32(inst); return;
|
||||
case O::V_FREXP_EXP_I32_F32: V_FREXP_EXP_I32_F32(inst); return;
|
||||
case O::V_CVT_OFF_F32_I4: V_CVT_OFF_F32_I4(inst); return;
|
||||
case O::V_CVT_PKRTZ_F16_F32: V_CVT_PKRTZ_F16_F32(inst); return;
|
||||
case O::V_CVT_PKNORM_I16_F32:
|
||||
V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackSnorm2x16);
|
||||
return true;
|
||||
return;
|
||||
case O::V_CVT_PKNORM_U16_F32:
|
||||
V_CVT_PKNORM_F32(inst, IR::ValueOpcode::PackUnorm2x16);
|
||||
return true;
|
||||
case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return true;
|
||||
case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return true;
|
||||
return;
|
||||
case O::V_CVT_PK_U8_F32: V_CVT_PK_U8_F32(inst); return;
|
||||
case O::V_PACK_B32_F16: V_PACK_B32_F16(inst); return;
|
||||
case O::V_CVT_PK_U16_U32:
|
||||
case O::V_CVT_PK_I16_I32: return PackB16(inst, false, false);
|
||||
|
||||
@@ -390,6 +398,7 @@ bool Translator::EmitVector(const Decoder::Instruction& inst) {
|
||||
|
||||
case O::V_FREXP_MANT_F32: return V_FREXP_MANT_F32(inst);
|
||||
case O::V_RCP_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecip32);
|
||||
case O::V_RCP_F64: return FloatUnary(inst, IR::ValueOpcode::FPRecip64);
|
||||
case O::V_RCP_IFLAG_F32: return FloatUnary(inst, IR::ValueOpcode::FPRecipIFlag32);
|
||||
case O::V_FRACT_F32: return FloatUnary(inst, IR::ValueOpcode::FPFract32);
|
||||
case O::V_TRUNC_F32: return FloatUnary(inst, IR::ValueOpcode::FPTrunc32);
|
||||
@@ -406,6 +415,7 @@ bool Translator::EmitVector(const Decoder::Instruction& inst) {
|
||||
case O::V_SUB_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, false);
|
||||
case O::V_SUBREV_F32: return FloatBinary(inst, IR::ValueOpcode::FPSub32, true);
|
||||
case O::V_MUL_F32: return FloatBinary(inst, IR::ValueOpcode::FPMul32, false);
|
||||
case O::V_MUL_F64: return FloatBinary(inst, IR::ValueOpcode::FPMul64, false);
|
||||
case O::V_MIN_F32: return FloatBinary(inst, IR::ValueOpcode::FPMin32, false);
|
||||
case O::V_MAX_F32: return FloatBinary(inst, IR::ValueOpcode::FPMax32, false);
|
||||
case O::V_LDEXP_F32: return FloatBinary(inst, IR::ValueOpcode::FPLdexp, false);
|
||||
@@ -414,6 +424,7 @@ bool Translator::EmitVector(const Decoder::Instruction& inst) {
|
||||
case O::V_MADAK_F32:
|
||||
case O::V_MAD_F32:
|
||||
case O::V_FMA_F32: return FloatTernary(inst, IR::ValueOpcode::FPFma32, false, true);
|
||||
case O::V_FMA_F64: return FloatTernary(inst, IR::ValueOpcode::FPFma64, false, false);
|
||||
case O::V_MIN3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMinTri32, false, false);
|
||||
case O::V_MAX3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMaxTri32, false, false);
|
||||
case O::V_MED3_F32: return FloatTernary(inst, IR::ValueOpcode::FPMedTri32, false, false);
|
||||
@@ -534,7 +545,7 @@ bool Translator::EmitVector(const Decoder::Instruction& inst) {
|
||||
case O::V_XAD_U32: return V_XAD_U32(inst);
|
||||
case O::V_LSHL_OR_B32: return V_LSHL_OR_B32(inst);
|
||||
case O::V_CNDMASK_B32: return V_CNDMASK_B32(inst);
|
||||
default: return false;
|
||||
default: return FailMissingTranslation(inst);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -81,7 +81,8 @@ bool EquivalentValue(const ResourcePlan& program, Value left, Value right,
|
||||
program.memory_info[li] != program.memory_info[ri]) {
|
||||
return false;
|
||||
}
|
||||
} else if (lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
|
||||
} else if (lhs->GetOpcode() != ValueOpcode::ReadConst &&
|
||||
lhs->Flags<uint64_t>() != rhs->Flags<uint64_t>()) {
|
||||
return false;
|
||||
}
|
||||
for (size_t index = 0; index < lhs->NumArgs(); index++) {
|
||||
@@ -454,10 +455,8 @@ void ValidateProgram(const Program& program, bool require_ssa) {
|
||||
ValueOpcodeName(inst.GetOpcode())));
|
||||
}
|
||||
if (memory.kind == ResourceKind::IndirectBuffer &&
|
||||
(memory.formatted || memory.typed ||
|
||||
(inst.GetOpcode() != ValueOpcode::LoadBufferU32x2 &&
|
||||
inst.GetOpcode() != ValueOpcode::LoadBufferU32x4))) {
|
||||
return Fail("indirect buffer requires a raw DWORD x2/x4 load");
|
||||
!memory.SupportsIndirectBufferLoad(inst.GetOpcode())) {
|
||||
return Fail("indirect buffer requires a raw DWORD x2/x3/x4 load");
|
||||
}
|
||||
if (buffer_components > 1u &&
|
||||
(!vector_buffer || memory.data_bits != 32u ||
|
||||
|
||||
@@ -67,8 +67,15 @@ struct MemoryInfo {
|
||||
bool image_r128 = false;
|
||||
bool idxen = false;
|
||||
bool offen = false;
|
||||
bool coherent = false;
|
||||
bool planning_only = false;
|
||||
|
||||
[[nodiscard]] bool SupportsIndirectBufferLoad(ValueOpcode opcode) const {
|
||||
return !formatted && !typed && data_bits == 32u &&
|
||||
(opcode == ValueOpcode::LoadBufferU32x2 || opcode == ValueOpcode::LoadBufferU32x3 ||
|
||||
opcode == ValueOpcode::LoadBufferU32x4);
|
||||
}
|
||||
|
||||
bool operator==(const MemoryInfo& other) const = default;
|
||||
};
|
||||
|
||||
@@ -140,6 +147,7 @@ struct SamplerResource {
|
||||
uint32_t first_use_pc = 0;
|
||||
bool force_point_filtering = false;
|
||||
bool depth_compare = false;
|
||||
bool integer_border = false;
|
||||
|
||||
bool operator==(const SamplerResource& other) const = default;
|
||||
};
|
||||
@@ -428,7 +436,7 @@ struct BindingLayout {
|
||||
};
|
||||
|
||||
struct ShaderInfo {
|
||||
static constexpr uint32_t MaxBuffers = 32;
|
||||
static constexpr uint32_t MaxBuffers = 64;
|
||||
static constexpr uint32_t MaxImages = 64;
|
||||
static constexpr uint32_t MaxSamplers = 32;
|
||||
static constexpr uint32_t MaxSampledPairs = 64;
|
||||
@@ -541,7 +549,7 @@ struct ResourcePlan {
|
||||
std::vector<SrtRead> srt_reads;
|
||||
std::vector<uint8_t> clean_flat_slots;
|
||||
bool requires_specialization_memory = false;
|
||||
bool has_address_writes = false;
|
||||
bool capture_specialization_reads = false;
|
||||
bool srt_plan_complete = false;
|
||||
bool resource_tracking_complete = false;
|
||||
ShaderInfo info;
|
||||
@@ -575,10 +583,12 @@ struct Program: ResourcePlan {
|
||||
CFG::FailureKind cfg_failure_kind = CFG::FailureKind::None;
|
||||
std::string fallback_reason;
|
||||
std::vector<BlockInfo> block_info;
|
||||
struct ScalarWrite { uint32_t pc; ScalarReg reg; };
|
||||
std::vector<ScalarWrite> scalar_writes;
|
||||
// Typed memory and export instructions reference shader-local metadata by dense index.
|
||||
// Decoder-only details (such as NSA register numbers) have already become IR operands.
|
||||
std::vector<ExportInfo> export_info;
|
||||
std::vector<Value> dynamic_reads;
|
||||
bool has_address_writes = false;
|
||||
bool shader_info_complete = false;
|
||||
BindingLayout bindings;
|
||||
bool binding_layout_complete = false;
|
||||
|
||||
@@ -30,6 +30,7 @@ std::string TypeName(Type type) {
|
||||
"ImageResource",
|
||||
"SamplerResource",
|
||||
"ImageAddress",
|
||||
"F64",
|
||||
};
|
||||
const auto bits = static_cast<uint32_t>(type);
|
||||
if (bits == 0) {
|
||||
|
||||
@@ -27,6 +27,7 @@ enum class Type : uint32_t {
|
||||
ImageResource = 1u << 17u,
|
||||
SamplerResource = 1u << 18u,
|
||||
ImageAddress = 1u << 19u,
|
||||
F64 = 1u << 20u,
|
||||
};
|
||||
|
||||
constexpr Type operator|(Type lhs, Type rhs) {
|
||||
|
||||
@@ -33,6 +33,7 @@ constexpr Type U32 = Type::U32;
|
||||
constexpr Type U64 = Type::U64;
|
||||
constexpr Type F16 = Type::F16;
|
||||
constexpr Type F32 = Type::F32;
|
||||
constexpr Type F64 = Type::F64;
|
||||
constexpr Type U32x2 = Type::U32x2;
|
||||
constexpr Type U32x3 = Type::U32x3;
|
||||
constexpr Type U32x4 = Type::U32x4;
|
||||
@@ -220,6 +221,8 @@ ImageOpcodeInfo ImageOpcodeInfoOf(ValueOpcode opcode) {
|
||||
case ValueOpcode::ImageAtomicAnd32:
|
||||
case ValueOpcode::ImageAtomicOr32:
|
||||
case ValueOpcode::ImageAtomicXor32:
|
||||
case ValueOpcode::ImageAtomicFMin32:
|
||||
case ValueOpcode::ImageAtomicFMax32:
|
||||
return {ImageAccess::Atomic, ImageResourceClass::Storage, false};
|
||||
default: return {};
|
||||
}
|
||||
|
||||
@@ -52,6 +52,10 @@ struct PermlaneFlags {
|
||||
static_assert(sizeof(PermlaneFlags) <= sizeof(uint64_t));
|
||||
static_assert(std::is_trivially_copyable_v<PermlaneFlags>);
|
||||
|
||||
struct FPCompareFlags {
|
||||
bool flush_input_denorms = false;
|
||||
};
|
||||
|
||||
struct MemoryFlags {
|
||||
uint32_t index = 0;
|
||||
uint32_t pc = 0;
|
||||
|
||||
@@ -4,6 +4,7 @@ VALUE_OPCODE(Identity, Opaque, Opaque)
|
||||
VALUE_OPCODE(Void, Void)
|
||||
VALUE_OPCODE(Reference, Void, U1)
|
||||
VALUE_OPCODE(ReferenceU32, Void, U32)
|
||||
VALUE_OPCODE(ConditionRef, U1, U1)
|
||||
|
||||
// Translation and register-state pseudo operations.
|
||||
VALUE_OPCODE(GetUserData, U32, ScalarReg)
|
||||
@@ -52,6 +53,8 @@ VALUE_OPCODE(BitCastU16F16, U16, F16)
|
||||
VALUE_OPCODE(BitCastF16U16, F16, U16)
|
||||
VALUE_OPCODE(BitCastU32F32, U32, F32)
|
||||
VALUE_OPCODE(BitCastF32U32, F32, U32)
|
||||
VALUE_OPCODE(BitCastU64F64, U64, F64)
|
||||
VALUE_OPCODE(BitCastF64U64, F64, U64)
|
||||
VALUE_OPCODE(ConvertU16U32, U16, U32)
|
||||
VALUE_OPCODE(ConvertU32U16, U32, U16)
|
||||
VALUE_OPCODE(ConvertU8U32, U8, U32)
|
||||
@@ -61,7 +64,11 @@ VALUE_OPCODE(ConvertF16F32, F16, F32)
|
||||
VALUE_OPCODE(ConvertS32F32, U32, F32)
|
||||
VALUE_OPCODE(ConvertU32F32, U32, F32)
|
||||
VALUE_OPCODE(ConvertF32S32, F32, U32)
|
||||
VALUE_OPCODE(ConvertF64S32, F64, U32)
|
||||
VALUE_OPCODE(ConvertF32F64, F32, F64)
|
||||
VALUE_OPCODE(ConvertF64F32, F64, F32)
|
||||
VALUE_OPCODE(ConvertF32U32, F32, U32)
|
||||
VALUE_OPCODE(ConvertF64U32, F64, U32)
|
||||
VALUE_OPCODE(CompositeConstructU64, U64, U32, U32)
|
||||
VALUE_OPCODE(CompositeConstructU32x2, U32x2, U32, U32)
|
||||
VALUE_OPCODE(CompositeConstructU32x3, U32x3, U32, U32, U32)
|
||||
@@ -166,6 +173,9 @@ VALUE_OPCODE(FPAdd32, F32, F32, F32)
|
||||
VALUE_OPCODE(FPSub32, F32, F32, F32)
|
||||
VALUE_OPCODE(FPFma32, F32, F32, F32, F32)
|
||||
VALUE_OPCODE(FPMul32, F32, F32, F32)
|
||||
VALUE_OPCODE(FPMul64, F64, F64, F64)
|
||||
VALUE_OPCODE(FPFma64, F64, F64, F64, F64)
|
||||
VALUE_OPCODE(FPRecip64, F64, F64)
|
||||
VALUE_OPCODE(FPMin32, F32, F32, F32)
|
||||
VALUE_OPCODE(FPMax32, F32, F32, F32)
|
||||
VALUE_OPCODE(FPMinTri32, F32, F32, F32, F32)
|
||||
@@ -267,8 +277,8 @@ VALUE_OPCODE(SharedAtomicUMax32, U32, U32, U32, U1)
|
||||
VALUE_OPCODE(SharedAtomicAnd32, U32, U32, U32, U1)
|
||||
VALUE_OPCODE(SharedAtomicOr32, U32, U32, U32, U1)
|
||||
VALUE_OPCODE(SharedAtomicXor32, U32, U32, U32, U1)
|
||||
VALUE_OPCODE(DataAppend, U32, U32, U1, U32, U32)
|
||||
VALUE_OPCODE(DataConsume, U32, U32, U1, U32, U32)
|
||||
VALUE_OPCODE(DataAppend, U32, U32, U1)
|
||||
VALUE_OPCODE(DataConsume, U32, U32, U1)
|
||||
VALUE_OPCODE(SwizzleU32, U32, U32, U32, U1)
|
||||
|
||||
VALUE_OPCODE(ImageQueryDimensions, U32x4, ImageResource, ImageAddress)
|
||||
@@ -284,6 +294,8 @@ VALUE_OPCODE(ImageAtomicUMax32, U32, ImageResource, ImageAddress, U32, U1)
|
||||
VALUE_OPCODE(ImageAtomicAnd32, U32, ImageResource, ImageAddress, U32, U1)
|
||||
VALUE_OPCODE(ImageAtomicOr32, U32, ImageResource, ImageAddress, U32, U1)
|
||||
VALUE_OPCODE(ImageAtomicXor32, U32, ImageResource, ImageAddress, U32, U1)
|
||||
VALUE_OPCODE(ImageAtomicFMin32, U32, ImageResource, ImageAddress, U32, U1)
|
||||
VALUE_OPCODE(ImageAtomicFMax32, U32, ImageResource, ImageAddress, U32, U1)
|
||||
VALUE_OPCODE(GetAttribute, U32, U32, U32)
|
||||
VALUE_OPCODE(GetInterpolationParameter, U32, U32, U32, U32)
|
||||
VALUE_OPCODE(SetAttribute, Void, U32x4, U1)
|
||||
|
||||
@@ -46,28 +46,54 @@ void AddBinding(BindingLayout& layout, DescriptorBindingKind kind,
|
||||
layout.descriptors.push_back({kind, std::move(resources)});
|
||||
}
|
||||
|
||||
bool UsesGds(const Program& program) {
|
||||
} // namespace
|
||||
|
||||
bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers) {
|
||||
std::array<bool, ShaderInfo::MaxBuffers> live_buffers {};
|
||||
bool uses_gds = false;
|
||||
for (const auto* block: program.blocks) {
|
||||
for (const auto& inst: *block) {
|
||||
if (SharedAccessOf(inst.GetOpcode()) == SharedAccess::None) {
|
||||
const auto op = inst.GetOpcode();
|
||||
if (BufferAccessOf(op) == BufferAccess::None &&
|
||||
SharedAccessOf(op) == SharedAccess::None) {
|
||||
continue;
|
||||
}
|
||||
const auto index = inst.Flags<MemoryFlags>().index;
|
||||
if (index >= program.memory_info.size()) {
|
||||
BindingFail("typed shader contains invalid shared-memory metadata");
|
||||
BindingFail("typed shader contains invalid memory metadata");
|
||||
}
|
||||
const auto kind = program.memory_info[index].kind;
|
||||
if (kind != ResourceKind::Lds && kind != ResourceKind::Gds) {
|
||||
BindingFail("typed shader contains invalid shared-memory metadata");
|
||||
const auto& memory = program.memory_info[index];
|
||||
if (memory.planning_only) {
|
||||
continue;
|
||||
}
|
||||
uses_gds |= kind == ResourceKind::Gds;
|
||||
if (SharedAccessOf(op) != SharedAccess::None) {
|
||||
if (memory.kind != ResourceKind::Lds && memory.kind != ResourceKind::Gds) {
|
||||
BindingFail("typed shader contains invalid shared-memory metadata");
|
||||
}
|
||||
uses_gds |= memory.kind == ResourceKind::Gds;
|
||||
} else if (memory.kind == ResourceKind::Buffer || memory.kind == ResourceKind::ScalarBuffer) {
|
||||
EXIT_IF(memory.resource >= program.info.buffers.size());
|
||||
live_buffers.at(memory.resource) = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
|
||||
if (live_buffers.at(i)) {
|
||||
buffers.push_back(i);
|
||||
}
|
||||
}
|
||||
return uses_gds;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
bool UsesFlattenedSrt(const Program& program) {
|
||||
return std::ranges::any_of(program.blocks, [](const Block* block) {
|
||||
return std::ranges::any_of(*block, [](const Inst& inst) {
|
||||
return inst.GetOpcode() == ValueOpcode::ReadConst;
|
||||
});
|
||||
}) || std::ranges::any_of(program.info.images, [](const ImageResource& image) {
|
||||
return image.indirect_search_iterations != 0u;
|
||||
});
|
||||
}
|
||||
|
||||
void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
|
||||
if (!program.shader_info_complete || program.binding_layout_complete) {
|
||||
@@ -76,18 +102,17 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
|
||||
: "binding layout already allocated");
|
||||
}
|
||||
BindingLayout next;
|
||||
std::vector<uint32_t> buffers;
|
||||
const bool uses_gds = CollectMemoryResources(program, buffers);
|
||||
next.user_data_registers = CollectUserData(program);
|
||||
next.memory_offset_dword = static_cast<uint32_t>(next.user_data_registers.size());
|
||||
next.memory_offset_count = static_cast<uint32_t>(program.info.buffers.size());
|
||||
next.memory_offset_count = static_cast<uint32_t>(buffers.size());
|
||||
next.push_data_start_dword =
|
||||
PushData::StartFor(push_data_start_dword, next.ShaderDataDwords());
|
||||
|
||||
if (!program.info.buffers.empty()) {
|
||||
std::vector<uint32_t> resources(program.info.buffers.size());
|
||||
for (uint32_t i = 0; i < resources.size(); i++) {
|
||||
resources[i] = i;
|
||||
}
|
||||
AddBinding(next, DescriptorBindingKind::Buffers, std::move(resources));
|
||||
if (!buffers.empty()) {
|
||||
// Draw binding accesses this first group directly when memory_offset_count is nonzero.
|
||||
AddBinding(next, DescriptorBindingKind::Buffers, std::move(buffers));
|
||||
}
|
||||
|
||||
std::array<std::vector<uint32_t>, ImageBindingCount> image_groups;
|
||||
@@ -123,19 +148,14 @@ void AllocateBindings(Program& program, uint32_t push_data_start_dword) {
|
||||
}
|
||||
AddBinding(next, DescriptorBindingKind::Samplers, std::move(resources));
|
||||
}
|
||||
if (UsesGds(program)) {
|
||||
if (uses_gds) {
|
||||
AddBinding(next, DescriptorBindingKind::Gds);
|
||||
}
|
||||
if (program.info.uses_dma) {
|
||||
AddBinding(next, DescriptorBindingKind::BdaPagetable);
|
||||
AddBinding(next, DescriptorBindingKind::FaultBuffer);
|
||||
}
|
||||
const bool uses_flattened_runtime =
|
||||
!program.srt_reads.empty() ||
|
||||
std::ranges::any_of(program.info.images, [](const ImageResource& image) {
|
||||
return image.indirect_search_iterations != 0u;
|
||||
});
|
||||
if (uses_flattened_runtime) {
|
||||
if (UsesFlattenedSrt(program)) {
|
||||
AddBinding(next, DescriptorBindingKind::FlattenedSrt);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,10 @@ namespace Libs::Graphics::ShaderRecompiler::IR {
|
||||
|
||||
void AllocateBindings(Program& program, uint32_t push_data_start_dword = 0);
|
||||
|
||||
// Collect live direct buffer indices and report whether the shader accesses GDS.
|
||||
bool CollectMemoryResources(const Program& program, std::vector<uint32_t>& buffers);
|
||||
bool UsesFlattenedSrt(const Program& program);
|
||||
|
||||
const DescriptorBinding* FindBinding(const BindingLayout& layout, DescriptorBindingKind kind);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::IR
|
||||
|
||||
@@ -630,6 +630,11 @@ void FoldInstruction(Block& block, Block::iterator instruction,
|
||||
}
|
||||
}
|
||||
return;
|
||||
case ValueOpcode::ConditionRef: {
|
||||
const auto value = Arg(inst, 0);
|
||||
if (IsImmediate(value, Type::U1)) Replace(inst, value);
|
||||
return;
|
||||
}
|
||||
case ValueOpcode::LogicalNot: {
|
||||
const auto value = Arg(inst, 0);
|
||||
if (IsImmediate(value, Type::U1)) {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "common/assert.h"
|
||||
#include "graphics/guest_gpu/gpu_format.h"
|
||||
#include "graphics/shader/recompiler/BufferFormat.h"
|
||||
#include "graphics/shader/recompiler/ir/ShaderIR.h"
|
||||
#include "graphics/shader/shaderBindings.h"
|
||||
|
||||
@@ -106,14 +107,16 @@ Prospero::BufferFormat ImageConversionFormat(Prospero::BufferFormat format) {
|
||||
: Prospero::BufferFormat::kInvalid;
|
||||
}
|
||||
|
||||
bool RequiresPointSampler(const ImageResource& image) {
|
||||
return image.numeric_class == Prospero::TextureNumericClass::Sint ||
|
||||
image.conversion_format != Prospero::BufferFormat::kInvalid;
|
||||
}
|
||||
enum class SamplerClass : uint8_t { Float, Integer, PointInteger };
|
||||
|
||||
bool RequiresPointSampler(const ResourceSpecialization::Image& image) {
|
||||
return image.numeric_class == Prospero::TextureNumericClass::Sint ||
|
||||
image.conversion_format != Prospero::BufferFormat::kInvalid;
|
||||
template <typename Image>
|
||||
SamplerClass ClassifySampler(const Image& image) {
|
||||
if (image.numeric_class == Prospero::TextureNumericClass::Sint ||
|
||||
image.conversion_format != Prospero::BufferFormat::kInvalid) {
|
||||
return SamplerClass::PointInteger;
|
||||
}
|
||||
return image.numeric_class == Prospero::TextureNumericClass::Uint ? SamplerClass::Integer
|
||||
: SamplerClass::Float;
|
||||
}
|
||||
|
||||
bool DescriptorIsCube(const DescriptorValue& descriptor) {
|
||||
@@ -222,9 +225,9 @@ void MarkCleanFlatSlots(const ResourcePlan& program, const DescriptorSource* sou
|
||||
}
|
||||
}
|
||||
|
||||
bool ReadScalarTable(uint64_t base, uint64_t size, uint32_t dynamic_offset,
|
||||
bool ReadScalarTable(uint64_t base, uint64_t size, uint64_t dynamic_offset,
|
||||
const SrtRuntime& runtime, std::span<uint32_t> words) {
|
||||
const auto offset = static_cast<uint64_t>(dynamic_offset) & ~uint64_t {3};
|
||||
const auto offset = dynamic_offset & ~uint64_t {3};
|
||||
const auto count = std::min<uint64_t>(words.size(), offset < size ? (size - offset) / 4u : 0u);
|
||||
std::ranges::fill(words.subspan(count), 0u);
|
||||
if (count == 0u) {
|
||||
@@ -303,26 +306,24 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
|
||||
material.Stride() != indirect.selector_stride) {
|
||||
return false;
|
||||
}
|
||||
// Enumerate every wrapped scalar-buffer offset that can pass the descriptor bounds.
|
||||
const auto step = std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u);
|
||||
const auto residue = static_cast<uint64_t>(indirect.selector_offset) % step;
|
||||
const auto limit = std::min<uint64_t>(UINT32_MAX, material.GetSize() + 3u);
|
||||
const auto probe_count = residue <= limit ? (limit - residue) / step + 1u : 0u;
|
||||
// The first aligned offset includes the immediate added after shader U32 arithmetic.
|
||||
const auto step = std::max<uint64_t>(4u,
|
||||
std::gcd<uint64_t>(indirect.selector_stride, uint64_t {1} << 32u));
|
||||
const uint64_t first = indirect.selector_offset;
|
||||
const auto size = material.GetSize();
|
||||
const auto limit = std::min(first + (uint64_t {1} << 32u) - step, size >= 4u ? size - 4u : 0u);
|
||||
const auto probe_count = size >= 4u && first <= limit ? (limit - first) / step + 1u : 0u;
|
||||
if (probe_count > MaxIndirectImageProbes) {
|
||||
return false;
|
||||
}
|
||||
keys.reserve(static_cast<size_t>(probe_count) + 1u);
|
||||
keys.push_back(0u);
|
||||
for (uint64_t offset = residue; offset <= limit && probe_count != 0u; offset += step) {
|
||||
for (uint64_t probe = 0, offset = first; probe < probe_count; ++probe, offset += step) {
|
||||
uint32_t key = 0;
|
||||
if (!ReadScalarTable(material.Base48(), material.GetSize(),
|
||||
static_cast<uint32_t>(offset), runtime, {&key, 1})) {
|
||||
if (!ReadScalarTable(material.Base48(), size, offset, runtime, {&key, 1})) {
|
||||
return false;
|
||||
}
|
||||
keys.push_back(key);
|
||||
if (limit - offset < step) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::ranges::sort(keys);
|
||||
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
||||
@@ -379,8 +380,13 @@ bool MaterializeIndirectImage(const ResourcePlan& program,
|
||||
} // namespace
|
||||
|
||||
struct SamplerPlan {
|
||||
std::array<uint32_t, ShaderInfo::MaxSamplers> point_sampler {};
|
||||
uint32_t sampler_count = 0;
|
||||
struct Binding {
|
||||
uint32_t source;
|
||||
SamplerClass type;
|
||||
};
|
||||
std::array<std::array<uint32_t, 3>, ShaderInfo::MaxSamplers> mapping;
|
||||
std::array<Binding, ShaderInfo::MaxSamplers> bindings;
|
||||
uint32_t sampler_count = 0;
|
||||
};
|
||||
|
||||
struct ImageRemap {
|
||||
@@ -400,6 +406,9 @@ struct ImageRemap {
|
||||
template <typename T>
|
||||
void Apply(std::vector<T>& images) const {
|
||||
EXIT_IF(images.size() != source_count);
|
||||
if (count == source_count) {
|
||||
return;
|
||||
}
|
||||
for (uint32_t index = 0; index < source_count; index++) {
|
||||
if (indices[index] != UINT32_MAX && indices[index] != index) {
|
||||
images[indices[index]] = std::move(images[index]);
|
||||
@@ -446,6 +455,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
: Prospero::BufferFormat::kInvalid,
|
||||
.descriptor_swizzle =
|
||||
program.info.buffers[i].formatted ? descriptor.DstSelXYZW() : DstSel(4, 5, 6, 7),
|
||||
.zero_stride_oob = descriptor.OutOfBounds() == 0u && stride == 0u,
|
||||
});
|
||||
}
|
||||
for (uint32_t i = 0; i < specialization.images.size(); i++) {
|
||||
@@ -485,7 +495,8 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
image.cube = DescriptorIsCube(descriptor);
|
||||
const auto format =
|
||||
static_cast<Prospero::BufferFormat>((descriptor.dwords[1] >> 20u) & 0x1ffu);
|
||||
if (base.atomic && format != Prospero::BufferFormat::k32UInt) {
|
||||
if (base.atomic && format != Prospero::BufferFormat::k32UInt &&
|
||||
format != Prospero::BufferFormat::k32Float) {
|
||||
return SpecializationFail(
|
||||
fmt::format("atomic image descriptor {} uses unsupported format {}", i,
|
||||
static_cast<uint32_t>(format)));
|
||||
@@ -514,7 +525,7 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
fmt::format("storage image descriptor {} uses unsupported format {}", i,
|
||||
static_cast<uint32_t>(format)));
|
||||
}
|
||||
if (raw_sint_storage) {
|
||||
if (raw_sint_storage || base.atomic) {
|
||||
image.numeric_class = Prospero::TextureNumericClass::Uint;
|
||||
}
|
||||
} else if (image.numeric_class == Prospero::TextureNumericClass::Unsupported ||
|
||||
@@ -589,11 +600,9 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
if (!BuildSamplerPlan(program.info, specialization.images, sampler_plan)) {
|
||||
return SpecializationFail("specialized sampler layout exceeds its resource limit");
|
||||
}
|
||||
for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
|
||||
const auto target = sampler_plan.point_sampler[index];
|
||||
if (target != UINT32_MAX && target >= program.info.samplers.size()) {
|
||||
snapshot.samplers.push_back(snapshot.samplers[index]);
|
||||
}
|
||||
for (uint32_t index = static_cast<uint32_t>(program.info.samplers.size());
|
||||
index < sampler_plan.sampler_count; index++) {
|
||||
snapshot.samplers.push_back(snapshot.samplers[sampler_plan.bindings[index].source]);
|
||||
}
|
||||
ImageRemap(specialization).Apply(snapshot.images);
|
||||
return true;
|
||||
@@ -601,29 +610,29 @@ static bool BuildResourceSpecialization(const ResourcePlan& program, ResourceSna
|
||||
|
||||
template <typename Images>
|
||||
bool BuildSamplerPlan(const ShaderInfo& base, const Images& images, SamplerPlan& plan) {
|
||||
if (base.samplers.size() > plan.point_sampler.size()) {
|
||||
if (base.samplers.size() > plan.mapping.size()) {
|
||||
return false;
|
||||
}
|
||||
std::array<uint8_t, ShaderInfo::MaxSamplers> usage {};
|
||||
plan.point_sampler.fill(UINT32_MAX);
|
||||
plan.sampler_count = static_cast<uint32_t>(base.samplers.size());
|
||||
for (const auto& pair: base.sampled_pairs) {
|
||||
if (pair.image >= images.size() || pair.sampler >= base.samplers.size()) {
|
||||
return false;
|
||||
}
|
||||
usage[pair.sampler] |= RequiresPointSampler(images[pair.image]) ? 2u : 1u;
|
||||
usage[pair.sampler] |= 1u << static_cast<uint32_t>(ClassifySampler(images[pair.image]));
|
||||
}
|
||||
for (uint32_t index = 0; index < base.samplers.size(); index++) {
|
||||
if ((usage[index] & 2u) == 0u) {
|
||||
continue;
|
||||
}
|
||||
if ((usage[index] & 1u) == 0u) {
|
||||
plan.point_sampler[index] = index;
|
||||
} else {
|
||||
if (plan.sampler_count >= ShaderInfo::MaxSamplers) {
|
||||
return false;
|
||||
}
|
||||
plan.point_sampler[index] = plan.sampler_count++;
|
||||
auto& mapping = plan.mapping[index];
|
||||
mapping.fill(UINT32_MAX);
|
||||
const auto classes = usage[index] == 0u ? 1u : usage[index];
|
||||
bool first = true;
|
||||
for (uint32_t type = 0; type < mapping.size(); type++) {
|
||||
if ((classes & (1u << type)) == 0u) continue;
|
||||
const auto target = first ? index : plan.sampler_count++;
|
||||
if (target >= ShaderInfo::MaxSamplers) return false;
|
||||
mapping[type] = target;
|
||||
plan.bindings[target] = {index, static_cast<SamplerClass>(type)};
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
@@ -856,7 +865,6 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
||||
plan.memory_info = program.memory_info;
|
||||
plan.srt_plan_complete = program.srt_plan_complete;
|
||||
plan.resource_tracking_complete = program.resource_tracking_complete;
|
||||
plan.has_address_writes = program.has_address_writes;
|
||||
|
||||
std::unordered_map<const Inst*, Inst*> cloned;
|
||||
std::function<Value(Value)> Clone = [&](Value value) -> Value {
|
||||
@@ -915,17 +923,25 @@ ResourcePlan ExtractResourcePlan(const Program& program) {
|
||||
plan.uniform_fill.values[i] = Clone(plan.uniform_fill.values[i]);
|
||||
}
|
||||
plan.clean_flat_slots.resize(plan.srt_reads.size());
|
||||
bool masked_image = false;
|
||||
for (const auto& image: plan.info.images) {
|
||||
const auto* source = Source(plan, image.source);
|
||||
if (source == nullptr || !source->indirect_image.has_value()) {
|
||||
continue;
|
||||
}
|
||||
plan.requires_specialization_memory = true;
|
||||
masked_image |= !source->indirect_image->selector_mask.IsEmpty();
|
||||
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->material_source),
|
||||
plan.clean_flat_slots, source->indirect_image->selector_mask);
|
||||
MarkCleanFlatSlots(plan, Source(plan, source->indirect_image->table_source),
|
||||
plan.clean_flat_slots);
|
||||
}
|
||||
if (masked_image) {
|
||||
plan.resource_tracking_complete &= !program.has_address_writes &&
|
||||
!std::ranges::any_of(plan.info.images, &ImageResource::written);
|
||||
plan.capture_specialization_reads =
|
||||
std::ranges::any_of(plan.info.buffers, &BufferResource::written);
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
@@ -935,18 +951,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
(program.requires_specialization_memory && runtime.read_specialization_memory == nullptr)) {
|
||||
return false;
|
||||
}
|
||||
const bool masked_image = std::ranges::any_of(program.info.images, [&](const auto& image) {
|
||||
const auto* source = Source(program, image.source);
|
||||
return source != nullptr && source->indirect_image.has_value() &&
|
||||
!source->indirect_image->selector_mask.IsEmpty();
|
||||
});
|
||||
if (masked_image &&
|
||||
(program.has_address_writes ||
|
||||
std::ranges::any_of(program.info.images, &ImageResource::written))) {
|
||||
return false;
|
||||
}
|
||||
const bool capture_reads = masked_image &&
|
||||
std::ranges::any_of(program.info.buffers, &BufferResource::written);
|
||||
const bool capture_reads = program.capture_specialization_reads;
|
||||
auto& reads = program.specialization_reads;
|
||||
ReadCapture capture {runtime, reads};
|
||||
SrtRuntime observed = runtime;
|
||||
@@ -1001,10 +1006,10 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
}
|
||||
}
|
||||
snapshot.images.resize(program.info.images.size());
|
||||
specialization.images.clear();
|
||||
specialization.images.reserve(program.info.images.size());
|
||||
for (const auto& image: program.info.images) {
|
||||
specialization.images.push_back({
|
||||
specialization.images.resize(program.info.images.size());
|
||||
for (uint32_t i = 0; i < program.info.images.size(); ++i) {
|
||||
const auto& image = program.info.images[i];
|
||||
specialization.images[i] = {
|
||||
.numeric_class = image.numeric_class,
|
||||
.dimension = image.dimension,
|
||||
.mip_count = image.mip_count,
|
||||
@@ -1014,10 +1019,7 @@ bool MaterializeResources(const ResourcePlan& program, const SrtRuntime& runtime
|
||||
.indirect_mapping_offset = image.indirect_mapping_offset,
|
||||
.indirect_search_iterations = image.indirect_search_iterations,
|
||||
.cube = image.cube,
|
||||
});
|
||||
}
|
||||
for (uint32_t i = 0; i < program.info.images.size(); ++i) {
|
||||
const auto& image = program.info.images[i];
|
||||
};
|
||||
const auto* source = Source(program, image.source);
|
||||
if (source == nullptr) {
|
||||
return false;
|
||||
@@ -1102,25 +1104,18 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
|
||||
auto samplers = program.info.samplers;
|
||||
auto sampled_pairs = program.info.sampled_pairs;
|
||||
samplers.reserve(sampler_plan.sampler_count);
|
||||
for (uint32_t index = 0; index < program.info.samplers.size(); index++) {
|
||||
const auto target = sampler_plan.point_sampler[index];
|
||||
if (target == UINT32_MAX) {
|
||||
continue;
|
||||
}
|
||||
if (target == index) {
|
||||
samplers[index].force_point_filtering = true;
|
||||
} else {
|
||||
EXIT_IF(target != samplers.size());
|
||||
auto sampler = samplers[index];
|
||||
sampler.force_point_filtering = true;
|
||||
samplers.push_back(std::move(sampler));
|
||||
for (uint32_t index = 0; index < sampler_plan.sampler_count; index++) {
|
||||
const auto& binding = sampler_plan.bindings[index];
|
||||
if (index >= program.info.samplers.size()) {
|
||||
samplers.push_back(program.info.samplers[binding.source]);
|
||||
}
|
||||
samplers[index].force_point_filtering = binding.type == SamplerClass::PointInteger;
|
||||
samplers[index].integer_border = binding.type != SamplerClass::Float;
|
||||
}
|
||||
for (auto& pair: sampled_pairs) {
|
||||
if (RequiresPointSampler(images[pair.image])) {
|
||||
EXIT_IF(sampler_plan.point_sampler[pair.sampler] == UINT32_MAX);
|
||||
pair.sampler = sampler_plan.point_sampler[pair.sampler];
|
||||
}
|
||||
const auto type = static_cast<uint32_t>(ClassifySampler(images[pair.image]));
|
||||
pair.sampler = sampler_plan.mapping[pair.sampler][type];
|
||||
EXIT_IF(pair.sampler == UINT32_MAX);
|
||||
samplers[pair.sampler].depth_compare |= images[pair.image].depth_compare;
|
||||
}
|
||||
|
||||
@@ -1129,6 +1124,49 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
|
||||
for (auto* block: program.blocks) {
|
||||
for (auto it = block->begin(); it != block->end(); ++it) {
|
||||
auto& inst = *it;
|
||||
if (BufferAccessOf(inst.GetOpcode()) == BufferAccess::Read) {
|
||||
const auto& memory = memory_info[inst.Flags<MemoryFlags>().index];
|
||||
if (memory.kind == ResourceKind::Buffer &&
|
||||
specialization.buffers[memory.resource].zero_stride_oob) {
|
||||
// Bounds mode 0 checks offset >= stride, so zero stride
|
||||
// makes every vector read out of bounds regardless of its address.
|
||||
const auto count = BufferComponentCount(inst.GetOpcode());
|
||||
std::array<Value, 4> values {Value(0u), Value(0u), Value(0u), Value(0u)};
|
||||
if (memory.formatted && !memory.typed) {
|
||||
const auto& buffer = buffers[memory.resource];
|
||||
const auto format = Format::GetFormatInfo(buffer.descriptor_format);
|
||||
for (uint32_t component = 0; component < count; component++) {
|
||||
if (format.type == Format::ComponentType::Unknown ||
|
||||
GetDstSel(buffer.descriptor_swizzle, component) != 1u) continue;
|
||||
const auto one = Format::FormattedConstantBits(
|
||||
format, Format::FormattedSourceKind::One);
|
||||
values[component] = Value(&*block->PrependNewInst(
|
||||
it, ValueOpcode::SelectU32,
|
||||
{inst.Arg(inst.NumArgs() - 1), Value(one), Value(0u)}));
|
||||
}
|
||||
}
|
||||
Value result = values[0];
|
||||
switch (inst.GetType()) {
|
||||
case Type::U8: result = Value(uint8_t {0}); break;
|
||||
case Type::U16: result = Value(uint16_t {0}); break;
|
||||
case Type::U32x2:
|
||||
result = Value(&*block->PrependNewInst(it, ValueOpcode::CompositeConstructU32x2,
|
||||
{values[0], values[1]}));
|
||||
break;
|
||||
case Type::U32x3:
|
||||
result = Value(&*block->PrependNewInst(it, ValueOpcode::CompositeConstructU32x3,
|
||||
{values[0], values[1], values[2]}));
|
||||
break;
|
||||
case Type::U32x4:
|
||||
result = Value(&*block->PrependNewInst(it, ValueOpcode::CompositeConstructU32x4,
|
||||
{values[0], values[1], values[2], values[3]}));
|
||||
break;
|
||||
default: break;
|
||||
}
|
||||
inst.ReplaceUsesWith(result);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const auto image_opcode = ImageOpcodeInfoOf(inst.GetOpcode());
|
||||
if (image_opcode.access == ImageAccess::None) {
|
||||
continue;
|
||||
@@ -1155,10 +1193,11 @@ void ApplyResourceSpecialization(Program& program, const ResourceSpecialization&
|
||||
inst.ReplaceUsesWith(Value(&*result));
|
||||
continue;
|
||||
}
|
||||
if (image_opcode.needs_sampler && RequiresPointSampler(image) &&
|
||||
if (image_opcode.needs_sampler &&
|
||||
memory.sampler < program.info.samplers.size()) {
|
||||
EXIT_IF(sampler_plan.point_sampler[memory.sampler] == UINT32_MAX);
|
||||
memory.sampler = sampler_plan.point_sampler[memory.sampler];
|
||||
const auto type = static_cast<uint32_t>(ClassifySampler(image));
|
||||
memory.sampler = sampler_plan.mapping[memory.sampler][type];
|
||||
EXIT_IF(memory.sampler == UINT32_MAX);
|
||||
}
|
||||
EXIT_IF(image.indirect_root == memory.resource &&
|
||||
inst.GetOpcode() != ValueOpcode::ImageSampleRaw);
|
||||
|
||||
@@ -12,6 +12,7 @@ struct ResourceSpecialization {
|
||||
uint32_t packed_stride = 0;
|
||||
Prospero::BufferFormat descriptor_format = Prospero::BufferFormat::kInvalid;
|
||||
uint32_t descriptor_swizzle = DstSel(4, 5, 6, 7);
|
||||
bool zero_stride_oob = false;
|
||||
bool operator==(const Buffer&) const = default;
|
||||
};
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -5,9 +5,8 @@
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::IR {
|
||||
|
||||
// Collects immutable resource topology from typed SSA handles, interns their resolved dwords in
|
||||
// descriptor_sources, then writes dense indices to handle flags and MemoryInfo.
|
||||
void TrackResources(Program& program);
|
||||
// Resolves native descriptor sources, plans their scalar reads, and assigns dense resource bindings.
|
||||
void TrackResources(Program& program, const Decoder::Program& decoded, const CFG::Graph& native_cfg);
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::IR
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <fmt/format.h>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace Libs::Graphics::ShaderRecompiler::IR {
|
||||
@@ -23,21 +22,6 @@ namespace {
|
||||
|
||||
constexpr uint64_t AddressMask = 0x0000ffffffffffffull;
|
||||
|
||||
const char* StageName(ShaderType stage) {
|
||||
switch (stage) {
|
||||
case ShaderType::Vertex: return "vertex";
|
||||
case ShaderType::Pixel: return "pixel";
|
||||
case ShaderType::Fetch: return "fetch";
|
||||
case ShaderType::Compute: return "compute";
|
||||
default: return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
std::string Diagnostic(const ResourcePlan& program, uint32_t pc, const std::string& message) {
|
||||
return fmt::format("shader SRT: hash=0x{:016x} stage={} pc=0x{:08x} {}", program.shader_hash,
|
||||
StageName(program.stage), pc, message);
|
||||
}
|
||||
|
||||
bool AddSignedAddress(uint64_t base, int64_t offset, uint64_t& result) {
|
||||
if (base > AddressMask) {
|
||||
return false;
|
||||
@@ -89,6 +73,7 @@ bool IsRuntimeSelect(ValueOpcode op) {
|
||||
|
||||
bool IsRuntimeUniformOp(ValueOpcode op) {
|
||||
switch (op) {
|
||||
case ValueOpcode::ConditionRef:
|
||||
case ValueOpcode::BitCastU32F32:
|
||||
case ValueOpcode::BitCastF32U32:
|
||||
case ValueOpcode::ConvertU32F32:
|
||||
@@ -312,151 +297,6 @@ private:
|
||||
std::unordered_set<const Inst*> m_validated_dependencies;
|
||||
};
|
||||
|
||||
class PlanBuilder {
|
||||
public:
|
||||
explicit PlanBuilder(Program& program): m_program(program) {}
|
||||
|
||||
void Run() {
|
||||
m_program.srt_reads.clear();
|
||||
m_program.dynamic_reads.clear();
|
||||
for (auto* block: m_program.blocks) {
|
||||
for (auto& inst: *block) {
|
||||
const auto op = inst.GetOpcode();
|
||||
if (op == ValueOpcode::LoadAddressU32 || op == ValueOpcode::ReadConstBuffer) {
|
||||
const auto flags = inst.Flags<MemoryFlags>();
|
||||
if (flags.index < m_program.memory_info.size()) {
|
||||
const auto kind = m_program.memory_info[flags.index].kind;
|
||||
const bool crosswired = (op == ValueOpcode::LoadAddressU32 &&
|
||||
kind == ResourceKind::ScalarBuffer) ||
|
||||
(op == ValueOpcode::ReadConstBuffer &&
|
||||
kind == ResourceKind::ScalarAddress);
|
||||
if (crosswired) {
|
||||
Fail(flags.pc,
|
||||
fmt::format("{} has incompatible scalar memory metadata",
|
||||
ValueOpcodeName(op)));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (IsDescriptorHandle(inst.GetOpcode())) {
|
||||
for (size_t index = 0; index < inst.NumArgs(); index++) {
|
||||
Collect(inst.Arg(index), 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto* block: m_program.blocks) {
|
||||
for (auto& inst: *block) {
|
||||
if (inst.GetOpcode() == ValueOpcode::LoadAddressU32 && IsRawRead(m_program, inst) &&
|
||||
inst.Arg(1).Resolve().IsImmediate() &&
|
||||
ValidateRuntimeValue(m_program, Value(&inst))) {
|
||||
Collect(Value(&inst), inst.Flags<MemoryFlags>().pc);
|
||||
}
|
||||
}
|
||||
}
|
||||
PatchReads();
|
||||
}
|
||||
|
||||
private:
|
||||
struct Patch {
|
||||
Inst* inst = nullptr;
|
||||
uint32_t slot = 0;
|
||||
bool keep = false;
|
||||
};
|
||||
|
||||
[[noreturn]] void Fail(uint32_t pc, const std::string& message) const {
|
||||
const auto diagnostic = Diagnostic(m_program, pc, message);
|
||||
EXIT("shader SRT planning failed: %s", diagnostic.c_str());
|
||||
std::abort();
|
||||
}
|
||||
|
||||
void Collect(Value value, uint32_t use_pc) {
|
||||
value = value.Resolve();
|
||||
if (value.IsImmediate()) {
|
||||
return;
|
||||
}
|
||||
auto* inst = value.TryInstruction();
|
||||
if (inst == nullptr) {
|
||||
Fail(use_pc, "invalid typed planning value");
|
||||
}
|
||||
const auto cycle = std::ranges::find(m_visiting, inst);
|
||||
if (cycle != m_visiting.end()) {
|
||||
const auto contains_phi = std::any_of(cycle, m_visiting.end(), [](const Inst* value) {
|
||||
return value->GetOpcode() == ValueOpcode::Phi;
|
||||
});
|
||||
if (contains_phi) {
|
||||
return;
|
||||
}
|
||||
Fail(use_pc, fmt::format("cyclic typed planning value {} without a phi",
|
||||
ValueOpcodeName(inst->GetOpcode())));
|
||||
}
|
||||
if (std::ranges::find(m_visited, inst) != m_visited.end()) {
|
||||
return;
|
||||
}
|
||||
m_visiting.push_back(inst);
|
||||
for (size_t index = 0; index < inst->NumArgs(); index++) {
|
||||
Collect(inst->Arg(index), use_pc);
|
||||
}
|
||||
m_visiting.pop_back();
|
||||
m_visited.push_back(inst);
|
||||
if (!IsRawRead(m_program, *inst)) {
|
||||
return;
|
||||
}
|
||||
const auto offset = inst->Arg(1).Resolve();
|
||||
if (!offset.IsImmediate() || offset.GetType() != Type::U32) {
|
||||
if (std::ranges::find(m_program.dynamic_reads, value) ==
|
||||
m_program.dynamic_reads.end()) {
|
||||
m_program.dynamic_reads.push_back(value);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (uint32_t slot = 0; slot < m_program.srt_reads.size(); slot++) {
|
||||
if (EquivalentValue(m_program, value, m_program.srt_reads[slot].value)) {
|
||||
m_patches.push_back({inst, slot, false});
|
||||
return;
|
||||
}
|
||||
}
|
||||
const auto slot = static_cast<uint32_t>(m_program.srt_reads.size());
|
||||
m_program.srt_reads.push_back({value, slot});
|
||||
m_patches.push_back({inst, slot, true});
|
||||
}
|
||||
|
||||
void PatchReads() {
|
||||
for (const auto& patch: m_patches) {
|
||||
auto* block = patch.inst->Parent();
|
||||
auto& list = block->Instructions();
|
||||
auto where =
|
||||
std::ranges::find_if(list, [&](const Inst& inst) { return &inst == patch.inst; });
|
||||
const auto resource =
|
||||
Value(&*block->PrependNewInst(where, ValueOpcode::GetSrtResource));
|
||||
const auto flat = Value(&*block->PrependNewInst(where, ValueOpcode::ReadConst,
|
||||
{resource, Value(patch.slot)}));
|
||||
const auto uses = patch.inst->Uses();
|
||||
for (const auto& use: uses) {
|
||||
use.user->SetArg(use.operand, flat);
|
||||
}
|
||||
for (auto& info: m_program.block_info) {
|
||||
if (info.condition.Resolve() == Value(patch.inst)) {
|
||||
info.condition = flat;
|
||||
}
|
||||
if (info.indirect_target.Resolve() == Value(patch.inst)) {
|
||||
info.indirect_target = flat;
|
||||
}
|
||||
}
|
||||
if (patch.keep) {
|
||||
const auto memory = patch.inst->Flags<MemoryFlags>().index;
|
||||
if (memory < m_program.memory_info.size()) {
|
||||
m_program.memory_info[memory].planning_only = true;
|
||||
}
|
||||
block->AppendNewInst(ValueOpcode::ReferenceU32, {Value(patch.inst)});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Program& m_program;
|
||||
std::vector<Inst*> m_visiting;
|
||||
std::vector<Inst*> m_visited;
|
||||
std::vector<Patch> m_patches;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -616,16 +456,15 @@ bool SrtWalker::EvaluateRawRead(const Inst& inst, uint64_t& result) {
|
||||
return false;
|
||||
}
|
||||
const auto byte_offset =
|
||||
static_cast<uint64_t>(immediate) + static_cast<uint32_t>(offset);
|
||||
const auto aligned = byte_offset & ~uint64_t {3};
|
||||
(static_cast<uint64_t>(immediate) & ~uint64_t {3}) + (static_cast<uint32_t>(offset) & ~3u);
|
||||
const auto stride = (static_cast<uint32_t>(high) >> 16u) & 0x3fffu;
|
||||
const auto size = stride == 0u
|
||||
? static_cast<uint64_t>(static_cast<uint32_t>(records))
|
||||
: static_cast<uint64_t>(stride) * static_cast<uint32_t>(records);
|
||||
if (aligned > size || size - aligned < sizeof(uint32_t)) {
|
||||
if (byte_offset > size || size - byte_offset < sizeof(uint32_t)) {
|
||||
return false;
|
||||
}
|
||||
address = ((base & ~uint64_t {3}) + byte_offset) & ~uint64_t {3};
|
||||
address = (base & ~uint64_t {3}) + byte_offset;
|
||||
} else {
|
||||
const auto relative = (immediate & ~int64_t {3}) +
|
||||
static_cast<int64_t>(static_cast<uint32_t>(offset) & ~3u);
|
||||
@@ -780,14 +619,18 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::FPOrdLessThanEqual32:
|
||||
if (binary()) {
|
||||
result = Float32(a) <= Float32(b);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::FPOrdGreaterThanEqual32:
|
||||
if (binary()) {
|
||||
result = Float32(a) >= Float32(b);
|
||||
const auto operand = [&](uint64_t bits) {
|
||||
if (inst.Flags<FPCompareFlags>().flush_input_denorms &&
|
||||
(bits & 0x7fffffffu) < 0x00800000u) {
|
||||
bits &= 0x80000000u;
|
||||
}
|
||||
return Float32(bits);
|
||||
};
|
||||
result = inst.GetOpcode() == ValueOpcode::FPOrdLessThanEqual32
|
||||
? operand(a) <= operand(b)
|
||||
: operand(a) >= operand(b);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -970,6 +813,7 @@ bool SrtWalker::EvaluateInst(const Inst& inst, uint64_t& result) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
case ValueOpcode::ConditionRef: return Arg(inst, 0, result);
|
||||
case ValueOpcode::LogicalNot:
|
||||
if (Arg(inst, 0, a)) {
|
||||
result = a == 0u;
|
||||
@@ -1061,13 +905,5 @@ bool ValidateRuntimeValue(const ResourcePlan& program, Value value, RuntimeValue
|
||||
return RuntimeValidator(program, type).Run(value);
|
||||
}
|
||||
|
||||
void BuildSrtPlan(Program& program) {
|
||||
if (program.resource_tracking_complete) {
|
||||
EXIT("shader SRT planning failed: cannot rebuild SRT after resource tracking");
|
||||
}
|
||||
program.srt_plan_complete = false;
|
||||
PlanBuilder(program).Run();
|
||||
program.srt_plan_complete = true;
|
||||
}
|
||||
|
||||
} // namespace Libs::Graphics::ShaderRecompiler::IR
|
||||
@@ -21,9 +21,6 @@ struct SrtRuntime {
|
||||
|
||||
enum class RuntimeValueType { Any, Integer };
|
||||
|
||||
// Collects reachable ReadConst values. Immediate offsets receive compact flat-buffer slots;
|
||||
// dynamic offsets remain explicit and are never assigned a fake slot.
|
||||
void BuildSrtPlan(Program& program);
|
||||
bool ValidateRuntimeValue(const ResourcePlan& program, Value value,
|
||||
RuntimeValueType type = RuntimeValueType::Any);
|
||||
// Uses the strict reader for values that affect shader specialization.
|
||||
|
||||
@@ -388,6 +388,8 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
|
||||
|
||||
EXIT_IF(attrib == nullptr || buffer == nullptr);
|
||||
|
||||
const bool debug_dump = Config::GraphicsDebugDumpEnabled();
|
||||
|
||||
for (uint32_t i = 0; i < num_input_semantics; i++) {
|
||||
const auto& in = input_semantics[i];
|
||||
|
||||
@@ -396,7 +398,7 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
|
||||
uint32_t reg = in.hardware_mapping;
|
||||
uint32_t size = in.size_in_elements;
|
||||
|
||||
if (Config::GraphicsDebugDumpEnabled()) {
|
||||
if (debug_dump) {
|
||||
LOGF("reg = %u, size = %u, va[%u] = 0x%08" PRIx32 "\n", reg, size, i,
|
||||
attrib[in.semantic]);
|
||||
}
|
||||
@@ -407,16 +409,6 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
|
||||
uint32_t offset = (attrib[in.semantic] >> 14u) & 0xfffu;
|
||||
uint32_t fetch_index = (attrib[in.semantic] >> 26u) & 0x1u;
|
||||
|
||||
if (fetch_index != 0) {
|
||||
static std::atomic<uint64_t> log_count = 0;
|
||||
auto log_id = log_count.fetch_add(1);
|
||||
if (log_id < 64) {
|
||||
LOGF("\t temporary: PS5 vertex attrib semantic %u uses fetch index %u, buffer "
|
||||
"index %zu\n",
|
||||
static_cast<uint32_t>(in.semantic), fetch_index, index);
|
||||
}
|
||||
}
|
||||
|
||||
EXIT_NOT_IMPLEMENTED(index >= ShaderVertexInputInfo::RES_MAX);
|
||||
|
||||
const auto* sharp = &buffer[index * 4];
|
||||
@@ -437,14 +429,6 @@ static void ShaderApplyAttribSemantics(ShaderVertexInputInfo& info,
|
||||
const auto format_raw = static_cast<uint32_t>(format);
|
||||
const auto buffer_format = format_raw >> 2u;
|
||||
const auto channels = (format_raw & 3u) + 1u;
|
||||
static std::atomic<uint64_t> log_count = 0;
|
||||
auto log_id = log_count.fetch_add(1);
|
||||
if (log_id < 64) {
|
||||
LOGF("\t PS5 vertex attrib semantic %u uses attrib format %u -> buffer "
|
||||
"format %u, offset %u, buffer index %zu\n",
|
||||
static_cast<uint32_t>(in.semantic), static_cast<uint32_t>(format),
|
||||
static_cast<uint32_t>(buffer_format), offset, index);
|
||||
}
|
||||
// AGC vertex formats encode the buffer format above the two channel-count bits.
|
||||
// The fetch prolog selects X001, XY01, XYZ1, or XYZW from that count.
|
||||
r.fields[3] = (r.fields[3] & ~((0x7fu << 12u) | 0xfffu)) |
|
||||
@@ -638,6 +622,7 @@ static void ShaderGetStaticInputInfoCS(const HW::ComputeShaderInfo& regs,
|
||||
info.threads_num[2] = regs.cs_regs.num_thread_z;
|
||||
info.lds_size_dwords = static_cast<uint32_t>(regs.cs_regs.lds_size) * 128u;
|
||||
info.scratch_size_dwords = data.scratch_size_dwords;
|
||||
info.float_mode = regs.cs_regs.float_mode;
|
||||
info.group_id[0] = regs.cs_regs.tgid_x_en != 0;
|
||||
info.group_id[1] = regs.cs_regs.tgid_y_en != 0;
|
||||
info.group_id[2] = regs.cs_regs.tgid_z_en != 0;
|
||||
@@ -743,7 +728,7 @@ void BuildStageStaticKey(const ShaderPixelInputInfo& info, std::vector<uint32_t>
|
||||
void BuildStageStaticKey(const ShaderComputeInputInfo& info, std::vector<uint32_t>& key) {
|
||||
key.clear();
|
||||
key.push_back(info.workgroup_register);
|
||||
key.push_back(info.wave_size);
|
||||
key.push_back(info.wave_size | (static_cast<uint32_t>(info.float_mode) << 8u));
|
||||
key.push_back(info.host_subgroup_size);
|
||||
key.push_back(info.thread_ids_num);
|
||||
key.push_back(info.lds_size_dwords);
|
||||
|
||||
@@ -145,6 +145,7 @@ struct ShaderVertexInputInfo {
|
||||
};
|
||||
|
||||
struct ShaderComputeInputInfo: ShaderWorkgroupInputInfo {
|
||||
uint8_t float_mode = 0xc0;
|
||||
uint32_t dispatch_threads_num[3] = {0, 0, 0};
|
||||
bool group_id[3] = {false, false, false};
|
||||
bool dispatch_thread_dimensions = false;
|
||||
|
||||
+43
-14
@@ -1,5 +1,6 @@
|
||||
#include "kernel/fileSystem.h"
|
||||
|
||||
#include "common/archive.h"
|
||||
#include "common/assert.h"
|
||||
#include "common/common.h"
|
||||
#include "common/dateTime.h"
|
||||
@@ -41,6 +42,7 @@ public:
|
||||
struct MountPair {
|
||||
std::filesystem::path dir;
|
||||
std::string point;
|
||||
std::shared_ptr<Common::ArchiveReader> archive;
|
||||
};
|
||||
|
||||
MountPoints() { EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread()); }
|
||||
@@ -260,6 +262,7 @@ void MountPoints::Mount(const std::filesystem::path& folder, const std::string&
|
||||
MountPair p;
|
||||
p.dir = folder;
|
||||
p.point = point_str;
|
||||
p.archive = Common::OpenArchive(folder);
|
||||
|
||||
m_mount_pairs.push_back(std::move(p));
|
||||
}
|
||||
@@ -346,11 +349,13 @@ std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name)
|
||||
}
|
||||
|
||||
const auto native_rel_path = Common::PathFromUtf8(rel_path);
|
||||
if (Common::IsArchivePath(p.dir)) {
|
||||
return p.dir / native_rel_path;
|
||||
}
|
||||
|
||||
#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
|
||||
if (HasWindowsForbiddenFilenameCharacter(rel_path)) {
|
||||
::printf("FileSystem: Windows-incompatible guest filename: %s\n",
|
||||
mounted_name.c_str());
|
||||
::printf("FileSystem: Windows-incompatible guest filename: %s\n", mounted_name.c_str());
|
||||
}
|
||||
return p.dir / native_rel_path;
|
||||
#else
|
||||
@@ -358,7 +363,8 @@ std::filesystem::path MountPoints::ResolvePath(const std::string& mounted_name)
|
||||
#endif
|
||||
}
|
||||
|
||||
return mounted_name;
|
||||
// Unmounted guest paths must not fall through to the host filesystem.
|
||||
return {};
|
||||
}
|
||||
|
||||
void Initialize() {
|
||||
@@ -461,14 +467,24 @@ int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode) {
|
||||
}
|
||||
|
||||
file->real_name = g_mount_points->ResolvePath(file->name);
|
||||
if (file->real_name.empty()) {
|
||||
g_files->DeleteDescriptor(descriptor);
|
||||
return KERNEL_ERROR_ENOENT;
|
||||
}
|
||||
if (Common::IsArchivePath(file->real_name) &&
|
||||
(rw_mode != Common::File::Mode::Read || trunc || creat)) {
|
||||
g_files->DeleteDescriptor(descriptor);
|
||||
return KERNEL_ERROR_EROFS;
|
||||
}
|
||||
|
||||
if (trunc && rw_mode == Common::File::Mode::Read) {
|
||||
g_files->DeleteDescriptor(descriptor);
|
||||
return KERNEL_ERROR_EACCES;
|
||||
}
|
||||
|
||||
bool dir_exist = Common::File::IsDirectoryExisting(file->real_name);
|
||||
bool file_exist = Common::File::IsFileExisting(file->real_name);
|
||||
const auto info = Common::File::GetInfo(file->real_name);
|
||||
const bool dir_exist = info && !info->is_file;
|
||||
const bool file_exist = info && info->is_file;
|
||||
|
||||
// A missing path opened without O_CREAT is ENOENT
|
||||
if (!creat && !dir_exist && !file_exist) {
|
||||
@@ -666,7 +682,7 @@ int64_t KYTY_SYSV_ABI KernelWrite(int d, const void* buf, size_t nbytes) {
|
||||
|
||||
auto* file = g_files->GetFile(d);
|
||||
|
||||
if (file == nullptr) {
|
||||
if (file == nullptr || !file->opened || !file->writable) {
|
||||
return KERNEL_ERROR_EBADF;
|
||||
}
|
||||
|
||||
@@ -881,7 +897,7 @@ int64_t KYTY_SYSV_ABI KernelPwrite(int d, const void* buf, size_t nbytes, int64_
|
||||
|
||||
auto* file = g_files->GetFile(d);
|
||||
|
||||
if (file == nullptr) {
|
||||
if (file == nullptr || !file->opened || !file->writable) {
|
||||
return KERNEL_ERROR_EBADF;
|
||||
}
|
||||
|
||||
@@ -1058,15 +1074,13 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
|
||||
|
||||
auto real_file_name = g_mount_points->ResolvePath(path);
|
||||
|
||||
bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
|
||||
bool is_file = Common::File::IsFileExisting(real_file_name);
|
||||
|
||||
if (!is_dir && !is_file) {
|
||||
const auto info = Common::File::GetInfo(real_file_name);
|
||||
if (!info) {
|
||||
LOGF("\t file not found\n");
|
||||
return KERNEL_ERROR_ENOENT;
|
||||
}
|
||||
|
||||
EXIT_NOT_IMPLEMENTED(is_dir && is_file);
|
||||
const bool is_dir = !info->is_file;
|
||||
|
||||
FileStat stat {};
|
||||
stat.st_mode = 0000777u | (is_dir ? 0040000u : 0100000u);
|
||||
@@ -1080,7 +1094,7 @@ int KYTY_SYSV_ABI KernelStat(const char* path, FileStat* sb) {
|
||||
stat.st_blksize = 512;
|
||||
stat.st_blocks = stat.st_size / 512;
|
||||
} else {
|
||||
stat.st_size = static_cast<int64_t>(Common::File::Size(real_file_name));
|
||||
stat.st_size = static_cast<int64_t>(info->size);
|
||||
stat.st_blksize = 512;
|
||||
stat.st_blocks = (stat.st_size + 511) / 512;
|
||||
|
||||
@@ -1217,6 +1231,9 @@ int KYTY_SYSV_ABI KernelUnlink(const char* path) {
|
||||
}
|
||||
|
||||
auto real_file_name = g_mount_points->ResolvePath(path);
|
||||
if (Common::IsArchivePath(real_file_name)) {
|
||||
return KERNEL_ERROR_EROFS;
|
||||
}
|
||||
|
||||
bool is_dir = Common::File::IsDirectoryExisting(real_file_name);
|
||||
bool is_file = Common::File::IsFileExisting(real_file_name);
|
||||
@@ -1254,8 +1271,11 @@ int KYTY_SYSV_ABI KernelRename(const char* from, const char* to) {
|
||||
auto to_path = std::string(to);
|
||||
auto real_from = g_mount_points->ResolvePath(from_path);
|
||||
auto real_to = g_mount_points->ResolvePath(to_path);
|
||||
if (Common::IsArchivePath(real_from) || Common::IsArchivePath(real_to)) {
|
||||
return KERNEL_ERROR_EROFS;
|
||||
}
|
||||
|
||||
if (!Common::File::IsFileExisting(real_from)) {
|
||||
if (real_to.empty() || !Common::File::IsFileExisting(real_from)) {
|
||||
return KERNEL_ERROR_ENOENT;
|
||||
}
|
||||
|
||||
@@ -1335,6 +1355,12 @@ int KYTY_SYSV_ABI KernelMkdir(const char* path, uint16_t mode) {
|
||||
path, mode);
|
||||
|
||||
auto real_name = g_mount_points->ResolvePath(std::string(path));
|
||||
if (real_name.empty()) {
|
||||
return KERNEL_ERROR_ENOENT;
|
||||
}
|
||||
if (Common::IsArchivePath(real_name)) {
|
||||
return KERNEL_ERROR_EROFS;
|
||||
}
|
||||
|
||||
if (Common::File::IsDirectoryExisting(real_name)) {
|
||||
return KERNEL_ERROR_EEXIST;
|
||||
@@ -1361,6 +1387,9 @@ int KYTY_SYSV_ABI KernelRmdir(const char* path) {
|
||||
LOGF("\t path = %s\n", path);
|
||||
|
||||
auto real_name = g_mount_points->ResolvePath(std::string(path));
|
||||
if (Common::IsArchivePath(real_name)) {
|
||||
return KERNEL_ERROR_EROFS;
|
||||
}
|
||||
|
||||
if (!Common::File::IsDirectoryExisting(real_name)) {
|
||||
return KERNEL_ERROR_ENOENT;
|
||||
|
||||
@@ -54,6 +54,7 @@ struct Lifecycle {
|
||||
|
||||
void Mount(const std::filesystem::path& folder, const std::string& point);
|
||||
void Umount(const std::string& folder_or_point);
|
||||
// Returns an empty path when the guest path has no mounted filesystem.
|
||||
std::filesystem::path GetRealFilename(const std::string& mounted_file_name);
|
||||
|
||||
int KYTY_SYSV_ABI KernelOpen(const char* path, int flags, uint16_t mode);
|
||||
|
||||
+182
-69
@@ -78,8 +78,7 @@ static Graphics::RenderContext& GetGpuResources() {
|
||||
}
|
||||
|
||||
static bool IsGpuAddressRange(uint64_t vaddr, uint64_t size) {
|
||||
constexpr uint64_t GPU_ADDRESS_LIMIT = 1ull << 40u;
|
||||
return vaddr != 0 && size != 0 && vaddr < GPU_ADDRESS_LIMIT && size < GPU_ADDRESS_LIMIT - vaddr;
|
||||
return Graphics::GuestRange {vaddr, size}.Valid();
|
||||
}
|
||||
|
||||
static void MapGpuRange(uint64_t vaddr, uint64_t size) {
|
||||
@@ -660,6 +659,9 @@ static bool g_test_fail_next_fixed_reserve_range_add = false;
|
||||
|
||||
class PhysicalMemory {
|
||||
public:
|
||||
enum class AllocationKind { Direct, Pooled, Automatic };
|
||||
using PhysicalRanges = std::vector<std::pair<uint64_t, uint64_t>>;
|
||||
|
||||
struct AllocatedBlock {
|
||||
uint64_t start_addr;
|
||||
uint64_t size;
|
||||
@@ -671,7 +673,7 @@ public:
|
||||
VirtualMemory::Mode mode;
|
||||
GpuAccessMode gpu_mode;
|
||||
int memory_type;
|
||||
bool pool_expansion;
|
||||
AllocationKind kind;
|
||||
char name[KERNEL_MAXIMUM_NAME_LENGTH];
|
||||
};
|
||||
|
||||
@@ -690,7 +692,10 @@ public:
|
||||
}
|
||||
|
||||
bool Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment,
|
||||
uint64_t* phys_addr_out, int memory_type, bool pool_expansion = false);
|
||||
uint64_t* phys_addr_out, int memory_type,
|
||||
AllocationKind kind = AllocationKind::Direct);
|
||||
bool ReserveAutomatic(uint64_t size, PhysicalRanges* ranges);
|
||||
void RestoreAutomatic(const PhysicalRanges& ranges);
|
||||
bool Available(uint64_t search_start, uint64_t search_end, size_t alignment,
|
||||
uint64_t* phys_addr_out, uint64_t* size_out);
|
||||
bool Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size,
|
||||
@@ -718,12 +723,14 @@ public:
|
||||
[[nodiscard]] const std::vector<AllocatedBlock>& GetMappings() const { return m_mappings; }
|
||||
|
||||
private:
|
||||
void ConsumeFreeRange(std::map<uint64_t, uint64_t>::iterator range, uint64_t start,
|
||||
uint64_t size);
|
||||
void AddFreeRange(uint64_t start, uint64_t size);
|
||||
static void RemoveFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
|
||||
uint64_t size);
|
||||
static void AddFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start, uint64_t size);
|
||||
void ReclaimAutomatic(uint64_t start, uint64_t size);
|
||||
|
||||
std::map<uint64_t, AllocatedBlock> m_physical;
|
||||
std::map<uint64_t, uint64_t> m_free;
|
||||
std::map<uint64_t, uint64_t> m_automatic_free;
|
||||
std::vector<AllocatedBlock> m_mappings;
|
||||
Common::Mutex m_mutex;
|
||||
};
|
||||
@@ -1093,7 +1100,7 @@ void SetFlexibleMemorySize(uint64_t size) {
|
||||
}
|
||||
|
||||
bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t len, size_t alignment,
|
||||
uint64_t* phys_addr_out, int memory_type, bool pool_expansion) {
|
||||
uint64_t* phys_addr_out, int memory_type, AllocationKind kind) {
|
||||
if (phys_addr_out == nullptr) {
|
||||
return false;
|
||||
}
|
||||
@@ -1128,10 +1135,13 @@ bool PhysicalMemory::Alloc(uint64_t search_start, uint64_t search_end, size_t le
|
||||
b.prot = 0;
|
||||
b.mode = VirtualMemory::Mode::NoAccess;
|
||||
b.memory_type = memory_type;
|
||||
b.pool_expansion = pool_expansion;
|
||||
b.kind = kind;
|
||||
|
||||
ConsumeFreeRange(range, free_pos, len);
|
||||
RemoveFreeRange(m_free, free_pos, len);
|
||||
EXIT_IF(!m_physical.emplace(b.start_addr, b).second);
|
||||
if (kind == AllocationKind::Automatic) {
|
||||
AddFreeRange(m_automatic_free, free_pos, len);
|
||||
}
|
||||
|
||||
*phys_addr_out = free_pos;
|
||||
return true;
|
||||
@@ -1180,35 +1190,99 @@ bool PhysicalMemory::Available(uint64_t search_start, uint64_t search_end, size_
|
||||
return true;
|
||||
}
|
||||
|
||||
void PhysicalMemory::ConsumeFreeRange(std::map<uint64_t, uint64_t>::iterator range, uint64_t start,
|
||||
uint64_t size) {
|
||||
const auto range_start = range->first;
|
||||
const auto range_end = range->first + range->second;
|
||||
m_free.erase(range);
|
||||
if (range_start < start) {
|
||||
m_free.emplace(range_start, start - range_start);
|
||||
void PhysicalMemory::RemoveFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
|
||||
uint64_t size) {
|
||||
const auto end = start + size;
|
||||
auto range = ranges.upper_bound(start);
|
||||
if (range != ranges.begin()) {
|
||||
--range;
|
||||
}
|
||||
if (start + size < range_end) {
|
||||
m_free.emplace(start + size, range_end - start - size);
|
||||
while (range != ranges.end() && range->first < end) {
|
||||
const auto range_start = range->first;
|
||||
const auto range_end = range_start + range->second;
|
||||
if (range_end <= start) {
|
||||
++range;
|
||||
continue;
|
||||
}
|
||||
range = ranges.erase(range);
|
||||
if (range_start < start) {
|
||||
ranges.emplace(range_start, start - range_start);
|
||||
}
|
||||
if (end < range_end) {
|
||||
ranges.emplace(end, range_end - end);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void PhysicalMemory::AddFreeRange(uint64_t start, uint64_t size) {
|
||||
void PhysicalMemory::AddFreeRange(std::map<uint64_t, uint64_t>& ranges, uint64_t start,
|
||||
uint64_t size) {
|
||||
auto end = start + size;
|
||||
auto next = m_free.lower_bound(start);
|
||||
if (next != m_free.begin()) {
|
||||
auto next = ranges.lower_bound(start);
|
||||
if (next != ranges.begin()) {
|
||||
auto previous = std::prev(next);
|
||||
if (previous->first + previous->second >= start) {
|
||||
start = previous->first;
|
||||
end = std::max(end, previous->first + previous->second);
|
||||
next = m_free.erase(previous);
|
||||
next = ranges.erase(previous);
|
||||
}
|
||||
}
|
||||
while (next != m_free.end() && next->first <= end) {
|
||||
while (next != ranges.end() && next->first <= end) {
|
||||
end = std::max(end, next->first + next->second);
|
||||
next = m_free.erase(next);
|
||||
next = ranges.erase(next);
|
||||
}
|
||||
ranges.emplace(start, end - start);
|
||||
}
|
||||
|
||||
bool PhysicalMemory::ReserveAutomatic(uint64_t size, PhysicalRanges* ranges) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
for (const auto& [start, available]: m_automatic_free) {
|
||||
const auto length = std::min(size, available);
|
||||
ranges->emplace_back(start, length);
|
||||
size -= length;
|
||||
if (size == 0) {
|
||||
for (const auto& [offset, length]: *ranges) {
|
||||
RemoveFreeRange(m_automatic_free, offset, length);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
ranges->clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
void PhysicalMemory::RestoreAutomatic(const PhysicalRanges& ranges) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
for (const auto& [start, size]: ranges) {
|
||||
AddFreeRange(m_automatic_free, start, size);
|
||||
}
|
||||
}
|
||||
|
||||
void PhysicalMemory::ReclaimAutomatic(uint64_t start, uint64_t size) {
|
||||
const auto end = start + size;
|
||||
auto owner = m_physical.upper_bound(start);
|
||||
if (owner != m_physical.begin()) {
|
||||
--owner;
|
||||
}
|
||||
bool automatic = false;
|
||||
for (; owner != m_physical.end() && owner->first < end; ++owner) {
|
||||
const auto& block = owner->second;
|
||||
if (block.kind != AllocationKind::Automatic || block.start_addr + block.size <= start) {
|
||||
continue;
|
||||
}
|
||||
const auto first = std::max(start, block.start_addr);
|
||||
const auto last = std::min(end, block.start_addr + block.size);
|
||||
AddFreeRange(m_automatic_free, first, last - first);
|
||||
automatic = true;
|
||||
}
|
||||
if (automatic) {
|
||||
for (const auto& mapping: m_mappings) {
|
||||
const auto first = std::max(start, mapping.start_addr);
|
||||
const auto last = std::min(end, mapping.start_addr + mapping.size);
|
||||
if (first < last) {
|
||||
RemoveFreeRange(m_automatic_free, first, last - first);
|
||||
}
|
||||
}
|
||||
}
|
||||
m_free.emplace(start, end - start);
|
||||
}
|
||||
|
||||
bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64_t* size,
|
||||
@@ -1225,18 +1299,22 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
|
||||
}
|
||||
auto it = std::prev(next);
|
||||
auto& b = it->second;
|
||||
if (b.pool_expansion || start < b.start_addr || start >= b.start_addr + b.size ||
|
||||
len > b.start_addr + b.size - start) {
|
||||
if (b.kind == AllocationKind::Pooled || start < b.start_addr ||
|
||||
start >= b.start_addr + b.size || len > b.start_addr + b.size - start) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (b.kind == AllocationKind::Automatic) {
|
||||
RemoveFreeRange(m_automatic_free, start, len);
|
||||
}
|
||||
|
||||
if (start == b.start_addr && len == b.size) {
|
||||
*vaddr = b.map_vaddr;
|
||||
*size = b.map_size;
|
||||
*gpu_mode = b.gpu_mode;
|
||||
|
||||
m_physical.erase(it);
|
||||
AddFreeRange(start, len);
|
||||
AddFreeRange(m_free, start, len);
|
||||
return true;
|
||||
}
|
||||
if (start > b.start_addr && start + len < b.start_addr + b.size) {
|
||||
@@ -1261,7 +1339,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
|
||||
}
|
||||
|
||||
m_physical.emplace(right.start_addr, right);
|
||||
AddFreeRange(start, len);
|
||||
AddFreeRange(m_free, start, len);
|
||||
return true;
|
||||
}
|
||||
if (start == b.start_addr && len < b.size) {
|
||||
@@ -1278,7 +1356,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
|
||||
remaining.map_size -= len;
|
||||
}
|
||||
m_physical.emplace(remaining.start_addr, remaining);
|
||||
AddFreeRange(start, len);
|
||||
AddFreeRange(m_free, start, len);
|
||||
return true;
|
||||
}
|
||||
if (start > b.start_addr && start + len == b.start_addr + b.size) {
|
||||
@@ -1290,7 +1368,7 @@ bool PhysicalMemory::Release(uint64_t start, size_t len, uint64_t* vaddr, uint64
|
||||
if (b.map_vaddr != 0) {
|
||||
b.map_size = b.size;
|
||||
}
|
||||
AddFreeRange(start, len);
|
||||
AddFreeRange(m_free, start, len);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1311,6 +1389,7 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
|
||||
return false;
|
||||
}
|
||||
auto first = std::prev(next);
|
||||
bool has_automatic = false;
|
||||
while (current < phys_addr + len) {
|
||||
auto block = m_physical.upper_bound(current);
|
||||
if (block == m_physical.begin()) {
|
||||
@@ -1318,10 +1397,11 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
|
||||
}
|
||||
--block;
|
||||
const auto block_end = block->second.start_addr + block->second.size;
|
||||
if (block->second.pool_expansion || current < block->second.start_addr ||
|
||||
if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
|
||||
current >= block_end) {
|
||||
return false;
|
||||
}
|
||||
has_automatic |= block->second.kind == AllocationKind::Automatic;
|
||||
current = std::min<uint64_t>(phys_addr + len, block_end);
|
||||
if (current < phys_addr + len) {
|
||||
const auto following = std::next(block);
|
||||
@@ -1342,6 +1422,9 @@ bool PhysicalMemory::Map(uint64_t vaddr, uint64_t phys_addr, size_t len, int pro
|
||||
mapping.mode = mode;
|
||||
mapping.gpu_mode = gpu_mode;
|
||||
m_mappings.push_back(mapping);
|
||||
if (has_automatic) {
|
||||
RemoveFreeRange(m_automatic_free, phys_addr, len);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -1362,7 +1445,7 @@ bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
|
||||
}
|
||||
--block;
|
||||
const auto block_end = block->second.start_addr + block->second.size;
|
||||
if (block->second.pool_expansion || current < block->second.start_addr ||
|
||||
if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
|
||||
current >= block_end) {
|
||||
return false;
|
||||
}
|
||||
@@ -1380,11 +1463,12 @@ bool PhysicalMemory::CanMapDirect(uint64_t phys_addr, size_t len) {
|
||||
bool PhysicalMemory::ReleasePoolExpansion(uint64_t phys_addr, size_t len) {
|
||||
Common::LockGuard lock(m_mutex);
|
||||
const auto it = m_physical.find(phys_addr);
|
||||
if (it == m_physical.end() || !it->second.pool_expansion || it->second.size != len) {
|
||||
if (it == m_physical.end() || it->second.kind != AllocationKind::Pooled ||
|
||||
it->second.size != len) {
|
||||
return false;
|
||||
}
|
||||
m_physical.erase(it);
|
||||
AddFreeRange(phys_addr, len);
|
||||
AddFreeRange(m_free, phys_addr, len);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1407,7 +1491,7 @@ bool PhysicalMemory::GetAllocatedSpan(uint64_t phys_addr, size_t len,
|
||||
}
|
||||
--block;
|
||||
const auto block_end = block->second.start_addr + block->second.size;
|
||||
if (block->second.pool_expansion || current < block->second.start_addr ||
|
||||
if (block->second.kind == AllocationKind::Pooled || current < block->second.start_addr ||
|
||||
current >= block_end) {
|
||||
blocks->clear();
|
||||
return false;
|
||||
@@ -1463,49 +1547,39 @@ bool PhysicalMemory::Unmap(uint64_t vaddr, uint64_t size, GpuAccessMode* gpu_mod
|
||||
}
|
||||
};
|
||||
|
||||
size_t index = 0;
|
||||
for (auto& b: m_mappings) {
|
||||
if (b.map_vaddr == vaddr && b.map_size == size) {
|
||||
*gpu_mode = b.gpu_mode;
|
||||
const auto host_vaddr = b.host_vaddr;
|
||||
const auto host_size = b.host_size;
|
||||
|
||||
for (size_t index = 0; index < m_mappings.size(); ++index) {
|
||||
auto& b = m_mappings[index];
|
||||
if (vaddr < b.map_vaddr || vaddr >= b.map_vaddr + b.map_size ||
|
||||
size > b.map_vaddr + b.map_size - vaddr) {
|
||||
continue;
|
||||
}
|
||||
*gpu_mode = b.gpu_mode;
|
||||
const auto phys_addr = b.start_addr + vaddr - b.map_vaddr;
|
||||
const auto host_vaddr = b.host_vaddr;
|
||||
const auto host_size = b.host_size;
|
||||
if (vaddr == b.map_vaddr && size == b.map_size) {
|
||||
m_mappings.erase(m_mappings.begin() + static_cast<std::ptrdiff_t>(index));
|
||||
set_host_release_if_last(host_vaddr, host_size);
|
||||
|
||||
return true;
|
||||
}
|
||||
if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) {
|
||||
*gpu_mode = b.gpu_mode;
|
||||
|
||||
} else if (vaddr > b.map_vaddr && vaddr + size < b.map_vaddr + b.map_size) {
|
||||
AllocatedBlock right = b;
|
||||
right.start_addr += (vaddr + size) - b.map_vaddr;
|
||||
right.size = b.map_vaddr + b.map_size - (vaddr + size);
|
||||
right.map_size = right.size;
|
||||
right.map_vaddr = vaddr + size;
|
||||
|
||||
b.size = vaddr - b.map_vaddr;
|
||||
b.map_size = b.size;
|
||||
m_mappings.push_back(right);
|
||||
return true;
|
||||
}
|
||||
if (vaddr == b.map_vaddr && size < b.map_size) {
|
||||
*gpu_mode = b.gpu_mode;
|
||||
|
||||
} else if (vaddr == b.map_vaddr) {
|
||||
b.start_addr += size;
|
||||
b.size -= size;
|
||||
b.map_vaddr += size;
|
||||
b.map_size -= size;
|
||||
return true;
|
||||
}
|
||||
if (vaddr > b.map_vaddr && vaddr + size == b.map_vaddr + b.map_size) {
|
||||
*gpu_mode = b.gpu_mode;
|
||||
|
||||
} else {
|
||||
b.size = vaddr - b.map_vaddr;
|
||||
b.map_size = b.size;
|
||||
return true;
|
||||
}
|
||||
index++;
|
||||
ReclaimAutomatic(phys_addr, size);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
@@ -2726,10 +2800,8 @@ int KYTY_SYSV_ABI KernelDirectMemoryQuery(int64_t offset, int flags, void* info,
|
||||
return OK;
|
||||
}
|
||||
|
||||
int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len,
|
||||
size_t alignment, int memory_type,
|
||||
int64_t* phys_addr_out) {
|
||||
PRINT_NAME();
|
||||
int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len, size_t alignment,
|
||||
int memory_type, int64_t* phys_addr_out, bool automatic) {
|
||||
|
||||
std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
|
||||
|
||||
@@ -2748,7 +2820,10 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc
|
||||
}
|
||||
|
||||
uint64_t addr = 0;
|
||||
if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type)) {
|
||||
const auto kind = automatic ? PhysicalMemory::AllocationKind::Automatic
|
||||
: PhysicalMemory::AllocationKind::Direct;
|
||||
if (!g_physical_memory->Alloc(search_start, search_end, len, alignment, &addr, memory_type,
|
||||
kind)) {
|
||||
LOGF_COLOR(Log::Color::Red, "\t[Fail]\n");
|
||||
return KERNEL_ERROR_EAGAIN;
|
||||
}
|
||||
@@ -2760,6 +2835,14 @@ int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t searc
|
||||
return OK;
|
||||
}
|
||||
|
||||
int KYTY_SYSV_ABI KernelAllocateDirectMemory(int64_t search_start, int64_t search_end, size_t len,
|
||||
size_t alignment, int memory_type,
|
||||
int64_t* phys_addr_out) {
|
||||
PRINT_NAME();
|
||||
return AllocateDirectMemory(search_start, search_end, len, alignment, memory_type,
|
||||
phys_addr_out);
|
||||
}
|
||||
|
||||
int KYTY_SYSV_ABI KernelAllocateMainDirectMemory(size_t len, size_t alignment, int memory_type,
|
||||
int64_t* phys_addr_out) {
|
||||
PRINT_NAME();
|
||||
@@ -3082,6 +3165,36 @@ int KYTY_SYSV_ABI KernelMapDirectMemory2(void** addr, size_t len, int type, int
|
||||
return ret;
|
||||
}
|
||||
|
||||
int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot) {
|
||||
std::lock_guard<std::recursive_mutex> memory_operation_lock(g_memory_operation_mutex);
|
||||
constexpr uint64_t page_size = 0x4000;
|
||||
if (size == 0 || (vaddr & (page_size - 1)) != 0 || (size & (page_size - 1)) != 0 ||
|
||||
vaddr < kExtendedMemoryBase || vaddr >= kExtendedMemoryBase + kExtendedMemorySize ||
|
||||
size > kExtendedMemoryBase + kExtendedMemorySize - vaddr) {
|
||||
return KERNEL_ERROR_EINVAL;
|
||||
}
|
||||
|
||||
PhysicalMemory::PhysicalRanges ranges;
|
||||
if (!g_physical_memory->ReserveAutomatic(size, &ranges)) {
|
||||
return KERNEL_ERROR_EAGAIN;
|
||||
}
|
||||
uint64_t mapped = 0;
|
||||
for (const auto& [offset, length]: ranges) {
|
||||
void* address = reinterpret_cast<void*>(vaddr + mapped);
|
||||
const int result = KernelMapDirectMemory2(&address, length, type, prot, 0x10,
|
||||
static_cast<int64_t>(offset), page_size);
|
||||
if (result != OK) {
|
||||
if (mapped != 0) {
|
||||
EXIT_IF(KernelMunmap(vaddr, mapped) != OK);
|
||||
}
|
||||
g_physical_memory->RestoreAutomatic(ranges);
|
||||
return result;
|
||||
}
|
||||
mapped += length;
|
||||
}
|
||||
return OK;
|
||||
}
|
||||
|
||||
int KYTY_SYSV_ABI KernelMapNamedDirectMemory(void** addr, size_t len, int prot, int flags,
|
||||
int64_t direct_memory_start, size_t alignment,
|
||||
const char* name) {
|
||||
@@ -3892,7 +4005,7 @@ int KYTY_SYSV_ABI KernelMemoryPoolExpand(int64_t search_start, int64_t search_en
|
||||
uint64_t phys_addr = 0;
|
||||
if (!g_physical_memory->Alloc(static_cast<uint64_t>(search_start),
|
||||
static_cast<uint64_t>(search_end), len, effective_alignment,
|
||||
&phys_addr, 0, true)) {
|
||||
&phys_addr, 0, PhysicalMemory::AllocationKind::Pooled)) {
|
||||
return KERNEL_ERROR_ENOMEM;
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,8 @@ struct Lifecycle {
|
||||
using callback_func_t = void (*)(uintptr_t addr, size_t size);
|
||||
|
||||
constexpr uint32_t KERNEL_MAXIMUM_NAME_LENGTH = 32;
|
||||
constexpr uint64_t kExtendedMemoryBase = 0x080000000000ull;
|
||||
constexpr uint64_t kExtendedMemorySize = 512ull * 1024 * 1024 * 1024;
|
||||
|
||||
struct VirtualQueryInfo {
|
||||
uintptr_t start;
|
||||
@@ -107,6 +109,9 @@ static_assert(sizeof(KernelMemoryPoolBlockStats) == 16,
|
||||
|
||||
void RegisterCallbacks(callback_func_t alloc_func, callback_func_t free_func);
|
||||
void SetFlexibleMemorySize(uint64_t size);
|
||||
int AllocateDirectMemory(int64_t search_start, int64_t search_end, size_t size, size_t alignment,
|
||||
int memory_type, int64_t* phys_addr_out, bool automatic = false);
|
||||
int MapAutomaticMemory(uint64_t vaddr, size_t size, int type, int prot);
|
||||
bool TryWriteBacking(uint64_t vaddr, const void* data, uint64_t size);
|
||||
bool TryReadBacking(uint64_t vaddr, void* data, uint64_t size);
|
||||
bool TryReadGpuCleanBacking(uint64_t vaddr, void* data, uint64_t size);
|
||||
|
||||
Loaded 100 of 164 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user