fix(windows): add experimental guest red-zone protection.

Port the static instruction patcher from shadPS4.

3rdparty/CMakeLists.txt

@@ -2,6 +2,32 @@ set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
 
 add_subdirectory(cpuinfo)
 
+# The Windows guest red-zone patcher needs a decoder/encoder and a small
+# trampoline assembler. Pin the same revisions used by shadPS4's implementation.
+include(FetchContent)
+
+set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+FetchContent_Declare(
+	xbyak
+	GIT_REPOSITORY https://github.com/herumi/xbyak.git
+	GIT_TAG 44a72f369268f7d552650891b296693e91db86bb
+	GIT_SHALLOW FALSE
+)
+
+set(ZYDIS_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(ZYDIS_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
+set(ZYDIS_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+set(ZYDIS_BUILD_DOXYGEN OFF CACHE BOOL "" FORCE)
+FetchContent_Declare(
+	zydis
+	GIT_REPOSITORY https://github.com/zyantific/zydis.git
+	GIT_TAG 120e0e705f8e3b507dc49377ac2879979f0d545c
+	GIT_SUBMODULES_RECURSE TRUE
+	GIT_SHALLOW FALSE
+)
+
+FetchContent_MakeAvailable(xbyak zydis)
+
 set(SDL_SHARED OFF CACHE BOOL "" FORCE)
 add_subdirectory(SDL2)
 

CMakeLists.txt

@@ -292,7 +292,7 @@ if (KYTY_CLANG_CL)
 	set_source_files_properties(${kyty_emulator_src} PROPERTIES COMPILE_FLAGS "-Wno-pragma-pack -Wno-deprecated-declarations -D_TIMESPEC_DEFINED")
 endif()
 
-set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL2-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version)
+set(kyty_emulator_link_libraries common Vulkan::Headers spirv-tools-opt spirv-tools SDL2-static imgui xxhash FFmpeg::ffmpeg fmt::fmt nlohmann_json::nlohmann_json LibAtrac9 kyty_git_version Zydis::Zydis xbyak::xbyak)
 
 # Linux system libraries required by the static FFmpeg archive.
 if(LINUX)
@@ -514,6 +514,8 @@ if(BUILD_TESTING)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
+	add_test(NAME guest_red_zone_patcher
+		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests> --red-zone-patcher-only)
 	add_test(NAME command_scheduler_timeline
 		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --scheduler-only)
 	add_test(NAME stream_buffer_ring

src/common/emulatorConfig.cpp

@@ -101,6 +101,12 @@ bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+bool RedZoneProtectionEnabled() {
+	return g_config->red_zone_protection_enabled;
+}
+#endif
+
 const Keymap& GetKeymap() {
 	return g_config->keymap;
 }

src/common/emulatorConfig.h

@@ -45,7 +45,10 @@ struct ConfigOptions {
 	bool                   spirv_debug_printf_enabled  = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
-	Keymap                 keymap;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	bool red_zone_protection_enabled = false;
+#endif
+	Keymap keymap;
 };
 
 void Load(const ConfigOptions& cfg);
@@ -76,6 +79,9 @@ bool SpirvDebugPrintfEnabled();
 
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+bool RedZoneProtectionEnabled();
+#endif
 
 const Keymap& GetKeymap();
 

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -2135,7 +2135,7 @@ KYTY_CP_OP_PARSER(CpOpClearState) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0001200);
 	EXIT_NOT_IMPLEMENTED((buffer[0] & ~0xfu) != 0);
 
-	cp.Reset();
+	cp.ApplyContextStateOperation(ContextStateOperation::Clear);
 
 	return 1;
 }

src/launcher/forms/configuration_edit_dialog.ui

@@ -88,6 +88,16 @@
            </property>
           </widget>
          </item>
+         <item row="2" column="1">
+          <widget class="QCheckBox" name="checkBox_red_zone_protection">
+           <property name="toolTip">
+            <string>Helps prevent crashes in some games on Windows</string>
+           </property>
+           <property name="text">
+            <string>Windows SysV red zone crash protection (Experimental)</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </widget>
       </item>

src/launcher/include/configuration.h

@@ -99,7 +99,10 @@ public:
 	QString                printf_output_file          = "_kyty.txt";
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   renderdoc_enabled           = false;
-	QStringList            host_input_mapping;
+#if defined(_WIN32)
+	bool red_zone_protection_enabled = false;
+#endif
+	QStringList host_input_mapping;
 
 	QString elf = QStringLiteral("eboot.bin");
 
@@ -119,7 +122,10 @@ public:
 		printf_output_file          = other.printf_output_file;
 		profiler_direction          = other.profiler_direction;
 		renderdoc_enabled           = other.renderdoc_enabled;
-		host_input_mapping          = other.host_input_mapping;
+#if defined(_WIN32)
+		red_zone_protection_enabled = other.red_zone_protection_enabled;
+#endif
+		host_input_mapping = other.host_input_mapping;
 	}
 
 	void CopyFrom(const Configuration& other) {
@@ -156,6 +162,9 @@ public:
 		KYTY_CFG_SET(printf_output_file);
 		KYTY_CFG_SET(profiler_direction);
 		KYTY_CFG_SET(renderdoc_enabled);
+#if defined(_WIN32)
+		KYTY_CFG_SET(red_zone_protection_enabled);
+#endif
 		s->setValue("host_input_mapping", host_input_mapping);
 		KYTY_CFG_SET(elf);
 	}
@@ -183,6 +192,10 @@ public:
 		KYTY_CFG_GET(printf_output_file);
 		KYTY_CFG_GET(profiler_direction);
 		KYTY_CFG_GET(renderdoc_enabled);
+#if defined(_WIN32)
+		red_zone_protection_enabled =
+		    s->value("red_zone_protection_enabled", red_zone_protection_enabled).toBool();
+#endif
 		host_input_mapping = s->value("host_input_mapping", host_input_mapping).toStringList();
 		elf                = s->value("elf", elf).toString();
 	}

src/launcher/src/configurationEditDialog.cpp

@@ -164,6 +164,11 @@ void ConfigurationEditDialog::Init(const Configuration& info) {
 	m_ui->checkBox_shader_validation->setChecked(info.shader_validation_enabled);
 	m_ui->checkBox_vulkan_validation->setChecked(info.vulkan_validation_enabled);
 	m_ui->checkBox_renderdoc_capture->setChecked(info.renderdoc_enabled);
+#if defined(_WIN32)
+	m_ui->checkBox_red_zone_protection->setChecked(info.red_zone_protection_enabled);
+#else
+	m_ui->checkBox_red_zone_protection->setVisible(false);
+#endif
 	ListInit(m_ui->comboBox_shader_optimization_type, info.shader_optimization_type);
 	ListInit(m_ui->comboBox_shader_log_direction, info.shader_log_direction);
 	m_ui->lineEdit_shader_log_folder->setText(info.shader_log_folder);
@@ -286,7 +291,10 @@ static void UpdateInfo(Configuration& info, Ui::ConfigurationEditDialog& ui) {
 	info.vulkan_validation_enabled = ui.checkBox_vulkan_validation->isChecked();
 	info.shader_validation_enabled = ui.checkBox_shader_validation->isChecked();
 	info.renderdoc_enabled         = ui.checkBox_renderdoc_capture->isChecked();
-	info.shader_optimization_type  = TextToEnum<Configuration::ShaderOptimizationType>(
+#if defined(_WIN32)
+	info.red_zone_protection_enabled = ui.checkBox_red_zone_protection->isChecked();
+#endif
+	info.shader_optimization_type = TextToEnum<Configuration::ShaderOptimizationType>(
 	    ui.comboBox_shader_optimization_type->currentText());
 	info.shader_log_direction = TextToEnum<Configuration::ShaderLogDirection>(
 	    ui.comboBox_shader_log_direction->currentText());

src/launcher/src/mainDialog.cpp

@@ -220,6 +220,11 @@ static QStringList CreateEmulatorArgs(const Configuration& info) {
 	args << "--printf-output-file" << info.printf_output_file;
 	args << "--profiler-direction" << EnumToText(info.profiler_direction);
 	args << "--spirv-debug-printf" << "false";
+#if defined(_WIN32)
+	if (info.red_zone_protection_enabled) {
+		args << "--redzone";
+	}
+#endif
 	for (const auto& binding: info.host_input_mapping) {
 		args << "--keymap" << binding;
 	}

src/libs/libKernel.cpp

@@ -516,6 +516,7 @@ static void QueuePendingSignal(Pthread thread, int signum) {
 	PthreadQueuePendingSignal(thread, signum);
 }
 
+#if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS
 static void WaitForSignalDispatch(Pthread thread, int signum) {
 	constexpr auto DISPATCH_WAIT_STEP = std::chrono::microseconds(1000);
 	constexpr auto DISPATCH_WAIT_MAX  = std::chrono::milliseconds(2);
@@ -526,6 +527,7 @@ static void WaitForSignalDispatch(Pthread thread, int signum) {
 		waited += DISPATCH_WAIT_STEP;
 	}
 }
+#endif
 
 static bool TakePendingSignal(Pthread thread, int signum) {
 	return PthreadTakePendingSignal(thread, signum);
@@ -1070,60 +1072,6 @@ static void SignalApcHandler(void* arg1, void* arg2, void* /*arg3*/, PCONTEXT co
 	}
 }
 
-static bool DispatchSignalWithSuspendedThreadContext(HANDLE target_thread, Pthread thread,
-                                                     int signum, exception_handler_func_t handler) {
-	if (target_thread == nullptr || thread == nullptr || handler == nullptr) {
-		return false;
-	}
-
-	if (!PthreadHasPendingSignal(thread, signum)) {
-		return true;
-	}
-
-	if (SuspendThread(target_thread) == static_cast<DWORD>(-1)) {
-		return false;
-	}
-
-	bool    dispatched = false;
-	CONTEXT host_ctx {};
-	host_ctx.ContextFlags = CONTEXT_FULL | CONTEXT_SEGMENTS;
-	if (GetThreadContext(target_thread, &host_ctx) != FALSE) {
-		if (TakePendingSignal(thread, signum)) {
-			auto   ctx                  = CreateSignalUcontext(&host_ctx);
-			auto   guest_context        = IsGuestCodeAddress(ctx.uc_mcontext.mc_rip);
-			HANDLE helper_thread_handle = nullptr;
-			if (guest_context) {
-				if (DuplicateHandle(GetCurrentProcess(), target_thread, GetCurrentProcess(),
-				                    &helper_thread_handle, THREAD_SUSPEND_RESUME, FALSE,
-				                    0) == FALSE) {
-					ResumeThread(target_thread);
-					return false;
-				}
-			} else {
-				SanitizeNonGuestSignalUcontext(&ctx, thread);
-				ResumeThread(target_thread);
-			}
-
-			std::thread([helper_thread_handle, thread, signum, handler, ctx]() mutable {
-				SignalDispatchScope scope;
-				auto*               previous_self = PthreadSwapSelfForSignal(thread);
-				handler(signum, &ctx);
-				PthreadSwapSelfForSignal(previous_self);
-				if (helper_thread_handle != nullptr) {
-					ResumeThread(helper_thread_handle);
-					CloseHandle(helper_thread_handle);
-				}
-			}).detach();
-			dispatched = true;
-			return true;
-		}
-		dispatched = true;
-	}
-
-	ResumeThread(target_thread);
-
-	return dispatched;
-}
 #endif
 
 void KernelDispatchPendingSignalForCurrentThread() {
@@ -1206,8 +1154,7 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 		}
 
 		HANDLE target_thread =
-		    OpenThread(THREAD_SET_CONTEXT | THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT, FALSE,
+		    OpenThread(THREAD_SET_CONTEXT, FALSE, static_cast<DWORD>(target_thread_id));
-		               static_cast<DWORD>(target_thread_id));
 		if (target_thread == nullptr) {
 			return KERNEL_ERROR_EINVAL;
 		}
@@ -1231,10 +1178,6 @@ static int KYTY_SYSV_ABI KernelRaiseException(Pthread thread, int signum) {
 
 		Common::CondVar::SignalThread(PthreadGetUniqueId(thread));
 		PthreadWakeForSignal(thread);
-		WaitForSignalDispatch(thread, signum);
-		if (PthreadHasPendingSignal(thread, signum)) {
-			DispatchSignalWithSuspendedThreadContext(target_thread, thread, signum, handler);
-		}
 		CloseHandle(target_thread);
 		return OK;
 #elif defined(__x86_64__)

src/loader/elf.h

@@ -53,6 +53,7 @@ constexpr Elf64_Half ET_DYNAMIC = 0xfe18; // Shared
 constexpr Elf64_Word PT_LOAD          = 1;
 constexpr Elf64_Word PT_DYNAMIC       = 2;
 constexpr Elf64_Word PT_TLS           = 7;
+constexpr Elf64_Word PT_GNU_EH_FRAME  = 0x6474e550;
 constexpr Elf64_Word PT_OS_DYNLIBDATA = 0x61000000;
 constexpr Elf64_Word PT_OS_PROCPARAM  = 0x61000001;
 constexpr Elf64_Word PT_OS_RELRO      = 0x61000010;

src/loader/redZonePatcher.h

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#ifndef KYTY_LOADER_RED_ZONE_PATCHER_H_
+#define KYTY_LOADER_RED_ZONE_PATCHER_H_
+
+#include "common/common.h"
+
+#include <span>
+#include <vector>
+
+namespace Loader {
+
+struct RedZonePatchResult {
+	uint64_t function_count                           = 0;
+	uint64_t instruction_count                        = 0;
+	uint64_t red_zone_function_count                  = 0;
+	uint64_t memory_instruction_count                 = 0;
+	uint64_t short_memory_instruction_count           = 0;
+	uint64_t patched_memory_instruction_count         = 0;
+	uint64_t stack_dependent_memory_instruction_count = 0;
+	uint64_t control_flow_memory_instruction_count    = 0;
+	uint64_t unrelocatable_memory_instruction_count   = 0;
+	uint64_t indirect_red_zone_function_count         = 0;
+};
+
+void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
+                                uint64_t trampoline_area_size);
+void UnregisterRedZonePatchModule(void* module_ptr);
+
+RedZonePatchResult PatchRedZoneMemoryInstructions(uint64_t segment_addr, uint64_t segment_size,
+                                                  std::span<const uintptr_t> function_starts);
+
+bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
+                                 std::vector<uintptr_t>* function_starts);
+
+} // namespace Loader
+
+#endif /* KYTY_LOADER_RED_ZONE_PATCHER_H_ */

src/loader/runtimeLinker.cpp

@@ -19,6 +19,7 @@
 #include "loader/elf.h"
 #include "loader/gamePatch.h"
 #include "loader/jit.h"
+#include "loader/redZonePatcher.h"
 #include "loader/symbolDatabase.h"
 #include "loader/x64InstructionEmulator.h"
 
@@ -2202,17 +2203,39 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
 	program->mapped_size = program->base_size_aligned + tls_handler_size;
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	const bool         use_red_zone_protection  = Config::RedZoneProtectionEnabled();
+	constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
+	if (use_red_zone_protection) {
+		EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
+		program->mapped_size += RED_ZONE_TRAMPOLINE_SIZE;
+	}
+#endif
+
 	program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
 	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
 	    Common::PathToString(program->file_name.filename()).c_str());
+	EXIT_IF(program->base_vaddr == 0);
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	if (use_red_zone_protection) {
+		program->red_zone_trampoline_vaddr = program->base_vaddr + program->base_size_aligned;
+		program->red_zone_trampoline_size  = RED_ZONE_TRAMPOLINE_SIZE;
+		RegisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr),
+		                           program->base_size_aligned,
+		                           reinterpret_cast<void*>(program->red_zone_trampoline_vaddr),
+		                           program->red_zone_trampoline_size);
+	}
+#endif
 	if (!is_shared) {
 		program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		program->tls.handler_vaddr += program->red_zone_trampoline_size;
+#endif
 	}
 
 	g_desired_base_addr += CODE_BASE_INCR * (1 + program->mapped_size / CODE_BASE_INCR);
 
-	EXIT_IF(program->base_vaddr == 0);
 	EXIT_IF(program->base_size_aligned < program->base_size);
 	LOGF("base_vaddr             = 0x%016" PRIx64 "\n"
 	     "base_size              = 0x%016" PRIx64 "\n"
@@ -2228,6 +2251,11 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	}
 
 	// program->elf->SetBaseVAddr(program->base_vaddr);
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	std::vector<std::pair<uint64_t, uint64_t>> executable_segments;
+	uint64_t                                   eh_frame_header_addr = 0;
+	uint64_t                                   eh_frame_header_size = 0;
+#endif
 
 	for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) {
 		if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) {
@@ -2250,6 +2278,11 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 			if (Common::VirtualMemory::IsExecute(mode)) {
 				PatchProgram(program, segment_addr, segment_memory_size);
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+				if (use_red_zone_protection) {
+					executable_segments.emplace_back(segment_addr, segment_file_size);
+				}
+#endif
 			}
 
 			if (!skip_protect) {
@@ -2284,8 +2317,42 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 
 			program->proc_param_vaddr = phdr[i].p_vaddr + program->base_vaddr;
 		}
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		if (use_red_zone_protection && phdr[i].p_type == PT_GNU_EH_FRAME) {
+			eh_frame_header_addr = phdr[i].p_vaddr + program->base_vaddr;
+			eh_frame_header_size = phdr[i].p_memsz;
+		}
+#endif
 	}
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	if (use_red_zone_protection) {
+		std::vector<uintptr_t> function_starts;
+		if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
+		                                 &function_starts)) {
+			LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n",
+			     Common::PathToString(program->file_name).c_str());
+		}
+		for (const auto& [segment_addr, segment_size]: executable_segments) {
+			const auto result =
+			    PatchRedZoneMemoryInstructions(segment_addr, segment_size, function_starts);
+			LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
+			     ", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
+			     ", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
+			     Common::PathToString(program->file_name.filename()).c_str(), result.function_count,
+			     result.red_zone_function_count, result.memory_instruction_count,
+			     result.patched_memory_instruction_count, result.short_memory_instruction_count,
+			     result.stack_dependent_memory_instruction_count,
+			     result.control_flow_memory_instruction_count,
+			     result.unrelocatable_memory_instruction_count);
+			Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
+		}
+		Common::VirtualMemory::FlushInstructionCache(program->red_zone_trampoline_vaddr,
+		                                             program->red_zone_trampoline_size);
+	}
+#endif
+
 	if (!is_shared) {
 		SetupTlsHandler(program);
 	}
@@ -2311,6 +2378,11 @@ void RuntimeLinker::DeleteProgram(Program* p) {
 
 	if (program->base_vaddr != 0 || program->mapped_size != 0) {
 		EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		if (program->red_zone_trampoline_size != 0) {
+			UnregisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr));
+		}
+#endif
 		EXIT_IF(
 		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
 	}

src/loader/runtimeLinker.h

@@ -114,15 +114,19 @@ struct Program {
 	Program();
 	~Program();
 
-	int32_t                         unique_id = -1;
+	int32_t                      unique_id = -1;
-	RuntimeLinker*                  rt        = nullptr;
+	RuntimeLinker*               rt        = nullptr;
-	std::filesystem::path           file_name;
+	std::filesystem::path        file_name;
-	std::unique_ptr<Elf64>          elf;
+	std::unique_ptr<Elf64>       elf;
-	std::unique_ptr<DynamicInfo>    dynamic_info;
+	std::unique_ptr<DynamicInfo> dynamic_info;
-	uint64_t                        base_vaddr        = 0;
+	uint64_t                     base_vaddr        = 0;
-	uint64_t                        base_size         = 0;
+	uint64_t                     base_size         = 0;
-	uint64_t                        base_size_aligned = 0;
+	uint64_t                     base_size_aligned = 0;
-	uint64_t                        mapped_size       = 0;
+	uint64_t                     mapped_size       = 0;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	uint64_t red_zone_trampoline_vaddr = 0;
+	uint64_t red_zone_trampoline_size  = 0;
+#endif
 	std::unique_ptr<SymbolDatabase> export_symbols;
 	std::unique_ptr<SymbolDatabase> import_symbols;
 	ThreadLocalStorage              tls;

src/main.cpp

@@ -64,6 +64,9 @@ static void PrintUsage() {
 	::printf("  --spirv-debug-printf <true|false>    Enable SPIR-V debug printf.\n");
 	::printf(
 	    "  --readback-linear-images <true|false> Read back writable linear images on submit.\n");
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	::printf("  --redzone                            Protect the guest SysV red zone.\n");
+#endif
 	::printf("  --keymap <Pad=Key>                   DualSense mapping; may be repeated.\n");
 	::printf("  --rd                                 Enable RenderDoc capture.\n");
 }
@@ -138,6 +141,13 @@ static bool ParseArgs(int argc, char* argv[], RunOptions& options, bool& show_he
 			continue;
 		}
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+		if (arg == "--redzone") {
+			options.config.red_zone_protection_enabled = true;
+			continue;
+		}
+#endif
+
 		if (!Common::StartsWith(arg, "--")) {
 			::printf("game input must be provided with --game\n");
 			return false;

tests/ShaderRecompilerComputeTests.cpp

@@ -18265,39 +18265,67 @@ void CheckPm4ContextStateOperations(RenderContext& renderer) {
 	            processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
 	        "push changed live Cx state");
 	processor.GetCtx().SetRenderTargetMask(0x01020304);
-	Require("Pm4ContextState", "push-pop restore",
+        Require("Pm4ContextState", "push-pop restore",
-	        invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
+                invoke(ContextStateOperation::Pop) ==
-	            processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
+                        Pm4ProcessResult::Complete &&
-	        "ordinary push/pop did not restore Cx state");
+                    processor.GetCtx().GetRenderTargetMask() == 0x1234abcd,
+                "ordinary push/pop did not restore Cx state");
 
-	Require("Pm4ContextState", "clear",
+        Require("Pm4ContextState", "clear",
-	        invoke(ContextStateOperation::Clear) == Pm4ProcessResult::Complete &&
+                invoke(ContextStateOperation::Clear) ==
-	            !processor.GetCtx().GetRenderControl().depth_clear_enable &&
+                        Pm4ProcessResult::Complete &&
-	            processor.GetCtx().GetRenderTargetMask() == HW::Context {}.GetRenderTargetMask() &&
+                    !processor.GetCtx().GetRenderControl().depth_clear_enable &&
-	            processor.GetUcfg().GetPrimType() == 0x35 &&
+                    processor.GetCtx().GetRenderTargetMask() ==
-	            processor.GetShCtx().GetPs().ps_regs.data_addr == shader_address,
+                        HW::Context{}.GetRenderTargetMask() &&
-	        "clear reset state outside the Cx register domain");
+                    processor.GetUcfg().GetPrimType() == 0x35 &&
+                    processor.GetShCtx().GetPs().ps_regs.data_addr ==
+                        shader_address,
+                "clear reset state outside the Cx register domain");
 
-	processor.GetCtx().SetRenderTargetMask(0xabcdef01);
+        processor.GetCtx().SetRenderTargetMask(0x0badc0de);
-	Require("Pm4ContextState", "push before processor reset",
+        Require("Pm4ContextState", "push before clear-state packet",
-	        invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete,
+                invoke(ContextStateOperation::Push) ==
-	        "push before reset failed");
+                    Pm4ProcessResult::Complete,
-	processor.Reset();
+                "push before CLEAR_STATE failed");
-	processor.GetCtx().SetRenderTargetMask(0x76543210);
+        processor.GetCtx().SetRenderTargetMask(0xfeedface);
-	Require("Pm4ContextState", "processor reset discards push",
+        std::array<uint32_t, 2> clear_state_packet{0xc0001200, 0};
-	        invoke(ContextStateOperation::Push) == Pm4ProcessResult::Complete &&
+        Pm4Execution clear_state_execution;
-	            invoke(ContextStateOperation::Pop) == Pm4ProcessResult::Complete &&
+        Require("Pm4ContextState", "clear-state packet",
-	            processor.GetCtx().GetRenderTargetMask() == 0x76543210,
+                processor.Process(
-	        "processor reset retained an invalid saved Cx state");
+                    clear_state_execution, clear_state_packet.data(),
+                    clear_state_packet.size()) == Pm4ProcessResult::Complete &&
+                    processor.GetCtx().GetRenderTargetMask() ==
+                        HW::Context{}.GetRenderTargetMask(),
+                "CLEAR_STATE did not clear the current Cx state");
+        Require("Pm4ContextState", "pop after clear-state packet",
+                invoke(ContextStateOperation::Pop) ==
+                        Pm4ProcessResult::Complete &&
+                    processor.GetCtx().GetRenderTargetMask() == 0x0badc0de,
+                "CLEAR_STATE discarded the pushed Cx state");
 
-	Require("Pm4ContextState", "HLE packet size",
+        processor.GetCtx().SetRenderTargetMask(0xabcdef01);
-	        Gen5::GraphicsDcbContextStateOpGetSize(0) == 20 &&
+        Require("Pm4ContextState", "push before processor reset",
-	            Gen5::GraphicsDcbContextStateOpGetSize(1) == 108 &&
+                invoke(ContextStateOperation::Push) ==
-	            Gen5::GraphicsDcbContextStateOpGetSize(2) == 108 &&
+                    Pm4ProcessResult::Complete,
-	            Gen5::GraphicsDcbContextStateOpGetSize(3) == 128 &&
+                "push before reset failed");
-	            Gen5::GraphicsDcbContextStateOpGetSize(4) == 0,
+        processor.Reset();
-	        "context-state HLE sizes do not match libSceAgc");
+        processor.GetCtx().SetRenderTargetMask(0x76543210);
-	std::printf("[host]    %-32s ok\n", "Pm4ContextState");
+        Require("Pm4ContextState", "processor reset discards push",
+                invoke(ContextStateOperation::Push) ==
+                        Pm4ProcessResult::Complete &&
+                    invoke(ContextStateOperation::Pop) ==
+                        Pm4ProcessResult::Complete &&
+                    processor.GetCtx().GetRenderTargetMask() == 0x76543210,
+                "processor reset retained an invalid saved Cx state");
+
+        Require("Pm4ContextState", "HLE packet size",
+                Gen5::GraphicsDcbContextStateOpGetSize(0) == 20 &&
+                    Gen5::GraphicsDcbContextStateOpGetSize(1) == 108 &&
+                    Gen5::GraphicsDcbContextStateOpGetSize(2) == 108 &&
+                    Gen5::GraphicsDcbContextStateOpGetSize(3) == 128 &&
+                    Gen5::GraphicsDcbContextStateOpGetSize(4) == 0,
+                "context-state HLE sizes do not match libSceAgc");
+        std::printf("[host]    %-32s ok\n", "Pm4ContextState");
 }
 
 void CheckPm4WaitResume(RenderContext& renderer) {

tests/VirtualMemoryAllocationTests.cpp

@@ -8,15 +8,28 @@
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
+#include "loader/redZonePatcher.h"
 #include "loader/runtimeLinker.h"
 #include "loader/systemContent.h"
 
+#include <array>
 #include <cinttypes>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
 #include <filesystem>
 #include <string>
+#include <vector>
+
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#ifdef DeleteFile
+#undef DeleteFile
+#endif
+#endif
 
 namespace {
 
@@ -133,6 +146,105 @@ void InitSubsystems() {
 	initialized = true;
 }
 
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+void* g_red_zone_fault_page = nullptr;
+
+LONG CALLBACK RedZoneFaultHandler(EXCEPTION_POINTERS* exception) {
+	if (exception == nullptr || exception->ExceptionRecord == nullptr ||
+	    exception->ContextRecord == nullptr ||
+	    exception->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION ||
+	    reinterpret_cast<void*>(exception->ExceptionRecord->ExceptionInformation[1]) !=
+	        g_red_zone_fault_page) {
+		return EXCEPTION_CONTINUE_SEARCH;
+	}
+
+	// Model the Windows exception stack footprint that prompted the static patch:
+	// data below the interrupted RSP is not part of the Windows ABI contract.
+	*reinterpret_cast<uint64_t*>(exception->ContextRecord->Rsp - 0x18) = 0;
+	DWORD old_protection = 0;
+	if (VirtualProtect(g_red_zone_fault_page, 0x1000, PAGE_READWRITE, &old_protection) == FALSE) {
+		return EXCEPTION_CONTINUE_SEARCH;
+	}
+	return EXCEPTION_CONTINUE_EXECUTION;
+}
+
+void TestWindowsGuestRedZoneStaticPatcher() {
+	const char* test = "WindowsGuestRedZoneStaticPatcher";
+	constexpr uint64_t SENTINEL = 0x1122334455667788ull;
+	constexpr uint64_t CODE_SIZE = 0x4000;
+	constexpr uint64_t TRAMPOLINE_SIZE = 0x4000;
+	const auto mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x0000000902000000ull, CODE_SIZE + TRAMPOLINE_SIZE,
+	    Common::VirtualMemory::Mode::ExecuteReadWrite, "red_zone_patcher_test");
+	Check(test, mapping != 0, "failed to allocate patch test code");
+
+	std::vector<uint8_t> code;
+	const auto emit = [&code](std::initializer_list<uint8_t> bytes) {
+		code.insert(code.end(), bytes.begin(), bytes.end());
+	};
+	const auto emit64 = [&code](uint64_t value) {
+		const auto offset = code.size();
+		code.resize(offset + sizeof(value));
+		std::memcpy(code.data() + offset, &value, sizeof(value));
+	};
+	emit({0x48, 0xb8});
+	emit64(SENTINEL);                         // movabs rax, sentinel
+	emit({0x48, 0x89, 0x44, 0x24, 0xe8});     // mov [rsp-0x18], rax
+	emit({0x48, 0x8b, 0x07});                 // mov rax, [rdi] (faultable, 3 bytes)
+	emit({0x48, 0x8b, 0x44, 0x24, 0xe8});     // mov rax, [rsp-0x18]
+	emit({0x48, 0xb9});
+	emit64(SENTINEL);                         // movabs rcx, sentinel
+	emit({0x48, 0x39, 0xc8});                 // cmp rax, rcx
+	emit({0x0f, 0x94, 0xc0});                 // sete al
+	emit({0x0f, 0xb6, 0xc0, 0xc3});           // movzx eax, al; ret
+	Check(test, code.size() < CODE_SIZE, "generated patch test code is too large");
+	std::memcpy(reinterpret_cast<void*>(mapping), code.data(), code.size());
+	Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, code.size()),
+	      "failed to flush generated test code");
+
+	g_red_zone_fault_page = VirtualAlloc(nullptr, 0x1000, MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS);
+	Check(test, g_red_zone_fault_page != nullptr, "failed to allocate fault page");
+	auto* handler = AddVectoredExceptionHandler(1, RedZoneFaultHandler);
+	Check(test, handler != nullptr, "failed to install test exception handler");
+
+	using GuestFunction = uint64_t(KYTY_SYSV_ABI*)(const uint64_t*);
+	const auto function = reinterpret_cast<GuestFunction>(mapping);
+	const bool unpatched_was_corrupted = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 0;
+
+	DWORD old_protection = 0;
+	Check(test, VirtualProtect(g_red_zone_fault_page, 0x1000, PAGE_NOACCESS, &old_protection) != FALSE,
+	      "failed to reset fault page protection");
+	Loader::RegisterRedZonePatchModule(reinterpret_cast<void*>(mapping), CODE_SIZE,
+	                                   reinterpret_cast<void*>(mapping + CODE_SIZE),
+	                                   TRAMPOLINE_SIZE);
+	const std::array<uintptr_t, 1> function_starts = {static_cast<uintptr_t>(mapping)};
+	const auto result = Loader::PatchRedZoneMemoryInstructions(
+	    mapping, code.size(), function_starts);
+	Check(test, Common::VirtualMemory::FlushInstructionCache(mapping, CODE_SIZE + TRAMPOLINE_SIZE),
+	      "failed to flush patched test code");
+	const bool patched_preserved = function(static_cast<const uint64_t*>(g_red_zone_fault_page)) == 1;
+
+	Loader::UnregisterRedZonePatchModule(reinterpret_cast<void*>(mapping));
+	RemoveVectoredExceptionHandler(handler);
+	VirtualFree(g_red_zone_fault_page, 0, MEM_RELEASE);
+	g_red_zone_fault_page = nullptr;
+	const bool freed = Libs::LibKernel::Memory::FreeGuestMemory(mapping, CODE_SIZE + TRAMPOLINE_SIZE);
+
+	Check(test, unpatched_was_corrupted, "test harness did not reproduce red-zone corruption");
+	Check(test, result.red_zone_function_count == 1 && result.memory_instruction_count >= 1 &&
+	                result.patched_memory_instruction_count >= 1 &&
+	                result.unrelocatable_memory_instruction_count == 0,
+	      "static patcher did not cover the faultable instruction");
+	Check(test, patched_preserved, "patched fault still corrupted the guest red zone");
+	Check(test, freed, "failed to free patch test code");
+	std::printf("[host]    %-48s ok\n", test);
+}
+#else
+void TestWindowsGuestRedZoneStaticPatcher() {
+	std::printf("[host]    %-48s skipped\n", "WindowsGuestRedZoneStaticPatcher");
+}
+#endif
+
 void RunTest(void (*test_func)()) {
 	if (g_failed_tests != 0) {
 		return;
@@ -2332,9 +2444,14 @@ void TestModuleRelocationUsesWritableHostMapping() {
 
 } // namespace
 
-int main() {
+int main(int argc, char** argv) {
 	InitSubsystems();
+	if (argc == 2 && std::strcmp(argv[1], "--red-zone-patcher-only") == 0) {
+		RunTest(TestWindowsGuestRedZoneStaticPatcher);
+		return g_failed_tests == 0 ? 0 : 1;
+	}
 
+	RunTest(TestWindowsGuestRedZoneStaticPatcher);
 	RunTest(TestProsperoArgumentAndInfoSizeContracts);
 	RunTest(TestGuestAddressSpaceOwnsReservationsBeforeBacking);
 	RunTest(TestGuestAddressSpaceHasNoFixedFallback);