net: emulate Windows PEEK and WAITALL receives

Co-authored-by: Firas Ben Mbarek <firasbenmbarek@outlook.com>
Co-authored-by: rudy-07 <kanhaiffco2007@gmail.com>
Co-authored-by: Matthew Hurley <171001439+theantipopau@users.noreply.github.com>
Co-authored-by: usopp-sama <go4av05.pvtx@gmail.com>
Co-authored-by: wruwami <wruwami@users.noreply.github.com>

3rdparty/CMakeLists.txt

@@ -25,7 +25,7 @@ FetchContent_MakeAvailable(opus)
 set(BUILD_TESTING "${kyty_build_testing_saved}")
 install(FILES "${opus_SOURCE_DIR}/COPYING" DESTINATION "licenses/opus")
 
-# The Windows guest red-zone patcher needs a decoder/encoder and a small
+# The guest instruction patcher needs a decoder/encoder and a small
 # trampoline assembler. Pin the same revisions used by shadPS4's implementation.
 set(XBYAK_BUILD_TESTS OFF CACHE BOOL "" FORCE)
 FetchContent_Declare(
@@ -48,6 +48,7 @@ FetchContent_Declare(
 )
 
 FetchContent_MakeAvailable(xbyak zydis)
+target_compile_definitions(xbyak INTERFACE XBYAK_NO_EXCEPTION)
 
 # ZArchive provides random-access reads for compressed game dumps. Build only
 # the reader/writer library and its zstd dependency; the command-line tool is

src/libs/network.cpp

@@ -849,20 +849,38 @@ using SocketIoLength                                = size_t;
 #endif
 
 struct SocketTransport {
-	explicit SocketTransport(NativeSocket value): socket(value) {}
+	SocketTransport(NativeSocket value, int socket_type): socket(value), type(socket_type) {}
 	~SocketTransport() { Close(); }
+	bool HasPendingReceive() const {
+#if defined(_WIN32)
+		return buffered_bytes.load(std::memory_order_relaxed) != 0 ||
+		       receive_shutdown.load(std::memory_order_relaxed);
+#else
+		return false;
+#endif
+	}
 	int Close() {
-		if (socket == INVALID_NATIVE_SOCKET) {
+#if defined(_WIN32)
+		receive_shutdown.store(true, std::memory_order_relaxed);
+#endif
+		const auto value = socket.exchange(INVALID_NATIVE_SOCKET);
+		if (value == INVALID_NATIVE_SOCKET) {
 			return 0;
 		}
-		const auto value = std::exchange(socket, INVALID_NATIVE_SOCKET);
 #if defined(_WIN32)
 		return closesocket(value);
 #else
 		return ::close(value);
 #endif
 	}
-	NativeSocket socket;
+	std::atomic<NativeSocket> socket;
+	const int                type;
+#if defined(_WIN32)
+	std::mutex         receive_mutex;
+	std::vector<char>  receive_buffer;
+	std::atomic_size_t buffered_bytes   = 0;
+	std::atomic_bool   receive_shutdown = false;
+#endif
 };
 
 struct P2pEndpoint {
@@ -1164,7 +1182,8 @@ static int ConvertHostSockaddr(const sockaddr_storage* addr, SocketLength addrle
 	return 0;
 }
 
-static bool GetSocketBackend(int guest_fd, NativeSocket* out, SocketSlot* state = nullptr) {
+static bool GetSocketBackend(int guest_fd, NativeSocket* out, SocketSlot* state = nullptr,
+                             std::shared_ptr<SocketTransport>* transport = nullptr) {
 	EXIT_IF(out == nullptr);
 
 	if (guest_fd < 0 || guest_fd >= SOCKET_FD_MAX) {
@@ -1188,6 +1207,9 @@ static bool GetSocketBackend(int guest_fd, NativeSocket* out, SocketSlot* state
 	if (state != nullptr) {
 		*state = slot;
 	}
+	if (transport != nullptr) {
+		*transport = slot.transport;
+	}
 	return true;
 }
 
@@ -1234,6 +1256,63 @@ static int AllocSocketFd(const std::shared_ptr<SocketTransport>& transport, bool
 	return -1;
 }
 
+static int SelectSockets(int nfds, fd_set* readfds, fd_set* writefds, fd_set* exceptfds,
+                         timeval* timeout,
+                         const std::vector<std::shared_ptr<SocketTransport>>& stream_reads) {
+#if defined(_WIN32)
+	if (!stream_reads.empty()) {
+		const auto requested_read   = *readfds;
+		const auto requested_write  = writefds != nullptr ? *writefds : fd_set {};
+		const auto requested_except = exceptfds != nullptr ? *exceptfds : fd_set {};
+		std::optional<std::chrono::steady_clock::time_point> deadline;
+		if (timeout != nullptr) {
+			deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout->tv_sec) +
+			           std::chrono::microseconds(timeout->tv_usec);
+		}
+		for (;;) {
+			*readfds = requested_read;
+			if (writefds != nullptr) {
+				*writefds = requested_write;
+			}
+			if (exceptfds != nullptr) {
+				*exceptfds = requested_except;
+			}
+			const bool ready = std::any_of(stream_reads.begin(), stream_reads.end(),
+			    [](const auto& transport) { return transport->HasPendingReceive(); });
+			// A concurrent peek can move native bytes into the guest buffer after this check.
+			// Bound native waits so both readiness sources are rechecked without holding receive locks.
+			auto wait = std::chrono::microseconds(ready ? 0 : 10'000);
+			if (deadline) {
+				const auto remaining = std::chrono::duration_cast<std::chrono::microseconds>(
+				    *deadline - std::chrono::steady_clock::now());
+				wait = std::min(wait, std::max(remaining, std::chrono::microseconds::zero()));
+			}
+			timeval slice {};
+			slice.tv_usec = static_cast<long>(wait.count());
+			int result = ::select(nfds, readfds, writefds, exceptfds, &slice);
+			if (result < 0) {
+				return result;
+			}
+			for (const auto& transport: stream_reads) {
+				const auto socket = transport->socket.load();
+				if (socket == INVALID_NATIVE_SOCKET) {
+					WSASetLastError(WSAENOTSOCK);
+					return SOCKET_ERROR;
+				}
+				if (transport->HasPendingReceive() && !FD_ISSET(socket, readfds)) {
+					FD_SET(socket, readfds);
+					result++;
+				}
+			}
+			if (result != 0 || (deadline && std::chrono::steady_clock::now() >= *deadline)) {
+				return result;
+			}
+		}
+	}
+#endif
+	return ::select(nfds, readfds, writefds, exceptfds, timeout);
+}
+
 static bool GuestFdIsSet(const void* fds, int fd) {
 	if (fds == nullptr || fd < 0 || fd >= SOCKET_FD_MAX) {
 		return false;
@@ -1621,6 +1700,7 @@ int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int t
 	FD_ZERO(&host_except);
 
 	std::vector<HostRegistration> host_registrations;
+	std::vector<std::shared_ptr<SocketTransport>> stream_reads;
 	host_registrations.reserve(registrations.size());
 	int host_nfds = 0;
 	for (const auto& registration: registrations) {
@@ -1641,6 +1721,11 @@ int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int t
 #endif
 		if ((registration.event.events & EPOLL_IN) != 0) {
 			FD_SET(socket, &host_read);
+#if defined(_WIN32)
+			if (state.transport->type == SOCK_STREAM) {
+				stream_reads.push_back(state.transport);
+			}
+#endif
 		}
 		if ((registration.event.events & EPOLL_OUT) != 0) {
 			FD_SET(socket, &host_write);
@@ -1660,8 +1745,8 @@ int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int t
 		host_timeout.tv_usec = timeout % 1000000;
 		host_timeout_ptr     = &host_timeout;
 	}
-
-	const int result = ::select(host_nfds, &host_read, &host_write, &host_except, host_timeout_ptr);
+	const int result = SelectSockets(host_nfds, &host_read, &host_write, &host_except,
+	                                 host_timeout_ptr, stream_reads);
 	if (result < 0) {
 		return SetHostSocketError();
 	}
@@ -1727,7 +1812,12 @@ int KYTY_SYSV_ABI SocketClose(int s) {
 	}
 	RemoveSocketFromEpolls(s);
 
-	if (slot.transport.use_count() == 1 && slot.transport->Close() != 0) {
+	bool close_transport = slot.transport.use_count() == 1;
+#if defined(_WIN32)
+	// Retained receive/readiness state must not defer native TCP close cancellation.
+	close_transport |= !slot.p2p && slot.transport->type == SOCK_STREAM;
+#endif
+	if (close_transport && slot.transport->Close() != 0) {
 		return NET_ERROR_EBADF;
 	}
 
@@ -1762,7 +1852,7 @@ int KYTY_SYSV_ABI Socket(int family, int type, int protocol) {
 		return SetHostSocketError();
 	}
 
-	auto transport = std::make_shared<SocketTransport>(socket);
+	auto transport = std::make_shared<SocketTransport>(socket, p2p ? SOCK_DGRAM : type);
 	if (p2p) {
 		// Logical sockets keep their own blocking mode; the shared transport never blocks.
 #if defined(_WIN32)
@@ -1945,7 +2035,7 @@ int KYTY_SYSV_ABI Accept(int s, void* addr, uint32_t* addrlen) {
 		return SetHostSocketError();
 	}
 
-	auto      transport = std::make_shared<SocketTransport>(accepted);
+	auto      transport = std::make_shared<SocketTransport>(accepted, SOCK_STREAM);
 	const int fd        = AllocSocketFd(transport);
 	if (fd < 0) {
 		*Posix::GetErrorAddr() = Posix::POSIX_EMFILE;
@@ -1974,8 +2064,9 @@ int KYTY_SYSV_ABI Shutdown(int s, int how) {
 	     "\t how = %d\n",
 	     s, how);
 
-	NativeSocket socket = INVALID_NATIVE_SOCKET;
-	if (!GetSocketBackend(s, &socket)) {
+	NativeSocket                    socket = INVALID_NATIVE_SOCKET;
+	std::shared_ptr<SocketTransport> transport;
+	if (!GetSocketBackend(s, &socket, nullptr, &transport)) {
 		return -1;
 	}
 
@@ -1988,6 +2079,9 @@ int KYTY_SYSV_ABI Shutdown(int s, int how) {
 	if (::shutdown(socket, how) == SOCKET_ERROR) {
 		return SetHostSocketError();
 	}
+	if (how == SD_RECEIVE || how == SD_BOTH) {
+		transport->receive_shutdown.store(true, std::memory_order_relaxed);
+	}
 
 	return 0;
 #else
@@ -2205,6 +2299,77 @@ int64_t KYTY_SYSV_ABI Sendto(int s, const void* buf, uint64_t len, int flags, co
 	return result;
 }
 
+#if defined(_WIN32)
+static int ReceiveStream(SocketTransport& transport, char* output, int length, int flags,
+                         bool dontwait) {
+	std::unique_lock lock(transport.receive_mutex, std::defer_lock);
+	if (dontwait) {
+		if (!lock.try_lock()) {
+			return SetGuestSocketError(Posix::POSIX_EWOULDBLOCK);
+		}
+	} else {
+		lock.lock();
+	}
+	if (transport.receive_shutdown.load(std::memory_order_relaxed)) {
+		return SetGuestSocketError(Posix::POSIX_ESHUTDOWN);
+	}
+	auto&           buffered = transport.receive_buffer;
+	const bool      peek     = (flags & MSG_PEEK) != 0;
+	const bool      waitall  = (flags & MSG_WAITALL) != 0;
+	int             copied   = static_cast<int>(std::min<size_t>(buffered.size(), length));
+	if (copied != 0) {
+		std::memcpy(output, buffered.data(), copied);
+		if (!peek) {
+			buffered.erase(buffered.begin(), buffered.begin() + copied);
+			transport.buffered_bytes.store(buffered.size(), std::memory_order_relaxed);
+		}
+	}
+	if (copied == length || (copied != 0 && !waitall)) {
+		return copied;
+	}
+
+	// Winsock rejects PEEK | WAITALL. Consume into transport-owned storage when peeking,
+	// and let ordinary recv report EOF, timeouts, or nonblocking partial progress.
+	flags &= ~(MSG_PEEK | MSG_WAITALL);
+	do {
+		if (transport.receive_shutdown.load(std::memory_order_relaxed)) {
+			return copied != 0 ? copied : SetGuestSocketError(Posix::POSIX_ESHUTDOWN);
+		}
+		const auto socket = transport.socket.load();
+		if (dontwait) {
+			fd_set readable {};
+			FD_SET(socket, &readable);
+			timeval immediate {};
+			const int ready = ::select(0, &readable, nullptr, nullptr, &immediate);
+			if (ready <= 0) {
+				return copied != 0 ? copied : (ready < 0 ? SetHostSocketError() :
+				    SetGuestSocketError(Posix::POSIX_EWOULDBLOCK));
+			}
+		}
+		const auto previous_size = buffered.size();
+		const int  requested     = peek ? std::min(length - copied, 64 * 1024) : length - copied;
+		char*      destination   = output + copied;
+		if (peek) {
+			buffered.resize(previous_size + requested);
+			destination = buffered.data() + previous_size;
+		}
+		const int received = ::recv(socket, destination, requested, flags);
+		if (peek) {
+			if (received > 0) {
+				std::memcpy(output + copied, destination, received);
+			}
+			buffered.resize(previous_size + std::max(received, 0));
+			transport.buffered_bytes.store(buffered.size(), std::memory_order_relaxed);
+		}
+		if (received <= 0) {
+			return copied != 0 || received == 0 ? copied : SetHostSocketError();
+		}
+		copied += received;
+	} while (waitall && copied < length);
+	return copied;
+}
+#endif
+
 int64_t KYTY_SYSV_ABI Recv(int s, void* buf, uint64_t len, int flags) {
 	return Recvfrom(s, buf, len, flags, nullptr, nullptr);
 }
@@ -2227,8 +2392,9 @@ int64_t KYTY_SYSV_ABI Recvfrom(int s, void* buf, uint64_t len, int flags, void*
 		return -1;
 	}
 
-	NativeSocket socket = INVALID_NATIVE_SOCKET;
-	if (!GetSocketBackend(s, &socket)) {
+	NativeSocket                    socket = INVALID_NATIVE_SOCKET;
+	std::shared_ptr<SocketTransport> transport;
+	if (!GetSocketBackend(s, &socket, nullptr, &transport)) {
 		return -1;
 	}
 
@@ -2242,11 +2408,27 @@ int64_t KYTY_SYSV_ABI Recvfrom(int s, void* buf, uint64_t len, int flags, void*
 	sockaddr_storage host_addr {};
 	SocketLength     host_addrlen = sizeof(host_addr);
 	int64_t          result       = 0;
-	if (addr == nullptr) {
-		result = ::recv(socket, static_cast<char*>(buf), host_len, host_flags);
-	} else {
-		result = ::recvfrom(socket, static_cast<char*>(buf), host_len, host_flags,
-		                    reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
+#if defined(_WIN32)
+	if (transport->type == SOCK_STREAM) {
+		if (addr != nullptr &&
+		    ::getpeername(socket, reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen) != 0) {
+			return SetHostSocketError();
+		}
+		result = ReceiveStream(*transport, static_cast<char*>(buf), host_len, host_flags,
+		                       (flags & 0x80) != 0);
+		if (result < 0) {
+			return result;
+		}
+	} else
+#endif
+	{
+		transport.reset();
+		if (addr == nullptr) {
+			result = ::recv(socket, static_cast<char*>(buf), host_len, host_flags);
+		} else {
+			result = ::recvfrom(socket, static_cast<char*>(buf), host_len, host_flags,
+			                    reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
+		}
 	}
 #if defined(_WIN32)
 	if (result < 0 && WSAGetLastError() == WSAEMSGSIZE) {
@@ -2303,6 +2485,7 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 	FD_ZERO(&host_except);
 
 	std::vector<std::pair<int, NativeSocket>> descriptors;
+	std::vector<std::shared_ptr<SocketTransport>> stream_reads;
 	int host_nfds = 0;
 	for (int fd = 0; fd < nfds; fd++) {
 		const bool read = GuestFdIsSet(readfds, fd);
@@ -2311,13 +2494,14 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 		if (!read && !write && !except) {
 			continue;
 		}
-		NativeSocket socket = INVALID_NATIVE_SOCKET;
+		NativeSocket                    socket = INVALID_NATIVE_SOCKET;
+		std::shared_ptr<SocketTransport> transport;
 #if !defined(_WIN32)
 		if (fd < 3) {
 			socket = fd;
 		} else
 #endif
-		if (!GetSocketBackend(fd, &socket)) {
+		if (!GetSocketBackend(fd, &socket, nullptr, &transport)) {
 			return -1;
 		}
 #if defined(_WIN32)
@@ -2335,6 +2519,11 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 		descriptors.emplace_back(fd, socket);
 		if (read) {
 			FD_SET(socket, &host_read);
+#if defined(_WIN32)
+			if (transport->type == SOCK_STREAM) {
+				stream_reads.push_back(std::move(transport));
+			}
+#endif
 		}
 		if (write) {
 			FD_SET(socket, &host_write);
@@ -2365,9 +2554,10 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 	} else
 #endif
 	{
-		result = ::select(host_nfds, readfds != nullptr ? &host_read : nullptr,
-		                  writefds != nullptr ? &host_write : nullptr,
-		                  exceptfds != nullptr ? &host_except : nullptr, host_timeout_ptr);
+		result = SelectSockets(host_nfds, readfds != nullptr ? &host_read : nullptr,
+		                       writefds != nullptr ? &host_write : nullptr,
+		                       exceptfds != nullptr ? &host_except : nullptr, host_timeout_ptr,
+		                       stream_reads);
 		if (result < 0) {
 			return SetHostSocketError();
 		}
@@ -2376,15 +2566,19 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 	GuestFdZero(readfds, nfds);
 	GuestFdZero(writefds, nfds);
 	GuestFdZero(exceptfds, nfds);
+	result = 0;
 	for (const auto& [fd, socket]: descriptors) {
 		if (FD_ISSET(socket, &host_read)) {
 			GuestFdSet(readfds, fd);
+			result++;
 		}
 		if (FD_ISSET(socket, &host_write)) {
 			GuestFdSet(writefds, fd);
+			result++;
 		}
 		if (FD_ISSET(socket, &host_except)) {
 			GuestFdSet(exceptfds, fd);
+			result++;
 		}
 	}
 	return result;

src/loader/guestInstructionPatcher.cpp

@@ -1,12 +1,11 @@
 // SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "loader/redZonePatcher.h"
+#include "loader/guestInstructionPatcher.h"
 
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/virtualMemory.h"
-#include "loader/x64InstructionEmulator.h"
 
 #include <Zydis/Zydis.h>
 #include <algorithm>
@@ -22,7 +21,7 @@
 #include <set>
 #include <unordered_set>
 #include <vector>
-#if defined(_WIN32)
+#if !defined(__APPLE__)
 #include <xbyak/xbyak.h>
 #include <xbyak/xbyak_util.h>
 #endif
@@ -34,7 +33,7 @@
 #undef max
 #endif
 
-#if defined(_WIN32)
+#if !defined(__APPLE__)
 using namespace Xbyak::util;
 #endif
 
@@ -53,7 +52,7 @@ using s64 = int64_t;
 
 constexpr size_t NearJumpSize = 5;
 
-#if defined(_WIN32)
+#if !defined(__APPLE__)
 
 struct PatchModule {
 	std::mutex           mutex {};
@@ -62,7 +61,7 @@ struct PatchModule {
 	std::set<u8*>        patched;
 	Xbyak::CodeGenerator patch_gen;
 	Xbyak::CodeGenerator trampoline_gen;
-	bool                 trampoline_exhausted = false;
+	bool                 trampoline_exhaustion_reported = false;
 
 	PatchModule(u8* module_ptr, u64 module_size, u8* trampoline_ptr, u64 trampoline_size)
 	    : start(module_ptr), end(module_ptr + module_size), patch_gen(module_size, module_ptr),
@@ -81,14 +80,14 @@ static PatchModule* GetContainingModule(const void* ptr) {
 	return address >= module->start && address < module->end ? module : nullptr;
 }
 
-static bool HandleTrampolineError(PatchModule* module, const Xbyak::Error& error) {
-	if (static_cast<int>(error) != Xbyak::ERR_CODE_IS_TOO_BIG) {
+static bool HandleTrampolineError(PatchModule* module, int error) {
+	if (error != Xbyak::ERR_CODE_IS_TOO_BIG) {
 		return false;
 	}
-	if (!module->trampoline_exhausted) {
-		LOGF("Windows guest red-zone trampoline space exhausted for module %p\n",
+	if (!module->trampoline_exhaustion_reported) {
+		LOGF("Guest instruction trampoline space exhausted for module %p\n",
 		     static_cast<void*>(module->start));
-		module->trampoline_exhausted = true;
+		module->trampoline_exhaustion_reported = true;
 	}
 	return true;
 }
@@ -103,9 +102,6 @@ static ZydisDecoder& GetDecoder() {
 	return decoder;
 }
 
-#endif
-#if defined(_WIN32)
-
 namespace {
 
 constexpr size_t GuestRedZoneSize = 128;
@@ -136,11 +132,39 @@ struct DecodedFunction {
 	bool                                        requires_conservative_red_zone_tracking {};
 };
 
-struct InstructionRewrite {
-	bool protect_red_zone {};
-	bool protected_indirect_call {};
+enum class InstructionReplacement {
+	None,
+	ReciprocalSquareRoot,
+	ExtractQ,
+	InsertQ,
+	ReadProcessorId,
+	CacheLineWriteBack
 };
 
+struct InstructionRewrite {
+	bool                   protect_red_zone {};
+	bool                   protected_indirect_call {};
+	InstructionReplacement replacement {};
+};
+
+InstructionPatchCounts& ReplacementCounts(GuestInstructionPatchResult& result,
+                                          InstructionReplacement       replacement) {
+	ASSERT(replacement != InstructionReplacement::None);
+	switch (replacement) {
+		case InstructionReplacement::ReciprocalSquareRoot: return result.reciprocal_sqrt;
+		case InstructionReplacement::ExtractQ: return result.extrq;
+		case InstructionReplacement::InsertQ: return result.insertq;
+		case InstructionReplacement::ReadProcessorId: return result.rdpid;
+		default: return result.clwb;
+	}
+}
+
+bool UsesInstructionTrap(InstructionReplacement replacement, bool trap_replacements) {
+	return replacement != InstructionReplacement::None &&
+	       (trap_replacements || replacement == InstructionReplacement::InsertQ ||
+	        replacement == InstructionReplacement::ReadProcessorId);
+}
+
 bool IsStackPointerRegister(ZydisRegister reg) {
 	return reg != ZYDIS_REGISTER_NONE &&
 	       ZydisRegisterGetLargestEnclosing(ZYDIS_MACHINE_MODE_LONG_64, reg) == ZYDIS_REGISTER_RSP;
@@ -210,10 +234,11 @@ DecodedCodeInstruction DecodeCodeInstruction(uintptr_t address, uintptr_t end) {
 
 	if (decoded.changes_stack_pointer) {
 		const auto& operands = decoded.operands;
-		if (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV &&
-		    operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER &&
+		if (operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER &&
 		    IsStackPointerRegister(operands[0].reg.value) &&
-		    operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY) {
+		    (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_RDPID ||
+		     (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV &&
+		      operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY))) {
 			decoded.replaces_stack_pointer = true;
 		} else if ((decoded.instruction.mnemonic == ZYDIS_MNEMONIC_ADD ||
 		            decoded.instruction.mnemonic == ZYDIS_MNEMONIC_SUB) &&
@@ -266,7 +291,7 @@ DecodedCodeInstruction DecodeCodeInstruction(uintptr_t address, uintptr_t end) {
 		const s64 access_start = operand.mem.disp.value;
 		const s64 access_size  = std::max<s64>(operand.size / 8, 1);
 		const s64 range_start  = std::max(access_start, -static_cast<s64>(GuestRedZoneSize));
-		const s64 range_end    = std::min(access_start + access_size, 0LL);
+		const s64 range_end    = std::min<s64>(access_start + access_size, 0);
 		for (s64 offset = range_start; offset < range_end; ++offset) {
 			const size_t bit = static_cast<size_t>(offset + static_cast<s64>(GuestRedZoneSize));
 			if ((operand.actions & ZYDIS_OPERAND_ACTION_MASK_READ) != 0) {
@@ -647,13 +672,17 @@ void AnalyzeRedZoneLiveness(DecodedFunction& function) {
 }
 
 bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded,
-                                Xbyak::CodeGenerator&         generator) {
+                                Xbyak::CodeGenerator& generator, s64 stack_adjustment = 0,
+                                ZydisMnemonic replacement = ZYDIS_MNEMONIC_INVALID) {
 	ZydisEncoderRequest request;
 	if (!ZYAN_SUCCESS(ZydisEncoderDecodedInstructionToEncoderRequest(
 	        &decoded.instruction, decoded.operands.data(),
 	        decoded.instruction.operand_count_visible, &request))) {
 		return false;
 	}
+	if (replacement != ZYDIS_MNEMONIC_INVALID) {
+		request.mnemonic = replacement;
+	}
 
 	for (u8 index = 0; index < decoded.instruction.operand_count_visible; ++index) {
 		const auto& operand = decoded.operands[index];
@@ -673,6 +702,9 @@ bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded,
 				return false;
 			}
 			request.operands[index].mem.displacement = static_cast<ZyanI64>(absolute_address);
+		} else if (operand.type == ZYDIS_OPERAND_TYPE_MEMORY &&
+		           IsStackPointerRegister(operand.mem.base)) {
+			request.operands[index].mem.displacement += stack_adjustment;
 		}
 	}
 
@@ -738,9 +770,9 @@ bool GenerateProtectedIndirectCall(const DecodedCodeInstruction& decoded,
 	return true;
 }
 
-void CollectRedZoneMemoryInstructions(const DecodedFunction& function,
+void CollectRedZoneMemoryInstructions(const DecodedFunction&                   function,
                                       std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
-                                      RedZonePatchResult& result) {
+                                      GuestInstructionPatchResult&             result) {
 	if (!function.uses_red_zone) {
 		return;
 	}
@@ -777,98 +809,284 @@ void CollectRedZoneMemoryInstructions(const DecodedFunction& function,
 	}
 }
 
-struct ReciprocalSquareRootSite {
-	uintptr_t address;
-	u8        length;
-	bool      requires_red_zone_protection;
-};
+// Compute 1/sqrt in double precision, rounded to float, with denormals treated as
+// signed zero. Preserve guest MXCSR, RFLAGS and the scratch register's full YMM.
+void GenerateReciprocalSquareRoot(const DecodedCodeInstruction& decoded,
+                                  Xbyak::CodeGenerator&         generator) {
+	const int destination   = decoded.operands[0].reg.value - ZYDIS_REGISTER_XMM0;
+	const int source        = decoded.operands[1].reg.value - ZYDIS_REGISTER_XMM0;
+	int       scratch_index = 0;
+	while (scratch_index == destination || scratch_index == source) {
+		++scratch_index;
+	}
+	const Xbyak::Ymm scratch(scratch_index);
+	Xbyak::Label     one;
+	Xbyak::Label     done;
+	constexpr size_t SpillSize = 48;
 
-void CollectReciprocalSquareRoots(const DecodedFunction& function,
-                                 std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
-                                 std::vector<ReciprocalSquareRootSite>& sites) {
+	// Guest code uses the SysV red zone on both hosts; put our spills below it.
+	// LEA/MOV/SIMD leave RFLAGS intact, including when the guest red zone is live.
+	generator.lea(rsp, ptr[rsp - GuestRedZoneSize - SpillSize]);
+	generator.vmovdqu(ptr[rsp], scratch);
+	generator.stmxcsr(ptr[rsp + 32]);
+	generator.mov(dword[rsp + 36], 0x1fc0); // round to nearest, exceptions masked, DAZ
+	generator.ldmxcsr(ptr[rsp + 36]);
+	generator.vcvtps2pd(scratch, Xbyak::Xmm(source));
+	generator.vsqrtpd(scratch, scratch);
+	generator.vbroadcastsd(Xbyak::Ymm(destination), ptr[rip + one]);
+	generator.vdivpd(scratch, Xbyak::Ymm(destination), scratch);
+	generator.vcvtpd2ps(Xbyak::Xmm(destination), scratch); // Clears upper YMM lanes.
+	generator.ldmxcsr(ptr[rsp + 32]);
+	generator.vmovdqu(scratch, ptr[rsp]);
+	generator.lea(rsp, ptr[rsp + GuestRedZoneSize + SpillSize]);
+	generator.jmp(done);
+	// A failed label reference can have an incomplete displacement. Do not bind it.
+	if (Xbyak::GetError() != 0) {
+		return;
+	}
+	generator.L(one);
+	generator.dq(0x3ff0000000000000ULL);
+	generator.L(done);
+}
+
+bool IsSupportedBitFieldInstruction(const DecodedCodeInstruction& decoded) {
+	const bool insert = decoded.instruction.mnemonic == ZYDIS_MNEMONIC_INSERTQ;
+	if (!insert && decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) {
+		return false;
+	}
+	const auto is_xmm = [](const ZydisDecodedOperand& operand) {
+		return operand.type == ZYDIS_OPERAND_TYPE_REGISTER &&
+		       operand.reg.value >= ZYDIS_REGISTER_XMM0 &&
+		       operand.reg.value <= ZYDIS_REGISTER_XMM15;
+	};
+	const size_t immediate_index = insert ? 2 : 1;
+	const bool   immediate =
+	    decoded.instruction.operand_count_visible == immediate_index + 2 &&
+	    decoded.operands[immediate_index].type == ZYDIS_OPERAND_TYPE_IMMEDIATE &&
+	    decoded.operands[immediate_index + 1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE;
+	if (!is_xmm(decoded.operands[0]) || (insert && !is_xmm(decoded.operands[1])) ||
+	    (!immediate &&
+	     !(decoded.instruction.operand_count_visible == 2 && is_xmm(decoded.operands[1])))) {
+		return false;
+	}
+	// Match the trap handler's encoding support: 66/F2 [REX] 0F 78/79 ModRM [imm8 imm8].
+	const auto*  code   = reinterpret_cast<const u8*>(decoded.address);
+	const size_t opcode = (code[1] & 0xf0u) == 0x40u ? 2 : 1;
+	return code[0] == (insert ? 0xf2 : 0x66) && code[opcode] == 0x0f &&
+	       code[opcode + 1] == (immediate ? 0x78 : 0x79) &&
+	       decoded.instruction.length == opcode + (immediate ? 5 : 3);
+}
+
+void GenerateExtractQ(const DecodedCodeInstruction& decoded, Xbyak::CodeGenerator& generator) {
+	const Xbyak::Xmm destination(decoded.operands[0].reg.value - ZYDIS_REGISTER_XMM0);
+	// Keep spills below the guest red zone on both hosts, and restore all modified flags/GPRs.
+	generator.lea(rsp, ptr[rsp - GuestRedZoneSize]);
+	generator.pushfq();
+	generator.push(rax);
+	generator.push(rcx);
+	generator.push(rdx);
+	if (decoded.operands[1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE) {
+		const u32 length = decoded.operands[1].imm.value.u & 63u;
+		const u32 index  = decoded.operands[2].imm.value.u & 63u;
+		const u64 mask   = length == 0 ? UINT64_MAX : (u64 {1} << length) - 1;
+		generator.movq(rax, destination);
+		if (index != 0) {
+			generator.shr(rax, index);
+		}
+		generator.mov(rdx, mask);
+	} else {
+		const Xbyak::Xmm source(decoded.operands[1].reg.value - ZYDIS_REGISTER_XMM0);
+		generator.movq(rax, source);
+		generator.mov(ecx, eax);
+		generator.neg(ecx);
+		generator.mov(rdx, UINT64_MAX);
+		// CL shifts use six bits: -length gives 64-length, with zero retaining all 64 bits.
+		generator.shr(rdx, cl);
+		generator.shr(rax, 8);
+		generator.mov(ecx, eax);
+		generator.movq(rax, destination);
+		generator.shr(rax, cl);
+	}
+	generator.and_(rax, rdx);
+	// Match the emulator: clear the upper XMM half, preserving the upper YMM half.
+	generator.movq(destination, rax);
+	generator.pop(rdx);
+	generator.pop(rcx);
+	generator.pop(rax);
+	generator.popfq();
+	generator.lea(rsp, ptr[rsp + GuestRedZoneSize]);
+}
+
+void CollectAmdInstructions(const DecodedFunction&                   function,
+                            std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
+                            GuestInstructionPatchResult&             result,
+                            GuestInstructionHostFeatures             host_features) {
 	for (const auto& [address, decoded]: function.instructions) {
-		if (!X64InstructionEmulator::IsReciprocalSquareRoot(decoded.instruction,
-		                                                    decoded.operands.data())) {
+		const auto&            instruction = decoded.instruction;
+		InstructionReplacement replacement {};
+		if (instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS &&
+		    instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX &&
+		    instruction.raw.vex.offset == 0 && decoded.operands[0].size == 128 &&
+		    decoded.operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER) {
+			replacement = InstructionReplacement::ReciprocalSquareRoot;
+		} else if (!host_features.sse4a && IsSupportedBitFieldInstruction(decoded)) {
+			replacement = instruction.mnemonic == ZYDIS_MNEMONIC_EXTRQ
+			                  ? InstructionReplacement::ExtractQ
+			                  : InstructionReplacement::InsertQ;
+		} else if (!host_features.rdpid && instruction.mnemonic == ZYDIS_MNEMONIC_RDPID) {
+			replacement = InstructionReplacement::ReadProcessorId;
+		} else if (!host_features.clwb && instruction.mnemonic == ZYDIS_MNEMONIC_CLWB) {
+			replacement = InstructionReplacement::CacheLineWriteBack;
+		} else {
 			continue;
 		}
-		const bool protect_red_zone = decoded.red_zone_live.any();
-		sites.push_back({address, decoded.instruction.length, protect_red_zone});
-		if (protect_red_zone) {
-			rewrite_sites[address].protect_red_zone = true;
-		}
+		++ReplacementCounts(result, replacement).found;
+		rewrite_sites[address].replacement = replacement;
 	}
 }
 
-uint64_t ApplyReciprocalSquareRootPatches(const PatchModule& module,
-                                        std::span<const ReciprocalSquareRootSite> sites,
-                                        uint64_t trampoline_addr, uint64_t trampoline_size) {
-	// Validate every required relocation before introducing any traps.
-	for (const auto& site: sites) {
-		if (site.requires_red_zone_protection &&
-		    !module.patched.contains(reinterpret_cast<u8*>(site.address))) {
-			EXIT("Cannot preserve the guest red zone at emulated VRSQRTPS 0x%016" PRIx64 "\n",
-			     static_cast<u64>(site.address));
-		}
+void MarkInstructionTrap(u8* code, const ZydisDecodedInstruction& instruction,
+                         InstructionReplacement replacement) {
+	if (replacement == InstructionReplacement::ReciprocalSquareRoot) {
+		// Clear a reserved VEX.vvvv bit, retaining both register operands for the handler.
+		code[instruction.raw.vex.size - 1] &= ~0x08u;
 	}
-
-	uint64_t patched = 0;
-	for (const auto& site: sites) {
-		if (!module.patched.contains(reinterpret_cast<u8*>(site.address))) {
-			patched += X64InstructionEmulator::PatchReciprocalSquareRoots(site.address, site.length);
-		}
-	}
-	return patched +
-	       X64InstructionEmulator::PatchReciprocalSquareRoots(trampoline_addr, trampoline_size);
+	// Other replacements already raise #UD on hosts without their CPU feature.
 }
 
-void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& function,
-                                const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
-                                RedZonePatchResult& result) {
+bool GenerateInstructionTrap(const DecodedCodeInstruction& decoded,
+                             InstructionReplacement replacement, Xbyak::CodeGenerator& generator,
+                             s64 stack_adjustment) {
+	if (replacement == InstructionReplacement::CacheLineWriteBack) {
+		return EncodeRelocatedInstruction(decoded, generator, stack_adjustment);
+	}
+	std::array<u8, ZYDIS_MAX_INSTRUCTION_LENGTH> code {};
+	std::memcpy(code.data(), reinterpret_cast<const void*>(decoded.address),
+	            decoded.instruction.length);
+	MarkInstructionTrap(code.data(), decoded.instruction, replacement);
+	generator.db(code.data(), decoded.instruction.length);
+	return true;
+}
+
+bool NeedsTrapRedZoneProtection(const DecodedCodeInstruction& decoded) {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	return decoded.red_zone_live.any();
+#else
+	return false;
+#endif
+}
+
+void TrapUnrelocatedInstructions(const PatchModule& module, const DecodedFunction& function,
+                                 const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
+                                 GuestInstructionPatchResult&                   result) {
+	for (const auto& [address, rewrite]: rewrite_sites) {
+		if (rewrite.replacement == InstructionReplacement::None ||
+		    module.patched.contains(reinterpret_cast<u8*>(address))) {
+			continue;
+		}
+		const auto& decoded = function.instructions.at(address);
+		if (NeedsTrapRedZoneProtection(decoded)) {
+			if (rewrite.replacement != InstructionReplacement::ReciprocalSquareRoot) {
+				// Leaving an unsupported instruction unchanged still traps with a live red zone.
+				EXIT("AMD CPU compatibility: cannot safely trap %s at %p (guest red zone is "
+				     "live)\n",
+				     ZydisMnemonicGetString(decoded.instruction.mnemonic),
+				     reinterpret_cast<void*>(address));
+			}
+			continue;
+		}
+		MarkInstructionTrap(reinterpret_cast<u8*>(address), decoded.instruction,
+		                    rewrite.replacement);
+		++ReplacementCounts(result, rewrite.replacement).trapped;
+	}
+}
+
+void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& function,
+                               const std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
+                               GuestInstructionPatchResult&                   result) {
 	struct RelocationSpan {
 		std::vector<const DecodedCodeInstruction*> instructions;
 		uintptr_t                                  patch_start {};
 		uintptr_t                                  continuation {};
 		size_t                                     patch_size {};
+		bool                                       trap_replacements {};
 	};
 
-	const auto emit_span = [&](const RelocationSpan& span) -> std::optional<size_t> {
-		const size_t trampoline_offset = module->trampoline_gen.getSize();
-		if (module->trampoline_exhausted) {
-			return std::nullopt;
-		}
-		try {
+	const auto emit_span = [&](RelocationSpan& span) -> std::optional<size_t> {
+		auto&        generator         = module->trampoline_gen;
+		const size_t trampoline_offset = generator.getSize();
+		const bool   has_replacements =
+		    std::ranges::any_of(span.instructions, [&](const auto* decoded) {
+			    const auto rewrite = rewrite_sites.find(decoded->address);
+			    return rewrite != rewrite_sites.end() &&
+			           rewrite->second.replacement != InstructionReplacement::None;
+		    });
+		for (bool trap_replacements: {false, true}) {
+			span.trap_replacements = trap_replacements;
+			Xbyak::ClearError();
+			bool encoded = true;
 			for (const auto* decoded: span.instructions) {
-				const auto rewrite = rewrite_sites.find(decoded->address);
+				const auto rewrite     = rewrite_sites.find(decoded->address);
+				const auto replacement = rewrite != rewrite_sites.end()
+				                             ? rewrite->second.replacement
+				                             : InstructionReplacement::None;
+				const bool uses_trap   = UsesInstructionTrap(replacement, trap_replacements);
 				const bool protected_indirect_call =
 				    rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call;
-				const bool protect_red_zone = rewrite != rewrite_sites.end() &&
-				                              rewrite->second.protect_red_zone &&
-				                              !protected_indirect_call;
+				const bool protect_red_zone =
+				    (rewrite != rewrite_sites.end() && rewrite->second.protect_red_zone &&
+				     !protected_indirect_call) ||
+				    ((uses_trap || replacement == InstructionReplacement::CacheLineWriteBack) &&
+				     NeedsTrapRedZoneProtection(*decoded));
 				if (protect_red_zone) {
-					module->trampoline_gen.lea(rsp, ptr[rsp - GuestRedZoneSize]);
+					generator.lea(rsp, ptr[rsp - GuestRedZoneSize]);
 				}
-				if (protected_indirect_call) {
-					if (!GenerateProtectedIndirectCall(*decoded, module->trampoline_gen)) {
-						module->trampoline_gen.setSize(trampoline_offset);
-						return std::nullopt;
+				if (replacement != InstructionReplacement::None) {
+					if (uses_trap) {
+						encoded = GenerateInstructionTrap(*decoded, replacement, generator,
+						                                  protect_red_zone ? GuestRedZoneSize : 0);
+					} else if (replacement == InstructionReplacement::ReciprocalSquareRoot) {
+						GenerateReciprocalSquareRoot(*decoded, generator);
+					} else if (replacement == InstructionReplacement::ExtractQ) {
+						GenerateExtractQ(*decoded, generator);
+					} else {
+						// CLFLUSH writes back dirty data too; invalidation is a permitted stronger
+						// action.
+						encoded = EncodeRelocatedInstruction(
+						    *decoded, generator, protect_red_zone ? GuestRedZoneSize : 0,
+						    ZYDIS_MNEMONIC_CLFLUSH);
 					}
-				} else if (!EncodeRelocatedInstruction(*decoded, module->trampoline_gen)) {
-					module->trampoline_gen.setSize(trampoline_offset);
-					return std::nullopt;
+				} else if (protected_indirect_call) {
+					encoded = GenerateProtectedIndirectCall(*decoded, generator);
+				} else {
+					encoded = EncodeRelocatedInstruction(*decoded, generator);
 				}
 				if (protect_red_zone && !decoded->replaces_stack_pointer) {
-					module->trampoline_gen.lea(rsp, ptr[rsp + GuestRedZoneSize]);
+					generator.lea(rsp, ptr[rsp + GuestRedZoneSize]);
+				}
+				if (!encoded || Xbyak::GetError() != 0) {
+					break;
 				}
 			}
-			module->trampoline_gen.jmp(reinterpret_cast<void*>(span.continuation));
-		} catch (const Xbyak::Error& error) {
-			module->trampoline_gen.setSize(trampoline_offset);
-			if (HandleTrampolineError(module, error)) {
-				return std::nullopt;
+			if (encoded && Xbyak::GetError() == 0) {
+				generator.jmp(reinterpret_cast<void*>(span.continuation));
+				if (Xbyak::GetError() == 0) {
+					return trampoline_offset;
+				}
+			}
+			const int error = Xbyak::GetError();
+			// Roll back before retrying this uncommitted span with smaller trap replacements.
+			generator.reset();
+			generator.setSize(trampoline_offset);
+			if (error != 0 && !HandleTrampolineError(module, error)) {
+				EXIT("Guest instruction trampoline generation failed: %s\n",
+				     Xbyak::ConvertErrorToString(error));
+			}
+			if (error != Xbyak::ERR_CODE_IS_TOO_BIG || !has_replacements) {
+				break;
 			}
-			throw;
 		}
-		return trampoline_offset;
+		return std::nullopt;
 	};
 
 	const auto record_rewrites = [&](const RelocationSpan& span) {
@@ -881,6 +1099,14 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 			if (rewrite->second.protect_red_zone && decoded->accesses_memory) {
 				++result.patched_memory_instruction_count;
 			}
+			if (rewrite->second.replacement != InstructionReplacement::None) {
+				auto& counts = ReplacementCounts(result, rewrite->second.replacement);
+				if (UsesInstructionTrap(rewrite->second.replacement, span.trap_replacements)) {
+					++counts.trapped;
+				} else {
+					++counts.native;
+				}
+			}
 		}
 	};
 
@@ -983,6 +1209,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 		patch_gen.setSize(span.patch_start - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
 		patch_gen.jmp(trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
 		patch_gen.nop(span.patch_size - NearJumpSize);
+		EXIT_IF(Xbyak::GetError() != 0);
 
 		record_rewrites(span);
 		patched_spans.emplace_back(span.patch_start, span.continuation);
@@ -1021,7 +1248,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 			continue;
 		}
 
-		const RelocationSpan site_span {
+		RelocationSpan site_span {
 		    .instructions = {&site_instruction->second},
 		    .patch_start  = site,
 		    .continuation = site + site_instruction->second.instruction.length,
@@ -1049,12 +1276,14 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 			patch_gen.reset();
 			patch_gen.setSize(*relay_slot - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
 			patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
+			EXIT_IF(Xbyak::GetError() != 0);
 
 			patch_gen.reset();
 			patch_gen.setSize(site - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
 			patch_gen.jmp(reinterpret_cast<void*>(*relay_slot),
 			              Xbyak::CodeGenerator::LabelType::T_SHORT);
 			patch_gen.nop(site_span.patch_size - ShortJumpSize);
+			EXIT_IF(Xbyak::GetError() != 0);
 
 			record_rewrites(site_span);
 			patched_spans.emplace_back(site_span.patch_start, site_span.continuation);
@@ -1187,6 +1416,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 			patch_gen.reset();
 			patch_gen.setSize(relay_address - reinterpret_cast<uintptr_t>(patch_gen.getCode()));
 			patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
+			EXIT_IF(Xbyak::GetError() != 0);
 			std::erase(relay_slots, relay_address);
 			std::erase(short_relay_slots, relay_address);
 		} else {
@@ -1201,6 +1431,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 			patch_gen.jmp(host_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
 			patch_gen.jmp(site_trampoline, Xbyak::CodeGenerator::LabelType::T_NEAR);
 			patch_gen.nop(host_span->patch_size - NearJumpSize * 2);
+			EXIT_IF(Xbyak::GetError() != 0);
 		}
 
 		uintptr_t jump_target = relay_address;
@@ -1210,6 +1441,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 			                  reinterpret_cast<uintptr_t>(patch_gen.getCode()));
 			patch_gen.jmp(reinterpret_cast<void*>(jump_target),
 			              Xbyak::CodeGenerator::LabelType::T_SHORT);
+			EXIT_IF(Xbyak::GetError() != 0);
 			jump_target       = final_short_jump;
 			const auto parent = relay_parent.find(final_short_jump);
 			ASSERT(parent != relay_parent.end());
@@ -1223,6 +1455,7 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 		patch_gen.jmp(reinterpret_cast<void*>(jump_target),
 		              Xbyak::CodeGenerator::LabelType::T_SHORT);
 		patch_gen.nop(site_span.patch_size - ShortJumpSize);
+		EXIT_IF(Xbyak::GetError() != 0);
 
 		if (host_span) {
 			record_rewrites(*host_span);
@@ -1240,10 +1473,30 @@ void RelocateRedZoneInstructions(PatchModule* module, const DecodedFunction& fun
 }
 } // namespace
 
-RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
-                                          std::span<const uintptr_t> function_starts,
-                                          bool protect_memory, bool emulate_rsqrt) {
-	RedZonePatchResult result {};
+GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
+	static const auto features = [] {
+		const Xbyak::util::Cpu cpu;
+		uint32_t               leaf[4] {};
+		Xbyak::util::Cpu::getCpuid(0, leaf);
+		bool rdpid = false;
+		if (leaf[0] >= 7) {
+			Xbyak::util::Cpu::getCpuidEx(7, 0, leaf);
+			rdpid = (leaf[2] & (1u << 22)) != 0;
+		}
+		return GuestInstructionHostFeatures {cpu.has(Xbyak::util::Cpu::tSSE4a), rdpid,
+		                                     cpu.has(Xbyak::util::Cpu::tCLWB)};
+	}();
+	return features;
+}
+
+GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
+                                                   std::span<const uintptr_t> function_starts,
+                                                   bool protect_memory, bool emulate_amd,
+                                                   GuestInstructionHostFeatures host_features) {
+	GuestInstructionPatchResult result {};
+	if (!protect_memory && !emulate_amd) {
+		return result;
+	}
 	auto*              module = GetContainingModule(reinterpret_cast<void*>(segment_addr));
 	if (module == nullptr || function_starts.empty()) {
 		return result;
@@ -1262,8 +1515,11 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
 	starts.erase(unique_end, starts.end());
 
 	std::unique_lock lock {module->mutex};
-	const size_t trampoline_begin = module->trampoline_gen.getSize();
-	std::vector<ReciprocalSquareRootSite> reciprocal_sqrt_sites;
+	const size_t     trampoline_begin = module->trampoline_gen.getSize();
+	bool analyze_red_zone = protect_memory;
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	analyze_red_zone |= emulate_amd;
+#endif
 	for (size_t function_index = 0; function_index < starts.size(); ++function_index) {
 		const uintptr_t function_start = starts[function_index];
 		const uintptr_t function_end =
@@ -1274,7 +1530,9 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
 
 		++result.function_count;
 		auto function = DecodeFunction(function_start, function_end, segment_addr, segment_end);
-		AnalyzeRedZoneLiveness(function);
+		if (analyze_red_zone) {
+			AnalyzeRedZoneLiveness(function);
+		}
 		result.instruction_count += function.instructions.size();
 
 		std::map<uintptr_t, InstructionRewrite> rewrite_sites;
@@ -1286,22 +1544,17 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
 		if (protect_memory) {
 			CollectRedZoneMemoryInstructions(function, rewrite_sites, result);
 		}
-		if (emulate_rsqrt) {
-			CollectReciprocalSquareRoots(function, rewrite_sites, reciprocal_sqrt_sites);
+		if (emulate_amd) {
+			CollectAmdInstructions(function, rewrite_sites, result, host_features);
 		}
 		if (!rewrite_sites.empty()) {
-			RelocateRedZoneInstructions(module, function, rewrite_sites, result);
+			RelocateGuestInstructions(module, function, rewrite_sites, result);
+			TrapUnrelocatedInstructions(*module, function, rewrite_sites, result);
 		}
 	}
-	// Preserve valid instruction encodings throughout CFG analysis and relocation.
-	// Only now turn reciprocal roots into traps, including the relocated copies.
 	const auto trampoline_addr =
 	    reinterpret_cast<u64>(module->trampoline_gen.getCode()) + trampoline_begin;
 	const auto trampoline_size = module->trampoline_gen.getSize() - trampoline_begin;
-	if (emulate_rsqrt) {
-		result.reciprocal_sqrt_instruction_count = ApplyReciprocalSquareRootPatches(
-		    *module, reciprocal_sqrt_sites, trampoline_addr, trampoline_size);
-	}
 	Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
 	if (trampoline_size != 0) {
 		Common::VirtualMemory::FlushInstructionCache(trampoline_addr, trampoline_size);
@@ -1311,7 +1564,12 @@ RedZonePatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
 
 #else
 
-RedZonePatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool) {
+GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
+	return {};
+}
+
+GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool,
+                                                   GuestInstructionHostFeatures) {
 	return {};
 }
 
@@ -1492,9 +1750,9 @@ bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_fram
 	return true;
 }
 
-void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
-                                uint64_t trampoline_area_size) {
-#if defined(_WIN32)
+void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
+                                         void* trampoline_area_ptr, uint64_t trampoline_area_size) {
+#if !defined(__APPLE__)
 	EXIT_IF(module_ptr == nullptr || module_size == 0 || trampoline_area_ptr == nullptr ||
 	        trampoline_area_size == 0);
 	const auto module_addr = reinterpret_cast<u64>(module_ptr);
@@ -1511,8 +1769,8 @@ void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* tr
 #endif
 }
 
-void UnregisterRedZonePatchModule(void* module_ptr) {
-#if defined(_WIN32)
+void UnregisterGuestInstructionPatchModule(void* module_ptr) {
+#if !defined(__APPLE__)
 	g_patch_modules.erase(reinterpret_cast<u64>(module_ptr));
 #else
 	(void)module_ptr;

src/loader/guestInstructionPatcher.h

@@ -0,0 +1,70 @@
+// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#ifndef KYTY_LOADER_GUEST_INSTRUCTION_PATCHER_H_
+#define KYTY_LOADER_GUEST_INSTRUCTION_PATCHER_H_
+
+#include "common/common.h"
+
+#include <span>
+#include <vector>
+
+namespace Loader {
+
+struct GuestInstructionHostFeatures {
+	bool sse4a = false;
+	bool rdpid = false;
+	bool clwb  = false;
+};
+
+GuestInstructionHostFeatures GetGuestInstructionHostFeatures();
+
+struct InstructionPatchCounts {
+	uint64_t found   = 0;
+	uint64_t native  = 0;
+	uint64_t trapped = 0;
+
+	uint64_t Skipped() const { return found - native - trapped; }
+
+	InstructionPatchCounts& operator+=(const InstructionPatchCounts& other) {
+		found += other.found;
+		native += other.native;
+		trapped += other.trapped;
+		return *this;
+	}
+};
+
+struct GuestInstructionPatchResult {
+	uint64_t               function_count                           = 0;
+	uint64_t               instruction_count                        = 0;
+	uint64_t               red_zone_function_count                  = 0;
+	uint64_t               memory_instruction_count                 = 0;
+	uint64_t               short_memory_instruction_count           = 0;
+	uint64_t               patched_memory_instruction_count         = 0;
+	uint64_t               stack_dependent_memory_instruction_count = 0;
+	uint64_t               control_flow_memory_instruction_count    = 0;
+	uint64_t               unrelocatable_memory_instruction_count   = 0;
+	uint64_t               indirect_red_zone_function_count         = 0;
+	InstructionPatchCounts reciprocal_sqrt;
+	InstructionPatchCounts extrq;
+	InstructionPatchCounts insertq;
+	InstructionPatchCounts rdpid;
+	InstructionPatchCounts clwb;
+};
+
+void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
+                                         void* trampoline_area_ptr, uint64_t trampoline_area_size);
+void UnregisterGuestInstructionPatchModule(void* module_ptr);
+
+// Apply enabled instruction fixes using native trampolines or safe trap fallbacks.
+GuestInstructionPatchResult PatchGuestInstructions(
+    uint64_t segment_addr, uint64_t segment_size, std::span<const uintptr_t> function_starts,
+    bool protect_memory, bool emulate_amd,
+    GuestInstructionHostFeatures host_features = GetGuestInstructionHostFeatures());
+
+bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
+                                 std::vector<uintptr_t>* function_starts);
+
+} // namespace Loader
+
+#endif /* KYTY_LOADER_GUEST_INSTRUCTION_PATCHER_H_ */

src/loader/redZonePatcher.h

@@ -1,42 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2026 shadPS4 Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#ifndef KYTY_LOADER_RED_ZONE_PATCHER_H_
-#define KYTY_LOADER_RED_ZONE_PATCHER_H_
-
-#include "common/common.h"
-
-#include <span>
-#include <vector>
-
-namespace Loader {
-
-struct RedZonePatchResult {
-	uint64_t function_count                           = 0;
-	uint64_t instruction_count                        = 0;
-	uint64_t red_zone_function_count                   = 0;
-	uint64_t memory_instruction_count                 = 0;
-	uint64_t short_memory_instruction_count           = 0;
-	uint64_t patched_memory_instruction_count         = 0;
-	uint64_t stack_dependent_memory_instruction_count = 0;
-	uint64_t control_flow_memory_instruction_count    = 0;
-	uint64_t unrelocatable_memory_instruction_count   = 0;
-	uint64_t indirect_red_zone_function_count         = 0;
-	uint64_t reciprocal_sqrt_instruction_count        = 0;
-};
-
-void RegisterRedZonePatchModule(void* module_ptr, uint64_t module_size, void* trampoline_area_ptr,
-                                uint64_t trampoline_area_size);
-void UnregisterRedZonePatchModule(void* module_ptr);
-
-// Analyze and protect fault sites before replacing reciprocal roots with traps.
-RedZonePatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size,
-                                          std::span<const uintptr_t> function_starts,
-                                          bool protect_memory, bool emulate_rsqrt);
-
-bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
-                                 std::vector<uintptr_t>* function_starts);
-
-} // namespace Loader
-
-#endif /* KYTY_LOADER_RED_ZONE_PATCHER_H_ */

src/loader/runtimeLinker.cpp

@@ -17,8 +17,8 @@
 #include "kernel/pthread.h"
 #include "loader/elf.h"
 #include "loader/gamePatch.h"
+#include "loader/guestInstructionPatcher.h"
 #include "loader/jit.h"
-#include "loader/redZonePatcher.h"
 #include "loader/symbolDatabase.h"
 #include "loader/x64InstructionEmulator.h"
 
@@ -1812,38 +1812,39 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	uint64_t tls_handler_size = is_shared ? 0 : Jit::SafeCall::GetSize();
 	EXIT_IF(tls_handler_size > UINT64_MAX - program->base_size_aligned);
 	program->mapped_size = program->base_size_aligned + tls_handler_size;
-	const bool emulate_rsqrt = Config::AmdCpuEnabled();
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	const bool         protect_memory_faults   = Config::RedZoneProtectionEnabled();
-	const bool         use_red_zone_protection = protect_memory_faults || emulate_rsqrt;
-	constexpr uint64_t RED_ZONE_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
-	if (use_red_zone_protection) {
-		EXIT_IF(RED_ZONE_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
-		program->mapped_size += RED_ZONE_TRAMPOLINE_SIZE;
-	}
+#if !defined(__APPLE__)
+	const bool emulate_amd = Config::AmdCpuEnabled();
+#else
+	const bool emulate_amd = false;
 #endif
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	const bool protect_memory_faults = Config::RedZoneProtectionEnabled();
+#else
+	const bool protect_memory_faults = false;
+#endif
+	const bool patch_guest_instructions = protect_memory_faults || emulate_amd;
+
+	constexpr uint64_t INSTRUCTION_TRAMPOLINE_SIZE = 8u * 1024u * 1024u;
+	if (patch_guest_instructions) {
+		EXIT_IF(INSTRUCTION_TRAMPOLINE_SIZE > UINT64_MAX - program->mapped_size);
+		program->mapped_size += INSTRUCTION_TRAMPOLINE_SIZE;
+	}
 
 	program->base_vaddr = Libs::LibKernel::Memory::AllocateProgramMemory(
 	    g_desired_base_addr, program->mapped_size, Common::VirtualMemory::Mode::ExecuteReadWrite,
 	    Common::PathToString(program->file_name.filename()).c_str());
 	EXIT_IF(program->base_vaddr == 0);
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	if (use_red_zone_protection) {
-		program->red_zone_trampoline_vaddr = program->base_vaddr + program->base_size_aligned;
-		program->red_zone_trampoline_size  = RED_ZONE_TRAMPOLINE_SIZE;
-		RegisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr),
-		                           program->base_size_aligned,
-		                           reinterpret_cast<void*>(program->red_zone_trampoline_vaddr),
-		                           program->red_zone_trampoline_size);
+	if (patch_guest_instructions) {
+		const auto trampoline_addr           = program->base_vaddr + program->base_size_aligned;
+		program->instruction_trampoline_size = INSTRUCTION_TRAMPOLINE_SIZE;
+		RegisterGuestInstructionPatchModule(
+		    reinterpret_cast<void*>(program->base_vaddr), program->base_size_aligned,
+		    reinterpret_cast<void*>(trampoline_addr), program->instruction_trampoline_size);
 	}
-#endif
 	if (!is_shared) {
-		program->tls.handler_vaddr = program->base_vaddr + program->base_size_aligned;
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		program->tls.handler_vaddr += program->red_zone_trampoline_size;
-#endif
+		program->tls.handler_vaddr =
+		    program->base_vaddr + program->base_size_aligned + program->instruction_trampoline_size;
 	}
 
 	g_desired_base_addr += CODE_BASE_INCR * (1 + program->mapped_size / CODE_BASE_INCR);
@@ -1863,10 +1864,8 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 	}
 
 	std::vector<std::pair<uint64_t, uint64_t>> executable_segments;
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	uint64_t                                   eh_frame_header_addr = 0;
 	uint64_t                                   eh_frame_header_size = 0;
-#endif
 
 	for (Elf64_Half i = 0; i < ehdr->e_phnum; i++) {
 		if (phdr[i].p_memsz != 0 && (phdr[i].p_type == PT_LOAD || phdr[i].p_type == PT_OS_RELRO)) {
@@ -1925,52 +1924,68 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 			program->proc_param_vaddr = phdr[i].p_vaddr + program->base_vaddr;
 		}
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		if (use_red_zone_protection && phdr[i].p_type == PT_GNU_EH_FRAME) {
+		if (patch_guest_instructions && phdr[i].p_type == PT_GNU_EH_FRAME) {
 			eh_frame_header_addr = phdr[i].p_vaddr + program->base_vaddr;
 			eh_frame_header_size = phdr[i].p_memsz;
 		}
-#endif
 	}
 
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	std::vector<uintptr_t> function_starts;
-	if (use_red_zone_protection) {
-		if (!DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
-		                                 &function_starts)) {
-			LOGF("Windows guest red-zone patching could not decode function boundaries for %s\n",
-			     Common::PathToString(program->file_name).c_str());
+	if (patch_guest_instructions) {
+		std::vector<uintptr_t> function_starts;
+		const bool             have_function_starts =
+		    DecodeEhFrameFunctionStarts(eh_frame_header_addr, eh_frame_header_size,
+		                                &function_starts) &&
+		    !function_starts.empty();
+		const auto module_name = Common::PathToString(program->file_name.filename());
+		if (!have_function_starts) {
+			Log::WriteToConsoleAndLog(
+			    fmt::format("{}: {} not patched (function boundaries unavailable)\n",
+			                emulate_amd ? "AMD CPU compatibility" : "Guest red-zone protection",
+			                module_name));
 		}
-	}
-#endif
-	for (const auto& [segment_addr, segment_size]: executable_segments) {
-		uint64_t reciprocal_sqrt_count = 0;
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		if (use_red_zone_protection) {
-			const auto result =
-			    PatchGuestInstructions(segment_addr, segment_size, function_starts,
-			                           protect_memory_faults, emulate_rsqrt);
-			LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
-			     ", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
-			     ", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
-			     Common::PathToString(program->file_name.filename()).c_str(), result.function_count,
-			     result.red_zone_function_count, result.memory_instruction_count,
-			     result.patched_memory_instruction_count, result.short_memory_instruction_count,
-			     result.stack_dependent_memory_instruction_count,
-			     result.control_flow_memory_instruction_count,
-			     result.unrelocatable_memory_instruction_count);
-			reciprocal_sqrt_count = result.reciprocal_sqrt_instruction_count;
+		GuestInstructionPatchResult totals {};
+		for (const auto& [segment_addr, segment_size]: executable_segments) {
+			const auto result = PatchGuestInstructions(segment_addr, segment_size, function_starts,
+			                                           protect_memory_faults, emulate_amd);
+			totals.reciprocal_sqrt += result.reciprocal_sqrt;
+			totals.extrq += result.extrq;
+			totals.insertq += result.insertq;
+			totals.rdpid += result.rdpid;
+			totals.clwb += result.clwb;
+			if (protect_memory_faults) {
+				LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
+				     ", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
+				     ", control=%" PRIu64 ", unrelocatable=%" PRIu64 "\n",
+				     module_name.c_str(),
+				     result.function_count, result.red_zone_function_count,
+				     result.memory_instruction_count, result.patched_memory_instruction_count,
+				     result.short_memory_instruction_count,
+				     result.stack_dependent_memory_instruction_count,
+				     result.control_flow_memory_instruction_count,
+				     result.unrelocatable_memory_instruction_count);
+			}
 		}
-#else
-		if (emulate_rsqrt) {
-			reciprocal_sqrt_count =
-			    X64InstructionEmulator::PatchReciprocalSquareRoots(segment_addr, segment_size);
-			Common::VirtualMemory::FlushInstructionCache(segment_addr, segment_size);
-		}
-#endif
-		if (reciprocal_sqrt_count != 0) {
-			LOGF("Guest VRSQRTPS emulation: %s, instructions=%" PRIu64 "\n",
-			     Common::PathToString(program->file_name.filename()).c_str(), reciprocal_sqrt_count);
+		if (emulate_amd && have_function_starts) {
+			InstructionPatchCounts combined {};
+			std::string            details;
+			for (const auto& [name, counts]: {std::pair {"VRSQRTPS", totals.reciprocal_sqrt},
+			                                  {"EXTRQ", totals.extrq},
+			                                  {"INSERTQ", totals.insertq},
+			                                  {"RDPID", totals.rdpid},
+			                                  {"CLWB", totals.clwb}}) {
+				combined += counts;
+				if (!details.empty()) details += "; ";
+				details += fmt::format("{}: native={}, trapped={}, skipped={}", name, counts.native,
+				                       counts.trapped, counts.Skipped());
+			}
+			const auto  found   = combined.found;
+			const auto  skipped = combined.Skipped();
+			const char* status  = found == 0         ? "no matching instructions"
+			                      : skipped == found ? "not patched"
+			                      : skipped != 0     ? "partially patched"
+			                                         : "patched";
+			Log::WriteToConsoleAndLog(
+			    fmt::format("AMD CPU compatibility: {} {} ({})\n", module_name, status, details));
 		}
 	}
 
@@ -1999,11 +2014,9 @@ void RuntimeLinker::DeleteProgram(Program* p) {
 
 	if (program->base_vaddr != 0 || program->mapped_size != 0) {
 		EXIT_IF(program->base_vaddr == 0 || program->mapped_size == 0);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-		if (program->red_zone_trampoline_size != 0) {
-			UnregisterRedZonePatchModule(reinterpret_cast<void*>(program->base_vaddr));
+		if (program->instruction_trampoline_size != 0) {
+			UnregisterGuestInstructionPatchModule(reinterpret_cast<void*>(program->base_vaddr));
 		}
-#endif
 		EXIT_IF(
 		    !Libs::LibKernel::Memory::FreeGuestMemory(program->base_vaddr, program->mapped_size));
 	}

src/loader/runtimeLinker.h

@@ -123,10 +123,7 @@ struct Program {
 	uint64_t                     base_size         = 0;
 	uint64_t                     base_size_aligned = 0;
 	uint64_t                     mapped_size       = 0;
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	uint64_t red_zone_trampoline_vaddr = 0;
-	uint64_t red_zone_trampoline_size  = 0;
-#endif
+	uint64_t instruction_trampoline_size = 0;
 	std::unique_ptr<SymbolDatabase> export_symbols;
 	std::unique_ptr<SymbolDatabase> import_symbols;
 	ThreadLocalStorage              tls;

src/loader/x64InstructionEmulator.cpp

@@ -5,10 +5,9 @@
 #include <Zydis/Zydis.h>
 #include <bit>
 #include <cstring>
-#if !defined(__APPLE__)
 #include <emmintrin.h>
 #include <xmmintrin.h>
-#endif
+#include <cpuid.h>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
@@ -407,12 +406,12 @@ struct Context {
 	void                   Advance(size_t length) { native->Rip += length; }
 	[[nodiscard]] void*    Xmm(uint8_t index) const { return &native->Xmm0 + index; }
 
-	void LoadGprs(uint64_t (&gpr)[16]) const {
-		const uint64_t registers[] = {native->Rax, native->Rcx, native->Rdx, native->Rbx,
-		                              native->Rsp, native->Rbp, native->Rsi, native->Rdi,
-		                              native->R8,  native->R9,  native->R10, native->R11,
-		                              native->R12, native->R13, native->R14, native->R15};
-		std::memcpy(gpr, registers, sizeof(gpr));
+	[[nodiscard]] auto& Gpr(uint8_t index) const {
+		DWORD64* registers[] = {&native->Rax, &native->Rcx, &native->Rdx, &native->Rbx,
+		                        &native->Rsp, &native->Rbp, &native->Rsi, &native->Rdi,
+		                        &native->R8,  &native->R9,  &native->R10, &native->R11,
+		                        &native->R12, &native->R13, &native->R14, &native->R15};
+		return *registers[index];
 	}
 
 	void ClearUpperYmm(uint8_t index) const {
@@ -438,6 +437,14 @@ struct Context {
 	void Advance(size_t length) {
 		native->uc_mcontext->__ss.__rip += static_cast<uint64_t>(length);
 	}
+	[[nodiscard]] auto& Gpr(uint8_t index) const {
+		auto*     state       = &native->uc_mcontext->__ss;
+		uint64_t* registers[] = {&state->__rax, &state->__rcx, &state->__rdx, &state->__rbx,
+		                         &state->__rsp, &state->__rbp, &state->__rsi, &state->__rdi,
+		                         &state->__r8,  &state->__r9,  &state->__r10, &state->__r11,
+		                         &state->__r12, &state->__r13, &state->__r14, &state->__r15};
+		return *registers[index];
+	}
 	// Darwin names the XMM file __fpu_xmm0..__fpu_xmm15 instead of exposing an array.
 	[[nodiscard]] void* Xmm(uint8_t index) const {
 		auto* fs = &native->uc_mcontext->__fs;
@@ -477,13 +484,11 @@ struct Context {
 		return native->uc_mcontext.fpregs->_xmm[index].element;
 	}
 
-	void LoadGprs(uint64_t (&gpr)[16]) const {
+	[[nodiscard]] auto& Gpr(uint8_t index) const {
 		constexpr int registers[] = {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP,
 		                             REG_RSI, REG_RDI, REG_R8,  REG_R9,  REG_R10, REG_R11,
 		                             REG_R12, REG_R13, REG_R14, REG_R15};
-		for (size_t i = 0; i < 16; ++i) {
-			gpr[i] = static_cast<uint64_t>(native->uc_mcontext.gregs[registers[i]]);
-		}
+		return native->uc_mcontext.gregs[registers[index]];
 	}
 
 	void ClearUpperYmm(uint8_t index) const {
@@ -507,6 +512,12 @@ struct Context {
 		}
 	}
 #endif
+
+	void LoadGprs(uint64_t (&gpr)[16]) const {
+		for (uint8_t i = 0; i < 16; ++i) {
+			gpr[i] = static_cast<uint64_t>(Gpr(i));
+		}
+	}
 };
 
 #if !defined(__APPLE__)
@@ -575,8 +586,8 @@ static bool TryEmulateSse4a(Context& context) {
 	if (rip[offset] != 0x0f) {
 		return false;
 	}
-	const bool register_extract = prefix == 0x66 && rip[offset + 1] == 0x79;
-	if (rip[offset + 1] != 0x78 && !register_extract) {
+	const bool register_form = rip[offset + 1] == 0x79;
+	if (rip[offset + 1] != 0x78 && !register_form) {
 		return false;
 	}
 
@@ -590,7 +601,7 @@ static bool TryEmulateSse4a(Context& context) {
 
 	// Immediate EXTRQ encodes its destination in r/m; the two-register form uses reg.
 	uint8_t dest_index = reg;
-	if (prefix == 0x66 && !register_extract) {
+	if (prefix == 0x66 && !register_form) {
 		dest_index = rm;
 	}
 	auto* dest_xmm = context.Xmm(dest_index);
@@ -599,15 +610,17 @@ static bool TryEmulateSse4a(Context& context) {
 		return false;
 	}
 	uint64_t dest[2] {};
-	uint64_t source = 0;
+	uint64_t source[2] {};
 	std::memcpy(dest, dest_xmm, sizeof(dest));
-	std::memcpy(&source, src_xmm, sizeof(source));
+	std::memcpy(source, src_xmm, sizeof(source));
 	uint8_t length             = 0;
 	uint8_t index              = 0;
 	size_t  instruction_length = offset + 3;
-	if (register_extract) {
-		length = static_cast<uint8_t>(source);
-		index  = static_cast<uint8_t>(source >> 8u);
+	if (register_form) {
+		// EXTRQ controls occupy the low quadword; INSERTQ uses the upper quadword.
+		const uint64_t controls = source[prefix == 0x66 ? 0 : 1];
+		length                  = static_cast<uint8_t>(controls);
+		index                   = static_cast<uint8_t>(controls >> 8u);
 	} else {
 		length = rip[offset + 3];
 		index  = rip[offset + 4];
@@ -617,13 +630,79 @@ static bool TryEmulateSse4a(Context& context) {
 		dest[0] = ExtractBitField(dest[0], length, index);
 		dest[1] = 0;
 	} else {
-		dest[0] = InsertBitField(dest[0], source, length, index);
+		dest[0] = InsertBitField(dest[0], source[0], length, index);
 	}
 	std::memcpy(dest_xmm, dest, sizeof(dest));
 	context.Advance(instruction_length);
 	return true;
 }
 
+static bool ReadTscAux(uint32_t& auxiliary) {
+	uint32_t eax = 0;
+	uint32_t ebx = 0;
+	uint32_t ecx = 0;
+	uint32_t edx = 0;
+	if (!__get_cpuid(0x80000001u, &eax, &ebx, &ecx, &edx) || (edx & (1u << 27u)) == 0) {
+		return false;
+	}
+	asm volatile("rdtscp" : "=a"(eax), "=d"(edx), "=c"(auxiliary));
+	return true;
+}
+
+static void FlushCacheLine(uint64_t address, ZydisRegister segment) {
+	// Preserve FS/GS addressing rather than treating a segment-relative offset as a pointer.
+	if (segment == ZYDIS_REGISTER_FS) {
+		asm volatile("clflush %%fs:(%0)" : : "r"(address) : "memory");
+	} else if (segment == ZYDIS_REGISTER_GS) {
+		asm volatile("clflush %%gs:(%0)" : : "r"(address) : "memory");
+	} else {
+		_mm_clflush(reinterpret_cast<const void*>(address));
+	}
+}
+
+static bool TryEmulateCpuExtensions(Context& context) {
+	ZydisDecoder            decoder {};
+	ZydisDecodedInstruction instruction {};
+	ZydisDecodedOperand     operands[ZYDIS_MAX_OPERAND_COUNT] {};
+	if (!ZYAN_SUCCESS(
+	        ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)) ||
+	    !ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, reinterpret_cast<const void*>(context.Rip()),
+	                                         ZYDIS_MAX_INSTRUCTION_LENGTH, &instruction,
+	                                         operands))) {
+		return false;
+	}
+	if (instruction.mnemonic == ZYDIS_MNEMONIC_RDPID &&
+	    operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER) {
+		const auto reg_class = ZydisRegisterGetClass(operands[0].reg.value);
+		const auto index     = ZydisRegisterGetId(operands[0].reg.value);
+		uint32_t   auxiliary = 0;
+		if ((reg_class != ZYDIS_REGCLASS_GPR32 && reg_class != ZYDIS_REGCLASS_GPR64) || index < 0 ||
+		    index >= 16 || !ReadTscAux(auxiliary)) {
+			return false;
+		}
+		// RDPID and guest RDTSCP observe the same host TSC_AUX; only RDPID's destination changes.
+		context.Gpr(static_cast<uint8_t>(index)) = auxiliary;
+	} else if (instruction.mnemonic == ZYDIS_MNEMONIC_CLWB &&
+	           operands[0].type == ZYDIS_OPERAND_TYPE_MEMORY) {
+		ZydisRegisterContext registers {};
+		for (uint8_t i = 0; i < 16; ++i) {
+			const auto value                         = static_cast<uint64_t>(context.Gpr(i));
+			registers.values[ZYDIS_REGISTER_RAX + i] = value;
+			registers.values[ZYDIS_REGISTER_EAX + i] = static_cast<uint32_t>(value);
+		}
+		uint64_t address = 0;
+		if (!ZYAN_SUCCESS(ZydisCalcAbsoluteAddressEx(&instruction, &operands[0], context.Rip(),
+		                                             &registers, &address))) {
+			return false;
+		}
+		FlushCacheLine(address, operands[0].mem.segment);
+	} else {
+		return false;
+	}
+	context.Advance(instruction.length);
+	return true;
+}
+
 #if !defined(__APPLE__)
 
 static bool TryEmulateMonitorxMwaitx(Context& context) {
@@ -713,46 +792,6 @@ static bool TryEmulateReciprocalSquareRoot(Context& context) {
 
 #endif
 
-bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction,
-                            const ZydisDecodedOperand* operands) {
-	return instruction.mnemonic == ZYDIS_MNEMONIC_VRSQRTPS &&
-	       instruction.encoding == ZYDIS_INSTRUCTION_ENCODING_VEX &&
-	       instruction.raw.vex.offset == 0 && operands[0].size == 128 &&
-	       operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER;
-}
-
-uint64_t PatchReciprocalSquareRoots(uint64_t address, uint64_t size) {
-	uint64_t patched = 0;
-#if !defined(__APPLE__)
-	ZydisDecoder decoder {};
-	if (!ZYAN_SUCCESS(
-	        ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64))) {
-		return 0;
-	}
-	for (uint64_t offset = 0; offset < size;) {
-		auto*                   code = reinterpret_cast<uint8_t*>(address + offset);
-		ZydisDecodedInstruction instruction {};
-		ZydisDecodedOperand     operands[ZYDIS_MAX_OPERAND_COUNT] {};
-		if (!ZYAN_SUCCESS(
-		        ZydisDecoderDecodeFull(&decoder, code, size - offset, &instruction, operands))) {
-			++offset;
-			continue;
-		}
-		if (IsReciprocalSquareRoot(instruction, operands)) {
-			// vvvv is reserved (must be 1111b). Clear one bit to route this
-			// otherwise intact instruction through the illegal-instruction emulator.
-			code[instruction.raw.vex.size - 1] &= ~0x08u;
-			++patched;
-		}
-		offset += instruction.length;
-	}
-#else
-	(void)address;
-	(void)size;
-#endif
-	return patched;
-}
-
 bool TryEmulate(void* native_context) {
 	if (native_context == nullptr) {
 		return false;
@@ -769,13 +808,10 @@ bool TryEmulate(void* native_context) {
 	Context context {static_cast<ucontext_t*>(native_context)};
 #endif
 #if !defined(__APPLE__)
-	if (TryEmulateReciprocalSquareRoot(context)) {
-		return true;
-	}
-	return TryEmulateMonitorxMwaitx(context) || TryEmulateSse4a(context) ||
-	       TryEmulateShaNi(context);
+	return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) ||
+	       TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context);
 #else
-	return TryEmulateSse4a(context);
+	return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context);
 #endif
 }
 

src/loader/x64InstructionEmulator.h

@@ -1,14 +1,8 @@
 #ifndef KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
 #define KYTY_LOADER_X64_INSTRUCTION_EMULATOR_H_
 
-#include <Zydis/DecoderTypes.h>
-#include <cstdint>
-
 namespace Loader::X64InstructionEmulator {
 
-[[nodiscard]] bool IsReciprocalSquareRoot(const ZydisDecodedInstruction& instruction,
-                                         const ZydisDecodedOperand* operands);
-uint64_t           PatchReciprocalSquareRoots(uint64_t address, uint64_t size);
 [[nodiscard]] bool TryEmulate(void* native_context);
 
 } // namespace Loader::X64InstructionEmulator

tests/KernelFileSystemTests.cpp

@@ -25,10 +25,12 @@
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
+#include <future>
 #include <limits>
 #include <memory>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 
 namespace Libs::LibKernelApr {
@@ -811,6 +813,210 @@ void CheckAprPaths(const std::filesystem::path &root) {
   FileSystem::Umount("/app0");
 }
 
+std::array<int, 2> CreateTcpPair() {
+  namespace Net = Libs::Network::Net;
+  // Guest sockaddr_in: length, family, network-order port/address, padding.
+  std::array<uint8_t, 16> address {16, 2, 0, 0, 127, 0, 0, 1};
+  const int listener = Net::Socket(2, 1, 0);
+  Check(listener >= 0, "create loopback listener");
+  Check(Net::Bind(listener, address.data(), address.size()) == 0, "bind loopback");
+  Check(Net::Listen(listener, 1) == 0, "listen on loopback");
+  uint32_t address_size = address.size();
+  Check(Net::Getsockname(listener, address.data(), &address_size) == 0,
+        "get assigned loopback port");
+  const int writer = Net::Socket(2, 1, 0);
+  Check(writer >= 0 && Net::Connect(writer, address.data(), address_size) == 0,
+        "connect wake socket");
+  const int reader = Net::Accept(listener, nullptr, nullptr);
+  Check(reader >= 0, "accept wake socket");
+  Check(Net::SocketClose(listener) == 0, "close listener");
+  return {reader, writer};
+}
+
+#if defined(_WIN32)
+void CheckSocketReceiveConcurrency() {
+  namespace Net = Libs::Network::Net;
+  const auto [reader, writer] = CreateTcpPair();
+  const int timeout_ms = 3000;
+  Check(Net::Setsockopt(reader, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0,
+        "bound the blocked receive concurrency fixture");
+  std::promise<void> entered;
+  std::promise<int64_t> completed;
+  auto entered_future = entered.get_future();
+  auto completed_future = completed.get_future();
+  std::jthread receiver([&] {
+    char byte = 0;
+    entered.set_value();
+    completed.set_value(Net::Recv(reader, &byte, 1, 0x42));
+  });
+  Check(entered_future.wait_for(std::chrono::seconds(2)) == std::future_status::ready,
+        "start blocked reader");
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  char byte = 0;
+  const auto before_nonblocking = std::chrono::steady_clock::now();
+  Check(Net::Recv(reader, &byte, 1, 0x80) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK &&
+            std::chrono::steady_clock::now() - before_nonblocking < std::chrono::milliseconds(500) &&
+            completed_future.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout,
+        "DONTWAIT does not wait behind another blocked reader");
+  const auto before_close = std::chrono::steady_clock::now();
+  Check(Net::SocketClose(reader) == 0, "close succeeds while another thread receives");
+  Check(std::chrono::steady_clock::now() - before_close < std::chrono::seconds(1),
+        "close does not wait for the blocked receive timeout");
+  Check(completed_future.wait_for(std::chrono::seconds(1)) == std::future_status::ready,
+        "close wakes the blocked receive within one second");
+  Check(completed_future.get() <= 0, "closed receive reports EOF or error");
+  Check(std::chrono::steady_clock::now() - before_close < std::chrono::seconds(1),
+        "closing a socket wakes its blocked receive before the receive timeout");
+  receiver.join();
+  Check(Net::SocketClose(writer) == 0, "close blocked receive writer");
+
+  const auto [shutdown_reader, shutdown_writer] = CreateTcpPair();
+  Check(Net::Setsockopt(shutdown_reader, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0 &&
+            Net::Send(shutdown_writer, "x", 1, 0) == 1 &&
+            Net::Recv(shutdown_reader, &byte, 1, 0x42) == 1 && byte == 'x' &&
+            Net::Shutdown(shutdown_reader, 0) == 0 &&
+            Net::Recv(shutdown_reader, &byte, 1, 0) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_ESHUTDOWN,
+        "receive shutdown rejects bytes retained by an earlier peek");
+  Check(Net::SocketClose(shutdown_reader) == 0 && Net::SocketClose(shutdown_writer) == 0,
+        "close receive shutdown fixture");
+}
+
+void CheckSocketReceiveBuffer(int reader, int writer) {
+  namespace Net = Libs::Network::Net;
+  constexpr int PeekWaitAll = 0x42;
+  constexpr char Payload[] = "abcdef";
+  const int timeout_ms = 2000;
+  Check(Net::Setsockopt(reader, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0,
+        "bound blocking receive regressions with SO_RCVTIMEO");
+  Check(Net::Send(writer, Payload, 2, 0) == 2, "send first TCP fragment");
+  std::jthread sender([&] {
+    std::this_thread::sleep_for(std::chrono::milliseconds(20));
+    Check(Net::Send(writer, Payload + 2, 4, 0) == 4, "send second TCP fragment");
+  });
+  std::array<char, 8> received {};
+  Check(Net::Recv(reader, received.data(), 6, PeekWaitAll) == 6 &&
+            std::memcmp(received.data(), Payload, 6) == 0,
+        "combined peek waits for fragmented TCP bytes");
+  sender.join();
+  received.fill(0);
+  Check(Net::Recv(reader, received.data(), 6, PeekWaitAll) == 6 &&
+            std::memcmp(received.data(), Payload, 6) == 0,
+        "repeated combined peek preserves all bytes");
+
+  const int epoll = Net::EpollCreate("buffered receive", 0);
+  Net::NetEpollEvent registration {};
+  registration.events = 1;
+  registration.data.u64 = 0x12345678;
+  Check(epoll >= 0 && Net::EpollControl(epoll, 1, reader, &registration) == 0,
+        "register buffered socket readability");
+  const auto check_readable = [&](bool expected) {
+    std::array<uint64_t, 16> readable {};
+    const auto bit = uint64_t {1} << (reader % 64);
+    readable[reader / 64] = bit;
+    const std::array<int64_t, 2> timeout {expected ? 1 : 0, 0};
+    Check(Net::Select(reader + 1, readable.data(), nullptr, nullptr, timeout.data()) ==
+              (expected ? 1 : 0) && readable[reader / 64] == (expected ? bit : 0),
+          "select reflects buffered bytes and their removal");
+    if (expected) {
+      auto writable = readable;
+      Check(Net::Select(reader + 1, readable.data(), writable.data(), nullptr,
+                        timeout.data()) == 2 && readable[reader / 64] == bit &&
+                writable[reader / 64] == bit,
+            "select counts readable and writable bits once each");
+    }
+    Net::NetEpollEvent event {};
+    Check(Net::EpollWait(epoll, &event, 1, expected ? 1000000 : 0) ==
+              (expected ? 1 : 0) &&
+              (!expected || (event.events == 1 && event.data.u64 == registration.data.u64)),
+          "epoll reflects buffered bytes and preserves registration data");
+  };
+  check_readable(true);
+  std::array<uint8_t, 16> peer {}, expected_peer {};
+  uint32_t peer_size = peer.size(), expected_peer_size = expected_peer.size();
+  Check(Net::Getsockname(writer, expected_peer.data(), &expected_peer_size) == 0 &&
+            Net::Recv(reader, received.data(), 0, PeekWaitAll) == 0 &&
+            Net::Recvfrom(reader, received.data(), 2, 0, peer.data(), &peer_size) == 2 &&
+            peer_size == expected_peer_size && peer == expected_peer &&
+            std::memcmp(received.data(), Payload, 2) == 0 &&
+            FileSystem::KernelRead(reader, received.data(), 2) == 2 &&
+            std::memcmp(received.data(), Payload + 2, 2) == 0,
+        "zero-length peek preserves bytes; recvfrom returns peer and shares kernel read buffer");
+  check_readable(true);
+  Check(Net::Send(writer, "gh", 2, 0) == 2 &&
+            Net::Recv(reader, received.data(), 4, 0x40) == 4 &&
+            std::memcmp(received.data(), "efgh", 4) == 0,
+        "waitall joins buffered bytes with newly received bytes");
+  check_readable(false);
+  Check(Net::Send(writer, "mn", 2, 0) == 2, "send plain peek fixture");
+  const auto before_peek = std::chrono::steady_clock::now();
+  Check(Net::Recv(reader, received.data(), 4, 2) == 2 &&
+            std::memcmp(received.data(), "mn", 2) == 0 &&
+            std::chrono::steady_clock::now() - before_peek < std::chrono::milliseconds(500) &&
+            Net::Recv(reader, received.data(), 4, 0) == 2 &&
+            std::memcmp(received.data(), "mn", 2) == 0,
+        "plain peek returns available bytes promptly and preserves them for ordinary receive");
+
+  const int enabled = 1, disabled = 0;
+  Check(Net::Setsockopt(reader, 0xffff, 0x1200, &enabled, sizeof(enabled)) == 0 &&
+            Net::Recv(reader, received.data(), 4, PeekWaitAll) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK,
+        "empty nonblocking combined peek returns would-block");
+  Check(Net::Send(writer, "ij", 2, 0) == 2, "send nonblocking receive fixture");
+  check_readable(true);
+  *Libs::Posix::GetErrorAddr() = Libs::Posix::POSIX_EINVAL;
+  for (int i = 0; i < 2; ++i) {
+    Check(Net::Recv(reader, received.data(), 4, PeekWaitAll) == 2 &&
+              std::memcmp(received.data(), "ij", 2) == 0 &&
+              *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EINVAL,
+          "nonblocking combined peek preserves a short result and successful errno");
+  }
+  Check(Net::Recv(reader, received.data(), 4, 0) == 2 &&
+            std::memcmp(received.data(), "ij", 2) == 0,
+        "ordinary receive consumes a short buffered result");
+  check_readable(false);
+  Check(Net::Setsockopt(reader, 0xffff, 0x1200, &disabled, sizeof(disabled)) == 0 &&
+            Net::EpollDestroy(epoll) == 0,
+        "restore blocking receives and destroy epoll");
+  Check(Net::Recv(reader, received.data(), 4, PeekWaitAll | 0x80) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK,
+        "per-call nonblocking combined peek returns would-block on a blocking socket");
+  Check(Net::Send(writer, "kl", 2, 0) == 2 && Net::Shutdown(writer, 1) == 0,
+        "send a final partial payload and EOF");
+  for (int i = 0; i < 2; ++i) {
+    Check(Net::Recv(reader, received.data(), 4, PeekWaitAll) == 2 &&
+              std::memcmp(received.data(), "kl", 2) == 0,
+          "EOF ends waitall while repeated peeks preserve the short payload");
+  }
+  Check(Net::Recv(reader, received.data(), 4, 0x40) == 2 &&
+            std::memcmp(received.data(), "kl", 2) == 0 &&
+            Net::Recv(reader, received.data(), 4, PeekWaitAll) == 0,
+        "draining the final buffered payload exposes EOF");
+
+  const int datagram = Net::Socket(2, 2, 0);
+  const int datagram_writer = Net::Socket(2, 2, 0);
+  std::array<uint8_t, 16> address {16, 2, 0, 0, 127, 0, 0, 1};
+  uint32_t address_size = address.size();
+  Check(datagram >= 0 && datagram_writer >= 0 &&
+            Net::Bind(datagram, address.data(), address.size()) == 0 &&
+            Net::Getsockname(datagram, address.data(), &address_size) == 0 &&
+            Net::Setsockopt(datagram, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0 &&
+            Net::Sendto(datagram_writer, Payload, 6, 0, address.data(), address_size) == 6,
+        "create bounded UDP truncation fixture");
+  *Libs::Posix::GetErrorAddr() = Libs::Posix::POSIX_EINVAL;
+  Check(Net::Recvfrom(datagram, received.data(), 2, 0, nullptr, nullptr) == 2 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EINVAL &&
+            std::memcmp(received.data(), Payload, 2) == 0 &&
+            Net::Setsockopt(datagram, 0xffff, 0x1200, &enabled, sizeof(enabled)) == 0 &&
+            Net::Recv(datagram, received.data(), received.size(), 0) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK,
+        "UDP truncation returns its prefix and discards the rest of the datagram");
+  Check(Net::SocketClose(datagram) == 0 && Net::SocketClose(datagram_writer) == 0,
+        "close UDP truncation fixture");
+}
+#endif
+
 void CheckSocketWakeup() {
   namespace Net = Libs::Network::Net;
   Loader::SymbolDatabase symbols;
@@ -829,21 +1035,7 @@ void CheckSocketWakeup() {
   const auto net_send = reinterpret_cast<Send>(send_symbol->vaddr);
   const auto net_recv = reinterpret_cast<Recv>(recv_symbol->vaddr);
   auto *net_errno = reinterpret_cast<Errno>(errno_symbol->vaddr)();
-  // Guest sockaddr_in: length, family, network-order port/address, padding.
-  std::array<uint8_t, 16> address {16, 2, 0, 0, 127, 0, 0, 1};
-  const int listener = Net::Socket(2, 1, 0);
-  Check(listener >= 0, "create loopback listener");
-  Check(Net::Bind(listener, address.data(), address.size()) == 0, "bind loopback");
-  Check(Net::Listen(listener, 1) == 0, "listen on loopback");
-  uint32_t address_size = address.size();
-  Check(Net::Getsockname(listener, address.data(), &address_size) == 0,
-        "get assigned loopback port");
-  const int writer = Net::Socket(2, 1, 0);
-  Check(writer >= 0 && Net::Connect(writer, address.data(), address_size) == 0,
-        "connect wake socket");
-  const int reader = Net::Accept(listener, nullptr, nullptr);
-  Check(reader >= 0, "accept wake socket");
-  Check(Net::SocketClose(listener) == 0, "close listener");
+  const auto [reader, writer] = CreateTcpPair();
   const int enabled = 1;
   Check(Net::Setsockopt(writer, 6, 1, &enabled, sizeof(enabled)) == 0,
         "enable TCP_NODELAY");
@@ -911,6 +1103,10 @@ void CheckSocketWakeup() {
             *net_errno == Libs::Posix::POSIX_EPIPE,
         "Net send reports a broken pipe without raising host SIGPIPE");
   Check(Net::SocketClose(disconnected) == 0, "close unconnected socket");
+#endif
+#if defined(_WIN32)
+  CheckSocketReceiveBuffer(reader, writer);
+  CheckSocketReceiveConcurrency();
 #endif
   Check(Net::SocketClose(writer) == 0, "close wake writer");
   *net_errno = Libs::Posix::POSIX_EINVAL;

tests/macosSse4aTests.cpp

@@ -8,6 +8,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <initializer_list>
 #include <sys/mman.h>
 #include <sys/ucontext.h>
 #include <unistd.h>
@@ -61,6 +62,21 @@ constexpr uint8_t kInsertqXmm10Xmm9Len16Index8[] = {
 	0xc3,                                     // ret
 };
 
+constexpr uint8_t kInsertqXmm10Xmm9[] = {
+	0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9
+	0xf3, 0x44, 0x0f, 0x6f, 0x12, // movdqu (%rdx), %xmm10
+	0xf2, 0x45, 0x0f, 0x79, 0xd1, // insertq xmm10, xmm9
+	0xf3, 0x44, 0x0f, 0x7f, 0x17, // movdqu %xmm10, (%rdi)
+	0xc3,
+};
+
+constexpr uint8_t kInsertqXmm9Xmm9[] = {
+	0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9
+	0xf2, 0x45, 0x0f, 0x79, 0xc9, // insertq xmm9, xmm9
+	0xf3, 0x44, 0x0f, 0x7f, 0x0f, // movdqu %xmm9, (%rdi)
+	0xc3,
+};
+
 constexpr uint8_t kUd2[] = {
 	0x0f, 0x0b, // ud2
 	0xc3,       // ret
@@ -192,6 +208,31 @@ void TestRexHighRegisterInsertqWithIndex() {
 	Check(out[1] == 0xbbbbbbbbbbbbbbbbull, "insertq xmm10, xmm9, 16, 8 leaves the upper 64 bits untouched");
 }
 
+void TestInsertqRegister() {
+	const auto separate = MapCode<PairBlobFn>(kInsertqXmm10Xmm9);
+	const auto aliased  = MapCode<PairBlobFn>(kInsertqXmm9Xmm9);
+	for (const auto fn: {separate, aliased}) {
+		for (const uint64_t controls: {0xc4c8ull, 0xc0c0ull, 0xffc1ull}) {
+			const uint64_t source[2] {0x0123456789abcdefull, 0xdeadbeef00000000ull | controls};
+			const uint64_t destination[2] {0x8877665544332211ull, 0xfedcba9876543210ull};
+			const auto*    initial  = fn == aliased ? source : destination;
+			uint64_t       expected = initial[0];
+			const auto     length   = (controls & 63) == 0 ? 64 : controls & 63;
+			const auto     index    = (controls >> 8) & 63;
+			for (uint64_t bit = 0; bit < length && index + bit < 64; ++bit) {
+				const uint64_t mask = uint64_t {1} << (index + bit);
+				expected = (expected & ~mask) | (((source[0] >> bit) & 1) << (index + bit));
+			}
+			uint64_t  out[2] {};
+			const int before = IllegalHits();
+			fn(out, source, destination);
+			Check(IllegalHits() == before + 1, "register INSERTQ raises one SIGILL");
+			Check(out[0] == expected && out[1] == initial[1],
+			      "register INSERTQ uses upper source controls and preserves upper destination");
+		}
+	}
+}
+
 void TestUnknownInstructionRefused() {
 	g_refused.store(false, std::memory_order_relaxed);
 	g_refuse_skip.store(2, std::memory_order_relaxed);
@@ -213,6 +254,7 @@ int main() {
 	TestInsertqImmediate();
 	TestRexHighRegisterExtrq();
 	TestRexHighRegisterInsertqWithIndex();
+	TestInsertqRegister();
 	TestUnknownInstructionRefused();
 	std::printf("macosSse4aTests: all passed\n");
 	return 0;