net: emulate Windows PEEK and WAITALL receives

Co-authored-by: Firas Ben Mbarek <firasbenmbarek@outlook.com>
Co-authored-by: rudy-07 <kanhaiffco2007@gmail.com>
Co-authored-by: Matthew Hurley <171001439+theantipopau@users.noreply.github.com>
Co-authored-by: usopp-sama <go4av05.pvtx@gmail.com>
Co-authored-by: wruwami <wruwami@users.noreply.github.com>

src/libs/network.cpp

@@ -849,20 +849,38 @@ using SocketIoLength                                = size_t;
 #endif
 
 struct SocketTransport {
-	explicit SocketTransport(NativeSocket value): socket(value) {}
+	SocketTransport(NativeSocket value, int socket_type): socket(value), type(socket_type) {}
 	~SocketTransport() { Close(); }
+	bool HasPendingReceive() const {
+#if defined(_WIN32)
+		return buffered_bytes.load(std::memory_order_relaxed) != 0 ||
+		       receive_shutdown.load(std::memory_order_relaxed);
+#else
+		return false;
+#endif
+	}
 	int Close() {
-		if (socket == INVALID_NATIVE_SOCKET) {
+#if defined(_WIN32)
+		receive_shutdown.store(true, std::memory_order_relaxed);
+#endif
+		const auto value = socket.exchange(INVALID_NATIVE_SOCKET);
+		if (value == INVALID_NATIVE_SOCKET) {
 			return 0;
 		}
-		const auto value = std::exchange(socket, INVALID_NATIVE_SOCKET);
 #if defined(_WIN32)
 		return closesocket(value);
 #else
 		return ::close(value);
 #endif
 	}
-	NativeSocket socket;
+	std::atomic<NativeSocket> socket;
+	const int                type;
+#if defined(_WIN32)
+	std::mutex         receive_mutex;
+	std::vector<char>  receive_buffer;
+	std::atomic_size_t buffered_bytes   = 0;
+	std::atomic_bool   receive_shutdown = false;
+#endif
 };
 
 struct P2pEndpoint {
@@ -1164,7 +1182,8 @@ static int ConvertHostSockaddr(const sockaddr_storage* addr, SocketLength addrle
 	return 0;
 }
 
-static bool GetSocketBackend(int guest_fd, NativeSocket* out, SocketSlot* state = nullptr) {
+static bool GetSocketBackend(int guest_fd, NativeSocket* out, SocketSlot* state = nullptr,
+                             std::shared_ptr<SocketTransport>* transport = nullptr) {
 	EXIT_IF(out == nullptr);
 
 	if (guest_fd < 0 || guest_fd >= SOCKET_FD_MAX) {
@@ -1188,6 +1207,9 @@ static bool GetSocketBackend(int guest_fd, NativeSocket* out, SocketSlot* state
 	if (state != nullptr) {
 		*state = slot;
 	}
+	if (transport != nullptr) {
+		*transport = slot.transport;
+	}
 	return true;
 }
 
@@ -1234,6 +1256,63 @@ static int AllocSocketFd(const std::shared_ptr<SocketTransport>& transport, bool
 	return -1;
 }
 
+static int SelectSockets(int nfds, fd_set* readfds, fd_set* writefds, fd_set* exceptfds,
+                         timeval* timeout,
+                         const std::vector<std::shared_ptr<SocketTransport>>& stream_reads) {
+#if defined(_WIN32)
+	if (!stream_reads.empty()) {
+		const auto requested_read   = *readfds;
+		const auto requested_write  = writefds != nullptr ? *writefds : fd_set {};
+		const auto requested_except = exceptfds != nullptr ? *exceptfds : fd_set {};
+		std::optional<std::chrono::steady_clock::time_point> deadline;
+		if (timeout != nullptr) {
+			deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout->tv_sec) +
+			           std::chrono::microseconds(timeout->tv_usec);
+		}
+		for (;;) {
+			*readfds = requested_read;
+			if (writefds != nullptr) {
+				*writefds = requested_write;
+			}
+			if (exceptfds != nullptr) {
+				*exceptfds = requested_except;
+			}
+			const bool ready = std::any_of(stream_reads.begin(), stream_reads.end(),
+			    [](const auto& transport) { return transport->HasPendingReceive(); });
+			// A concurrent peek can move native bytes into the guest buffer after this check.
+			// Bound native waits so both readiness sources are rechecked without holding receive locks.
+			auto wait = std::chrono::microseconds(ready ? 0 : 10'000);
+			if (deadline) {
+				const auto remaining = std::chrono::duration_cast<std::chrono::microseconds>(
+				    *deadline - std::chrono::steady_clock::now());
+				wait = std::min(wait, std::max(remaining, std::chrono::microseconds::zero()));
+			}
+			timeval slice {};
+			slice.tv_usec = static_cast<long>(wait.count());
+			int result = ::select(nfds, readfds, writefds, exceptfds, &slice);
+			if (result < 0) {
+				return result;
+			}
+			for (const auto& transport: stream_reads) {
+				const auto socket = transport->socket.load();
+				if (socket == INVALID_NATIVE_SOCKET) {
+					WSASetLastError(WSAENOTSOCK);
+					return SOCKET_ERROR;
+				}
+				if (transport->HasPendingReceive() && !FD_ISSET(socket, readfds)) {
+					FD_SET(socket, readfds);
+					result++;
+				}
+			}
+			if (result != 0 || (deadline && std::chrono::steady_clock::now() >= *deadline)) {
+				return result;
+			}
+		}
+	}
+#endif
+	return ::select(nfds, readfds, writefds, exceptfds, timeout);
+}
+
 static bool GuestFdIsSet(const void* fds, int fd) {
 	if (fds == nullptr || fd < 0 || fd >= SOCKET_FD_MAX) {
 		return false;
@@ -1621,6 +1700,7 @@ int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int t
 	FD_ZERO(&host_except);
 
 	std::vector<HostRegistration> host_registrations;
+	std::vector<std::shared_ptr<SocketTransport>> stream_reads;
 	host_registrations.reserve(registrations.size());
 	int host_nfds = 0;
 	for (const auto& registration: registrations) {
@@ -1641,6 +1721,11 @@ int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int t
 #endif
 		if ((registration.event.events & EPOLL_IN) != 0) {
 			FD_SET(socket, &host_read);
+#if defined(_WIN32)
+			if (state.transport->type == SOCK_STREAM) {
+				stream_reads.push_back(state.transport);
+			}
+#endif
 		}
 		if ((registration.event.events & EPOLL_OUT) != 0) {
 			FD_SET(socket, &host_write);
@@ -1660,8 +1745,8 @@ int KYTY_SYSV_ABI EpollWait(int eid, NetEpollEvent* events, int maxevents, int t
 		host_timeout.tv_usec = timeout % 1000000;
 		host_timeout_ptr     = &host_timeout;
 	}
-
-	const int result = ::select(host_nfds, &host_read, &host_write, &host_except, host_timeout_ptr);
+	const int result = SelectSockets(host_nfds, &host_read, &host_write, &host_except,
+	                                 host_timeout_ptr, stream_reads);
 	if (result < 0) {
 		return SetHostSocketError();
 	}
@@ -1727,7 +1812,12 @@ int KYTY_SYSV_ABI SocketClose(int s) {
 	}
 	RemoveSocketFromEpolls(s);
 
-	if (slot.transport.use_count() == 1 && slot.transport->Close() != 0) {
+	bool close_transport = slot.transport.use_count() == 1;
+#if defined(_WIN32)
+	// Retained receive/readiness state must not defer native TCP close cancellation.
+	close_transport |= !slot.p2p && slot.transport->type == SOCK_STREAM;
+#endif
+	if (close_transport && slot.transport->Close() != 0) {
 		return NET_ERROR_EBADF;
 	}
 
@@ -1762,7 +1852,7 @@ int KYTY_SYSV_ABI Socket(int family, int type, int protocol) {
 		return SetHostSocketError();
 	}
 
-	auto transport = std::make_shared<SocketTransport>(socket);
+	auto transport = std::make_shared<SocketTransport>(socket, p2p ? SOCK_DGRAM : type);
 	if (p2p) {
 		// Logical sockets keep their own blocking mode; the shared transport never blocks.
 #if defined(_WIN32)
@@ -1945,7 +2035,7 @@ int KYTY_SYSV_ABI Accept(int s, void* addr, uint32_t* addrlen) {
 		return SetHostSocketError();
 	}
 
-	auto      transport = std::make_shared<SocketTransport>(accepted);
+	auto      transport = std::make_shared<SocketTransport>(accepted, SOCK_STREAM);
 	const int fd        = AllocSocketFd(transport);
 	if (fd < 0) {
 		*Posix::GetErrorAddr() = Posix::POSIX_EMFILE;
@@ -1974,8 +2064,9 @@ int KYTY_SYSV_ABI Shutdown(int s, int how) {
 	     "\t how = %d\n",
 	     s, how);
 
-	NativeSocket socket = INVALID_NATIVE_SOCKET;
-	if (!GetSocketBackend(s, &socket)) {
+	NativeSocket                    socket = INVALID_NATIVE_SOCKET;
+	std::shared_ptr<SocketTransport> transport;
+	if (!GetSocketBackend(s, &socket, nullptr, &transport)) {
 		return -1;
 	}
 
@@ -1988,6 +2079,9 @@ int KYTY_SYSV_ABI Shutdown(int s, int how) {
 	if (::shutdown(socket, how) == SOCKET_ERROR) {
 		return SetHostSocketError();
 	}
+	if (how == SD_RECEIVE || how == SD_BOTH) {
+		transport->receive_shutdown.store(true, std::memory_order_relaxed);
+	}
 
 	return 0;
 #else
@@ -2205,6 +2299,77 @@ int64_t KYTY_SYSV_ABI Sendto(int s, const void* buf, uint64_t len, int flags, co
 	return result;
 }
 
+#if defined(_WIN32)
+static int ReceiveStream(SocketTransport& transport, char* output, int length, int flags,
+                         bool dontwait) {
+	std::unique_lock lock(transport.receive_mutex, std::defer_lock);
+	if (dontwait) {
+		if (!lock.try_lock()) {
+			return SetGuestSocketError(Posix::POSIX_EWOULDBLOCK);
+		}
+	} else {
+		lock.lock();
+	}
+	if (transport.receive_shutdown.load(std::memory_order_relaxed)) {
+		return SetGuestSocketError(Posix::POSIX_ESHUTDOWN);
+	}
+	auto&           buffered = transport.receive_buffer;
+	const bool      peek     = (flags & MSG_PEEK) != 0;
+	const bool      waitall  = (flags & MSG_WAITALL) != 0;
+	int             copied   = static_cast<int>(std::min<size_t>(buffered.size(), length));
+	if (copied != 0) {
+		std::memcpy(output, buffered.data(), copied);
+		if (!peek) {
+			buffered.erase(buffered.begin(), buffered.begin() + copied);
+			transport.buffered_bytes.store(buffered.size(), std::memory_order_relaxed);
+		}
+	}
+	if (copied == length || (copied != 0 && !waitall)) {
+		return copied;
+	}
+
+	// Winsock rejects PEEK | WAITALL. Consume into transport-owned storage when peeking,
+	// and let ordinary recv report EOF, timeouts, or nonblocking partial progress.
+	flags &= ~(MSG_PEEK | MSG_WAITALL);
+	do {
+		if (transport.receive_shutdown.load(std::memory_order_relaxed)) {
+			return copied != 0 ? copied : SetGuestSocketError(Posix::POSIX_ESHUTDOWN);
+		}
+		const auto socket = transport.socket.load();
+		if (dontwait) {
+			fd_set readable {};
+			FD_SET(socket, &readable);
+			timeval immediate {};
+			const int ready = ::select(0, &readable, nullptr, nullptr, &immediate);
+			if (ready <= 0) {
+				return copied != 0 ? copied : (ready < 0 ? SetHostSocketError() :
+				    SetGuestSocketError(Posix::POSIX_EWOULDBLOCK));
+			}
+		}
+		const auto previous_size = buffered.size();
+		const int  requested     = peek ? std::min(length - copied, 64 * 1024) : length - copied;
+		char*      destination   = output + copied;
+		if (peek) {
+			buffered.resize(previous_size + requested);
+			destination = buffered.data() + previous_size;
+		}
+		const int received = ::recv(socket, destination, requested, flags);
+		if (peek) {
+			if (received > 0) {
+				std::memcpy(output + copied, destination, received);
+			}
+			buffered.resize(previous_size + std::max(received, 0));
+			transport.buffered_bytes.store(buffered.size(), std::memory_order_relaxed);
+		}
+		if (received <= 0) {
+			return copied != 0 || received == 0 ? copied : SetHostSocketError();
+		}
+		copied += received;
+	} while (waitall && copied < length);
+	return copied;
+}
+#endif
+
 int64_t KYTY_SYSV_ABI Recv(int s, void* buf, uint64_t len, int flags) {
 	return Recvfrom(s, buf, len, flags, nullptr, nullptr);
 }
@@ -2227,8 +2392,9 @@ int64_t KYTY_SYSV_ABI Recvfrom(int s, void* buf, uint64_t len, int flags, void*
 		return -1;
 	}
 
-	NativeSocket socket = INVALID_NATIVE_SOCKET;
-	if (!GetSocketBackend(s, &socket)) {
+	NativeSocket                    socket = INVALID_NATIVE_SOCKET;
+	std::shared_ptr<SocketTransport> transport;
+	if (!GetSocketBackend(s, &socket, nullptr, &transport)) {
 		return -1;
 	}
 
@@ -2242,11 +2408,27 @@ int64_t KYTY_SYSV_ABI Recvfrom(int s, void* buf, uint64_t len, int flags, void*
 	sockaddr_storage host_addr {};
 	SocketLength     host_addrlen = sizeof(host_addr);
 	int64_t          result       = 0;
-	if (addr == nullptr) {
-		result = ::recv(socket, static_cast<char*>(buf), host_len, host_flags);
-	} else {
-		result = ::recvfrom(socket, static_cast<char*>(buf), host_len, host_flags,
-		                    reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
+#if defined(_WIN32)
+	if (transport->type == SOCK_STREAM) {
+		if (addr != nullptr &&
+		    ::getpeername(socket, reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen) != 0) {
+			return SetHostSocketError();
+		}
+		result = ReceiveStream(*transport, static_cast<char*>(buf), host_len, host_flags,
+		                       (flags & 0x80) != 0);
+		if (result < 0) {
+			return result;
+		}
+	} else
+#endif
+	{
+		transport.reset();
+		if (addr == nullptr) {
+			result = ::recv(socket, static_cast<char*>(buf), host_len, host_flags);
+		} else {
+			result = ::recvfrom(socket, static_cast<char*>(buf), host_len, host_flags,
+			                    reinterpret_cast<sockaddr*>(&host_addr), &host_addrlen);
+		}
 	}
 #if defined(_WIN32)
 	if (result < 0 && WSAGetLastError() == WSAEMSGSIZE) {
@@ -2303,6 +2485,7 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 	FD_ZERO(&host_except);
 
 	std::vector<std::pair<int, NativeSocket>> descriptors;
+	std::vector<std::shared_ptr<SocketTransport>> stream_reads;
 	int host_nfds = 0;
 	for (int fd = 0; fd < nfds; fd++) {
 		const bool read = GuestFdIsSet(readfds, fd);
@@ -2311,13 +2494,14 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 		if (!read && !write && !except) {
 			continue;
 		}
-		NativeSocket socket = INVALID_NATIVE_SOCKET;
+		NativeSocket                    socket = INVALID_NATIVE_SOCKET;
+		std::shared_ptr<SocketTransport> transport;
 #if !defined(_WIN32)
 		if (fd < 3) {
 			socket = fd;
 		} else
 #endif
-		if (!GetSocketBackend(fd, &socket)) {
+		if (!GetSocketBackend(fd, &socket, nullptr, &transport)) {
 			return -1;
 		}
 #if defined(_WIN32)
@@ -2335,6 +2519,11 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 		descriptors.emplace_back(fd, socket);
 		if (read) {
 			FD_SET(socket, &host_read);
+#if defined(_WIN32)
+			if (transport->type == SOCK_STREAM) {
+				stream_reads.push_back(std::move(transport));
+			}
+#endif
 		}
 		if (write) {
 			FD_SET(socket, &host_write);
@@ -2365,9 +2554,10 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 	} else
 #endif
 	{
-		result = ::select(host_nfds, readfds != nullptr ? &host_read : nullptr,
-		                  writefds != nullptr ? &host_write : nullptr,
-		                  exceptfds != nullptr ? &host_except : nullptr, host_timeout_ptr);
+		result = SelectSockets(host_nfds, readfds != nullptr ? &host_read : nullptr,
+		                       writefds != nullptr ? &host_write : nullptr,
+		                       exceptfds != nullptr ? &host_except : nullptr, host_timeout_ptr,
+		                       stream_reads);
 		if (result < 0) {
 			return SetHostSocketError();
 		}
@@ -2376,15 +2566,19 @@ int KYTY_SYSV_ABI Select(int nfds, void* readfds, void* writefds, void* exceptfd
 	GuestFdZero(readfds, nfds);
 	GuestFdZero(writefds, nfds);
 	GuestFdZero(exceptfds, nfds);
+	result = 0;
 	for (const auto& [fd, socket]: descriptors) {
 		if (FD_ISSET(socket, &host_read)) {
 			GuestFdSet(readfds, fd);
+			result++;
 		}
 		if (FD_ISSET(socket, &host_write)) {
 			GuestFdSet(writefds, fd);
+			result++;
 		}
 		if (FD_ISSET(socket, &host_except)) {
 			GuestFdSet(exceptfds, fd);
+			result++;
 		}
 	}
 	return result;

src/loader/guestInstructionPatcher.cpp

@@ -132,7 +132,14 @@ struct DecodedFunction {
 	bool                                        requires_conservative_red_zone_tracking {};
 };
 
-enum class InstructionReplacement { None, ReciprocalSquareRoot, ExtractQ };
+enum class InstructionReplacement {
+	None,
+	ReciprocalSquareRoot,
+	ExtractQ,
+	InsertQ,
+	ReadProcessorId,
+	CacheLineWriteBack
+};
 
 struct InstructionRewrite {
 	bool                   protect_red_zone {};
@@ -143,8 +150,19 @@ struct InstructionRewrite {
 InstructionPatchCounts& ReplacementCounts(GuestInstructionPatchResult& result,
                                           InstructionReplacement       replacement) {
 	ASSERT(replacement != InstructionReplacement::None);
-	return replacement == InstructionReplacement::ReciprocalSquareRoot ? result.reciprocal_sqrt
-	                                                                   : result.extrq;
+	switch (replacement) {
+		case InstructionReplacement::ReciprocalSquareRoot: return result.reciprocal_sqrt;
+		case InstructionReplacement::ExtractQ: return result.extrq;
+		case InstructionReplacement::InsertQ: return result.insertq;
+		case InstructionReplacement::ReadProcessorId: return result.rdpid;
+		default: return result.clwb;
+	}
+}
+
+bool UsesInstructionTrap(InstructionReplacement replacement, bool trap_replacements) {
+	return replacement != InstructionReplacement::None &&
+	       (trap_replacements || replacement == InstructionReplacement::InsertQ ||
+	        replacement == InstructionReplacement::ReadProcessorId);
 }
 
 bool IsStackPointerRegister(ZydisRegister reg) {
@@ -216,10 +234,11 @@ DecodedCodeInstruction DecodeCodeInstruction(uintptr_t address, uintptr_t end) {
 
 	if (decoded.changes_stack_pointer) {
 		const auto& operands = decoded.operands;
-		if (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV &&
-		    operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER &&
+		if (operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER &&
 		    IsStackPointerRegister(operands[0].reg.value) &&
-		    operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY) {
+		    (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_RDPID ||
+		     (decoded.instruction.mnemonic == ZYDIS_MNEMONIC_MOV &&
+		      operands[1].type == ZYDIS_OPERAND_TYPE_MEMORY))) {
 			decoded.replaces_stack_pointer = true;
 		} else if ((decoded.instruction.mnemonic == ZYDIS_MNEMONIC_ADD ||
 		            decoded.instruction.mnemonic == ZYDIS_MNEMONIC_SUB) &&
@@ -653,13 +672,17 @@ void AnalyzeRedZoneLiveness(DecodedFunction& function) {
 }
 
 bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded,
-                                Xbyak::CodeGenerator&         generator) {
+                                Xbyak::CodeGenerator& generator, s64 stack_adjustment = 0,
+                                ZydisMnemonic replacement = ZYDIS_MNEMONIC_INVALID) {
 	ZydisEncoderRequest request;
 	if (!ZYAN_SUCCESS(ZydisEncoderDecodedInstructionToEncoderRequest(
 	        &decoded.instruction, decoded.operands.data(),
 	        decoded.instruction.operand_count_visible, &request))) {
 		return false;
 	}
+	if (replacement != ZYDIS_MNEMONIC_INVALID) {
+		request.mnemonic = replacement;
+	}
 
 	for (u8 index = 0; index < decoded.instruction.operand_count_visible; ++index) {
 		const auto& operand = decoded.operands[index];
@@ -679,6 +702,9 @@ bool EncodeRelocatedInstruction(const DecodedCodeInstruction& decoded,
 				return false;
 			}
 			request.operands[index].mem.displacement = static_cast<ZyanI64>(absolute_address);
+		} else if (operand.type == ZYDIS_OPERAND_TYPE_MEMORY &&
+		           IsStackPointerRegister(operand.mem.base)) {
+			request.operands[index].mem.displacement += stack_adjustment;
 		}
 	}
 
@@ -823,8 +849,9 @@ void GenerateReciprocalSquareRoot(const DecodedCodeInstruction& decoded,
 	generator.L(done);
 }
 
-bool IsSupportedExtractQ(const DecodedCodeInstruction& decoded) {
-	if (decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) {
+bool IsSupportedBitFieldInstruction(const DecodedCodeInstruction& decoded) {
+	const bool insert = decoded.instruction.mnemonic == ZYDIS_MNEMONIC_INSERTQ;
+	if (!insert && decoded.instruction.mnemonic != ZYDIS_MNEMONIC_EXTRQ) {
 		return false;
 	}
 	const auto is_xmm = [](const ZydisDecodedOperand& operand) {
@@ -832,18 +859,20 @@ bool IsSupportedExtractQ(const DecodedCodeInstruction& decoded) {
 		       operand.reg.value >= ZYDIS_REGISTER_XMM0 &&
 		       operand.reg.value <= ZYDIS_REGISTER_XMM15;
 	};
-	const bool immediate = decoded.instruction.operand_count_visible == 3 &&
-	                       decoded.operands[1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE &&
-	                       decoded.operands[2].type == ZYDIS_OPERAND_TYPE_IMMEDIATE;
-	if (!is_xmm(decoded.operands[0]) ||
+	const size_t immediate_index = insert ? 2 : 1;
+	const bool   immediate =
+	    decoded.instruction.operand_count_visible == immediate_index + 2 &&
+	    decoded.operands[immediate_index].type == ZYDIS_OPERAND_TYPE_IMMEDIATE &&
+	    decoded.operands[immediate_index + 1].type == ZYDIS_OPERAND_TYPE_IMMEDIATE;
+	if (!is_xmm(decoded.operands[0]) || (insert && !is_xmm(decoded.operands[1])) ||
 	    (!immediate &&
 	     !(decoded.instruction.operand_count_visible == 2 && is_xmm(decoded.operands[1])))) {
 		return false;
 	}
-	// Match the trap handler's encoding support: 66 [REX] 0F 78/79 ModRM [imm8 imm8].
+	// Match the trap handler's encoding support: 66/F2 [REX] 0F 78/79 ModRM [imm8 imm8].
 	const auto*  code   = reinterpret_cast<const u8*>(decoded.address);
 	const size_t opcode = (code[1] & 0xf0u) == 0x40u ? 2 : 1;
-	return code[0] == 0x66 && code[opcode] == 0x0f &&
+	return code[0] == (insert ? 0xf2 : 0x66) && code[opcode] == 0x0f &&
 	       code[opcode + 1] == (immediate ? 0x78 : 0x79) &&
 	       decoded.instruction.length == opcode + (immediate ? 5 : 3);
 }
@@ -890,8 +919,8 @@ void GenerateExtractQ(const DecodedCodeInstruction& decoded, Xbyak::CodeGenerato
 
 void CollectAmdInstructions(const DecodedFunction&                   function,
                             std::map<uintptr_t, InstructionRewrite>& rewrite_sites,
-                            GuestInstructionPatchResult&             result) {
-	static const bool host_lacks_sse4a = !Xbyak::util::Cpu().has(Xbyak::util::Cpu::tSSE4a);
+                            GuestInstructionPatchResult&             result,
+                            GuestInstructionHostFeatures             host_features) {
 	for (const auto& [address, decoded]: function.instructions) {
 		const auto&            instruction = decoded.instruction;
 		InstructionReplacement replacement {};
@@ -900,8 +929,14 @@ void CollectAmdInstructions(const DecodedFunction&                   function,
 		    instruction.raw.vex.offset == 0 && decoded.operands[0].size == 128 &&
 		    decoded.operands[1].type == ZYDIS_OPERAND_TYPE_REGISTER) {
 			replacement = InstructionReplacement::ReciprocalSquareRoot;
-		} else if (host_lacks_sse4a && IsSupportedExtractQ(decoded)) {
-			replacement = InstructionReplacement::ExtractQ;
+		} else if (!host_features.sse4a && IsSupportedBitFieldInstruction(decoded)) {
+			replacement = instruction.mnemonic == ZYDIS_MNEMONIC_EXTRQ
+			                  ? InstructionReplacement::ExtractQ
+			                  : InstructionReplacement::InsertQ;
+		} else if (!host_features.rdpid && instruction.mnemonic == ZYDIS_MNEMONIC_RDPID) {
+			replacement = InstructionReplacement::ReadProcessorId;
+		} else if (!host_features.clwb && instruction.mnemonic == ZYDIS_MNEMONIC_CLWB) {
+			replacement = InstructionReplacement::CacheLineWriteBack;
 		} else {
 			continue;
 		}
@@ -916,16 +951,21 @@ void MarkInstructionTrap(u8* code, const ZydisDecodedInstruction& instruction,
 		// Clear a reserved VEX.vvvv bit, retaining both register operands for the handler.
 		code[instruction.raw.vex.size - 1] &= ~0x08u;
 	}
-	// EXTRQ already raises #UD on hosts without SSE4a; keep its original encoding.
+	// Other replacements already raise #UD on hosts without their CPU feature.
 }
 
-void GenerateInstructionTrap(const DecodedCodeInstruction& decoded,
-                             InstructionReplacement replacement, Xbyak::CodeGenerator& generator) {
+bool GenerateInstructionTrap(const DecodedCodeInstruction& decoded,
+                             InstructionReplacement replacement, Xbyak::CodeGenerator& generator,
+                             s64 stack_adjustment) {
+	if (replacement == InstructionReplacement::CacheLineWriteBack) {
+		return EncodeRelocatedInstruction(decoded, generator, stack_adjustment);
+	}
 	std::array<u8, ZYDIS_MAX_INSTRUCTION_LENGTH> code {};
 	std::memcpy(code.data(), reinterpret_cast<const void*>(decoded.address),
 	            decoded.instruction.length);
 	MarkInstructionTrap(code.data(), decoded.instruction, replacement);
 	generator.db(code.data(), decoded.instruction.length);
+	return true;
 }
 
 bool NeedsTrapRedZoneProtection(const DecodedCodeInstruction& decoded) {
@@ -946,11 +986,11 @@ void TrapUnrelocatedInstructions(const PatchModule& module, const DecodedFunctio
 		}
 		const auto& decoded = function.instructions.at(address);
 		if (NeedsTrapRedZoneProtection(decoded)) {
-			if (rewrite.replacement == InstructionReplacement::ExtractQ) {
-				// Unlike VRSQRTPS, leaving EXTRQ unchanged would still trap and corrupt the red
-				// zone.
-				EXIT("AMD CPU compatibility: cannot safely trap EXTRQ at %p (guest red zone is "
+			if (rewrite.replacement != InstructionReplacement::ReciprocalSquareRoot) {
+				// Leaving an unsupported instruction unchanged still traps with a live red zone.
+				EXIT("AMD CPU compatibility: cannot safely trap %s at %p (guest red zone is "
 				     "live)\n",
+				     ZydisMnemonicGetString(decoded.instruction.mnemonic),
 				     reinterpret_cast<void*>(address));
 			}
 			continue;
@@ -990,23 +1030,31 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct
 				const auto replacement = rewrite != rewrite_sites.end()
 				                             ? rewrite->second.replacement
 				                             : InstructionReplacement::None;
+				const bool uses_trap   = UsesInstructionTrap(replacement, trap_replacements);
 				const bool protected_indirect_call =
 				    rewrite != rewrite_sites.end() && rewrite->second.protected_indirect_call;
 				const bool protect_red_zone =
 				    (rewrite != rewrite_sites.end() && rewrite->second.protect_red_zone &&
 				     !protected_indirect_call) ||
-				    (replacement != InstructionReplacement::None && trap_replacements &&
+				    ((uses_trap || replacement == InstructionReplacement::CacheLineWriteBack) &&
 				     NeedsTrapRedZoneProtection(*decoded));
 				if (protect_red_zone) {
 					generator.lea(rsp, ptr[rsp - GuestRedZoneSize]);
 				}
 				if (replacement != InstructionReplacement::None) {
-					if (trap_replacements) {
-						GenerateInstructionTrap(*decoded, replacement, generator);
+					if (uses_trap) {
+						encoded = GenerateInstructionTrap(*decoded, replacement, generator,
+						                                  protect_red_zone ? GuestRedZoneSize : 0);
 					} else if (replacement == InstructionReplacement::ReciprocalSquareRoot) {
 						GenerateReciprocalSquareRoot(*decoded, generator);
-					} else {
+					} else if (replacement == InstructionReplacement::ExtractQ) {
 						GenerateExtractQ(*decoded, generator);
+					} else {
+						// CLFLUSH writes back dirty data too; invalidation is a permitted stronger
+						// action.
+						encoded = EncodeRelocatedInstruction(
+						    *decoded, generator, protect_red_zone ? GuestRedZoneSize : 0,
+						    ZYDIS_MNEMONIC_CLFLUSH);
 					}
 				} else if (protected_indirect_call) {
 					encoded = GenerateProtectedIndirectCall(*decoded, generator);
@@ -1053,7 +1101,7 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct
 			}
 			if (rewrite->second.replacement != InstructionReplacement::None) {
 				auto& counts = ReplacementCounts(result, rewrite->second.replacement);
-				if (span.trap_replacements) {
+				if (UsesInstructionTrap(rewrite->second.replacement, span.trap_replacements)) {
 					++counts.trapped;
 				} else {
 					++counts.native;
@@ -1425,9 +1473,26 @@ void RelocateGuestInstructions(PatchModule* module, const DecodedFunction& funct
 }
 } // namespace
 
+GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
+	static const auto features = [] {
+		const Xbyak::util::Cpu cpu;
+		uint32_t               leaf[4] {};
+		Xbyak::util::Cpu::getCpuid(0, leaf);
+		bool rdpid = false;
+		if (leaf[0] >= 7) {
+			Xbyak::util::Cpu::getCpuidEx(7, 0, leaf);
+			rdpid = (leaf[2] & (1u << 22)) != 0;
+		}
+		return GuestInstructionHostFeatures {cpu.has(Xbyak::util::Cpu::tSSE4a), rdpid,
+		                                     cpu.has(Xbyak::util::Cpu::tCLWB)};
+	}();
+	return features;
+}
+
 GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment_size,
                                                    std::span<const uintptr_t> function_starts,
-                                                   bool protect_memory, bool emulate_amd) {
+                                                   bool protect_memory, bool emulate_amd,
+                                                   GuestInstructionHostFeatures host_features) {
 	GuestInstructionPatchResult result {};
 	if (!protect_memory && !emulate_amd) {
 		return result;
@@ -1480,7 +1545,7 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment
 			CollectRedZoneMemoryInstructions(function, rewrite_sites, result);
 		}
 		if (emulate_amd) {
-			CollectAmdInstructions(function, rewrite_sites, result);
+			CollectAmdInstructions(function, rewrite_sites, result, host_features);
 		}
 		if (!rewrite_sites.empty()) {
 			RelocateGuestInstructions(module, function, rewrite_sites, result);
@@ -1499,8 +1564,12 @@ GuestInstructionPatchResult PatchGuestInstructions(u64 segment_addr, u64 segment
 
 #else
 
-GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool,
-                                                   bool) {
+GuestInstructionHostFeatures GetGuestInstructionHostFeatures() {
+	return {};
+}
+
+GuestInstructionPatchResult PatchGuestInstructions(u64, u64, std::span<const uintptr_t>, bool, bool,
+                                                   GuestInstructionHostFeatures) {
 	return {};
 }
 

src/loader/guestInstructionPatcher.h

@@ -11,6 +11,14 @@
 
 namespace Loader {
 
+struct GuestInstructionHostFeatures {
+	bool sse4a = false;
+	bool rdpid = false;
+	bool clwb  = false;
+};
+
+GuestInstructionHostFeatures GetGuestInstructionHostFeatures();
+
 struct InstructionPatchCounts {
 	uint64_t found   = 0;
 	uint64_t native  = 0;
@@ -39,6 +47,9 @@ struct GuestInstructionPatchResult {
 	uint64_t               indirect_red_zone_function_count         = 0;
 	InstructionPatchCounts reciprocal_sqrt;
 	InstructionPatchCounts extrq;
+	InstructionPatchCounts insertq;
+	InstructionPatchCounts rdpid;
+	InstructionPatchCounts clwb;
 };
 
 void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
@@ -46,9 +57,10 @@ void RegisterGuestInstructionPatchModule(void* module_ptr, uint64_t module_size,
 void UnregisterGuestInstructionPatchModule(void* module_ptr);
 
 // Apply enabled instruction fixes using native trampolines or safe trap fallbacks.
-GuestInstructionPatchResult PatchGuestInstructions(uint64_t segment_addr, uint64_t segment_size,
-                                                   std::span<const uintptr_t> function_starts,
-                                                   bool protect_memory, bool emulate_amd);
+GuestInstructionPatchResult PatchGuestInstructions(
+    uint64_t segment_addr, uint64_t segment_size, std::span<const uintptr_t> function_starts,
+    bool protect_memory, bool emulate_amd,
+    GuestInstructionHostFeatures host_features = GetGuestInstructionHostFeatures());
 
 bool DecodeEhFrameFunctionStarts(uint64_t eh_frame_header_addr, uint64_t eh_frame_header_size,
                                  std::vector<uintptr_t>* function_starts);

src/loader/runtimeLinker.cpp

@@ -1949,6 +1949,9 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 			                                           protect_memory_faults, emulate_amd);
 			totals.reciprocal_sqrt += result.reciprocal_sqrt;
 			totals.extrq += result.extrq;
+			totals.insertq += result.insertq;
+			totals.rdpid += result.rdpid;
+			totals.clwb += result.clwb;
 			if (protect_memory_faults) {
 				LOGF("Windows guest red-zone patching: %s, functions=%" PRIu64 ", red_zone=%" PRIu64
 				     ", memory=%" PRIu64 ", patched=%" PRIu64 ", short=%" PRIu64 ", stack=%" PRIu64
@@ -1963,18 +1966,26 @@ void RuntimeLinker::LoadProgramToMemory(Program* program) {
 			}
 		}
 		if (emulate_amd && have_function_starts) {
-			const auto  found   = totals.reciprocal_sqrt.found + totals.extrq.found;
-			const auto  skipped = totals.reciprocal_sqrt.Skipped() + totals.extrq.Skipped();
+			InstructionPatchCounts combined {};
+			std::string            details;
+			for (const auto& [name, counts]: {std::pair {"VRSQRTPS", totals.reciprocal_sqrt},
+			                                  {"EXTRQ", totals.extrq},
+			                                  {"INSERTQ", totals.insertq},
+			                                  {"RDPID", totals.rdpid},
+			                                  {"CLWB", totals.clwb}}) {
+				combined += counts;
+				if (!details.empty()) details += "; ";
+				details += fmt::format("{}: native={}, trapped={}, skipped={}", name, counts.native,
+				                       counts.trapped, counts.Skipped());
+			}
+			const auto  found   = combined.found;
+			const auto  skipped = combined.Skipped();
 			const char* status  = found == 0         ? "no matching instructions"
 			                      : skipped == found ? "not patched"
 			                      : skipped != 0     ? "partially patched"
 			                                         : "patched";
-			Log::WriteToConsoleAndLog(fmt::format(
-			    "AMD CPU compatibility: {} {} (VRSQRTPS: native={}, trapped={}, skipped={}; "
-			    "EXTRQ: native={}, trapped={}, skipped={})\n",
-			    module_name, status, totals.reciprocal_sqrt.native, totals.reciprocal_sqrt.trapped,
-			    totals.reciprocal_sqrt.Skipped(), totals.extrq.native, totals.extrq.trapped,
-			    totals.extrq.Skipped()));
+			Log::WriteToConsoleAndLog(
+			    fmt::format("AMD CPU compatibility: {} {} ({})\n", module_name, status, details));
 		}
 	}
 

src/loader/x64InstructionEmulator.cpp

@@ -2,12 +2,12 @@
 
 #include "common/common.h"
 
+#include <Zydis/Zydis.h>
 #include <bit>
 #include <cstring>
-#if !defined(__APPLE__)
 #include <emmintrin.h>
 #include <xmmintrin.h>
-#endif
+#include <cpuid.h>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
@@ -406,12 +406,12 @@ struct Context {
 	void                   Advance(size_t length) { native->Rip += length; }
 	[[nodiscard]] void*    Xmm(uint8_t index) const { return &native->Xmm0 + index; }
 
-	void LoadGprs(uint64_t (&gpr)[16]) const {
-		const uint64_t registers[] = {native->Rax, native->Rcx, native->Rdx, native->Rbx,
-		                              native->Rsp, native->Rbp, native->Rsi, native->Rdi,
-		                              native->R8,  native->R9,  native->R10, native->R11,
-		                              native->R12, native->R13, native->R14, native->R15};
-		std::memcpy(gpr, registers, sizeof(gpr));
+	[[nodiscard]] auto& Gpr(uint8_t index) const {
+		DWORD64* registers[] = {&native->Rax, &native->Rcx, &native->Rdx, &native->Rbx,
+		                        &native->Rsp, &native->Rbp, &native->Rsi, &native->Rdi,
+		                        &native->R8,  &native->R9,  &native->R10, &native->R11,
+		                        &native->R12, &native->R13, &native->R14, &native->R15};
+		return *registers[index];
 	}
 
 	void ClearUpperYmm(uint8_t index) const {
@@ -437,6 +437,14 @@ struct Context {
 	void Advance(size_t length) {
 		native->uc_mcontext->__ss.__rip += static_cast<uint64_t>(length);
 	}
+	[[nodiscard]] auto& Gpr(uint8_t index) const {
+		auto*     state       = &native->uc_mcontext->__ss;
+		uint64_t* registers[] = {&state->__rax, &state->__rcx, &state->__rdx, &state->__rbx,
+		                         &state->__rsp, &state->__rbp, &state->__rsi, &state->__rdi,
+		                         &state->__r8,  &state->__r9,  &state->__r10, &state->__r11,
+		                         &state->__r12, &state->__r13, &state->__r14, &state->__r15};
+		return *registers[index];
+	}
 	// Darwin names the XMM file __fpu_xmm0..__fpu_xmm15 instead of exposing an array.
 	[[nodiscard]] void* Xmm(uint8_t index) const {
 		auto* fs = &native->uc_mcontext->__fs;
@@ -476,13 +484,11 @@ struct Context {
 		return native->uc_mcontext.fpregs->_xmm[index].element;
 	}
 
-	void LoadGprs(uint64_t (&gpr)[16]) const {
+	[[nodiscard]] auto& Gpr(uint8_t index) const {
 		constexpr int registers[] = {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP,
 		                             REG_RSI, REG_RDI, REG_R8,  REG_R9,  REG_R10, REG_R11,
 		                             REG_R12, REG_R13, REG_R14, REG_R15};
-		for (size_t i = 0; i < 16; ++i) {
-			gpr[i] = static_cast<uint64_t>(native->uc_mcontext.gregs[registers[i]]);
-		}
+		return native->uc_mcontext.gregs[registers[index]];
 	}
 
 	void ClearUpperYmm(uint8_t index) const {
@@ -506,6 +512,12 @@ struct Context {
 		}
 	}
 #endif
+
+	void LoadGprs(uint64_t (&gpr)[16]) const {
+		for (uint8_t i = 0; i < 16; ++i) {
+			gpr[i] = static_cast<uint64_t>(Gpr(i));
+		}
+	}
 };
 
 #if !defined(__APPLE__)
@@ -574,8 +586,8 @@ static bool TryEmulateSse4a(Context& context) {
 	if (rip[offset] != 0x0f) {
 		return false;
 	}
-	const bool register_extract = prefix == 0x66 && rip[offset + 1] == 0x79;
-	if (rip[offset + 1] != 0x78 && !register_extract) {
+	const bool register_form = rip[offset + 1] == 0x79;
+	if (rip[offset + 1] != 0x78 && !register_form) {
 		return false;
 	}
 
@@ -589,7 +601,7 @@ static bool TryEmulateSse4a(Context& context) {
 
 	// Immediate EXTRQ encodes its destination in r/m; the two-register form uses reg.
 	uint8_t dest_index = reg;
-	if (prefix == 0x66 && !register_extract) {
+	if (prefix == 0x66 && !register_form) {
 		dest_index = rm;
 	}
 	auto* dest_xmm = context.Xmm(dest_index);
@@ -598,15 +610,17 @@ static bool TryEmulateSse4a(Context& context) {
 		return false;
 	}
 	uint64_t dest[2] {};
-	uint64_t source = 0;
+	uint64_t source[2] {};
 	std::memcpy(dest, dest_xmm, sizeof(dest));
-	std::memcpy(&source, src_xmm, sizeof(source));
+	std::memcpy(source, src_xmm, sizeof(source));
 	uint8_t length             = 0;
 	uint8_t index              = 0;
 	size_t  instruction_length = offset + 3;
-	if (register_extract) {
-		length = static_cast<uint8_t>(source);
-		index  = static_cast<uint8_t>(source >> 8u);
+	if (register_form) {
+		// EXTRQ controls occupy the low quadword; INSERTQ uses the upper quadword.
+		const uint64_t controls = source[prefix == 0x66 ? 0 : 1];
+		length                  = static_cast<uint8_t>(controls);
+		index                   = static_cast<uint8_t>(controls >> 8u);
 	} else {
 		length = rip[offset + 3];
 		index  = rip[offset + 4];
@@ -616,13 +630,79 @@ static bool TryEmulateSse4a(Context& context) {
 		dest[0] = ExtractBitField(dest[0], length, index);
 		dest[1] = 0;
 	} else {
-		dest[0] = InsertBitField(dest[0], source, length, index);
+		dest[0] = InsertBitField(dest[0], source[0], length, index);
 	}
 	std::memcpy(dest_xmm, dest, sizeof(dest));
 	context.Advance(instruction_length);
 	return true;
 }
 
+static bool ReadTscAux(uint32_t& auxiliary) {
+	uint32_t eax = 0;
+	uint32_t ebx = 0;
+	uint32_t ecx = 0;
+	uint32_t edx = 0;
+	if (!__get_cpuid(0x80000001u, &eax, &ebx, &ecx, &edx) || (edx & (1u << 27u)) == 0) {
+		return false;
+	}
+	asm volatile("rdtscp" : "=a"(eax), "=d"(edx), "=c"(auxiliary));
+	return true;
+}
+
+static void FlushCacheLine(uint64_t address, ZydisRegister segment) {
+	// Preserve FS/GS addressing rather than treating a segment-relative offset as a pointer.
+	if (segment == ZYDIS_REGISTER_FS) {
+		asm volatile("clflush %%fs:(%0)" : : "r"(address) : "memory");
+	} else if (segment == ZYDIS_REGISTER_GS) {
+		asm volatile("clflush %%gs:(%0)" : : "r"(address) : "memory");
+	} else {
+		_mm_clflush(reinterpret_cast<const void*>(address));
+	}
+}
+
+static bool TryEmulateCpuExtensions(Context& context) {
+	ZydisDecoder            decoder {};
+	ZydisDecodedInstruction instruction {};
+	ZydisDecodedOperand     operands[ZYDIS_MAX_OPERAND_COUNT] {};
+	if (!ZYAN_SUCCESS(
+	        ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)) ||
+	    !ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, reinterpret_cast<const void*>(context.Rip()),
+	                                         ZYDIS_MAX_INSTRUCTION_LENGTH, &instruction,
+	                                         operands))) {
+		return false;
+	}
+	if (instruction.mnemonic == ZYDIS_MNEMONIC_RDPID &&
+	    operands[0].type == ZYDIS_OPERAND_TYPE_REGISTER) {
+		const auto reg_class = ZydisRegisterGetClass(operands[0].reg.value);
+		const auto index     = ZydisRegisterGetId(operands[0].reg.value);
+		uint32_t   auxiliary = 0;
+		if ((reg_class != ZYDIS_REGCLASS_GPR32 && reg_class != ZYDIS_REGCLASS_GPR64) || index < 0 ||
+		    index >= 16 || !ReadTscAux(auxiliary)) {
+			return false;
+		}
+		// RDPID and guest RDTSCP observe the same host TSC_AUX; only RDPID's destination changes.
+		context.Gpr(static_cast<uint8_t>(index)) = auxiliary;
+	} else if (instruction.mnemonic == ZYDIS_MNEMONIC_CLWB &&
+	           operands[0].type == ZYDIS_OPERAND_TYPE_MEMORY) {
+		ZydisRegisterContext registers {};
+		for (uint8_t i = 0; i < 16; ++i) {
+			const auto value                         = static_cast<uint64_t>(context.Gpr(i));
+			registers.values[ZYDIS_REGISTER_RAX + i] = value;
+			registers.values[ZYDIS_REGISTER_EAX + i] = static_cast<uint32_t>(value);
+		}
+		uint64_t address = 0;
+		if (!ZYAN_SUCCESS(ZydisCalcAbsoluteAddressEx(&instruction, &operands[0], context.Rip(),
+		                                             &registers, &address))) {
+			return false;
+		}
+		FlushCacheLine(address, operands[0].mem.segment);
+	} else {
+		return false;
+	}
+	context.Advance(instruction.length);
+	return true;
+}
+
 #if !defined(__APPLE__)
 
 static bool TryEmulateMonitorxMwaitx(Context& context) {
@@ -729,9 +809,9 @@ bool TryEmulate(void* native_context) {
 #endif
 #if !defined(__APPLE__)
 	return TryEmulateReciprocalSquareRoot(context) || TryEmulateMonitorxMwaitx(context) ||
-	       TryEmulateSse4a(context) || TryEmulateShaNi(context);
+	       TryEmulateSse4a(context) || TryEmulateShaNi(context) || TryEmulateCpuExtensions(context);
 #else
-	return TryEmulateSse4a(context);
+	return TryEmulateSse4a(context) || TryEmulateCpuExtensions(context);
 #endif
 }
 

tests/KernelFileSystemTests.cpp

@@ -25,10 +25,12 @@
 #include <cstdlib>
 #include <cstring>
 #include <filesystem>
+#include <future>
 #include <limits>
 #include <memory>
 #include <string>
 #include <string_view>
+#include <thread>
 #include <vector>
 
 namespace Libs::LibKernelApr {
@@ -811,6 +813,210 @@ void CheckAprPaths(const std::filesystem::path &root) {
   FileSystem::Umount("/app0");
 }
 
+std::array<int, 2> CreateTcpPair() {
+  namespace Net = Libs::Network::Net;
+  // Guest sockaddr_in: length, family, network-order port/address, padding.
+  std::array<uint8_t, 16> address {16, 2, 0, 0, 127, 0, 0, 1};
+  const int listener = Net::Socket(2, 1, 0);
+  Check(listener >= 0, "create loopback listener");
+  Check(Net::Bind(listener, address.data(), address.size()) == 0, "bind loopback");
+  Check(Net::Listen(listener, 1) == 0, "listen on loopback");
+  uint32_t address_size = address.size();
+  Check(Net::Getsockname(listener, address.data(), &address_size) == 0,
+        "get assigned loopback port");
+  const int writer = Net::Socket(2, 1, 0);
+  Check(writer >= 0 && Net::Connect(writer, address.data(), address_size) == 0,
+        "connect wake socket");
+  const int reader = Net::Accept(listener, nullptr, nullptr);
+  Check(reader >= 0, "accept wake socket");
+  Check(Net::SocketClose(listener) == 0, "close listener");
+  return {reader, writer};
+}
+
+#if defined(_WIN32)
+void CheckSocketReceiveConcurrency() {
+  namespace Net = Libs::Network::Net;
+  const auto [reader, writer] = CreateTcpPair();
+  const int timeout_ms = 3000;
+  Check(Net::Setsockopt(reader, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0,
+        "bound the blocked receive concurrency fixture");
+  std::promise<void> entered;
+  std::promise<int64_t> completed;
+  auto entered_future = entered.get_future();
+  auto completed_future = completed.get_future();
+  std::jthread receiver([&] {
+    char byte = 0;
+    entered.set_value();
+    completed.set_value(Net::Recv(reader, &byte, 1, 0x42));
+  });
+  Check(entered_future.wait_for(std::chrono::seconds(2)) == std::future_status::ready,
+        "start blocked reader");
+  std::this_thread::sleep_for(std::chrono::milliseconds(50));
+  char byte = 0;
+  const auto before_nonblocking = std::chrono::steady_clock::now();
+  Check(Net::Recv(reader, &byte, 1, 0x80) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK &&
+            std::chrono::steady_clock::now() - before_nonblocking < std::chrono::milliseconds(500) &&
+            completed_future.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout,
+        "DONTWAIT does not wait behind another blocked reader");
+  const auto before_close = std::chrono::steady_clock::now();
+  Check(Net::SocketClose(reader) == 0, "close succeeds while another thread receives");
+  Check(std::chrono::steady_clock::now() - before_close < std::chrono::seconds(1),
+        "close does not wait for the blocked receive timeout");
+  Check(completed_future.wait_for(std::chrono::seconds(1)) == std::future_status::ready,
+        "close wakes the blocked receive within one second");
+  Check(completed_future.get() <= 0, "closed receive reports EOF or error");
+  Check(std::chrono::steady_clock::now() - before_close < std::chrono::seconds(1),
+        "closing a socket wakes its blocked receive before the receive timeout");
+  receiver.join();
+  Check(Net::SocketClose(writer) == 0, "close blocked receive writer");
+
+  const auto [shutdown_reader, shutdown_writer] = CreateTcpPair();
+  Check(Net::Setsockopt(shutdown_reader, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0 &&
+            Net::Send(shutdown_writer, "x", 1, 0) == 1 &&
+            Net::Recv(shutdown_reader, &byte, 1, 0x42) == 1 && byte == 'x' &&
+            Net::Shutdown(shutdown_reader, 0) == 0 &&
+            Net::Recv(shutdown_reader, &byte, 1, 0) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_ESHUTDOWN,
+        "receive shutdown rejects bytes retained by an earlier peek");
+  Check(Net::SocketClose(shutdown_reader) == 0 && Net::SocketClose(shutdown_writer) == 0,
+        "close receive shutdown fixture");
+}
+
+void CheckSocketReceiveBuffer(int reader, int writer) {
+  namespace Net = Libs::Network::Net;
+  constexpr int PeekWaitAll = 0x42;
+  constexpr char Payload[] = "abcdef";
+  const int timeout_ms = 2000;
+  Check(Net::Setsockopt(reader, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0,
+        "bound blocking receive regressions with SO_RCVTIMEO");
+  Check(Net::Send(writer, Payload, 2, 0) == 2, "send first TCP fragment");
+  std::jthread sender([&] {
+    std::this_thread::sleep_for(std::chrono::milliseconds(20));
+    Check(Net::Send(writer, Payload + 2, 4, 0) == 4, "send second TCP fragment");
+  });
+  std::array<char, 8> received {};
+  Check(Net::Recv(reader, received.data(), 6, PeekWaitAll) == 6 &&
+            std::memcmp(received.data(), Payload, 6) == 0,
+        "combined peek waits for fragmented TCP bytes");
+  sender.join();
+  received.fill(0);
+  Check(Net::Recv(reader, received.data(), 6, PeekWaitAll) == 6 &&
+            std::memcmp(received.data(), Payload, 6) == 0,
+        "repeated combined peek preserves all bytes");
+
+  const int epoll = Net::EpollCreate("buffered receive", 0);
+  Net::NetEpollEvent registration {};
+  registration.events = 1;
+  registration.data.u64 = 0x12345678;
+  Check(epoll >= 0 && Net::EpollControl(epoll, 1, reader, &registration) == 0,
+        "register buffered socket readability");
+  const auto check_readable = [&](bool expected) {
+    std::array<uint64_t, 16> readable {};
+    const auto bit = uint64_t {1} << (reader % 64);
+    readable[reader / 64] = bit;
+    const std::array<int64_t, 2> timeout {expected ? 1 : 0, 0};
+    Check(Net::Select(reader + 1, readable.data(), nullptr, nullptr, timeout.data()) ==
+              (expected ? 1 : 0) && readable[reader / 64] == (expected ? bit : 0),
+          "select reflects buffered bytes and their removal");
+    if (expected) {
+      auto writable = readable;
+      Check(Net::Select(reader + 1, readable.data(), writable.data(), nullptr,
+                        timeout.data()) == 2 && readable[reader / 64] == bit &&
+                writable[reader / 64] == bit,
+            "select counts readable and writable bits once each");
+    }
+    Net::NetEpollEvent event {};
+    Check(Net::EpollWait(epoll, &event, 1, expected ? 1000000 : 0) ==
+              (expected ? 1 : 0) &&
+              (!expected || (event.events == 1 && event.data.u64 == registration.data.u64)),
+          "epoll reflects buffered bytes and preserves registration data");
+  };
+  check_readable(true);
+  std::array<uint8_t, 16> peer {}, expected_peer {};
+  uint32_t peer_size = peer.size(), expected_peer_size = expected_peer.size();
+  Check(Net::Getsockname(writer, expected_peer.data(), &expected_peer_size) == 0 &&
+            Net::Recv(reader, received.data(), 0, PeekWaitAll) == 0 &&
+            Net::Recvfrom(reader, received.data(), 2, 0, peer.data(), &peer_size) == 2 &&
+            peer_size == expected_peer_size && peer == expected_peer &&
+            std::memcmp(received.data(), Payload, 2) == 0 &&
+            FileSystem::KernelRead(reader, received.data(), 2) == 2 &&
+            std::memcmp(received.data(), Payload + 2, 2) == 0,
+        "zero-length peek preserves bytes; recvfrom returns peer and shares kernel read buffer");
+  check_readable(true);
+  Check(Net::Send(writer, "gh", 2, 0) == 2 &&
+            Net::Recv(reader, received.data(), 4, 0x40) == 4 &&
+            std::memcmp(received.data(), "efgh", 4) == 0,
+        "waitall joins buffered bytes with newly received bytes");
+  check_readable(false);
+  Check(Net::Send(writer, "mn", 2, 0) == 2, "send plain peek fixture");
+  const auto before_peek = std::chrono::steady_clock::now();
+  Check(Net::Recv(reader, received.data(), 4, 2) == 2 &&
+            std::memcmp(received.data(), "mn", 2) == 0 &&
+            std::chrono::steady_clock::now() - before_peek < std::chrono::milliseconds(500) &&
+            Net::Recv(reader, received.data(), 4, 0) == 2 &&
+            std::memcmp(received.data(), "mn", 2) == 0,
+        "plain peek returns available bytes promptly and preserves them for ordinary receive");
+
+  const int enabled = 1, disabled = 0;
+  Check(Net::Setsockopt(reader, 0xffff, 0x1200, &enabled, sizeof(enabled)) == 0 &&
+            Net::Recv(reader, received.data(), 4, PeekWaitAll) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK,
+        "empty nonblocking combined peek returns would-block");
+  Check(Net::Send(writer, "ij", 2, 0) == 2, "send nonblocking receive fixture");
+  check_readable(true);
+  *Libs::Posix::GetErrorAddr() = Libs::Posix::POSIX_EINVAL;
+  for (int i = 0; i < 2; ++i) {
+    Check(Net::Recv(reader, received.data(), 4, PeekWaitAll) == 2 &&
+              std::memcmp(received.data(), "ij", 2) == 0 &&
+              *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EINVAL,
+          "nonblocking combined peek preserves a short result and successful errno");
+  }
+  Check(Net::Recv(reader, received.data(), 4, 0) == 2 &&
+            std::memcmp(received.data(), "ij", 2) == 0,
+        "ordinary receive consumes a short buffered result");
+  check_readable(false);
+  Check(Net::Setsockopt(reader, 0xffff, 0x1200, &disabled, sizeof(disabled)) == 0 &&
+            Net::EpollDestroy(epoll) == 0,
+        "restore blocking receives and destroy epoll");
+  Check(Net::Recv(reader, received.data(), 4, PeekWaitAll | 0x80) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK,
+        "per-call nonblocking combined peek returns would-block on a blocking socket");
+  Check(Net::Send(writer, "kl", 2, 0) == 2 && Net::Shutdown(writer, 1) == 0,
+        "send a final partial payload and EOF");
+  for (int i = 0; i < 2; ++i) {
+    Check(Net::Recv(reader, received.data(), 4, PeekWaitAll) == 2 &&
+              std::memcmp(received.data(), "kl", 2) == 0,
+          "EOF ends waitall while repeated peeks preserve the short payload");
+  }
+  Check(Net::Recv(reader, received.data(), 4, 0x40) == 2 &&
+            std::memcmp(received.data(), "kl", 2) == 0 &&
+            Net::Recv(reader, received.data(), 4, PeekWaitAll) == 0,
+        "draining the final buffered payload exposes EOF");
+
+  const int datagram = Net::Socket(2, 2, 0);
+  const int datagram_writer = Net::Socket(2, 2, 0);
+  std::array<uint8_t, 16> address {16, 2, 0, 0, 127, 0, 0, 1};
+  uint32_t address_size = address.size();
+  Check(datagram >= 0 && datagram_writer >= 0 &&
+            Net::Bind(datagram, address.data(), address.size()) == 0 &&
+            Net::Getsockname(datagram, address.data(), &address_size) == 0 &&
+            Net::Setsockopt(datagram, 0xffff, 0x1006, &timeout_ms, sizeof(timeout_ms)) == 0 &&
+            Net::Sendto(datagram_writer, Payload, 6, 0, address.data(), address_size) == 6,
+        "create bounded UDP truncation fixture");
+  *Libs::Posix::GetErrorAddr() = Libs::Posix::POSIX_EINVAL;
+  Check(Net::Recvfrom(datagram, received.data(), 2, 0, nullptr, nullptr) == 2 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EINVAL &&
+            std::memcmp(received.data(), Payload, 2) == 0 &&
+            Net::Setsockopt(datagram, 0xffff, 0x1200, &enabled, sizeof(enabled)) == 0 &&
+            Net::Recv(datagram, received.data(), received.size(), 0) == -1 &&
+            *Libs::Posix::GetErrorAddr() == Libs::Posix::POSIX_EWOULDBLOCK,
+        "UDP truncation returns its prefix and discards the rest of the datagram");
+  Check(Net::SocketClose(datagram) == 0 && Net::SocketClose(datagram_writer) == 0,
+        "close UDP truncation fixture");
+}
+#endif
+
 void CheckSocketWakeup() {
   namespace Net = Libs::Network::Net;
   Loader::SymbolDatabase symbols;
@@ -829,21 +1035,7 @@ void CheckSocketWakeup() {
   const auto net_send = reinterpret_cast<Send>(send_symbol->vaddr);
   const auto net_recv = reinterpret_cast<Recv>(recv_symbol->vaddr);
   auto *net_errno = reinterpret_cast<Errno>(errno_symbol->vaddr)();
-  // Guest sockaddr_in: length, family, network-order port/address, padding.
-  std::array<uint8_t, 16> address {16, 2, 0, 0, 127, 0, 0, 1};
-  const int listener = Net::Socket(2, 1, 0);
-  Check(listener >= 0, "create loopback listener");
-  Check(Net::Bind(listener, address.data(), address.size()) == 0, "bind loopback");
-  Check(Net::Listen(listener, 1) == 0, "listen on loopback");
-  uint32_t address_size = address.size();
-  Check(Net::Getsockname(listener, address.data(), &address_size) == 0,
-        "get assigned loopback port");
-  const int writer = Net::Socket(2, 1, 0);
-  Check(writer >= 0 && Net::Connect(writer, address.data(), address_size) == 0,
-        "connect wake socket");
-  const int reader = Net::Accept(listener, nullptr, nullptr);
-  Check(reader >= 0, "accept wake socket");
-  Check(Net::SocketClose(listener) == 0, "close listener");
+  const auto [reader, writer] = CreateTcpPair();
   const int enabled = 1;
   Check(Net::Setsockopt(writer, 6, 1, &enabled, sizeof(enabled)) == 0,
         "enable TCP_NODELAY");
@@ -911,6 +1103,10 @@ void CheckSocketWakeup() {
             *net_errno == Libs::Posix::POSIX_EPIPE,
         "Net send reports a broken pipe without raising host SIGPIPE");
   Check(Net::SocketClose(disconnected) == 0, "close unconnected socket");
+#endif
+#if defined(_WIN32)
+  CheckSocketReceiveBuffer(reader, writer);
+  CheckSocketReceiveConcurrency();
 #endif
   Check(Net::SocketClose(writer) == 0, "close wake writer");
   *net_errno = Libs::Posix::POSIX_EINVAL;

tests/VirtualMemoryAllocationTests.cpp

@@ -26,16 +26,25 @@
 #include <filesystem>
 #include <string>
 #include <thread>
+#include <utility>
 #include <vector>
 
 #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <csignal>
 #include <immintrin.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <x86intrin.h>
+#endif
+#include <Zydis/Zydis.h>
 #include <xbyak/xbyak.h>
 #include <xbyak/xbyak_util.h>
 #endif
 
 #if defined(__linux__)
+#include <asm/prctl.h>
+#include <sys/syscall.h>
 #include <sys/uio.h>
 #include <ucontext.h>
 #include <unistd.h>
@@ -2900,6 +2909,140 @@ struct InstructionTestScope {
 	}
 };
 
+struct SavedInstructionContext {
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	CONTEXT native {};
+	auto&   Gpr(size_t index) {
+		const std::array registers {&native.Rax, &native.Rcx, &native.Rdx, &native.Rbx,
+		                            &native.Rsp, &native.Rbp, &native.Rsi, &native.Rdi,
+		                            &native.R8,  &native.R9,  &native.R10, &native.R11,
+		                            &native.R12, &native.R13, &native.R14, &native.R15};
+		return *registers[index];
+	}
+	auto& Rip() { return native.Rip; }
+	auto& Flags() { return native.EFlags; }
+	void* Xmm() { return &native.Xmm0; }
+#else
+	ucontext_t    native {};
+	_libc_fpstate fpstate {};
+	SavedInstructionContext() { native.uc_mcontext.fpregs = &fpstate; }
+	auto& Gpr(size_t index) {
+		constexpr std::array registers {REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP,
+		                                REG_RSI, REG_RDI, REG_R8,  REG_R9,  REG_R10, REG_R11,
+		                                REG_R12, REG_R13, REG_R14, REG_R15};
+		return native.uc_mcontext.gregs[registers[index]];
+	}
+	auto& Rip() { return native.uc_mcontext.gregs[REG_RIP]; }
+	auto& Flags() { return native.uc_mcontext.gregs[REG_EFL]; }
+	void* Xmm() { return fpstate._xmm; }
+#endif
+};
+
+void TestCpuExtensionContexts() {
+	const char*              test = "CpuExtensionContexts";
+	SavedInstructionContext  context;
+	constexpr uint64_t       sentinel = 0x8192a3b4c5d6e700ull;
+	std::array<uint8_t, 256> xmm;
+	for (size_t i = 0; i < xmm.size(); ++i) {
+		xmm[i] = static_cast<uint8_t>(i);
+	}
+	std::memcpy(context.Xmm(), xmm.data(), xmm.size());
+	const auto check_state = [&](const std::array<uint64_t, 16>& expected) {
+		for (size_t reg = 0; reg < expected.size(); ++reg) {
+			Check(test, static_cast<uint64_t>(context.Gpr(reg)) == expected[reg],
+			      "CPU extension changed an unexpected GPR");
+		}
+		Check(test,
+		      context.Flags() == 0x897 && std::memcmp(context.Xmm(), xmm.data(), xmm.size()) == 0,
+		      "CPU extension changed RFLAGS or XMM state");
+	};
+	for (uint8_t encoding = 0; encoding < 32; ++encoding) {
+		const uint8_t destination = encoding & 15;
+		bool          checked     = false;
+		for (size_t attempt = 0; attempt < 100 && !checked; ++attempt) {
+			std::array<uint64_t, 16> expected;
+			for (size_t reg = 0; reg < expected.size(); ++reg) {
+				context.Gpr(reg) = expected[reg] = sentinel | reg;
+			}
+			context.Flags() = 0x897;
+			std::array<uint8_t, 16> instruction {
+			    0xf3, static_cast<uint8_t>(0x40 | ((encoding >> 4) << 3) | (destination >> 3)),
+			    0x0f, 0xc7, static_cast<uint8_t>(0xf8 | (destination & 7))};
+			context.Rip()   = reinterpret_cast<uintptr_t>(instruction.data());
+			uint32_t before = 0;
+			uint32_t after  = 0;
+			__rdtscp(&before);
+			Check(test, EmulateInstructionContext(&context.native), "RDPID fallback was rejected");
+			__rdtscp(&after);
+			Check(test, context.Rip() == reinterpret_cast<uintptr_t>(instruction.data() + 5),
+			      "RDPID advanced RIP incorrectly");
+			if (before == after) {
+				expected[destination] = before;
+				check_state(expected);
+				checked = true;
+			}
+		}
+		Check(test, checked, "thread migrated repeatedly during RDPID comparison");
+	}
+
+	constexpr uint64_t size    = 0x4000;
+	const auto         mapping = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x30000000, size, Common::VirtualMemory::Mode::ReadWrite, "clwb_context_test");
+	Check(test, mapping != 0 && mapping + size <= UINT32_MAX,
+	      "failed to allocate low CLWB fixture");
+	InstructionTestScope restore {mapping, size};
+	auto*                instruction = reinterpret_cast<uint8_t*>(mapping);
+	auto*                data        = instruction + 0x200;
+	std::memset(data, 0xa5, 64);
+	const auto flush = [&](std::initializer_list<uint8_t> bytes, int base, uint64_t value,
+	                       int index = -1, uint64_t index_value = 0) {
+		std::array<uint64_t, 16> expected;
+		for (size_t reg = 0; reg < expected.size(); ++reg) {
+			context.Gpr(reg) = expected[reg] = sentinel | reg;
+		}
+		if (base >= 0) {
+			context.Gpr(base) = expected[base] = value;
+		}
+		if (index >= 0) {
+			context.Gpr(index) = expected[index] = index_value;
+		}
+		context.Flags() = 0x897;
+		std::fill_n(instruction, 16, 0);
+		std::copy(bytes.begin(), bytes.end(), instruction);
+		context.Rip() = mapping;
+		Check(test, EmulateInstructionContext(&context.native), "CLWB fallback was rejected");
+		Check(test, static_cast<uint64_t>(context.Rip()) == mapping + bytes.size(),
+		      "CLWB advanced RIP incorrectly");
+		check_state(expected);
+		Check(test, std::all_of(data, data + 64, [](uint8_t byte) { return byte == 0xa5; }),
+		      "CLWB changed guest memory");
+	};
+	const auto address = reinterpret_cast<uintptr_t>(data);
+	flush({0x66, 0x0f, 0xae, 0x37}, 7, address);                 // [rdi]
+	flush({0x66, 0x0f, 0xae, 0x74, 0x24, 0xf8}, 4, address + 8); // [rsp-8]
+	flush({0x66, 0x41, 0x0f, 0xae, 0x34, 0x24}, 12, address);    // [r12]
+	flush({0x66, 0x43, 0x0f, 0xae, 0x74, 0xac, 0x20}, 12, address - 0x20 - 12, 13,
+	      3);                                                    // [r12+r13*4+32]
+	flush({0x66, 0x41, 0x0f, 0xae, 0x35, 0xf7, 1, 0, 0}, -1, 0); // RIP+503; REX.B ignored
+	flush({0x67, 0x66, 0x0f, 0xae, 0x37}, 7, address | (uint64_t {1} << 32)); // [edi]
+	const auto byte = [&](int shift) { return static_cast<uint8_t>(address >> shift); };
+	flush({0x67, 0x66, 0x0f, 0xae, 0x35, 0xf7, 1, 0, 0}, -1, 0); // EIP+503
+	flush({0x66, 0x41, 0x0f, 0xae, 0x34, 0x25, byte(0), byte(8), byte(16), byte(24)}, -1,
+	      0); // SIB absolute; REX.B ignored
+#if defined(__linux__)
+	for (const auto segment:
+	     {std::pair {ARCH_GET_FS, uint8_t {0x64}}, std::pair {ARCH_GET_GS, uint8_t {0x65}}}) {
+		unsigned long base = 0;
+		Check(test, syscall(SYS_arch_prctl, segment.first, &base) == 0,
+		      "failed to read segment base");
+		flush({segment.second, 0x66, 0x0f, 0xae, 0x37}, 7, address - base);
+	}
+#else
+	flush({0x65, 0x66, 0x0f, 0xae, 0x37}, 7, address - reinterpret_cast<uintptr_t>(NtCurrentTeb()));
+#endif
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 void TestPackedReciprocalSquareRoot() {
 	const char* test = "PackedReciprocalSquareRoot";
 	constexpr uint64_t code_size = 0x4000;
@@ -3229,12 +3372,12 @@ void TestPackedReciprocalSquareRoot() {
 	std::printf("[host]    %-48s ok\n", test);
 }
 
-void TestPackedBitFieldExtract() {
-	const char*        test      = "PackedBitFieldExtract";
+void TestPackedBitField(bool insert) {
+	const char*        test      = insert ? "PackedBitFieldInsert" : "PackedBitFieldExtract";
 	constexpr uint64_t code_size = 0x80000;
 	const auto         mapping   = Libs::LibKernel::Memory::AllocateProgramMemory(
-	    0x904000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, "extrq_test");
-	Check(test, mapping != 0, "failed to allocate EXTRQ test code");
+	    0x904000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite, "bitfield_test");
+	Check(test, mapping != 0, "failed to allocate SSE4a bitfield test code");
 	InstructionTestScope restore {mapping, code_size * 2};
 	restore.InstallHandler(test);
 	const bool host_sse4a      = Xbyak::util::Cpu().has(Xbyak::util::Cpu::tSSE4a);
@@ -3243,19 +3386,23 @@ void TestPackedBitFieldExtract() {
 		                                            reinterpret_cast<void*>(mapping + code_size),
 		                                            capacity);
 	};
-	// Xbyak has no EXTRQ mnemonic. Emit the two canonical SSE4a encodings directly.
-	const auto extrq = [](Xbyak::CodeGenerator& code, int destination, int source,
-	                      uint8_t length = 0, uint8_t index = 0) {
-		code.db(0x66);
+	// Xbyak has no SSE4a bitfield mnemonics. A negative source selects the immediate form.
+	const auto emit_bit_field = [insert](Xbyak::CodeGenerator& code, int destination, int source,
+	                                     uint8_t length = 0, uint8_t index = 0) {
+		const bool immediate = source < 0;
+		if (insert && immediate) {
+			source = 9;
+		}
+		code.db(insert ? 0xf2 : 0x66);
 		const int rex = source < 0 ? (destination >> 3) : ((destination >> 3) << 2) | (source >> 3);
 		if (rex != 0) {
 			code.db(0x40 | rex);
 		}
 		code.db(0x0f);
-		code.db(source < 0 ? 0x78 : 0x79);
+		code.db(immediate ? 0x78 : 0x79);
 		code.db(source < 0 ? 0xc0 | (destination & 7)
 		                   : 0xc0 | ((destination & 7) << 3) | (source & 7));
-		if (source < 0) {
+		if (immediate) {
 			code.db(length);
 			code.db(index);
 		}
@@ -3270,6 +3417,16 @@ void TestPackedBitFieldExtract() {
 		}
 		return result;
 	};
+	const auto insert_bits = [](uint64_t destination, uint64_t source, uint32_t length,
+	                            uint32_t index) {
+		const uint32_t count = (length & 63u) == 0 ? 64 : length & 63u;
+		const uint32_t first = index & 63u;
+		for (uint32_t bit = 0; bit < count && first + bit < 64; ++bit) {
+			const uint64_t mask = uint64_t {1} << (first + bit);
+			destination         = (destination & ~mask) | (((source >> bit) & 1u) << (first + bit));
+		}
+		return destination;
+	};
 	struct Result {
 		std::array<uint64_t, 4>  destination;
 		std::array<uint64_t, 4>  source;
@@ -3299,7 +3456,8 @@ void TestPackedBitFieldExtract() {
 	for (const auto registers: {std::array {1, 8}, std::array {8, 9}, std::array {3, 3},
 	                            std::array {15, 0}, std::array {8, -1}}) {
 		const auto           destination = registers[0];
-		const auto           source      = registers[1];
+		const auto           source      = insert && registers[1] < 0 ? 9 : registers[1];
+		const bool           immediate   = registers[1] < 0;
 		Xbyak::CodeGenerator code(code_size, reinterpret_cast<void*>(mapping));
 		const std::array     saved {code.rbx, code.rbp, code.r12, code.r13, code.r14, code.r15};
 		const std::array gprs {code.rax, code.rcx, code.rdx, code.rbx, code.rbp, code.r8, code.r9,
@@ -3322,7 +3480,7 @@ void TestPackedBitFieldExtract() {
 			code.mov(gprs[index], gpr_sentinel | index);
 		}
 		code.mov(code.qword[code.rsi + offsetof(Result, stack_before)], code.rsp);
-		extrq(code, destination, source, 0xc8, 0xc4);
+		emit_bit_field(code, destination, registers[1], 0xc8, 0xc4);
 		code.mov(code.qword[code.rsi + offsetof(Result, stack_after)], code.rsp);
 		for (size_t index = 0; index < gprs.size(); ++index) {
 			code.mov(code.qword[code.rsi + offsetof(Result, gprs) + index * 8], gprs[index]);
@@ -3344,7 +3502,7 @@ void TestPackedBitFieldExtract() {
 			code.pop(*reg);
 		}
 		code.ret();
-		Check(test, Xbyak::GetError() == 0, "failed to generate EXTRQ state fixture");
+		Check(test, Xbyak::GetError() == 0, "failed to generate SSE4a bitfield state fixture");
 		const std::vector<uint8_t> original(code.getCode(), code.getCurr());
 		register_module(code_size);
 		const std::vector<uint8_t> trampoline(code.getCode() + code_size,
@@ -3352,45 +3510,51 @@ void TestPackedBitFieldExtract() {
 		const auto                 disabled =
 		    Loader::PatchGuestInstructions(mapping, code.getSize(), function_starts, false, false);
 		Check(test,
-		      disabled.extrq.found == 0 &&
+		      disabled.extrq.found == 0 && disabled.insertq.found == 0 &&
 		          std::equal(original.begin(), original.end(), code.getCode()) &&
 		          std::equal(trampoline.begin(), trampoline.end(), code.getCode() + code_size),
-		      "disabled AMD patching changed EXTRQ code or trampolines");
+		      "disabled AMD patching changed SSE4a bitfield code or trampolines");
 		for (const bool fallback: {false, true}) {
+			if (insert && fallback) {
+				continue; // INSERTQ always uses the trap path.
+			}
 			std::memcpy(reinterpret_cast<void*>(mapping), original.data(), original.size());
 			register_module(fallback ? 32 : code_size);
-			const auto counts = Loader::PatchGuestInstructions(mapping, code.getSize(),
-			                                                   function_starts, false, true)
-			                        .extrq;
+			const auto patched = Loader::PatchGuestInstructions(mapping, code.getSize(),
+			                                                    function_starts, false, true);
+			const auto counts  = insert ? patched.insertq : patched.extrq;
 			if (host_sse4a) {
 				Check(test,
 				      counts.found == 0 &&
 				          std::equal(original.begin(), original.end(), code.getCode()),
-				      "SSE4a host received unnecessary EXTRQ patches");
+				      "SSE4a host received unnecessary SSE4a bitfield patches");
 				std::printf("[host]    %-48s skipped (native SSE4a host)\n", test);
 				return;
 			}
-			std::printf("[host]    EXTRQ xmm%d,%d %s found=%" PRIu64 " native=%" PRIu64
+			std::printf("[host]    bitfield xmm%d,%d %s found=%" PRIu64 " native=%" PRIu64
 			            " trapped=%" PRIu64 "\n",
 			            destination, source, fallback ? "fallback" : "replacement", counts.found,
 			            counts.native, counts.trapped);
 			Check(test,
 			      Xbyak::GetError() == 0 && counts.found == 1 && counts.Skipped() == 0 &&
-			          counts.native == (fallback ? 0u : 1u) &&
-			          counts.trapped == (fallback ? 1u : 0u),
-			      "EXTRQ native/fallback patch counts differ");
+			          counts.native == (fallback || insert ? 0u : 1u) &&
+			          counts.trapped == (fallback || insert ? 1u : 0u),
+			      "SSE4a bitfield native/fallback patch counts differ");
 			const auto traps_before = g_instruction_traps;
 			uint64_t   calls        = 0;
 			for (uint32_t length = 0; length < 64; ++length) {
 				for (uint32_t index = 0; index < 64; ++index) {
-					if (source < 0 && (length != 8 || index != 4)) {
+					if (immediate && (length != 8 || index != 4)) {
 						continue;
 					}
 					for (const uint64_t pattern: {UINT64_MAX, uint64_t {0x0123456789abcdef}}) {
 						const uint64_t controls = (pattern & ~uint64_t {0xffff}) |
 						                          ((index | 0xc0u) << 8) | (length | 0xc0u);
-						input[0]                = source == destination ? controls : pattern;
-						input[4]                = controls;
+						input[0] = !insert && source == destination ? controls : pattern;
+						input[4] = insert ? ~pattern : controls;
+						if (insert) {
+							input[source == destination ? 1 : 5] = controls;
+						}
 						Result output {};
 						_mm_setcsr(0x5fa5);
 						function(input.data(), &output);
@@ -3398,38 +3562,47 @@ void TestPackedBitFieldExtract() {
 						_mm_setcsr(restore.mxcsr);
 						++calls;
 						Check(test,
-						      output.destination[0] == extract(input[0], length, index) &&
-						          output.destination[1] == 0 && output.destination[2] == input[2] &&
+						      output.destination[0] ==
+						              (insert ? insert_bits(input[0],
+						                                    input[source == destination ? 0 : 4],
+						                                    length, index)
+						                      : extract(input[0], length, index)) &&
+						          output.destination[1] == (insert ? input[1] : 0) &&
+						          output.destination[2] == input[2] &&
 						          output.destination[3] == input[3],
-						      "EXTRQ result or destination lanes differ");
+						      "SSE4a bitfield result or destination lanes differ");
 						Check(test,
 						      source < 0 || source == destination
 						          ? output.source == output.destination
 						          : std::equal(output.source.begin(), output.source.end(),
 						                       input.begin() + 4),
-						      "EXTRQ changed its separate source register");
+						      "SSE4a bitfield changed its separate source register");
 						Check(test,
 						      mxcsr == 0x5fa5 && (output.flags & 0x8d5u) == 0x895u &&
 						          output.stack_before == output.stack_after,
-						      "EXTRQ changed MXCSR, RFLAGS or RSP");
+						      "SSE4a bitfield changed MXCSR, RFLAGS or RSP");
 						for (size_t reg = 0; reg < output.gprs.size(); ++reg) {
 							Check(test, output.gprs[reg] == (gpr_sentinel | reg),
-							      "EXTRQ corrupted a guest GPR");
+							      "SSE4a bitfield corrupted a guest GPR");
 						}
 						for (size_t slot = 0; slot < output.red_zone.size(); ++slot) {
 							Check(test,
 							      output.red_zone[slot] == (red_zone_sentinel | ((slot + 1) * 8)),
-							      "EXTRQ corrupted the guest red zone");
+							      "SSE4a bitfield corrupted the guest red zone");
 						}
 					}
 				}
 			}
 			Check(test,
 			      static_cast<uint64_t>(g_instruction_traps - traps_before) ==
-			          (fallback ? calls : 0),
-			      "EXTRQ execution did not match native/fallback selection");
+			          (fallback || insert ? calls : 0),
+			      "SSE4a bitfield execution did not match native/fallback selection");
 		}
 	}
+	if (insert) {
+		std::printf("[host]    %-48s ok (all 4096 length/index pairs)\n", test);
+		return;
+	}
 
 	// Cover every immediate length/index pair without re-decoding thousands of large fixtures.
 	Xbyak::CodeGenerator   code(code_size, reinterpret_cast<void*>(mapping));
@@ -3438,7 +3611,7 @@ void TestPackedBitFieldExtract() {
 		for (uint32_t index = 0; index < 64; ++index) {
 			starts.push_back(mapping + code.getSize());
 			code.vmovups(code.ymm8, code.ptr[code.rdi]);
-			extrq(code, 8, -1, length | 0xc0, index | 0xc0);
+			emit_bit_field(code, 8, -1, length | 0xc0, index | 0xc0);
 			code.vmovups(code.ptr[code.rsi], code.ymm8);
 			code.vzeroupper();
 			code.ret();
@@ -3480,7 +3653,7 @@ void TestPackedBitFieldExtract() {
 	code.vmovups(code.ymm1, code.ptr[code.rdi]);
 	code.vmovups(code.ymm2, code.ptr[code.rdi + 32]);
 	code.lea(code.rax, code.ptr[code.rip + continuation]);
-	extrq(code, 1, 2);
+	emit_bit_field(code, 1, 2);
 	code.jmp(code.rax);
 	code.L(continuation);
 	code.vmovups(code.ptr[code.rsi], code.ymm1);
@@ -3508,7 +3681,7 @@ void TestPackedBitFieldExtract() {
 	code.vmovups(code.ymm1, code.ptr[code.rdi]);
 	code.vmovups(code.ymm2, code.ptr[code.rdi + 32]);
 	code.vmovups(code.ymm3, code.ptr[code.rdi + 64]);
-	extrq(code, 1, 2);
+	emit_bit_field(code, 1, 2);
 	code.vrsqrtps(code.xmm4, code.xmm3);
 	code.vmovups(code.ptr[code.rsi], code.ymm1);
 	code.vmovups(code.ptr[code.rsi + 32], code.ymm4);
@@ -3538,6 +3711,170 @@ void TestPackedBitFieldExtract() {
 	std::printf("[host]    %-48s ok (all 4096 length/index pairs)\n", test);
 }
 
+void TestPackedBitFieldExtract() {
+	TestPackedBitField(false);
+}
+
+void TestPackedBitFieldInsert() {
+	TestPackedBitField(true);
+}
+
+void TestCpuExtensionPatches() {
+	const char*        test      = "CpuExtensionPatches";
+	constexpr uint64_t code_size = 0x4000;
+	const auto         mapping   = Libs::LibKernel::Memory::AllocateProgramMemory(
+	    0x906000000, code_size * 2, Common::VirtualMemory::Mode::ExecuteReadWrite,
+	    "cpu_extension_test");
+	Check(test, mapping != 0, "failed to allocate CPU extension fixture");
+	InstructionTestScope restore {mapping, code_size * 2};
+	restore.InstallHandler(test);
+	const auto                     host = Loader::GetGuestInstructionHostFeatures();
+	const std::array<uintptr_t, 1> starts {mapping};
+	const auto                     register_module = [&](uint64_t capacity) {
+		std::memset(reinterpret_cast<void*>(mapping + code_size), 0, code_size);
+		Loader::RegisterGuestInstructionPatchModule(reinterpret_cast<void*>(mapping), code_size,
+		                                            reinterpret_cast<void*>(mapping + code_size),
+		                                            capacity);
+	};
+	Xbyak::CodeGenerator code(code_size, reinterpret_cast<void*>(mapping));
+	using GuestFunction = void(KYTY_SYSV_ABI*)(const uint64_t*, uint64_t*);
+	const auto function = reinterpret_cast<GuestFunction>(mapping);
+	for (const bool writes_rsp: {false, true}) {
+		code.reset();
+		code.push(code.r12);
+		code.mov(code.r12, code.rsp);
+		code.vmovups(code.ymm8, code.ptr[code.rdi]);
+		code.vmovups(code.ymm9, code.ptr[code.rdi + 32]);
+		code.mov(code.r10, 0x123456789abcdef0ull);
+		code.mov(code.qword[code.rsp - 8], code.r10);
+		code.clwb(code.ptr[code.rsp - 8]);
+		for (const uint8_t byte: {0xf2, 0x45, 0x0f, 0x79, 0xc1}) {
+			code.db(byte); // INSERTQ xmm8,xmm9
+		}
+		code.db(0xf3); // RDPID rax/rsp
+		code.db(0x48);
+		code.db(0x0f);
+		code.db(0xc7);
+		code.db(writes_rsp ? 0xfc : 0xf8);
+		if (writes_rsp) {
+			code.mov(code.rax, code.rsp);
+			code.mov(code.rsp, code.r12);
+		}
+		code.mov(code.qword[code.rsi], code.rax);
+		code.mov(code.rax, code.qword[code.rsp - 8]);
+		code.mov(code.qword[code.rsi + 8], code.rax);
+		code.vmovups(code.ptr[code.rsi + 16], code.ymm8);
+		code.vzeroupper();
+		code.pop(code.r12);
+		code.ret();
+		Check(test, Xbyak::GetError() == 0, "failed to generate mixed CPU extension fixture");
+		const std::vector<uint8_t> original(code.getCode(), code.getCurr());
+		register_module(code_size);
+		const auto disabled =
+		    Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, false, {});
+		Check(test,
+		      disabled.insertq.found == 0 && disabled.rdpid.found == 0 &&
+		          disabled.clwb.found == 0 &&
+		          std::equal(original.begin(), original.end(), code.getCode()) &&
+		          std::all_of(code.getCode() + code_size, code.getCode() + code_size * 2,
+		                      [](uint8_t byte) { return byte == 0; }),
+		      "disabled AMD option changed CPU extension code or trampolines");
+		const auto supported = Loader::PatchGuestInstructions(mapping, code.getSize(), starts,
+		                                                      false, true, {true, true, true});
+		Check(test,
+		      supported.insertq.found == 0 && supported.rdpid.found == 0 &&
+		          supported.clwb.found == 0 &&
+		          std::equal(original.begin(), original.end(), code.getCode()),
+		      "available host CPU extensions were unnecessarily patched");
+		const auto enabled =
+		    Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, true, {});
+		Check(test,
+		      enabled.clwb.found == 1 && enabled.clwb.native == 1 && enabled.clwb.trapped == 0 &&
+		          enabled.insertq.found == 1 && enabled.insertq.native == 0 &&
+		          enabled.insertq.trapped == 1 && enabled.rdpid.found == 1 &&
+		          enabled.rdpid.native == 0 && enabled.rdpid.trapped == 1,
+		      "mixed native CLWB and trapped INSERTQ/RDPID counts differ");
+		// Verify the actual stack-relative CLFLUSH operand, since flushing the wrong mapped
+		// cache line would otherwise leave the sentinel unchanged too.
+		ZydisDecoder decoder {};
+		Check(test,
+		      ZYAN_SUCCESS(
+		          ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64)),
+		      "failed to initialize decoder");
+		bool found_flush = false;
+		for (size_t offset = 0; offset < 512 && !found_flush;) {
+			ZydisDecodedInstruction instruction {};
+			ZydisDecodedOperand     operands[ZYDIS_MAX_OPERAND_COUNT] {};
+			Check(test,
+			      ZYAN_SUCCESS(ZydisDecoderDecodeFull(&decoder, code.getCode() + code_size + offset,
+			                                          code_size - offset, &instruction, operands)),
+			      "failed to decode CPU extension trampoline");
+			if (instruction.mnemonic == ZYDIS_MNEMONIC_CLFLUSH) {
+				found_flush = true;
+				Check(test,
+				      operands[0].mem.base == ZYDIS_REGISTER_RSP &&
+#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+				          operands[0].mem.disp.value == 120,
+#else
+				          operands[0].mem.disp.value == -8,
+#endif
+				      "CLWB replacement did not preserve its guest stack address");
+			}
+			offset += instruction.length;
+		}
+		Check(test, found_flush, "CLWB native replacement did not emit CLFLUSH");
+		const std::array<uint64_t, 8> input {0x1111, 0xdeadbeef, 0x12345678, 0x87654321,
+		                                     0xab,   0xffffc4c8, 0,          0};
+		std::array<uint64_t, 6>       output {};
+		bool                          checked_aux = false;
+		for (size_t attempt = 0; attempt < 100 && !checked_aux; ++attempt) {
+			uint32_t   before = 0;
+			uint32_t   after  = 0;
+			const auto traps  = g_instruction_traps;
+			__rdtscp(&before);
+			function(input.data(), output.data());
+			__rdtscp(&after);
+			Check(test, g_instruction_traps == traps + !host.sse4a + !host.rdpid,
+			      "mixed CPU extension fixture used an unexpected trap path");
+			Check(test,
+			      output[1] == 0x123456789abcdef0ull && output[2] == 0x1ab1 &&
+			          (host.sse4a || output[3] == input[1]) &&
+			          std::equal(output.begin() + 4, output.end(), input.begin() + 2),
+			      "mixed CPU extensions corrupted INSERTQ state or the guest red zone");
+			if (before == after) {
+				Check(test, output[0] == before,
+				      "RDPID patch lost TSC_AUX or adjusted its RSP destination");
+				checked_aux = true;
+			}
+		}
+		Check(test, checked_aux, "thread migrated repeatedly during patched RDPID comparison");
+	}
+
+	code.reset();
+	code.clwb(code.ptr[code.rdi]);
+	code.ret();
+	Check(test, Xbyak::GetError() == 0, "failed to generate exhausted CLWB fixture");
+	const std::vector<uint8_t> original(code.getCode(), code.getCurr());
+	register_module(1);
+	const auto exhausted =
+	    Loader::PatchGuestInstructions(mapping, code.getSize(), starts, false, true, {}).clwb;
+	Check(test,
+	      exhausted.found == 1 && exhausted.native == 0 && exhausted.trapped == 1 &&
+	          std::equal(original.begin(), original.end(), code.getCode()) &&
+	          Xbyak::GetError() == 0,
+	      "exhausted CLWB replacement did not retain its natural trap fallback");
+	std::array<uint64_t, 8> data;
+	data.fill(0x123456789abcdef0ull);
+	const auto traps = g_instruction_traps;
+	function(data.data(), nullptr);
+	Check(test,
+	      g_instruction_traps == traps + !host.clwb &&
+	          std::all_of(data.begin(), data.end(),
+	                      [](uint64_t value) { return value == 0x123456789abcdef0ull; }),
+	      "CLWB direct fallback changed memory or used an unexpected trap path");
+	std::printf("[host]    %-48s ok\n", test);
+}
+
 #endif
 
 #if defined(__x86_64__) || defined(_M_X64)
@@ -3714,6 +4051,12 @@ int main(int argc, char** argv) {
 	}
 #endif
 #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	if (argc == 2 && std::strcmp(argv[1], "--cpu-extensions-only") == 0) {
+		RunTest(TestCpuExtensionContexts);
+		RunTest(TestPackedBitFieldInsert);
+		RunTest(TestCpuExtensionPatches);
+		return g_failed_tests == 0 ? 0 : 1;
+	}
 	if (argc == 2 && std::strcmp(argv[1], "--extrq-only") == 0) {
 		RunTest(TestPackedBitFieldExtract);
 		return g_failed_tests == 0 ? 0 : 1;
@@ -3734,6 +4077,9 @@ int main(int argc, char** argv) {
 #if defined(__linux__) || KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	RunTest(TestPackedReciprocalSquareRoot);
 	RunTest(TestPackedBitFieldExtract);
+	RunTest(TestCpuExtensionContexts);
+	RunTest(TestPackedBitFieldInsert);
+	RunTest(TestCpuExtensionPatches);
 #endif
 	RunTest(TestWindowsGuestRedZoneStaticPatcher);
 	RunTest(TestProsperoArgumentAndInfoSizeContracts);

tests/macosSse4aTests.cpp

@@ -8,6 +8,7 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <initializer_list>
 #include <sys/mman.h>
 #include <sys/ucontext.h>
 #include <unistd.h>
@@ -61,6 +62,21 @@ constexpr uint8_t kInsertqXmm10Xmm9Len16Index8[] = {
 	0xc3,                                     // ret
 };
 
+constexpr uint8_t kInsertqXmm10Xmm9[] = {
+	0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9
+	0xf3, 0x44, 0x0f, 0x6f, 0x12, // movdqu (%rdx), %xmm10
+	0xf2, 0x45, 0x0f, 0x79, 0xd1, // insertq xmm10, xmm9
+	0xf3, 0x44, 0x0f, 0x7f, 0x17, // movdqu %xmm10, (%rdi)
+	0xc3,
+};
+
+constexpr uint8_t kInsertqXmm9Xmm9[] = {
+	0xf3, 0x44, 0x0f, 0x6f, 0x0e, // movdqu (%rsi), %xmm9
+	0xf2, 0x45, 0x0f, 0x79, 0xc9, // insertq xmm9, xmm9
+	0xf3, 0x44, 0x0f, 0x7f, 0x0f, // movdqu %xmm9, (%rdi)
+	0xc3,
+};
+
 constexpr uint8_t kUd2[] = {
 	0x0f, 0x0b, // ud2
 	0xc3,       // ret
@@ -192,6 +208,31 @@ void TestRexHighRegisterInsertqWithIndex() {
 	Check(out[1] == 0xbbbbbbbbbbbbbbbbull, "insertq xmm10, xmm9, 16, 8 leaves the upper 64 bits untouched");
 }
 
+void TestInsertqRegister() {
+	const auto separate = MapCode<PairBlobFn>(kInsertqXmm10Xmm9);
+	const auto aliased  = MapCode<PairBlobFn>(kInsertqXmm9Xmm9);
+	for (const auto fn: {separate, aliased}) {
+		for (const uint64_t controls: {0xc4c8ull, 0xc0c0ull, 0xffc1ull}) {
+			const uint64_t source[2] {0x0123456789abcdefull, 0xdeadbeef00000000ull | controls};
+			const uint64_t destination[2] {0x8877665544332211ull, 0xfedcba9876543210ull};
+			const auto*    initial  = fn == aliased ? source : destination;
+			uint64_t       expected = initial[0];
+			const auto     length   = (controls & 63) == 0 ? 64 : controls & 63;
+			const auto     index    = (controls >> 8) & 63;
+			for (uint64_t bit = 0; bit < length && index + bit < 64; ++bit) {
+				const uint64_t mask = uint64_t {1} << (index + bit);
+				expected = (expected & ~mask) | (((source[0] >> bit) & 1) << (index + bit));
+			}
+			uint64_t  out[2] {};
+			const int before = IllegalHits();
+			fn(out, source, destination);
+			Check(IllegalHits() == before + 1, "register INSERTQ raises one SIGILL");
+			Check(out[0] == expected && out[1] == initial[1],
+			      "register INSERTQ uses upper source controls and preserves upper destination");
+		}
+	}
+}
+
 void TestUnknownInstructionRefused() {
 	g_refused.store(false, std::memory_order_relaxed);
 	g_refuse_skip.store(2, std::memory_order_relaxed);
@@ -213,6 +254,7 @@ int main() {
 	TestInsertqImmediate();
 	TestRexHighRegisterExtrq();
 	TestRexHighRegisterInsertqWithIndex();
+	TestInsertqRegister();
 	TestUnknownInstructionRefused();
 	std::printf("macosSse4aTests: all passed\n");
 	return 0;