kernel: implement AIO request polling

This reverts commit 7012634f32.

3rdparty/renderdoc/renderdoc_app.h

@@ -0,0 +1,875 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2026 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#pragma once
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Documentation for the API is available at https://renderdoc.org/docs/in_application_api.html
+//
+
+#if !defined(RENDERDOC_NO_STDINT)
+#include <stdint.h>
+#endif
+
+#if defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER)
+#define RENDERDOC_CC __cdecl
+#elif defined(__linux__) || defined(__FreeBSD__) || defined(__sun__) || defined(__OpenBSD__)
+#define RENDERDOC_CC
+#elif defined(__APPLE__)
+#define RENDERDOC_CC
+#else
+#error "Unknown platform"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// Constants not used directly in below API
+
+// This is a GUID/magic value used for when applications pass a path where shader debug
+// information can be found to match up with a stripped shader.
+// the define can be used like so: const GUID RENDERDOC_ShaderDebugMagicValue =
+// RENDERDOC_ShaderDebugMagicValue_value
+#define RENDERDOC_ShaderDebugMagicValue_struct                                \
+  {                                                                           \
+    0xeab25520, 0x6670, 0x4865, 0x84, 0x29, 0x6c, 0x8, 0x51, 0x54, 0x00, 0xff \
+  }
+
+// as an alternative when you want a byte array (assuming x86 endianness):
+#define RENDERDOC_ShaderDebugMagicValue_bytearray                                                 \
+  {                                                                                               \
+    0x20, 0x55, 0xb2, 0xea, 0x70, 0x66, 0x65, 0x48, 0x84, 0x29, 0x6c, 0x8, 0x51, 0x54, 0x00, 0xff \
+  }
+
+// truncated version when only a uint64_t is available (e.g. Vulkan tags):
+#define RENDERDOC_ShaderDebugMagicValue_truncated 0x48656670eab25520ULL
+
+// this is a magic value for vulkan user tags to indicate which dispatchable API objects are which
+// for object annotations
+#define RENDERDOC_APIObjectAnnotationHelper 0xfbb3b337b664d0adULL
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// RenderDoc capture options
+//
+
+typedef enum RENDERDOC_CaptureOption
+{
+  // Allow the application to enable vsync
+  //
+  // Default - enabled
+  //
+  // 1 - The application can enable or disable vsync at will
+  // 0 - vsync is force disabled
+  eRENDERDOC_Option_AllowVSync = 0,
+
+  // Allow the application to enable fullscreen
+  //
+  // Default - enabled
+  //
+  // 1 - The application can enable or disable fullscreen at will
+  // 0 - fullscreen is force disabled
+  eRENDERDOC_Option_AllowFullscreen = 1,
+
+  // Record API debugging events and messages
+  //
+  // Default - disabled
+  //
+  // 1 - Enable built-in API debugging features and records the results into
+  //     the capture, which is matched up with events on replay
+  // 0 - no API debugging is forcibly enabled
+  eRENDERDOC_Option_APIValidation = 2,
+  eRENDERDOC_Option_DebugDeviceMode = 2,    // deprecated name of this enum
+
+  // Capture CPU callstacks for API events
+  //
+  // Default - disabled
+  //
+  // 1 - Enables capturing of callstacks
+  // 0 - no callstacks are captured
+  eRENDERDOC_Option_CaptureCallstacks = 3,
+
+  // When capturing CPU callstacks, only capture them from actions.
+  // This option does nothing without the above option being enabled
+  //
+  // Default - disabled
+  //
+  // 1 - Only captures callstacks for actions.
+  //     Ignored if CaptureCallstacks is disabled
+  // 0 - Callstacks, if enabled, are captured for every event.
+  eRENDERDOC_Option_CaptureCallstacksOnlyDraws = 4,
+  eRENDERDOC_Option_CaptureCallstacksOnlyActions = 4,
+
+  // Specify a delay in seconds to wait for a debugger to attach, after
+  // creating or injecting into a process, before continuing to allow it to run.
+  //
+  // 0 indicates no delay, and the process will run immediately after injection
+  //
+  // Default - 0 seconds
+  //
+  eRENDERDOC_Option_DelayForDebugger = 5,
+
+  // Verify buffer access. This includes checking the memory returned by a Map() call to
+  // detect any out-of-bounds modification, as well as initialising buffers with undefined contents
+  // to a marker value to catch use of uninitialised memory.
+  //
+  // NOTE: This option is only valid for OpenGL and D3D11. Explicit APIs such as D3D12 and Vulkan do
+  // not do the same kind of interception & checking and undefined contents are really undefined.
+  //
+  // Default - disabled
+  //
+  // 1 - Verify buffer access
+  // 0 - No verification is performed, and overwriting bounds may cause crashes or corruption in
+  //     RenderDoc.
+  eRENDERDOC_Option_VerifyBufferAccess = 6,
+
+  // The old name for eRENDERDOC_Option_VerifyBufferAccess was eRENDERDOC_Option_VerifyMapWrites.
+  // This option now controls the filling of uninitialised buffers with 0xdddddddd which was
+  // previously always enabled
+  eRENDERDOC_Option_VerifyMapWrites = eRENDERDOC_Option_VerifyBufferAccess,
+
+  // Hooks any system API calls that create child processes, and injects
+  // RenderDoc into them recursively with the same options.
+  //
+  // Default - disabled
+  //
+  // 1 - Hooks into spawned child processes
+  // 0 - Child processes are not hooked by RenderDoc
+  eRENDERDOC_Option_HookIntoChildren = 7,
+
+  // By default RenderDoc only includes resources in the final capture necessary
+  // for that frame, this allows you to override that behaviour.
+  //
+  // Default - disabled
+  //
+  // 1 - all live resources at the time of capture are included in the capture
+  //     and available for inspection
+  // 0 - only the resources referenced by the captured frame are included
+  eRENDERDOC_Option_RefAllResources = 8,
+
+  // **NOTE**: As of RenderDoc v1.1 this option has been deprecated. Setting or
+  // getting it will be ignored, to allow compatibility with older versions.
+  // In v1.1 the option acts as if it's always enabled.
+  //
+  // By default RenderDoc skips saving initial states for resources where the
+  // previous contents don't appear to be used, assuming that writes before
+  // reads indicate previous contents aren't used.
+  //
+  // Default - disabled
+  //
+  // 1 - initial contents at the start of each captured frame are saved, even if
+  //     they are later overwritten or cleared before being used.
+  // 0 - unless a read is detected, initial contents will not be saved and will
+  //     appear as black or empty data.
+  eRENDERDOC_Option_SaveAllInitials = 9,
+
+  // In APIs that allow for the recording of command lists to be replayed later,
+  // RenderDoc may choose to not capture command lists before a frame capture is
+  // triggered, to reduce overheads. This means any command lists recorded once
+  // and replayed many times will not be available and may cause a failure to
+  // capture.
+  //
+  // NOTE: This is only true for APIs where multithreading is difficult or
+  // discouraged. Newer APIs like Vulkan and D3D12 will ignore this option
+  // and always capture all command lists since the API is heavily oriented
+  // around it and the overheads have been reduced by API design.
+  //
+  // 1 - All command lists are captured from the start of the application
+  // 0 - Command lists are only captured if their recording begins during
+  //     the period when a frame capture is in progress.
+  eRENDERDOC_Option_CaptureAllCmdLists = 10,
+
+  // Mute API debugging output when the API validation mode option is enabled
+  //
+  // Default - enabled
+  //
+  // 1 - Mute any API debug messages from being displayed or passed through
+  // 0 - API debugging is displayed as normal
+  eRENDERDOC_Option_DebugOutputMute = 11,
+
+  // Option to allow vendor extensions to be used even when they may be
+  // incompatible with RenderDoc and cause corrupted replays or crashes.
+  //
+  // Default - inactive
+  //
+  // No values are documented, this option should only be used when absolutely
+  // necessary as directed by a RenderDoc developer.
+  eRENDERDOC_Option_AllowUnsupportedVendorExtensions = 12,
+
+  // Define a soft memory limit which some APIs may aim to keep overhead under where
+  // possible. Anything above this limit will where possible be saved directly to disk during
+  // capture.
+  // This will cause increased disk space use (which may cause a capture to fail if disk space is
+  // exhausted) as well as slower capture times.
+  //
+  // Not all memory allocations may be deferred like this so it is not a guarantee of a memory
+  // limit.
+  //
+  // Units are in MBs, suggested values would range from 200MB to 1000MB.
+  //
+  // Default - 0 Megabytes
+  eRENDERDOC_Option_SoftMemoryLimit = 13,
+} RENDERDOC_CaptureOption;
+
+// Sets an option that controls how RenderDoc behaves on capture.
+//
+// Returns 1 if the option and value are valid
+// Returns 0 if either is invalid and the option is unchanged
+typedef int(RENDERDOC_CC *pRENDERDOC_SetCaptureOptionU32)(RENDERDOC_CaptureOption opt, uint32_t val);
+typedef int(RENDERDOC_CC *pRENDERDOC_SetCaptureOptionF32)(RENDERDOC_CaptureOption opt, float val);
+
+// Gets the current value of an option as a uint32_t
+//
+// If the option is invalid, 0xffffffff is returned
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetCaptureOptionU32)(RENDERDOC_CaptureOption opt);
+
+// Gets the current value of an option as a float
+//
+// If the option is invalid, -FLT_MAX is returned
+typedef float(RENDERDOC_CC *pRENDERDOC_GetCaptureOptionF32)(RENDERDOC_CaptureOption opt);
+
+typedef enum RENDERDOC_InputButton
+{
+  // '0' - '9' matches ASCII values
+  eRENDERDOC_Key_0 = 0x30,
+  eRENDERDOC_Key_1 = 0x31,
+  eRENDERDOC_Key_2 = 0x32,
+  eRENDERDOC_Key_3 = 0x33,
+  eRENDERDOC_Key_4 = 0x34,
+  eRENDERDOC_Key_5 = 0x35,
+  eRENDERDOC_Key_6 = 0x36,
+  eRENDERDOC_Key_7 = 0x37,
+  eRENDERDOC_Key_8 = 0x38,
+  eRENDERDOC_Key_9 = 0x39,
+
+  // 'A' - 'Z' matches ASCII values
+  eRENDERDOC_Key_A = 0x41,
+  eRENDERDOC_Key_B = 0x42,
+  eRENDERDOC_Key_C = 0x43,
+  eRENDERDOC_Key_D = 0x44,
+  eRENDERDOC_Key_E = 0x45,
+  eRENDERDOC_Key_F = 0x46,
+  eRENDERDOC_Key_G = 0x47,
+  eRENDERDOC_Key_H = 0x48,
+  eRENDERDOC_Key_I = 0x49,
+  eRENDERDOC_Key_J = 0x4A,
+  eRENDERDOC_Key_K = 0x4B,
+  eRENDERDOC_Key_L = 0x4C,
+  eRENDERDOC_Key_M = 0x4D,
+  eRENDERDOC_Key_N = 0x4E,
+  eRENDERDOC_Key_O = 0x4F,
+  eRENDERDOC_Key_P = 0x50,
+  eRENDERDOC_Key_Q = 0x51,
+  eRENDERDOC_Key_R = 0x52,
+  eRENDERDOC_Key_S = 0x53,
+  eRENDERDOC_Key_T = 0x54,
+  eRENDERDOC_Key_U = 0x55,
+  eRENDERDOC_Key_V = 0x56,
+  eRENDERDOC_Key_W = 0x57,
+  eRENDERDOC_Key_X = 0x58,
+  eRENDERDOC_Key_Y = 0x59,
+  eRENDERDOC_Key_Z = 0x5A,
+
+  // leave the rest of the ASCII range free
+  // in case we want to use it later
+  eRENDERDOC_Key_NonPrintable = 0x100,
+
+  eRENDERDOC_Key_Divide,
+  eRENDERDOC_Key_Multiply,
+  eRENDERDOC_Key_Subtract,
+  eRENDERDOC_Key_Plus,
+
+  eRENDERDOC_Key_F1,
+  eRENDERDOC_Key_F2,
+  eRENDERDOC_Key_F3,
+  eRENDERDOC_Key_F4,
+  eRENDERDOC_Key_F5,
+  eRENDERDOC_Key_F6,
+  eRENDERDOC_Key_F7,
+  eRENDERDOC_Key_F8,
+  eRENDERDOC_Key_F9,
+  eRENDERDOC_Key_F10,
+  eRENDERDOC_Key_F11,
+  eRENDERDOC_Key_F12,
+
+  eRENDERDOC_Key_Home,
+  eRENDERDOC_Key_End,
+  eRENDERDOC_Key_Insert,
+  eRENDERDOC_Key_Delete,
+  eRENDERDOC_Key_PageUp,
+  eRENDERDOC_Key_PageDn,
+
+  eRENDERDOC_Key_Backspace,
+  eRENDERDOC_Key_Tab,
+  eRENDERDOC_Key_PrtScrn,
+  eRENDERDOC_Key_Pause,
+
+  eRENDERDOC_Key_Max,
+} RENDERDOC_InputButton;
+
+// Sets which key or keys can be used to toggle focus between multiple windows
+//
+// If keys is NULL or num is 0, toggle keys will be disabled
+typedef void(RENDERDOC_CC *pRENDERDOC_SetFocusToggleKeys)(RENDERDOC_InputButton *keys, int num);
+
+// Sets which key or keys can be used to capture the next frame
+//
+// If keys is NULL or num is 0, captures keys will be disabled
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureKeys)(RENDERDOC_InputButton *keys, int num);
+
+typedef enum RENDERDOC_OverlayBits
+{
+  // This single bit controls whether the overlay is enabled or disabled globally
+  eRENDERDOC_Overlay_Enabled = 0x1,
+
+  // Show the average framerate over several seconds as well as min/max
+  eRENDERDOC_Overlay_FrameRate = 0x2,
+
+  // Show the current frame number
+  eRENDERDOC_Overlay_FrameNumber = 0x4,
+
+  // Show a list of recent captures, and how many captures have been made
+  eRENDERDOC_Overlay_CaptureList = 0x8,
+
+  // Default values for the overlay mask
+  eRENDERDOC_Overlay_Default = (eRENDERDOC_Overlay_Enabled | eRENDERDOC_Overlay_FrameRate |
+                                eRENDERDOC_Overlay_FrameNumber | eRENDERDOC_Overlay_CaptureList),
+
+  // Enable all bits
+  eRENDERDOC_Overlay_All = 0x7ffffff,
+
+  // Disable all bits
+  eRENDERDOC_Overlay_None = 0,
+} RENDERDOC_OverlayBits;
+
+// returns the overlay bits that have been set
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetOverlayBits)(void);
+// sets the overlay bits with an and & or mask
+typedef void(RENDERDOC_CC *pRENDERDOC_MaskOverlayBits)(uint32_t And, uint32_t Or);
+
+// this function will attempt to remove RenderDoc's hooks in the application.
+//
+// Note: that this can only work correctly if done immediately after
+// the module is loaded, before any API work happens. RenderDoc will remove its
+// injected hooks and shut down. Behaviour is undefined if this is called
+// after any API functions have been called, and there is still no guarantee of
+// success.
+typedef void(RENDERDOC_CC *pRENDERDOC_RemoveHooks)(void);
+
+// DEPRECATED: compatibility for code compiled against pre-1.4.1 headers.
+typedef pRENDERDOC_RemoveHooks pRENDERDOC_Shutdown;
+
+// This function will unload RenderDoc's crash handler.
+//
+// If you use your own crash handler and don't want RenderDoc's handler to
+// intercede, you can call this function to unload it and any unhandled
+// exceptions will pass to the next handler.
+typedef void(RENDERDOC_CC *pRENDERDOC_UnloadCrashHandler)(void);
+
+// Sets the capture file path template
+//
+// pathtemplate is a UTF-8 string that gives a template for how captures will be named
+// and where they will be saved.
+//
+// Any extension is stripped off the path, and captures are saved in the directory
+// specified, and named with the filename and the frame number appended. If the
+// directory does not exist it will be created, including any parent directories.
+//
+// If pathtemplate is NULL, the template will remain unchanged
+//
+// Example:
+//
+// SetCaptureFilePathTemplate("my_captures/example");
+//
+// Capture #1 -> my_captures/example_frame123.rdc
+// Capture #2 -> my_captures/example_frame456.rdc
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureFilePathTemplate)(const char *pathtemplate);
+
+// returns the current capture path template, see SetCaptureFileTemplate above, as a UTF-8 string
+typedef const char *(RENDERDOC_CC *pRENDERDOC_GetCaptureFilePathTemplate)(void);
+
+// DEPRECATED: compatibility for code compiled against pre-1.1.2 headers.
+typedef pRENDERDOC_SetCaptureFilePathTemplate pRENDERDOC_SetLogFilePathTemplate;
+typedef pRENDERDOC_GetCaptureFilePathTemplate pRENDERDOC_GetLogFilePathTemplate;
+
+// returns the number of captures that have been made
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetNumCaptures)(void);
+
+// This function returns the details of a capture, by index. New captures are added
+// to the end of the list.
+//
+// filename will be filled with the absolute path to the capture file, as a UTF-8 string
+// pathlength will be written with the length in bytes of the filename string
+// timestamp will be written with the time of the capture, in seconds since the Unix epoch
+//
+// Any of the parameters can be NULL and they'll be skipped.
+//
+// The function will return 1 if the capture index is valid, or 0 if the index is invalid
+// If the index is invalid, the values will be unchanged
+//
+// Note: when captures are deleted in the UI they will remain in this list, so the
+// capture path may not exist anymore.
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetCapture)(uint32_t idx, char *filename,
+                                                      uint32_t *pathlength, uint64_t *timestamp);
+
+// Sets the comments associated with a capture file. These comments are displayed in the
+// UI program when opening.
+//
+// filePath should be a path to the capture file to add comments to. If set to NULL or ""
+// the most recent capture file created made will be used instead.
+// comments should be a NULL-terminated UTF-8 string to add as comments.
+//
+// Any existing comments will be overwritten.
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureFileComments)(const char *filePath,
+                                                              const char *comments);
+
+// returns 1 if the RenderDoc UI is connected to this application, 0 otherwise
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_IsTargetControlConnected)(void);
+
+// DEPRECATED: compatibility for code compiled against pre-1.1.1 headers.
+// This was renamed to IsTargetControlConnected in API 1.1.1, the old typedef is kept here for
+// backwards compatibility with old code, it is castable either way since it's ABI compatible
+// as the same function pointer type.
+typedef pRENDERDOC_IsTargetControlConnected pRENDERDOC_IsRemoteAccessConnected;
+
+// This function will launch the Replay UI associated with the RenderDoc library injected
+// into the running application.
+//
+// if connectTargetControl is 1, the Replay UI will be launched with a command line parameter
+// to connect to this application
+// cmdline is the rest of the command line, as a UTF-8 string. E.g. a captures to open
+// if cmdline is NULL, the command line will be empty.
+//
+// returns the PID of the replay UI if successful, 0 if not successful.
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_LaunchReplayUI)(uint32_t connectTargetControl,
+                                                          const char *cmdline);
+
+// RenderDoc can return a higher version than requested if it's backwards compatible,
+// this function returns the actual version returned. If a parameter is NULL, it will be
+// ignored and the others will be filled out.
+typedef void(RENDERDOC_CC *pRENDERDOC_GetAPIVersion)(int *major, int *minor, int *patch);
+
+// Requests that the replay UI show itself (if hidden or not the current top window). This can be
+// used in conjunction with IsTargetControlConnected and LaunchReplayUI to intelligently handle
+// showing the UI after making a capture.
+//
+// This will return 1 if the request was successfully passed on, though it's not guaranteed that
+// the UI will be on top in all cases depending on OS rules. It will return 0 if there is no current
+// target control connection to make such a request, or if there was another error
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_ShowReplayUI)(void);
+
+//////////////////////////////////////////////////////////////////////////
+// Capturing functions
+//
+
+// A device pointer is a pointer to the API's root handle.
+//
+// This would be an ID3D11Device, HGLRC/GLXContext, ID3D12Device, etc
+typedef void *RENDERDOC_DevicePointer;
+
+// A window handle is the OS's native window handle
+//
+// This would be an HWND, GLXDrawable, etc
+typedef void *RENDERDOC_WindowHandle;
+
+// A helper macro for Vulkan, where the device handle cannot be used directly.
+//
+// Passing the VkInstance to this macro will return the RENDERDOC_DevicePointer to use.
+//
+// Specifically, the value needed is the dispatch table pointer, which sits as the first
+// pointer-sized object in the memory pointed to by the VkInstance. Thus we cast to a void** and
+// indirect once.
+#define RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(inst) (*((void **)(inst)))
+
+// This sets the RenderDoc in-app overlay in the API/window pair as 'active' and it will
+// respond to keypresses. Neither parameter can be NULL
+typedef void(RENDERDOC_CC *pRENDERDOC_SetActiveWindow)(RENDERDOC_DevicePointer device,
+                                                       RENDERDOC_WindowHandle wndHandle);
+
+// capture the next frame on whichever window and API is currently considered active
+typedef void(RENDERDOC_CC *pRENDERDOC_TriggerCapture)(void);
+
+// capture the next N frames on whichever window and API is currently considered active
+typedef void(RENDERDOC_CC *pRENDERDOC_TriggerMultiFrameCapture)(uint32_t numFrames);
+
+// When choosing either a device pointer or a window handle to capture, you can pass NULL.
+// Passing NULL specifies a 'wildcard' match against anything. This allows you to specify
+// any API rendering to a specific window, or a specific API instance rendering to any window,
+// or in the simplest case of one window and one API, you can just pass NULL for both.
+//
+// In either case, if there are two or more possible matching (device,window) pairs it
+// is undefined which one will be captured.
+//
+// Note: for headless rendering you can pass NULL for the window handle and either specify
+// a device pointer or leave it NULL as above.
+
+// Immediately starts capturing API calls on the specified device pointer and window handle.
+//
+// If there is no matching thing to capture (e.g. no supported API has been initialised),
+// this will do nothing.
+//
+// The results are undefined (including crashes) if two captures are started overlapping,
+// even on separate devices and/oror windows.
+typedef void(RENDERDOC_CC *pRENDERDOC_StartFrameCapture)(RENDERDOC_DevicePointer device,
+                                                         RENDERDOC_WindowHandle wndHandle);
+
+// Returns whether or not a frame capture is currently ongoing anywhere.
+//
+// This will return 1 if a capture is ongoing, and 0 if there is no capture running
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_IsFrameCapturing)(void);
+
+// Ends capturing immediately.
+//
+// This will return 1 if the capture succeeded, and 0 if there was an error capturing.
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_EndFrameCapture)(RENDERDOC_DevicePointer device,
+                                                           RENDERDOC_WindowHandle wndHandle);
+
+// Ends capturing immediately and discard any data stored without saving to disk.
+//
+// This will return 1 if the capture was discarded, and 0 if there was an error or no capture
+// was in progress
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_DiscardFrameCapture)(RENDERDOC_DevicePointer device,
+                                                               RENDERDOC_WindowHandle wndHandle);
+
+// Only valid to be called between a call to StartFrameCapture and EndFrameCapture. Gives a custom
+// title to the capture produced which will be displayed in the UI.
+//
+// If multiple captures are ongoing, this title will be applied to the first capture to end after
+// this call. The second capture to end will have no title, unless this function is called again.
+//
+// Calling this function has no effect if no capture is currently running, and if it is called
+// multiple times only the last title will be used.
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureTitle)(const char *title);
+
+// Annotations API:
+//
+// These functions allow you to specify annotations either on a per-command level, or a per-object
+// level.
+//
+// Basic types of annotations are supported, as well as vector versions and references to API objects.
+//
+// The annotations are stored as keys, with the key being a dot-separated path allowing arbitrary
+// nesting and user organisation. The keys are sorted in human order so `foo.2.bar` will be displayed
+// before `foo.10.bar` to allow creation of arrays if desired.
+//
+// Deleting an annotation can be done by assigning an empty value to it.
+
+// the type of an annotation value, or Empty to delete an annotation
+typedef enum RENDERDOC_AnnotationType
+{
+  eRENDERDOC_Empty,
+  eRENDERDOC_Bool,
+  eRENDERDOC_Int32,
+  eRENDERDOC_UInt32,
+  eRENDERDOC_Int64,
+  eRENDERDOC_UInt64,
+  eRENDERDOC_Float,
+  eRENDERDOC_Double,
+  eRENDERDOC_String,
+  eRENDERDOC_APIObject,
+  eRENDERDOC_AnnotationMax = 0x7FFFFFFF,
+} RENDERDOC_AnnotationType;
+
+// a union with vector annotation value data
+typedef union RENDERDOC_AnnotationVectorValue
+{
+  bool boolean[4];
+  int32_t int32[4];
+  int64_t int64[4];
+  uint32_t uint32[4];
+  uint64_t uint64[4];
+  float float32[4];
+  double float64[4];
+} RENDERDOC_AnnotationVectorValue;
+
+// a union with scalar annotation value data
+typedef union RENDERDOC_AnnotationValue
+{
+  bool boolean;
+  int32_t int32;
+  int64_t int64;
+  uint32_t uint32;
+  uint64_t uint64;
+  float float32;
+  double float64;
+
+  RENDERDOC_AnnotationVectorValue vector;
+
+  const char *string;
+  void *apiObject;
+} RENDERDOC_AnnotationValue;
+
+// a struct for specifying a GL object, as we don't have pointers we can use so instead we specify a
+// pointer to this struct giving both the type and the name
+typedef struct RENDERDOC_GLResourceReference
+{
+  // this is the same GLenum identifier as passed to glObjectLabel
+  uint32_t identifier;
+  uint32_t name;
+} GLResourceReference;
+
+// simple C++ helpers to avoid the need for a temporary objects for value passing and GL object specification
+#ifdef __cplusplus
+struct RDGLObjectHelper
+{
+  RENDERDOC_GLResourceReference gl;
+
+  RDGLObjectHelper(uint32_t identifier, uint32_t name)
+  {
+    gl.identifier = identifier;
+    gl.name = name;
+  }
+
+  operator RENDERDOC_GLResourceReference *() { return &gl; }
+};
+
+struct RDAnnotationHelper
+{
+  RENDERDOC_AnnotationValue val;
+
+  RDAnnotationHelper(bool b) { val.boolean = b; }
+  RDAnnotationHelper(int32_t i) { val.int32 = i; }
+  RDAnnotationHelper(int64_t i) { val.int64 = i; }
+  RDAnnotationHelper(uint32_t i) { val.uint32 = i; }
+  RDAnnotationHelper(uint64_t i) { val.uint64 = i; }
+  RDAnnotationHelper(float f) { val.float32 = f; }
+  RDAnnotationHelper(double d) { val.float64 = d; }
+  RDAnnotationHelper(const char *s) { val.string = s; }
+
+  operator RENDERDOC_AnnotationValue *() { return &val; }
+};
+#endif
+
+// The device is specified in the same way as other API calls that take a RENDERDOC_DevicePointer
+// to specify the device.
+//
+// The object or queue/commandbuffer will depend on the graphics API in question.
+//
+// Return value:
+// 0 - The annotation was applied successfully.
+// 1 - The device is unknown/invalid
+// 2 - The device is valid but the annotation is not supported for API-specific reasons, such as an
+//     unrecognised or invalid object or queue/commandbuffer
+// 3 - The call is ill-formed or invalid e.g. empty is specified with a value pointer, or non-empty
+//     is specified with a NULL value pointer
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_SetObjectAnnotation)(RENDERDOC_DevicePointer device,
+                                                               void *object, const char *key,
+                                                               RENDERDOC_AnnotationType valueType,
+                                                               uint32_t valueVectorWidth,
+                                                               const RENDERDOC_AnnotationValue *value);
+
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_SetCommandAnnotation)(
+    RENDERDOC_DevicePointer device, void *queueOrCommandBuffer, const char *key,
+    RENDERDOC_AnnotationType valueType, uint32_t valueVectorWidth,
+    const RENDERDOC_AnnotationValue *value);
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// RenderDoc API versions
+//
+
+// RenderDoc uses semantic versioning (http://semver.org/).
+//
+// MAJOR version is incremented when incompatible API changes happen.
+// MINOR version is incremented when functionality is added in a backwards-compatible manner.
+// PATCH version is incremented when backwards-compatible bug fixes happen.
+//
+// Note that this means the API returned can be higher than the one you might have requested.
+// e.g. if you are running against a newer RenderDoc that supports 1.0.1, it will be returned
+// instead of 1.0.0. You can check this with the GetAPIVersion entry point
+typedef enum RENDERDOC_Version
+{
+  eRENDERDOC_API_Version_1_0_0 = 10000,    // RENDERDOC_API_1_0_0 = 1 00 00
+  eRENDERDOC_API_Version_1_0_1 = 10001,    // RENDERDOC_API_1_0_1 = 1 00 01
+  eRENDERDOC_API_Version_1_0_2 = 10002,    // RENDERDOC_API_1_0_2 = 1 00 02
+  eRENDERDOC_API_Version_1_1_0 = 10100,    // RENDERDOC_API_1_1_0 = 1 01 00
+  eRENDERDOC_API_Version_1_1_1 = 10101,    // RENDERDOC_API_1_1_1 = 1 01 01
+  eRENDERDOC_API_Version_1_1_2 = 10102,    // RENDERDOC_API_1_1_2 = 1 01 02
+  eRENDERDOC_API_Version_1_2_0 = 10200,    // RENDERDOC_API_1_2_0 = 1 02 00
+  eRENDERDOC_API_Version_1_3_0 = 10300,    // RENDERDOC_API_1_3_0 = 1 03 00
+  eRENDERDOC_API_Version_1_4_0 = 10400,    // RENDERDOC_API_1_4_0 = 1 04 00
+  eRENDERDOC_API_Version_1_4_1 = 10401,    // RENDERDOC_API_1_4_1 = 1 04 01
+  eRENDERDOC_API_Version_1_4_2 = 10402,    // RENDERDOC_API_1_4_2 = 1 04 02
+  eRENDERDOC_API_Version_1_5_0 = 10500,    // RENDERDOC_API_1_5_0 = 1 05 00
+  eRENDERDOC_API_Version_1_6_0 = 10600,    // RENDERDOC_API_1_6_0 = 1 06 00
+  eRENDERDOC_API_Version_1_7_0 = 10700,    // RENDERDOC_API_1_7_0 = 1 07 00
+} RENDERDOC_Version;
+
+// API version changelog:
+//
+// 1.0.0 - initial release
+// 1.0.1 - Bugfix: IsFrameCapturing() was returning false for captures that were triggered
+//         by keypress or TriggerCapture, instead of Start/EndFrameCapture.
+// 1.0.2 - Refactor: Renamed eRENDERDOC_Option_DebugDeviceMode to eRENDERDOC_Option_APIValidation
+// 1.1.0 - Add feature: TriggerMultiFrameCapture(). Backwards compatible with 1.0.x since the new
+//         function pointer is added to the end of the struct, the original layout is identical
+// 1.1.1 - Refactor: Renamed remote access to target control (to better disambiguate from remote
+//         replay/remote server concept in replay UI)
+// 1.1.2 - Refactor: Renamed "log file" in function names to just capture, to clarify that these
+//         are captures and not debug logging files. This is the first API version in the v1.0
+//         branch.
+// 1.2.0 - Added feature: SetCaptureFileComments() to add comments to a capture file that will be
+//         displayed in the UI program on load.
+// 1.3.0 - Added feature: New capture option eRENDERDOC_Option_AllowUnsupportedVendorExtensions
+//         which allows users to opt-in to allowing unsupported vendor extensions to function.
+//         Should be used at the user's own risk.
+//         Refactor: Renamed eRENDERDOC_Option_VerifyMapWrites to
+//         eRENDERDOC_Option_VerifyBufferAccess, which now also controls initialisation to
+//         0xdddddddd of uninitialised buffer contents.
+// 1.4.0 - Added feature: DiscardFrameCapture() to discard a frame capture in progress and stop
+//         capturing without saving anything to disk.
+// 1.4.1 - Refactor: Renamed Shutdown to RemoveHooks to better clarify what is happening
+// 1.4.2 - Refactor: Renamed 'draws' to 'actions' in callstack capture option.
+// 1.5.0 - Added feature: ShowReplayUI() to request that the replay UI show itself if connected
+// 1.6.0 - Added feature: SetCaptureTitle() which can be used to set a title for a
+//         capture made with StartFrameCapture() or EndFrameCapture()
+// 1.7.0 - Added feature: SetObjectAnnotation() / SetCommandAnnotation() for adding rich
+//         annotations to objects and command streams
+
+typedef struct RENDERDOC_API_1_7_0
+{
+  pRENDERDOC_GetAPIVersion GetAPIVersion;
+
+  pRENDERDOC_SetCaptureOptionU32 SetCaptureOptionU32;
+  pRENDERDOC_SetCaptureOptionF32 SetCaptureOptionF32;
+
+  pRENDERDOC_GetCaptureOptionU32 GetCaptureOptionU32;
+  pRENDERDOC_GetCaptureOptionF32 GetCaptureOptionF32;
+
+  pRENDERDOC_SetFocusToggleKeys SetFocusToggleKeys;
+  pRENDERDOC_SetCaptureKeys SetCaptureKeys;
+
+  pRENDERDOC_GetOverlayBits GetOverlayBits;
+  pRENDERDOC_MaskOverlayBits MaskOverlayBits;
+
+  // Shutdown was renamed to RemoveHooks in 1.4.1.
+  // These unions allow old code to continue compiling without changes
+  union
+  {
+    pRENDERDOC_Shutdown Shutdown;
+    pRENDERDOC_RemoveHooks RemoveHooks;
+  };
+  pRENDERDOC_UnloadCrashHandler UnloadCrashHandler;
+
+  // Get/SetLogFilePathTemplate was renamed to Get/SetCaptureFilePathTemplate in 1.1.2.
+  // These unions allow old code to continue compiling without changes
+  union
+  {
+    // deprecated name
+    pRENDERDOC_SetLogFilePathTemplate SetLogFilePathTemplate;
+    // current name
+    pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate;
+  };
+  union
+  {
+    // deprecated name
+    pRENDERDOC_GetLogFilePathTemplate GetLogFilePathTemplate;
+    // current name
+    pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate;
+  };
+
+  pRENDERDOC_GetNumCaptures GetNumCaptures;
+  pRENDERDOC_GetCapture GetCapture;
+
+  pRENDERDOC_TriggerCapture TriggerCapture;
+
+  // IsRemoteAccessConnected was renamed to IsTargetControlConnected in 1.1.1.
+  // This union allows old code to continue compiling without changes
+  union
+  {
+    // deprecated name
+    pRENDERDOC_IsRemoteAccessConnected IsRemoteAccessConnected;
+    // current name
+    pRENDERDOC_IsTargetControlConnected IsTargetControlConnected;
+  };
+  pRENDERDOC_LaunchReplayUI LaunchReplayUI;
+
+  pRENDERDOC_SetActiveWindow SetActiveWindow;
+
+  pRENDERDOC_StartFrameCapture StartFrameCapture;
+  pRENDERDOC_IsFrameCapturing IsFrameCapturing;
+  pRENDERDOC_EndFrameCapture EndFrameCapture;
+
+  // new function in 1.1.0
+  pRENDERDOC_TriggerMultiFrameCapture TriggerMultiFrameCapture;
+
+  // new function in 1.2.0
+  pRENDERDOC_SetCaptureFileComments SetCaptureFileComments;
+
+  // new function in 1.4.0
+  pRENDERDOC_DiscardFrameCapture DiscardFrameCapture;
+
+  // new function in 1.5.0
+  pRENDERDOC_ShowReplayUI ShowReplayUI;
+
+  // new function in 1.6.0
+  pRENDERDOC_SetCaptureTitle SetCaptureTitle;
+
+  // new functions in 1.7.0
+  pRENDERDOC_SetObjectAnnotation SetObjectAnnotation;
+  pRENDERDOC_SetCommandAnnotation SetCommandAnnotation;
+} RENDERDOC_API_1_7_0;
+
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_1;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_2;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_1;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_2;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_2_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_3_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_1;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_2;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_5_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_6_0;
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// RenderDoc API entry point
+//
+// This entry point can be obtained via GetProcAddress/dlsym if RenderDoc is available.
+//
+// The name is the same as the typedef - "RENDERDOC_GetAPI"
+//
+// This function is not thread safe, and should not be called on multiple threads at once.
+// Ideally, call this once as early as possible in your application's startup, before doing
+// any API work, since some configuration functionality etc has to be done also before
+// initialising any APIs.
+//
+// Parameters:
+//   version is a single value from the RENDERDOC_Version above.
+//
+//   outAPIPointers will be filled out with a pointer to the corresponding struct of function
+//   pointers.
+//
+// Returns:
+//   1 - if the outAPIPointers has been filled with a pointer to the API struct requested
+//   0 - if the requested version is not supported or the arguments are invalid.
+//
+typedef int(RENDERDOC_CC *pRENDERDOC_GetAPI)(RENDERDOC_Version version, void **outAPIPointers);
+
+#ifdef __cplusplus
+}    // extern "C"
+#endif

CMakeLists.txt

@@ -203,14 +203,21 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
 	"${KYTY_SOURCE_DIR}/graphics/shader/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.h"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.h"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.h"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/cfg/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/cfg/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/decode/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/decode/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/translate/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/translate/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/backend/spirv/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/backend/spirv/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.inc"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/presentation/*.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/presentation/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/presentation/window/*.cpp"
@@ -247,7 +254,7 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 	add_custom_command(
 		OUTPUT "${shader_header}"
 		COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}"
-		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
 		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
@@ -256,6 +263,22 @@ foreach(shader_name IN LISTS gpu_tiler_shader_names)
 	)
 	list(APPEND gpu_tiler_shader_headers "${shader_header}")
 endforeach()
+
+set(fault_buffer_shader_source "${gpu_tiler_shader_dir}/fault_buffer_process.comp")
+set(fault_buffer_shader_spv "${gpu_tiler_generated_dir}/fault_buffer_process.spv")
+set(fault_buffer_shader_header "${gpu_tiler_generated_dir}/fault_buffer_process_spv.h")
+add_custom_command(
+	OUTPUT "${fault_buffer_shader_header}"
+	COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_tiler_generated_dir}"
+	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
+	        -DCACHING_PAGEBITS=14 -DMAX_PAGE_FAULTS=1024
+	        -o "${fault_buffer_shader_spv}" "${fault_buffer_shader_source}"
+	COMMAND ${CMAKE_COMMAND} -DINPUT=${fault_buffer_shader_spv} -DOUTPUT=${fault_buffer_shader_header}
+	        -DSYMBOL=FAULT_BUFFER_PROCESS_SPV -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
+	DEPENDS "${fault_buffer_shader_source}"
+	VERBATIM
+)
+list(APPEND kyty_emulator_src "${fault_buffer_shader_header}")
 list(APPEND kyty_emulator_src ${gpu_tiler_shader_headers})
 
 set(gpu_blit_generated_dir "${PROJECT_BINARY_DIR}/gpu_blit_shaders")
@@ -271,7 +294,7 @@ foreach(shader_source IN LISTS gpu_blit_shader_sources)
 	add_custom_command(
 		OUTPUT "${shader_header}"
 		COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_blit_generated_dir}"
-		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+		COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 		        "-I${gpu_tiler_shader_dir}" -o "${shader_spv}" "${shader_source}"
 		COMMAND ${CMAKE_COMMAND} -DINPUT=${shader_spv} -DOUTPUT=${shader_header}
 		        -DSYMBOL=${shader_symbol} -P "${KYTY_SOURCE_DIR}/embed_spirv.cmake"
@@ -322,6 +345,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/xxhash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
+	${KYTY_THIRD_PARTY_DIR}/renderdoc
 	${KYTY_THIRD_PARTY_DIR}/stb
 )
 
@@ -367,9 +391,8 @@ add_kyty_full_emulator_test(shader_cfg_tests "${KYTY_TESTS_DIR}/shaderCfgTests.c
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ScalarProvenanceTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ReadLaneElimination.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 )
 target_link_libraries(scalar_provenance_tests fmt::fmt)
 target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
@@ -413,9 +436,8 @@ add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL
 	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/shaderStageRuntime.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 )
 target_link_libraries(shader_stage_runtime_tests fmt::fmt)
 target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers})
@@ -424,13 +446,12 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtPatcher.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceTracking.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/BindingLayout.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/shaderPixelParameter.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/BindingLayout.cpp"
 )
 target_link_libraries(resource_tracking_tests fmt::fmt)
 target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
@@ -451,6 +472,9 @@ add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
 target_link_libraries(event_queue_lifetime_tests common fmt::fmt)
 target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
 
+add_kyty_full_emulator_test(kernel_file_system_tests
+	"${KYTY_TESTS_DIR}/KernelFileSystemTests.cpp")
+
 add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
 	"${KYTY_SOURCE_DIR}/kernel/syncOnAddress.cpp"
@@ -476,7 +500,7 @@ set(gpu_test_ms_depth_header
 add_custom_command(
 	OUTPUT "${gpu_test_ms_depth_header}"
 	COMMAND ${CMAKE_COMMAND} -E make_directory "${gpu_test_generated_dir}"
-	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.0 -Os
+	COMMAND "${KYTY_GLSLANG_VALIDATOR}" -V --target-env vulkan1.3 -Os
 	        -o "${gpu_test_ms_depth_spv}" "${gpu_test_ms_depth_source}"
 	COMMAND ${CMAKE_COMMAND} -DINPUT=${gpu_test_ms_depth_spv}
 	        -DOUTPUT=${gpu_test_ms_depth_header} -DSYMBOL=GPU_TEST_MS_DEPTH_SPV
@@ -511,9 +535,12 @@ if(BUILD_TESTING)
 	add_test(NAME shader_stage_runtime COMMAND $<TARGET_FILE:shader_stage_runtime_tests>)
 	add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
+	add_test(NAME kernel_file_system COMMAND $<TARGET_FILE:kernel_file_system_tests>)
 	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
+	add_test(NAME shader_recompiler_alignbyte
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME guest_red_zone_patcher
@@ -562,9 +589,11 @@ if(BUILD_TESTING)
 		shader_stage_runtime_tests
 		resource_tracking_tests
 		event_queue_lifetime_tests
+		kernel_file_system_tests
 		sync_on_address_tests
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
+		audio_out2_port_tests
 	)
 endif()
 

src/common/emulatorConfig.cpp

@@ -93,6 +93,10 @@ bool SpirvDebugPrintfEnabled() {
 	return g_config->spirv_debug_printf_enabled;
 }
 
+bool GpuAssistedValidationEnabled() {
+	return g_config->gpu_assisted_validation_enabled && g_config->vulkan_validation_enabled;
+}
+
 bool RenderDocEnabled() {
 	return g_config->renderdoc_enabled;
 }
@@ -101,6 +105,10 @@ bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }
 
+bool PlayGoHackEnabled() {
+	return g_config->playgo_hack_enabled;
+}
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled() {
 	return g_config->red_zone_protection_enabled;

src/common/emulatorConfig.h

@@ -49,8 +49,10 @@ struct ConfigOptions {
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   spirv_debug_printf_enabled  = false;
+	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
+	bool                   playgo_hack_enabled         = false;
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	bool red_zone_protection_enabled = false;
 #endif
@@ -83,8 +85,11 @@ ProfilerDirection GetProfilerDirection();
 
 bool SpirvDebugPrintfEnabled();
 
+bool GpuAssistedValidationEnabled();
+
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
+bool PlayGoHackEnabled();
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled();
 #endif

src/common/hostException.cpp

@@ -2,7 +2,6 @@
 
 #include <atomic>
 #include <cstdio>
-#include <cstdlib>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
@@ -28,54 +27,14 @@ namespace Common::HostException {
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
-static thread_local bool    g_in_exception_filter = false;
 
 static_assert(decltype(g_handler)::is_always_lock_free);
 static_assert(decltype(g_install_state)::is_always_lock_free);
-
-[[noreturn]] static void FailFast(const char* reason) noexcept {
-	std::fputs("HostException fail-fast: ", stderr);
-	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
-	std::fputc('\n', stderr);
-	std::fflush(stderr);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
-#endif
-	std::_Exit(321);
-}
-
-class FilterScope final {
-public:
-	FilterScope() noexcept {
-		if (g_in_exception_filter) {
-			FailFast("nested exception while resolving a host fault");
-		}
-		g_in_exception_filter = true;
-	}
-
-	~FilterScope() { g_in_exception_filter = false; }
-
-	KYTY_CLASS_NO_COPY(FilterScope);
-};
-
-static Handler LoadInstalledHandler() noexcept {
-	if (g_install_state.load(std::memory_order_acquire) == 0) {
-		FailFast("host exception handler is not installed");
-	}
-
-	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
-	return handler;
-}
 #endif
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
-static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
-	FilterScope filter_scope;
-
+static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) noexcept {
 	auto* exception_record = exception->ExceptionRecord;
 
 	if (exception_record->ExceptionCode == DBG_PRINTEXCEPTION_C ||
@@ -105,12 +64,6 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	} else if (exception_record->ExceptionCode == EXCEPTION_ILLEGAL_INSTRUCTION) {
 		info.type = ExceptionType::IllegalInstruction;
 	} else {
-		printf("Unhandled win exception: code=0x%08" PRIx32 ", addr=0x%016" PRIx64
-		       ", rip=0x%016" PRIx64 ", rsp=0x%016" PRIx64 ", rbp=0x%016" PRIx64 "\n",
-		       static_cast<uint32_t>(exception_record->ExceptionCode),
-		       reinterpret_cast<uint64_t>(exception_record->ExceptionAddress),
-		       exception->ContextRecord->Rip, exception->ContextRecord->Rsp,
-		       exception->ContextRecord->Rbp);
 		return EXCEPTION_CONTINUE_SEARCH;
 	}
 
@@ -131,28 +84,21 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	info.r14 = exception->ContextRecord->R14;
 	info.r15 = exception->ContextRecord->R15;
 
-	const auto handler = LoadInstalledHandler();
-
-	return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler != nullptr && handler(info)) {
+		return EXCEPTION_CONTINUE_EXECUTION;
+	}
+	return EXCEPTION_CONTINUE_SEARCH;
 }
 
 #elif defined(__APPLE__)
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
-static thread_local bool    g_in_exception_filter = false;
 
 static_assert(decltype(g_handler)::is_always_lock_free);
 static_assert(decltype(g_install_state)::is_always_lock_free);
 
-[[noreturn]] static void FailFast(const char* reason) noexcept {
-	std::fputs("HostException fail-fast: ", stderr);
-	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
-	std::fputc('\n', stderr);
-	std::fflush(stderr);
-	std::_Exit(321);
-}
-
 // Translate the x86-64 page-fault error code (mcontext __es.__err) into an access type.
 // bit 1 (0x2) = write, bit 4 (0x10) = instruction fetch, otherwise a read.
 static AccessViolationType DecodeAccess(uint64_t err) {
@@ -170,11 +116,6 @@ static AccessViolationType DecodeAccess(uint64_t err) {
 // re-executing the faulting instruction against the now-fixed protection. An unresolved
 // fault restores the default disposition so the retry terminates the process.
 static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
-	if (g_in_exception_filter) {
-		FailFast("nested exception while resolving a host fault");
-	}
-	g_in_exception_filter = true;
-
 	auto*       uc = static_cast<ucontext_t*>(uctx);
 	const auto* mc = uc->uc_mcontext;
 	const auto& ss = mc->__ss;
@@ -210,14 +151,7 @@ static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
 	info.r15 = ss.__r15;
 
 	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
-
-	const bool resolved   = handler(info);
-	g_in_exception_filter = false;
-
-	if (resolved) {
+	if (handler != nullptr && handler(info)) {
 		return; // retry the faulting instruction against the fixed mapping
 	}
 
@@ -244,8 +178,6 @@ static void ChainToDefault(int signal_number) noexcept {
 }
 
 static void SignalHandler(int signal_number, siginfo_t* signal_info, void* native_context) {
-	FilterScope filter_scope;
-
 	auto* context = static_cast<ucontext_t*>(native_context);
 	auto* gregs   = context->uc_mcontext.gregs;
 
@@ -289,9 +221,8 @@ static void SignalHandler(int signal_number, siginfo_t* signal_info, void* nativ
 	info.r14 = static_cast<uint64_t>(gregs[REG_R14]);
 	info.r15 = static_cast<uint64_t>(gregs[REG_R15]);
 
-	const auto handler = LoadInstalledHandler();
-
-	if (handler(info)) {
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler != nullptr && handler(info)) {
 		return;
 	}
 
@@ -313,7 +244,7 @@ bool InstallHandler(Handler handler) {
 	g_handler.store(handler, std::memory_order_release);
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) {
+	if (AddVectoredExceptionHandler(0, ExceptionFilter) == nullptr) {
 		g_handler.store(nullptr, std::memory_order_release);
 		g_install_state.store(0, std::memory_order_release);
 		printf("AddVectoredExceptionHandler() failed\n");

src/common/slotVector.h

@@ -0,0 +1,132 @@
+#ifndef EMULATOR_SRC_COMMON_SLOTVECTOR_H_
+#define EMULATOR_SRC_COMMON_SLOTVECTOR_H_
+
+#include "common/assert.h"
+#include "common/abi.h"
+
+#include <compare>
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <limits>
+#include <optional>
+#include <utility>
+#include <vector>
+
+namespace Common {
+
+struct SlotId {
+	static constexpr uint32_t INVALID_INDEX = std::numeric_limits<uint32_t>::max();
+
+	constexpr SlotId() noexcept = default;
+	constexpr SlotId(uint32_t value) noexcept: index(value), generation(1) {}
+	constexpr SlotId(uint32_t value, uint32_t slot_generation) noexcept
+	    : index(value), generation(slot_generation) {}
+
+	[[nodiscard]] constexpr explicit operator bool() const noexcept { return index != INVALID_INDEX; }
+	constexpr auto operator<=>(const SlotId&) const noexcept = default;
+
+	uint32_t index = INVALID_INDEX;
+	uint32_t generation = 0;
+};
+
+// Stable-address slot storage for cache resources that are intentionally non-movable.
+template <typename T>
+class SlotVector {
+public:
+	SlotVector() = default;
+	KYTY_CLASS_NO_COPY(SlotVector);
+
+	[[nodiscard]] T& operator[](SlotId id) noexcept {
+		EXIT_IF(!is_allocated(id));
+		return *m_values[id.index].value;
+	}
+
+	[[nodiscard]] const T& operator[](SlotId id) const noexcept {
+		EXIT_IF(!is_allocated(id));
+		return *m_values[id.index].value;
+	}
+
+	[[nodiscard]] T* try_get(SlotId id) noexcept {
+		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+	}
+
+	[[nodiscard]] const T* try_get(SlotId id) const noexcept {
+		return is_allocated(id) ? &*m_values[id.index].value : nullptr;
+	}
+
+	[[nodiscard]] bool is_allocated(SlotId id) const noexcept {
+		return id && id.index < m_values.size() &&
+		       m_values[id.index].generation == id.generation && m_values[id.index].value.has_value();
+	}
+
+	template <typename... Args>
+	[[nodiscard]] SlotId insert(Args&&... args) {
+		uint32_t index = 0;
+		if (m_free_list.empty()) {
+			index = static_cast<uint32_t>(m_values.size());
+			m_values.emplace_back();
+			m_values.back().value.emplace(std::forward<Args>(args)...);
+		} else {
+			index = m_free_list.back();
+			m_free_list.pop_back();
+			EXIT_IF(m_values[index].value.has_value());
+			m_values[index].value.emplace(std::forward<Args>(args)...);
+		}
+		++m_size;
+		return SlotId {index, m_values[index].generation};
+	}
+
+	void erase(SlotId id) noexcept {
+		EXIT_IF(!is_allocated(id));
+		auto& slot = m_values[id.index];
+		slot.value.reset();
+		if (++slot.generation == 0) {
+			slot.generation = 1;
+		}
+		m_free_list.push_back(id.index);
+		--m_size;
+	}
+
+	[[nodiscard]] size_t size() const noexcept { return m_size; }
+	[[nodiscard]] size_t capacity() const noexcept { return m_values.size(); }
+
+	template <typename F>
+	void ForEach(F&& fn) {
+		for (uint32_t index = 0; index < m_values.size(); ++index) {
+			if (m_values[index].value) {
+				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+			}
+		}
+	}
+
+	template <typename F>
+	void ForEach(F&& fn) const {
+		for (uint32_t index = 0; index < m_values.size(); ++index) {
+			if (m_values[index].value) {
+				fn(SlotId {index, m_values[index].generation}, *m_values[index].value);
+			}
+		}
+	}
+
+private:
+	struct Slot {
+		std::optional<T> value;
+		uint32_t         generation = 1;
+	};
+
+	std::deque<Slot>             m_values;
+	std::vector<uint32_t>        m_free_list;
+	size_t                       m_size = 0;
+};
+
+} // namespace Common
+
+template <>
+struct std::hash<Common::SlotId> {
+	[[nodiscard]] size_t operator()(Common::SlotId id) const noexcept {
+		return std::hash<uint64_t> {}((static_cast<uint64_t>(id.generation) << 32u) | id.index);
+	}
+};
+
+#endif // EMULATOR_SRC_COMMON_SLOTVECTOR_H_

src/emulator.cpp

@@ -15,6 +15,7 @@
 #include "kernel/fileSystem.h"
 #include "kernel/memory.h"
 #include "kernel/pthread.h"
+#include "kytyGitVersion.h"
 #include "libs/agc.h"
 #include "libs/audio.h"
 #include "libs/controller.h"
@@ -26,13 +27,31 @@
 
 #include <cstdlib>
 #include <filesystem>
+#include <thread>
 
 namespace Emulator {
 
 static void PrintSystemInfo() {
-	Common::SystemInfo info = Common::GetSystemInfo();
+	const Common::SystemInfo info = Common::GetSystemInfo();
 
-	LOGF("ProcessorName = %s\n", info.ProcessorName.c_str());
+#if defined(__APPLE__)
+	static constexpr auto platform_name = "macOS";
+#elif KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
+	static constexpr auto platform_name = "Windows";
+#elif KYTY_PLATFORM == KYTY_PLATFORM_LINUX
+	static constexpr auto platform_name = "Linux";
+#else
+	static constexpr auto platform_name = "Unknown";
+#endif
+
+	LOGF("Build\n"
+	     "  version: %s\n\n"
+	     "Host\n"
+	     "  os:      %s\n"
+	     "  cpu:     %s\n"
+	     "  threads: %u\n\n",
+	     KYTY_BUILD_LABEL, platform_name, info.ProcessorName.c_str(),
+	     std::thread::hardware_concurrency());
 }
 
 static void KytyClose() {

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -7,6 +7,7 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 
 #include <cstdint>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -30,10 +31,9 @@ private:
 	friend class CommandProcessor;
 
 	struct BufferCursor {
-		uint32_t* next_packet         = nullptr;
-		uint32_t  remaining_dw        = 0;
-		uint32_t  total_dw            = 0;
-		uint32_t  deferred_advance_dw = 0;
+		std::span<const uint32_t> commands;
+		uint32_t                  offset_dw           = 0;
+		uint32_t                  deferred_advance_dw = 0;
 	};
 
 	std::vector<BufferCursor> m_buffer_stack;
@@ -50,7 +50,8 @@ public:
 		int64_t flip_arg  = 0;
 	};
 
-	explicit CommandProcessor(RenderContext& renderer): m_renderer(renderer) {}
+	CommandProcessor(RenderContext& renderer, int interrupt_event_id)
+	    : m_renderer(renderer), m_interrupt_event_id(interrupt_event_id) {}
 	~CommandProcessor() = default;
 
 	KYTY_CLASS_NO_COPY(CommandProcessor);
@@ -103,7 +104,7 @@ public:
 	                    uint32_t mode);
 	void DispatchIndirect(uint32_t data_offset, uint32_t mode);
 	void WaitFlipDone(uint32_t video_out_handle, uint32_t display_buffer_index);
-	void TriggerEvent(uint32_t event_type, uint32_t event_index);
+	void TriggerEvent(uint32_t event_type, uint32_t event_index, uint64_t event_address = 0);
 
 	void SetUserDataMarker(HW::UserSgprType type) { m_user_data_marker = type; }
 	[[nodiscard]] HW::UserSgprType GetUserDataMarker() const { return m_user_data_marker; }
@@ -112,6 +113,7 @@ public:
 	void SetCeComplete(bool complete) { m_ce_complete = complete; }
 	void WaitCe();
 	void WaitDeDiff(uint32_t diff);
+	void WaitForRewind(bool valid);
 	void IncrementDe();
 	void IncrementCe();
 
@@ -130,13 +132,14 @@ public:
 	                    const volatile void* address, uint32_t count_in_dwords);
 	[[nodiscard]] bool ShouldSkipPredicatedPackets() const { return m_predicate_skip; }
 
-	Pm4ProcessResult Process(Pm4Execution& execution, uint32_t* buffer, uint32_t size_dw);
-	void             ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw);
+	Pm4ProcessResult Process(Pm4Execution& execution, std::span<const uint32_t> commands);
+	void             ProcessIndirectBuffer(std::span<const uint32_t> commands);
 
 	void SetFlip(const FlipInfo& flip) { m_flip = flip; }
 
 	[[nodiscard]] uint64_t GetSubmitId() const { return m_submit_id; }
 	void                   SetSubmitId(uint64_t submit_id) { m_submit_id = submit_id; }
+	[[nodiscard]] bool     IsAsyncComputeQueue() const { return m_interrupt_event_id >= 0x20; }
 
 private:
 	template <typename T>
@@ -150,10 +153,10 @@ private:
 	                          uint32_t render_target_slice_offset, uint32_t first_vertex,
 	                          uint32_t first_instance);
 
-	CommandScheduler&    GetScheduler() const { return m_renderer.GetCommandScheduler(); }
-	RenderCommandBuffer& CurrentBuffer() { return GetScheduler().Current(); }
-	void                 CheckBuffer() const { GetScheduler().CheckActive(); }
-	GpuResourceManager&  GetGpuResources() const { return m_renderer.GetGpuResources(); }
+	CommandScheduler&   GetScheduler() const { return m_renderer.GetCommandScheduler(); }
+	CommandBuffer&      CurrentBuffer() { return GetScheduler().Current(); }
+	void                CheckBuffer() const { GetScheduler().CheckActive(); }
+	GpuResourceManager& GetGpuResources() const { return m_renderer.GetGpuResources(); }
 
 	RenderContext&   m_renderer;
 	HW::Context      m_ctx;
@@ -176,9 +179,11 @@ private:
 
 	uint32_t m_const_ram[0x3000] = {0};
 
-	FlipInfo m_flip;
-	uint64_t m_submit_id      = 0;
-	bool     m_predicate_skip = false;
+	FlipInfo  m_flip;
+	const int m_interrupt_event_id;
+	uint64_t  m_submit_id                   = 0;
+	uint64_t  m_synthetic_occlusion_counter = 0;
+	bool      m_predicate_skip              = false;
 };
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -35,7 +35,6 @@ constexpr auto MakeContextDispatchTable() {
 	}
 	g_hw_ctx_func[Pm4::CB_TARGET_MASK]                 = HwCtxSetRenderTargetMask;
 	g_hw_ctx_func[Pm4::PA_SC_GENERIC_SCISSOR_TL]       = HwCtxSetGenericScissor;
-	g_hw_ctx_func[Pm4::CB_BLEND_RED]                   = HwCtxSetBlendColor;
 	g_hw_ctx_func[Pm4::CB_DCC_CONTROL]                 = HwCtxSetCbDccControl;
 	g_hw_ctx_func[Pm4::DB_STENCIL_CONTROL]             = HwCtxSetStencilControl;
 	g_hw_ctx_func[Pm4::DB_STENCILREFMASK]              = HwCtxSetStencilMask;
@@ -82,7 +81,6 @@ constexpr auto MakeContextDispatchTable() {
 	g_hw_ctx_func[Pm4::VGT_PRIMITIVEID_RESET]                 = HwCtxSetPrimitiveIdReset;
 	g_hw_ctx_func[Pm4::PA_CL_OBJPRIM_ID_CNTL]                 = HwCtxSetObjprimIdControl;
 	g_hw_ctx_func[Pm4::VGT_SHADER_STAGES_EN]                  = HwCtxSetShaderStages;
-	g_hw_ctx_func[Pm4::PA_CL_GB_VERT_CLIP_ADJ]                = HwCtxSetGuardBands;
 
 	for (uint32_t slot = 0; slot < 8; slot++) {
 		g_hw_ctx_func[Pm4::CB_COLOR0_BASE + slot * 15] = HwCtxSetRenderTarget;
@@ -217,6 +215,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_COND_EXEC]                 = CpOpCondExec;
 	g_cp_op_func[Pm4::IT_SET_PREDICATION]           = CpOpSetPredication;
 	g_cp_op_func[Pm4::IT_WRITE_DATA]                = CpOpWriteData;
+	g_cp_op_func[Pm4::IT_WAIT_REG_MEM]              = CpOpWaitRegMem32;
 	g_cp_op_func[Pm4::IT_INDIRECT_BUFFER]           = CpOpIndirectBuffer;
 	g_cp_op_func[Pm4::IT_COPY_DATA]                 = CpOpCopyData;
 	g_cp_op_func[Pm4::IT_EVENT_WRITE]               = CpOpEventWrite;
@@ -224,6 +223,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_EVENT_WRITE_EOS]           = CpOpEventWriteEos;
 	g_cp_op_func[Pm4::IT_DMA_DATA]                  = CpOpDmaData;
 	g_cp_op_func[Pm4::IT_ACQUIRE_MEM]               = CpOpAcquireMem;
+	g_cp_op_func[Pm4::IT_REWIND]                    = CpOpRewind;
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG]           = CpOpSetContextReg;
 	g_cp_op_func[Pm4::IT_SET_SH_REG]                = CpOpSetShaderReg;
 	g_cp_op_func[Pm4::IT_SET_UCONFIG_REG]           = CpOpSetUconfigReg;
@@ -238,6 +238,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_WAIT_ON_CE_COUNTER]        = CpOpWaitOnCeCounter;
 	g_cp_op_func[Pm4::IT_WAIT_ON_DE_COUNTER_DIFF]   = CpOpWaitOnDeCounterDiff;
 	g_cp_op_func[Pm4::IT_GET_LOD_STATS]             = CpOpGetLodStats;
+	g_cp_op_func[Pm4::IT_WAIT_REG_MEM_64]           = CpOpWaitRegMem64;
 
 	return g_cp_op_func;
 }
@@ -246,12 +247,10 @@ constexpr auto MakeCustomOpcodeDispatchTable() {
 	std::array<cp_op_parser_func_t, Pm4::R_NUM> g_cp_op_custom_func {};
 
 	g_cp_op_custom_func[Pm4::R_DISPATCH_RESET] = CpOpDispatchReset;
-	g_cp_op_custom_func[Pm4::R_WAIT_MEM_32]    = CpOpWaitRegMem32;
 	g_cp_op_custom_func[Pm4::R_WAIT_FLIP_DONE] = CpOpWaitFlipDone;
 	g_cp_op_custom_func[Pm4::R_PUSH_MARKER]    = CpOpPushMarker;
 	g_cp_op_custom_func[Pm4::R_POP_MARKER]     = CpOpPopMarker;
 	g_cp_op_custom_func[Pm4::R_ACQUIRE_MEM]    = CpOpAcquireMem;
-	g_cp_op_custom_func[Pm4::R_WAIT_MEM_64]    = CpOpWaitRegMem64;
 	g_cp_op_custom_func[Pm4::R_FLIP]           = CpOpFlip;
 	g_cp_op_custom_func[Pm4::R_RELEASE_MEM]    = CpOpReleaseMem;
 	g_cp_op_custom_func[Pm4::R_CONTEXT_STATE]  = CpOpContextState;

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -56,7 +56,6 @@ uint32_t HwCtxSetWindowScissor(CommandProcessor&, uint32_t, uint32_t, const uint
 uint32_t HwCtxSetClipRect(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetRenderTargetMask(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetGenericScissor(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
-uint32_t HwCtxSetBlendColor(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetCbDccControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetStencilControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetStencilMask(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
@@ -90,7 +89,6 @@ uint32_t HwCtxSetDrawPayloadControl(CommandProcessor&, uint32_t, uint32_t, const
 uint32_t HwCtxSetPrimitiveIdReset(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetObjprimIdControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetShaderStages(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
-uint32_t HwCtxSetGuardBands(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetRenderTarget(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetColorInfo(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
 uint32_t HwCtxSetBlendControl(CommandProcessor&, uint32_t, uint32_t, const uint32_t*, uint32_t);
@@ -133,6 +131,7 @@ uint32_t CpOpDrawIndirectMulti(CommandProcessor&, uint32_t, const uint32_t*, uin
 uint32_t CpOpDrawIndex(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDrawIndexOffset(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpPfpSyncMe(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpRewind(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpIndexType(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpNumInstances(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDrawIndexAuto(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -71,19 +71,6 @@ constexpr uint32_t NormalizeRegisterOffset(uint32_t raw_offset) {
 	return raw_offset & ~RegisterSelectorMask;
 }
 
-// Indirect Cx descriptors retain their selector. Selector 1 offsets 0..31 address the
-// SPI_PS_INPUT_CNTL register bank; ordinary context-register offsets remain unchanged.
-constexpr uint32_t DecodeIndirectCxRegisterOffset(uint32_t raw_offset) {
-	const auto offset = NormalizeRegisterOffset(raw_offset);
-	return (raw_offset & RegisterSelectorMask) == Pm4::CX_PS_SHADER_USAGE_BASE && offset < 32u
-	           ? Pm4::SPI_PS_INPUT_CNTL_0 + offset
-	           : offset;
-}
-
-static_assert(DecodeIndirectCxRegisterOffset(Pm4::CX_PS_SHADER_USAGE_BASE + 2u) ==
-              Pm4::SPI_PS_INPUT_CNTL_0 + 2u);
-static_assert(DecodeIndirectCxRegisterOffset(Pm4::DB_Z_INFO) == Pm4::DB_Z_INFO);
-
 bool ReleaseMemGcrNeedsBarrier(uint32_t eop_event_type, uint32_t gcr_cntl) {
 	return eop_event_type != 0x28u ||
 	       (gcr_cntl & (GcrGl2MetadataInvalidate | GcrGl0VectorInvalidate | GcrGl1Invalidate |
@@ -530,22 +517,6 @@ KYTY_HW_CTX_PARSER(HwCtxSetAaMask) {
 	return num_values;
 }
 
-KYTY_HW_CTX_PARSER(HwCtxSetBlendColor) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0046900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::CB_BLEND_RED);
-
-	HW::BlendColor r;
-
-	r.red   = *reinterpret_cast<const float*>(&buffer[0]);
-	r.green = *reinterpret_cast<const float*>(&buffer[1]);
-	r.blue  = *reinterpret_cast<const float*>(&buffer[2]);
-	r.alpha = *reinterpret_cast<const float*>(&buffer[3]);
-
-	cp.GetCtx().SetBlendColor(r);
-
-	return 4;
-}
-
 KYTY_HW_CTX_PARSER(HwCtxSetBlendControl) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900);
 
@@ -859,20 +830,6 @@ KYTY_HW_CTX_PARSER(HwCtxSetClipRect) {
 	return value_count;
 }
 
-KYTY_HW_CTX_PARSER(HwCtxSetGuardBands) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0046900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_CL_GB_VERT_CLIP_ADJ);
-
-	auto vert_clip    = *reinterpret_cast<const float*>(&buffer[0]); // PA_CL_GB_VERT_CLIP_ADJ
-	auto vert_discard = *reinterpret_cast<const float*>(&buffer[1]); // PA_CL_GB_VERT_DISC_ADJ
-	auto horz_clip    = *reinterpret_cast<const float*>(&buffer[2]); // PA_CL_GB_HORZ_CLIP_ADJ
-	auto horz_discard = *reinterpret_cast<const float*>(&buffer[3]); // PA_CL_GB_HORZ_DISC_ADJ
-
-	cp.GetCtx().SetGuardBands(horz_clip, vert_clip, horz_discard, vert_discard);
-
-	return 4;
-}
-
 KYTY_HW_CTX_PARSER(HwCtxSetHardwareScreenOffset) {
 	EXIT_NOT_IMPLEMENTED(cmd_id != 0xC0016900);
 	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::PA_SU_HARDWARE_SCREEN_OFFSET);
@@ -1248,23 +1205,16 @@ KYTY_HW_CTX_PARSER(HwCtxSetStencilInfo) {
 }
 
 KYTY_HW_CTX_PARSER(HwCtxSetStencilMask) {
-	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0026900);
-	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::DB_STENCILREFMASK);
+	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
+	EXIT_NOT_IMPLEMENTED(num_values == 0);
+	EXIT_NOT_IMPLEMENTED(cmd_offset < Pm4::DB_STENCILREFMASK);
+	EXIT_NOT_IMPLEMENTED(cmd_offset + num_values - 1u > Pm4::DB_STENCILREFMASK_BF);
 
-	HW::StencilMask r;
+	for (uint32_t i = 0; i < num_values; i++) {
+		g_hw_ctx_indirect_func[(cmd_offset + i) & (Pm4::CX_NUM - 1)](cp, cmd_offset + i, buffer[i]);
+	}
 
-	r.stencil_testval      = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILTESTVAL);
-	r.stencil_mask         = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILMASK);
-	r.stencil_writemask    = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILWRITEMASK);
-	r.stencil_opval        = KYTY_PM4_GET(buffer[0], DB_STENCILREFMASK, STENCILOPVAL);
-	r.stencil_testval_bf   = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILTESTVAL_BF);
-	r.stencil_mask_bf      = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILMASK_BF);
-	r.stencil_writemask_bf = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILWRITEMASK_BF);
-	r.stencil_opval_bf     = KYTY_PM4_GET(buffer[1], DB_STENCILREFMASK_BF, STENCILOPVAL_BF);
-
-	cp.GetCtx().SetStencilMask(r);
-
-	return 2;
+	return num_values;
 }
 
 KYTY_HW_CTX_PARSER(HwCtxSetViewportScaleOffset) {
@@ -1367,10 +1317,6 @@ static void HwShSetCsRegister(CommandProcessor& cp, uint32_t cmd_offset, uint32_
 			                         Pm4::COMPUTE_PGM_RSRC1_BULKY_MASK) != 0u;
 			cs_regs.threadgroup_configuration = ((value >> Pm4::COMPUTE_PGM_RSRC1_WGP_MODE_SHIFT) &
 			                                     Pm4::COMPUTE_PGM_RSRC1_WGP_MODE_MASK) != 0u;
-			cs_regs.wave_size                 = (((value >> Pm4::COMPUTE_PGM_RSRC1_W32_EN_SHIFT) &
-			                                      Pm4::COMPUTE_PGM_RSRC1_W32_EN_MASK) != 0u)
-			                                        ? 32u
-			                                        : 64u;
 			break;
 		case Pm4::COMPUTE_PGM_RSRC2:
 			cs_regs.scratch_en     = ((value >> Pm4::COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT) &
@@ -1627,16 +1573,10 @@ KYTY_HW_UC_PARSER(HwUcSetIaMultiVgtParam) {
 	return num_values;
 }
 
-static void HwIgnoreMultiPrimIbReset([[maybe_unused]] uint32_t value) {}
-
 KYTY_HW_UC_PARSER(HwUcSetMultiPrimIbReset) {
-	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
-
-	for (uint32_t i = 0; i < num_values; i++) {
-		HwIgnoreMultiPrimIbReset(buffer[i]);
-	}
-
-	return num_values;
+	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::GE_MULTI_PRIM_IB_RESET_EN);
+	cp.GetUcfg().SetPrimitiveResetControl(buffer[0]);
+	return 1;
 }
 
 static void HwUcIgnoreBorderColorTableAddr([[maybe_unused]] uint32_t cmd_offset,
@@ -1836,6 +1776,17 @@ KYTY_CP_OP_PARSER(CpOpPfpSyncMe) {
 	return 1;
 }
 
+KYTY_CP_OP_PARSER(CpOpRewind) {
+	KYTY_PROFILER_FUNCTION();
+
+	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0005900);
+	EXIT_NOT_IMPLEMENTED((buffer[0] & ~0x81000000u) != 0);
+
+	cp.WaitForRewind((buffer[0] & 0x80000000u) != 0);
+
+	return 1;
+}
+
 KYTY_CP_OP_PARSER(CpOpSetPredication) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -1908,11 +1859,11 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	uint64_t reference   = buffer[5] | (static_cast<uint64_t>(buffer[6]) << 32u);
 	uint32_t mode        = buffer[0] & 0x3u;
 	uint32_t function    = (buffer[0] >> 8u) & 0x7u;
-	auto*    then_buffer = reinterpret_cast<uint32_t*>((buffer[7] & 0xfffffffcu) |
-	                                                   (static_cast<uint64_t>(buffer[8]) << 32u));
+	auto*    then_buffer = reinterpret_cast<const uint32_t*>(
+	    (buffer[7] & 0xfffffffcu) | (static_cast<uint64_t>(buffer[8]) << 32u));
 	uint32_t then_num_dw = buffer[9] & 0xfffffu;
-	auto*    else_buffer = reinterpret_cast<uint32_t*>((buffer[10] & 0xfffffffcu) |
-	                                                   (static_cast<uint64_t>(buffer[11]) << 32u));
+	auto*    else_buffer = reinterpret_cast<const uint32_t*>(
+	    (buffer[10] & 0xfffffffcu) | (static_cast<uint64_t>(buffer[11]) << 32u));
 	uint32_t else_num_dw = buffer[12] & 0xfffffu;
 
 	EXIT_NOT_IMPLEMENTED(compare_addr == nullptr);
@@ -1926,10 +1877,10 @@ KYTY_CP_OP_PARSER(CpOpBranch) {
 	     reinterpret_cast<uint64_t>(else_buffer), else_num_dw);
 
 	if (take_then) {
-		cp.ProcessIndirectBuffer(then_buffer, then_num_dw);
+		cp.ProcessIndirectBuffer({then_buffer, then_num_dw});
 	} else if (mode == 2 && else_num_dw != 0) {
 		EXIT_NOT_IMPLEMENTED(else_buffer == nullptr);
-		cp.ProcessIndirectBuffer(else_buffer, else_num_dw);
+		cp.ProcessIndirectBuffer({else_buffer, else_num_dw});
 	}
 
 	return payload_dw;
@@ -2196,12 +2147,19 @@ KYTY_CP_OP_PARSER(CpOpEventWrite) {
 
 	EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != Pm4::IT_EVENT_WRITE);
 
-	uint32_t event_index = (buffer[0] >> 8u) & 0x7u;
-	uint32_t event_type  = (buffer[0]) & 0x3fu;
+	const auto packet_size_dw = KYTY_PM4_LEN(cmd_id);
+	uint32_t   event_index    = (buffer[0] >> 8u) & 0x7u;
+	uint32_t   event_type     = (buffer[0]) & 0x3fu;
+	uint64_t   event_address  = 0;
 
-	cp.TriggerEvent(event_type, event_index);
+	if (event_type == 0x39u) {
+		EXIT_NOT_IMPLEMENTED(packet_size_dw != 4u);
+		event_address = buffer[1] | (static_cast<uint64_t>(buffer[2]) << 32u);
+	}
 
-	return KYTY_PM4_LEN(cmd_id) - 1u;
+	cp.TriggerEvent(event_type, event_index, event_address);
+
+	return packet_size_dw - 1u;
 }
 
 KYTY_CP_OP_PARSER(CpOpEventWriteEop) {
@@ -2363,7 +2321,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 	}
 
 	auto* indirect_buffer =
-	    reinterpret_cast<uint32_t*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
+	    reinterpret_cast<const uint32_t*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
 	uint32_t                     indirect_num_dw = control & 0xfffffu;
 	static std::atomic<uint32_t> indirect_log_count {0};
 	if (indirect_log_count.fetch_add(1) < 128) {
@@ -2383,7 +2341,7 @@ KYTY_CP_OP_PARSER(CpOpIndirectBuffer) {
 
 	GraphicsDbgDumpDcb("ci", indirect_num_dw, indirect_buffer);
 
-	cp.ProcessIndirectBuffer(indirect_buffer, indirect_num_dw);
+	cp.ProcessIndirectBuffer({indirect_buffer, indirect_num_dw});
 
 	return 3;
 }
@@ -2406,10 +2364,10 @@ KYTY_CP_OP_PARSER(CpOpIndirectCxRegs) {
 		EXIT("indirect CX registers have null address, num_regs = %" PRIu32 "\n", indirect_num_dw);
 	}
 	for (uint32_t i = 0; i < indirect_num_dw; i++, indirect_buffer += 2) {
-		// Keep the encoded offset for packet control values, and use the decoded offset only
-		// for register dispatch.
+		// Keep the encoded offset for packet control values, and use the normalized offset
+		// only for register dispatch.
 		auto raw_cmd_offset = indirect_buffer[0];
-		auto cmd_offset     = DecodeIndirectCxRegisterOffset(raw_cmd_offset);
+		auto cmd_offset     = NormalizeRegisterOffset(raw_cmd_offset);
 		auto value          = indirect_buffer[1];
 
 		if (HwCtxTrySetFakeRegister(cmd_offset, value)) {
@@ -2859,6 +2817,20 @@ KYTY_CP_OP_PARSER(CpOpSetShaderReg) {
 	auto pfunc = g_hw_sh_func[cmd_offset];
 
 	if (pfunc == nullptr) {
+		auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
+		bool handled    = num_values != 0;
+		for (uint32_t i = 0; i < num_values; i++) {
+			if (cmd_offset + i >= Pm4::SH_NUM || g_hw_sh_indirect_func[cmd_offset + i] == nullptr) {
+				handled = false;
+				break;
+			}
+		}
+		if (handled) {
+			for (uint32_t i = 0; i < num_values; i++) {
+				g_hw_sh_indirect_func[cmd_offset + i](cp, cmd_offset + i, buffer[1 + i]);
+			}
+			return num_values + 1u;
+		}
 		EXIT("unknown shader register\n\t%05" PRIx32 ":\n\tcmd_id = %08" PRIx32
 		     "\n\tcmd_offset = %08" PRIx32 "\n",
 		     num_dw - dw, cmd_id, cmd_offset);
@@ -2872,6 +2844,10 @@ KYTY_CP_OP_PARSER(CpOpSetShaderReg) {
 KYTY_CP_OP_PARSER(CpOpSetUconfigReg) {
 	KYTY_PROFILER_FUNCTION();
 
+	if (Gen5::AgcIsInternalDataPacket(cmd_id, buffer)) {
+		return KYTY_PM4_LEN(cmd_id) - 1u;
+	}
+
 	auto raw_cmd_offset = buffer[0];
 	auto cmd_offset     = NormalizeRegisterOffset(raw_cmd_offset);
 
@@ -2962,19 +2938,21 @@ template <typename T>
 static uint32_t CpOpWaitRegMemSized(CommandProcessor& cp, uint32_t cmd_id, const uint32_t* buffer) {
 	static_assert(sizeof(T) == sizeof(uint32_t) || sizeof(T) == sizeof(uint64_t));
 
-	constexpr auto value_dw   = static_cast<uint32_t>(sizeof(T) / sizeof(uint32_t));
+	constexpr auto value_dw = static_cast<uint32_t>(sizeof(T) / sizeof(uint32_t));
 	constexpr auto payload_dw = 4u + value_dw * 2u;
 	constexpr auto packet_dw  = payload_dw + 1u;
-	constexpr auto register_id =
-	    (sizeof(T) == sizeof(uint32_t) ? Pm4::R_WAIT_MEM_32 : Pm4::R_WAIT_MEM_64);
+	constexpr auto opcode =
+	    (sizeof(T) == sizeof(uint32_t) ? Pm4::IT_WAIT_REG_MEM : Pm4::IT_WAIT_REG_MEM_64);
 
-	EXIT_NOT_IMPLEMENTED(KYTY_PM4_R(cmd_id) != register_id);
+	EXIT_NOT_IMPLEMENTED(((cmd_id >> 8u) & 0xffu) != opcode || KYTY_PM4_R(cmd_id) != 0u);
 	EXIT_NOT_IMPLEMENTED(KYTY_PM4_LEN(cmd_id) != packet_dw);
 
-	auto* addr = reinterpret_cast<const T*>(buffer[0] | (static_cast<uint64_t>(buffer[1]) << 32u));
-	auto  mask = CpOpWaitRegMemReadValue<T>(buffer + 2u);
-	auto  ref  = CpOpWaitRegMemReadValue<T>(buffer + 2u + value_dw);
-	auto  ctrl = buffer[2u + value_dw * 2u];
+	constexpr auto address_align_mask = (sizeof(T) == sizeof(uint32_t) ? 0x3u : 0x7u);
+	auto  ctrl = buffer[0];
+	auto* addr = reinterpret_cast<const T*>((buffer[1] & ~address_align_mask) |
+	                                        (static_cast<uint64_t>(buffer[2] & 0x3ffffu) << 32u));
+	auto  ref  = CpOpWaitRegMemReadValue<T>(buffer + 3u);
+	auto  mask = CpOpWaitRegMemReadValue<T>(buffer + 3u + value_dw);
 	auto  poll = buffer[3u + value_dw * 2u];
 
 	EXIT_NOT_IMPLEMENTED((ctrl & 0x10u) == 0);
@@ -3546,7 +3524,7 @@ void GraphicsInitJmpTablesCxIndirect() {
 		cp.GetCtx().SetReuseOff(value);
 	};
 	g_hw_ctx_indirect_func[Pm4::VGT_MULTI_PRIM_IB_RESET_INDX] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwIgnoreMultiPrimIbReset(value);
+		cp.GetCtx().SetPrimitiveResetIndex(value);
 	};
 	g_hw_ctx_indirect_func[Pm4::VGT_TESS_DISTRIBUTION] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
 		cp.GetCtx().SetTessDistribution(value);
@@ -4424,7 +4402,7 @@ void GraphicsInitJmpTablesUcIndirect() {
 		HwUcIgnoreIaMultiVgtParam(value);
 	};
 	g_hw_uc_indirect_func[Pm4::GE_MULTI_PRIM_IB_RESET_EN] = [](KYTY_HW_UC_INDIRECT_ARGS) {
-		HwIgnoreMultiPrimIbReset(value);
+		cp.GetUcfg().SetPrimitiveResetControl(value);
 	};
 	g_hw_uc_indirect_func[Pm4::TA_CS_BC_BASE_ADDR] = [](KYTY_HW_UC_INDIRECT_ARGS) {
 		HwUcIgnoreBorderColorTableAddr(cmd_offset, value);

src/graphics/guest_gpu/gpu_format.cpp

@@ -9,79 +9,80 @@ namespace Libs::Graphics::Prospero {
 namespace {
 
 struct FormatInfo {
-	BufferFormat format;
-	uint32_t     bytes_per_element;
-	uint32_t     block_compressed_bytes_per_block;
-	uint32_t     render_target_bytes_per_element;
-	bool         supported_sampled_texture_format;
-	bool         unsigned_integer_texture_format;
+	BufferFormat       format;
+	uint32_t           bytes_per_element;
+	uint32_t           block_compressed_bytes_per_block;
+	uint32_t           render_target_bytes_per_element;
+	bool               sampled_texture;
+	bool               uint_texture;
 };
 
 constexpr FormatInfo kFormatInfo[] = {
-    {BufferFormat::k8UNorm, 1, 0, 1, true, false},
-    {BufferFormat::k8SNorm, 0, 0, 1, false, false},
-    {BufferFormat::k8UInt, 1, 0, 1, true, true},
-    {BufferFormat::k16UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k16SNorm, 2, 0, 2, true, false},
-    {BufferFormat::k16UInt, 2, 0, 2, true, true},
-    {BufferFormat::k16SInt, 2, 0, 2, false, false},
-    {BufferFormat::k16Float, 2, 0, 2, true, false},
-    {BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
-    {BufferFormat::k8_8UInt, 2, 0, 2, true, true},
-    {BufferFormat::k8_8SInt, 2, 0, 2, false, false},
-    {BufferFormat::k32UInt, 4, 0, 4, true, true},
-    {BufferFormat::k32SInt, 4, 0, 4, false, false},
-    {BufferFormat::k32Float, 4, 0, 4, true, false},
-    {BufferFormat::k16_16UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k16_16SNorm, 4, 0, 4, true, false},
-    {BufferFormat::k16_16UInt, 4, 0, 4, true, true},
-    {BufferFormat::k16_16SInt, 4, 0, 4, false, false},
-    {BufferFormat::k16_16Float, 4, 0, 4, true, false},
-    {BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
-    {BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
-    {BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
-    {BufferFormat::k8_8_8_8SNorm, 4, 0, 4, true, false},
-    {BufferFormat::k8_8_8_8UInt, 4, 0, 4, true, true},
-    {BufferFormat::k8_8_8_8SInt, 4, 0, 4, false, false},
-    {BufferFormat::k32_32UInt, 8, 0, 8, true, true},
-    {BufferFormat::k32_32SInt, 8, 0, 8, false, false},
-    {BufferFormat::k32_32Float, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16UNorm, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16SNorm, 8, 0, 8, true, false},
-    {BufferFormat::k16_16_16_16UInt, 8, 0, 8, true, true},
-    {BufferFormat::k16_16_16_16SInt, 8, 0, 8, false, false},
-    {BufferFormat::k16_16_16_16Float, 8, 0, 8, true, false},
-    {BufferFormat::k32_32_32UInt, 12, 0, 12, true, true},
-    {BufferFormat::k32_32_32SInt, 12, 0, 12, false, false},
-    {BufferFormat::k32_32_32Float, 12, 0, 12, true, false},
-    {BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
-    {BufferFormat::k32_32_32_32SInt, 16, 0, 16, false, false},
-    {BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
-    {BufferFormat::k8Srgb, 1, 0, 0, true, false},
-    {BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
-    {BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
-    {BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
-    {BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
-    {BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
-    {BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
-    {BufferFormat::kFmask8_S4_F4, 1, 0, 1, true, false},
-    {BufferFormat::kBc1UNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc1Srgb, 0, 8, 0, true, false},
-    {BufferFormat::kBc2UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc2Srgb, 0, 16, 0, true, false},
-    {BufferFormat::kBc3UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc3Srgb, 0, 16, 0, true, false},
-    {BufferFormat::kBc4UNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc4SNorm, 0, 8, 0, true, false},
-    {BufferFormat::kBc5UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc5SNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc6UFloat, 0, 16, 0, true, false},
-    {BufferFormat::kBc6SFloat, 0, 16, 0, true, false},
-    {BufferFormat::kBc7UNorm, 0, 16, 0, true, false},
-    {BufferFormat::kBc7Srgb, 0, 16, 0, true, false},
+	{BufferFormat::k8UNorm, 1, 0, 1, true, false},
+	{BufferFormat::k8SNorm, 0, 0, 1, false, false},
+	{BufferFormat::k8UInt, 1, 0, 1, true, true},
+	{BufferFormat::k16UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k16SNorm, 2, 0, 2, true, false},
+	{BufferFormat::k16UInt, 2, 0, 2, true, true},
+	{BufferFormat::k16SInt, 2, 0, 2, false, false},
+	{BufferFormat::k16Float, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8SNorm, 2, 0, 2, true, false},
+	{BufferFormat::k8_8UInt, 2, 0, 2, true, true},
+	{BufferFormat::k8_8SInt, 2, 0, 2, false, false},
+	{BufferFormat::k32UInt, 4, 0, 4, true, true},
+	{BufferFormat::k32SInt, 4, 0, 4, false, false},
+	{BufferFormat::k32Float, 4, 0, 4, true, false},
+	{BufferFormat::k16_16UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k16_16SNorm, 4, 0, 4, true, false},
+	{BufferFormat::k16_16UInt, 4, 0, 4, true, true},
+	{BufferFormat::k16_16SInt, 4, 0, 4, false, false},
+	{BufferFormat::k16_16Float, 4, 0, 4, true, false},
+	{BufferFormat::k11_11_10UInt, 4, 0, 4, true, true},
+	{BufferFormat::k11_11_10Float, 4, 0, 4, true, false},
+	{BufferFormat::k10_10_10_2UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k10_10_10_2UInt, 4, 0, 4, true, true},
+	{BufferFormat::k8_8_8_8UNorm, 4, 0, 4, true, false},
+	{BufferFormat::k8_8_8_8SNorm, 4, 0, 4, true, false},
+	{BufferFormat::k8_8_8_8UInt, 4, 0, 4, true, true},
+	{BufferFormat::k8_8_8_8SInt, 4, 0, 4, false, false},
+	{BufferFormat::k32_32UInt, 8, 0, 8, true, true},
+	{BufferFormat::k32_32SInt, 8, 0, 8, false, false},
+	{BufferFormat::k32_32Float, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16UNorm, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16SNorm, 8, 0, 8, true, false},
+	{BufferFormat::k16_16_16_16UInt, 8, 0, 8, true, true},
+	{BufferFormat::k16_16_16_16SInt, 8, 0, 8, false, false},
+	{BufferFormat::k16_16_16_16Float, 8, 0, 8, true, false},
+	{BufferFormat::k32_32_32UInt, 12, 0, 12, true, true},
+	{BufferFormat::k32_32_32SInt, 12, 0, 12, false, false},
+	{BufferFormat::k32_32_32Float, 12, 0, 12, true, false},
+	{BufferFormat::k32_32_32_32UInt, 16, 0, 16, true, true},
+	{BufferFormat::k32_32_32_32SInt, 16, 0, 16, false, false},
+	{BufferFormat::k32_32_32_32Float, 16, 0, 16, true, false},
+	{BufferFormat::k8Srgb, 1, 0, 0, true, false},
+	{BufferFormat::k8_8Srgb, 2, 0, 0, true, false},
+	{BufferFormat::k8_8_8_8Srgb, 4, 0, 4, true, false},
+	{BufferFormat::k9_9_9_5Float, 4, 0, 0, true, false},
+	{BufferFormat::k5_6_5UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k5_5_5_1UNorm, 2, 0, 2, true, false},
+	{BufferFormat::k1_5_5_5UNorm, 0, 0, 2, false, false},
+	{BufferFormat::k4_4_4_4UNorm, 2, 0, 2, true, false},
+	{BufferFormat::kFmask8_S4_F4, 1, 0, 1, true, false},
+	{BufferFormat::kBc1UNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc1Srgb, 0, 8, 0, true, false},
+	{BufferFormat::kBc2UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc2Srgb, 0, 16, 0, true, false},
+	{BufferFormat::kBc3UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc3Srgb, 0, 16, 0, true, false},
+	{BufferFormat::kBc4UNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc4SNorm, 0, 8, 0, true, false},
+	{BufferFormat::kBc5UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc5SNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc6UFloat, 0, 16, 0, true, false},
+	{BufferFormat::kBc6SFloat, 0, 16, 0, true, false},
+	{BufferFormat::kBc7UNorm, 0, 16, 0, true, false},
+	{BufferFormat::kBc7Srgb, 0, 16, 0, true, false},
 };
 
 constexpr uint8_t RenderTargetTypeBit(ChannelType type) {
@@ -201,18 +202,22 @@ uint32_t RenderTargetBytesPerElement(BufferFormat format) {
 	return info != nullptr ? info->render_target_bytes_per_element : 0;
 }
 
-bool IsSupportedTextureFormat(BufferFormat format) {
-	const auto* info = FindFormatInfo(format);
-	return info != nullptr && info->supported_sampled_texture_format;
-}
-
-bool IsUintTextureFormat(BufferFormat format) {
-	const auto* info = FindFormatInfo(format);
-	return info != nullptr && info->unsigned_integer_texture_format;
-}
-
 bool IsFmaskTextureFormat(BufferFormat format) {
 	return format == BufferFormat::kFmask8_S4_F4;
 }
 
+bool IsSampledTextureFormat(BufferFormat format) {
+	const auto* info = FindFormatInfo(format);
+	return info != nullptr && info->sampled_texture;
+}
+
+bool IsUintTextureFormat(BufferFormat format) {
+	const auto* info = FindFormatInfo(format);
+	return info != nullptr && info->uint_texture;
+}
+
+BufferFormat RemapTextureFormat(BufferFormat format) {
+	return format == BufferFormat::k11_11_10UInt ? BufferFormat::k32UInt : format;
+}
+
 } // namespace Libs::Graphics::Prospero

src/graphics/guest_gpu/gpu_format.h

@@ -41,9 +41,10 @@ RenderTargetFormatEncoding ResolveRenderTargetFormat(ChannelLayout layout, Chann
 uint32_t                   NumBytesPerElement(BufferFormat format);
 uint32_t                   BlockCompressedBytesPerBlock(BufferFormat format);
 uint32_t                   RenderTargetBytesPerElement(BufferFormat format);
-bool                       IsSupportedTextureFormat(BufferFormat format);
-bool                       IsUintTextureFormat(BufferFormat format);
 bool                       IsFmaskTextureFormat(BufferFormat format);
+bool                       IsSampledTextureFormat(BufferFormat format);
+bool                       IsUintTextureFormat(BufferFormat format);
+BufferFormat               RemapTextureFormat(BufferFormat format);
 
 } // namespace Libs::Graphics::Prospero
 

src/graphics/guest_gpu/graphicsRun.cpp

@@ -13,6 +13,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/sync.h"
+#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/shader/shader.h"
@@ -26,6 +27,7 @@
 #include <cstdio>
 #include <deque>
 #include <memory>
+#include <mutex>
 #include <semaphore>
 #include <thread>
 #include <vector>
@@ -36,7 +38,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr;
 static thread_local Pm4Execution*     g_current_execution = nullptr;
 static thread_local bool              g_gpu_mutex_owned   = false;
 static thread_local bool              g_gpu_thread        = false;
-static thread_local GpuState*         g_gpu_state         = nullptr;
+static thread_local GuestGpu*         g_gpu_state         = nullptr;
 
 class GpuMutexLock final {
 public:
@@ -59,115 +61,23 @@ private:
 	Common::Mutex& m_mutex;
 };
 
-struct OwnedCmdBuffer {
-	OwnedCmdBuffer() = default;
-	explicit OwnedCmdBuffer(const uint32_t* data, uint32_t count) {
-		EXIT_IF(data == nullptr && count != 0);
-		if (count != 0) {
-			m_words.assign(data, data + count);
-		}
-	}
-
-	[[nodiscard]] bool      Empty() const noexcept { return m_words.empty(); }
-	[[nodiscard]] uint32_t  Size() const noexcept { return static_cast<uint32_t>(m_words.size()); }
-	[[nodiscard]] uint32_t* Data() noexcept { return m_words.data(); }
-
-private:
-	std::vector<uint32_t> m_words;
-};
-
-class GpuState {
-public:
-	static constexpr uint32_t ComputePipeCount     = 7;
-	static constexpr uint32_t QueuesPerComputePipe = 8;
-	static constexpr uint32_t ComputeQueueCount    = ComputePipeCount * QueuesPerComputePipe;
-	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
-
-	explicit GpuState(RenderContext& renderer): m_renderer(renderer) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-		m_gfx_cp = std::make_unique<CommandProcessor>(renderer);
-		m_thread = std::jthread(ThreadRun, this);
-	}
-	~GpuState();
-
-	KYTY_CLASS_NO_COPY(GpuState);
-
-	void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
-	            uint32_t num_const_dw, bool trigger_agc_interrupt_on_done);
-	void SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
-	                   bool trigger_agc_interrupt_on_done);
-	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
-	void Shutdown();
-	[[nodiscard]] bool        IsStopping();
-	void                      SendCommand(Common::UniqueFunction<void>&& command);
-	void                      SendCommandSync(Common::UniqueFunction<void>&& command);
-	int                       GetFrameNum();
-	[[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; }
-
-private:
-	enum class SubmissionType { Graphics, Compute, FlipPreparation };
-
-	struct Submission {
-		SubmissionType type     = SubmissionType::Graphics;
-		uint32_t       queue_id = 0;
-		OwnedCmdBuffer commands;
-		OwnedCmdBuffer constant_commands;
-		Pm4Execution   command_execution;
-		Pm4Execution   constant_execution;
-		bool           trigger_agc_interrupt_on_done = false;
-		bool           reset_processor               = false;
-		bool           started                       = false;
-		bool           command_complete              = false;
-		bool           constant_complete             = false;
-		bool           blocked                       = false;
-		uint64_t       flip_request_id               = 0;
-	};
-
-	void              Enqueue(Submission submission);
-	void              WaitForIdle();
-	void              ProcessCommands();
-	bool              Process(Submission& submission);
-	static void       ThreadRun(void* data);
-	CommandProcessor& GetProcessor(uint32_t queue_id);
-
-	RenderContext&                                 m_renderer;
-	Common::Mutex                                  m_submission_mutex;
-	Common::Mutex                                  m_queue_mutex;
-	std::mutex                                     m_shutdown_mutex;
-	Common::CondVar                                m_work_available;
-	Common::CondVar                                m_idle;
-	std::array<std::deque<Submission>, QueueCount> m_queues;
-	std::deque<Common::UniqueFunction<void>>       m_commands;
-	std::atomic_uint32_t                           m_pending_commands {0};
-	uint32_t                                       m_next_queue        = 0;
-	uint32_t                                       m_submission_count  = 0;
-	bool                                           m_processing        = false;
-	bool                                           m_graphics_done     = true;
-	bool                                           m_accepting         = true;
-	bool                                           m_stopping          = false;
-	bool                                           m_shutdown_complete = false;
-
-	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
-	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;
-
-	uint64_t        m_submit_id = 0;
-	std::atomic_int m_done_num  = 0;
-	std::jthread    m_thread;
-
-	friend class CommandProcessor;
-};
-
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
 	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
 }
 
-GpuState::~GpuState() {
+GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
+	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
+	GraphicsInitJmpTables();
+	m_gfx_cp = std::make_unique<CommandProcessor>(renderer, 0);
+	m_thread = std::jthread(ThreadRun, this);
+}
+
+GuestGpu::~GuestGpu() {
 	Shutdown();
 }
 
-void GpuState::Shutdown() {
+void GuestGpu::Shutdown() {
 	std::lock_guard shutdown_lock(m_shutdown_mutex);
 	if (m_shutdown_complete) {
 		return;
@@ -184,12 +94,12 @@ void GpuState::Shutdown() {
 	m_shutdown_complete = true;
 }
 
-bool GpuState::IsStopping() {
+bool GuestGpu::IsStopping() {
 	Common::LockGuard lock(m_queue_mutex);
 	return m_stopping;
 }
 
-void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
+void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
 	EXIT_IF(!command);
 	if (IsGpuThread()) {
 		command();
@@ -202,7 +112,7 @@ void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
 	m_work_available.Signal();
 }
 
-void GpuState::ProcessCommands() {
+void GuestGpu::ProcessCommands() {
 	EXIT_IF(!IsGpuThread());
 	while (m_pending_commands.load(std::memory_order_acquire) != 0) {
 		Common::UniqueFunction<void> command;
@@ -217,7 +127,7 @@ void GpuState::ProcessCommands() {
 	}
 }
 
-void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
+void GuestGpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	EXIT_IF(!command);
 	if (IsGpuThread()) {
 		command();
@@ -231,38 +141,37 @@ void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	done.acquire();
 }
 
-void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
-                      uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
+void GuestGpu::Submit(std::span<const uint32_t> draw_commands,
+                      std::span<const uint32_t> constant_commands) {
+	if (draw_commands.empty()) {
+		return;
+	}
 	GpuMutexLock lock(m_submission_mutex);
 	Submission   submission;
-	submission.type                          = SubmissionType::Graphics;
-	submission.queue_id                      = 0;
-	submission.commands                      = OwnedCmdBuffer(cmd_draw_buffer, num_draw_dw);
-	submission.constant_commands             = OwnedCmdBuffer(cmd_const_buffer, num_const_dw);
-	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
-	submission.reset_processor               = m_graphics_done;
-	m_graphics_done                          = false;
+	submission.type              = SubmissionType::Graphics;
+	submission.queue_id          = 0;
+	submission.commands          = draw_commands;
+	submission.constant_commands = constant_commands;
+	submission.reset_processor   = m_graphics_done;
+	m_graphics_done              = false;
 	Enqueue(std::move(submission));
 }
 
-void GpuState::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
-                             bool trigger_agc_interrupt_on_done) {
+void GuestGpu::SubmitCompute(uint32_t queue, std::span<const uint32_t> commands) {
+	EXIT_IF(commands.empty());
 	GpuMutexLock lock(m_submission_mutex);
 
-	constexpr uint32_t compute_queue_base = 0x20u;
-	EXIT_NOT_IMPLEMENTED(queue < compute_queue_base ||
-	                     queue >= compute_queue_base + ComputeQueueCount);
+	EXIT_NOT_IMPLEMENTED(queue < ComputeQueueBase || queue >= ComputeQueueBase + ComputeQueueCount);
 
-	const auto compute_queue = queue - compute_queue_base;
+	const auto compute_queue = queue - ComputeQueueBase;
 	Submission submission;
-	submission.type                          = SubmissionType::Compute;
-	submission.queue_id                      = 1 + compute_queue;
-	submission.commands                      = OwnedCmdBuffer(cmd_buffer, num_dw);
-	submission.trigger_agc_interrupt_on_done = trigger_agc_interrupt_on_done;
+	submission.type     = SubmissionType::Compute;
+	submission.queue_id = 1 + compute_queue;
+	submission.commands = commands;
 	Enqueue(std::move(submission));
 }
 
-void GpuState::SubmitFlipPreparation(uint64_t request_id) {
+void GuestGpu::SubmitFlipPreparation(uint64_t request_id) {
 	GpuMutexLock lock(m_submission_mutex);
 	Submission   submission;
 	submission.type            = SubmissionType::FlipPreparation;
@@ -273,7 +182,7 @@ void GpuState::SubmitFlipPreparation(uint64_t request_id) {
 	Enqueue(std::move(submission));
 }
 
-void GpuState::Done() {
+void GuestGpu::Done() {
 	GpuMutexLock lock(m_submission_mutex);
 	if (!IsGpuThread()) {
 		WaitForIdle();
@@ -282,18 +191,18 @@ void GpuState::Done() {
 	m_done_num++;
 }
 
-int GpuState::GetFrameNum() {
+int GuestGpu::GetFrameNum() const {
 	return m_done_num;
 }
 
-CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) {
+CommandProcessor& GuestGpu::GetProcessor(uint32_t queue_id) {
 	EXIT_IF(queue_id >= QueueCount);
 	if (queue_id == 0) {
 		return *m_gfx_cp;
 	}
 	auto& processor = m_compute_cp[queue_id - 1];
 	if (processor == nullptr) {
-		processor = std::make_unique<CommandProcessor>(m_renderer);
+		processor = std::make_unique<CommandProcessor>(m_renderer, ComputeQueueBase + queue_id - 1);
 	}
 	return *processor;
 }
@@ -346,8 +255,7 @@ void CommandProcessor::BufferFlush() {
 }
 
 void CommandProcessor::BufferFlushAndWait() {
-	auto& submitted = GetScheduler().FlushAndGetSubmitted();
-	submitted.WaitForFenceOnly();
+	GetScheduler().FlushAndWait();
 }
 
 void CommandProcessor::BufferWait() {
@@ -374,6 +282,12 @@ void CommandProcessor::WaitDeDiff(uint32_t diff) {
 	}
 }
 
+void CommandProcessor::WaitForRewind(bool valid) {
+	if (!valid) {
+		SuspendPm4();
+	}
+}
+
 void CommandProcessor::IncrementDe() {
 	m_de_count++;
 }
@@ -426,10 +340,8 @@ template void CommandProcessor::WaitRegMem<uint64_t>(uint32_t, const uint64_t*,
 
 void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw_num,
                                  uint32_t write_control) {
-	const uint32_t dst_sel      = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
-	const uint32_t cache_policy = (write_control >> 25u) & 0x3u;
-	const uint32_t increment    = (write_control >> 16u) & 0x1u;
-	const uint32_t write_confirm = (write_control >> 20u) & 0x1u;
+	const uint32_t dst_sel = ((write_control >> 30u) & 0x1u) | ((write_control >> 7u) & 0x1eu);
+	const bool     write_one_address = ((write_control >> 16u) & 0x1u) != 0;
 
 	switch (dst_sel) {
 		case 0:
@@ -439,16 +351,17 @@ void CommandProcessor::WriteData(uint32_t* dst, const uint32_t* src, uint32_t dw
 		case 6: break;
 		default: EXIT("unsupported writeData destination selector 0x%02" PRIx32 "\n", dst_sel);
 	}
-	EXIT_NOT_IMPLEMENTED(increment != 0);
-
-	if (cache_policy > 3 || write_confirm > 1) {
-		LOGF("\t warning: unexpected write_data control 0x%08" PRIx32 "\n", write_control);
-	}
 	if (dw_num == 0) {
 		return;
 	}
 
-	memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
+	if (write_one_address) {
+		for (uint32_t i = 0; i < dw_num; i++) {
+			dst[0] = src[i];
+		}
+	} else {
+		memcpy(dst, src, static_cast<size_t>(dw_num) * sizeof(uint32_t));
+	}
 }
 
 void CommandProcessor::WriteReferenceClock(uint64_t dst_address, uint32_t num_bytes) {
@@ -489,6 +402,13 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 			default: return false;
 		}
 	};
+	if (dst_sel == 2) {
+		// kNowhere discards the GL2 prefetch destination without a guest-visible write.
+		if (src_sel != 3) {
+			EXIT("unsupported dmaData nowhere source selector 0x%02" PRIx8 "\n", src_sel);
+		}
+		return;
+	}
 	bool dst_gds = false;
 	if (!decode_gds(dst_sel, dst_gds)) {
 		EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel);
@@ -511,7 +431,7 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 	                        dst_gds, src_gds);
 }
 
-void GpuState::Enqueue(Submission submission) {
+void GuestGpu::Enqueue(Submission submission) {
 	EXIT_IF(submission.queue_id >= QueueCount);
 	Common::LockGuard lock(m_queue_mutex);
 	EXIT_IF(!m_accepting);
@@ -520,15 +440,15 @@ void GpuState::Enqueue(Submission submission) {
 	m_work_available.Signal();
 }
 
-void GpuState::WaitForIdle() {
+void GuestGpu::WaitForIdle() {
 	Common::LockGuard lock(m_queue_mutex);
 	while (m_processing || !m_commands.empty() || m_submission_count != 0) {
 		m_idle.Wait(&m_queue_mutex);
 	}
 }
 
-void GpuState::ThreadRun(void* data) {
-	auto* gpu = static_cast<GpuState*>(data);
+void GuestGpu::ThreadRun(void* data) {
+	auto* gpu = static_cast<GuestGpu*>(data);
 	EXIT_IF(gpu == nullptr);
 	KYTY_PROFILER_THREAD("Thread_Gpu");
 	g_gpu_thread = true;
@@ -624,9 +544,13 @@ void GpuState::ThreadRun(void* data) {
 	}
 }
 
-bool GpuState::Process(Submission& submission) {
-	auto&      cp          = GetProcessor(submission.queue_id);
+bool GuestGpu::Process(Submission& submission) {
 	const bool first_slice = !submission.started;
+	if (first_slice && RenderDocCaptureRequested()) {
+		Common::LockGuard render_lock(m_renderer.GetMutex());
+		RenderDocStartCapture();
+	}
+	auto& cp = GetProcessor(submission.queue_id);
 
 	if (first_slice && submission.reset_processor) {
 		cp.Reset();
@@ -645,21 +569,20 @@ bool GpuState::Process(Submission& submission) {
 	switch (submission.type) {
 		case SubmissionType::Graphics: {
 			bool progressed = false;
-			submission.constant_complete |= submission.constant_commands.Empty();
+			submission.constant_complete |= submission.constant_commands.empty();
 			for (;;) {
 				bool round_progress = false;
 				if (!submission.constant_complete) {
 					submission.constant_complete =
-					    cp.Process(
-					        submission.constant_execution, submission.constant_commands.Data(),
-					        submission.constant_commands.Size()) == Pm4ProcessResult::Complete;
+					    cp.Process(submission.constant_execution, submission.constant_commands) ==
+					    Pm4ProcessResult::Complete;
 					round_progress |= submission.constant_execution.MadeProgress();
 				}
 				cp.SetCeComplete(submission.constant_complete);
 				if (!submission.command_complete) {
 					submission.command_complete =
-					    cp.Process(submission.command_execution, submission.commands.Data(),
-					               submission.commands.Size()) == Pm4ProcessResult::Complete;
+					    cp.Process(submission.command_execution, submission.commands) ==
+					    Pm4ProcessResult::Complete;
 					round_progress |= submission.command_execution.MadeProgress();
 				}
 				progressed |= round_progress;
@@ -676,14 +599,11 @@ bool GpuState::Process(Submission& submission) {
 			} else if (complete) {
 				m_renderer.GetGpuResources().RunGarbageCollector();
 			}
-			if (complete && submission.trigger_agc_interrupt_on_done) {
-				m_renderer.TriggerEopEvent(0);
-			}
 			break;
 		}
 		case SubmissionType::Compute: {
-			const auto      num_dw                  = submission.commands.Size();
-			auto*           buffer                  = submission.commands.Data();
+			const auto      num_dw = static_cast<uint32_t>(submission.commands.size());
+			const auto*     buffer = submission.commands.data();
 			static uint32_t compute_batch_log_count = 0;
 			if (first_slice && num_dw <= 128 && compute_batch_log_count++ < 32) {
 				LOGF("compute direct batch: data=0x%016" PRIx64 ", num_dw=%" PRIu32 "\n",
@@ -695,7 +615,7 @@ bool GpuState::Process(Submission& submission) {
 			if (first_slice) {
 				GraphicsDbgDumpDcb("cc", num_dw, buffer);
 			}
-			complete = cp.Process(submission.command_execution, buffer, num_dw) ==
+			complete = cp.Process(submission.command_execution, submission.commands) ==
 			           Pm4ProcessResult::Complete;
 			if (submission.command_execution.MadeProgress()) {
 				if (complete) {
@@ -705,9 +625,6 @@ bool GpuState::Process(Submission& submission) {
 			} else if (complete) {
 				m_renderer.GetGpuResources().RunGarbageCollector();
 			}
-			if (complete && submission.trigger_agc_interrupt_on_done) {
-				Sync::TriggerAgcUserInterrupt();
-			}
 			break;
 		}
 		case SubmissionType::FlipPreparation:
@@ -719,12 +636,13 @@ bool GpuState::Process(Submission& submission) {
 	return complete;
 }
 
-Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* buffer,
-                                           uint32_t size_dw) {
+Pm4ProcessResult CommandProcessor::Process(Pm4Execution&             execution,
+                                           std::span<const uint32_t> commands) {
 	KYTY_PROFILER_BLOCK("CommandProcessor::Process");
 	EXIT_IF(g_current_execution != nullptr);
-	if (execution.m_buffer_stack.empty() && size_dw != 0) {
-		execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0});
+	EXIT_IF(commands.size() > UINT32_MAX);
+	if (execution.m_buffer_stack.empty() && !commands.empty()) {
+		execution.m_buffer_stack.push_back({commands});
 	}
 	execution.m_suspended     = false;
 	execution.m_made_progress = false;
@@ -749,14 +667,14 @@ Pm4ProcessResult CommandProcessor::Process(Pm4Execution& execution, uint32_t* bu
 	                                        : Pm4ProcessResult::Blocked;
 }
 
-void CommandProcessor::ProcessIndirectBuffer(uint32_t* buffer, uint32_t size_dw) {
+void CommandProcessor::ProcessIndirectBuffer(std::span<const uint32_t> commands) {
 	EXIT_IF(g_current_execution == nullptr);
-	if (size_dw == 0) {
+	if (commands.empty()) {
 		return;
 	}
 	auto&      execution  = *g_current_execution;
 	const auto stop_depth = execution.m_buffer_stack.size();
-	execution.m_buffer_stack.push_back({buffer, size_dw, size_dw, 0});
+	execution.m_buffer_stack.push_back({commands});
 	ProcessPm4(execution, stop_depth);
 }
 
@@ -772,29 +690,28 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 		}
 		const auto buffer_index = execution.m_buffer_stack.size() - 1;
 		auto&      cursor       = execution.m_buffer_stack[buffer_index];
+		EXIT_IF(cursor.offset_dw > cursor.commands.size());
 		if (cursor.deferred_advance_dw != 0) {
-			EXIT_IF(cursor.deferred_advance_dw > cursor.remaining_dw);
-			cursor.next_packet += cursor.deferred_advance_dw;
-			cursor.remaining_dw -= cursor.deferred_advance_dw;
+			EXIT_IF(cursor.deferred_advance_dw > cursor.commands.size() - cursor.offset_dw);
+			cursor.offset_dw += cursor.deferred_advance_dw;
 			cursor.deferred_advance_dw = 0;
 			execution.m_made_progress  = true;
 			continue;
 		}
-		if (cursor.remaining_dw == 0) {
+		if (cursor.offset_dw == cursor.commands.size()) {
 			execution.m_buffer_stack.pop_back();
 			continue;
 		}
 
-		auto* const packet        = cursor.next_packet;
-		const auto  remaining_dw  = cursor.remaining_dw;
-		const auto  total_dw      = cursor.total_dw;
-		const auto  packet_header = packet[0];
-		const auto  opcode        = (packet_header >> 8u) & 0xffu;
+		const auto* const packet        = cursor.commands.data() + cursor.offset_dw;
+		const auto        total_dw      = static_cast<uint32_t>(cursor.commands.size());
+		const auto        remaining_dw  = total_dw - cursor.offset_dw;
+		const auto        packet_header = packet[0];
+		const auto        opcode        = (packet_header >> 8u) & 0xffu;
 		EXIT_NOT_IMPLEMENTED(remaining_dw > total_dw);
 
 		if (packet_header == 0x80000000u) {
-			cursor.next_packet++;
-			cursor.remaining_dw--;
+			cursor.offset_dw++;
 			execution.m_made_progress = true;
 			continue;
 		}
@@ -829,8 +746,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 					     dst, val, packet[1], packet[2]);
 				}
 			}
-			cursor.next_packet += packet_dw;
-			cursor.remaining_dw -= packet_dw;
+			cursor.offset_dw += packet_dw;
 			execution.m_made_progress = true;
 			continue;
 		}
@@ -863,8 +779,7 @@ void CommandProcessor::ProcessPm4(Pm4Execution& execution, size_t stop_depth) {
 			return;
 		}
 		EXIT_IF(execution.m_buffer_stack.size() != buffer_index + 1);
-		execution.m_buffer_stack[buffer_index].next_packet += packet_dw;
-		execution.m_buffer_stack[buffer_index].remaining_dw -= packet_dw;
+		execution.m_buffer_stack[buffer_index].offset_dw += packet_dw;
 		execution.m_made_progress = true;
 	}
 }
@@ -1163,6 +1078,8 @@ void CommandProcessor::DrawIndirectMulti(uint32_t data_offset, uint32_t max_coun
 
 void CommandProcessor::DispatchDirect(uint32_t thread_group_x, uint32_t thread_group_y,
                                       uint32_t thread_group_z, uint32_t mode) {
+	m_sh_ctx.SetCsWaveSize(Pm4::ComputeWaveSize(mode));
+
 	uint32_t frame_num = 0;
 	// uint32_t local_x   = 1;
 	// uint32_t local_y   = 1;
@@ -1305,7 +1222,14 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 	switch (interrupt_selector) {
 		case 0x00:
 		case 0x03: with_interrupt = false; break;
-		case 0x01: Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id); return;
+		case 0x01:
+			if (!IsAsyncComputeQueue()) {
+				Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id,
+				                                 interrupt_context_id);
+				return;
+			}
+			with_interrupt = true;
+			break;
 		case 0x02: with_interrupt = true; break;
 		default: EXIT("unknown interrupt_selector\n");
 	}
@@ -1318,10 +1242,11 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		if (with_interrupt) {
 			if (with_writeback) {
 				Sync::WriteAtEndOfPipeWithInterruptWriteBack32(m_submit_id, CurrentBuffer(), dst,
-				                                               data, interrupt_context_id);
+				                                               data, m_interrupt_event_id,
+				                                               interrupt_context_id);
 			} else {
 				Sync::WriteAtEndOfPipeWithInterrupt32(m_submit_id, CurrentBuffer(), dst, data,
-				                                      interrupt_context_id);
+				                                      m_interrupt_event_id, interrupt_context_id);
 			}
 		} else if (with_writeback) {
 			Sync::WriteAtEndOfPipeWithWriteBack32(m_submit_id, CurrentBuffer(), dst, data);
@@ -1336,12 +1261,12 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 				if (eop_event_type == 0x2f && cache_action == 0x00 && event_index == 0x06) {
 					auto* dst = static_cast<uint32_t*>(dst_gpu_addr);
 					SynchronizeGpu();
-					Sync::ReadGds(m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
+					Sync::ReadGds(*m_renderer.GetBufferCache().GetGdsBuffer(), dst, value & 0xffffu,
 					              value >> 16u);
 					Sync::WriteAtEndOfPipeGds32(m_submit_id, CurrentBuffer(), dst, value & 0xffffu,
 					                            value >> 16u);
 					if (with_interrupt) {
-						m_renderer.TriggerEopEvent(interrupt_context_id);
+						m_renderer.TriggerInterrupt(m_interrupt_event_id, interrupt_context_id);
 					}
 					return;
 				}
@@ -1367,10 +1292,12 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 					if (with_interrupt) {
 						if (with_writeback) {
 							Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
-							    m_submit_id, CurrentBuffer(), dst, value, interrupt_context_id);
+							    m_submit_id, CurrentBuffer(), dst, value, m_interrupt_event_id,
+							    interrupt_context_id);
 						} else {
 							Sync::WriteAtEndOfPipeWithInterrupt64(m_submit_id, CurrentBuffer(), dst,
-							                                      value, interrupt_context_id);
+							                                      value, m_interrupt_event_id,
+							                                      interrupt_context_id);
 						}
 					} else if (with_writeback) {
 						Sync::WriteAtEndOfPipeWithWriteBack64(m_submit_id, CurrentBuffer(), dst,
@@ -1449,26 +1376,35 @@ void CommandProcessor::WriteAtEndOfPipe(uint32_t cache_policy, uint32_t event_wr
 		case 0x04:
 			if constexpr (sizeof(T) == sizeof(uint64_t)) {
 				const auto clock = Sync::ReadReferenceClock();
-				std::memcpy(dst_gpu_addr, &clock, sizeof(clock));
+				auto*      dst   = static_cast<uint64_t*>(dst_gpu_addr);
+				std::memcpy(dst, &clock, sizeof(clock));
 				switch (cache_action) {
 					case 0x00:
-						if (((eop_event_type == 0x04 && event_index == 0x05) ||
-						     (eop_event_type == 0x28 && event_index == 0x00)) &&
-						    !with_interrupt) {
-							Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
-							                                   static_cast<uint64_t*>(dst_gpu_addr),
-							                                   clock);
+						if ((eop_event_type == 0x04 && event_index == 0x05) ||
+						    (eop_event_type == 0x28 && event_index == 0x00)) {
+							if (with_interrupt) {
+								Sync::WriteAtEndOfPipeWithInterrupt64(
+								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    interrupt_context_id);
+							} else {
+								Sync::WriteAtEndOfPipeClockCounter(m_submit_id, CurrentBuffer(),
+								                                   dst, clock);
+							}
 							return;
 						}
 						break;
 					case 0x38:
-						if (((eop_event_type == 0x04 &&
-						      (event_index == 0x00 || event_index == 0x05)) ||
-						     (eop_event_type == 0x28 && event_index == 0x00)) &&
-						    !with_interrupt) {
-							Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
-							    m_submit_id, CurrentBuffer(), static_cast<uint64_t*>(dst_gpu_addr),
-							    clock);
+						if ((eop_event_type == 0x04 &&
+						     (event_index == 0x00 || event_index == 0x05)) ||
+						    (eop_event_type == 0x28 && event_index == 0x00)) {
+							if (with_interrupt) {
+								Sync::WriteAtEndOfPipeWithInterruptWriteBack64(
+								    m_submit_id, CurrentBuffer(), dst, clock, m_interrupt_event_id,
+								    interrupt_context_id);
+							} else {
+								Sync::WriteAtEndOfPipeClockCounterWithWriteBack(
+								    m_submit_id, CurrentBuffer(), dst, clock);
+							}
 							return;
 						}
 						break;
@@ -1525,15 +1461,17 @@ void CommandProcessor::EmitGlobalBarrier() {
 void CommandProcessor::TriggerEopEventAtEndOfPipe(uint32_t interrupt_context_id) {
 	CheckBuffer();
 
-	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), interrupt_context_id);
+	Sync::TriggerEopEventAtEndOfPipe(CurrentBuffer(), m_interrupt_event_id, interrupt_context_id);
 }
 
-void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
+void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index,
+                                    uint64_t event_address) {
 	if (GraphicsRunDebugDumpEnabled()) {
 		LOGF("CommandProcessor::TriggerEvent()\n"
 		     "\t event_type  = 0x%08" PRIx32 "\n"
-		     "\t event_index = 0x%08" PRIx32 "\n",
-		     event_type, event_index);
+		     "\t event_index = 0x%08" PRIx32 "\n"
+		     "\t address     = 0x%016" PRIx64 "\n",
+		     event_type, event_index, event_address);
 	}
 
 	const auto valid_cache_event_index = event_index == 0x00000000 || event_index == 0x00000007;
@@ -1570,12 +1508,36 @@ void CommandProcessor::TriggerEvent(uint32_t event_type, uint32_t event_index) {
 		case 0x0000001a:
 		case 0x0000001b:
 		case 0x00000038:
-		case 0x00000039:
 		case 0x0000003a:
 			LOGF("\t temporary: ignoring unsupported event_write type 0x%08" PRIx32
 			     ", index 0x%08" PRIx32 "\n",
 			     event_type, event_index);
 			break;
+		case 0x00000039: {
+			if (event_index != 0x00000001 || event_address == 0 || (event_address & 0x7u) != 0) {
+				EXIT("invalid occlusion-counter dump: index=0x%08" PRIx32 ", address=0x%016" PRIx64
+				     "\n",
+				     event_index, event_address);
+			}
+			static std::once_flag warning_once;
+			std::call_once(warning_once, [] {
+				std::printf("Warning: game uses occlusion queries, which are currently treated as "
+				            "always visible; GPU usage may be higher and FPS may be lower.\n");
+			});
+
+			// Until host occlusion queries are implemented, publish an always-visible result. The
+			// PS5 layout contains one interleaved begin/end pair per DB, and bit 63 marks a result
+			// ready.
+			constexpr uint64_t ready_bit    = 1ull << 63u;
+			constexpr uint64_t counter_mask = ready_bit - 1u;
+			auto*              results      = reinterpret_cast<volatile uint64_t*>(event_address);
+			const auto         value        = ready_bit | m_synthetic_occlusion_counter;
+			for (uint32_t db = 0; db < 16u; db++) {
+				results[db * 2u] = value;
+			}
+			m_synthetic_occlusion_counter = (m_synthetic_occlusion_counter + 1u) & counter_mask;
+			break;
+		}
 		default:
 			EXIT("unknown event type: 0x%08" PRIx32 ", 0x%08" PRIx32 "\n", event_type, event_index);
 	}
@@ -1638,7 +1600,7 @@ void CommandProcessor::FlipWithInterrupt(uint32_t eop_event_type, uint32_t cache
 	                                         m_flip.flip_arg);
 	Sync::WriteAtEndOfPipeWithInterruptWriteBackFlip32(
 	    m_submit_id, command, static_cast<uint32_t*>(dst_gpu_addr), value, m_flip.handle,
-	    m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request);
+	    m_flip.index, m_flip.flip_mode, m_flip.flip_arg, request, m_interrupt_event_id);
 	GetScheduler().Flush();
 }
 
@@ -1659,60 +1621,11 @@ void CommandProcessor::PrepareCpuFlip(uint64_t request_id) {
 }
 
 void CommandProcessor::SynchronizeGpu() {
-	GetScheduler().FinishCurrent();
+	GetScheduler().Finish();
 }
 
-Gpu::Gpu(RenderContext& renderer) {
-	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	GraphicsInitJmpTables();
-	m_state = std::make_unique<GpuState>(renderer);
-}
-
-Gpu::~Gpu() = default;
-
-void Gpu::Shutdown() {
-	m_state->Shutdown();
-}
-
-bool Gpu::IsStopping() {
-	return m_state->IsStopping();
-}
-
-void Gpu::SendCommand(Common::UniqueFunction<void>&& command) {
-	m_state->SendCommand(std::move(command));
-}
-
-void Gpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
-	m_state->SendCommandSync(std::move(command));
-}
-
-void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
-                 uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) {
-	EXIT_IF(draw_commands == nullptr || draw_size_dw == 0);
-	m_state->Submit(draw_commands, draw_size_dw, constant_commands, constant_size_dw,
-	                trigger_agc_interrupt_on_done);
-}
-
-void Gpu::SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw,
-                        bool trigger_agc_interrupt_on_done) {
-	EXIT_IF(commands == nullptr || size_dw == 0);
-	m_state->SubmitCompute(queue, commands, size_dw, trigger_agc_interrupt_on_done);
-}
-
-void Gpu::SubmitFlipPreparation(uint64_t request_id) {
-	m_state->SubmitFlipPreparation(request_id);
-}
-
-void Gpu::Done() {
-	m_state->Done();
-}
-
-int Gpu::GetFrameNum() const {
-	return m_state->GetFrameNum();
-}
-
-bool Gpu::IsGpuThread() noexcept {
-	return GpuState::IsGpuThread();
+bool GuestGpu::IsGpuThread() noexcept {
+	return g_gpu_thread;
 }
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/graphicsRun.h

@@ -3,38 +3,100 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/threads.h"
 #include "common/uniqueFunction.h"
+#include "graphics/guest_gpu/command_processor/commandProcessor.h"
 
+#include <array>
+#include <atomic>
+#include <cstdint>
+#include <deque>
 #include <memory>
+#include <mutex>
+#include <span>
+#include <thread>
 
 namespace Libs::Graphics {
 
-class GpuState;
 class RenderContext;
 
-class Gpu final {
+class GuestGpu final {
 public:
-	explicit Gpu(RenderContext& renderer);
-	~Gpu();
-	KYTY_CLASS_NO_COPY(Gpu);
+	explicit GuestGpu(RenderContext& renderer);
+	~GuestGpu();
+	KYTY_CLASS_NO_COPY(GuestGpu);
 
 	void               Shutdown();
 	[[nodiscard]] bool IsStopping();
 	void               SendCommand(Common::UniqueFunction<void>&& command);
 	void               SendCommandSync(Common::UniqueFunction<void>&& command);
 
-	void Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
-	            uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done = false);
-	void SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw,
-	                   bool trigger_agc_interrupt_on_done = false);
-	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
+	// Submitted command memory is borrowed and must remain valid until GPU execution completes.
+	void              Submit(std::span<const uint32_t> draw_commands,
+	                         std::span<const uint32_t> constant_commands);
+	void              SubmitCompute(uint32_t queue, std::span<const uint32_t> commands);
+	void              SubmitFlipPreparation(uint64_t request_id);
+	void              Done();
 	[[nodiscard]] int GetFrameNum() const;
 
 	[[nodiscard]] static bool IsGpuThread() noexcept;
 
 private:
-	std::unique_ptr<GpuState> m_state;
+	static constexpr uint32_t ComputePipeCount     = 7;
+	static constexpr uint32_t QueuesPerComputePipe = 8;
+	static constexpr uint32_t ComputeQueueCount    = ComputePipeCount * QueuesPerComputePipe;
+	static constexpr uint32_t ComputeQueueBase     = 0x20;
+	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
+
+	enum class SubmissionType { Graphics, Compute, FlipPreparation };
+
+	struct Submission {
+		SubmissionType            type     = SubmissionType::Graphics;
+		uint32_t                  queue_id = 0;
+		std::span<const uint32_t> commands;
+		std::span<const uint32_t> constant_commands;
+		Pm4Execution              command_execution;
+		Pm4Execution              constant_execution;
+		bool                      reset_processor   = false;
+		bool                      started           = false;
+		bool                      command_complete  = false;
+		bool                      constant_complete = false;
+		bool                      blocked           = false;
+		uint64_t                  flip_request_id   = 0;
+	};
+
+	void              Enqueue(Submission submission);
+	void              WaitForIdle();
+	void              ProcessCommands();
+	bool              Process(Submission& submission);
+	static void       ThreadRun(void* data);
+	CommandProcessor& GetProcessor(uint32_t queue_id);
+
+	RenderContext&                                 m_renderer;
+	Common::Mutex                                  m_submission_mutex;
+	Common::Mutex                                  m_queue_mutex;
+	std::mutex                                     m_shutdown_mutex;
+	Common::CondVar                                m_work_available;
+	Common::CondVar                                m_idle;
+	std::array<std::deque<Submission>, QueueCount> m_queues;
+	std::deque<Common::UniqueFunction<void>>       m_commands;
+	std::atomic_uint32_t                           m_pending_commands {0};
+	uint32_t                                       m_next_queue        = 0;
+	uint32_t                                       m_submission_count  = 0;
+	bool                                           m_processing        = false;
+	bool                                           m_graphics_done     = true;
+	bool                                           m_accepting         = true;
+	bool                                           m_stopping          = false;
+	bool                                           m_shutdown_complete = false;
+
+	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
+	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;
+
+	uint64_t        m_submit_id = 0;
+	std::atomic_int m_done_num  = 0;
+	std::jthread    m_thread;
+
+	friend class CommandProcessor;
 };
 } // namespace Libs::Graphics
 

src/graphics/guest_gpu/hardwareContext.h

@@ -295,10 +295,7 @@ struct ClipControl {
 	bool    linear_attribute_clip_enable        = false;
 	bool    force_viewport_index_from_vs_enable = false;
 
-	[[nodiscard]] bool IsZClipModeRepresentable() const {
-		return min_z_clip_disable == max_z_clip_disable;
-	}
-	[[nodiscard]] bool IsZClipEnabled() const { return !min_z_clip_disable; }
+	[[nodiscard]] bool IsZClipEnabled() const { return !min_z_clip_disable && !max_z_clip_disable; }
 };
 
 struct DepthControl {
@@ -941,8 +938,10 @@ public:
 	[[nodiscard]] uint8_t GetStencilClearValue() const { return m_stencil_clear_value; }
 	void SetStencilClearValue(uint8_t clear_value) { m_stencil_clear_value = clear_value; }
 
-	[[nodiscard]] float GetLineWidth() const { return m_line_width; }
-	void                SetLineWidth(float width) { m_line_width = width; }
+	[[nodiscard]] float    GetLineWidth() const { return m_line_width; }
+	void                   SetLineWidth(float width) { m_line_width = width; }
+	[[nodiscard]] uint32_t GetPrimitiveResetIndex() const { return m_primitive_reset_index; }
+	void SetPrimitiveResetIndex(uint32_t index) { m_primitive_reset_index = index; }
 
 	[[nodiscard]] const ShaderRegisters& GetShaderRegisters() const { return m_sh_regs; }
 
@@ -987,7 +986,8 @@ public:
 	void SetScShaderControl(uint32_t value) { m_sh_regs.m_paScShaderControl = value; }
 
 private:
-	float m_line_width = 1.0f;
+	float    m_line_width            = 1.0f;
+	uint32_t m_primitive_reset_index = 0xffffffffu;
 
 	BlendControl    m_blend_control[8];
 	BlendColor      m_blend_color;
@@ -1035,6 +1035,8 @@ public:
 	[[nodiscard]] uint32_t GetIndexOffset() const { return m_index_offset; }
 	void                   SetObjectId(uint32_t object_id) { m_object_id = object_id; }
 	[[nodiscard]] uint32_t GetObjectId() const { return m_object_id; }
+	void SetPrimitiveResetControl(uint32_t control) { m_primitive_reset_control = control; }
+	[[nodiscard]] uint32_t GetPrimitiveResetControl() const { return m_primitive_reset_control; }
 
 	[[nodiscard]] const GeControl& GetGeControl() const { return m_ge_cntl; }
 	void                           SetGeControl(const GeControl& control) { m_ge_cntl = control; }
@@ -1049,9 +1051,10 @@ public:
 	void SetGdsOaAddress(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].address = value; }
 
 private:
-	Prospero::PrimitiveType m_prim_type    = Prospero::PrimitiveType::kNone;
-	uint32_t                m_index_offset = 0;
-	uint32_t                m_object_id    = 0;
+	Prospero::PrimitiveType m_prim_type               = Prospero::PrimitiveType::kNone;
+	uint32_t                m_index_offset            = 0;
+	uint32_t                m_object_id               = 0;
+	uint32_t                m_primitive_reset_control = 0;
 
 	GeControl    m_ge_cntl;
 	GeUserVgprEn m_ge_user_vgpr_en;
@@ -1086,6 +1089,7 @@ public:
 	void SetPsShaderChksum(uint32_t value) { m_ps.ps_regs.chksum = value; }
 
 	void SetCsShader(const CsStageRegisters& cs_regs) { m_cs.cs_regs = cs_regs; }
+	void SetCsWaveSize(uint8_t wave_size) { m_cs.cs_regs.wave_size = wave_size; }
 
 	void SetPsUserSgpr(uint32_t id, uint32_t value, UserSgprType type) {
 		m_ps.ps_user_sgpr.value[id] = value;

src/graphics/guest_gpu/pm4.cpp

@@ -21,12 +21,10 @@ constexpr auto MakeRegisterNames() {
 	names[R_DRAW_RESET]     = "R_DRAW_RESET";
 	names[R_WAIT_FLIP_DONE] = "R_WAIT_FLIP_DONE";
 	names[R_DISPATCH_RESET] = "R_DISPATCH_RESET";
-	names[R_WAIT_MEM_32]    = "R_WAIT_MEM_32";
 	names[R_PUSH_MARKER]    = "R_PUSH_MARKER";
 	names[R_POP_MARKER]     = "R_POP_MARKER";
 	names[R_ACQUIRE_MEM]    = "R_ACQUIRE_MEM";
 	names[R_WRITE_DATA]     = "R_WRITE_DATA";
-	names[R_WAIT_MEM_64]    = "R_WAIT_MEM_64";
 	names[R_FLIP]           = "R_FLIP";
 	names[R_RELEASE_MEM]    = "R_RELEASE_MEM";
 	names[R_DMA_DATA]       = "R_DMA_DATA";
@@ -62,6 +60,7 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_WRITE_DATA]                = "IT_WRITE_DATA";
 	names[IT_MEM_SEMAPHORE]             = "IT_MEM_SEMAPHORE";
 	names[IT_DRAW_INDEX_INDIRECT_MULTI] = "IT_DRAW_INDEX_INDIRECT_MULTI";
+	names[IT_WAIT_REG_MEM]               = "IT_WAIT_REG_MEM";
 	names[IT_INDIRECT_BUFFER]           = "IT_INDIRECT_BUFFER";
 	names[IT_COPY_DATA]                 = "IT_COPY_DATA";
 	names[IT_CP_DMA]                    = "IT_CP_DMA";
@@ -91,6 +90,7 @@ constexpr auto MakeOpcodeNames() {
 	names[IT_DISPATCH_DRAW_PREAMBLE]    = "IT_DISPATCH_DRAW_PREAMBLE";
 	names[IT_DISPATCH_DRAW]             = "IT_DISPATCH_DRAW";
 	names[IT_GET_LOD_STATS]             = "IT_GET_LOD_STATS";
+	names[IT_WAIT_REG_MEM_64]           = "IT_WAIT_REG_MEM_64";
 	names[IT_SET_CONTEXT_REG_INDIRECT]  = "IT_SET_CONTEXT_REG_INDIRECT";
 	return names;
 }
@@ -100,7 +100,7 @@ constexpr auto g_opcode_names   = MakeOpcodeNames();
 
 } // namespace
 
-void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw,
+void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_t start_dw,
                          uint32_t num_dw) {
 	// db_dump();
 
@@ -143,9 +143,9 @@ void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t star
 				if ((op == IT_SET_CONTEXT_REG_INDIRECT || op == IT_SET_SH_REG_INDIRECT ||
 				     op == IT_SET_UCONFIG_REG_INDIRECT) &&
 				    len == 4) {
-					auto* indirect_buffer =
-					    reinterpret_cast<uint32_t*>((static_cast<uint64_t>(cmd[0]) & 0xfffffffcu) |
-					                                (static_cast<uint64_t>(cmd[1]) << 32u));
+					auto* indirect_buffer = reinterpret_cast<const uint32_t*>(
+					    (static_cast<uint64_t>(cmd[0]) & 0xfffffffcu) |
+					    (static_cast<uint64_t>(cmd[1]) << 32u));
 					const uint32_t indirect_num_regs = cmd[3] & 0x3fffu;
 
 					for (uint32_t i = 0; indirect_buffer != nullptr && i < indirect_num_regs;

src/graphics/guest_gpu/pm4.h

@@ -42,6 +42,7 @@ constexpr uint32_t IT_WRITE_DATA                = 0x37;
 constexpr uint32_t IT_MEM_SEMAPHORE             = 0x39;
 constexpr uint32_t IT_DRAW_INDEX_INDIRECT_MULTI = 0x38;
 constexpr uint32_t IT_DISPATCH_DRAW_PREAMBLE    = 0x3A;
+constexpr uint32_t IT_WAIT_REG_MEM              = 0x3C;
 constexpr uint32_t IT_INDIRECT_BUFFER           = 0x3F;
 constexpr uint32_t IT_COPY_DATA                 = 0x40;
 constexpr uint32_t IT_CP_DMA                    = 0x41;
@@ -70,6 +71,7 @@ constexpr uint32_t IT_WAIT_ON_CE_COUNTER        = 0x86;
 constexpr uint32_t IT_WAIT_ON_DE_COUNTER_DIFF   = 0x88;
 constexpr uint32_t IT_DISPATCH_DRAW             = 0x8D;
 constexpr uint32_t IT_GET_LOD_STATS             = 0x8E;
+constexpr uint32_t IT_WAIT_REG_MEM_64           = 0x93;
 constexpr uint32_t IT_SET_CONTEXT_REG_INDIRECT  = 0x9F;
 
 /* Custom codes. Implemented via IT_NOP */
@@ -78,12 +80,10 @@ constexpr uint32_t R_ZERO           = 0x00;
 constexpr uint32_t R_DRAW_RESET     = 0x05;
 constexpr uint32_t R_WAIT_FLIP_DONE = 0x06;
 constexpr uint32_t R_DISPATCH_RESET = 0x09;
-constexpr uint32_t R_WAIT_MEM_32    = 0x0A;
 constexpr uint32_t R_PUSH_MARKER    = 0x0B;
 constexpr uint32_t R_POP_MARKER     = 0x0C;
 constexpr uint32_t R_ACQUIRE_MEM    = 0x14;
 constexpr uint32_t R_WRITE_DATA     = 0x15;
-constexpr uint32_t R_WAIT_MEM_64    = 0x16;
 constexpr uint32_t R_FLIP           = 0x17;
 constexpr uint32_t R_RELEASE_MEM    = 0x18;
 constexpr uint32_t R_DMA_DATA       = 0x19;
@@ -317,8 +317,8 @@ constexpr uint32_t FSR_RECURSIONS1              = 0xF2;
 constexpr uint32_t VGT_MULTI_PRIM_IB_RESET_INDX = 0x103;
 constexpr uint32_t CB_RMI_GL2_CACHE_CONTROL     = 0x104;
 constexpr uint32_t CB_BLEND_RED                 = 0x105;
-constexpr uint32_t CB_BLEND_GREEN               = 0x106;
-constexpr uint32_t CB_BLEND_BLUE                = 0x107;
+constexpr uint32_t CB_BLEND_BLUE                = 0x106;
+constexpr uint32_t CB_BLEND_GREEN               = 0x107;
 constexpr uint32_t CB_BLEND_ALPHA               = 0x108;
 constexpr uint32_t CB_DCC_CONTROL               = 0x109;
 
@@ -387,9 +387,6 @@ constexpr uint32_t SPI_SHADER_POS_FORMAT  = 0x1C3;
 constexpr uint32_t SPI_SHADER_Z_FORMAT    = 0x1C4;
 constexpr uint32_t SPI_SHADER_COL_FORMAT  = 0x1C5;
 
-// Indirect Cx descriptor selector for the 32-entry PS input-control register bank.
-constexpr uint32_t CX_PS_SHADER_USAGE_BASE = 0x10000000u;
-
 constexpr uint32_t CB_BLEND0_CONTROL                            = 0x1E0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_SHIFT       = 0;
 constexpr uint32_t CB_BLEND0_CONTROL_COLOR_SRCBLEND_MASK        = 0x1F;
@@ -984,8 +981,16 @@ constexpr uint32_t COMPUTE_PGM_RSRC1_FP16_OVFL_SHIFT  = 26;
 constexpr uint32_t COMPUTE_PGM_RSRC1_FP16_OVFL_MASK   = 0x1;
 constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_SHIFT   = 29;
 constexpr uint32_t COMPUTE_PGM_RSRC1_WGP_MODE_MASK    = 0x1;
-constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_SHIFT     = 30;
-constexpr uint32_t COMPUTE_PGM_RSRC1_W32_EN_MASK      = 0x1;
+
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT = 15;
+constexpr uint32_t COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK  = 0x1;
+
+[[nodiscard]] constexpr uint8_t ComputeWaveSize(uint32_t dispatch_initiator) {
+	return ((dispatch_initiator >> COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_SHIFT) &
+	        COMPUTE_DISPATCH_INITIATOR_CS_W32_EN_MASK) != 0u
+	           ? 32u
+	           : 64u;
+}
 
 constexpr uint32_t COMPUTE_PGM_RSRC2                      = 0x213;
 constexpr uint32_t COMPUTE_PGM_RSRC2_SCRATCH_EN_SHIFT     = 0;
@@ -1083,7 +1088,7 @@ constexpr uint32_t UC_NOP                       = 0x80003FFF;
 
 constexpr uint32_t UC_NUM = 0x3FFF + 1;
 
-void DumpPm4PacketStream(Common::File* file, uint32_t* cmd_buffer, uint32_t start_dw,
+void DumpPm4PacketStream(Common::File* file, const uint32_t* cmd_buffer, uint32_t start_dw,
                          uint32_t num_dw);
 
 } // namespace Libs::Graphics::Pm4

src/graphics/guest_gpu/tile.cpp

@@ -1351,6 +1351,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 			}
 			break;
 		}
+		case Prospero::TileMode::kStandard4KB:
 		case Prospero::TileMode::kStandard64KB:
 		case Prospero::TileMode::kPrt:
 		case Prospero::TileMode::kDepth:

src/graphics/host_gpu/graphicContext.h

@@ -3,20 +3,88 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/threads.h"
 #include "graphics/host_gpu/vulkanCommon.h" // IWYU pragma: export
-#include "graphics/host_gpu/vulkanInstance.h"
 
+#include <map>
+#include <mutex>
+#include <tuple>
 #include <vector>
 #include <vk_mem_alloc.h>
 
 namespace Libs::Graphics {
 
-class CommandBuffer;
 struct VulkanBuffer;
 struct VulkanImage;
 struct VulkanMemory;
 
-struct GraphicContext: public VulkanInstance {
+inline constexpr uint32_t VULKAN_TARGET_API_VERSION = VK_API_VERSION_1_3;
+
+struct GraphicContext {
+	vk::Instance                       instance                              = nullptr;
+	vk::DebugUtilsMessengerEXT         debug_messenger                       = nullptr;
+	vk::PhysicalDevice                 physical_device                       = nullptr;
+	vk::PhysicalDeviceProperties       physical_device_properties            = {};
+	vk::PhysicalDeviceMemoryProperties physical_device_memory_properties     = {};
+	vk::Device                         device                                = nullptr;
+	VmaAllocator                       allocator                             = nullptr;
+	bool                               memory_budget_ext_enabled             = false;
+	bool                               rt_extensions_enabled                 = false;
+	bool                               compute_subgroup_size_control_enabled = false;
+	bool                               compute_wave64_supported              = false;
+	bool                               sample_rate_shading_enabled           = false;
+	uint32_t                           subgroup_size                         = 0;
+	uint32_t                           min_subgroup_size                     = 0;
+	uint32_t                           max_subgroup_size                     = 0;
+	uint32_t                           max_push_descriptors                  = 0;
+	vk::ShaderStageFlags               required_subgroup_size_stages         = {};
+	Common::Mutex                      queue_mutex;
+	uint32_t                           queue_family = static_cast<uint32_t>(-1);
+	vk::Queue                          queue        = nullptr;
+
+	[[nodiscard]] const vk::PhysicalDeviceProperties& GetPhysicalDeviceProperties() const {
+		return physical_device_properties;
+	}
+
+	[[nodiscard]] const vk::PhysicalDeviceMemoryProperties&
+	GetPhysicalDeviceMemoryProperties() const {
+		return physical_device_memory_properties;
+	}
+
+	[[nodiscard]] vk::FormatProperties GetFormatProperties(vk::Format format) const {
+		std::scoped_lock lock(m_format_properties_mutex);
+		auto [it, inserted] = m_format_properties.try_emplace(format);
+		if (inserted) {
+			physical_device.getFormatProperties(format, &it->second);
+		}
+		return it->second;
+	}
+
+	[[nodiscard]] vk::Result GetImageFormatProperties(vk::Format format, vk::ImageType type,
+	                                                  vk::ImageTiling            tiling,
+	                                                  vk::ImageUsageFlags        usage,
+	                                                  vk::ImageCreateFlags       flags,
+	                                                  vk::ImageFormatProperties* properties) const {
+		using Key = std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
+		                       vk::ImageCreateFlags>;
+		std::scoped_lock lock(m_image_format_properties_mutex);
+		auto [it, inserted] =
+		    m_image_format_properties.try_emplace(Key {format, type, tiling, usage, flags});
+		if (inserted) {
+			it->second.first = physical_device.getImageFormatProperties(format, type, tiling, usage,
+			                                                            flags, &it->second.second);
+		}
+		if (properties != nullptr) {
+			*properties = it->second.second;
+		}
+		return it->second.first;
+	}
+
+	[[nodiscard]] vk::DeviceSize StorageMinAlignment() const {
+		const auto alignment = physical_device_properties.limits.minStorageBufferOffsetAlignment;
+		return alignment != 0 ? alignment : 1;
+	}
+
 	[[nodiscard]] bool CreateAllocator();
 	void               DestroyAllocator();
 	void               LogMemoryBudget() const;
@@ -24,7 +92,6 @@ struct GraphicContext: public VulkanInstance {
 	[[nodiscard]] uint64_t GetDeviceMemoryUsage() const;
 	[[nodiscard]] uint64_t GetTotalMemoryBudget() const;
 	void                   CreateBuffer(uint64_t size, VulkanBuffer& buffer);
-	void                   DeleteBuffer(VulkanBuffer& buffer);
 	[[nodiscard]] bool     CreateImage(const vk::ImageCreateInfo& info, VulkanImage& image);
 	void                   DeleteImage(VulkanImage& image);
 	void                   MapMemory(VulkanMemory& memory, void*& data);
@@ -36,6 +103,15 @@ struct GraphicContext: public VulkanInstance {
 
 	uint32_t screen_width  = 0;
 	uint32_t screen_height = 0;
+
+private:
+	mutable std::mutex                                 m_format_properties_mutex;
+	mutable std::map<vk::Format, vk::FormatProperties> m_format_properties;
+	mutable std::mutex                                 m_image_format_properties_mutex;
+	mutable std::map<std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
+	                            vk::ImageCreateFlags>,
+	                 std::pair<vk::Result, vk::ImageFormatProperties>>
+	    m_image_format_properties;
 };
 
 struct VulkanMemory {

src/graphics/host_gpu/rangeSet.h

@@ -53,12 +53,30 @@ public:
 		}
 	}
 
+	void Clear() { m_ranges.clear(); }
+
+	template <typename Func>
+	void ForEach(Func&& func) const {
+		for (const auto& [begin, end]: m_ranges) {
+			func(begin, end);
+		}
+	}
+
 	[[nodiscard]] std::vector<Range> Intersections(uint64_t address, uint64_t size) const {
 		std::vector<Range> result;
 		ForEachIntersection(address, size, [&result](Range range) { result.push_back(range); });
 		return result;
 	}
 
+	[[nodiscard]] bool Intersects(uint64_t address, uint64_t size) const {
+		const auto end = End(address, size);
+		auto       it  = m_ranges.lower_bound(address);
+		if (it != m_ranges.begin() && std::prev(it)->second > address) {
+			return true;
+		}
+		return it != m_ranges.end() && it->first < end;
+	}
+
 	[[nodiscard]] bool Contains(uint64_t address, uint64_t size) const {
 		const auto end = End(address, size);
 		auto       it  = m_ranges.upper_bound(address);
@@ -89,7 +107,7 @@ public:
 
 private:
 	static uint64_t End(uint64_t address, uint64_t size) {
-		if (address == 0 || size == 0 || size > UINT64_MAX - address) {
+		if (size == 0 || size > UINT64_MAX - address) {
 			EXIT("invalid range-set address or size\n");
 		}
 		return address + size;

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -3,12 +3,15 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/lruCache.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/memoryTracker.h"
 #include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/renderer/cache/faultManager.h"
+#include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/cache/streamBuffer.h"
 
 #include <map>
-#include <memory>
 #include <span>
 #include <utility>
 #include <vector>
@@ -16,91 +19,95 @@
 namespace Libs::Graphics {
 
 struct GraphicContext;
-class CommandBuffer;
 class CommandScheduler;
 class TextureCache;
 
-struct BufferBinding {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	uint64_t              offset = 0;
-};
-
-struct ImageBufferSource {
-	Buffer*  buffer = nullptr;
-	uint64_t offset = 0;
-};
+using BufferId = Common::SlotId;
+inline constexpr BufferId NULL_BUFFER_ID {0};
 
 class BufferCache {
 public:
-	static constexpr uint64_t CACHING_PAGE_SIZE = 16ull * 1024ull;
-	static constexpr uint64_t GetBufferOffset(uint64_t vaddr) {
-		return vaddr & (CACHING_PAGE_SIZE - 1);
-	}
+	static constexpr uint32_t CACHING_PAGEBITS  = 14;
+	static constexpr uint64_t CACHING_PAGESIZE  = uint64_t {1} << CACHING_PAGEBITS;
+	static constexpr uint64_t CACHING_NUMPAGES  = uint64_t {1} << (40 - CACHING_PAGEBITS);
+	static constexpr uint64_t BDA_PAGETABLE_SIZE =
+	    CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
 
 	BufferCache(GraphicContext& graphics, CommandScheduler& scheduler, PageManager& page_manager,
 	            TextureCache& texture_cache);
 	~BufferCache();
 	KYTY_CLASS_NO_COPY(BufferCache);
 
-	void                        InvalidateMemory(uint64_t vaddr, uint64_t size);
-	void                        ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
-	void                        UnmapMemory(uint64_t vaddr, uint64_t size);
-	[[nodiscard]] BufferBinding ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
-	                                         bool is_written = false, bool is_read = true,
-	                                         bool is_formatted = false);
-	[[nodiscard]] StreamBuffer& GetUtilityBuffer(MemoryUsage usage) noexcept;
-	[[nodiscard]] Buffer&       GetGdsBuffer() noexcept { return m_gds_buffer; }
-	[[nodiscard]] const Buffer& GetGdsBuffer() const noexcept { return m_gds_buffer; }
-	[[nodiscard]] BufferBinding UploadTransient(const void* data, uint64_t size,
-	                                            uint64_t alignment);
-	[[nodiscard]] std::shared_ptr<Buffer> ObtainNullBuffer();
-	[[nodiscard]] ImageBufferSource       ObtainBufferForImage(uint64_t vaddr, uint64_t size);
-	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds = false);
-	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds = false,
-	                bool src_gds = false);
+	void                   InvalidateMemory(uint64_t vaddr, uint64_t size);
+	void                   ReadMemory(uint64_t vaddr, uint64_t size, bool is_write = false);
+	[[nodiscard]] Buffer&  GetBuffer(BufferId id) { return m_slot_buffers[id]; }
+	[[nodiscard]] BufferId FindBuffer(uint64_t vaddr, uint64_t size);
+	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBuffer(uint64_t vaddr, uint64_t size,
+	                                                        bool     is_written,
+	                                                        bool     is_texel_buffer = false,
+	                                                        BufferId id              = {});
+	[[nodiscard]] StreamBuffer&                GetUtilityBuffer(MemoryUsage usage) noexcept {
+		switch (usage) {
+			case MemoryUsage::Upload: return m_staging_buffer;
+			case MemoryUsage::Stream: return m_stream_buffer;
+			case MemoryUsage::Download: return m_download_buffer;
+			case MemoryUsage::DeviceLocal: return m_device_buffer;
+		}
+		EXIT("BufferCache: invalid utility-buffer usage\n");
+	}
+	[[nodiscard]] const Buffer* GetGdsBuffer() const noexcept { return &m_gds_buffer; }
+	[[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept { return &m_bda_pagetable_buffer; }
+	[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return m_fault_manager.GetFaultBuffer(); }
+	[[nodiscard]] std::pair<Buffer*, uint64_t> ObtainBufferForImage(uint64_t vaddr, uint64_t size);
+	void FillBuffer(uint64_t vaddr, uint64_t size, uint32_t value, bool is_gds);
+	void CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t size, bool dst_gds,
+	                bool src_gds);
 	// Cache-index and exact dirty-range queries require GPU-thread serialization.
 	[[nodiscard]] bool IsRegionRegistered(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool HasGpuDirtyBytes(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionCpuModified(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsRegionGpuModified(uint64_t vaddr, uint64_t size);
+	void               ProcessFaultBuffer();
+	void               SynchronizeBuffersInRange(uint64_t vaddr, uint64_t size);
 	void               RunGarbageCollector();
 
 private:
 	friend struct BufferCacheTestAccess;
 
-	struct CacheRange {
-		uint64_t address = 0;
-		uint64_t size    = 0;
-	};
-	struct CachedBuffer;
 	struct DownloadCopy;
-	struct DownloadRange;
-	struct RetiredBuffer;
+	using PageTable = MultiLevelPageTable<BufferId, CACHING_PAGEBITS, 40, 16>;
+	static_assert(CACHING_PAGESIZE == (uint64_t {1} << PageTable::kPageBits));
 	static constexpr uint64_t               DOWNLOAD_ALIGNMENT = 64;
-	[[nodiscard]] static uint64_t           AlignDown(uint64_t value) noexcept;
-	[[nodiscard]] static uint64_t           AlignUp(uint64_t value);
 	[[nodiscard]] static constexpr uint64_t AlignDownload(uint64_t size) noexcept {
 		return (size + DOWNLOAD_ALIGNMENT - 1) & ~(DOWNLOAD_ALIGNMENT - 1);
 	}
 	[[nodiscard]] static std::pair<uint64_t, uint64_t> DownloadEnvelope(const DownloadCopy& copy);
-	[[nodiscard]] static bool ResolveOverlap(CacheRange& merged, CacheRange candidate) noexcept;
-	void Upload(CommandBuffer& command, Buffer& destination, uint64_t destination_offset,
-	            const void* source, uint64_t size);
-	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
-	                                              uint64_t size);
+	void WriteDataBuffer(Buffer& buffer, uint64_t address, const void* source, uint64_t size);
+	void TouchBuffer(const Buffer& buffer);
+	[[nodiscard]] BufferId CreateBuffer(uint64_t vaddr, uint64_t size);
+	void                   Register(BufferId id);
+	void Unregister(BufferId id);
+	template <bool insert>
+	void ChangeRegister(BufferId id);
+	void DeleteBuffer(BufferId id);
+	[[nodiscard]] bool SynchronizeBuffer(Buffer& buffer, uint64_t vaddr, uint64_t size,
+	                                     bool is_written, bool is_texel_buffer);
+	[[nodiscard]] vk::Buffer UploadCopies(Buffer& buffer, std::span<vk::BufferCopy> copies,
+	                                      uint64_t total_size);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
-	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
-	void PublishDownloads(std::span<const DownloadRange> downloads);
-	void QueueGarbageDownload(std::span<const DownloadCopy> copies, RetiredBuffer retire);
+	void DownloadBufferMemory(std::span<const DownloadCopy> copies);
 	void WriteHostMemory(uint64_t vaddr, std::span<const uint8_t> data);
 	void ReadMemoryOnGpu(uint64_t vaddr, uint64_t size, bool is_write);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
+	FaultManager                                      m_fault_manager;
 	Buffer                                            m_gds_buffer;
-	std::shared_ptr<Buffer>                           m_null_buffer;
-	std::map<uint64_t, std::unique_ptr<CachedBuffer>> m_buffers;
+	Buffer                                            m_bda_pagetable_buffer;
+	Common::SlotVector<Buffer>                        m_slot_buffers;
+	Common::LeastRecentlyUsedCache<BufferId, uint64_t> m_lru_cache;
+	std::map<uint64_t, BufferId>                      m_buffers;
+	PageTable                                         m_page_table;
 	RangeSet                                          m_gpu_modified_ranges;
 	MemoryTracker                                     m_memory_tracker;
 	StreamBuffer                                      m_staging_buffer;
@@ -108,7 +115,7 @@ private:
 	StreamBuffer                                      m_download_buffer;
 	StreamBuffer                                      m_device_buffer;
 	TextureCache&                                     m_texture_cache;
-	uint64_t                                          m_total_used_memory = 0;
+	uint64_t                                          m_total_used_memory  = 0;
 	uint64_t m_trigger_gc_memory  = 1ull * 1024 * 1024 * 1024;
 	uint64_t m_critical_gc_memory = 2ull * 1024 * 1024 * 1024;
 	uint64_t m_gc_tick            = 0;

src/graphics/host_gpu/renderer/cache/faultManager.cpp

@@ -0,0 +1,167 @@
+#include "graphics/host_gpu/renderer/cache/faultManager.h"
+
+#include "common/assert.h"
+#include "common/logging/log.h"
+#include "gpu_tiler_shaders/fault_buffer_process_spv.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/commandScheduler.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+
+#include <bit>
+#include <cinttypes>
+#include <cstring>
+#include <limits>
+
+namespace Libs::Graphics {
+
+namespace {
+
+constexpr size_t MaxPageFaults    = 1024;
+constexpr size_t PageFaultAreaSize = MaxPageFaults * sizeof(uint64_t);
+
+} // namespace
+
+FaultManager::FaultManager(GraphicContext& graphics, CommandScheduler& scheduler,
+                           BufferCache& buffer_cache, uint32_t caching_pagebits,
+                           uint64_t caching_num_pages)
+    : m_graphics(graphics), m_scheduler(scheduler), m_buffer_cache(buffer_cache),
+      m_caching_pagesize(uint64_t {1} << caching_pagebits),
+      m_caching_num_pages(caching_num_pages), m_fault_buffer_size(caching_num_pages / 8),
+      m_fault_buffer(graphics, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags,
+                     m_fault_buffer_size),
+      m_download_buffer(graphics, scheduler, MemoryUsage::Download, 0, AllFlags,
+                        MaxPendingFaults * PageFaultAreaSize) {
+	SetVulkanObjectNameF(m_graphics.device, m_fault_buffer.Handle(), "Fault Buffer");
+
+	const vk::DescriptorSetLayoutBinding bindings[] {
+	    {0, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
+	    {1, vk::DescriptorType::eStorageBuffer, 1, vk::ShaderStageFlagBits::eCompute, nullptr},
+	};
+	vk::DescriptorSetLayoutCreateInfo layout_info {};
+	layout_info.sType        = vk::StructureType::eDescriptorSetLayoutCreateInfo;
+	layout_info.flags        = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR;
+	layout_info.bindingCount = std::size(bindings);
+	layout_info.pBindings    = bindings;
+	RequireVulkanSuccess(
+	    m_graphics.device.createDescriptorSetLayout(&layout_info, nullptr,
+	                                                &m_fault_process_desc_layout),
+	    "create fault-buffer descriptor layout");
+
+	vk::ShaderModuleCreateInfo module_info {};
+	module_info.sType    = vk::StructureType::eShaderModuleCreateInfo;
+	module_info.codeSize = std::size(FAULT_BUFFER_PROCESS_SPV) * sizeof(uint32_t);
+	module_info.pCode    = FAULT_BUFFER_PROCESS_SPV;
+	vk::ShaderModule module = nullptr;
+	RequireVulkanSuccess(m_graphics.device.createShaderModule(&module_info, nullptr, &module),
+	                     "create fault-buffer shader module");
+
+	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
+	pipeline_layout_info.sType          = vk::StructureType::ePipelineLayoutCreateInfo;
+	pipeline_layout_info.setLayoutCount = 1;
+	pipeline_layout_info.pSetLayouts    = &m_fault_process_desc_layout;
+	RequireVulkanSuccess(
+	    m_graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
+	                                           &m_fault_process_pipeline_layout),
+	    "create fault-buffer pipeline layout");
+
+	vk::PipelineShaderStageCreateInfo stage {};
+	stage.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
+	stage.stage  = vk::ShaderStageFlagBits::eCompute;
+	stage.module = module;
+	stage.pName  = "main";
+	vk::ComputePipelineCreateInfo pipeline_info {};
+	pipeline_info.sType  = vk::StructureType::eComputePipelineCreateInfo;
+	pipeline_info.stage  = stage;
+	pipeline_info.layout = m_fault_process_pipeline_layout;
+	const auto result = m_graphics.device.createComputePipelines(
+	    nullptr, 1, &pipeline_info, nullptr, &m_fault_process_pipeline);
+	m_graphics.device.destroyShaderModule(module, nullptr);
+	RequireVulkanSuccess(result, "create fault-buffer pipeline");
+	SetVulkanObjectNameF(m_graphics.device, m_fault_process_pipeline, "Fault Buffer Parser");
+}
+
+FaultManager::~FaultManager() {
+	m_graphics.device.destroyPipeline(m_fault_process_pipeline, nullptr);
+	m_graphics.device.destroyPipelineLayout(m_fault_process_pipeline_layout, nullptr);
+	m_graphics.device.destroyDescriptorSetLayout(m_fault_process_desc_layout, nullptr);
+}
+
+void FaultManager::ProcessFaultBuffer() {
+	if (const auto wait_tick = m_fault_areas[m_current_area]; wait_tick != 0) {
+		m_scheduler.Wait(wait_tick);
+		m_scheduler.PopPendingOperations();
+	}
+
+	const auto offset = m_current_area * PageFaultAreaSize;
+	auto*      mapped = m_download_buffer.Mapped().data() + offset;
+	std::memset(mapped, 0, PageFaultAreaSize);
+	m_download_buffer.Flush(offset, PageFaultAreaSize);
+
+	vk::BufferMemoryBarrier2 pre_barrier {};
+	pre_barrier.sType         = vk::StructureType::eBufferMemoryBarrier2;
+	pre_barrier.srcStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
+	pre_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
+	pre_barrier.dstStageMask  = vk::PipelineStageFlagBits2::eComputeShader;
+	pre_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderRead;
+	pre_barrier.buffer        = m_fault_buffer.Handle();
+	pre_barrier.offset        = 0;
+	pre_barrier.size          = m_fault_buffer_size;
+	auto post_barrier         = pre_barrier;
+	post_barrier.srcStageMask  = vk::PipelineStageFlagBits2::eComputeShader;
+	post_barrier.srcAccessMask = vk::AccessFlagBits2::eShaderWrite;
+	post_barrier.dstStageMask  = vk::PipelineStageFlagBits2::eAllCommands;
+	post_barrier.dstAccessMask = vk::AccessFlagBits2::eShaderWrite;
+
+	const vk::DescriptorBufferInfo infos[] {
+	    {m_fault_buffer.Handle(), 0, m_fault_buffer_size},
+	    {m_download_buffer.Handle(), offset, PageFaultAreaSize},
+	};
+	std::array<vk::WriteDescriptorSet, 2> writes {};
+	for (uint32_t index = 0; index < writes.size(); ++index) {
+		writes[index].sType           = vk::StructureType::eWriteDescriptorSet;
+		writes[index].dstBinding      = index;
+		writes[index].descriptorCount = 1;
+		writes[index].descriptorType  = vk::DescriptorType::eStorageBuffer;
+		writes[index].pBufferInfo     = &infos[index];
+	}
+
+	m_scheduler.EndRendering();
+	auto command = m_scheduler.Current().Handle();
+	vk::DependencyInfo dependency {};
+	dependency.sType                    = vk::StructureType::eDependencyInfo;
+	dependency.dependencyFlags          = vk::DependencyFlagBits::eByRegion;
+	dependency.bufferMemoryBarrierCount = 1;
+	dependency.pBufferMemoryBarriers    = &pre_barrier;
+	command.pipelineBarrier2(dependency);
+	command.bindPipeline(vk::PipelineBindPoint::eCompute, m_fault_process_pipeline);
+	command.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute,
+	                             m_fault_process_pipeline_layout, 0, writes);
+	const auto num_threads    = m_caching_num_pages / 32;
+	const auto num_workgroups = (num_threads + 63) / 64;
+	command.dispatch(static_cast<uint32_t>(num_workgroups), 1, 1);
+	dependency.pBufferMemoryBarriers = &post_barrier;
+	command.pipelineBarrier2(dependency);
+
+	const auto area = m_current_area;
+	m_scheduler.DeferOperation([this, mapped, offset, area] {
+		m_download_buffer.Invalidate(offset, PageFaultAreaSize);
+		m_fault_ranges.Clear();
+		const auto* faults = std::bit_cast<const uint64_t*>(mapped);
+		const auto  count  = static_cast<uint32_t>(faults[0]);
+		for (uint32_t index = 1; index <= count; ++index) {
+			m_fault_ranges.Add(faults[index], m_caching_pagesize);
+			LOGF("Accessed non-GPU cached memory at 0x%016" PRIx64 "\n", faults[index]);
+		}
+		m_fault_ranges.ForEach([this](uint64_t start, uint64_t end) {
+			EXIT_IF(end - start > std::numeric_limits<uint32_t>::max());
+			(void)m_buffer_cache.FindBuffer(start, end - start);
+		});
+		m_fault_areas[area] = 0;
+	});
+
+	m_fault_areas[m_current_area++] = m_scheduler.CurrentTick();
+	m_current_area %= MaxPendingFaults;
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/faultManager.h

@@ -0,0 +1,46 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_
+
+#include "common/abi.h"
+#include "graphics/host_gpu/rangeSet.h"
+#include "graphics/host_gpu/renderer/cache/streamBuffer.h"
+
+#include <array>
+#include <cstdint>
+
+namespace Libs::Graphics {
+
+class BufferCache;
+
+class FaultManager {
+	static constexpr size_t MaxPendingFaults = 8;
+
+public:
+	FaultManager(GraphicContext& graphics, CommandScheduler& scheduler, BufferCache& buffer_cache,
+	             uint32_t caching_pagebits, uint64_t caching_num_pages);
+	~FaultManager();
+	KYTY_CLASS_NO_COPY(FaultManager);
+
+	[[nodiscard]] Buffer* GetFaultBuffer() noexcept { return &m_fault_buffer; }
+	void                  ProcessFaultBuffer();
+
+private:
+	GraphicContext&                            m_graphics;
+	CommandScheduler&                          m_scheduler;
+	BufferCache&                               m_buffer_cache;
+	RangeSet                                   m_fault_ranges;
+	uint64_t                                   m_caching_pagesize;
+	uint64_t                                   m_caching_num_pages;
+	uint64_t                                   m_fault_buffer_size;
+	Buffer                                     m_fault_buffer;
+	Buffer                                     m_download_buffer;
+	std::array<uint64_t, MaxPendingFaults>      m_fault_areas {};
+	uint32_t                                   m_current_area = 0;
+	vk::DescriptorSetLayout                    m_fault_process_desc_layout = nullptr;
+	vk::Pipeline                               m_fault_process_pipeline = nullptr;
+	vk::PipelineLayout                         m_fault_process_pipeline_layout = nullptr;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_CACHE_FAULTMANAGER_H_

src/graphics/host_gpu/renderer/cache/gpuResourceManager.cpp

@@ -60,10 +60,10 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	const auto unmap = [this, vaddr, size] {
 		if (m_scheduler.Active()) {
 			const auto tick = m_scheduler.CurrentTick();
-			m_scheduler.FinishCurrent();
+			m_scheduler.Finish();
 			m_scheduler.WaitPriorityOperations(tick);
 		}
-		m_buffer_cache.UnmapMemory(vaddr, size);
+		m_buffer_cache.InvalidateMemory(vaddr, size);
 		m_texture_cache.UnmapMemory(vaddr, size);
 		m_page_manager.OnGpuUnmap(vaddr, size);
 		std::lock_guard lock(m_mapped_ranges_mutex);
@@ -76,7 +76,19 @@ void GpuResourceManager::UnmapMemory(uint64_t vaddr, uint64_t size) {
 	m_gpu->SendCommandSync(unmap);
 }
 
+void GpuResourceManager::PrepareBda() {
+	std::shared_lock lock(m_mapped_ranges_mutex);
+	m_mapped_ranges.ForEach([this](uint64_t start, uint64_t end) {
+		m_buffer_cache.SynchronizeBuffersInRange(start, end - start);
+	});
+	m_fault_process_pending = true;
+}
+
 void GpuResourceManager::RunGarbageCollector() {
+	if (m_fault_process_pending) {
+		m_fault_process_pending = false;
+		m_buffer_cache.ProcessFaultBuffer();
+	}
 	m_texture_cache.ProcessDownloadImages();
 	m_texture_cache.RunGarbageCollector();
 	m_buffer_cache.RunGarbageCollector();

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -13,7 +13,7 @@
 namespace Libs::Graphics {
 
 class CommandScheduler;
-class Gpu;
+class GuestGpu;
 
 class GpuResourceManager {
 public:
@@ -23,13 +23,14 @@ public:
 
 	[[nodiscard]] BufferCache&  GetBufferCache() { return m_buffer_cache; }
 	[[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; }
-	void                        SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
+	void                        SetGpu(GuestGpu* gpu) noexcept { m_gpu = gpu; }
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
 	[[nodiscard]] bool IsMapped(uint64_t vaddr, uint64_t size) const noexcept;
 	void               MapMemory(uint64_t vaddr, uint64_t size);
 	void               UnmapMemory(uint64_t vaddr, uint64_t size);
+	void               PrepareBda();
 	void               RunGarbageCollector();
 
 private:
@@ -39,7 +40,8 @@ private:
 	TextureCache              m_texture_cache;
 	mutable std::shared_mutex m_mapped_ranges_mutex;
 	RangeSet                  m_mapped_ranges;
-	Gpu*                      m_gpu = nullptr;
+	GuestGpu*                 m_gpu = nullptr;
+	bool                      m_fault_process_pending = false;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/streamBuffer.cpp

@@ -73,8 +73,12 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
 	buffer_info.usage       = flags;
 	buffer_info.sharingMode = vk::SharingMode::eExclusive;
 
+	const bool with_bda = bool(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress);
+	const VmaAllocationCreateFlags bda_flag =
+	    with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
 	VmaAllocationCreateInfo allocation_info {};
-	allocation_info.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | AllocationFlags(usage);
+	allocation_info.flags =
+	    VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | AllocationFlags(usage);
 	allocation_info.usage = AllocationUsage(usage);
 	allocation_info.preferredFlags = usage == MemoryUsage::DeviceLocal
 	                                     ? VkMemoryPropertyFlags {}
@@ -99,6 +103,13 @@ Buffer::Buffer(GraphicContext& graphics, CommandScheduler& scheduler, MemoryUsag
 	m_buffer->memory.type            = allocation_result.memoryType;
 	m_buffer->memory.unique_id       = VulkanNextMemoryUniqueId();
 	graphics.device.getBufferMemoryRequirements(m_buffer->buffer, &m_buffer->memory.requirements);
+	if (static_cast<bool>(flags & vk::BufferUsageFlagBits::eShaderDeviceAddress)) {
+		vk::BufferDeviceAddressInfo address_info {};
+		address_info.sType  = vk::StructureType::eBufferDeviceAddressInfo;
+		address_info.buffer = m_buffer->buffer;
+		m_device_address    = graphics.device.getBufferAddress(address_info);
+		EXIT_IF(m_device_address == 0);
+	}
 
 	VkMemoryPropertyFlags properties = 0;
 	vmaGetAllocationMemoryProperties(graphics.allocator, m_buffer->memory.allocation, &properties);
@@ -122,6 +133,11 @@ vk::Buffer Buffer::Handle() const noexcept {
 	return m_buffer->buffer;
 }
 
+vk::DeviceAddress Buffer::BufferDeviceAddress() const noexcept {
+	EXIT_IF(m_device_address == 0);
+	return m_device_address;
+}
+
 bool Buffer::IsInBounds(uint64_t address, uint64_t size) const noexcept {
 	return address >= m_cpu_address && size <= m_size && address - m_cpu_address <= m_size - size;
 }
@@ -141,6 +157,15 @@ void Buffer::Flush(uint64_t offset, uint64_t size) {
 	}
 }
 
+void Buffer::Invalidate(uint64_t offset, uint64_t size) {
+	EXIT_IF(m_usage != MemoryUsage::Download || offset > m_size || size > m_size - offset);
+	if (!m_is_coherent && size != 0) {
+		const auto result =
+		    vmaInvalidateAllocation(m_graphics->allocator, m_buffer->memory.allocation, offset, size);
+		EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
+	}
+}
+
 vk::BufferMemoryBarrier Buffer::Barrier(uint64_t offset, uint64_t size, vk::AccessFlags source,
                                         vk::AccessFlags destination) const {
 	if (Handle() == nullptr || size == 0 || offset > m_size || size > m_size - offset) {
@@ -310,16 +335,6 @@ void StreamBuffer::Commit() {
 	watch.tick        = tick;
 }
 
-void StreamBuffer::Invalidate(uint64_t offset, uint64_t size) {
-	EXIT_IF(Usage() != MemoryUsage::Download || offset > Size() || size > Size() - offset);
-	if (IsCoherent() || size == 0) {
-		return;
-	}
-	const auto result = vmaInvalidateAllocation(Graphics().allocator,
-	                                            NativeBuffer().memory.allocation, offset, size);
-	EXIT_NOT_IMPLEMENTED(static_cast<vk::Result>(result) != vk::Result::eSuccess);
-}
-
 uint64_t StreamBuffer::Copy(const void* source, uint64_t size, uint64_t alignment) {
 	EXIT_IF(source == nullptr);
 	const auto [data, offset] = Map(size, alignment);

src/graphics/host_gpu/renderer/cache/streamBuffer.h

@@ -48,12 +48,14 @@ public:
 	[[nodiscard]] bool               IsCoherent() const noexcept { return m_is_coherent; }
 	[[nodiscard]] MemoryUsage        Usage() const noexcept { return m_usage; }
 	[[nodiscard]] uint64_t           CpuAddress() const noexcept { return m_cpu_address; }
+	[[nodiscard]] vk::DeviceAddress BufferDeviceAddress() const noexcept;
 	[[nodiscard]] uint64_t           Offset(uint64_t address) const noexcept {
 		return address - m_cpu_address;
 	}
 	[[nodiscard]] bool IsInBounds(uint64_t address, uint64_t size) const noexcept;
 	void               Write(uint64_t offset, const void* source, uint64_t size);
 	void               Flush(uint64_t offset, uint64_t size);
+	void               Invalidate(uint64_t offset, uint64_t size);
 	void CopyFrom(CommandBuffer& command, const Buffer& source, uint64_t source_offset,
 	              uint64_t destination_offset, uint64_t size,
 	              vk::AccessFlags source_before      = vk::AccessFlagBits::eMemoryWrite,
@@ -65,6 +67,10 @@ public:
 	                                                   vk::AccessFlagBits::eMemoryWrite);
 	void Fill(uint64_t offset, uint64_t size, uint32_t value);
 
+	// BufferCache state lives directly on the resource.
+	bool   is_deleted = false;
+	size_t lru_id     = 0;
+
 protected:
 	[[nodiscard]] GraphicContext&   Graphics() const noexcept { return *m_graphics; }
 	[[nodiscard]] CommandScheduler& Scheduler() const noexcept { return *m_scheduler; }
@@ -80,6 +86,7 @@ private:
 	MemoryUsage                   m_usage       = MemoryUsage::DeviceLocal;
 	uint64_t                      m_cpu_address = 0;
 	uint64_t                      m_size        = 0;
+	vk::DeviceAddress             m_device_address = 0;
 	std::unique_ptr<VulkanBuffer> m_buffer;
 	std::span<uint8_t>            m_mapped;
 	bool                          m_is_coherent = false;
@@ -93,9 +100,6 @@ public:
 	[[nodiscard]] std::pair<uint8_t*, uint64_t> Map(uint64_t size, uint64_t alignment = 0,
 	                                                bool allow_wait = true);
 	void                                        Commit();
-	// Download mappings become visible to the CPU only after their GPU completion tick is free.
-	// Call this from the scheduler's deferred completion operation before reading Mapped().
-	void                   Invalidate(uint64_t offset, uint64_t size);
 	[[nodiscard]] uint64_t Copy(const void* source, uint64_t size, uint64_t alignment = 0);
 
 private:

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -4,19 +4,17 @@
 #include "common/abi.h"
 #include "common/common.h"
 #include "common/lruCache.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/pageManager.h"
 #include "graphics/host_gpu/regionManager.h"
 #include "graphics/host_gpu/renderer/cache/multiLevelPageTable.h"
 #include "graphics/host_gpu/renderer/image/blitHelper.h"
 #include "graphics/host_gpu/renderer/image/image.h"
+#include "graphics/host_gpu/renderer/image/tiler.h"
 
-#include <compare>
 #include <map>
-#include <memory>
-#include <optional>
-#include <set>
-#include <span>
-#include <utility>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -27,8 +25,6 @@ class BufferCache;
 class CommandBuffer;
 class CommandScheduler;
 class RenderExecutor;
-class StreamBuffer;
-class TileManager;
 struct TextureCacheTestAccess;
 
 class TextureCache {
@@ -59,20 +55,26 @@ public:
 	[[nodiscard]] vk::ImageView FindTexture(ImageId id, const ImageDesc& desc);
 	[[nodiscard]] vk::ImageView FindRenderTarget(ImageId id, const ImageDesc& desc);
 	[[nodiscard]] vk::ImageView FindDepthTarget(ImageId id, const ImageDesc& desc);
-	[[nodiscard]] Image&        GetImage(ImageId id);
-	[[nodiscard]] const Image&  GetImage(ImageId id) const;
-	void                        MarkGpuWritten(ImageId id);
+	[[nodiscard]] Image&        GetImage(ImageId id) {
+		auto& image = m_slot_images[id];
+		TouchImage(image);
+		return image;
+	}
+	void MarkGpuWritten(ImageId id);
 
 	[[nodiscard]] bool ClearImageFromBuffer(CommandBuffer& command, uint64_t address, uint64_t size,
 	                                        uint32_t packed_clear);
 	void               InvalidateMemory(uint64_t address, uint64_t size);
-	[[nodiscard]] bool InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
-	                                           bool formatted_buffer_write = false);
+	void               InvalidateMemoryFromGPU(uint64_t address, uint64_t size);
 	[[nodiscard]] RegionInfo QueryRegion(uint64_t address, uint64_t size);
 
 	[[nodiscard]] bool IsMeta(uint64_t address);
-	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice);
+	[[nodiscard]] bool IsMetaCleared(uint64_t address, uint32_t slice,
+	                                 uint32_t* fill_value = nullptr);
 	[[nodiscard]] bool ClearMeta(uint64_t address);
+	// Returns true when registered DCC absorbed the fill and the caller may skip the dispatch.
+	// False may still record PendingDcc state, but the guest dispatch must execute.
+	[[nodiscard]] bool TryConsumeDccFill(uint64_t address, uint64_t size, uint32_t fill_value);
 	[[nodiscard]] bool TouchMeta(uint64_t address, uint32_t slice, bool is_clear);
 
 	void UnmapMemory(uint64_t address, uint64_t size);
@@ -84,13 +86,18 @@ private:
 	struct ColorTransferPlan;
 	struct DownloadPlan;
 
-	struct Slot {
-		std::shared_ptr<Image> image;
-		uint32_t               generation = 1;
-	};
-
 	struct MetaDataInfo {
+		// A guest metadata-fill dispatch may initialize DCC before its render target is bound.
+		// PendingDcc retains that exact fill until FindRenderTarget classifies the address,
+		// without exposing an unconfirmed buffer address to the normal metadata heuristics.
+		// Keep all surface metadata in one entry so CMask/FMask can be
+		// registered beside HTile and DCC without introducing parallel tracking paths.
+		enum class Type : uint8_t { PendingDcc, CMask, FMask, HTile, Dcc };
+
+		Type     type       = Type::PendingDcc;
 		uint32_t clear_mask = 0;
+		uint32_t fill_value = 0xffffffffu;
+		uint64_t fill_size  = 0;
 	};
 
 	struct OverlapResult {
@@ -102,25 +109,21 @@ private:
 	using ImageIds       = InlinePageOwnerList<ImageId, 16>;
 	using ImagePageTable = MultiLevelPageTable<ImageIds, 20, 40, 10>;
 
-	[[nodiscard]] Image&                 ResolveImage(ImageId id);
-	[[nodiscard]] const Image&           ResolveImage(ImageId id) const;
-	[[nodiscard]] std::shared_ptr<Image> ResolveOwner(ImageId id) const;
-	[[nodiscard]] ImageId                InsertImage(const ImageInfo& info);
-	[[nodiscard]] ImageId                GetNullImage(const ImageDesc& desc);
-	void                                 RegisterImage(ImageId id);
-	void                                 UnregisterImage(ImageId id);
-	void                                 DeleteImage(ImageId id);
-	void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
-	void RetainImage(CommandBuffer& command, ImageId id);
-	void TouchImage(Image& image);
-	void TrackImage(ImageId id);
-	void TrackImageHead(ImageId id);
-	void TrackImageTail(ImageId id);
-	void UntrackImage(ImageId id);
-	void UntrackImageHead(ImageId id);
-	void UntrackImageTail(ImageId id);
-	void TrackImageDownload(ImageId id);
-	void TrackImageDownloadLocked(ImageId id, Image& image);
+	[[nodiscard]] ImageId     InsertImage(const ImageInfo& info);
+	[[nodiscard]] ImageId     GetNullImage(const ImageDesc& desc);
+	void                      RegisterImage(ImageId id);
+	void                      UnregisterImage(ImageId id);
+	void                      DeleteImage(ImageId id);
+	void                      FreeImage(ImageId id);
+	void                      TouchImage(Image& image);
+	void                      TrackImage(ImageId id);
+	void                      TrackImageHead(ImageId id);
+	void                      TrackImageTail(ImageId id);
+	void                      UntrackImage(ImageId id);
+	void                      UntrackImageHead(ImageId id);
+	void                      UntrackImageTail(ImageId id);
+	void                      MarkAsMaybeDirty(ImageId id, Image& image);
+	void                      TrackImageDownload(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
 	                                      bool exact_format);
 	[[nodiscard]] static BindingType UploadBinding(const Image& image);
@@ -149,31 +152,24 @@ private:
 	[[nodiscard]] bool CopyD16(Image& destination, Image& source);
 	void               CopyImage(ImageId destination, ImageId source);
 	void               AssociateStencil(ImageId depth, GuestRange stencil);
-	void               AssociateStencilLocked(ImageId depth, GuestRange stencil);
 	void CopyImageMip(ImageId destination, ImageId source, uint32_t mip, uint32_t layer);
 	void ValidateImageDesc(const ImageDesc& desc) const;
 
-	void InvalidateCpuAliases(uint64_t address, uint64_t size);
-	void ClearGpuModified(ImageId id);
-
-	void                                        DownloadImage(ImageId id);
-	[[nodiscard]] bool                          TryDownloadImage(ImageId id);
-	[[nodiscard]] std::pair<uint8_t*, uint64_t> MapDownload(uint64_t size, uint64_t alignment);
-	void QueueDownload(GuestRange range, StreamBuffer& download, uint8_t* mapped, uint64_t offset);
+	void               InvalidateCpuAliases(uint64_t address, uint64_t size);
+	[[nodiscard]] bool TryDownloadImage(ImageId id);
 
 	GraphicContext&                                   m_graphics;
 	CommandScheduler&                                 m_scheduler;
 	TrackingSpinLock                                  m_lock;
 	PageManager&                                      m_page_manager;
 	BlitHelper                                        m_blit_helper;
-	std::unique_ptr<TileManager>                      m_tiler;
+	TileManager                                       m_tiler;
 	BufferCache&                                      m_buffer_cache;
-	std::vector<Slot>                                 m_slots;
-	std::vector<uint32_t>                             m_free_slots;
+	Common::SlotVector<Image>                         m_slot_images;
 	ImagePageTable                                    m_image_page_table;
-	std::map<vk::Format, ImageId>                     m_null_images;
+	std::unordered_map<vk::Format, ImageId>           m_null_images;
 	Common::LeastRecentlyUsedCache<ImageId, uint64_t> m_lru_cache;
-	std::set<ImageId>                                 m_download_images;
+	std::unordered_set<ImageId>                       m_download_images;
 	std::map<uint64_t, MetaDataInfo>                  m_surface_metas;
 	uint64_t                                          m_total_used_memory  = 0;
 	uint64_t                                          m_trigger_gc_memory  = 0;

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -9,20 +9,42 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <algorithm>
+#include <array>
 #include <atomic>
 
 namespace Libs::Graphics {
 
 static std::atomic<uint32_t> g_render_color_log_count = 0;
 
+static void ResolveDccClearInfo(RenderColorInfo& info, vk::Format format, bool has_dcc,
+                                uint32_t packed_clear) {
+	// Register-backed DCC clears use the target's packed clear value. Decode one-word guest
+	// formats here; unsupported encodings remain tracked without unsafe materialization.
+	info.metadata_clear_supported =
+	    has_dcc && DecodePackedColorClear(format, packed_clear, info.color_clear_value);
+	switch (format) {
+		case vk::Format::eR8G8B8A8Unorm:
+		case vk::Format::eR8G8B8A8Srgb:
+		case vk::Format::eB8G8R8A8Unorm:
+		case vk::Format::eB8G8R8A8Srgb:
+		case vk::Format::eA2B10G10R10UnormPack32:
+		case vk::Format::eA2R10G10B10UnormPack32:
+			info.metadata_fixed_clear_supported = has_dcc;
+			break;
+		default: info.metadata_fixed_clear_supported = false; break;
+	}
+	if (!info.metadata_clear_supported) {
+		info.color_clear_value = {};
+	}
+}
+
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer& buffer,
                                               RenderColorInfo& r,
                                               uint32_t         render_target_slice_offset,
                                               uint32_t render_target_slot, bool ignore_target_mask,
@@ -55,20 +77,22 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		}
 
 		// No color output
-		r.type               = RenderColorType::NoColorOutput;
-		r.desc               = {};
-		r.base_addr          = 0;
-		r.image_id           = {};
-		r.image_view         = nullptr;
-		r.format             = vk::Format::eUndefined;
-		r.extent             = {};
-		r.base_mip_level     = 0;
-		r.base_array_layer   = 0;
-		r.buffer_size        = 0;
-		r.samples            = 1;
-		r.export_mapping     = {};
-		r.color_clear_enable = false;
-		r.color_clear_value  = {};
+		r.type                           = RenderColorType::NoColorOutput;
+		r.desc                           = {};
+		r.base_addr                      = 0;
+		r.image_id                       = {};
+		r.image_view                     = nullptr;
+		r.format                         = vk::Format::eUndefined;
+		r.extent                         = {};
+		r.base_mip_level                 = 0;
+		r.base_array_layer               = 0;
+		r.buffer_size                    = 0;
+		r.samples                        = 1;
+		r.export_mapping                 = {};
+		r.color_clear_enable             = false;
+		r.metadata_clear_supported       = false;
+		r.metadata_fixed_clear_supported = false;
+		r.color_clear_value              = {};
 		return;
 	}
 	const auto samples = render_sample_count(rt.attrib.num_fragments);
@@ -107,9 +131,9 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		}
 	}
 
-	// CB_COLOR_CONTROL describes the color-buffer operation / ROP3 logic op.
-	// ROP3 Copy is the normal color write path, not a render-target clear.
-	// SRGB clear words are still encoded as normalized component values.
+	// Color-control state selects the color-buffer operation and logical blend operation.
+	// The normal copy operation is a regular color write, not an attachment clear.
+	// Nonlinear clear values are still stored as normalized components.
 	// Fast color clears are metadata driven and must be handled explicitly when
 	// that metadata path is implemented; render-pass load must preserve contents.
 	r.color_clear_enable = false;
@@ -120,22 +144,38 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	uint32_t   pitch  = 0;
 	uint64_t   size   = 0;
 	bool       tile   = false;
-	const bool volume = rt.attrib3.dimension == 2;
-	if (rt.attrib3.dimension != 1 && !volume) {
+	static constexpr std::array image_types {Prospero::ImageType::kColor1D,
+	                                         Prospero::ImageType::kColor2D,
+	                                         Prospero::ImageType::kColor3D};
+	if (rt.attrib3.dimension >= image_types.size()) {
 		EXIT("unsupported render-target dimension: %u\n", rt.attrib3.dimension);
 	}
+	const auto image_type = image_types[rt.attrib3.dimension];
+	const bool is_1d      = image_type == Prospero::ImageType::kColor1D;
+	const bool volume     = image_type == Prospero::ImageType::kColor3D;
+	if (is_1d && rt.attrib2.height != 0) {
+		EXIT("1D render target has nonzero height: %u\n", rt.attrib2.height);
+	}
 	if (!volume && rt.attrib3.depth != 0) {
-		EXIT("2D render target has nonzero depth: %u\n", rt.attrib3.depth);
+		EXIT("non-3D render target has nonzero depth: %u\n", rt.attrib3.depth);
+	}
+	if (is_1d && samples != 1) {
+		EXIT("multisampled 1D render targets are unsupported\n");
 	}
 	if (volume && samples != 1) {
 		EXIT("multisampled 3D render targets are unsupported\n");
 	}
-	const uint32_t depth      = volume ? rt.attrib3.depth + 1u : 1u;
-	const bool     standard64 = rt.attrib3.tile_mode == Prospero::TileMode::kStandard64KB;
+	const uint32_t depth        = volume ? rt.attrib3.depth + 1u : 1u;
+	const bool     standard4    = rt.attrib3.tile_mode == Prospero::TileMode::kStandard4KB;
+	const bool     standard64   = rt.attrib3.tile_mode == Prospero::TileMode::kStandard64KB;
+	const bool     depth_tile   = rt.attrib3.tile_mode == Prospero::TileMode::kDepth;
+	const bool     texture_tile = standard4 || standard64 || depth_tile;
 
 	switch (rt.attrib3.tile_mode) {
 		case Prospero::TileMode::kLinear:
+		case Prospero::TileMode::kStandard4KB:
 		case Prospero::TileMode::kStandard64KB:
+		case Prospero::TileMode::kDepth:
 		case Prospero::TileMode::kRenderTarget:
 			tile = !RenderIsColorTileModeLinear(rt.attrib3.tile_mode);
 			break;
@@ -147,6 +187,9 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (samples > 1 && (!tile || levels != 1)) {
 		EXIT("multisampled render targets require a single-mip tiled surface\n");
 	}
+	if (texture_tile && samples != 1) {
+		EXIT("texture-tiled color render targets do not support multisampling\n");
+	}
 
 	width  = rt.attrib2.width + 1;
 	height = rt.attrib2.height + 1;
@@ -157,31 +200,38 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		EXIT("render-target format has no valid element size\n");
 	}
 	const auto transfer_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
-	if (standard64 &&
-	    (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 || levels != 1 ||
-	     rt.view.current_mip_level != 0 || view.base_layer != 0 || view.image_layers != 1 ||
-	     samples != 1 || bytes_per_element != 4 || rt.pitch.pitch_div8_minus1 != 0 ||
-	     (rt.base.addr & 0xffffu) != 0 || rt.info.fmask_compression_enable ||
-	     rt.info.fmask_data_compression_disable || rt.info.fmask_one_frag_mode ||
-	     rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable ||
-	     rt.info.cmask_is_linear != 0 || rt.info.cmask_addr_type != 0 || rt.info.alt_tile_mode ||
-	     rt.cmask.addr != 0 || rt.fmask.addr != 0 || rt.dcc_addr.addr != 0 ||
-	     rt.dcc.data_write_on_dcc_clear_to_reg)) {
-		EXIT("unsupported Standard64KB render target: addr=0x%016" PRIx64
+	TileTextureBlockLayout texture_tile_layout {};
+	if (texture_tile &&
+	    (!TileGetTextureBlockLayout(transfer_format, rt.attrib3.tile_mode, volume,
+	                                texture_tile_layout) ||
+	     rt.pitch.pitch_div8_minus1 != 0 ||
+	     (rt.base.addr & (texture_tile_layout.block.block_size - 1u)) != 0 ||
+	     rt.info.fmask_compression_enable || rt.info.fmask_data_compression_disable ||
+	     rt.info.fmask_one_frag_mode || rt.info.cmask_fast_clear_enable ||
+	     rt.info.dcc_compression_enable || rt.info.cmask_is_linear != 0 ||
+	     rt.info.cmask_addr_type != 0 || rt.info.alt_tile_mode || rt.cmask.addr != 0 ||
+	     rt.fmask.addr != 0 || rt.dcc_addr.addr != 0 || rt.dcc.data_write_on_dcc_clear_to_reg)) {
+		EXIT("unsupported texture-tiled render target: addr=0x%016" PRIx64 " tile=%u"
 		     " dimension=%u depth=%u levels=%u layer=%u/%u samples=%u fragments=%u bpe=%u"
 		     " cmask=0x%016" PRIx64 " fmask=0x%016" PRIx64 " dcc=0x%016" PRIx64 "\n",
-		     rt.base.addr, rt.attrib3.dimension, rt.attrib3.depth, levels, view.base_layer,
-		     view.image_layers, rt.attrib.num_samples, rt.attrib.num_fragments, bytes_per_element,
-		     rt.cmask.addr, rt.fmask.addr, rt.dcc_addr.addr);
+		     rt.base.addr, static_cast<uint32_t>(rt.attrib3.tile_mode), rt.attrib3.dimension,
+		     rt.attrib3.depth, levels, view.base_layer, view.image_layers, rt.attrib.num_samples,
+		     rt.attrib.num_fragments, bytes_per_element, rt.cmask.addr, rt.fmask.addr,
+		     rt.dcc_addr.addr);
+	}
+	if ((standard64 || depth_tile) && (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 ||
+	                                   view.base_layer != 0 || view.image_layers != 1)) {
+		EXIT("unsupported 64KB texture-tiled render-target view: dimension=%u depth=%u"
+		     " layer=%u/%u\n",
+		     rt.attrib3.dimension, rt.attrib3.depth, view.base_layer, view.image_layers);
 	}
 	if (rt.pitch.pitch_div8_minus1 != 0) {
 		pitch = (rt.pitch.pitch_div8_minus1 + 1u) << 3u;
 	} else if (tile) {
 		if (volume) {
 			pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
-		} else if (standard64) {
-			pitch =
-			    TileGetTexturePitch(Prospero::BufferFormat::k32Float, width, rt.attrib3.tile_mode);
+		} else if (texture_tile) {
+			pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
 		} else {
 			pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
 		}
@@ -214,10 +264,10 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	} else if (tile) {
 		TileSizeAlign layout {};
 		bool          valid_layout = false;
-		if (standard64) {
-			TileGetTextureSize(Prospero::BufferFormat::k32Float, width, height, levels,
-			                   rt.attrib3.tile_mode, &layout, mip_sizes, mip_padded);
-			valid_layout = layout.size != 0 && layout.align == 65536;
+		if (texture_tile) {
+			TileGetTextureSize(transfer_format, width, height, levels, rt.attrib3.tile_mode,
+			                   &layout, mip_sizes, mip_padded);
+			valid_layout = layout.size != 0 && layout.align == texture_tile_layout.block.block_size;
 		} else {
 			valid_layout =
 			    levels == 1 ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element,
@@ -287,13 +337,20 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	desc.info.data         = {rt.base.addr, backing_size};
 	desc.info.pixel_format = target_format.format;
 	desc.info.guest_format = transfer_format;
-	desc.info.type         = volume ? Prospero::ImageType::kColor3D : Prospero::ImageType::kColor2D;
+	desc.info.type         = image_type;
 	desc.info.extent       = {width, height, depth};
 	desc.info.resources    = {levels, volume ? 1u : view.image_layers};
 	desc.info.pitch        = pitch;
 	desc.info.bytes_per_block = bytes_per_element;
 	desc.info.samples         = samples;
 	desc.info.tile_mode       = rt.attrib3.tile_mode;
+	const bool has_dcc        = rt.info.dcc_compression_enable && rt.dcc_addr.addr != 0;
+	if (has_dcc) {
+		// DCC uses a separate metadata allocation. Carry its address through ImageInfo so
+		// TextureCache can associate metadata fills observed before target registration.
+		desc.info.metadata.kind          = ImageMetadataKind::Dcc;
+		desc.info.metadata.range.address = rt.dcc_addr.addr;
+	}
 	for (uint32_t level = 0; level < levels; level++) {
 		if (volume) {
 			const auto& mip             = volume_layout.mips[level];
@@ -319,8 +376,13 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		};
 	}
 	desc.view_info.format = target_format.format;
-	desc.view_info.type =
-	    view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
+	if (is_1d) {
+		desc.view_info.type =
+		    view.layer_count == 1 ? vk::ImageViewType::e1D : vk::ImageViewType::e1DArray;
+	} else {
+		desc.view_info.type =
+		    view.layer_count == 1 ? vk::ImageViewType::e2D : vk::ImageViewType::e2DArray;
+	}
 	desc.view_info.aspect      = vk::ImageAspectFlagBits::eColor;
 	desc.view_info.base_level  = rt.view.current_mip_level;
 	desc.view_info.level_count = 1;
@@ -340,7 +402,7 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	r.samples                  = samples;
 	r.export_mapping           = target_format.export_mapping;
 	r.color_clear_enable       = false;
-	r.color_clear_value        = {};
+	ResolveDccClearInfo(r, target_format.format, has_dcc, rt.clear_word0.word0);
 	BindRenderTarget(r.image_id);
 }
 

src/graphics/host_gpu/renderer/colorRenderTarget.h

@@ -10,8 +10,6 @@
 
 namespace Libs::Graphics {
 
-class RenderCommandBuffer;
-
 enum class RenderColorType {
 	NoColorOutput,
 	RenderTexture,
@@ -31,7 +29,9 @@ struct RenderColorInfo {
 	uint32_t                        target_slot      = 0;
 	uint32_t                        samples          = 1;
 	Prospero::ColorComponentMapping export_mapping;
-	bool                            color_clear_enable = false;
+	bool                            color_clear_enable             = false;
+	bool                            metadata_clear_supported       = false;
+	bool                            metadata_fixed_clear_supported = false;
 	vk::ClearColorValue             color_clear_value {};
 };
 

src/graphics/host_gpu/renderer/commandScheduler.cpp

@@ -1,96 +1,97 @@
 #include "graphics/host_gpu/renderer/commandScheduler.h"
 
 #include "common/assert.h"
+#include "common/logging/log.h"
 #include "graphics/host_gpu/graphicContext.h"
 
 #include <algorithm>
+#include <cstdio>
+#include <optional>
 
 namespace Libs::Graphics {
 
 static thread_local CommandScheduler* g_deferred_callback_scheduler = nullptr;
 
-void CommandSlot::Reset() {
-	EXIT_IF(buffer == nullptr);
-	const auto result = buffer.reset(vk::CommandBufferResetFlagBits::eReleaseResources);
-	if (result != vk::Result::eSuccess) {
-		EXIT("failed to reset Vulkan command buffer: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
-	}
+namespace {
+
+void ReportVulkanFatal(const char* what, vk::Result result, uint64_t tick, uint32_t debug_op,
+                       uint64_t debug_submit, uint32_t arg0, uint32_t arg1, uint32_t arg2,
+                       uint32_t arg3, uint64_t arg4) {
+	LOGF("%s failed: %s (%d), tick=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	     what, VulkanToString(result).c_str(), static_cast<int>(result), tick, debug_op,
+	     debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::printf("%s failed: %s (%d), tick=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	            " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	            what, VulkanToString(result).c_str(), static_cast<int>(result), tick, debug_op,
+	            debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::fflush(stdout);
 }
 
-CommandScheduler::CommandPool::~CommandPool() {
-	Destroy();
-}
-
-void CommandScheduler::CommandPool::Create(GraphicContext& graphics) {
-	EXIT_IF(m_pool != nullptr || m_graphics != nullptr ||
-	        graphics.queue_family == static_cast<uint32_t>(-1));
-	m_graphics = &graphics;
+} // namespace
 
+CommandScheduler::CommandPool::CommandPool(GraphicContext& graphics, MasterSemaphore& master)
+    : m_graphics(graphics), m_master(master) {
+	EXIT_IF(graphics.queue_family == static_cast<uint32_t>(-1));
 	vk::CommandPoolCreateInfo create {};
 	create.sType            = vk::StructureType::eCommandPoolCreateInfo;
 	create.queueFamilyIndex = graphics.queue_family;
-	create.flags            = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
+	create.flags            = vk::CommandPoolCreateFlagBits::eTransient |
+	                          vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
 	const auto result       = graphics.device.createCommandPool(&create, nullptr, &m_pool);
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || m_pool == nullptr);
 }
 
-CommandSlot* CommandScheduler::CommandPool::CreateSlot() {
-	EXIT_IF(m_graphics == nullptr);
-	auto& graphics = *m_graphics;
+CommandScheduler::CommandPool::~CommandPool() {
+	m_graphics.device.destroyCommandPool(m_pool, nullptr);
+}
+
+size_t CommandScheduler::CommandPool::Grow() {
+	const auto first = m_ticks.size();
+	m_ticks.resize(first + GrowStep);
+	m_buffers.resize(first + GrowStep);
 
 	vk::CommandBufferAllocateInfo allocate {};
 	allocate.sType              = vk::StructureType::eCommandBufferAllocateInfo;
 	allocate.commandPool        = m_pool;
 	allocate.level              = vk::CommandBufferLevel::ePrimary;
-	allocate.commandBufferCount = 1;
-	vk::CommandBuffer buffer    = nullptr;
-	EXIT_IF(graphics.device.allocateCommandBuffers(&allocate, &buffer) != vk::Result::eSuccess);
-
-	vk::FenceCreateInfo fence_create {};
-	fence_create.sType = vk::StructureType::eFenceCreateInfo;
-	fence_create.flags = vk::FenceCreateFlagBits::eSignaled;
-	vk::Fence fence    = nullptr;
-	if (graphics.device.createFence(&fence_create, nullptr, &fence) != vk::Result::eSuccess) {
-		graphics.device.freeCommandBuffers(m_pool, 1, &buffer);
-		EXIT("failed to create command-buffer fence\n");
-	}
-
-	auto& slot      = m_slots.emplace_back();
-	slot.pool_mutex = &m_mutex;
-	slot.id         = static_cast<uint32_t>(m_slots.size() - 1);
-	slot.buffer     = buffer;
-	slot.fence      = fence;
-	return &slot;
+	allocate.commandBufferCount = static_cast<uint32_t>(GrowStep);
+	EXIT_IF(m_graphics.device.allocateCommandBuffers(&allocate, m_buffers.data() + first) !=
+	        vk::Result::eSuccess);
+	return first;
 }
 
-CommandSlot* CommandScheduler::CommandPool::Allocate(GraphicContext& graphics) {
-	Common::LockGuard lock(m_mutex);
-	if (m_pool == nullptr) {
-		Create(graphics);
-	}
-	EXIT_IF(m_graphics != &graphics);
-	auto  found = std::ranges::find_if(m_slots, [](const auto& slot) { return !slot.busy; });
-	auto* slot  = found != m_slots.end() ? &*found : CreateSlot();
-	slot->busy  = true;
-	slot->Reset();
-	return slot;
-}
+vk::CommandBuffer CommandScheduler::CommandPool::Commit() {
+	auto       gpu_tick = m_master.KnownGpuTick();
+	const auto search   = [this, &gpu_tick](size_t begin, size_t end) -> std::optional<size_t> {
+		for (size_t index = begin; index < end; ++index) {
+			if (gpu_tick >= m_ticks[index]) {
+				m_ticks[index] = m_master.CurrentTick();
+				return index;
+			}
+		}
+		return std::nullopt;
+	};
 
-void CommandScheduler::CommandPool::Destroy() {
-	Common::LockGuard lock(m_mutex);
-	if (m_pool == nullptr) {
-		return;
+	auto found = search(m_hint, m_ticks.size());
+	if (!found) {
+		m_master.Refresh();
+		gpu_tick = m_master.KnownGpuTick();
+		found    = search(m_hint, m_ticks.size());
 	}
-	EXIT_IF(std::ranges::any_of(m_slots, [](const auto& slot) { return slot.busy; }));
-	EXIT_IF(m_graphics == nullptr);
-	for (const auto& slot: m_slots) {
-		m_graphics->device.destroyFence(slot.fence, nullptr);
+	if (!found) {
+		found = search(0, m_hint);
 	}
-	m_graphics->device.destroyCommandPool(m_pool, nullptr);
-	m_slots.clear();
-	m_pool     = nullptr;
-	m_graphics = nullptr;
+	if (!found) {
+		found           = Grow();
+		m_ticks[*found] = m_master.CurrentTick();
+	}
+
+	m_hint = *found + 1;
+	if (m_hint == m_ticks.size()) {
+		m_hint = 0;
+	}
+	return m_buffers[*found];
 }
 
 bool CommandScheduler::InDeferredOperation() noexcept {
@@ -99,10 +100,8 @@ bool CommandScheduler::InDeferredOperation() noexcept {
 
 CommandScheduler::CommandScheduler(RenderContext& context, GraphicContext& graphics)
     : m_master(graphics), m_context(context), m_graphics(graphics),
-      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {
-	m_buffers.reserve(CommandBufferGrowStep);
-	m_buffer_ticks.reserve(CommandBufferGrowStep);
-}
+      m_command_pool(graphics, m_master), m_command(*this),
+      m_priority_thread([this](std::stop_token stop) { PriorityOperationsThread(stop); }) {}
 
 CommandScheduler::~CommandScheduler() {
 	Shutdown();
@@ -128,9 +127,11 @@ void CommandScheduler::Shutdown() {
 		}
 		m_operation_state = OperationState::Draining;
 	}
-	if (Active() && m_recording) {
-		Finish();
+	if (!m_command.IsInvalid()) {
+		Submit();
 	}
+	m_master.Wait(CurrentTick() - 1);
+	PopPendingOperations();
 	DrainPriorityOperations();
 	m_priority_thread.request_stop();
 	m_operation_available.notify_all();
@@ -156,14 +157,10 @@ void CommandScheduler::Begin(HW::Context& registers, HW::UserConfig& user_config
 	m_user_config = &user_config;
 	m_shaders     = &shaders;
 
-	if (!Active()) {
-		m_current = static_cast<int>(GrowCommandBuffers());
-	}
-
-	BindCurrent();
-	if (!m_recording) {
-		Current().Begin();
-		m_recording = true;
+	if (m_command.IsInvalid()) {
+		BeginNext();
+	} else {
+		BindCurrent();
 	}
 }
 
@@ -172,7 +169,7 @@ void CommandScheduler::BeginRendering(const RenderState& state) {
 }
 
 void CommandScheduler::EndRendering() {
-	if (Active() && m_recording) {
+	if (Active() && !m_command.IsInvalid()) {
 		Current().EndRendering();
 	}
 }
@@ -183,55 +180,40 @@ void CommandScheduler::Flush() {
 }
 
 void CommandScheduler::Flush(SubmitInfo& submit) {
-	SubmitCurrent(submit);
+	Submit(submit);
 	BeginNext();
 }
 
-CommandBuffer& CommandScheduler::FlushAndGetSubmitted() {
-	SubmitInfo submit;
-	auto&      submitted = SubmitCurrent(submit);
+void CommandScheduler::FlushAndWait() {
+	const auto tick = Submit();
+	m_master.Wait(tick);
 	BeginNext();
-	return submitted;
 }
 
 void CommandScheduler::Finish() {
 	CheckActive();
-	const auto tick = CurrentTick();
-	if (m_recording) {
-		SubmitInfo submit;
-		SubmitCurrent(submit);
+	if (!m_command.IsInvalid()) {
+		Submit();
 	}
-	for (auto& buffer: m_buffers) {
-		buffer->WaitForFenceAndReset();
-	}
-	m_master.Wait(tick);
+	m_master.Wait(CurrentTick() - 1);
+	BeginNext();
 	PopPendingOperations();
-	BindCurrent();
-	Current().Begin();
-	m_recording = true;
-}
-
-void CommandScheduler::FinishCurrent() {
-	SubmitInfo submit;
-	auto&      submitted = SubmitCurrent(submit);
-	submitted.WaitForFenceAndReset();
-	m_master.Refresh();
-	PopPendingOperations();
-	submitted.Begin();
-	m_recording = true;
 }
 
 void CommandScheduler::Wait(uint64_t tick) {
-	CheckActive();
 	EXIT_IF(tick > CurrentTick());
-	if (tick >= CurrentTick()) {
+	if (tick == CurrentTick()) {
+		CheckActive();
 		// A stream-buffer wrap can wait while a draw is being prepared through a reference to
-		// Current(). Recycle the same command object so that reference remains valid.
-		FinishCurrent();
-		return;
+		// Current(). The wrapper stays stable while its pooled Vulkan buffer is retired. Deferred
+		// resources are released only at the next GPU operation boundary.
+		const auto submitted_tick = Submit();
+		EXIT_IF(submitted_tick != tick);
+		m_master.Wait(tick);
+		BeginNext();
+	} else {
+		m_master.Wait(tick);
 	}
-	m_master.Wait(tick);
-	PopPendingOperations();
 }
 
 void CommandScheduler::PopPendingOperations() {
@@ -243,17 +225,18 @@ void CommandScheduler::PopPendingOperations(bool refresh_gpu_tick) {
 		m_master.Refresh();
 	}
 	for (;;) {
-		Common::UniqueFunction<void> callback;
+		PendingOperation operation;
 		{
 			std::lock_guard lock(m_operation_mutex);
 			if (m_pending_operations.empty() ||
 			    !m_master.IsFree(m_pending_operations.front().tick)) {
 				return;
 			}
-			callback = std::move(m_pending_operations.front().callback);
+			operation = std::move(m_pending_operations.front());
 			m_pending_operations.pop();
 		}
-		RunOperation(std::move(callback));
+		WaitPriorityOperations(operation.tick);
+		RunOperation(std::move(operation.callback));
 	}
 }
 
@@ -359,86 +342,81 @@ bool CommandScheduler::IsFree(uint64_t tick) {
 	return m_master.IsFree(tick);
 }
 
-CommandSlot* CommandScheduler::AllocateCommandBuffer() {
-	return m_command_pool.Allocate(m_graphics);
-}
-
-uint64_t CommandScheduler::NextSubmitSequence() noexcept {
-	return m_submit_sequence.fetch_add(1, std::memory_order_relaxed) + 1;
-}
-
 void CommandScheduler::CheckActive() const {
-	EXIT_IF(!Active() || static_cast<size_t>(m_current) >= m_buffers.size());
+	EXIT_IF(!Active());
 }
 
-RenderCommandBuffer& CommandScheduler::Current() const {
+CommandBuffer& CommandScheduler::Current() {
 	CheckActive();
-	EXIT_IF(m_buffers[m_current] == nullptr);
-	return *m_buffers[m_current];
+	return m_command;
 }
 
-void CommandScheduler::BindCurrent() const {
+void CommandScheduler::BindCurrent() {
 	EXIT_IF(m_registers == nullptr || m_user_config == nullptr || m_shaders == nullptr);
-	Current().Bind(*m_registers, *m_user_config, *m_shaders);
+	m_command.Bind(*m_registers, *m_user_config, *m_shaders);
 }
 
-CommandBuffer& CommandScheduler::SubmitCurrent(SubmitInfo& submit) {
-	CheckActive();
-	EXIT_IF(!m_recording);
-	auto& submitted = Current();
-	submitted.End();
-	const auto signal_tick = m_master.NextTick();
-	submit.AddSignal(m_master.Handle(), signal_tick);
-	submitted.Execute(submit);
-	m_buffer_ticks[static_cast<size_t>(m_current)] = signal_tick;
-	m_recording                                    = false;
-	return submitted;
+CommandBuffer& CommandScheduler::BeginCommand() {
+	EXIT_IF(!m_command.IsInvalid());
+	m_command.m_buffer = m_command_pool.Commit();
+	m_command.Begin();
+	return m_command;
 }
 
-int CommandScheduler::FindReusableBuffer(uint64_t gpu_tick) const {
-	EXIT_IF(m_buffer_ticks.size() != m_buffers.size());
-	const auto buffer_count = m_buffers.size();
-	for (size_t offset = 1; offset <= buffer_count; ++offset) {
-		const auto candidate = (static_cast<size_t>(m_current) + offset) % buffer_count;
-		if (gpu_tick >= m_buffer_ticks[candidate]) {
-			return static_cast<int>(candidate);
-		}
+uint64_t CommandScheduler::Submit(SubmitInfo submit) {
+	EXIT_IF(m_command.IsInvalid());
+	EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores ||
+	        submit.num_signal_semaphores >= SubmitInfo::MaxSemaphores);
+
+	m_command.End();
+	const auto buffer   = m_command.m_buffer;
+	auto&      graphics = m_graphics;
+	EXIT_IF(graphics.queue == nullptr);
+
+	vk::Result result;
+	uint64_t   tick;
+	{
+		Common::LockGuard lock(graphics.queue_mutex);
+		tick = m_master.NextTick();
+		submit.AddSignal(m_master.Handle(), tick);
+
+		vk::TimelineSemaphoreSubmitInfo timeline_info {};
+		timeline_info.sType                     = vk::StructureType::eTimelineSemaphoreSubmitInfo;
+		timeline_info.waitSemaphoreValueCount   = submit.num_wait_semaphores;
+		timeline_info.pWaitSemaphoreValues      = submit.wait_ticks.data();
+		timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores;
+		timeline_info.pSignalSemaphoreValues    = submit.signal_ticks.data();
+
+		vk::SubmitInfo submit_info {};
+		submit_info.sType                = vk::StructureType::eSubmitInfo;
+		submit_info.pNext                = &timeline_info;
+		submit_info.waitSemaphoreCount   = submit.num_wait_semaphores;
+		submit_info.pWaitSemaphores      = submit.wait_semaphores.data();
+		submit_info.pWaitDstStageMask    = submit.wait_stages.data();
+		submit_info.commandBufferCount   = 1;
+		submit_info.pCommandBuffers      = &buffer;
+		submit_info.signalSemaphoreCount = submit.num_signal_semaphores;
+		submit_info.pSignalSemaphores    = submit.signal_semaphores.data();
+
+		result = graphics.queue.submit(1, &submit_info, nullptr);
 	}
-	return -1;
-}
 
-size_t CommandScheduler::GrowCommandBuffers() {
-	const auto first = m_buffers.size();
-	const auto end   = first + CommandBufferGrowStep;
-	m_buffers.reserve(end);
-	m_buffer_ticks.reserve(end);
-	for (size_t i = first; i < end; ++i) {
-		m_buffers.emplace_back(std::make_unique<RenderCommandBuffer>(*this));
-		m_buffer_ticks.push_back(0);
+	if (result != vk::Result::eSuccess) {
+		ReportVulkanFatal("vkQueueSubmit", result, tick, m_command.m_debug_op,
+		                  m_command.m_debug_submit_id, m_command.m_debug_arg0,
+		                  m_command.m_debug_arg1, m_command.m_debug_arg2, m_command.m_debug_arg3,
+		                  m_command.m_debug_arg4);
 	}
-	return first;
+	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
+
+	m_command.m_buffer = nullptr;
+	return tick;
 }
 
 void CommandScheduler::BeginNext() {
-	EXIT_IF(m_recording);
-
-	auto candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	bool refreshed = false;
-	if (candidate < 0) {
-		m_master.Refresh();
-		refreshed = true;
-		candidate = FindReusableBuffer(m_master.KnownGpuTick());
-	}
-	if (candidate < 0) {
-		candidate = static_cast<int>(GrowCommandBuffers());
-	}
-
-	m_current = candidate;
-	Current().WaitForFenceAndReset();
-	PopPendingOperations(!refreshed);
+	EXIT_IF(!m_command.IsInvalid());
 	BindCurrent();
-	Current().Begin();
-	m_recording = true;
+	BeginCommand();
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/commandScheduler.h

@@ -6,11 +6,7 @@
 #include "graphics/host_gpu/renderer/masterSemaphore.h"
 #include "graphics/host_gpu/renderer/render.h"
 
-#include <array>
-#include <atomic>
 #include <condition_variable>
-#include <deque>
-#include <memory>
 #include <mutex>
 
 #include <queue>
@@ -20,16 +16,6 @@
 
 namespace Libs::Graphics {
 
-struct CommandSlot {
-	Common::Mutex*    pool_mutex = nullptr;
-	uint32_t          id         = 0;
-	vk::CommandBuffer buffer     = nullptr;
-	vk::Fence         fence      = nullptr;
-	bool              busy       = false;
-
-	void Reset();
-};
-
 class CommandScheduler {
 public:
 	CommandScheduler(RenderContext& context, GraphicContext& graphics);
@@ -41,9 +27,10 @@ public:
 	void           EndRendering();
 	void           Flush();
 	void           Flush(SubmitInfo& submit);
-	CommandBuffer& FlushAndGetSubmitted();
+	void           FlushAndWait();
 	void           Finish();
-	void           FinishCurrent();
+	CommandBuffer& BeginCommand();
+	uint64_t       Submit(SubmitInfo submit = {});
 	// Deferred callbacks can observe an externally owned drain, but cannot initiate shutdown:
 	// the priority runner cannot join itself.
 	void                      Shutdown();
@@ -55,34 +42,35 @@ public:
 	void                      DeferPriorityOperation(Common::UniqueFunction<void>&& operation);
 	[[nodiscard]] static bool InDeferredOperation() noexcept;
 
-	[[nodiscard]] bool            Active() const noexcept { return m_current >= 0; }
-	void                          CheckActive() const;
-	RenderCommandBuffer&          Current() const;
-	[[nodiscard]] uint64_t        CurrentTick() const noexcept { return m_master.CurrentTick(); }
-	[[nodiscard]] bool            IsFree(uint64_t tick);
-	[[nodiscard]] RenderContext&  Context() const noexcept { return m_context; }
-	[[nodiscard]] GraphicContext& Graphics() const noexcept { return m_graphics; }
+	[[nodiscard]] bool             Active() const noexcept { return m_registers != nullptr; }
+	void                           CheckActive() const;
+	CommandBuffer&                 Current();
+	[[nodiscard]] uint64_t         CurrentTick() const noexcept { return m_master.CurrentTick(); }
+	[[nodiscard]] bool             IsFree(uint64_t tick);
+	[[nodiscard]] MasterSemaphore& GetMasterSemaphore() noexcept { return m_master; }
+	[[nodiscard]] RenderContext&   Context() const noexcept { return m_context; }
+	[[nodiscard]] GraphicContext&  Graphics() const noexcept { return m_graphics; }
 
 private:
-	static constexpr size_t CommandBufferGrowStep = 4;
-
 	class CommandPool {
 	public:
-		CommandPool() = default;
+		CommandPool(GraphicContext& graphics, MasterSemaphore& master);
 		~CommandPool();
 		KYTY_CLASS_NO_COPY(CommandPool);
 
-		CommandSlot* Allocate(GraphicContext& graphics);
+		vk::CommandBuffer Commit();
 
 	private:
-		void         Create(GraphicContext& graphics);
-		CommandSlot* CreateSlot();
-		void         Destroy();
+		static constexpr size_t GrowStep = 4;
 
-		GraphicContext*         m_graphics = nullptr;
-		Common::Mutex           m_mutex;
-		vk::CommandPool         m_pool = nullptr;
-		std::deque<CommandSlot> m_slots;
+		size_t Grow();
+
+		GraphicContext&                m_graphics;
+		MasterSemaphore&               m_master;
+		vk::CommandPool                m_pool = nullptr;
+		std::vector<vk::CommandBuffer> m_buffers;
+		std::vector<uint64_t>          m_ticks;
+		size_t                         m_hint = 0;
 	};
 
 	enum class OperationState { Open, Draining, Closed };
@@ -92,39 +80,28 @@ private:
 		uint64_t                     tick = 0;
 	};
 
-	void                       BindCurrent() const;
-	CommandBuffer&             SubmitCurrent(SubmitInfo& submit);
-	void                       BeginNext();
-	void                       PopPendingOperations(bool refresh_gpu_tick);
-	[[nodiscard]] int          FindReusableBuffer(uint64_t gpu_tick) const;
-	[[nodiscard]] size_t       GrowCommandBuffers();
-	void                       PriorityOperationsThread(std::stop_token stop);
-	void                       RunOperation(Common::UniqueFunction<void>&& operation);
-	[[nodiscard]] CommandSlot* AllocateCommandBuffer();
-	[[nodiscard]] uint64_t     NextSubmitSequence() noexcept;
+	void BindCurrent();
+	void BeginNext();
+	void PopPendingOperations(bool refresh_gpu_tick);
+	void PriorityOperationsThread(std::stop_token stop);
+	void RunOperation(Common::UniqueFunction<void>&& operation);
 
-	MasterSemaphore                                   m_master;
-	RenderContext&                                    m_context;
-	GraphicContext&                                   m_graphics;
-	CommandPool                                       m_command_pool;
-	std::vector<std::unique_ptr<RenderCommandBuffer>> m_buffers;
-	std::vector<uint64_t>                             m_buffer_ticks;
-	std::queue<PendingOperation>                      m_pending_operations;
-	std::queue<PendingOperation>                      m_priority_operations;
-	std::mutex                                        m_operation_mutex;
-	std::condition_variable                           m_operation_available;
-	std::jthread                                      m_priority_thread;
-	bool                                              m_priority_active      = false;
-	uint64_t                                          m_priority_active_tick = 0;
-	OperationState                                    m_operation_state      = OperationState::Open;
-	int                                               m_current              = -1;
-	bool                                              m_recording            = false;
-	HW::Context*                                      m_registers            = nullptr;
-	HW::UserConfig*                                   m_user_config          = nullptr;
-	HW::Shader*                                       m_shaders              = nullptr;
-	std::atomic<uint64_t>                             m_submit_sequence      = 0;
-
-	friend class CommandBuffer;
+	MasterSemaphore              m_master;
+	RenderContext&               m_context;
+	GraphicContext&              m_graphics;
+	CommandPool                  m_command_pool;
+	CommandBuffer                m_command;
+	std::queue<PendingOperation> m_pending_operations;
+	std::queue<PendingOperation> m_priority_operations;
+	std::mutex                   m_operation_mutex;
+	std::condition_variable      m_operation_available;
+	std::jthread                 m_priority_thread;
+	bool                         m_priority_active      = false;
+	uint64_t                     m_priority_active_tick = 0;
+	OperationState               m_operation_state      = OperationState::Open;
+	HW::Context*                 m_registers            = nullptr;
+	HW::UserConfig*              m_user_config          = nullptr;
+	HW::Shader*                  m_shaders              = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/context.cpp

@@ -1,7 +1,5 @@
 #include "common/assert.h"
 #include "common/common.h"
-#include "common/emulatorConfig.h"
-#include "common/logging/log.h"
 #include "common/profiler.h"
 #include "common/threads.h"
 #include "graphics/host_gpu/graphicContext.h"
@@ -9,7 +7,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vma.h"
@@ -18,98 +15,28 @@
 #include <algorithm>
 #include <bit>
 #include <cstring>
-#include <memory>
 namespace Libs::Graphics {
 
-FenceResourceRetainer::~FenceResourceRetainer() {
-	if (!m_resources.empty()) {
-		EXIT("fence resource retainer destroyed before release\n");
-	}
-}
-
-void FenceResourceRetainer::Retain(std::shared_ptr<void> resource) {
-	if (resource == nullptr) {
-		EXIT("cannot retain a null fence resource\n");
-	}
-	if (std::ranges::none_of(m_resources, [&resource](const auto& retained) {
-		    return retained.get() == resource.get();
-	    })) {
-		m_resources.push_back(std::move(resource));
-	}
-}
-
-void FenceResourceRetainer::ReleaseAfterFence() noexcept {
-	m_resources.clear();
-}
-
 CommandBuffer::CommandBuffer(CommandScheduler& scheduler)
-    : m_context(scheduler.Context()), m_scheduler(scheduler), m_graphics(scheduler.Graphics()),
-      m_slot(scheduler.AllocateCommandBuffer()) {}
-
-CommandBuffer::~CommandBuffer() {
-	Release();
-}
+    : m_context(scheduler.Context()), m_graphics(scheduler.Graphics()) {}
 
 bool CommandBuffer::IsInvalid() const {
-	return m_slot == nullptr;
+	return m_buffer == nullptr;
 }
 
 vk::CommandBuffer CommandBuffer::Handle() const {
 	EXIT_IF(IsInvalid());
-
-	const auto handle = m_slot->buffer;
-	EXIT_IF(handle == nullptr);
-	return handle;
+	return m_buffer;
 }
 
-void CommandBuffer::Release() {
-	EXIT_IF(IsInvalid());
-
-	Common::LockGuard lock(*m_slot->pool_mutex);
-
-	WaitForFence();
-
-	m_slot->busy = false;
-	m_slot->Reset();
-	ReleaseResourcesAfterFence();
-	m_slot = nullptr;
-
-	EXIT_NOT_IMPLEMENTED(!IsInvalid());
-}
-
-void CommandBuffer::RetireBufferAfterFence(std::unique_ptr<VulkanBuffer> buffer) {
-	if (IsInvalid() || m_execute || buffer == nullptr || buffer->buffer == nullptr) {
-		EXIT("cannot retire a buffer on an invalid or submitted command buffer\n");
-	}
-	m_retired_buffers.push_back(std::move(buffer));
-}
-
-void CommandBuffer::RetainResourceUntilFence(std::shared_ptr<void> resource) {
-	if (IsInvalid() || m_execute) {
-		EXIT("cannot retain a resource on an invalid or submitted command buffer\n");
-	}
-	m_fence_resources.Retain(std::move(resource));
-}
-
-void CommandBuffer::RecycleDescriptorAfterFence(VulkanDescriptorSet& set) {
-	m_descriptor_sets_after_fence.push_back(&set);
-}
-
-void CommandBuffer::RecycleDescriptorsAfterFence() {
-	for (auto* set: m_descriptor_sets_after_fence) {
-		m_context.GetDescriptorCache().Recycle(*set);
-	}
-	m_descriptor_sets_after_fence.clear();
-}
-
-void CommandBuffer::Begin() const {
-	EXIT_IF(m_rendering);
+void CommandBuffer::Begin() {
+	EXIT_IF(m_rendering || IsInvalid());
 	auto buffer = Handle();
 
 	vk::CommandBufferBeginInfo begin_info {};
 	begin_info.sType            = vk::StructureType::eCommandBufferBeginInfo;
 	begin_info.pNext            = nullptr;
-	begin_info.flags            = {};
+	begin_info.flags            = vk::CommandBufferUsageFlagBits::eOneTimeSubmit;
 	begin_info.pInheritanceInfo = nullptr;
 
 	auto result = buffer.begin(&begin_info);
@@ -137,125 +64,6 @@ void CommandBuffer::SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0,
 	m_debug_arg4      = arg4;
 }
 
-void CommandBuffer::Execute(const SubmitInfo& submit) {
-	EXIT_IF(IsInvalid());
-	EXIT_IF(m_execute);
-	EXIT_IF(submit.num_wait_semaphores > SubmitInfo::MaxSemaphores ||
-	        submit.num_signal_semaphores > SubmitInfo::MaxSemaphores);
-
-	auto buffer = Handle();
-	auto fence  = m_slot->fence;
-
-	vk::TimelineSemaphoreSubmitInfo timeline_info {};
-	timeline_info.sType                     = vk::StructureType::eTimelineSemaphoreSubmitInfo;
-	timeline_info.waitSemaphoreValueCount   = submit.num_wait_semaphores;
-	timeline_info.pWaitSemaphoreValues      = submit.wait_ticks.data();
-	timeline_info.signalSemaphoreValueCount = submit.num_signal_semaphores;
-	timeline_info.pSignalSemaphoreValues    = submit.signal_ticks.data();
-
-	vk::SubmitInfo submit_info {};
-	submit_info.sType                = vk::StructureType::eSubmitInfo;
-	submit_info.pNext                = &timeline_info;
-	submit_info.waitSemaphoreCount   = submit.num_wait_semaphores;
-	submit_info.pWaitSemaphores      = submit.wait_semaphores.data();
-	submit_info.pWaitDstStageMask    = submit.wait_stages.data();
-	submit_info.commandBufferCount   = 1;
-	submit_info.pCommandBuffers      = &buffer;
-	submit_info.signalSemaphoreCount = submit.num_signal_semaphores;
-	submit_info.pSignalSemaphores    = submit.signal_semaphores.data();
-
-	auto& graphics = m_graphics;
-	EXIT_IF(graphics.queue == nullptr);
-
-	auto result = graphics.device.resetFences(1, &fence);
-	if (result != vk::Result::eSuccess) {
-		LOGF("vkResetFences failed before submit: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	if (Config::GraphicsDebugDumpEnabled()) {
-		LOGF("vkQueueSubmit begin: slot=%u waits=%u signals=%u debug_op=%u debug_submit=%" PRIu64
-		     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     m_slot->id, submit.num_wait_semaphores, submit.num_signal_semaphores, m_debug_op,
-		     m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-
-	{
-		Common::LockGuard lock(graphics.queue_mutex);
-		m_submit_seq = m_scheduler.NextSubmitSequence();
-		result       = graphics.queue.submit(1, &submit_info, fence);
-	}
-
-	m_execute      = true;
-	m_fence_waited = false;
-
-	if (result != vk::Result::eSuccess) {
-		LOGF("vkQueueSubmit failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-}
-
-void CommandBuffer::WaitForFence() {
-	FinalizeFence(false);
-}
-
-void CommandBuffer::WaitForFenceOnly() {
-	EXIT_IF(IsInvalid());
-	if (!m_execute || m_fence_waited) {
-		return;
-	}
-	auto device = m_graphics.device;
-	auto result = device.waitForFences(1, &m_slot->fence, VK_TRUE, UINT64_MAX);
-	if (result != vk::Result::eSuccess) {
-		LOGF("vkWaitForFences failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-	m_fence_waited = true;
-}
-
-void CommandBuffer::WaitForFenceAndReset() {
-	FinalizeFence(true);
-}
-
-void CommandBuffer::FinalizeFence(bool reset_recording) {
-	const bool was_executed = m_execute;
-	WaitForFenceOnly();
-	if (was_executed) {
-		m_execute      = false;
-		m_fence_waited = false;
-		if (reset_recording) {
-			Common::LockGuard lock(*m_slot->pool_mutex);
-			m_slot->Reset();
-		}
-	}
-	if (was_executed) {
-		ReleaseResourcesAfterFence();
-	}
-	DeleteBuffersAfterFence();
-}
-
-void CommandBuffer::ReleaseResourcesAfterFence() {
-	RecycleDescriptorsAfterFence();
-	m_fence_resources.ReleaseAfterFence();
-}
-
-void CommandBuffer::DeleteBuffersAfterFence() {
-	for (const auto& buffer: m_retired_buffers) {
-		m_graphics.DeleteBuffer(*buffer);
-	}
-	m_retired_buffers.clear();
-}
-
 void CommandBuffer::BeginRendering(const RenderState& state) const {
 	EXIT_IF(state.width == 0 || state.height == 0 || state.num_layers == 0 ||
 	        state.num_color_attachments > RENDER_COLOR_ATTACHMENTS_MAX);

src/graphics/host_gpu/renderer/debug.cpp

@@ -27,7 +27,7 @@ static bool RenderTargetMaskHasMrt(uint32_t mask) {
 	return (mask & ~0x0fu) != 0;
 }
 
-static bool RenderTargetMaskHasBoundMrt(const RenderCommandBuffer& buffer) {
+static bool RenderTargetMaskHasBoundMrt(const CommandBuffer& buffer) {
 	const auto& hw   = buffer.GetRegisters();
 	const auto  mask = hw.GetRenderTargetMask();
 
@@ -45,7 +45,7 @@ static bool RenderTargetMaskHasBoundMrt(const RenderCommandBuffer& buffer) {
 	return bound_targets > 1;
 }
 
-uint32_t render_target_first_bound_slot(const RenderCommandBuffer& buffer) {
+uint32_t render_target_first_bound_slot(const CommandBuffer& buffer) {
 	const auto& hw   = buffer.GetRegisters();
 	const auto  mask = hw.GetRenderTargetMask();
 	for (uint32_t i = 0; i < 8; i++) {
@@ -71,14 +71,16 @@ void uc_print(const char* func, const HW::UserConfig& uc) {
 	LOGF("\t GetPrimType()         = 0x%08" PRIx32 "\n"
 	     "\t GetIndexOffset()      = 0x%08" PRIx32 "\n"
 	     "\t GetObjectId()         = 0x%08" PRIx32 "\n"
+	     "\t primitive_reset       = 0x%08" PRIx32 "\n"
 	     "\t primitive_group_size  = 0x%04" PRIx16 "\n"
 	     "\t vertex_group_size     = 0x%04" PRIx16 "\n"
 	     "\t en_user_vgpr1         = %s\n"
 	     "\t en_user_vgpr2         = %s\n"
 	     "\t en_user_vgpr3         = %s\n",
 	     static_cast<uint32_t>(uc.GetPrimType()), uc.GetIndexOffset(), uc.GetObjectId(),
-	     ge_cntl.primitive_group_size, ge_cntl.vertex_group_size, user_en.vgpr1 ? "true" : "false",
-	     user_en.vgpr2 ? "true" : "false", user_en.vgpr3 ? "true" : "false");
+	     uc.GetPrimitiveResetControl(), ge_cntl.primitive_group_size, ge_cntl.vertex_group_size,
+	     user_en.vgpr1 ? "true" : "false", user_en.vgpr2 ? "true" : "false",
+	     user_en.vgpr3 ? "true" : "false");
 }
 
 void uc_check(const HW::UserConfig& uc) {
@@ -264,20 +266,23 @@ static void RtCheck(const HW::RenderTarget& rt) {
 				logged = true;
 			}
 		}
+		const bool native_2x_one_frag =
+		    rt.info.fmask_compression_enable && !rt.info.fmask_data_compression_disable &&
+		    rt.info.cmask_fast_clear_enable && rt.fmask.addr != 0 && rt.cmask.addr != 0 &&
+		    rt.attrib.num_samples == 1 && rt.attrib.num_fragments == 1;
+		EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode && !native_2x_one_frag);
 		if (rt.info.fmask_compression_enable) {
 			EXIT_NOT_IMPLEMENTED(rt.attrib.num_samples == 0 && rt.attrib.num_fragments == 0);
 			static bool logged = false;
 			if (!logged) {
-				LOGF("RenderTarget: temporary: ignoring PS5 FMASK metadata for single-sample MSAA "
-				     "fallback, fmask=0x%016" PRIx64 "\n",
+				LOGF("RenderTarget: using native Vulkan MSAA without guest FMASK metadata, "
+				     "fmask=0x%016" PRIx64 "\n",
 				     rt.fmask.addr);
 				logged = true;
 			}
 		}
 
-		// EXIT_NOT_IMPLEMENTED(rt.info.fmask_compression_mode != 0x00000000);
 		EXIT_NOT_IMPLEMENTED(rt.info.fmask_data_compression_disable != false);
-		EXIT_NOT_IMPLEMENTED(rt.info.fmask_one_frag_mode != false);
 
 		if (rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable) {
 			static bool logged = false;
@@ -559,14 +564,11 @@ static void ZCheck(const HW::DepthRenderTarget& z, const HW::DepthControl& dc,
 				     z.depth_view.stencil_write_disable ? "true" : "false");
 			}
 		}
-		EXIT_NOT_IMPLEMENTED(z.z_read_base_addr != z.z_write_base_addr);
-		EXIT_NOT_IMPLEMENTED(z.stencil_read_base_addr != z.stencil_write_base_addr);
-		EXIT_NOT_IMPLEMENTED(z.z_write_base_addr == 0);
-		if (z.stencil_info.format != Prospero::StencilFormat::kInvalid &&
-		    z.stencil_write_base_addr == 0) {
-			LOGF("\t warning: stencil format is set without a stencil base address, continuing "
-			     "without stencil attachment\n");
-		}
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.depth_write_disable &&
+		                     z.z_read_base_addr != z.z_write_base_addr);
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.stencil_write_disable &&
+		                     z.stencil_read_base_addr != z.stencil_write_base_addr);
+		EXIT_NOT_IMPLEMENTED(!z.depth_view.depth_write_disable && z.z_write_base_addr == 0);
 		// EXIT_NOT_IMPLEMENTED(z.pitch_div8_minus1 != 0x000000ff);
 		// EXIT_NOT_IMPLEMENTED(z.height_div8_minus1 != 0x0000008f);
 		// EXIT_NOT_IMPLEMENTED(z.slice_div64_minus1 != 0x00008fff);
@@ -615,8 +617,7 @@ static void ClipCheck(const HW::ClipControl& c) {
 	// dx_linear_attr_clip_enable preserves linear (noperspective) attributes at clip-generated
 	// vertices, which Vulkan provides as part of clipping and interpolation.
 	EXIT_NOT_IMPLEMENTED(c.user_clip_planes != 0 || c.user_clip_plane_mode != 0 ||
-	                     c.vertex_kill_any || !c.IsZClipModeRepresentable() ||
-	                     c.user_clip_plane_negate_y || c.clip_disable ||
+	                     c.vertex_kill_any || c.user_clip_plane_negate_y || c.clip_disable ||
 	                     c.user_clip_plane_cull_only || c.cull_on_clipping_error_disable ||
 	                     c.force_viewport_index_from_vs_enable);
 }
@@ -640,17 +641,6 @@ static void RcPrint(const char* func, const HW::RenderControl& c) {
 static void RcCheck(const HW::RenderControl& c) {
 	// EXIT_NOT_IMPLEMENTED(c.depth_clear_enable != false);
 	// EXIT_NOT_IMPLEMENTED(c.stencil_clear_enable != false);
-	if ((c.resummarize_enable || c.copy_centroid || c.copy_sample != 0)) {
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1) < 16) {
-			LOGF("\t warning: unsupported PS5 render-control metadata state, continuing: "
-			     "resummarize=%s copy_centroid=%s copy_sample=%" PRIu8 "\n",
-			     c.resummarize_enable ? "true" : "false", c.copy_centroid ? "true" : "false",
-			     c.copy_sample);
-		}
-		return;
-	}
-	EXIT_NOT_IMPLEMENTED(c.resummarize_enable != false);
 	// EXIT_NOT_IMPLEMENTED(c.stencil_compress_disable != false);
 	// EXIT_NOT_IMPLEMENTED(c.depth_compress_disable != false);
 	EXIT_NOT_IMPLEMENTED(c.copy_centroid != false);
@@ -898,12 +888,10 @@ void LogDrawPhase(const char* draw_name, const char* phase) {
 	}
 }
 
-void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
-                      uint32_t ps_crc32) {
+void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id) {
 	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: %s VS=0x%08" PRIx32 "/0x%08" PRIx32 " PS=0x%08" PRIx32 "/0x%08" PRIx32
-		     "\n",
-		     phase, vs_hash0, vs_crc32, ps_hash0, ps_crc32);
+		LOGF("PipelineTrace: %s VS=%" PRIu64 " PS=%" PRIu64 "\n", phase, vertex_program_id,
+		     pixel_program_id);
 	}
 }
 
@@ -1167,7 +1155,7 @@ ScissorRect calc_final_scissor(const HW::ScreenViewport& vp, const HW::ScanModeC
 	return ScissorRectClamp(final, extent.width, extent.height);
 }
 
-void hw_check(const RenderCommandBuffer& buffer) {
+void hw_check(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
@@ -1240,7 +1228,7 @@ void hw_check(const RenderCommandBuffer& buffer) {
 	// EXIT_NOT_IMPLEMENTED(hw.GetStencilClearValue() != 0);
 }
 
-void hw_print(const RenderCommandBuffer& buffer) {
+void hw_print(const CommandBuffer& buffer) {
 	const auto& hw      = buffer.GetRegisters();
 	const auto  rt_slot = render_target_first_bound_slot(buffer);
 	const auto& rt      = hw.GetRenderTarget(rt_slot);
@@ -1265,9 +1253,10 @@ void hw_print(const RenderCommandBuffer& buffer) {
 		     "\t GetRenderTargetMask()   = 0x%08" PRIx32 "\n"
 		     "\t GetDepthClearValue()    = %f\n"
 		     "\t GetStencilClearValue()  = %" PRIu8 "\n"
-		     "\t GetLineWidth()          = %f\n",
+		     "\t GetLineWidth()          = %f\n"
+		     "\t primitive_reset_index   = 0x%08" PRIx32 "\n",
 		     hw.GetRenderTargetMask(), hw.GetDepthClearValue(), hw.GetStencilClearValue(),
-		     hw.GetLineWidth());
+		     hw.GetLineWidth(), hw.GetPrimitiveResetIndex());
 
 		LOGF("%s", Common::Concat(rt_print("RenderTraget:", rt), "").c_str());
 

src/graphics/host_gpu/renderer/debug.h

@@ -9,7 +9,7 @@
 
 namespace Libs::Graphics {
 
-class RenderCommandBuffer;
+class CommandBuffer;
 
 namespace HW {
 class Context;
@@ -28,15 +28,15 @@ struct ScissorRect {
 };
 
 uint32_t                 render_target_mask_slot(uint32_t mask, uint32_t slot);
-uint32_t                 render_target_first_bound_slot(const RenderCommandBuffer& buffer);
+uint32_t                 render_target_first_bound_slot(const CommandBuffer& buffer);
 bool                     graphics_debug_dump_enabled();
 void                     uc_print(const char* func, const HW::UserConfig& uc);
 void                     uc_check(const HW::UserConfig& uc);
 void                     sh_print(const char* func, const HW::Shader& uc);
 std::vector<std::string> rt_print(const char* func, const HW::RenderTarget& rt);
 bool                     RenderIsColorTileModeLinear(Prospero::TileMode tile_mode);
-void                     hw_print(const RenderCommandBuffer& buffer);
-void                     hw_check(const RenderCommandBuffer& buffer);
+void                     hw_print(const CommandBuffer& buffer);
+void                     hw_check(const CommandBuffer& buffer);
 void                     LogDrawPhase(const char* draw_name, const char* phase);
 ScissorRect calc_final_scissor(const HW::ScreenViewport& vp, const HW::ScanModeControl& smc,
                                vk::Extent2D extent);

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -13,7 +13,6 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
@@ -71,8 +70,8 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
 	return fail == replace_op || pass == replace_op || depth_fail == replace_op;
 }
 
-[[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(const RenderCommandBuffer& buffer,
-                                                                 const DepthFormatPolicy&   policy,
+[[nodiscard]] static vk::Format ResolveHostDepthAttachmentFormat(const CommandBuffer&     buffer,
+                                                                 const DepthFormatPolicy& policy,
                                                                  bool     has_stencil,
                                                                  uint32_t samples) {
 	auto&      graphics         = buffer.GetGraphics();
@@ -98,7 +97,7 @@ static bool UsesStencilOpValue(uint8_t fail, uint8_t pass, uint8_t depth_fail) {
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer& buffer,
                                               RenderDepthInfo& r) {
 	KYTY_PROFILER_FUNCTION();
 	(void)submit_id;
@@ -166,18 +165,18 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 			DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
 			           z.depth_view.slice_max);
 	}
-	if (rc.resummarize_enable || rc.copy_centroid || rc.copy_sample != 0 ||
-	    z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
-	    z.z_info.partially_resident || z.stencil_info.partially_resident ||
-	    z.z_info.max_mip_level != 0 || z.depth_view.current_mip_level != 0 ||
-	    z.depth_info.addr5_swizzle_mask != 0 || z.depth_info.array_mode != 0 ||
-	    z.depth_info.pipe_config != 0 || z.depth_info.bank_width != 0 ||
-	    z.depth_info.bank_height != 0 || z.depth_info.macro_tile_aspect != 0 ||
-	    z.depth_info.num_banks != 0 || z.htile_surface.linear != 0 ||
-	    z.htile_surface.full_cache != 0 || z.htile_surface.htile_uses_preload_win != 0 ||
-	    z.htile_surface.preload != 0 || z.htile_surface.prefetch_width != 0 ||
-	    z.htile_surface.prefetch_height != 0 || z.htile_surface.dst_outside_zero_to_one != 0 ||
-	    z.z_read_base_addr == 0 || z.z_write_base_addr != z.z_read_base_addr ||
+	if (rc.copy_centroid || rc.copy_sample != 0 || z.z_info.expclear_enabled ||
+	    z.stencil_info.expclear_enabled || z.z_info.partially_resident ||
+	    z.stencil_info.partially_resident || z.z_info.max_mip_level != 0 ||
+	    z.depth_view.current_mip_level != 0 || z.depth_info.addr5_swizzle_mask != 0 ||
+	    z.depth_info.array_mode != 0 || z.depth_info.pipe_config != 0 ||
+	    z.depth_info.bank_width != 0 || z.depth_info.bank_height != 0 ||
+	    z.depth_info.macro_tile_aspect != 0 || z.depth_info.num_banks != 0 ||
+	    z.htile_surface.linear != 0 || z.htile_surface.full_cache != 0 ||
+	    z.htile_surface.htile_uses_preload_win != 0 || z.htile_surface.preload != 0 ||
+	    z.htile_surface.prefetch_width != 0 || z.htile_surface.prefetch_height != 0 ||
+	    z.htile_surface.dst_outside_zero_to_one != 0 || z.z_read_base_addr == 0 ||
+	    (!z.depth_view.depth_write_disable && z.z_write_base_addr != z.z_read_base_addr) ||
 	    (z.z_read_base_addr & 0xffffu) != 0 ||
 	    dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways)) {
 		DepthFatal("unsupported depth register state");
@@ -185,8 +184,9 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	if (has_stencil) {
 		if (z.stencil_info.format != Prospero::StencilFormat::k8UInt || !htile_stencil_compat ||
 		    z.stencil_read_base_addr == 0 ||
-		    z.stencil_write_base_addr != z.stencil_read_base_addr ||
-		    (z.stencil_read_base_addr & 0xffffu) != 0 || z.depth_view.stencil_write_disable) {
+		    (!z.depth_view.stencil_write_disable &&
+		     z.stencil_write_base_addr != z.stencil_read_base_addr) ||
+		    (z.stencil_read_base_addr & 0xffffu) != 0) {
 			DepthFatal("unsupported stencil attachment state");
 		}
 	} else if (z.stencil_read_base_addr != 0 || z.stencil_write_base_addr != 0 ||
@@ -287,18 +287,22 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 	r.depth_min_bounds         = hw.GetDepthBoundsMin();
 	r.depth_max_bounds         = hw.GetDepthBoundsMax();
 
-	r.stencil_clear_enable = has_stencil && rc.stencil_clear_enable;
-	r.stencil_clear_value  = hw.GetStencilClearValue();
-	r.stencil_test_enable  = has_stencil && dc.stencil_enable;
+	r.stencil_clear_enable =
+	    has_stencil && rc.stencil_clear_enable && !z.depth_view.stencil_write_disable;
+	r.stencil_clear_value = hw.GetStencilClearValue();
+	r.stencil_test_enable = has_stencil && dc.stencil_enable;
 	if (r.stencil_test_enable) {
-		const uint8_t front_write_mask = rc.stencil_clear_enable ? 0 : sm.stencil_writemask;
-		const uint8_t back_write_mask  = rc.stencil_clear_enable ? 0 : sm.stencil_writemask_bf;
+		const bool stencil_ops_disabled =
+		    rc.stencil_clear_enable || z.depth_view.stencil_write_disable;
+		const uint8_t front_write_mask = stencil_ops_disabled ? 0 : sm.stencil_writemask;
+		const uint8_t back_write_mask  = stencil_ops_disabled ? 0 : sm.stencil_writemask_bf;
 		if (dc.stencilfunc > static_cast<uint8_t>(vk::CompareOp::eAlways) ||
 		    (dc.backface_enable &&
 		     dc.stencilfunc_bf > static_cast<uint8_t>(vk::CompareOp::eAlways)) ||
-		    (UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
+		    (front_write_mask != 0 &&
+		     UsesStencilOpValue(sc.stencil_fail, sc.stencil_zpass, sc.stencil_zfail) &&
 		     sm.stencil_opval != sm.stencil_testval) ||
-		    (dc.backface_enable &&
+		    (dc.backface_enable && back_write_mask != 0 &&
 		     UsesStencilOpValue(sc.stencil_fail_bf, sc.stencil_zpass_bf, sc.stencil_zfail_bf) &&
 		     sm.stencil_opval_bf != sm.stencil_testval_bf)) {
 			DepthFatal("unsupported stencil compare or replacement state");

src/graphics/host_gpu/renderer/depthRenderTarget.h

@@ -11,8 +11,6 @@
 
 namespace Libs::Graphics {
 
-class RenderCommandBuffer;
-
 inline constexpr bool depth_htile_stencil_acceleration_compatible(bool has_stencil, bool has_htile,
                                                                   bool htile_stencil_disabled) {
 	return htile_stencil_disabled || (has_stencil && has_htile);

src/graphics/host_gpu/renderer/image/image.cpp

@@ -736,15 +736,10 @@ Image::~Image() {
 	if (m_graphics == nullptr) {
 		return;
 	}
-	{
-		std::lock_guard lock(views.mutex);
-		for (auto& cached: views.views) {
-			if (cached.view != nullptr) {
-				m_graphics->device.destroyImageView(cached.view, nullptr);
-				cached.view = nullptr;
-			}
+	for (const auto& cached: views) {
+		if (cached.view != nullptr) {
+			m_graphics->device.destroyImageView(cached.view, nullptr);
 		}
-		views.views.clear();
 	}
 	if (backing.image != nullptr) {
 		m_graphics->DeleteImage(backing);

src/graphics/host_gpu/renderer/image/image.h

@@ -2,12 +2,12 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_IMAGE_H_
 
 #include "common/assert.h"
+#include "common/slotVector.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
 
 #include <compare>
 #include <limits>
-#include <mutex>
 #include <optional>
 #include <span>
 #include <utility>
@@ -19,29 +19,13 @@ class Buffer;
 class CommandScheduler;
 struct ImageTestAccess;
 
-struct ImageId {
-	uint32_t index      = std::numeric_limits<uint32_t>::max();
-	uint32_t generation = 0;
-
-	[[nodiscard]] explicit operator bool() const noexcept {
-		return index != std::numeric_limits<uint32_t>::max();
-	}
-	auto operator<=>(const ImageId&) const = default;
-};
+using ImageId = Common::SlotId;
 
 struct CachedImageView {
 	ImageViewInfo info;
 	vk::ImageView view = nullptr;
 };
 
-struct ImageViewCache {
-	ImageViewCache() = default;
-	KYTY_CLASS_NO_COPY(ImageViewCache);
-
-	std::mutex                   mutex;
-	std::vector<CachedImageView> views;
-};
-
 struct ImageUsage {
 	bool texture       = false;
 	bool storage       = false;
@@ -55,6 +39,7 @@ struct ImageBinding {
 	bool is_target     = false;
 	bool needs_rebind  = false;
 	bool force_general = false;
+	bool shader_write  = false;
 };
 
 class Image final {
@@ -156,7 +141,7 @@ public:
 
 	ImageInfo        info;
 	VulkanImage      backing;
-	ImageViewCache   views;
+	std::vector<CachedImageView> views;
 	ImageUsage       usage;
 	ImageBinding     binding;
 	bool             registered     = false;

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -212,7 +212,6 @@ struct DepthFormatPolicy {
 	Prospero::DepthFormat     depth_format;
 	Prospero::BufferFormat    guest_format;
 	uint32_t                  bytes_per_element;
-	vk::Format                sampled_view_format;
 	vk::Format                depth_attachment_format;
 	std::array<vk::Format, 3> stencil_attachment_formats;
 };
@@ -221,13 +220,11 @@ inline constexpr std::array<DepthFormatPolicy, 2> DEPTH_FORMAT_POLICIES {{
     {Prospero::DepthFormat::kZ16,
      Prospero::BufferFormat::k16UNorm,
      2,
-     vk::Format::eR16Unorm,
      vk::Format::eD16Unorm,
      {vk::Format::eD16UnormS8Uint, vk::Format::eD24UnormS8Uint, vk::Format::eD32SfloatS8Uint}},
     {Prospero::DepthFormat::kZ32F,
      Prospero::BufferFormat::k32Float,
      4,
-     vk::Format::eR32Sfloat,
      vk::Format::eD32Sfloat,
      {vk::Format::eD32SfloatS8Uint, vk::Format::eUndefined, vk::Format::eUndefined}},
 }};
@@ -317,25 +314,6 @@ inline bool ImageInfo::IsDepth() const noexcept {
 	return std::bit_cast<uint32_t>(static_cast<float>(value) / 65535.0f);
 }
 
-[[nodiscard]] inline constexpr bool
-IsSupportedSampledDepthFormat(vk::Format image_format, Prospero::BufferFormat guest_format,
-                              vk::Format view_format) noexcept {
-	const auto* policy = FindGuestDepthFormatPolicy(guest_format);
-	return policy != nullptr && view_format == policy->sampled_view_format &&
-	       (image_format == policy->depth_attachment_format ||
-	        IsStencilAttachmentFormat(*policy, image_format));
-}
-
-[[nodiscard]] inline constexpr bool IsSupportedSampledDepthFormat(vk::Format image_format,
-                                                                  vk::Format view_format) noexcept {
-	for (const auto& policy: DEPTH_FORMAT_POLICIES) {
-		if (IsSupportedSampledDepthFormat(image_format, policy.guest_format, view_format)) {
-			return true;
-		}
-	}
-	return false;
-}
-
 [[nodiscard]] inline constexpr bool IsSupportedDepthTargetFormat(const ImageInfo& info) {
 	const auto* policy = FindGuestDepthFormatPolicy(info.guest_format);
 	return policy != nullptr && info.bytes_per_block == policy->bytes_per_element &&

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -4,8 +4,6 @@
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/image.h"
 
-#include <mutex>
-
 namespace Libs::Graphics {
 
 namespace {
@@ -36,17 +34,6 @@ namespace {
 	}
 }
 
-[[nodiscard]] bool IsDepthViewFormat(vk::Format format) {
-	switch (format) {
-		case vk::Format::eD16Unorm:
-		case vk::Format::eR16Unorm:
-		case vk::Format::eD32Sfloat:
-		case vk::Format::eR32Sfloat:
-		case vk::Format::eR32Uint: return true;
-		default: return false;
-	}
-}
-
 [[nodiscard]] bool IsValidViewType(const VulkanImage& image, const ImageViewInfo& info) {
 	switch (image.image_type) {
 		case vk::ImageType::e1D:
@@ -327,13 +314,13 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 	const auto& image      = backing;
 	auto        normalized = view_info;
 	const bool  is_storage = static_cast<bool>(normalized.usage & vk::ImageUsageFlagBits::eStorage);
-	normalized.aspect      = FullAspectMask(image.format);
-	if (normalized.aspect & vk::ImageAspectFlagBits::eDepth &&
-	    IsDepthViewFormat(normalized.format)) {
+	const auto  image_aspect = FullAspectMask(image.format);
+	if (image_aspect & vk::ImageAspectFlagBits::eDepth &&
+	    ImageViewOps::IsFormatDepthCompatible(normalized.format)) {
 		normalized.format = image.format;
 		normalized.aspect = vk::ImageAspectFlagBits::eDepth;
 	}
-	if (normalized.aspect & vk::ImageAspectFlagBits::eStencil &&
+	if (image_aspect & vk::ImageAspectFlagBits::eStencil &&
 	    IsStencilViewFormat(normalized.format)) {
 		normalized.format = image.format;
 		normalized.aspect = vk::ImageAspectFlagBits::eStencil;
@@ -368,8 +355,7 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     image.layers);
 	}
 
-	std::lock_guard lock(views.mutex);
-	for (const auto& cached: views.views) {
+	for (const auto& cached: views) {
 		if (cached.info == normalized) {
 			return cached.view;
 		}
@@ -405,7 +391,7 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 		     view_info.level_count, view_info.base_layer, view_info.layer_count,
 		     static_cast<vk::ImageUsageFlags::MaskType>(view_info.usage));
 	}
-	views.views.push_back({normalized, view});
+	views.push_back({normalized, view});
 	return view;
 }
 

src/graphics/host_gpu/renderer/image/imageView.h

@@ -12,8 +12,25 @@ namespace ImageViewOps {
 
 [[nodiscard]] vk::ImageAspectFlags DepthAspectMask(vk::Format format);
 [[nodiscard]] bool                 FormatsCompatible(vk::Format base, vk::Format view) noexcept;
+
+[[nodiscard]] inline bool IsFormatDepthCompatible(vk::Format format) noexcept {
+	switch (format) {
+		case vk::Format::eD32Sfloat:
+		case vk::Format::eR32Sfloat:
+		case vk::Format::eR32Uint:
+		case vk::Format::eD16Unorm:
+		case vk::Format::eR16Unorm: return true;
+		default: return false;
+	}
+}
 } // namespace ImageViewOps
 
+[[nodiscard]] inline bool IsSupportedSampledDepthFormat(vk::Format image_format,
+                                                        vk::Format view_format) noexcept {
+	return DepthAspectTransferFormat(image_format) != vk::Format::eUndefined &&
+	       ImageViewOps::IsFormatDepthCompatible(view_format);
+}
+
 [[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle) noexcept {
 	if ((swizzle & ~0xfffu) != 0) {
 		return false;
@@ -86,8 +103,14 @@ SelectSampledDepthView(vk::Format image_format, vk::Format view_format, uint32_t
 
 [[nodiscard]] inline bool
 IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
-	return resource.kind == ShaderRecompiler::IR::ResourceKind::Image &&
-	       (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
+	if (resource.kind != ShaderRecompiler::IR::ResourceKind::Image &&
+	    resource.kind != ShaderRecompiler::IR::ResourceKind::ImageUint) {
+		return false;
+	}
+	if (resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint && resource.depth_compare) {
+		return false;
+	}
+	return (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray) &&
@@ -95,14 +118,6 @@ IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resou
 	       !resource.written && !resource.atomic;
 }
 
-[[nodiscard]] inline bool
-IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
-	return resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint &&
-	       resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read &&
-	       !resource.written && !resource.atomic && !resource.depth_compare;
-}
-
 inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
                                      uint32_t swizzle) noexcept {
 	if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
@@ -120,7 +135,11 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
-	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
+	       ((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
+	         resource.mip_count == 1u) ||
+	        (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage &&
+	         resource.mip_count != 0u)) &&
+	       resource.written &&
 	       (!resource.atomic ||
 	        (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
 	         resource.read)) &&

src/graphics/host_gpu/renderer/image/textureCommon.cpp

@@ -119,13 +119,16 @@ vk::ComponentMapping TextureGetComponentMapping(uint32_t swizzle) {
 	return components;
 }
 
-vk::Format TextureGetFormat(Prospero::BufferFormat format) {
-	const auto vk_format = VulkanFormat(format);
+SurfaceFormatInfo TextureGetSurfaceFormatInfo(Prospero::BufferFormat format) {
+	const auto backing_format    = Prospero::RemapTextureFormat(format);
+	const auto vk_format         = VulkanFormat(backing_format);
+	const auto conversion_format =
+	    backing_format != format ? format : Prospero::BufferFormat::kInvalid;
 	if (vk_format != vk::Format::eUndefined) {
-		return vk_format;
+		return SurfaceFormatInfo(vk_format, conversion_format);
 	}
 	EXIT("unknown format: fmt = %u\n", static_cast<uint32_t>(format));
-	return vk::Format::eUndefined;
+	return SurfaceFormatInfo(vk::Format::eUndefined, Prospero::BufferFormat::kInvalid);
 }
 
 namespace {

src/graphics/host_gpu/renderer/image/textureCommon.h

@@ -3,7 +3,7 @@
 
 #include "common/abi.h"
 #include "common/common.h"
-#include "graphics/guest_gpu/gpu_defs.h"
+#include "graphics/guest_gpu/gpu_format.h"
 #include "graphics/guest_gpu/tile.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
@@ -19,6 +19,15 @@ struct RenderTargetFormatInfo {
 	Prospero::ColorComponentMapping export_mapping;
 };
 
+struct SurfaceFormatInfo {
+	explicit constexpr SurfaceFormatInfo(vk::Format format,
+	                                     Prospero::BufferFormat conversion_format)
+	    : vk_format(format), conversion_format(conversion_format) {}
+
+	vk::Format             vk_format;
+	Prospero::BufferFormat conversion_format;
+};
+
 struct TextureUploadMipLayout {
 	uint64_t offset       = 0;
 	uint64_t size         = 0;
@@ -35,7 +44,7 @@ struct TextureUploadLayout {
 };
 
 vk::ComponentMapping   TextureGetComponentMapping(uint32_t swizzle);
-vk::Format             TextureGetFormat(Prospero::BufferFormat format);
+SurfaceFormatInfo      TextureGetSurfaceFormatInfo(Prospero::BufferFormat format);
 RenderTargetFormatInfo TextureGetRenderTargetFormat(Prospero::ChannelLayout layout,
                                                     Prospero::ChannelType   type,
                                                     Prospero::ChannelOrder  order);

src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp

@@ -1,300 +0,0 @@
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
-
-#include "common/assert.h"
-#include "common/profiler.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-#include <array>
-
-namespace Libs::Graphics {
-namespace {
-
-using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind;
-
-vk::ShaderStageFlags StageFlags(DescriptorCache::Stage stage) {
-	switch (stage) {
-		case DescriptorCache::Stage::Vertex: return vk::ShaderStageFlagBits::eVertex;
-		case DescriptorCache::Stage::Pixel: return vk::ShaderStageFlagBits::eFragment;
-		case DescriptorCache::Stage::Compute: return vk::ShaderStageFlagBits::eCompute;
-		default: EXIT("unknown descriptor stage\n");
-	}
-}
-
-bool IsSampledImage(BindingKind kind) {
-	switch (kind) {
-		case BindingKind::Sampled1D:
-		case BindingKind::Sampled1DArray:
-		case BindingKind::Sampled2D:
-		case BindingKind::Sampled2DArray:
-		case BindingKind::Sampled2DMsaa:
-		case BindingKind::Sampled2DMsaaArray:
-		case BindingKind::Sampled3D:
-		case BindingKind::SampledUint1D:
-		case BindingKind::SampledUint1DArray:
-		case BindingKind::SampledUint2D:
-		case BindingKind::SampledUint2DArray:
-		case BindingKind::SampledUint2DMsaa:
-		case BindingKind::SampledUint2DMsaaArray:
-		case BindingKind::SampledUint3D: return true;
-		default: return false;
-	}
-}
-
-bool IsStorageImage(BindingKind kind) {
-	switch (kind) {
-		case BindingKind::Storage1D:
-		case BindingKind::Storage1DArray:
-		case BindingKind::Storage2D:
-		case BindingKind::Storage2DArray:
-		case BindingKind::Storage3D:
-		case BindingKind::StorageUint1D:
-		case BindingKind::StorageUint1DArray:
-		case BindingKind::StorageUint2D:
-		case BindingKind::StorageUint2DArray:
-		case BindingKind::StorageUint3D: return true;
-		default: return false;
-	}
-}
-
-vk::DescriptorType DescriptorType(BindingKind kind) {
-	if (kind == BindingKind::Samplers) {
-		return vk::DescriptorType::eSampler;
-	}
-	if (IsSampledImage(kind)) {
-		return vk::DescriptorType::eSampledImage;
-	}
-	if (IsStorageImage(kind)) {
-		return vk::DescriptorType::eStorageImage;
-	}
-	return vk::DescriptorType::eStorageBuffer;
-}
-
-uint32_t DescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
-	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
-}
-
-std::vector<uint32_t> LayoutKey(DescriptorCache::Stage               stage,
-                                const ShaderRecompiler::IR::Program& program) {
-	std::vector<uint32_t> key;
-	key.reserve(1u + program.bindings.descriptors.size() * 4u);
-	key.push_back(static_cast<uint32_t>(stage));
-	for (const auto& binding: program.bindings.descriptors) {
-		key.push_back(static_cast<uint32_t>(binding.kind));
-		key.push_back(binding.binding);
-		key.push_back(DescriptorCount(binding));
-		key.push_back(static_cast<uint32_t>(DescriptorType(binding.kind)));
-	}
-	return key;
-}
-
-vk::DescriptorBufferInfo BufferInfo(const BufferView& view) {
-	EXIT_IF(view.buffer == nullptr);
-	return {view.buffer, view.offset, view.range};
-}
-
-} // namespace
-
-vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture) {
-	EXIT_IF(!texture.image_id || texture.image_view == nullptr ||
-	        texture.layout == vk::ImageLayout::eUndefined);
-	return {nullptr, texture.image_view, texture.layout};
-}
-
-DescriptorCache::~DescriptorCache() {
-	for (auto& [layout, sets]: m_free_sets_by_layout) {
-		(void)layout;
-		for (auto* set: sets) {
-			delete set;
-		}
-	}
-	for (const auto& [key, layout]: m_descriptor_set_layouts) {
-		(void)key;
-		m_graphics.device.destroyDescriptorSetLayout(layout, nullptr);
-	}
-	for (const auto& pool: m_pools) {
-		m_graphics.device.destroyDescriptorPool(pool.pool, nullptr);
-	}
-}
-
-vk::DescriptorSetLayout
-DescriptorCache::GetDescriptorSetLayoutInternal(Stage                                stage,
-                                                const ShaderRecompiler::IR::Program& program) {
-	const auto key = LayoutKey(stage, program);
-	if (const auto found = m_descriptor_set_layouts.find(key);
-	    found != m_descriptor_set_layouts.end()) {
-		return found->second;
-	}
-
-	std::vector<vk::DescriptorSetLayoutBinding> bindings;
-	bindings.reserve(program.bindings.descriptors.size());
-	for (const auto& descriptor: program.bindings.descriptors) {
-		bindings.push_back({descriptor.binding, DescriptorType(descriptor.kind),
-		                    DescriptorCount(descriptor), StageFlags(stage), nullptr});
-	}
-	if (bindings.empty()) {
-		return nullptr;
-	}
-
-	vk::DescriptorSetLayoutCreateInfo info {};
-	info.sType                     = vk::StructureType::eDescriptorSetLayoutCreateInfo;
-	info.bindingCount              = static_cast<uint32_t>(bindings.size());
-	info.pBindings                 = bindings.data();
-	vk::DescriptorSetLayout layout = nullptr;
-	const auto result = m_graphics.device.createDescriptorSetLayout(&info, nullptr, &layout);
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || layout == nullptr);
-	m_descriptor_set_layouts.emplace(key, layout);
-	return layout;
-}
-
-void DescriptorCache::CreatePool() {
-	KYTY_PROFILER_FUNCTION();
-	constexpr uint32_t           MaxSets = 512;
-	const vk::DescriptorPoolSize sizes[] = {
-	    {vk::DescriptorType::eStorageBuffer,
-	     MaxSets * (ShaderRecompiler::IR::ShaderInfo::MaxBuffers +
-	                ShaderRecompiler::IR::ShaderInfo::MaxAddresses + 3u)},
-	    {vk::DescriptorType::eSampledImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
-	    {vk::DescriptorType::eStorageImage, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxImages},
-	    {vk::DescriptorType::eSampler, MaxSets * ShaderRecompiler::IR::ShaderInfo::MaxSamplers},
-	};
-	vk::DescriptorPoolCreateInfo info {};
-	info.sType         = vk::StructureType::eDescriptorPoolCreateInfo;
-	info.poolSizeCount = static_cast<uint32_t>(std::size(sizes));
-	info.pPoolSizes    = sizes;
-	info.maxSets       = MaxSets;
-	const auto pool_id = static_cast<int>(m_pools.size());
-	auto&      pool    = m_pools.emplace_back();
-	const auto result  = m_graphics.device.createDescriptorPool(&info, nullptr, &pool.pool);
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess || pool.pool == nullptr);
-	pool.next_free_pool = m_first_free_pool;
-	m_first_free_pool   = pool_id;
-}
-
-VulkanDescriptorSet* DescriptorCache::Allocate(Stage                                stage,
-                                               const ShaderRecompiler::IR::Program& program) {
-	KYTY_PROFILER_FUNCTION();
-	Common::LockGuard lock(m_mutex);
-	const auto        layout = GetDescriptorSetLayoutInternal(stage, program);
-	EXIT_IF(layout == nullptr);
-	auto& free_sets = m_free_sets_by_layout[layout];
-	if (!free_sets.empty()) {
-		auto* result = free_sets.back();
-		free_sets.pop_back();
-		return result;
-	}
-
-	constexpr uint32_t Batch = 32;
-	for (int attempt = 0; attempt < 2; attempt++) {
-		for (int pool_id = m_first_free_pool; pool_id != -1;
-		     pool_id     = m_pools[pool_id].next_free_pool) {
-			auto&                                      pool = m_pools[pool_id];
-			std::array<vk::DescriptorSetLayout, Batch> layouts;
-			layouts.fill(layout);
-			std::array<vk::DescriptorSet, Batch> sets {};
-			vk::DescriptorSetAllocateInfo        info {};
-			info.sType              = vk::StructureType::eDescriptorSetAllocateInfo;
-			info.descriptorPool     = pool.pool;
-			info.descriptorSetCount = Batch;
-			info.pSetLayouts        = layouts.data();
-			if (m_graphics.device.allocateDescriptorSets(&info, sets.data()) ==
-			    vk::Result::eSuccess) {
-				free_sets.reserve(free_sets.size() + Batch - 1u);
-				for (uint32_t i = 0; i + 1u < Batch; i++) {
-					free_sets.push_back(new VulkanDescriptorSet {sets[i], layout, pool_id});
-				}
-				return new VulkanDescriptorSet {sets.back(), layout, pool_id};
-			}
-			m_first_free_pool = pool.next_free_pool;
-			break;
-		}
-		CreatePool();
-	}
-	return nullptr;
-}
-
-void DescriptorCache::Recycle(VulkanDescriptorSet& set) {
-	EXIT_IF(set.set == nullptr || set.layout == nullptr);
-	Common::LockGuard lock(m_mutex);
-	m_free_sets_by_layout[set.layout].push_back(&set);
-}
-
-VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage                                stage,
-                                                    const ShaderRecompiler::IR::Program& program,
-                                                    const NativeDescriptors&             data) {
-	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(data.buffers.size() != program.info.buffers.size() ||
-	        data.images.size() != program.info.images.size() ||
-	        data.samplers.size() != program.info.samplers.size() ||
-	        data.addresses.size() != program.info.addresses.size());
-	auto* set = Allocate(stage, program);
-	EXIT_NOT_IMPLEMENTED(set == nullptr);
-
-	const auto descriptor_count = program.info.buffers.size() + program.info.images.size() +
-	                              program.info.samplers.size() + program.info.addresses.size() + 3u;
-	std::vector<vk::DescriptorBufferInfo> buffer_infos;
-	std::vector<vk::DescriptorImageInfo>  image_infos;
-	std::vector<vk::WriteDescriptorSet>   writes;
-	buffer_infos.reserve(descriptor_count);
-	image_infos.reserve(descriptor_count);
-	writes.reserve(program.bindings.descriptors.size());
-
-	for (const auto& binding: program.bindings.descriptors) {
-		vk::WriteDescriptorSet write {};
-		write.sType             = vk::StructureType::eWriteDescriptorSet;
-		write.dstSet            = set->set;
-		write.dstBinding        = binding.binding;
-		write.descriptorType    = DescriptorType(binding.kind);
-		write.descriptorCount   = DescriptorCount(binding);
-		const auto buffer_start = buffer_infos.size();
-		const auto image_start  = image_infos.size();
-		switch (binding.kind) {
-			case BindingKind::Buffers:
-				for (const auto resource: binding.resources) {
-					buffer_infos.push_back(BufferInfo(data.buffers.at(resource)));
-				}
-				break;
-			case BindingKind::AddressMemory:
-				for (const auto resource: binding.resources) {
-					buffer_infos.push_back(BufferInfo(data.addresses.at(resource)));
-				}
-				break;
-			case BindingKind::FlattenedSrt:
-				buffer_infos.push_back(BufferInfo(data.flattened_srt));
-				break;
-			case BindingKind::UserData: buffer_infos.push_back(BufferInfo(data.user_data)); break;
-			case BindingKind::Gds: buffer_infos.push_back(BufferInfo(data.gds)); break;
-			case BindingKind::Samplers:
-				for (const auto resource: binding.resources) {
-					const auto sampler = data.samplers.at(resource);
-					EXIT_IF(sampler == nullptr);
-					image_infos.push_back({sampler, nullptr, vk::ImageLayout::eUndefined});
-				}
-				break;
-			default: {
-				for (const auto resource: binding.resources) {
-					const auto& texture = data.images.at(resource);
-					image_infos.push_back(MakeImageInfo(texture));
-				}
-				break;
-			}
-		}
-		if (buffer_infos.size() != buffer_start) {
-			write.pBufferInfo = buffer_infos.data() + buffer_start;
-		}
-		if (image_infos.size() != image_start) {
-			write.pImageInfo = image_infos.data() + image_start;
-		}
-		writes.push_back(write);
-	}
-	m_graphics.device.updateDescriptorSets(static_cast<uint32_t>(writes.size()), writes.data(), 0,
-	                                       nullptr);
-	return *set;
-}
-
-vk::DescriptorSetLayout
-DescriptorCache::GetDescriptorSetLayout(Stage stage, const ShaderRecompiler::IR::Program& program) {
-	Common::LockGuard lock(m_mutex);
-	return GetDescriptorSetLayoutInternal(stage, program);
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptorCache.h

@@ -1,114 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_
-
-#include "common/abi.h"
-#include "common/assert.h"
-#include "common/common.h"
-#include "common/threads.h"
-#include "graphics/host_gpu/graphicContext.h"
-#include "graphics/host_gpu/renderer/cache/textureCache.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/shaderBindings.h"
-
-#include <map>
-#include <memory>
-#include <span>
-#include <unordered_map>
-#include <vector>
-
-namespace Libs::Graphics {
-
-namespace ShaderRecompiler::IR {
-struct DescriptorValue;
-struct ImageResource;
-struct Program;
-struct ResourceSnapshot;
-} // namespace ShaderRecompiler::IR
-
-class CommandBuffer;
-struct DescriptorCacheTestAccess;
-struct ShaderStageRuntime;
-
-struct VulkanDescriptorSet {
-	vk::DescriptorSet       set     = nullptr;
-	vk::DescriptorSetLayout layout  = nullptr;
-	int                     pool_id = -1;
-};
-
-struct BufferView {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	vk::DeviceSize        offset = 0;
-	vk::DeviceSize        range  = VK_WHOLE_SIZE;
-};
-
-class DescriptorCache {
-public:
-	enum class Stage { Unknown, Vertex, Pixel, Compute };
-
-	struct TextureBinding {
-		ImageId                 image_id;
-		vk::ImageView           image_view = nullptr;
-		TextureCache::ImageDesc desc;
-		vk::ImageLayout         layout = vk::ImageLayout::eUndefined;
-	};
-
-	struct NativeDescriptors {
-		std::vector<BufferView>     buffers;
-		std::vector<TextureBinding> images;
-		std::vector<vk::Sampler>    samplers;
-		std::vector<BufferView>     addresses;
-		BufferView                  gds;
-		BufferView                  flattened_srt;
-		BufferView                  user_data;
-	};
-
-	struct PreparedBindings {
-		std::shared_ptr<const ShaderRecompiler::IR::Program>          program;
-		std::shared_ptr<const ShaderRecompiler::IR::ResourceSnapshot> snapshot;
-		NativeDescriptors                                             resources;
-		std::vector<uint32_t>                                         flattened_srt;
-		std::vector<uint32_t>                                         user_data;
-		vk::ShaderStageFlags                                          shader_stage;
-		Stage                                                         stage     = Stage::Unknown;
-		bool                                                          committed = false;
-	};
-
-	explicit DescriptorCache(GraphicContext& graphics): m_graphics(graphics) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	}
-	~DescriptorCache();
-	KYTY_CLASS_NO_COPY(DescriptorCache);
-
-	vk::DescriptorSetLayout GetDescriptorSetLayout(Stage                                stage,
-	                                               const ShaderRecompiler::IR::Program& program);
-	void                    Recycle(VulkanDescriptorSet& set);
-	VulkanDescriptorSet&    GetDescriptor(Stage stage, const ShaderRecompiler::IR::Program& program,
-	                                      const NativeDescriptors& descriptors);
-
-private:
-	friend struct DescriptorCacheTestAccess;
-
-	struct Pool {
-		vk::DescriptorPool pool           = nullptr;
-		int                next_free_pool = -1;
-	};
-
-	static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture);
-	void                           CreatePool();
-	VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program);
-	vk::DescriptorSetLayout
-	GetDescriptorSetLayoutInternal(Stage stage, const ShaderRecompiler::IR::Program& program);
-
-	GraphicContext&   m_graphics;
-	Common::Mutex     m_mutex;
-	std::vector<Pool> m_pools;
-	int               m_first_free_pool = -1;
-	std::unordered_map<vk::DescriptorSetLayout, std::vector<VulkanDescriptorSet*>>
-	                                                         m_free_sets_by_layout;
-	std::map<std::vector<uint32_t>, vk::DescriptorSetLayout> m_descriptor_set_layouts;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORCACHE_H_

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.cpp

@@ -0,0 +1,97 @@
+#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
+
+#include "common/assert.h"
+#include "common/profiler.h"
+#include "graphics/host_gpu/graphicContext.h"
+#include "graphics/host_gpu/renderer/masterSemaphore.h"
+
+namespace Libs::Graphics {
+namespace {
+
+constexpr uint32_t   DescriptorHeapCount = 1024;
+constexpr std::array DescriptorPoolSizes = {
+    vk::DescriptorPoolSize {vk::DescriptorType::eStorageBuffer, 8192},
+    vk::DescriptorPoolSize {vk::DescriptorType::eSampledImage, 8192},
+    vk::DescriptorPoolSize {vk::DescriptorType::eStorageImage, 1024},
+    vk::DescriptorPoolSize {vk::DescriptorType::eSampler, 1024},
+};
+
+} // namespace
+
+DescriptorHeap::DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore)
+    : m_graphics(graphics), m_master_semaphore(master_semaphore) {
+	CreatePool();
+}
+
+DescriptorHeap::~DescriptorHeap() {
+	m_graphics.device.destroyDescriptorPool(m_current_pool, nullptr);
+	for (const auto& [pool, tick]: m_pending_pools) {
+		m_master_semaphore.Wait(tick);
+		m_graphics.device.destroyDescriptorPool(pool, nullptr);
+	}
+}
+
+vk::DescriptorSet DescriptorHeap::Commit(vk::DescriptorSetLayout layout) {
+	KYTY_PROFILER_FUNCTION();
+	EXIT_IF(layout == nullptr);
+
+	auto& batch = m_sets[layout];
+	if (batch.size != 0) {
+		return batch.sets[--batch.size];
+	}
+	if (Allocate(layout, batch)) {
+		return batch.sets[--batch.size];
+	}
+
+	m_pending_pools.emplace_back(m_current_pool, m_master_semaphore.CurrentTick());
+	if (const auto& [pool, tick] = m_pending_pools.front(); m_master_semaphore.IsFree(tick)) {
+		m_current_pool = pool;
+		m_pending_pools.pop_front();
+		EXIT_IF(m_graphics.device.resetDescriptorPool(m_current_pool, {}) != vk::Result::eSuccess);
+	} else {
+		CreatePool();
+	}
+
+	m_sets.clear();
+	auto& fresh_batch = m_sets[layout];
+	EXIT_IF(!Allocate(layout, fresh_batch));
+	return fresh_batch.sets[--fresh_batch.size];
+}
+
+bool DescriptorHeap::Allocate(vk::DescriptorSetLayout layout, Batch& batch) {
+	std::array<vk::DescriptorSetLayout, DescriptorSetBatch> layouts;
+	layouts.fill(layout);
+
+	vk::DescriptorSetAllocateInfo allocate {};
+	allocate.sType          = vk::StructureType::eDescriptorSetAllocateInfo;
+	allocate.descriptorPool = m_current_pool;
+	allocate.pSetLayouts    = layouts.data();
+
+	for (;;) {
+		allocate.descriptorSetCount = batch.allocation;
+		const auto result = m_graphics.device.allocateDescriptorSets(&allocate, batch.sets.data());
+		if (result == vk::Result::eSuccess) {
+			batch.size = batch.allocation;
+			return true;
+		}
+		EXIT_IF(result != vk::Result::eErrorOutOfPoolMemory &&
+		        result != vk::Result::eErrorFragmentedPool);
+		if (batch.allocation == 1) {
+			return false;
+		}
+		batch.allocation /= 2;
+	}
+}
+
+void DescriptorHeap::CreatePool() {
+	vk::DescriptorPoolCreateInfo create {};
+	create.sType         = vk::StructureType::eDescriptorPoolCreateInfo;
+	create.flags         = {};
+	create.maxSets       = DescriptorHeapCount;
+	create.poolSizeCount = static_cast<uint32_t>(DescriptorPoolSizes.size());
+	create.pPoolSizes    = DescriptorPoolSizes.data();
+	EXIT_IF(m_graphics.device.createDescriptorPool(&create, nullptr, &m_current_pool) !=
+	        vk::Result::eSuccess);
+}
+
+} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptorHeap.h

@@ -0,0 +1,45 @@
+#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_
+#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_
+
+#include "common/common.h"
+#include "graphics/host_gpu/vulkanCommon.h"
+
+#include <array>
+#include <deque>
+#include <unordered_map>
+
+namespace Libs::Graphics {
+
+struct GraphicContext;
+class MasterSemaphore;
+
+class DescriptorHeap {
+public:
+	DescriptorHeap(GraphicContext& graphics, MasterSemaphore& master_semaphore);
+	~DescriptorHeap();
+	KYTY_CLASS_NO_COPY(DescriptorHeap);
+
+	[[nodiscard]] vk::DescriptorSet Commit(vk::DescriptorSetLayout layout);
+
+private:
+	static constexpr uint32_t DescriptorSetBatch = 32;
+
+	struct Batch {
+		std::array<vk::DescriptorSet, DescriptorSetBatch> sets {};
+		uint32_t                                          size       = 0;
+		uint32_t                                          allocation = DescriptorSetBatch;
+	};
+
+	[[nodiscard]] bool Allocate(vk::DescriptorSetLayout layout, Batch& batch);
+	void               CreatePool();
+
+	GraphicContext&                                     m_graphics;
+	MasterSemaphore&                                    m_master_semaphore;
+	vk::DescriptorPool                                  m_current_pool = nullptr;
+	std::deque<std::pair<vk::DescriptorPool, uint64_t>> m_pending_pools;
+	std::unordered_map<vk::DescriptorSetLayout, Batch>  m_sets;
+};
+
+} // namespace Libs::Graphics
+
+#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORHEAP_H_

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -17,16 +17,16 @@
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/image/textureCommon.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vma.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
+#include "kernel/memory.h"
 
 #include <algorithm>
 #include <atomic>
@@ -44,10 +44,99 @@
 
 namespace Libs::Graphics {
 
+namespace {
+
+using BindingKind = ShaderRecompiler::IR::DescriptorBindingKind;
+
+bool IsSampledImage(BindingKind kind) {
+	switch (kind) {
+		case BindingKind::Sampled1D:
+		case BindingKind::Sampled1DArray:
+		case BindingKind::Sampled2D:
+		case BindingKind::Sampled2DArray:
+		case BindingKind::Sampled2DMsaa:
+		case BindingKind::Sampled2DMsaaArray:
+		case BindingKind::Sampled3D:
+		case BindingKind::SampledUint1D:
+		case BindingKind::SampledUint1DArray:
+		case BindingKind::SampledUint2D:
+		case BindingKind::SampledUint2DArray:
+		case BindingKind::SampledUint2DMsaa:
+		case BindingKind::SampledUint2DMsaaArray:
+		case BindingKind::SampledUint3D: return true;
+		default: return false;
+	}
+}
+
+bool IsStorageImage(BindingKind kind) {
+	switch (kind) {
+		case BindingKind::Storage1D:
+		case BindingKind::Storage1DArray:
+		case BindingKind::Storage2D:
+		case BindingKind::Storage2DArray:
+		case BindingKind::Storage3D:
+		case BindingKind::StorageUint1D:
+		case BindingKind::StorageUint1DArray:
+		case BindingKind::StorageUint2D:
+		case BindingKind::StorageUint2DArray:
+		case BindingKind::StorageUint3D:
+		case BindingKind::StorageAtomic1D:
+		case BindingKind::StorageAtomic1DArray:
+		case BindingKind::StorageAtomic2D:
+		case BindingKind::StorageAtomic2DArray:
+		case BindingKind::StorageAtomic3D: return true;
+		default: return false;
+	}
+}
+
+} // namespace
+
+vk::DescriptorType NativeDescriptorType(BindingKind kind) {
+	if (kind == BindingKind::Samplers) {
+		return vk::DescriptorType::eSampler;
+	}
+	if (IsSampledImage(kind)) {
+		return vk::DescriptorType::eSampledImage;
+	}
+	if (IsStorageImage(kind)) {
+		return vk::DescriptorType::eStorageImage;
+	}
+	return vk::DescriptorType::eStorageBuffer;
+}
+
+uint32_t NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding) {
+	return binding.resources.empty() ? 1u : static_cast<uint32_t>(binding.resources.size());
+}
+
+vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture, uint32_t element) {
+	const auto view =
+	    texture.mip_views.empty()
+	        ? (element == 0u ? texture.image_view : vk::ImageView {})
+	        : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {});
+	EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
+	return {nullptr, view, texture.layout};
+}
+
+static const char* ShaderStageResourceName(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return "Vertex";
+		case ShaderType::Pixel: return "Pixel";
+		case ShaderType::Compute: return "Compute";
+		default: return "Unknown";
+	}
+}
+
+static vk::ShaderStageFlags NativeShaderStage(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return vk::ShaderStageFlagBits::eVertex;
+		case ShaderType::Pixel: return vk::ShaderStageFlagBits::eFragment;
+		case ShaderType::Compute: return vk::ShaderStageFlagBits::eCompute;
+		default: EXIT("unknown native shader stage\n");
+	}
+}
+
 static void BindNullStorageBuffer(RenderContext& context, BufferView& dst) {
-	auto owner = context.GetBufferCache().ObtainNullBuffer();
-	dst.buffer = owner->Handle();
-	dst.owner  = std::move(owner);
+	dst.buffer = context.GetBufferCache().GetBuffer(NULL_BUFFER_ID).Handle();
 	dst.offset = 0;
 	dst.range  = 16;
 }
@@ -78,10 +167,11 @@ static bool IsMultisampledTexture(Prospero::ImageType type) {
 	       type == Prospero::ImageType::kColor2DMsaaArray;
 }
 
-static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& command_buffer,
+static BufferView NativeStorageBuffer(RenderContext&                              context,
                                       const ShaderBufferResource&                 descriptor,
                                       const ShaderRecompiler::IR::BufferResource& resource,
-                                      uint32_t&                                   buffer_offset) {
+                                      ShaderType stage, uint32_t slot, uint32_t& buffer_offset,
+                                      BufferId id) {
 	BufferView result;
 	buffer_offset = 0;
 
@@ -91,69 +181,38 @@ static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& com
 	if (stride != 0 && records > UINT64_MAX / stride) {
 		EXIT("storage buffer descriptor footprint overflow\n");
 	}
-	const auto size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
-	if (address == 0 || size == 0) {
+	const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+	if (address == 0 || requested_size == 0) {
 		BindNullStorageBuffer(context, result);
 		return result;
 	}
+	const auto  size      = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
 	const auto& graphics  = context.GetGraphics();
 	const auto  alignment = graphics.StorageMinAlignment();
 	if (alignment == 0 ||
 	    size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange) {
 		EXIT("storage buffer range or device alignment is unsupported\n");
 	}
-	auto binding = context.GetBufferCache().ObtainBuffer(
-	    command_buffer, address, size, resource.written, resource.read, resource.formatted);
-	const auto aligned_offset = binding.offset - binding.offset % alignment;
-	const auto adjustment     = binding.offset - aligned_offset;
+	auto [buffer, offset] = context.GetBufferCache().ObtainBuffer(address, size, resource.written,
+	                                                              resource.formatted, id);
+	const auto aligned_offset = offset - offset % alignment;
+	const auto adjustment     = offset - aligned_offset;
 	const auto max_range      = graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange;
 	if (adjustment % sizeof(uint32_t) != 0 || adjustment >= 256 || size > max_range - adjustment) {
 		EXIT("storage buffer offset adjustment is unsupported\n");
 	}
 	buffer_offset = static_cast<uint32_t>(adjustment);
-	result.owner  = std::move(binding.owner);
-	result.buffer = binding.buffer;
+	result.buffer = buffer->Handle();
 	result.offset = aligned_offset;
 	result.range  = static_cast<vk::DeviceSize>(size + adjustment);
-	return result;
-}
-
-static BufferView
-NativeAddressBuffer(RenderContext& context, CommandBuffer& command_buffer,
-                    const ShaderRecompiler::IR::AddressResource&           resource,
-                    const ShaderRecompiler::IR::ResourceSnapshot::Address& address) {
-	BufferView result;
-	if (address.binding_base == 0) {
-		BindNullStorageBuffer(context, result);
-		return result;
+	if (resource.formatted && resource.written) {
+		context.GetTextureCache().InvalidateMemoryFromGPU(address, size);
 	}
-	if (resource.written) {
-		EXIT("writable address resources are unsupported\n");
-	}
-	const auto limit =
-	    resource.kind == ShaderRecompiler::IR::ResourceKind::Flat
-	        ? ShaderRecompiler::IR::FlatAddressWindowSize
-	        : static_cast<uint64_t>(
-	              context.GetGraphics().GetPhysicalDeviceProperties().limits.maxStorageBufferRange);
-	uint64_t   size   = 0;
-	const auto access = HostMemoryAccess::Mapped;
-	if (!HostMemoryQueryRange(address.binding_base, limit, access, size)) {
-		EXIT("address resource is not host-accessible: base=0x%016" PRIx64 "\n",
-		     address.binding_base);
-	}
-	const auto& graphics  = context.GetGraphics();
-	const auto  alignment = graphics.StorageMinAlignment();
-	if (alignment == 0 ||
-	    size > graphics.GetPhysicalDeviceProperties().limits.maxStorageBufferRange ||
-	    BufferCache::GetBufferOffset(address.binding_base) % alignment != 0) {
-		EXIT("address resource range or alignment is unsupported\n");
-	}
-	auto binding =
-	    context.GetBufferCache().ObtainBuffer(command_buffer, address.binding_base, size);
-	result.owner  = std::move(binding.owner);
-	result.buffer = binding.buffer;
-	result.offset = binding.offset;
-	result.range  = static_cast<vk::DeviceSize>(size);
+	SetVulkanObjectNameF(
+	    graphics.device, result.buffer,
+	    "Kyty.{}.StorageBuffer[slot={} guest=0x{:016x} size=0x{:x} access={} formatted={}]",
+	    ShaderStageResourceName(stage), slot, address, size,
+	    resource.written ? (resource.read ? "ReadWrite" : "Write") : "Read", resource.formatted);
 	return result;
 }
 
@@ -231,7 +290,8 @@ bool IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& r
 	       descriptor.BaseArray5() == 0;
 }
 
-bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image) {
+bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, const Image& image,
+                                      bool r128) {
 	const auto width        = static_cast<uint32_t>(descriptor.Width5()) + 1u;
 	const auto height       = static_cast<uint32_t>(descriptor.Height5()) + 1u;
 	const auto type         = descriptor.Type();
@@ -258,12 +318,12 @@ bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, c
 	                                  descriptor.BaseArray5() <= descriptor.Depth() &&
 	                                  descriptor.Depth() < image.info.resources.layers;
 	const bool levels_ok =
-	    multisampled
-	        ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
-	              descriptor.LastLevel() <= 3 && descriptor.MaxMip() == descriptor.LastLevel() &&
-	              image.info.resources.levels == 1 && image.info.samples == samples
-	        : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
-	              descriptor.MaxMip() == 0 && image.info.samples == 1;
+	    multisampled ? descriptor.BaseLevel() == 0 && descriptor.LastLevel() >= 1 &&
+	                       descriptor.LastLevel() <= 3 &&
+	                       (r128 || descriptor.MaxMip() == descriptor.LastLevel()) &&
+	                       image.info.resources.levels == 1 && image.info.samples == samples
+	                 : descriptor.BaseLevel() == 0 && descriptor.LastLevel() == 0 &&
+	                       (r128 || descriptor.MaxMip() == 0) && image.info.samples == 1;
 	return image.info.IsDepth() && width == image.info.extent.width &&
 	       height == image.info.extent.height &&
 	       (supported_2d || supported_array || supported_cube || supported_msaa_2d ||
@@ -273,7 +333,8 @@ bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor, c
 	       (!descriptor.MsaaDepth() || multisampled) && pitch >= width && pitch == image.info.pitch;
 }
 
-bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image) {
+bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, const Image& image,
+                                     bool r128) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
 	const uint32_t     field3_expected = descriptor.DstSelXYZW() |
@@ -286,10 +347,14 @@ bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor, co
 	                                     (static_cast<uint32_t>(descriptor.MaxMip()) << 4u);
 	const bool         common          = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
 	                                     (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	                                     descriptor.fields[3] == field3_expected &&
-	                                     descriptor.fields[4] == field4_expected &&
-	                                     descriptor.fields[5] == field5_expected;
-	if (!common || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
+	                                     descriptor.fields[3] == field3_expected;
+	if (r128) {
+		return common && descriptor.fields[4] == 0 && descriptor.fields[5] == 0 &&
+		       descriptor.fields[6] == 0 && descriptor.fields[7] == 0;
+	}
+	const bool full = common && descriptor.fields[4] == field4_expected &&
+	                  descriptor.fields[5] == field5_expected;
+	if (!full || (descriptor.fields[6] == 0 && descriptor.fields[7] != 0)) {
 		return false;
 	}
 	if (descriptor.fields[6] == 0) {
@@ -310,12 +375,11 @@ static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource
                                        vk::Format view_format, uint64_t size) {
 	const bool resource_ok = IsSupportedSampledDepthResource(resource);
 	const bool descriptor_ok =
-	    image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image);
+	    image != nullptr && IsSupportedDepthTargetDescriptor(descriptor, *image, resource.r128);
 	const bool encoding_ok =
-	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image);
+	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image, resource.r128);
 	const bool format_ok =
-	    image != nullptr &&
-	    IsSupportedSampledDepthFormat(image->info.pixel_format, descriptor.Format(), view_format);
+	    image != nullptr && IsSupportedSampledDepthFormat(image->info.pixel_format, view_format);
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
 		return;
 	}
@@ -403,13 +467,19 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_swizzle =
 	    IsValidImageSwizzle(swizzle) &&
 	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
+	const auto max_mip = resource.r128 ? descriptor.LastLevel() : descriptor.MaxMip();
+	const auto view_last_level =
+	    resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage
+	        ? descriptor.LastLevel()
+	        : std::min(descriptor.LastLevel(), max_mip);
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       descriptor.BaseLevel() == descriptor.LastLevel() &&
-	       descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
-	       supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
+	       descriptor.BaseLevel() <= view_last_level && view_last_level <= max_mip &&
+	       descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
+	       !descriptor.MsaaDepth();
 }
 
-static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descriptor) {
+static bool IsSupportedStorageTextureEncoding(const ShaderRecompiler::IR::ImageResource& resource,
+                                              const ShaderTextureResource& descriptor) {
 	constexpr uint32_t field1_reserved_mask = 0x200fff00u;
 	constexpr uint32_t field2_reserved_mask = 0xf0003000u;
 	constexpr uint32_t field5_expected      = 0x00700000u;
@@ -421,9 +491,14 @@ static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descr
 	                                     (static_cast<uint32_t>(descriptor.Type()) << 28u);
 	const uint32_t     expected_field4 =
 	    descriptor.Depth() | (static_cast<uint32_t>(descriptor.BaseArray5()) << 16u);
-	return (descriptor.fields[1] & field1_reserved_mask) == 0 &&
-	       (descriptor.fields[2] & field2_reserved_mask) == 0 &&
-	       descriptor.fields[3] == expected_field3 && descriptor.fields[4] == expected_field4 &&
+	const bool common = (descriptor.fields[1] & field1_reserved_mask) == 0 &&
+	                    (descriptor.fields[2] & field2_reserved_mask) == 0 &&
+	                    descriptor.fields[3] == expected_field3;
+	if (resource.r128) {
+		return common && descriptor.fields[4] == 0 && descriptor.fields[5] == 0 &&
+		       descriptor.fields[6] == 0 && descriptor.fields[7] == 0;
+	}
+	return common && descriptor.fields[4] == expected_field4 &&
 	       (descriptor.fields[5] & ~field5_max_mip_mask) == field5_expected;
 }
 
@@ -432,13 +507,13 @@ void ValidateStorageTexture(const ShaderRecompiler::IR::ImageResource& resource,
 	const auto format        = descriptor.Format();
 	const bool resource_ok   = IsSupportedStorageImageResource(resource);
 	const bool descriptor_ok = IsSupportedStorageTextureDescriptor(resource, descriptor);
-	const bool encoding_ok   = IsSupportedStorageTextureEncoding(descriptor);
+	const bool encoding_ok   = IsSupportedStorageTextureEncoding(resource, descriptor);
 	const bool uint_resource =
 	    resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint;
 	const bool raw_sint_storage = format == Prospero::BufferFormat::k32SInt && uint_resource &&
 	                              resource.written && !resource.read && !resource.atomic;
 	const bool format_ok =
-	    raw_sint_storage || (Prospero::IsSupportedTextureFormat(format) &&
+	    raw_sint_storage || (Prospero::IsSampledTextureFormat(format) &&
 	                         uint_resource == Prospero::IsUintTextureFormat(format) &&
 	                         (!resource.atomic || format == Prospero::BufferFormat::k32UInt));
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
@@ -547,15 +622,17 @@ static void PopulateTextureMipLayout(ImageInfo& info) {
 
 static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource& resource,
                                      const ShaderTextureResource& descriptor, vk::Format format,
-                                     bool storage, uint32_t view_levels, uint32_t image_layers) {
+                                     bool shader_conversion, bool storage, uint32_t view_levels,
+                                     uint32_t image_layers) {
 	ImageViewInfo view {};
 	view.format      = format;
 	view.aspect      = vk::ImageAspectFlagBits::eColor;
 	view.base_level  = descriptor.BaseLevel();
 	view.level_count = view_levels;
-	view.usage = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
-	view.mapping =
-	    storage ? vk::ComponentMapping {} : TextureGetComponentMapping(descriptor.DstSelXYZW());
+	view.usage   = storage ? vk::ImageUsageFlagBits::eStorage : vk::ImageUsageFlagBits::eSampled;
+	view.mapping = storage || shader_conversion
+	                   ? vk::ComponentMapping {}
+	                   : TextureGetComponentMapping(descriptor.DstSelXYZW());
 	switch (resource.dimension) {
 		case ShaderRecompiler::Decoder::ImageDimension::Dim1D:
 			view.type       = vk::ImageViewType::e1D;
@@ -601,9 +678,8 @@ static ImageViewInfo TextureViewInfo(const ShaderRecompiler::IR::ImageResource&
 	return view;
 }
 
-DescriptorCache::TextureBinding
-RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
-                               const ShaderRecompiler::IR::DescriptorValue& value) {
+TextureBinding RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
+                                              const ShaderRecompiler::IR::DescriptorValue& value) {
 	ShaderTextureResource descriptor;
 	CopyNativeDescriptor(value, descriptor.fields);
 	const bool storage = resource.written;
@@ -626,16 +702,20 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto last_level   = descriptor.LastLevel();
 	const auto type         = TextureType(descriptor);
 	const bool multisampled = IsMultisampledTexture(type);
-	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
-	const auto tile         = descriptor.TileMode();
-	const bool depth_tile   = tile == Prospero::TileMode::kDepth;
-	const bool msaa_tile    = depth_tile || tile == Prospero::TileMode::kRenderTarget;
-	const bool msaa_array   = type == Prospero::ImageType::kColor2DMsaaArray;
-	if ((!multisampled && (base_level > last_level || last_level >= levels)) ||
+	const auto max_mip      = resource.r128 ? last_level : descriptor.MaxMip();
+	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(max_mip) + 1u;
+	const bool dynamic_storage =
+	    storage && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
+	const auto view_last_level =
+	    !multisampled && !dynamic_storage ? std::min(last_level, max_mip) : last_level;
+	const auto tile       = descriptor.TileMode();
+	const bool depth_tile = tile == Prospero::TileMode::kDepth;
+	const bool msaa_tile  = depth_tile || tile == Prospero::TileMode::kRenderTarget;
+	const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
+	if ((!multisampled && (base_level > view_last_level || view_last_level >= levels)) ||
 	    (multisampled &&
-	     (base_level != 0 || last_level == 0 || last_level > 3 ||
-	      descriptor.MaxMip() != last_level || !msaa_tile ||
-	      (descriptor.MsaaDepth() && !depth_tile) ||
+	     (base_level != 0 || last_level == 0 || last_level > 3 || max_mip != last_level ||
+	      !msaa_tile || (descriptor.MsaaDepth() && !depth_tile) ||
 	      (!msaa_array && (descriptor.Depth() != 0 || descriptor.BaseArray5() != 0))))) {
 		EXIT("unsupported texture mip view: base=%u last=%u levels=%u max=%u type=%u tile=%u "
 		     "kind=%u dimension=%u mip_mode=%u read=%d written=%d "
@@ -650,12 +730,16 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	}
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto view_levels =
-	    multisampled ? 1u : static_cast<uint32_t>(last_level - base_level) + 1u;
-	const auto depth  = static_cast<uint32_t>(descriptor.Depth()) + 1u;
-	const auto format = descriptor.Format();
+	    multisampled ? 1u : static_cast<uint32_t>(view_last_level - base_level) + 1u;
+	const auto depth          = static_cast<uint32_t>(descriptor.Depth()) + 1u;
+	const auto format         = descriptor.Format();
+	const auto surface_format = TextureGetSurfaceFormatInfo(format);
+	const bool shader_conversion =
+	    surface_format.conversion_format != Prospero::BufferFormat::kInvalid;
 	const bool sampled_numeric_class =
-	    storage || ((resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) ==
-	                Prospero::IsUintTextureFormat(format));
+	    storage || (Prospero::IsSampledTextureFormat(format) &&
+	                (resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) ==
+	                    Prospero::IsUintTextureFormat(format));
 	if (!storage &&
 	    (resource.kind == ShaderRecompiler::IR::ResourceKind::Image ||
 	     resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint) &&
@@ -691,7 +775,7 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 		ValidateStorageTexture(resource, descriptor, size.size);
 	}
 
-	const auto pixel_format        = TextureGetFormat(format);
+	const auto pixel_format        = surface_format.vk_format;
 	const auto storage_view_format = storage && format == Prospero::BufferFormat::k32SInt
 	                                     ? vk::Format::eR32Uint
 	                                     : SrgbStorageViewFormat(pixel_format);
@@ -716,11 +800,11 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	} else {
 		PopulateTextureMipLayout(desc.info);
 	}
-	desc.view_info = TextureViewInfo(resource, descriptor, view_format, storage, view_levels,
-	                                 desc.info.resources.layers);
+	desc.view_info = TextureViewInfo(resource, descriptor, view_format, shader_conversion, storage,
+	                                 view_levels, desc.info.resources.layers);
 	desc.type = storage ? TextureCache::BindingType::Storage : TextureCache::BindingType::Texture;
 
-	auto       id                  = texture_cache.FindImage(desc);
+	auto       id                  = texture_cache.FindImage(desc, shader_conversion);
 	auto*      image               = &texture_cache.GetImage(id);
 	const bool stencil_association = static_cast<bool>(image->depth_id);
 	if (stencil_association) {
@@ -756,25 +840,25 @@ static vk::Sampler NativeSampler(RenderContext&                       context,
 	if (!depth_compare) {
 		descriptor.fields[0] &= ~(0x7u << 12u);
 	}
+	if (program.info.samplers[index].force_point_filtering) {
+		descriptor.SetPointFiltering();
+	}
 	return context.GetSamplerCache().GetSampler(descriptor);
 }
 
-static BufferView NativeUpload(RenderContext& context, CommandBuffer& command_buffer,
-                               std::span<const uint32_t> data) {
+static BufferView NativeUpload(RenderContext& context, std::span<const uint32_t> data) {
 	EXIT_IF(data.empty());
-	EXIT_IF(command_buffer.IsInvalid() || command_buffer.IsExecute());
-	auto binding = context.GetBufferCache().UploadTransient(data.data(), data.size_bytes(), 256);
-	return {.owner  = std::move(binding.owner),
-	        .buffer = binding.buffer,
-	        .offset = binding.offset,
-	        .range  = data.size_bytes()};
+	auto& command_buffer = context.GetCommandScheduler().Current();
+	EXIT_IF(command_buffer.IsInvalid());
+	auto&      buffer = context.GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+	const auto offset = buffer.Copy(data.data(), data.size_bytes(), 256);
+	return {.buffer = buffer.Handle(), .offset = offset, .range = data.size_bytes()};
 }
 
 void RenderExecutor::TrackImageBinding(ImageId id) {
-	auto image = m_context.GetTextureCache().ResolveOwner(id);
-	EXIT_IF(image == nullptr);
-	if (std::ranges::find(m_bound_images, image) == m_bound_images.end()) {
-		m_bound_images.push_back(std::move(image));
+	EXIT_IF(m_context.GetTextureCache().m_slot_images.try_get(id) == nullptr);
+	if (std::ranges::find(m_bound_images, id) == m_bound_images.end()) {
+		m_bound_images.push_back(id);
 	}
 }
 
@@ -784,9 +868,10 @@ void RenderExecutor::BindImage(ImageId id, bool storage) {
 		return;
 	}
 	if (image.binding.is_bound) {
-		image.binding.force_general |= storage;
+		image.binding.force_general |= image.binding.shader_write != storage;
 	}
 	image.binding.is_bound = true;
+	image.binding.shader_write |= storage;
 	TrackImageBinding(id);
 }
 
@@ -797,16 +882,15 @@ void RenderExecutor::BindRenderTarget(ImageId id) {
 }
 
 void RenderExecutor::ResetBindings() {
-	for (const auto& image: m_bound_images) {
-		image->binding = {};
+	for (const auto id: m_bound_images) {
+		if (auto* image = m_context.GetTextureCache().m_slot_images.try_get(id); image != nullptr) {
+			image->binding = {};
+		}
 	}
 	m_bound_images.clear();
 }
 
-DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&            buffer,
-                                                                  const ShaderStageRuntime& runtime,
-                                                                  vk::ShaderStageFlags shader_stage,
-                                                                  DescriptorCache::Stage stage) {
+PreparedBindings RenderExecutor::PrepareBindings(const ShaderStageRuntime& runtime) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(!runtime);
 	const auto& program  = *runtime.program;
@@ -816,12 +900,10 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&
 		EXIT("invalid native shader runtime snapshot: %s\n", error.c_str());
 	}
 
-	DescriptorCache::PreparedBindings prepared;
-	prepared.program      = runtime.program;
-	prepared.snapshot     = runtime.resources;
-	prepared.shader_stage = shader_stage;
-	prepared.stage        = stage;
-	auto& descriptors     = prepared.resources;
+	PreparedBindings prepared;
+	prepared.program  = runtime.program.get();
+	prepared.snapshot = runtime.resources.get();
+	auto& descriptors = prepared.resources;
 	descriptors.buffers.reserve(program.info.buffers.size());
 	descriptors.images.reserve(program.info.images.size());
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
@@ -846,44 +928,75 @@ DescriptorCache::PreparedBindings RenderExecutor::PrepareBindings(CommandBuffer&
 	prepared.user_data.resize(program.bindings.ShaderDataDwords());
 	if (ShaderRecompiler::IR::FindBinding(
 	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::Gds) != nullptr) {
-		descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer().Handle();
+		descriptors.gds.buffer = m_context.GetBufferCache().GetGdsBuffer()->Handle();
 	}
 	return prepared;
 }
 
-void RenderExecutor::RebindBuffers(CommandBuffer&                     buffer,
-                                   DescriptorCache::PreparedBindings& prepared) {
+void RenderExecutor::FindBuffers(PreparedBindings& prepared) {
+	KYTY_PROFILER_FUNCTION();
+	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
+	const auto& program  = *prepared.program;
+	const auto& snapshot = *prepared.snapshot;
+	auto&       cache    = m_context.GetBufferCache();
+
+	prepared.buffer_ids.clear();
+	prepared.buffer_ids.reserve(program.info.buffers.size());
+	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
+		ShaderBufferResource descriptor;
+		CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields);
+		const auto address = descriptor.Base48();
+		const auto stride  = descriptor.Stride();
+		const auto records = descriptor.NumRecords();
+		EXIT_IF(stride != 0 && records > UINT64_MAX / stride);
+		const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+		if (address == 0 || requested_size == 0) {
+			prepared.buffer_ids.emplace_back();
+			continue;
+		}
+		const auto size = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
+		prepared.buffer_ids.push_back(cache.FindBuffer(address, size));
+	}
+
+}
+
+void RenderExecutor::RebindBuffers(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
 	const auto& program   = *prepared.program;
 	const auto& snapshot  = *prepared.snapshot;
 	auto&       resources = prepared.resources;
 	const auto& layout    = program.bindings;
+	EXIT_IF(prepared.buffer_ids.size() != program.info.buffers.size());
 
 	resources.buffers.clear();
 	resources.buffers.reserve(program.info.buffers.size());
 	EXIT_IF(prepared.user_data.size() != layout.ShaderDataDwords());
-	std::fill(prepared.user_data.begin() + layout.buffer_offset_dword, prepared.user_data.end(), 0);
+	std::fill(prepared.user_data.begin() + layout.memory_offset_dword, prepared.user_data.end(), 0);
+	auto pack_memory_offset = [&](uint32_t index, uint32_t offset) {
+		const auto dword = layout.memory_offset_dword + index / 4u;
+		const auto shift = (index % 4u) * 8u;
+		prepared.user_data[dword] |= offset << shift;
+	};
 	for (uint32_t i = 0; i < program.info.buffers.size(); i++) {
 		ShaderBufferResource descriptor;
 		CopyNativeDescriptor(snapshot.buffers[i], descriptor.fields);
 		uint32_t buffer_offset = 0;
-		resources.buffers.push_back(NativeStorageBuffer(m_context, buffer, descriptor,
-		                                                program.info.buffers[i], buffer_offset));
-		const auto dword = layout.buffer_offset_dword + i / 4u;
-		const auto shift = (i % 4u) * 8u;
-		prepared.user_data[dword] |= buffer_offset << shift;
+		resources.buffers.push_back(NativeStorageBuffer(m_context, descriptor,
+		                                                program.info.buffers[i], program.stage, i,
+		                                                buffer_offset, prepared.buffer_ids[i]));
+		pack_memory_offset(i, buffer_offset);
 	}
-	resources.addresses.clear();
-	resources.addresses.reserve(program.info.addresses.size());
-	for (uint32_t i = 0; i < program.info.addresses.size(); i++) {
-		resources.addresses.push_back(NativeAddressBuffer(
-		    m_context, buffer, program.info.addresses[i], snapshot.addresses[i]));
+	if (!prepared.flattened_srt.empty()) {
+		resources.flattened_srt = NativeUpload(m_context, prepared.flattened_srt);
+	}
+	if (ShaderRecompiler::IR::FindBinding(
+	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) {
+		resources.user_data = NativeUpload(m_context, prepared.user_data);
 	}
 }
 
-void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
-                                  DescriptorCache::PreparedBindings& prepared) {
+void RenderExecutor::RebindImages(PreparedBindings& prepared) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(prepared.program == nullptr || prepared.snapshot == nullptr);
 	const auto& program  = *prepared.program;
@@ -892,7 +1005,7 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 	EXIT_IF(images.size() != program.info.images.size());
 	auto& texture_cache = m_context.GetTextureCache();
 	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		const auto old_image = texture_cache.ResolveOwner(images[i].image_id);
+		const auto old_image = texture_cache.m_slot_images.try_get(images[i].image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
@@ -902,8 +1015,29 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
 		}
-		auto& binding      = images[i];
-		binding.image_view = texture_cache.FindTexture(binding.image_id, binding.desc);
+	}
+	for (uint32_t i = 0; i < program.info.images.size(); i++) {
+		auto& binding = images[i];
+		binding.mip_views.clear();
+		const auto& resource = program.info.images[i];
+		if (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage) {
+			EXIT_IF(resource.mip_count == 0u ||
+			        resource.mip_count != binding.desc.view_info.level_count);
+			binding.mip_views.reserve(resource.mip_count);
+			for (uint32_t mip = 0; mip < resource.mip_count; mip++) {
+				auto desc = binding.desc;
+				desc.view_info.base_level += mip;
+				desc.view_info.level_count = 1;
+				binding.mip_views.push_back(texture_cache.FindTexture(binding.image_id, desc));
+			}
+			binding.image_view = binding.mip_views.front();
+		} else {
+			auto desc = binding.desc;
+			if (desc.type == TextureCache::BindingType::Storage) {
+				desc.view_info.level_count = 1;
+			}
+			binding.image_view = texture_cache.FindTexture(binding.image_id, desc);
+		}
 		auto&      image   = texture_cache.GetImage(binding.image_id);
 		const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
 		image.usage.storage |= storage;
@@ -912,118 +1046,221 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 }
 
 RenderExecutor::GraphicsBindings
-RenderExecutor::PrepareGraphicsBindings(CommandBuffer& buffer, const ShaderStageRuntime& vertex,
+RenderExecutor::PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
                                         const ShaderStageRuntime& pixel, bool pixel_active) {
 	GraphicsBindings bindings {
-	    .vertex = PrepareBindings(buffer, vertex, vk::ShaderStageFlagBits::eVertex,
-	                              DescriptorCache::Stage::Vertex),
+	    .vertex = PrepareBindings(vertex),
 	};
-	RebindBuffers(buffer, bindings.vertex);
-	RebindImages(buffer, bindings.vertex);
 	if (pixel_active) {
-		bindings.pixel.emplace(PrepareBindings(buffer, pixel, vk::ShaderStageFlagBits::eFragment,
-		                                       DescriptorCache::Stage::Pixel));
-		RebindBuffers(buffer, *bindings.pixel);
-		RebindImages(buffer, *bindings.pixel);
+		bindings.pixel.emplace(PrepareBindings(pixel));
+	}
+	FindBuffers(bindings.vertex);
+	if (bindings.pixel) {
+		FindBuffers(*bindings.pixel);
+	}
+	if (bindings.vertex.program->info.uses_dma ||
+	    (bindings.pixel && bindings.pixel->program->info.uses_dma)) {
+		m_context.GetGpuResources().PrepareBda();
+	}
+	RebindBuffers(bindings.vertex);
+	if (bindings.pixel) {
+		RebindBuffers(*bindings.pixel);
+	}
+	RebindImages(bindings.vertex);
+	if (bindings.pixel) {
+		RebindImages(*bindings.pixel);
 	}
 	return bindings;
 }
 
-static void RetainBindings(CommandBuffer& buffer, DescriptorCache::NativeDescriptors& resources) {
-	if (resources.flattened_srt.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(resources.flattened_srt.owner));
-	}
-	if (resources.user_data.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(resources.user_data.owner));
-	}
-	for (auto& view: resources.buffers) {
-		if (view.owner != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(view.owner));
-		}
-	}
-	for (auto& view: resources.addresses) {
-		if (view.owner != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(view.owner));
-		}
-	}
-}
-
 void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
                                     vk::PipelineBindPoint              pipeline_bind_point,
-                                    vk::PipelineLayout                 layout,
-                                    DescriptorCache::PreparedBindings& prepared) {
+                                    const PipelineCache::Pipeline&     pipeline,
+                                    std::span<PreparedBindings* const> prepared_bindings) {
 	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(prepared.program == nullptr || prepared.stage == DescriptorCache::Stage::Unknown ||
-	        prepared.committed);
-	const auto& program     = *prepared.program;
-	auto&       descriptors = prepared.resources;
-	if (!prepared.flattened_srt.empty()) {
-		descriptors.flattened_srt = NativeUpload(m_context, buffer, prepared.flattened_srt);
-	}
-	if (ShaderRecompiler::IR::FindBinding(
-	        program.bindings, ShaderRecompiler::IR::DescriptorBindingKind::UserData) != nullptr) {
-		descriptors.user_data = NativeUpload(m_context, buffer, prepared.user_data);
-	}
-	RetainBindings(buffer, descriptors);
-
-	auto       vk_buffer     = buffer.Handle();
-	const auto shader_stages = ShaderPipelineStages(prepared.shader_stage);
-	if (descriptors.gds.buffer != nullptr) {
-		buffer.EndRendering();
-		const auto barrier = MakeGdsDependency(descriptors.gds.buffer);
-		vk_buffer.pipelineBarrier(
-		    vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer |
-		        vk::PipelineStageFlagBits::eAllGraphics | vk::PipelineStageFlagBits::eComputeShader,
-		    shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr);
-	}
-
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		auto&       image   = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
-		auto&       binding = descriptors.images[i];
-		const auto& view    = binding.desc.view_info;
-		const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
-		                                   view.layer_count};
-		const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
-		if (image.info.data.Empty()) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              storage
-			                  ? vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite
-			                  : vk::AccessFlagBits2::eShaderRead,
-			              range, vk_buffer);
-		} else if ((image.binding.force_general || image.binding.is_target) &&
-		           !image.info.IsDepth()) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead |
-			                  vk::AccessFlagBits2::eColorAttachmentRead |
-			                  vk::AccessFlagBits2::eColorAttachmentWrite,
-			              {}, vk_buffer);
-		} else if (storage) {
-			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
-			              range, vk_buffer);
-		} else {
-			image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
-			                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
-			              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+	auto   vk_buffer        = buffer.Handle();
+	size_t descriptor_count = 0;
+	size_t write_count      = 0;
+	constexpr auto GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment;
+	const auto push_constant_stages = pipeline_bind_point == vk::PipelineBindPoint::eGraphics
+	                                      ? vk::ShaderStageFlags {GraphicsStages}
+	                                      : vk::ShaderStageFlags {vk::ShaderStageFlagBits::eCompute};
+	for (const auto* prepared: prepared_bindings) {
+		EXIT_IF(prepared == nullptr || prepared->program == nullptr ||
+		        prepared->snapshot == nullptr || prepared->committed);
+		write_count += prepared->program->bindings.descriptors.size();
+		for (const auto& binding: prepared->program->bindings.descriptors) {
+			descriptor_count += NativeDescriptorCount(binding);
 		}
-		binding.layout = image.backing.state.layout;
+		const auto shader_stage = NativeShaderStage(prepared->program->stage);
+		EXIT_IF((pipeline_bind_point == vk::PipelineBindPoint::eGraphics &&
+		         (shader_stage & GraphicsStages) == vk::ShaderStageFlags {}) ||
+		        (pipeline_bind_point == vk::PipelineBindPoint::eCompute &&
+		         shader_stage != vk::ShaderStageFlagBits::eCompute));
 	}
+	m_descriptor_buffers.clear();
+	m_descriptor_images.clear();
+	m_descriptor_writes.clear();
+	m_push_constants.fill(0);
+	m_descriptor_buffers.reserve(descriptor_count);
+	m_descriptor_images.reserve(descriptor_count);
+	m_descriptor_writes.reserve(write_count);
 
-	if (!program.bindings.descriptors.empty()) {
-		auto& set =
-		    m_context.GetDescriptorCache().GetDescriptor(prepared.stage, program, descriptors);
-		vk_buffer.bindDescriptorSets(pipeline_bind_point, layout, program.bindings.descriptor_set,
-		                             1, &set.set, 0, nullptr);
-		buffer.RecycleDescriptorAfterFence(set);
+	for (auto* prepared: prepared_bindings) {
+		const auto& program       = *prepared->program;
+		auto&       descriptors   = prepared->resources;
+		const auto  shader_stage  = NativeShaderStage(program.stage);
+		const auto  shader_stages = ShaderPipelineStages(shader_stage);
+		if (descriptors.gds.buffer != nullptr) {
+			buffer.EndRendering();
+			const auto barrier = MakeGdsDependency(descriptors.gds.buffer);
+			vk_buffer.pipelineBarrier(
+			    vk::PipelineStageFlagBits::eHost | vk::PipelineStageFlagBits::eTransfer |
+			        vk::PipelineStageFlagBits::eAllGraphics |
+			        vk::PipelineStageFlagBits::eComputeShader,
+			    shader_stages, vk::DependencyFlags {}, 0, nullptr, 1, &barrier, 0, nullptr);
+		}
+
+		for (uint32_t i = 0; i < program.info.images.size(); i++) {
+			auto& image   = m_context.GetTextureCache().GetImage(descriptors.images[i].image_id);
+			auto& binding = descriptors.images[i];
+			const auto&                 view = binding.desc.view_info;
+			const ImageSubresourceRange range {view.base_level, view.level_count, view.base_layer,
+			                                   view.layer_count};
+			const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
+			if (image.info.data.Empty()) {
+				image.Transit(vk::ImageLayout::eGeneral,
+				              storage ? vk::AccessFlagBits2::eShaderRead |
+				                            vk::AccessFlagBits2::eShaderWrite
+				                      : vk::AccessFlagBits2::eShaderRead,
+				              range, vk_buffer);
+			} else if ((image.binding.force_general || image.binding.is_target) &&
+			           !image.info.IsDepth()) {
+				const vk::AccessFlags2 storage_access = image.binding.shader_write
+				                                            ? vk::AccessFlagBits2::eShaderWrite
+				                                            : vk::AccessFlags2 {};
+				image.Transit(vk::ImageLayout::eGeneral,
+				              vk::AccessFlagBits2::eShaderRead | storage_access |
+				                  vk::AccessFlagBits2::eColorAttachmentRead |
+				                  vk::AccessFlagBits2::eColorAttachmentWrite,
+				              {}, vk_buffer);
+			} else if (storage) {
+				image.Transit(vk::ImageLayout::eGeneral,
+				              vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
+				              range, vk_buffer);
+			} else {
+				image.Transit(image.info.IsDepth() ? vk::ImageLayout::eDepthStencilReadOnlyOptimal
+				                                   : vk::ImageLayout::eShaderReadOnlyOptimal,
+				              vk::AccessFlagBits2::eShaderRead, range, vk_buffer);
+			}
+			binding.layout = image.backing.state.layout;
+		}
+
+		m_image_occurrences.assign(descriptors.images.size(), 0);
+		for (const auto& binding: program.bindings.descriptors) {
+			vk::WriteDescriptorSet write {};
+			write.sType          = vk::StructureType::eWriteDescriptorSet;
+			write.dstBinding     = ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind);
+			write.descriptorType = NativeDescriptorType(binding.kind);
+			write.descriptorCount   = NativeDescriptorCount(binding);
+			const auto buffer_start = m_descriptor_buffers.size();
+			const auto image_start  = m_descriptor_images.size();
+			switch (binding.kind) {
+				case BindingKind::Buffers:
+					for (const auto resource: binding.resources) {
+						const auto& view = descriptors.buffers.at(resource);
+						EXIT_IF(view.buffer == nullptr);
+						m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					}
+					break;
+				case BindingKind::BdaPagetable:
+				case BindingKind::FaultBuffer: {
+					auto& cache = m_context.GetBufferCache();
+					const auto* bda_buffer = binding.kind == BindingKind::BdaPagetable
+					                             ? cache.GetBdaPageTableBuffer()
+					                             : cache.GetFaultBuffer();
+					m_descriptor_buffers.emplace_back(bda_buffer->Handle(), 0, bda_buffer->Size());
+					break;
+				}
+				case BindingKind::FlattenedSrt:
+				case BindingKind::UserData:
+				case BindingKind::Gds: {
+					const auto& view =
+					    binding.kind == BindingKind::FlattenedSrt ? descriptors.flattened_srt
+					    : binding.kind == BindingKind::UserData   ? descriptors.user_data
+					                                              : descriptors.gds;
+					EXIT_IF(view.buffer == nullptr);
+					m_descriptor_buffers.emplace_back(view.buffer, view.offset, view.range);
+					break;
+				}
+				case BindingKind::Samplers:
+					for (const auto resource: binding.resources) {
+						const auto sampler = descriptors.samplers.at(resource);
+						EXIT_IF(sampler == nullptr);
+						m_descriptor_images.emplace_back(sampler, nullptr,
+						                                 vk::ImageLayout::eUndefined);
+					}
+					break;
+				default:
+					for (const auto resource: binding.resources) {
+						m_descriptor_images.push_back(MakeImageInfo(
+						    descriptors.images.at(resource), m_image_occurrences.at(resource)++));
+					}
+					break;
+			}
+			if (m_descriptor_buffers.size() != buffer_start) {
+				write.pBufferInfo = m_descriptor_buffers.data() + buffer_start;
+			}
+			if (m_descriptor_images.size() != image_start) {
+				write.pImageInfo = m_descriptor_images.data() + image_start;
+			}
+			m_descriptor_writes.push_back(write);
+		}
+		for (uint32_t i = 0; i < descriptors.images.size(); i++) {
+			const auto expected =
+			    descriptors.images[i].mip_views.empty()
+			        ? 1u
+			        : static_cast<uint32_t>(descriptors.images[i].mip_views.size());
+			EXIT_IF(m_image_occurrences[i] != expected);
+		}
+
+		if (program.bindings.push_constant_size != 0) {
+			const auto offset = program.bindings.push_constant_offset;
+			EXIT_IF(offset % sizeof(uint32_t) != 0 ||
+			        program.bindings.push_constant_size !=
+			            prepared->user_data.size() * sizeof(uint32_t) ||
+			        offset + program.bindings.push_constant_size >
+			            ShaderRecompiler::IR::NativePushConstantSize);
+			std::copy(prepared->user_data.begin(), prepared->user_data.end(),
+			          m_push_constants.begin() + offset / sizeof(uint32_t));
+		}
 	}
-	if (program.bindings.push_constant_size != 0) {
-		EXIT_IF(program.bindings.push_constant_size !=
-		        prepared.user_data.size() * sizeof(uint32_t));
-		vk_buffer.pushConstants(layout, prepared.shader_stage,
-		                        program.bindings.push_constant_offset,
-		                        program.bindings.push_constant_size, prepared.user_data.data());
+	vk_buffer.pushConstants(pipeline.pipeline_layout, push_constant_stages, 0,
+	                        ShaderRecompiler::IR::NativePushConstantSize,
+	                        m_push_constants.data());
+
+	if (!m_descriptor_writes.empty()) {
+		EXIT_IF(pipeline.descriptor_set_layout == nullptr);
+		if (pipeline.uses_push_descriptors) {
+			vk_buffer.pushDescriptorSetKHR(pipeline_bind_point, pipeline.pipeline_layout, 0,
+			                               static_cast<uint32_t>(m_descriptor_writes.size()),
+			                               m_descriptor_writes.data());
+		} else {
+			const auto set = m_context.GetDescriptorHeap().Commit(pipeline.descriptor_set_layout);
+			for (auto& write: m_descriptor_writes) {
+				write.dstSet = set;
+			}
+			m_context.GetGraphics().device.updateDescriptorSets(
+			    static_cast<uint32_t>(m_descriptor_writes.size()), m_descriptor_writes.data(), 0,
+			    nullptr);
+			vk_buffer.bindDescriptorSets(pipeline_bind_point, pipeline.pipeline_layout, 0, 1, &set,
+			                             0, nullptr);
+		}
+	}
+	for (auto* prepared: prepared_bindings) {
+		prepared->committed = true;
 	}
-	prepared.committed = true;
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/descriptors.h

@@ -2,6 +2,8 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_DESCRIPTORS_H_
 
 #include "common/assert.h"
+#include "graphics/host_gpu/renderer/cache/bufferCache.h"
+#include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/image/image.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
@@ -10,9 +12,54 @@
 #include <cstdint>
 #include <cstring>
 #include <type_traits>
+#include <vector>
 
 namespace Libs::Graphics {
 
+namespace ShaderRecompiler::IR {
+struct ResourceSnapshot;
+}
+
+struct BufferView {
+	vk::Buffer     buffer = nullptr;
+	vk::DeviceSize offset = 0;
+	vk::DeviceSize range  = VK_WHOLE_SIZE;
+};
+
+struct TextureBinding {
+	ImageId                    image_id;
+	vk::ImageView              image_view = nullptr;
+	TextureCache::ImageDesc    desc;
+	vk::ImageLayout            layout = vk::ImageLayout::eUndefined;
+	std::vector<vk::ImageView> mip_views;
+};
+
+struct NativeDescriptors {
+	std::vector<BufferView>     buffers;
+	std::vector<TextureBinding> images;
+	std::vector<vk::Sampler>    samplers;
+	BufferView                  gds;
+	BufferView                  flattened_srt;
+	BufferView                  user_data;
+};
+
+struct PreparedBindings {
+	const ShaderRecompiler::IR::Program*          program  = nullptr;
+	const ShaderRecompiler::IR::ResourceSnapshot* snapshot = nullptr;
+	NativeDescriptors                             resources;
+	std::vector<BufferId>                         buffer_ids;
+	std::vector<uint32_t>                         flattened_srt;
+	std::vector<uint32_t>                         user_data;
+	bool                                          committed = false;
+};
+
+[[nodiscard]] vk::DescriptorType
+NativeDescriptorType(ShaderRecompiler::IR::DescriptorBindingKind kind);
+[[nodiscard]] uint32_t
+NativeDescriptorCount(const ShaderRecompiler::IR::DescriptorBinding& binding);
+[[nodiscard]] vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
+                                                    uint32_t              element = 0);
+
 template <typename T>
 [[nodiscard]] T DecodeNativeDescriptor(const ShaderRecompiler::IR::DescriptorValue& value) {
 	static_assert(std::is_trivially_copyable_v<T>);
@@ -34,9 +81,9 @@ ResolveTargetTextureView(const ShaderRecompiler::IR::ImageResource& resource,
                          Prospero::ImageType type, uint32_t base_layer, uint32_t image_layers);
 
 [[nodiscard]] bool IsSupportedDepthTargetDescriptor(const ShaderTextureResource& descriptor,
-                                                    const Image&                 image);
+                                                    const Image& image, bool r128 = false);
 [[nodiscard]] bool IsSupportedDepthTextureEncoding(const ShaderTextureResource& descriptor,
-                                                   const Image&                 image);
+                                                   const Image& image, bool r128 = false);
 [[nodiscard]] bool
 IsSupportedSampledVideoOutView(const ShaderRecompiler::IR::ImageResource& resource,
                                const ShaderTextureResource& descriptor, const Image& image);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -10,11 +10,14 @@
 #include "graphics/host_gpu/renderer/image/imageView.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
+#include "graphics/shader/shaderCompiler.h"
 
 #include <atomic>
+#include <cstdio>
 #include <cstring>
 #include <span>
 #include <utility>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -36,16 +39,132 @@ void NormalizeStaticParamsForDynamicState(PipelineStaticParameters& static_param
 
 } // namespace
 
+struct PipelineCache::ProgramCache {
+	struct SourceKey {
+		ShaderType stage = ShaderType::Unknown;
+		uint64_t   hash  = 0;
+
+		bool operator==(const SourceKey&) const = default;
+	};
+
+	struct Permutation {
+		std::vector<uint32_t>                               static_key;
+		std::shared_ptr<const ShaderRecompiler::IR::Program> program;
+		ShaderProgram                                       handle;
+	};
+
+	struct SourceKeyHash {
+		std::size_t operator()(const SourceKey& key) const {
+			std::size_t hash = static_cast<std::size_t>(key.stage);
+			PipelineKeyHash::Mix(hash, static_cast<std::size_t>(key.hash));
+			if constexpr (sizeof(std::size_t) < sizeof(uint64_t)) {
+				PipelineKeyHash::Mix(hash, static_cast<std::size_t>(key.hash >> 32u));
+			}
+			return hash;
+		}
+	};
+
+	static uint64_t MixId(uint64_t hash, uint64_t value) {
+		return hash ^ (value + 0x9e3779b97f4a7c15ull + (hash << 6u) + (hash >> 2u));
+	}
+
+	static constexpr std::size_t MaxStaticKeyWords =
+	    5 + ShaderVertexInputInfo::RES_MAX * 17;
+
+	template <typename InputInfo>
+	ShaderProgram Get(const ShaderParams& params, InputInfo& input_info) {
+		constexpr ShaderType stage = [] {
+			if constexpr (std::is_same_v<InputInfo, ShaderVertexInputInfo>) {
+				return ShaderType::Vertex;
+			} else if constexpr (std::is_same_v<InputInfo, ShaderPixelInputInfo>) {
+				return ShaderType::Pixel;
+			} else {
+				static_assert(std::is_same_v<InputInfo, ShaderComputeInputInfo>);
+				return ShaderType::Compute;
+			}
+		}();
+
+		BuildStageStaticKey(input_info, key_scratch);
+		auto& permutations = programs[{stage, params.hash}];
+		for (const auto& permutation: permutations) {
+			if (permutation.static_key == key_scratch &&
+			    MaterializeProgram(permutation.program, params, input_info)) {
+				return permutation.handle;
+			}
+		}
+
+		const auto module = CompileProgram(device, params, input_info);
+		EXIT_IF(module == nullptr || !input_info.stage);
+		uint64_t id = MixId(MixId(params.hash, static_cast<uint64_t>(stage)), permutations.size());
+		if (id == 0) {
+			id = 1;
+		}
+		const ShaderProgram handle {.id = id, .module = module};
+		permutations.push_back({key_scratch, input_info.stage.program, handle});
+
+		std::printf("Num compiled %u shaders\n", ++num_compiled);
+		return handle;
+	}
+
+	explicit ProgramCache(vk::Device device): device(device) {
+		key_scratch.reserve(MaxStaticKeyWords);
+	}
+
+	std::unordered_map<SourceKey, std::vector<Permutation>, SourceKeyHash> programs;
+	std::vector<uint32_t> key_scratch;
+	vk::Device device;
+	uint32_t   num_compiled = 0;
+};
+
+PipelineCache::PipelineCache(GraphicContext& graphics)
+    : m_graphics(graphics), m_program_cache(std::make_unique<ProgramCache>(graphics.device)) {
+	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
+}
+
 PipelineCache::~PipelineCache() {
 	auto destroy = [this](const auto& pipelines) {
 		for (const auto& [key, pipeline]: pipelines) {
 			(void)key;
 			m_graphics.device.destroyPipeline(pipeline->pipeline, nullptr);
 			m_graphics.device.destroyPipelineLayout(pipeline->pipeline_layout, nullptr);
+			m_graphics.device.destroyDescriptorSetLayout(pipeline->descriptor_set_layout, nullptr);
 		}
 	};
 	destroy(m_graphics_pipelines);
 	destroy(m_compute_pipelines);
+	for (const auto& [key, permutations]: m_program_cache->programs) {
+		(void)key;
+		for (const auto& permutation: permutations) {
+			m_graphics.device.destroyShaderModule(permutation.handle.module, nullptr);
+		}
+	}
+}
+
+ShaderProgram PipelineCache::GetVertexProgram(const HW::VertexShaderInfo& regs,
+                                              const HW::ShaderRegisters&  sh,
+                                              ShaderVertexInputInfo&      input_info) {
+	const auto params = PrepareProgram(regs, sh, input_info);
+	Common::LockGuard lock(m_mutex);
+	return m_program_cache->Get(params, input_info);
+}
+
+ShaderProgram PipelineCache::GetPixelProgram(
+    const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
+    const ShaderVertexInputInfo&                        vertex_info,
+    std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
+    ShaderPixelInputInfo&                               input_info) {
+	const auto params = PrepareProgram(regs, sh, vertex_info, target_export_mapping, input_info);
+	Common::LockGuard lock(m_mutex);
+	return m_program_cache->Get(params, input_info);
+}
+
+ShaderProgram PipelineCache::GetComputeProgram(const HW::ComputeShaderInfo& regs,
+                                               const HW::ShaderRegisters&   sh,
+                                               ShaderComputeInputInfo&      input_info) {
+	input_info.needs_lds_barriers = !m_graphics.compute_wave64_supported;
+	const auto params             = PrepareProgram(regs, sh, input_info);
+	Common::LockGuard lock(m_mutex);
+	return m_program_cache->Get(params, input_info);
 }
 
 bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other) const noexcept {
@@ -53,23 +172,22 @@ bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other)
 }
 
 PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
-    RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
-    ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
-    ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, bool ps_active,
-    std::span<const uint32_t> vs_spirv, std::span<const uint32_t> ps_spirv) {
+    std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
+    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+    const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
+    bool primitive_restart_enable, const ShaderProgram& vertex_program,
+    const ShaderProgram& pixel_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200);
 
-	EXIT_IF(colors == nullptr);
-	EXIT_IF(color_count > RENDER_COLOR_ATTACHMENTS_MAX);
-	EXIT_IF(vs_spirv.empty());
-	EXIT_IF(ps_active && ps_spirv.empty());
+	EXIT_IF(colors.size() > RENDER_COLOR_ATTACHMENTS_MAX);
+	EXIT_IF(!vertex_program);
+	const bool ps_active = ps_input_info != nullptr;
+	EXIT_IF(ps_active && !pixel_program);
+	const auto color_count = static_cast<uint32_t>(colors.size());
 
 	Common::LockGuard lock(m_mutex);
-	auto&             ctx    = command.GetRegisters();
-	auto&             sh_ctx = command.GetShaders();
+	auto&             ctx = command.GetRegisters();
 
-	const auto&           vertex_info                              = sh_ctx.GetVs();
-	const auto&           ps_regs                                  = sh_ctx.GetPs();
 	const HW::BlendColor& bclr                                     = ctx.GetBlendColor();
 	uint32_t              color_mask[RENDER_COLOR_ATTACHMENTS_MAX] = {};
 	for (uint32_t i = 0; i < color_count; i++) {
@@ -80,11 +198,8 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 	}
 	const HW::ModeControl& mc = ctx.GetModeControl();
 
-	auto     vs_id = ShaderGetIdVS(vertex_info, vs_input_info, true);
-	ShaderId ps_id {};
-	if (ps_active) {
-		ps_id = ShaderGetIdPS(ps_regs, *ps_input_info, true);
-	}
+	const auto vs_id = vertex_program.id;
+	const auto ps_id = ps_active ? pixel_program.id : 0;
 
 	PipelineStaticParameters static_params {};
 	GraphicsPipeline         p {};
@@ -120,6 +235,12 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 			     depth.samples);
 		}
 	}
+	if (color_count == 0 && !with_depth) {
+		attachment_samples = render_sample_count(ctx.GetAaConfig().msaa_num_samples);
+		EXIT_IF(!static_cast<bool>(
+		    m_graphics.GetPhysicalDeviceProperties().limits.framebufferNoAttachmentsSampleCounts &
+		    vulkan_sample_count(attachment_samples)));
+	}
 	EXIT_IF(attachment_samples == 0 ||
 	        vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {});
 
@@ -130,12 +251,12 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 		}
 	}
 
-	const auto& clip_control = ctx.GetClipControl();
-	EXIT_NOT_IMPLEMENTED(!clip_control.IsZClipModeRepresentable());
-	static_params.negative_one_to_one = !clip_control.dx_clip_space;
-	static_params.depth_clip_enable   = clip_control.IsZClipEnabled();
-	static_params.topology            = topology;
-	static_params.samples             = attachment_samples;
+	const auto& clip_control               = ctx.GetClipControl();
+	static_params.negative_one_to_one      = !clip_control.dx_clip_space;
+	static_params.depth_clip_enable        = clip_control.IsZClipEnabled();
+	static_params.topology                 = topology;
+	static_params.primitive_restart_enable = primitive_restart_enable;
+	static_params.samples                  = attachment_samples;
 	static_params.sample_shading_enable =
 	    ps_active && attachment_samples > 1 && ps_input_info->ps_sample_shading;
 	if (static_params.sample_shading_enable && !m_graphics.sample_rate_shading_enabled) {
@@ -194,20 +315,17 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 		if (ps_active) {
 			ShaderDbgDumpInputInfo(*ps_input_info);
 		}
-		LOGF("PipelineTrace: shader binaries VS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " words=%" PRIu64 "\n",
-		     vs_id.hash0, vs_id.crc32, static_cast<uint64_t>(vs_spirv.size()), ps_id.hash0,
-		     ps_id.crc32, static_cast<uint64_t>(ps_spirv.size()));
+		LOGF("PipelineTrace: shader modules VS=%" PRIu64 " module=%p PS=%" PRIu64
+		     " module=%p\n",
+		     vs_id, static_cast<void*>(vertex_program.module), ps_id,
+		     static_cast<void*>(pixel_program.module));
 	}
 
 	auto cached = std::make_unique<GraphicsPipeline>(p);
-	LogPipelineTrace("CreatePipelineInternal begin", vs_id.hash0, vs_id.crc32, ps_id.hash0,
-	                 ps_id.crc32);
-	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, rendering, vs_input_info,
-	                       vs_spirv, ps_input_info, ps_spirv, static_params, vs_id.hash0,
-	                       vs_id.crc32, ps_id.hash0, ps_id.crc32, ps_active);
-	LogPipelineTrace("CreatePipelineInternal done", vs_id.hash0, vs_id.crc32, ps_id.hash0,
-	                 ps_id.crc32);
+	LogPipelineTrace("CreatePipelineInternal begin", vs_id, ps_id);
+	CreatePipelineInternal(m_graphics, *cached, rendering, vs_input_info, vertex_program.module,
+	                       ps_input_info, pixel_program.module, static_params);
+	LogPipelineTrace("CreatePipelineInternal done", vs_id, ps_id);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
 	EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr);
@@ -219,19 +337,16 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 }
 
 PipelineCache::ComputePipeline&
-PipelineCache::CreateComputePipeline(ShaderComputeInputInfo&      input_info,
-                                     const HW::ComputeShaderInfo& cs_regs,
-                                     std::span<const uint32_t>    cs_spirv) {
+PipelineCache::CreateComputePipeline(ShaderComputeInputInfo& input_info,
+                                     const ShaderProgram&    compute_program) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Compute)", profiler::colors::RedA100);
 
-	EXIT_IF(cs_spirv.empty());
+	EXIT_IF(!compute_program);
 
 	Common::LockGuard lock(m_mutex);
 
-	auto cs_id = ShaderGetIdCS(cs_regs, input_info, true);
-
 	ComputePipeline p {};
-	p.cs_shader_id = cs_id;
+	p.cs_shader_id = compute_program.id;
 
 	ComputePipelineKey key {};
 	key.cs_shader_id = p.cs_shader_id;
@@ -245,7 +360,7 @@ PipelineCache::CreateComputePipeline(ShaderComputeInputInfo&      input_info,
 	}
 
 	auto cached = std::make_unique<ComputePipeline>(p);
-	CreatePipelineInternal(m_graphics, m_descriptor_cache, *cached, input_info, cs_spirv);
+	CreatePipelineInternal(m_graphics, *cached, input_info, compute_program.module);
 
 	EXIT_NOT_IMPLEMENTED(cached->pipeline == nullptr);
 	EXIT_NOT_IMPLEMENTED(cached->pipeline_layout == nullptr);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -20,8 +20,7 @@ namespace Libs::Graphics {
 struct GraphicContext;
 struct RenderColorInfo;
 struct RenderDepthInfo;
-class RenderCommandBuffer;
-class DescriptorCache;
+class CommandBuffer;
 
 namespace HW {
 class Context;
@@ -38,6 +37,7 @@ struct PipelineStaticParameters {
 	bool                       depth_clip_enable        = true;
 	int                        scissor_ltrb[4]          = {0};
 	vk::PrimitiveTopology      topology                 = vk::PrimitiveTopology::ePointList;
+	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
 	bool                       with_depth               = false;
@@ -79,9 +79,9 @@ static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
 static_assert(sizeof(PipelineStaticParameters) ==
               sizeof(float[3]) + sizeof(float[3]) + sizeof(bool) * 2 + sizeof(int[4]) +
-                  sizeof(vk::PrimitiveTopology) + sizeof(uint32_t) + sizeof(bool) * 4 +
-                  sizeof(vk::CompareOp) + sizeof(bool) + sizeof(float) * 2 + sizeof(bool) +
-                  sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) +
+                  sizeof(vk::PrimitiveTopology) + sizeof(bool) + sizeof(uint32_t) +
+                  sizeof(bool) * 4 + sizeof(vk::CompareOp) + sizeof(bool) + sizeof(float) * 2 +
+                  sizeof(bool) + sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) +
                   sizeof(uint32_t[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(bool) * 3 +
                   sizeof(uint8_t[RENDER_COLOR_ATTACHMENTS_MAX]) * 6 +
                   sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) * 3 + sizeof(float) * 4);
@@ -95,44 +95,65 @@ struct PipelineRenderingState {
 	bool operator==(const PipelineRenderingState&) const = default;
 };
 
+struct ShaderProgram {
+	uint64_t         id     = 0;
+	vk::ShaderModule module = nullptr;
+
+	explicit operator bool() const {
+		return id != 0 && module != nullptr;
+	}
+};
+
 class PipelineCache {
 public:
-	PipelineCache(GraphicContext& graphics, DescriptorCache& descriptor_cache)
-	    : m_graphics(graphics), m_descriptor_cache(descriptor_cache) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	}
+	explicit PipelineCache(GraphicContext& graphics);
 	~PipelineCache();
 	KYTY_CLASS_NO_COPY(PipelineCache);
 
 	struct Pipeline {
-		vk::PipelineLayout pipeline_layout = nullptr;
-		vk::Pipeline       pipeline        = nullptr;
+		vk::PipelineLayout      pipeline_layout       = nullptr;
+		vk::Pipeline            pipeline              = nullptr;
+		vk::DescriptorSetLayout descriptor_set_layout = nullptr;
+		bool                    uses_push_descriptors = false;
 	};
 
 	struct GraphicsPipeline: Pipeline {
-		ShaderId vs_shader_id;
-		ShaderId ps_shader_id;
+		uint64_t vs_shader_id = 0;
+		uint64_t ps_shader_id = 0;
 	};
 
 	struct ComputePipeline: Pipeline {
-		ShaderId cs_shader_id;
+		uint64_t cs_shader_id = 0;
 	};
 
-	GraphicsPipeline&
-	CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
-	                       ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
-	                       ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
-	                       bool ps_active, std::span<const uint32_t> vs_spirv,
-	                       std::span<const uint32_t> ps_spirv);
-	ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo&      input_info,
-	                                       const HW::ComputeShaderInfo& cs_regs,
-	                                       std::span<const uint32_t>    cs_spirv);
+	ShaderProgram GetVertexProgram(const HW::VertexShaderInfo& regs,
+	                               const HW::ShaderRegisters&  sh,
+	                               ShaderVertexInputInfo&      input_info);
+	ShaderProgram
+	GetPixelProgram(const HW::PixelShaderInfo& regs, const HW::ShaderRegisters& sh,
+	                const ShaderVertexInputInfo&                        vertex_info,
+	                std::span<const Prospero::ColorComponentMapping, 8> target_export_mapping,
+	                ShaderPixelInputInfo&                               input_info);
+	ShaderProgram GetComputeProgram(const HW::ComputeShaderInfo& regs,
+	                                const HW::ShaderRegisters& sh,
+	                                ShaderComputeInputInfo& input_info);
+
+	GraphicsPipeline& CreateGraphicsPipeline(
+	    std::span<const RenderColorInfo> colors, const RenderDepthInfo& depth,
+	    const ShaderVertexInputInfo& vs_input_info, CommandBuffer& command,
+	    const ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
+	    bool primitive_restart_enable, const ShaderProgram& vertex_program,
+	    const ShaderProgram& pixel_program);
+	ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo& input_info,
+	                                       const ShaderProgram&    compute_program);
 
 private:
+	struct ProgramCache;
+
 	struct GraphicsPipelineKey {
 		PipelineRenderingState   rendering;
-		ShaderId                 vs_shader_id;
-		ShaderId                 ps_shader_id;
+		uint64_t                 vs_shader_id = 0;
+		uint64_t                 ps_shader_id = 0;
 		PipelineStaticParameters static_params;
 
 		bool operator==(const GraphicsPipelineKey& other) const {
@@ -142,7 +163,7 @@ private:
 	};
 
 	struct ComputePipelineKey {
-		ShaderId cs_shader_id;
+		uint64_t cs_shader_id = 0;
 
 		bool operator==(const ComputePipelineKey& other) const {
 			return cs_shader_id == other.cs_shader_id;
@@ -155,15 +176,6 @@ private:
 			        (hash >> 2u);
 		}
 
-		static void MixShaderId(std::size_t& hash, const ShaderId& id) {
-			Mix(hash, id.hash0);
-			Mix(hash, id.crc32);
-			Mix(hash, id.ids.size());
-			for (auto value: id.ids) {
-				Mix(hash, value);
-			}
-		}
-
 		static void MixStaticParams(std::size_t& hash, const PipelineStaticParameters& params) {
 			const auto* bytes = reinterpret_cast<const uint8_t*>(&params);
 			for (std::size_t i = 0; i < sizeof(params); i++) {
@@ -185,8 +197,8 @@ private:
 		std::size_t operator()(const GraphicsPipelineKey& key) const {
 			std::size_t hash = 0;
 			PipelineKeyHash::MixRendering(hash, key.rendering);
-			PipelineKeyHash::MixShaderId(hash, key.vs_shader_id);
-			PipelineKeyHash::MixShaderId(hash, key.ps_shader_id);
+			PipelineKeyHash::Mix(hash, key.vs_shader_id);
+			PipelineKeyHash::Mix(hash, key.ps_shader_id);
 			PipelineKeyHash::MixStaticParams(hash, key.static_params);
 			return hash;
 		}
@@ -195,13 +207,13 @@ private:
 	struct ComputePipelineKeyHash {
 		std::size_t operator()(const ComputePipelineKey& key) const {
 			std::size_t hash = 0;
-			PipelineKeyHash::MixShaderId(hash, key.cs_shader_id);
+			PipelineKeyHash::Mix(hash, key.cs_shader_id);
 			return hash;
 		}
 	};
 
-	GraphicContext&  m_graphics;
-	DescriptorCache& m_descriptor_cache;
+	GraphicContext&                m_graphics;
+	std::unique_ptr<ProgramCache> m_program_cache;
 	std::unordered_map<GraphicsPipelineKey, std::unique_ptr<GraphicsPipeline>,
 	                   GraphicsPipelineKeyHash>
 	    m_graphics_pipelines;
@@ -210,19 +222,15 @@ private:
 	Common::Mutex m_mutex;
 };
 
-void LogPipelineTrace(const char* phase, uint32_t vs_hash0, uint32_t vs_crc32, uint32_t ps_hash0,
-                      uint32_t ps_crc32);
+void LogPipelineTrace(const char* phase, uint64_t vertex_program_id, uint64_t pixel_program_id);
 void CreatePipelineInternal(
-    GraphicContext& graphics, DescriptorCache& descriptor_cache,
-    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
-    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
-    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
-    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
-    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active);
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::ComputePipeline& pipeline,
-                            const ShaderComputeInputInfo&   input_info,
-                            std::span<const uint32_t>       cs_shader);
+    GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline,
+    const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info,
+    vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info,
+	vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline,
+                            const ShaderComputeInputInfo& input_info,
+                            vk::ShaderModule              compute_module);
 
 } // namespace Libs::Graphics
 

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h

@@ -2,7 +2,7 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_
 
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
 #include <vector>
 

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp

@@ -1,60 +0,0 @@
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
-
-#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
-
-namespace Libs::Graphics {
-
-ShaderSubgroupCapabilities::ShaderSubgroupCapabilities(const GraphicContext& graphics)
-    : subgroup_size(graphics.subgroup_size), min_subgroup_size(graphics.min_subgroup_size),
-      max_subgroup_size(graphics.max_subgroup_size),
-      required_subgroup_size_stages(graphics.required_subgroup_size_stages),
-      subgroup_size_control_enabled(graphics.subgroup_size_control_enabled) {}
-
-ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size) {
-	// A 64-wide guest wave can't assume the host GPU's lane layout matches
-	// GCN's. AMD RDNA also reports a 64-wide subgroup but uses a different
-	// lane/pixel layout, which corrupted rendering under "native wave" mode.
-	// Per-invocation emulation works regardless of the host's lane layout.
-	if (guest_wave_size == 64u) {
-		return ShaderLaneMaskMode::PerInvocation;
-	}
-	return ShaderLaneMaskMode::NativeWave;
-}
-
-ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
-                                                    vk::ShaderStageFlagBits           stage,
-                                                    const ShaderRecompiler::IR::Program& program) {
-	const auto guest_wave_size = program.wave_size;
-	if (guest_wave_size != 32u && guest_wave_size != 64u) {
-		return {};
-	}
-	if (stage != vk::ShaderStageFlagBits::eCompute) {
-		const auto expected = SelectGraphicsLaneMaskMode(guest_wave_size);
-		if (program.lane_mask_mode != expected || (capabilities.subgroup_size != guest_wave_size &&
-		                                           expected != ShaderLaneMaskMode::PerInvocation)) {
-			return {};
-		}
-		return {expected == ShaderLaneMaskMode::NativeWave
-		            ? ShaderSubgroupMode::Natural
-		            : ShaderSubgroupMode::PerInvocationGraphics,
-		        0};
-	}
-	if (program.lane_mask_mode != ShaderLaneMaskMode::NativeWave) {
-		return {};
-	}
-	if (capabilities.subgroup_size == guest_wave_size) {
-		return {ShaderSubgroupMode::Natural, 0};
-	}
-	if (capabilities.subgroup_size_control_enabled &&
-	    (capabilities.required_subgroup_size_stages & stage) &&
-	    guest_wave_size >= capabilities.min_subgroup_size &&
-	    guest_wave_size <= capabilities.max_subgroup_size) {
-		return {ShaderSubgroupMode::Controlled, guest_wave_size};
-	}
-	if (!ShaderRecompiler::Spirv::ProgramRequiresExactSubgroupSize(program)) {
-		return {ShaderSubgroupMode::FlattenedMasks, 0};
-	}
-	return {};
-}
-
-} // namespace Libs::Graphics

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.h

@@ -1,40 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_
-
-#include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ir/ShaderIR.h"
-
-namespace Libs::Graphics {
-
-enum class ShaderSubgroupMode {
-	Natural,
-	Controlled,
-	PerInvocationGraphics,
-	FlattenedMasks,
-	Unsupported
-};
-
-struct ShaderSubgroupConfiguration {
-	ShaderSubgroupMode mode          = ShaderSubgroupMode::Unsupported;
-	uint32_t           required_size = 0;
-};
-
-struct ShaderSubgroupCapabilities {
-	explicit ShaderSubgroupCapabilities(const GraphicContext& graphics);
-
-	uint32_t             subgroup_size     = 0;
-	uint32_t             min_subgroup_size = 0;
-	uint32_t             max_subgroup_size = 0;
-	vk::ShaderStageFlags required_subgroup_size_stages;
-	bool                 subgroup_size_control_enabled = false;
-};
-
-ShaderLaneMaskMode SelectGraphicsLaneMaskMode(uint32_t guest_wave_size);
-
-ShaderSubgroupConfiguration ConfigureShaderSubgroup(const ShaderSubgroupCapabilities& capabilities,
-                                                    vk::ShaderStageFlagBits           stage,
-                                                    const ShaderRecompiler::IR::Program& program);
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERSUBGROUP_H_

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -6,9 +6,8 @@
 #include "graphics/guest_gpu/gpu_defs.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/debug.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
@@ -20,6 +19,7 @@
 #include <algorithm>
 #include <limits>
 #include <span>
+#include <vector>
 
 namespace Libs::Graphics {
 
@@ -389,116 +389,52 @@ static vk::BlendOp GetBlendOp(uint32_t op) {
 	return vk::BlendOp::eAdd;
 }
 
-static void CreateLayout(DescriptorCache&                   descriptor_cache,
-                         std::span<vk::DescriptorSetLayout> set_layouts, uint32_t& set_layouts_num,
-                         std::span<vk::PushConstantRange>     push_constant_info,
-                         uint32_t&                            push_constant_info_num,
-                         const ShaderRecompiler::IR::Program& program,
-                         vk::ShaderStageFlags vk_stage, DescriptorCache::Stage stage) {
-	const auto& bindings        = program.bindings;
-	const bool  need_descriptor = !bindings.descriptors.empty();
-
-	if (bindings.push_constant_size != 0) {
-		auto index = push_constant_info_num;
-
-		push_constant_info[index].stageFlags = vk_stage;
-		push_constant_info[index].offset     = bindings.push_constant_offset;
-		push_constant_info[index].size       = bindings.push_constant_size;
-		push_constant_info_num++;
-	}
-
-	if (need_descriptor) {
-		EXIT_IF(bindings.descriptor_set != set_layouts_num);
-
-		set_layouts[set_layouts_num] = descriptor_cache.GetDescriptorSetLayout(stage, program);
-		set_layouts_num++;
+static void AddLayoutBindings(std::vector<vk::DescriptorSetLayoutBinding>& descriptor_bindings,
+                              const ShaderRecompiler::IR::Program& program,
+                              vk::ShaderStageFlagBits              stage) {
+	for (const auto& binding: program.bindings.descriptors) {
+		descriptor_bindings.push_back(
+		    {ShaderRecompiler::IR::NativeBinding(program.stage, binding.kind),
+		     NativeDescriptorType(binding.kind), NativeDescriptorCount(binding), stage, nullptr});
 	}
 }
 
-static void ConfigureSubgroupSize(const GraphicContext& graphics, vk::ShaderStageFlagBits vk_stage,
-                                  const ShaderRecompiler::IR::Program&                   program,
-                                  vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo& required,
-                                  vk::PipelineShaderStageCreateInfo&                     stage) {
-	const auto config =
-	    ConfigureShaderSubgroup(ShaderSubgroupCapabilities {graphics}, vk_stage, program);
-	switch (config.mode) {
-		case ShaderSubgroupMode::Natural: return;
-		case ShaderSubgroupMode::PerInvocationGraphics: {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("Vulkan running %u-lane graphics shader stage 0x%08x with per-invocation "
-				     "EXEC/VCC on the host-native subgroup\n",
-				     program.wave_size, static_cast<uint32_t>(vk_stage));
-			}
-			return;
-		}
-		case ShaderSubgroupMode::FlattenedMasks: {
-			static std::atomic<uint32_t> log_count {0};
-			if (log_count.fetch_add(1, std::memory_order_relaxed) < 16) {
-				LOGF("Vulkan running %u-lane shader stage 0x%08x with flattened per-invocation "
-				     "EXEC/VCC\n",
-				     program.wave_size, static_cast<uint32_t>(vk_stage));
-			}
-			return;
-		}
-		case ShaderSubgroupMode::Controlled: break;
-		case ShaderSubgroupMode::Unsupported:
-		default:
-			EXIT("Vulkan cannot run %u-lane shader stage 0x%08x: default=%u min=%u max=%u "
-			     "controlled_stages=0x%08x\n",
-			     program.wave_size, static_cast<uint32_t>(vk_stage), graphics.subgroup_size,
-			     graphics.min_subgroup_size, graphics.max_subgroup_size,
-			     static_cast<vk::ShaderStageFlags::MaskType>(
-			         graphics.required_subgroup_size_stages));
+static void CreateDescriptorLayout(GraphicContext& graphics, PipelineCache::Pipeline& pipeline,
+                                   std::span<const vk::DescriptorSetLayoutBinding> bindings) {
+	uint32_t descriptor_count = 0;
+	for (const auto& binding: bindings) {
+		descriptor_count += binding.descriptorCount;
 	}
+	pipeline.uses_push_descriptors = descriptor_count <= graphics.max_push_descriptors;
 
-	required.sType = vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
-	required.requiredSubgroupSize = config.required_size;
-	stage.pNext                   = &required;
+	vk::DescriptorSetLayoutCreateInfo create {};
+	create.sType        = vk::StructureType::eDescriptorSetLayoutCreateInfo;
+	create.flags        = pipeline.uses_push_descriptors
+	                          ? vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR
+	                          : vk::DescriptorSetLayoutCreateFlags {};
+	create.bindingCount = static_cast<uint32_t>(bindings.size());
+	create.pBindings    = bindings.data();
+	EXIT_IF(graphics.device.createDescriptorSetLayout(
+	            &create, nullptr, &pipeline.descriptor_set_layout) != vk::Result::eSuccess);
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
 void CreatePipelineInternal(
-    GraphicContext& graphics, DescriptorCache& descriptor_cache,
-    PipelineCache::GraphicsPipeline& pipeline, const PipelineRenderingState& rendering,
-    const ShaderVertexInputInfo& vs_input_info, std::span<const uint32_t> vs_shader,
-    const ShaderPixelInputInfo* ps_input_info, std::span<const uint32_t> ps_shader,
-    const PipelineStaticParameters& static_params, uint32_t vs_hash0, uint32_t vs_crc32,
-    uint32_t ps_hash0, uint32_t ps_crc32, bool ps_active) {
-	EXIT_IF(ps_active && ps_input_info == nullptr);
+    GraphicContext& graphics, PipelineCache::GraphicsPipeline& pipeline,
+    const PipelineRenderingState& rendering, const ShaderVertexInputInfo& vs_input_info,
+    vk::ShaderModule vertex_module, const ShaderPixelInputInfo* ps_input_info,
+	vk::ShaderModule pixel_module, const PipelineStaticParameters& static_params) {
+	const bool ps_active = ps_input_info != nullptr;
+	EXIT_IF(vertex_module == nullptr || (ps_active && pixel_module == nullptr));
 
 	const bool rect_list = static_params.topology == vk::PrimitiveTopology::ePatchList;
 
-	vk::ShaderModule vert_shader_module         = nullptr;
 	vk::ShaderModule tess_control_shader_module = nullptr;
 	vk::ShaderModule tess_eval_shader_module    = nullptr;
-	vk::ShaderModule frag_shader_module         = nullptr;
 
 	vk::ShaderModuleCreateInfo create_info {};
-
 	create_info.sType = vk::StructureType::eShaderModuleCreateInfo;
-	create_info.pNext = nullptr;
-	create_info.flags = {};
-
-	create_info.codeSize = vs_shader.size() * 4;
-	create_info.pCode    = vs_shader.data();
-	auto result = graphics.device.createShaderModule(&create_info, nullptr, &vert_shader_module);
-	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: vkCreateShaderModule VS done result=%s module=%p\n",
-		     VulkanToString(result).c_str(), static_cast<void*>(vert_shader_module));
-	}
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	if (ps_active) {
-		create_info.codeSize = ps_shader.size() * 4;
-		create_info.pCode    = ps_shader.data();
-		result = graphics.device.createShaderModule(&create_info, nullptr, &frag_shader_module);
-		if (graphics_debug_dump_enabled()) {
-			LOGF("PipelineTrace: vkCreateShaderModule PS done result=%s module=%p\n",
-			     VulkanToString(result).c_str(), static_cast<void*>(frag_shader_module));
-		}
-		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-	}
+	vk::Result result {};
 	if (rect_list) {
 		const auto shaders =
 		    BuildRectListShaders(vs_input_info, ps_active ? ps_input_info : nullptr);
@@ -523,40 +459,27 @@ void CreatePipelineInternal(
 		EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	}
 
-	EXIT_NOT_IMPLEMENTED(vert_shader_module == nullptr);
 	EXIT_NOT_IMPLEMENTED(
 	    rect_list && (tess_control_shader_module == nullptr || tess_eval_shader_module == nullptr));
-	EXIT_NOT_IMPLEMENTED(ps_active && frag_shader_module == nullptr);
 
-	vk::PipelineShaderStageCreateInfo                     vert_shader_stage_info {};
-	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo vert_subgroup_size {};
+	vk::PipelineShaderStageCreateInfo vert_shader_stage_info {};
 
 	vert_shader_stage_info.sType               = vk::StructureType::ePipelineShaderStageCreateInfo;
 	vert_shader_stage_info.pNext               = nullptr;
 	vert_shader_stage_info.flags               = {};
 	vert_shader_stage_info.stage               = vk::ShaderStageFlagBits::eVertex;
-	vert_shader_stage_info.module              = vert_shader_module;
+	vert_shader_stage_info.module              = vertex_module;
 	vert_shader_stage_info.pName               = "main";
 	vert_shader_stage_info.pSpecializationInfo = nullptr;
-	EXIT_IF(!vs_input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eVertex, *vs_input_info.stage.program,
-	                      vert_subgroup_size, vert_shader_stage_info);
 
-	vk::PipelineShaderStageCreateInfo                     frag_shader_stage_info {};
-	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo frag_subgroup_size {};
+	vk::PipelineShaderStageCreateInfo frag_shader_stage_info {};
 	frag_shader_stage_info.sType               = vk::StructureType::ePipelineShaderStageCreateInfo;
 	frag_shader_stage_info.pNext               = nullptr;
 	frag_shader_stage_info.flags               = {};
 	frag_shader_stage_info.stage               = vk::ShaderStageFlagBits::eFragment;
-	frag_shader_stage_info.module              = frag_shader_module;
+	frag_shader_stage_info.module              = pixel_module;
 	frag_shader_stage_info.pName               = "main";
 	frag_shader_stage_info.pSpecializationInfo = nullptr;
-	if (ps_active) {
-		EXIT_IF(!ps_input_info->stage);
-		ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eFragment,
-		                      *ps_input_info->stage.program, frag_subgroup_size,
-		                      frag_shader_stage_info);
-	}
 
 	vk::PipelineShaderStageCreateInfo tess_control_shader_stage_info {};
 	tess_control_shader_stage_info.sType  = vk::StructureType::ePipelineShaderStageCreateInfo;
@@ -599,9 +522,10 @@ void CreatePipelineInternal(
 
 			uint32_t attr_size       = 4;
 			auto     registers_num   = vs_input_info.resources_dst[index].registers_num;
-			auto     used_components = (vs_input_info.resource_fetch_components[index] > 0
-			                                ? vs_input_info.resource_fetch_components[index]
-			                                : registers_num);
+			const auto compiled_components =
+			    vs_input_info.stage.program->info.vertex_fetch_components[index];
+			auto used_components =
+			    compiled_components > 0 ? static_cast<int>(compiled_components) : registers_num;
 			GetInputFormat(vs_input_info.resources[index], input_attr[index].format, attr_size,
 			               static_cast<uint32_t>(used_components));
 
@@ -726,7 +650,8 @@ void CreatePipelineInternal(
 	input_assembly.pNext    = nullptr;
 	input_assembly.flags    = {};
 	input_assembly.topology = static_params.topology;
-	input_assembly.primitiveRestartEnable = VK_FALSE;
+	input_assembly.primitiveRestartEnable =
+	    static_params.primitive_restart_enable ? VK_TRUE : VK_FALSE;
 
 	vk::Viewport viewport {};
 	viewport.x        = static_params.viewport_offset[0] - static_params.viewport_scale[0];
@@ -876,40 +801,36 @@ void CreatePipelineInternal(
 	color_blending.blendConstants[2] = static_params.blend_color_blue;
 	color_blending.blendConstants[3] = static_params.blend_color_alpha;
 
-	vk::DescriptorSetLayout set_layouts[2]  = {};
-	uint32_t                set_layouts_num = 0;
-
-	vk::PushConstantRange push_constant_info[2];
-	uint32_t              push_constant_info_num = 0;
-
 	EXIT_IF(!vs_input_info.stage);
-	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num, *vs_input_info.stage.program,
-	             vk::ShaderStageFlagBits::eVertex, DescriptorCache::Stage::Vertex);
+	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
+	AddLayoutBindings(descriptor_bindings, *vs_input_info.stage.program,
+	                  vk::ShaderStageFlagBits::eVertex);
 	if (ps_active) {
 		EXIT_IF(!ps_input_info->stage);
-		CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-		             push_constant_info_num, *ps_input_info->stage.program,
-		             vk::ShaderStageFlagBits::eFragment, DescriptorCache::Stage::Pixel);
+		AddLayoutBindings(descriptor_bindings, *ps_input_info->stage.program,
+		                  vk::ShaderStageFlagBits::eFragment);
 	}
+	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
+	constexpr auto GraphicsStages =
+	    vk::ShaderStageFlagBits::eVertex | vk::ShaderStageFlagBits::eFragment;
+	const vk::PushConstantRange push_constants {
+	    GraphicsStages, 0, ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;
 	pipeline_layout_info.pNext                  = nullptr;
 	pipeline_layout_info.flags                  = {};
-	pipeline_layout_info.setLayoutCount         = set_layouts_num;
-	pipeline_layout_info.pSetLayouts            = (set_layouts_num > 0 ? set_layouts : nullptr);
-	pipeline_layout_info.pushConstantRangeCount = push_constant_info_num;
-	pipeline_layout_info.pPushConstantRanges =
-	    push_constant_info_num > 0 ? push_constant_info : nullptr;
+	pipeline_layout_info.setLayoutCount         = 1;
+	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.pushConstantRangeCount = 1;
+	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
 	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=0x%08" PRIx32 "/0x%08" PRIx32
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " set_layouts=%" PRIu32 " push_constants=%" PRIu32
-		     "\n",
-		     vs_hash0, vs_crc32, ps_hash0, ps_crc32, set_layouts_num, push_constant_info_num);
+		LOGF("PipelineTrace: vkCreatePipelineLayout begin VS=%" PRIu64 " PS=%" PRIu64
+		     " set_layouts=1 push_constants=%" PRIu32 "\n",
+		     pipeline.vs_shader_id, pipeline.ps_shader_id, 1u);
 	}
 	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
 	                                              &pipeline.pipeline_layout);
@@ -1008,11 +929,12 @@ void CreatePipelineInternal(
 	EXIT_IF(pipeline.pipeline != nullptr);
 
 	if (graphics_debug_dump_enabled()) {
-		LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=0x%08" PRIx32 "/0x%08" PRIx32
-		     " PS=0x%08" PRIx32 "/0x%08" PRIx32 " topology=%" PRIu32 " color_mask=0x%08" PRIx32
+		LOGF("PipelineTrace: vkCreateGraphicsPipelines begin VS=%" PRIu64 " PS=%" PRIu64
+		     " topology=%" PRIu32 " color_mask=0x%08" PRIx32
 		     " depth=%s blend=%s dyn_states=%" PRIu32 " viewport=%f,%f %fx%f"
 		     " scissor=%d,%d %ux%u\n",
-		     vs_hash0, vs_crc32, ps_hash0, ps_crc32, static_cast<uint32_t>(static_params.topology),
+		     pipeline.vs_shader_id, pipeline.ps_shader_id,
+		     static_cast<uint32_t>(static_params.topology),
 		     static_params.color_mask[0], (static_params.with_depth ? "true" : "false"),
 		     (static_params.blend_enable[0] ? "true" : "false"), dynamic_states_count, viewport.x,
 		     viewport.y, viewport.width, viewport.height, scissor.offset.x, scissor.offset.y,
@@ -1028,40 +950,19 @@ void CreatePipelineInternal(
 
 	EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr);
 
-	if (frag_shader_module != nullptr) {
-		graphics.device.destroyShaderModule(frag_shader_module, nullptr);
-	}
 	if (tess_control_shader_module != nullptr) {
 		graphics.device.destroyShaderModule(tess_control_shader_module, nullptr);
 	}
 	if (tess_eval_shader_module != nullptr) {
 		graphics.device.destroyShaderModule(tess_eval_shader_module, nullptr);
 	}
-	graphics.device.destroyShaderModule(vert_shader_module, nullptr);
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descriptor_cache,
-                            PipelineCache::ComputePipeline& pipeline,
-                            const ShaderComputeInputInfo&   input_info,
-                            std::span<const uint32_t>       cs_shader) {
-	vk::ShaderModule comp_shader_module = nullptr;
-
-	vk::ShaderModuleCreateInfo create_info {};
-
-	create_info.sType    = vk::StructureType::eShaderModuleCreateInfo;
-	create_info.pNext    = nullptr;
-	create_info.flags    = {};
-	create_info.codeSize = cs_shader.size() * 4;
-	create_info.pCode    = cs_shader.data();
-	LOGF("PipelineTrace: vkCreateShaderModule CS begin words=%" PRIu64 "\n",
-	     static_cast<uint64_t>(cs_shader.size()));
-	auto result = graphics.device.createShaderModule(&create_info, nullptr, &comp_shader_module);
-	LOGF("PipelineTrace: vkCreateShaderModule CS done result=%s module=%p\n",
-	     VulkanToString(result).c_str(), static_cast<void*>(comp_shader_module));
-	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
-
-	EXIT_NOT_IMPLEMENTED(comp_shader_module == nullptr);
+void CreatePipelineInternal(GraphicContext& graphics, PipelineCache::ComputePipeline& pipeline,
+                            const ShaderComputeInputInfo& input_info,
+                            vk::ShaderModule              compute_module) {
+	EXIT_IF(compute_module == nullptr);
 
 	vk::PipelineShaderStageCreateInfo                     comp_shader_stage_info {};
 	vk::PipelineShaderStageRequiredSubgroupSizeCreateInfo comp_subgroup_size {};
@@ -1069,40 +970,41 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	comp_shader_stage_info.pNext               = nullptr;
 	comp_shader_stage_info.flags               = {};
 	comp_shader_stage_info.stage               = vk::ShaderStageFlagBits::eCompute;
-	comp_shader_stage_info.module              = comp_shader_module;
+	comp_shader_stage_info.module              = compute_module;
 	comp_shader_stage_info.pName               = "main";
 	comp_shader_stage_info.pSpecializationInfo = nullptr;
 	EXIT_IF(!input_info.stage);
-	ConfigureSubgroupSize(graphics, vk::ShaderStageFlagBits::eCompute, *input_info.stage.program,
-	                      comp_subgroup_size, comp_shader_stage_info);
+	const auto wave_size = input_info.stage.program->wave_size;
+	if (graphics.compute_subgroup_size_control_enabled &&
+	    wave_size >= graphics.min_subgroup_size && wave_size <= graphics.max_subgroup_size) {
+		comp_subgroup_size.sType =
+		    vk::StructureType::ePipelineShaderStageRequiredSubgroupSizeCreateInfo;
+		comp_subgroup_size.requiredSubgroupSize = wave_size;
+		comp_shader_stage_info.pNext            = &comp_subgroup_size;
+	}
 
-	vk::DescriptorSetLayout set_layouts[1]  = {};
-	uint32_t                set_layouts_num = 0;
-
-	vk::PushConstantRange push_constant_info[1];
-	uint32_t              push_constant_info_num = 0;
-
-	EXIT_IF(!input_info.stage);
-	CreateLayout(descriptor_cache, set_layouts, set_layouts_num, push_constant_info,
-	             push_constant_info_num, *input_info.stage.program,
-	             vk::ShaderStageFlagBits::eCompute, DescriptorCache::Stage::Compute);
+	std::vector<vk::DescriptorSetLayoutBinding> descriptor_bindings;
+	AddLayoutBindings(descriptor_bindings, *input_info.stage.program,
+	                  vk::ShaderStageFlagBits::eCompute);
+	CreateDescriptorLayout(graphics, pipeline, descriptor_bindings);
+	const vk::PushConstantRange push_constants {vk::ShaderStageFlagBits::eCompute, 0,
+	                                            ShaderRecompiler::IR::NativePushConstantSize};
 
 	vk::PipelineLayoutCreateInfo pipeline_layout_info {};
 	pipeline_layout_info.sType                  = vk::StructureType::ePipelineLayoutCreateInfo;
 	pipeline_layout_info.pNext                  = nullptr;
 	pipeline_layout_info.flags                  = {};
-	pipeline_layout_info.setLayoutCount         = set_layouts_num;
-	pipeline_layout_info.pSetLayouts            = (set_layouts_num > 0 ? set_layouts : nullptr);
-	pipeline_layout_info.pushConstantRangeCount = push_constant_info_num;
-	pipeline_layout_info.pPushConstantRanges =
-	    push_constant_info_num > 0 ? push_constant_info : nullptr;
+	pipeline_layout_info.setLayoutCount         = 1;
+	pipeline_layout_info.pSetLayouts            = &pipeline.descriptor_set_layout;
+	pipeline_layout_info.pushConstantRangeCount = 1;
+	pipeline_layout_info.pPushConstantRanges    = &push_constants;
 
 	EXIT_IF(pipeline.pipeline_layout != nullptr);
 
-	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=%u push_constants=%u\n",
-	     set_layouts_num, push_constant_info_num);
-	result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
-	                                              &pipeline.pipeline_layout);
+	LOGF("PipelineTrace: vkCreatePipelineLayout CS begin set_layouts=1 push_constants=%u\n",
+	     1u);
+	auto result = graphics.device.createPipelineLayout(&pipeline_layout_info, nullptr,
+	                                                  &pipeline.pipeline_layout);
 	LOGF("PipelineTrace: vkCreatePipelineLayout CS done result=%s layout=%p\n",
 	     VulkanToString(result).c_str(), static_cast<void*>(pipeline.pipeline_layout));
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
@@ -1128,8 +1030,6 @@ void CreatePipelineInternal(GraphicContext& graphics, DescriptorCache& descripto
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
 	EXIT_NOT_IMPLEMENTED(pipeline.pipeline == nullptr);
-
-	graphics.device.destroyShaderModule(comp_shader_module, nullptr);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/render.h

@@ -4,12 +4,14 @@
 #include "common/abi.h"
 #include "common/assert.h"
 #include "common/common.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptors.h"
+#include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/renderTarget.h"
 #include "graphics/host_gpu/vulkanCommon.h"
 
 #include <array>
-#include <memory>
+#include <optional>
+#include <span>
 #include <vector>
 
 namespace Libs::Graphics {
@@ -23,9 +25,6 @@ class Shader;
 struct GraphicContext;
 struct ShaderBufferResource;
 struct ShaderComputeInputInfo;
-struct CommandSlot;
-struct VulkanBuffer;
-struct VulkanDescriptorSet;
 struct RenderDepthInfo;
 struct RenderColorInfo;
 struct DrawCallInfo;
@@ -49,20 +48,6 @@ enum class CommandBufferDebugOp : uint32_t {
 	Unknown,
 };
 
-class FenceResourceRetainer {
-public:
-	FenceResourceRetainer() = default;
-	~FenceResourceRetainer();
-	KYTY_CLASS_NO_COPY(FenceResourceRetainer);
-
-	void               Retain(std::shared_ptr<void> resource);
-	void               ReleaseAfterFence() noexcept;
-	[[nodiscard]] bool Empty() const noexcept { return m_resources.empty(); }
-
-private:
-	std::vector<std::shared_ptr<void>> m_resources;
-};
-
 struct SubmitInfo {
 	static constexpr uint32_t MaxSemaphores = 3;
 
@@ -91,78 +76,52 @@ struct SubmitInfo {
 
 class CommandBuffer {
 public:
-	explicit CommandBuffer(CommandScheduler& scheduler);
-	~CommandBuffer();
+	~CommandBuffer() = default;
 
 	KYTY_CLASS_NO_COPY(CommandBuffer);
 
 	[[nodiscard]] bool IsInvalid() const;
 
-	void Begin() const;
-	void End() const;
-	void Execute(const SubmitInfo& submit = {});
 	void SetDebugInfo(uint32_t op, uint64_t submit_id, uint32_t arg0 = 0, uint32_t arg1 = 0,
 	                  uint32_t arg2 = 0, uint32_t arg3 = 0, uint64_t arg4 = 0);
 	void BeginRendering(const RenderState& state) const;
 	void EndRendering() const;
-	void WaitForFenceOnly();
-	void WaitForFence();
-	void WaitForFenceAndReset();
-	void RetireBufferAfterFence(std::unique_ptr<VulkanBuffer> buffer);
-	void RetainResourceUntilFence(std::shared_ptr<void> resource);
-	void RecycleDescriptorAfterFence(VulkanDescriptorSet& set);
 
 	[[nodiscard]] vk::CommandBuffer Handle() const;
 	[[nodiscard]] GraphicContext&   GetGraphics() const noexcept { return m_graphics; }
 	[[nodiscard]] RenderContext&    GetContext() const noexcept { return m_context; }
-	[[nodiscard]] bool              IsExecute() const { return m_execute; }
+	[[nodiscard]] HW::Context&      GetRegisters() const noexcept { return *m_registers; }
+	[[nodiscard]] HW::UserConfig&   GetUserConfig() const noexcept { return *m_user_config; }
+	[[nodiscard]] HW::Shader&       GetShaders() const noexcept { return *m_shaders; }
 
 private:
-	void Release();
-	void FinalizeFence(bool reset_recording);
-	void ReleaseResourcesAfterFence();
-	void DeleteBuffersAfterFence();
-	void RecycleDescriptorsAfterFence();
-
-	RenderContext&                             m_context;
-	CommandScheduler&                          m_scheduler;
-	GraphicContext&                            m_graphics;
-	CommandSlot*                               m_slot            = nullptr;
-	bool                                       m_execute         = false;
-	bool                                       m_fence_waited    = false;
-	uint64_t                                   m_submit_seq      = 0;
-	uint32_t                                   m_debug_op        = 0;
-	uint64_t                                   m_debug_submit_id = 0;
-	uint32_t                                   m_debug_arg0      = 0;
-	uint32_t                                   m_debug_arg1      = 0;
-	uint32_t                                   m_debug_arg2      = 0;
-	uint32_t                                   m_debug_arg3      = 0;
-	uint64_t                                   m_debug_arg4      = 0;
-	std::vector<std::unique_ptr<VulkanBuffer>> m_retired_buffers;
-	FenceResourceRetainer                      m_fence_resources;
-	std::vector<VulkanDescriptorSet*>          m_descriptor_sets_after_fence;
-	mutable RenderState                        m_render_state;
-	mutable bool                               m_rendering = false;
-};
-
-class RenderCommandBuffer final: public CommandBuffer {
-public:
-	explicit RenderCommandBuffer(CommandScheduler& scheduler): CommandBuffer(scheduler) {}
-
+	explicit CommandBuffer(CommandScheduler& scheduler);
 	void Bind(HW::Context& registers, HW::UserConfig& user_config, HW::Shader& shaders) noexcept {
 		m_registers   = &registers;
 		m_user_config = &user_config;
 		m_shaders     = &shaders;
 	}
 
-	[[nodiscard]] HW::Context&    GetRegisters() const noexcept { return *m_registers; }
-	[[nodiscard]] HW::UserConfig& GetUserConfig() const noexcept { return *m_user_config; }
-	[[nodiscard]] HW::Shader&     GetShaders() const noexcept { return *m_shaders; }
+	void Begin();
+	void End() const;
 
-private:
-	HW::Context*    m_registers   = nullptr;
-	HW::UserConfig* m_user_config = nullptr;
-	HW::Shader*     m_shaders     = nullptr;
+	RenderContext&      m_context;
+	GraphicContext&     m_graphics;
+	vk::CommandBuffer   m_buffer          = nullptr;
+	uint32_t            m_debug_op        = 0;
+	uint64_t            m_debug_submit_id = 0;
+	uint32_t            m_debug_arg0      = 0;
+	uint32_t            m_debug_arg1      = 0;
+	uint32_t            m_debug_arg2      = 0;
+	uint32_t            m_debug_arg3      = 0;
+	uint64_t            m_debug_arg4      = 0;
+	mutable RenderState m_render_state;
+	mutable bool        m_rendering   = false;
+	HW::Context*        m_registers   = nullptr;
+	HW::UserConfig*     m_user_config = nullptr;
+	HW::Shader*         m_shaders     = nullptr;
+
+	friend class CommandScheduler;
 };
 
 class RenderExecutor {
@@ -170,66 +129,69 @@ public:
 	explicit RenderExecutor(RenderContext& context): m_context(context) {}
 	KYTY_CLASS_NO_COPY(RenderExecutor);
 
-	void DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_type_and_size,
+	void DrawIndex(uint64_t submit_id, CommandBuffer& buffer, uint32_t index_type_and_size,
 	               uint32_t index_count, const void* index_addr, uint32_t flags, uint32_t type,
 	               uint32_t instance_count = 1, uint32_t render_target_slice_offset = 0,
 	               int32_t vertex_offset_add = 0, uint32_t first_instance = 0);
-	void DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
-	              uint32_t flags, uint32_t render_target_slice_offset = 0,
-	              uint32_t instance_count = 1, uint32_t first_vertex = 0,
-	              uint32_t first_instance = 0);
-	void DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t thread_group_x,
+	void DrawAuto(uint64_t submit_id, CommandBuffer& buffer, uint32_t index_count, uint32_t flags,
+	              uint32_t render_target_slice_offset = 0, uint32_t instance_count = 1,
+	              uint32_t first_vertex = 0, uint32_t first_instance = 0);
+	void DispatchDirect(uint64_t submit_id, CommandBuffer& buffer, uint32_t thread_group_x,
 	                    uint32_t thread_group_y, uint32_t thread_group_z, uint32_t mode);
 
-	[[nodiscard]] DescriptorCache::PreparedBindings
-	     PrepareBindings(CommandBuffer& buffer, const ShaderStageRuntime& runtime,
-	                     vk::ShaderStageFlags shader_stage, DescriptorCache::Stage stage);
-	void RebindBuffers(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings);
-	void RebindImages(CommandBuffer& buffer, DescriptorCache::PreparedBindings& bindings);
+	[[nodiscard]] PreparedBindings PrepareBindings(const ShaderStageRuntime& runtime);
+	void                           FindBuffers(PreparedBindings& bindings);
+	void                           RebindBuffers(PreparedBindings& bindings);
+	void                           RebindImages(PreparedBindings& bindings);
 	void CommitBindings(CommandBuffer& buffer, vk::PipelineBindPoint pipeline_bind_point,
-	                    vk::PipelineLayout layout, DescriptorCache::PreparedBindings& bindings);
+	                    const PipelineCache::Pipeline&     pipeline,
+	                    std::span<PreparedBindings* const> bindings);
 
 private:
 	struct GraphicsBindings {
-		DescriptorCache::PreparedBindings                vertex;
-		std::optional<DescriptorCache::PreparedBindings> pixel;
+		PreparedBindings                vertex;
+		std::optional<PreparedBindings> pixel;
 	};
 
-	[[nodiscard]] DescriptorCache::TextureBinding
-	ResolveTexture(const ShaderRecompiler::IR::ImageResource&   resource,
-	               const ShaderRecompiler::IR::DescriptorValue& value);
-	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(CommandBuffer&            buffer,
-	                                                       const ShaderStageRuntime& vertex,
+	[[nodiscard]] TextureBinding ResolveTexture(const ShaderRecompiler::IR::ImageResource& resource,
+	                                            const ShaderRecompiler::IR::DescriptorValue& value);
+	[[nodiscard]] GraphicsBindings PrepareGraphicsBindings(const ShaderStageRuntime& vertex,
 	                                                       const ShaderStageRuntime& pixel,
 	                                                       bool                      pixel_active);
-	void ResolveRenderColorTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+	void ResolveRenderColorTarget(uint64_t submit_id, CommandBuffer& buffer,
 	                              RenderColorInfo& target, uint32_t render_target_slice_offset = 0,
 	                              uint32_t render_target_slot = UINT32_MAX,
 	                              bool ignore_target_mask = false, bool exact_format = false);
-	void ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandBuffer& buffer,
+	void ResolveRenderDepthTarget(uint64_t submit_id, CommandBuffer& buffer,
 	                              RenderDepthInfo& target);
-	[[nodiscard]] bool PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
+	[[nodiscard]] bool PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& buffer,
 	                                          const DrawCallInfo& draw,
 	                                          uint32_t            render_target_slice_offset,
 	                                          bool log_setup_phases, DrawRenderState& state);
-	void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
-	                         const DrawCallInfo& draw, DrawRenderState& state,
-	                         vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
-	                         const DrawIndexBufferSource& index_source, bool log_pipeline_phase,
+	void ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer, const DrawCallInfo& draw,
+	                         DrawRenderState& state, vk::PrimitiveTopology topology,
+	                         const DrawEmitInfo& emit, const DrawIndexBufferSource& index_source,
+	                         bool primitive_restart_enable, bool log_pipeline_phase,
 	                         bool set_bind_debug, bool set_auto_debug);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth);
-	[[nodiscard]] bool        ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
+	[[nodiscard]] bool        ResolveColorTargets(uint64_t submit_id, CommandBuffer& buffer,
 	                                              uint32_t render_target_slice_offset);
 	void                      BindImage(ImageId id, bool storage);
 	void                      BindRenderTarget(ImageId id);
 	void                      TrackImageBinding(ImageId id);
 	void                      ResetBindings();
 	[[nodiscard]] bool        TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
-	                                                     const RenderCommandBuffer&    buffer);
+	                                                     const CommandBuffer&          buffer);
 
-	RenderContext&                      m_context;
-	std::vector<std::shared_ptr<Image>> m_bound_images;
+	RenderContext&                        m_context;
+	std::vector<ImageId>                  m_bound_images;
+	std::vector<vk::DescriptorBufferInfo> m_descriptor_buffers;
+	std::vector<vk::DescriptorImageInfo>  m_descriptor_images;
+	std::vector<vk::WriteDescriptorSet>   m_descriptor_writes;
+	std::vector<uint32_t>                 m_image_occurrences;
+	std::array<uint32_t, ShaderRecompiler::IR::NativePushConstantSize / sizeof(uint32_t)>
+	    m_push_constants {};
 
 	friend struct RenderExecutorTestAccess;
 };

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -11,16 +11,14 @@
 #include "graphics/guest_gpu/hardwareContext.h"
 #include "graphics/host_gpu/graphicContext.h"
 #include "graphics/host_gpu/renderer/image/imageInfo.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/descriptors.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
 #include "kernel/pthread.h"
@@ -46,7 +44,7 @@ static uint64_t BufferDescriptorSize(const ShaderBufferResource& descriptor) {
 }
 
 bool RenderExecutor::TryConsumeComputeMetaClear(const ShaderComputeInputInfo& input,
-                                                const RenderCommandBuffer&    buffer) {
+                                                const CommandBuffer&          buffer) {
 	const auto& program   = *input.stage.program;
 	const auto& resources = *input.stage.resources;
 	if (resources.buffers.size() != program.info.buffers.size()) {
@@ -83,9 +81,8 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
 	const auto& program   = *input.stage.program;
 	const auto& resources = *input.stage.resources;
 	if (program.info.buffers.size() != 1 || resources.buffers.size() != 1 ||
-	    !program.info.images.empty() || !program.info.samplers.empty() ||
-	    !program.info.addresses.empty() || !resources.images.empty() ||
-	    !resources.samplers.empty() || !resources.addresses.empty()) {
+	    !program.info.images.empty() || !program.info.samplers.empty() || program.info.uses_dma ||
+	    !resources.images.empty() || !resources.samplers.empty()) {
 		return false;
 	}
 	const auto& resource   = program.info.buffers.front();
@@ -115,7 +112,7 @@ bool ResolveComputeImageClear(const ShaderComputeInputInfo& input, uint32_t grou
 	    group_z == 1 && input.dispatch_threads_num[0] == group_x &&
 	    input.dispatch_threads_num[1] == 1 && input.dispatch_threads_num[2] == 1 &&
 	    input.group_id[0] && !input.group_id[1] && !input.group_id[2] &&
-	    input.thread_ids_num == 1 && input.wave_size == 32 && !input.tg_size_en && mode == 0x61u &&
+	    input.thread_ids_num == 1 && input.wave_size == 64 && !input.tg_size_en && mode == 0x61u &&
 	    group_x % input.threads_num[0] == 0 && descriptor.NumRecords() == group_x;
 	const auto size = BufferDescriptorSize(descriptor);
 	if (!full_dispatch || size == 0) {
@@ -139,6 +136,21 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com
 	}
 	auto& cache = command.GetContext().GetTextureCache();
 	if (!cache.ClearImageFromBuffer(command, descriptor.Base48(), size, packed_clear)) {
+		// Recognized metadata-fill shaders access DCC as an ordinary storage buffer and may run
+		// before the render target is bound. TryConsumeDccFill either consumes registered state
+		// or retains a PendingDcc fill while allowing the dispatch to run.
+		const bool registered_metadata =
+		    cache.TryConsumeDccFill(descriptor.Base48(), size, packed_clear);
+		static std::atomic<uint32_t> logged_metadata_clears {0};
+		if (logged_metadata_clears.fetch_add(1, std::memory_order_relaxed) < 32) {
+			LOGF("GraphicsRenderDispatchDirect: %s metadata clear shader=0x%016" PRIx64
+			     " addr=0x%016" PRIx64 " size=0x%016" PRIx64 " value=0x%08" PRIx32 "\n",
+			     registered_metadata ? "tracked" : "deferred", input.stage.program->shader_hash,
+			     descriptor.Base48(), size, packed_clear);
+		}
+		if (registered_metadata) {
+			return true;
+		}
 		return false;
 	}
 	static std::atomic<uint32_t> logged_clears {0};
@@ -150,10 +162,11 @@ static bool TryConsumeComputeImageClear(const ShaderComputeInputInfo& input, Com
 	return true;
 }
 
-void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::DispatchDirect(uint64_t submit_id, CommandBuffer& buffer,
                                     uint32_t thread_group_x, uint32_t thread_group_y,
                                     uint32_t thread_group_z, uint32_t mode) {
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ctx    = buffer.GetRegisters();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -194,16 +207,12 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 	const auto& cs_regs = sh_ctx.GetCs();
 	const auto& sh_regs = ctx.GetShaderRegisters();
 
-	ShaderComputeInputInfo    input_info {};
-	std::span<const uint32_t> cs_shader;
-	if (!ShaderCompileInfoCS(cs_regs, sh_regs, input_info, cs_shader)) {
-		EXIT("ShaderCompileInfoCS failed for dispatch with CS shader 0x%016" PRIx64 "\n",
-		     cs_regs.cs_regs.data_addr);
-	}
-
+	ShaderComputeInputInfo input_info {};
 	const bool use_thread_dimensions = (mode & DISPATCH_INITIATOR_USE_THREAD_DIMENSIONS) != 0;
+	input_info.dispatch_thread_dimensions = use_thread_dimensions;
+	const auto compute_program =
+	    m_context.GetPipelineCache().GetComputeProgram(cs_regs, sh_regs, input_info);
 	if (use_thread_dimensions) {
-		input_info.dispatch_thread_dimensions = true;
 		input_info.dispatch_threads_num[0]    = thread_group_x;
 		input_info.dispatch_threads_num[1]    = thread_group_y;
 		input_info.dispatch_threads_num[2]    = thread_group_z;
@@ -261,9 +270,10 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 			     r.Base40(), static_cast<uint32_t>(r.Type()), static_cast<uint32_t>(r.Format()),
 			     static_cast<uint32_t>(r.Width5()) + 1u, static_cast<uint32_t>(r.Height5()) + 1u,
 			     static_cast<uint32_t>(r.Depth()) + 1u,
-			     std::max<uint32_t>(static_cast<uint32_t>(r.LastLevel()),
-			                        static_cast<uint32_t>(r.MaxMip())) +
-			         1u,
+			     r.Type() == Prospero::ImageType::kColor2DMsaa ||
+			             r.Type() == Prospero::ImageType::kColor2DMsaaArray
+			         ? 1u
+			         : static_cast<uint32_t>(image.r128 ? r.LastLevel() : r.MaxMip()) + 1u,
 			     static_cast<uint32_t>(r.TileMode()));
 		}
 		for (uint32_t i = 0; i < program.info.samplers.size(); i++) {
@@ -317,14 +327,19 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 
 	buffer.EndRendering();
 	auto& pipeline =
-	    m_context.GetPipelineCache().CreateComputePipeline(input_info, sh_ctx.GetCs(), cs_shader);
-	auto bindings = PrepareBindings(buffer, input_info.stage, vk::ShaderStageFlagBits::eCompute,
-	                                DescriptorCache::Stage::Compute);
-	RebindBuffers(buffer, bindings);
-	RebindImages(buffer, bindings);
+	    m_context.GetPipelineCache().CreateComputePipeline(input_info, compute_program);
+	auto bindings = PrepareBindings(input_info.stage);
+	FindBuffers(bindings);
+	if (program.info.uses_dma) {
+		m_context.GetGpuResources().PrepareBda();
+	}
+	RebindBuffers(bindings);
+	RebindImages(bindings);
 
-	auto vk_buffer = buffer.Handle();
-	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline.pipeline_layout, bindings);
+	auto              vk_buffer        = buffer.Handle();
+	PreparedBindings* descriptor_stage = &bindings;
+	CommitBindings(buffer, vk::PipelineBindPoint::eCompute, pipeline,
+	               std::span {&descriptor_stage, 1u});
 	bool has_storage_writes = HasShaderBufferWrites(input_info.stage);
 	has_storage_writes =
 	    std::any_of(program.info.images.begin(), program.info.images.end(),

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -4,7 +4,6 @@
 #include "common/logging/log.h"
 #include "graphics/guest_gpu/graphicsRun.h"
 #include "graphics/presentation/videoOut.h"
-#include "kernel/pthread.h"
 #include "libs/errno.h"
 
 #include <algorithm>
@@ -13,8 +12,9 @@ namespace Libs::Graphics {
 
 RenderContext::RenderContext(GraphicContext& graphics)
     : m_graphics(graphics), m_render_executor(*this), m_command_scheduler(*this, graphics),
-      m_descriptor_cache(graphics), m_pipeline_cache(graphics, m_descriptor_cache),
-      m_sampler_cache(graphics), m_gpu_resources(graphics, m_command_scheduler) {
+      m_descriptor_heap(graphics, m_command_scheduler.GetMasterSemaphore()),
+      m_pipeline_cache(graphics), m_sampler_cache(graphics),
+      m_gpu_resources(graphics, m_command_scheduler) {
 	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
 }
 
@@ -26,7 +26,7 @@ RenderContext::~RenderContext() {
 void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) {
 	EXIT_IF(m_gpu != nullptr);
 	m_video_out = video_out;
-	m_gpu       = std::make_unique<Gpu>(*this);
+	m_gpu       = std::make_unique<GuestGpu>(*this);
 	m_gpu_resources.SetGpu(m_gpu.get());
 }
 
@@ -38,14 +38,14 @@ void RenderContext::ShutdownGpu() {
 	}
 	if (m_video_out != nullptr) {
 		if (m_command_scheduler.Active()) {
-			m_command_scheduler.FinishCurrent();
+			m_command_scheduler.Finish();
 		}
 		m_command_scheduler.DrainPriorityOperations();
 		m_video_out = nullptr;
 	}
 }
 
-Gpu& RenderContext::GetGpu() const {
+GuestGpu& RenderContext::GetGpu() const {
 	EXIT_IF(m_gpu == nullptr);
 	return *m_gpu;
 }
@@ -55,60 +55,54 @@ VideoOut::VideoOutDriver& RenderContext::GetVideoOut() const {
 	return *m_video_out;
 }
 
-void RenderContext::AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) {
-	auto queue = LibKernel::EventQueue::KernelPinEqueue(eq);
-	if (!queue) {
+void RenderContext::AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
+	Common::LockGuard lock(m_interrupt_mutex);
+
+	auto it = std::find_if(
+	    m_interrupt_eqs.begin(), m_interrupt_eqs.end(),
+	    [eq, event_id](const auto& entry) { return entry.eq == eq && entry.event_id == event_id; });
+	if (it != m_interrupt_eqs.end()) {
 		return;
 	}
 
-	Common::LockGuard lock(m_eop_mutex);
-
-	auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) {
-		return entry.eq == eq && entry.id == id;
-	});
-	if (it != m_eop_eqs.end()) {
-		return;
-	}
-
-	m_eop_eqs.push_back({eq, std::move(queue), id});
+	m_interrupt_eqs.push_back({eq, event_id});
 }
 
-void RenderContext::DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id) {
-	Common::LockGuard lock(m_eop_mutex);
+void RenderContext::DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id) {
+	Common::LockGuard lock(m_interrupt_mutex);
 
-	auto it = std::find_if(m_eop_eqs.begin(), m_eop_eqs.end(), [eq, id](const auto& entry) {
-		return entry.eq == eq && entry.id == id;
-	});
-	if (it == m_eop_eqs.end()) {
+	auto it = std::find_if(
+	    m_interrupt_eqs.begin(), m_interrupt_eqs.end(),
+	    [eq, event_id](const auto& entry) { return entry.eq == eq && entry.event_id == event_id; });
+	if (it == m_interrupt_eqs.end()) {
 		return;
 	}
 
-	m_eop_eqs.erase(it);
+	m_interrupt_eqs.erase(it);
 }
 
-void RenderContext::TriggerEopEvent(uint32_t context_id) {
-	std::vector<EopEqRegistration> registrations;
+void RenderContext::TriggerInterrupt(int event_id, uint32_t context_id) {
+	std::vector<InterruptEqRegistration> registrations;
 	{
-		Common::LockGuard lock(m_eop_mutex);
-		registrations = m_eop_eqs;
+		Common::LockGuard lock(m_interrupt_mutex);
+		for (const auto& registration: m_interrupt_eqs) {
+			if (registration.event_id == event_id) {
+				registrations.push_back(registration);
+			}
+		}
 	}
 
 	for (const auto& registration: registrations) {
 		const auto result = LibKernel::EventQueue::KernelTriggerEvent(
-		    registration.eq, static_cast<uintptr_t>(registration.id),
+		    registration.eq, static_cast<uintptr_t>(registration.event_id),
 		    LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS,
 		    reinterpret_cast<void*>(static_cast<uintptr_t>(context_id)));
 		if (result == LibKernel::KERNEL_ERROR_EBADF || result == LibKernel::KERNEL_ERROR_ENOENT) {
-			DeleteEopEq(registration.eq, registration.id);
+			DeleteInterruptEq(registration.eq, registration.event_id);
 			continue;
 		}
 		EXIT_NOT_IMPLEMENTED(result != OK);
 	}
-
-	auto tsc    = LibKernel::KernelReadTsc();
-	auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT,
-	                                                                  reinterpret_cast<void*>(tsc));
-	EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT);
 }
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderContext.h

@@ -10,7 +10,7 @@
 #include "graphics/host_gpu/renderer/cache/samplerCache.h"
 #include "graphics/host_gpu/renderer/cache/textureCache.h"
 #include "graphics/host_gpu/renderer/commandScheduler.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
+#include "graphics/host_gpu/renderer/pipeline/descriptorHeap.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "kernel/eventQueue.h"
 
@@ -23,8 +23,7 @@ class VideoOutDriver;
 
 namespace Libs::Graphics {
 
-constexpr int AGC_USER_INTERRUPT_EVENT = 0x1800;
-class Gpu;
+class GuestGpu;
 
 class RenderContext {
 public:
@@ -35,43 +34,42 @@ public:
 	[[nodiscard]] GraphicContext&           GetGraphics() const noexcept { return m_graphics; }
 	void                                    InitializeGpu(VideoOut::VideoOutDriver* video_out);
 	void                                    ShutdownGpu();
-	[[nodiscard]] Gpu&                      GetGpu() const;
+	[[nodiscard]] GuestGpu&                 GetGpu() const;
 	[[nodiscard]] VideoOut::VideoOutDriver& GetVideoOut() const;
 
 	Common::Mutex&      GetMutex() { return m_mutex; }
 	CommandScheduler&   GetCommandScheduler() { return m_command_scheduler; }
 	PipelineCache&      GetPipelineCache() { return m_pipeline_cache; }
-	DescriptorCache&    GetDescriptorCache() { return m_descriptor_cache; }
+	DescriptorHeap&     GetDescriptorHeap() { return m_descriptor_heap; }
 	SamplerCache&       GetSamplerCache() { return m_sampler_cache; }
 	GpuResourceManager& GetGpuResources() { return m_gpu_resources; }
 	BufferCache&        GetBufferCache() { return m_gpu_resources.GetBufferCache(); }
 	TextureCache&       GetTextureCache() { return m_gpu_resources.GetTextureCache(); }
 	RenderExecutor&     GetRenderExecutor() { return m_render_executor; }
 
-	void AddEopEq(LibKernel::EventQueue::KernelEqueue eq, int id);
-	void DeleteEopEq(LibKernel::EventQueue::KernelEqueue eq, int id);
-	void TriggerEopEvent(uint32_t context_id);
+	void AddInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
+	void DeleteInterruptEq(LibKernel::EventQueue::KernelEqueue eq, int event_id);
+	void TriggerInterrupt(int event_id, uint32_t context_id);
 
 private:
-	struct EopEqRegistration {
-		LibKernel::EventQueue::KernelEqueue    eq = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
-		LibKernel::EventQueue::KernelEqueueRef queue;
-		int                                    id = 0;
+	struct InterruptEqRegistration {
+		LibKernel::EventQueue::KernelEqueue eq       = LibKernel::EventQueue::KERNEL_EQUEUE_INVALID;
+		int                                 event_id = 0;
 	};
 
 	GraphicContext&           m_graphics;
 	Common::Mutex             m_mutex;
 	RenderExecutor            m_render_executor;
 	CommandScheduler          m_command_scheduler;
-	DescriptorCache           m_descriptor_cache;
+	DescriptorHeap            m_descriptor_heap;
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
 	GpuResourceManager        m_gpu_resources;
-	std::unique_ptr<Gpu>      m_gpu;
+	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 
-	Common::Mutex                  m_eop_mutex;
-	std::vector<EopEqRegistration> m_eop_eqs;
+	Common::Mutex                        m_interrupt_mutex;
+	std::vector<InterruptEqRegistration> m_interrupt_eqs;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -15,15 +15,13 @@
 #include "graphics/host_gpu/renderer/colorRenderTarget.h"
 #include "graphics/host_gpu/renderer/debug.h"
 #include "graphics/host_gpu/renderer/depthRenderTarget.h"
-#include "graphics/host_gpu/renderer/pipeline/descriptorCache.h"
 #include "graphics/host_gpu/renderer/pipeline/pipelineCache.h"
 #include "graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h"
-#include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
 #include "kernel/memory.h"
@@ -35,9 +33,11 @@
 #include <atomic>
 #include <bit>
 #include <cmath>
+#include <cstdio>
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <span>
 #include <unordered_map>
@@ -103,7 +103,7 @@ static bool IsDualSourceBlendFactor(uint32_t factor) {
 }
 
 static void LogFramebufferSkip(const char* draw_name, const RenderColorInfo& color,
-                               const RenderDepthInfo& depth, const RenderCommandBuffer& buffer,
+                               const RenderDepthInfo& depth, const CommandBuffer& buffer,
                                uint32_t index_count, uint32_t flags) {
 	const auto& ctx  = buffer.GetRegisters();
 	const auto& ucfg = buffer.GetUserConfig();
@@ -126,7 +126,7 @@ static void LogFramebufferSkip(const char* draw_name, const RenderColorInfo& col
 	    static_cast<uint32_t>(ucfg.GetPrimType()), index_count, flags);
 }
 
-static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer,
+static void LogMrtState(const char* draw_name, const CommandBuffer& buffer,
                         const ShaderPixelInputInfo& ps_input_info) {
 	const auto& ctx            = buffer.GetRegisters();
 	const auto& sh_regs        = ctx.GetShaderRegisters();
@@ -140,18 +140,6 @@ static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer
 	                   (bc0.separate_alpha_blend && (IsDualSourceBlendFactor(bc0.alpha_srcblend) ||
 	                                                 IsDualSourceBlendFactor(bc0.alpha_destblend)));
 
-	for (uint32_t i = 1; i < 8; i++) {
-		const auto& rt = ctx.GetRenderTarget(i);
-		if (rt.base.addr != 0 || ps_input_info.target_output_mode[i] != 0 ||
-		    ((rt_mask >> (i * 4u)) & 0x0fu) != 0 || ((cb_shader_mask >> (i * 4u)) & 0x0fu) != 0) {
-			interesting = true;
-		}
-	}
-
-	if (!interesting) {
-		return;
-	}
-
 	auto log_id = g_mrt_state_log_count.fetch_add(1);
 	if (log_id >= 32) {
 		return;
@@ -189,7 +177,7 @@ static void LogMrtState(const char* draw_name, const RenderCommandBuffer& buffer
 }
 
 static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& color,
-                               const RenderDepthInfo& depth, const RenderCommandBuffer& buffer,
+                               const RenderDepthInfo& depth, const CommandBuffer& buffer,
                                const ShaderPixelInputInfo& ps_input_info, uint32_t index_count,
                                uint32_t flags) {
 	const auto& ctx  = buffer.GetRegisters();
@@ -243,7 +231,7 @@ static void LogDrawTargetState(const char* draw_name, const RenderColorInfo& col
 	LogMrtState(draw_name, buffer, ps_input_info);
 }
 
-static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderColorInfo& color,
+static void LogDrawInputState(const CommandBuffer& buffer, const RenderColorInfo& color,
                               const ShaderVertexInputInfo& vs_input_info,
                               uint32_t index_type_and_size, uint32_t index_count,
                               const void* index_addr) {
@@ -321,89 +309,43 @@ static void LogDrawInputState(const RenderCommandBuffer& buffer, const RenderCol
 	}
 }
 
-static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandBuffer& buffer,
-                                                            const RenderColorInfo*     colors,
-                                                            uint32_t                   color_count,
-                                                            const RenderDepthInfo&     depth) {
+static void SetGraphicsDynamicParams(const CommandBuffer& buffer, vk::CommandBuffer vk_buffer,
+                                     const RenderColorInfo* colors, uint32_t color_count,
+                                     const RenderDepthInfo& depth) {
+	KYTY_PROFILER_FUNCTION();
+
 	EXIT_IF(colors == nullptr);
 	const auto& ctx = buffer.GetRegisters();
 
-	PipelineDynamicParameters ret {};
-	ret.color_write_count   = color_count;
-	ret.stencil_test_enable = depth.stencil_test_enable;
-
 	const auto&  vp = ctx.GetScreenViewport();
 	vk::Extent2D framebuffer_extent {};
 	if (color_count > 0 && colors[0].image_id) {
 		framebuffer_extent = colors[0].extent;
 	} else if (depth.image_id) {
 		framebuffer_extent = {depth.width, depth.height};
+	} else {
+		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
+		framebuffer_extent = {limits.maxFramebufferWidth, limits.maxFramebufferHeight};
 	}
 
 	const auto final_scissor = calc_final_scissor(vp, ctx.GetScanModeControl(), framebuffer_extent);
 
-	ret.viewport_scale[0]  = vp.viewports[0].xscale;
-	ret.viewport_scale[1]  = vp.viewports[0].yscale;
-	ret.viewport_scale[2]  = vp.viewports[0].zscale;
-	ret.viewport_offset[0] = vp.viewports[0].xoffset;
-	ret.viewport_offset[1] = vp.viewports[0].yoffset;
-	ret.viewport_offset[2] = vp.viewports[0].zoffset;
-	ret.scissor_ltrb[0]    = final_scissor.left;
-	ret.scissor_ltrb[1]    = final_scissor.top;
-	ret.scissor_ltrb[2]    = final_scissor.right;
-	ret.scissor_ltrb[3]    = final_scissor.bottom;
-	ret.line_width         = ctx.GetLineWidth();
-	ret.stencil_front      = depth.stencil_dynamic_front;
-	ret.stencil_back       = depth.stencil_dynamic_back;
-
-	const auto& mode        = ctx.GetModeControl();
-	const auto& poly_offset = ctx.GetPolyOffset();
-	const bool  use_front   = mode.poly_offset_front_enable && !mode.cull_front;
-	const bool  use_back    = mode.poly_offset_back_enable && !mode.cull_back;
-	ret.depth_bias_enable   = use_front || use_back;
-	if (ret.depth_bias_enable) {
-		// Vulkan has one bias for both faces. Prefer a visible front face when both are enabled.
-		const float guest_constant_factor =
-		    use_front ? poly_offset.front_offset : poly_offset.back_offset;
-		ret.depth_bias_constant_factor =
-		    ConvertPolygonOffsetConstantFactor(guest_constant_factor, poly_offset, depth.format);
-		ret.depth_bias_clamp = poly_offset.clamp;
-		ret.depth_bias_slope_factor =
-		    (use_front ? poly_offset.front_scale : poly_offset.back_scale) / 16.0f;
-	}
-
-	// CB_COLOR_CONTROL.operation controls special CB operations, not the normal color component
-	// write mask. Use CB_TARGET_MASK here so scanout passes with mode=Disable do not go black.
-	for (uint32_t i = 0; i < RENDER_COLOR_ATTACHMENTS_MAX; i++) {
-		ret.color_write_enable[i] =
-		    (i < color_count &&
-		     render_target_mask_slot(ctx.GetRenderTargetMask(), colors[i].target_slot) != 0);
-	}
-
-	return ret;
-}
-
-static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                             const PipelineDynamicParameters& dynamic_params) {
-	KYTY_PROFILER_FUNCTION();
-
 	vk::Viewport viewport {};
-	viewport.x        = dynamic_params.viewport_offset[0] - dynamic_params.viewport_scale[0];
-	viewport.y        = dynamic_params.viewport_offset[1] - dynamic_params.viewport_scale[1];
-	viewport.width    = dynamic_params.viewport_scale[0] * 2.0f;
-	viewport.height   = dynamic_params.viewport_scale[1] * 2.0f;
-	viewport.minDepth = dynamic_params.viewport_offset[2];
-	viewport.maxDepth = dynamic_params.viewport_scale[2] + dynamic_params.viewport_offset[2];
+	viewport.x        = vp.viewports[0].xoffset - vp.viewports[0].xscale;
+	viewport.y        = vp.viewports[0].yoffset - vp.viewports[0].yscale;
+	viewport.width    = vp.viewports[0].xscale * 2.0f;
+	viewport.height   = vp.viewports[0].yscale * 2.0f;
+	viewport.minDepth = vp.viewports[0].zoffset;
+	viewport.maxDepth = vp.viewports[0].zscale + vp.viewports[0].zoffset;
 	vk_buffer.setViewport(0, 1, &viewport);
 
 	vk::Rect2D scissor {};
-	scissor.offset = {dynamic_params.scissor_ltrb[0], dynamic_params.scissor_ltrb[1]};
-	scissor.extent = {
-	    static_cast<uint32_t>(dynamic_params.scissor_ltrb[2] - dynamic_params.scissor_ltrb[0]),
-	    static_cast<uint32_t>(dynamic_params.scissor_ltrb[3] - dynamic_params.scissor_ltrb[1])};
+	scissor.offset = {final_scissor.left, final_scissor.top};
+	scissor.extent = {static_cast<uint32_t>(final_scissor.right - final_scissor.left),
+	                  static_cast<uint32_t>(final_scissor.bottom - final_scissor.top)};
 	vk_buffer.setScissor(0, 1, &scissor);
 
-	float line_width = dynamic_params.line_width;
+	float line_width = ctx.GetLineWidth();
 	if (line_width != 1.0f) {
 		static bool logged = false;
 		if (!logged) {
@@ -416,26 +358,36 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe
 	}
 	vk_buffer.setLineWidth(line_width);
 
-	vk_buffer.setDepthBiasEnable(dynamic_params.depth_bias_enable ? VK_TRUE : VK_FALSE);
-	if (dynamic_params.depth_bias_enable) {
-		vk_buffer.setDepthBias(dynamic_params.depth_bias_constant_factor,
-		                       dynamic_params.depth_bias_clamp,
-		                       dynamic_params.depth_bias_slope_factor);
+	const auto& mode              = ctx.GetModeControl();
+	const auto& poly_offset       = ctx.GetPolyOffset();
+	const bool  use_front         = mode.poly_offset_front_enable && !mode.cull_front;
+	const bool  use_back          = mode.poly_offset_back_enable && !mode.cull_back;
+	const bool  depth_bias_enable = use_front || use_back;
+	vk_buffer.setDepthBiasEnable(depth_bias_enable ? VK_TRUE : VK_FALSE);
+	if (depth_bias_enable) {
+		// Vulkan has one bias for both faces. Prefer a visible front face when both are enabled.
+		const float guest_constant_factor =
+		    use_front ? poly_offset.front_offset : poly_offset.back_offset;
+		const float constant_factor =
+		    ConvertPolygonOffsetConstantFactor(guest_constant_factor, poly_offset, depth.format);
+		const float slope_factor =
+		    (use_front ? poly_offset.front_scale : poly_offset.back_scale) / 16.0f;
+		vk_buffer.setDepthBias(constant_factor, poly_offset.clamp, slope_factor);
 	}
 
-	if (dynamic_params.stencil_test_enable) {
+	if (depth.stencil_test_enable) {
 		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eFront,
-		                                dynamic_params.stencil_front.compareMask);
+		                                depth.stencil_dynamic_front.compareMask);
 		vk_buffer.setStencilCompareMask(vk::StencilFaceFlagBits::eBack,
-		                                dynamic_params.stencil_back.compareMask);
+		                                depth.stencil_dynamic_back.compareMask);
 		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eFront,
-		                              dynamic_params.stencil_front.writeMask);
+		                              depth.stencil_dynamic_front.writeMask);
 		vk_buffer.setStencilWriteMask(vk::StencilFaceFlagBits::eBack,
-		                              dynamic_params.stencil_back.writeMask);
+		                              depth.stencil_dynamic_back.writeMask);
 		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eFront,
-		                              dynamic_params.stencil_front.reference);
+		                              depth.stencil_dynamic_front.reference);
 		vk_buffer.setStencilReference(vk::StencilFaceFlagBits::eBack,
-		                              dynamic_params.stencil_back.reference);
+		                              depth.stencil_dynamic_back.reference);
 	}
 
 #if defined(__APPLE__)
@@ -443,11 +395,15 @@ static void SetDynamicParams(const RenderCommandBuffer& buffer, vk::CommandBuffe
 	// eColorWriteEnableEXT dynamic state and relies on the static colorWriteMask instead.
 #else
 	vk::Bool32 enable[RENDER_COLOR_ATTACHMENTS_MAX] = {};
-	for (uint32_t i = 0; i < dynamic_params.color_write_count; i++) {
-		enable[i] = (dynamic_params.color_write_enable[i] ? VK_TRUE : VK_FALSE);
+	// Color-control operation selects special color-buffer paths, not the normal component write
+	// mask. Attachment availability therefore follows the target write mask.
+	for (uint32_t i = 0; i < color_count; i++) {
+		enable[i] = render_target_mask_slot(ctx.GetRenderTargetMask(), colors[i].target_slot) != 0
+		                ? VK_TRUE
+		                : VK_FALSE;
 	}
-	if (dynamic_params.color_write_count != 0) {
-		vk_buffer.setColorWriteEnableEXT(dynamic_params.color_write_count, enable);
+	if (color_count != 0) {
+		vk_buffer.setColorWriteEnableEXT(color_count, enable);
 	}
 #endif
 }
@@ -465,7 +421,7 @@ static bool PixelShaderHasDepthOrCoverageSideEffects(const HW::ShaderRegisters&
 	       db.shader_execute_on_noop;
 }
 
-static bool ShouldSkipGeShader(const RenderCommandBuffer& buffer) {
+static bool ShouldSkipGeShader(const CommandBuffer& buffer) {
 	const auto& ctx         = buffer.GetRegisters();
 	const auto& ucfg        = buffer.GetUserConfig();
 	const auto& sh_ctx      = buffer.GetShaders();
@@ -503,6 +459,12 @@ static bool ShouldSkipGeShader(const RenderCommandBuffer& buffer) {
 	    sh_regs.m_geMaxOutputPerSubgroup > 0x00000040;
 
 	if (unsupported_stage_mask || unsupported_gs_stage || ge_group_size || ge_shader_regs) {
+		static std::once_flag warning_once;
+		std::call_once(warning_once, [] {
+			std::printf("Warning: game uses unsupported graphics pipelines; some draw calls were "
+			            "skipped.\n");
+		});
+
 		const auto log_id = g_shader_stage_log_count.fetch_add(1);
 		if (log_id < 32) {
 			LOGF("Skipping unsupported GE shader draw: stages=0x%08" PRIx32
@@ -521,15 +483,15 @@ static bool ShouldSkipGeShader(const RenderCommandBuffer& buffer) {
 }
 
 struct DrawRenderState {
-	RenderDepthInfo           depth_info;
-	RenderColorInfo           color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
-	uint32_t                  color_count                              = 0;
-	bool                      ps_active                                = true;
-	RenderState               rendering;
-	ShaderVertexInputInfo     vs_input_info;
-	ShaderPixelInputInfo      ps_input_info;
-	std::span<const uint32_t> vs_shader;
-	std::span<const uint32_t> ps_shader;
+	RenderDepthInfo       depth_info;
+	RenderColorInfo       color_info[RENDER_COLOR_ATTACHMENTS_MAX] = {};
+	uint32_t              color_count                              = 0;
+	bool                  ps_active                                = true;
+	RenderState           rendering;
+	ShaderVertexInputInfo vs_input_info;
+	ShaderPixelInputInfo  ps_input_info;
+	ShaderProgram         vertex_program;
+	ShaderProgram         pixel_program;
 };
 
 struct DrawCallInfo {
@@ -541,6 +503,70 @@ struct DrawCallInfo {
 	uint32_t             first_instance = 0;
 };
 
+static bool ResolveDccAttachmentClear(TextureCache& cache, const RenderColorInfo& target,
+                                      const ImageViewInfo& view, vk::ClearColorValue& clear_value) {
+	if (target.desc.info.metadata.kind != ImageMetadataKind::Dcc) {
+		return false;
+	}
+
+	// A DCC fast clear may update metadata only and leave the color allocation stale. Vulkan has
+	// no guest DCC state, so materialize the deferred value when the surface is next bound.
+	const auto metadata_address = target.desc.info.metadata.range.address;
+	uint32_t   metadata_value   = 0xffffffffu;
+	if (!cache.IsMetaCleared(metadata_address, view.base_layer, &metadata_value)) {
+		return false;
+	}
+
+	// Translate recognized constant and register-backed DCC states into a host clear value.
+	clear_value = {};
+	switch (static_cast<uint8_t>(metadata_value)) {
+		case 0x00: break;
+		case 0x20:
+			if (!target.metadata_clear_supported) {
+				return false;
+			}
+			clear_value = target.color_clear_value;
+			break;
+		case 0x40:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[3] = 1.0f;
+			break;
+		case 0x80:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[0] = 1.0f;
+			clear_value.float32[1] = 1.0f;
+			clear_value.float32[2] = 1.0f;
+			break;
+		case 0xc0:
+			if (!target.metadata_fixed_clear_supported) {
+				return false;
+			}
+			clear_value.float32[0] = 1.0f;
+			clear_value.float32[1] = 1.0f;
+			clear_value.float32[2] = 1.0f;
+			clear_value.float32[3] = 1.0f;
+			break;
+		default: return false;
+	}
+
+	for (uint32_t layer = 1; layer < view.layer_count; layer++) {
+		if (!cache.IsMetaCleared(metadata_address, view.base_layer + layer)) {
+			return false;
+		}
+	}
+	// Consume only after every layer in this Vulkan view can be materialized together.
+	for (uint32_t layer = 0; layer < view.layer_count; layer++) {
+		if (!cache.TouchMeta(metadata_address, view.base_layer + layer, false)) {
+			EXIT("failed to consume DCC clear state\n");
+		}
+	}
+	return true;
+}
+
 RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
                                                  uint32_t color_count, RenderDepthInfo& depth) {
 	EXIT_IF(colors == nullptr || color_count > RENDER_COLOR_ATTACHMENTS_MAX);
@@ -554,7 +580,7 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	for (uint32_t i = 0; i < color_count; i++) {
 		auto& target = colors[i];
 		EXIT_IF(!target.image_id);
-		const auto old_image = cache.ResolveOwner(target.image_id);
+		const auto old_image = cache.m_slot_images.try_get(target.image_id);
 		if (old_image == nullptr || (!old_image->registered && !old_image->info.data.Empty()) ||
 		    old_image->binding.needs_rebind) {
 			if (old_image != nullptr) {
@@ -565,6 +591,15 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		}
 		target.image_view = cache.FindRenderTarget(target.image_id, target.desc);
 		auto& image       = cache.GetImage(target.image_id);
+		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
+		                     "Kyty.MRT{}.Image[guest=0x{:016x} size=0x{:x} format={}]",
+		                     target.target_slot, image.info.data.address, image.info.data.size,
+		                     static_cast<uint32_t>(image.info.pixel_format));
+		SetVulkanObjectNameF(m_context.GetGraphics().device, target.image_view,
+		                     "Kyty.MRT{}.View[guest=0x{:016x} mip={} layer={}+{}]",
+		                     target.target_slot, image.info.data.address,
+		                     target.desc.view_info.base_level, target.desc.view_info.base_layer,
+		                     target.desc.view_info.layer_count);
 		EXIT_IF(image.backing.samples != target.samples || target.image_view == nullptr);
 		if (attachment_samples == 0) {
 			attachment_samples = target.samples;
@@ -588,10 +623,16 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.image_view   = target.image_view;
 		attachment.image_layout = layout;
 		attachment.clear_value  = target.color_clear_value.uint32;
-		attachment.is_clear     = target.color_clear_enable;
+		vk::ClearColorValue metadata_clear_value {};
+		const bool          metadata_clear =
+		    ResolveDccAttachmentClear(cache, target, view, metadata_clear_value);
+		if (metadata_clear) {
+			attachment.clear_value = metadata_clear_value.uint32;
+		}
+		attachment.is_clear = target.color_clear_enable || metadata_clear;
 	}
 	if (depth.image_id) {
-		const auto owner = cache.ResolveOwner(depth.image_id);
+		const auto owner = cache.m_slot_images.try_get(depth.image_id);
 		if (owner == nullptr || !owner->registered || owner->binding.needs_rebind) {
 			EXIT("depth target changed after render-state discovery\n");
 		}
@@ -608,6 +649,14 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 			EXIT("failed to consume HTile clear state\n");
 		}
 		auto& image = cache.GetImage(depth.image_id);
+		SetVulkanObjectNameF(m_context.GetGraphics().device, image.backing.image,
+		                     "Kyty.DepthTarget.Image[guest=0x{:016x} size=0x{:x} format={}]",
+		                     image.info.data.address, image.info.data.size,
+		                     static_cast<uint32_t>(image.info.pixel_format));
+		SetVulkanObjectNameF(m_context.GetGraphics().device, depth.image_view,
+		                     "Kyty.DepthTarget.View[guest=0x{:016x} layer={}+{}]",
+		                     image.info.data.address, depth.desc.view_info.base_layer,
+		                     depth.desc.view_info.layer_count);
 		EXIT_IF(depth.image_view == nullptr || image.backing.samples != depth.samples);
 		if (attachment_samples == 0) {
 			attachment_samples = depth.samples;
@@ -640,8 +689,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.has_stencil    = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
 		attachment.stencil_clear  = depth.stencil_clear_enable;
 	}
-	if (attachment_samples == 0 ||
-	    vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
+	if (color_count == 0 && !depth.image_id) {
+		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
+		state.width        = limits.maxFramebufferWidth;
+		state.height       = limits.maxFramebufferHeight;
+	} else if (attachment_samples == 0 ||
+	           vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
 		EXIT("render state has no valid attachments\n");
 	}
 	if (state.num_layers == std::numeric_limits<uint32_t>::max()) {
@@ -653,22 +706,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	return state;
 }
 
-static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer,
-                                     const DrawRenderState& state, const DrawCallInfo& draw) {
-	EXIT_IF(draw.name == nullptr);
-	const auto& ctx    = buffer.GetRegisters();
-	const auto& sh_ctx = buffer.GetShaders();
-
-	const bool with_depth = (state.depth_info.format != vk::Format::eUndefined &&
-	                         static_cast<bool>(state.depth_info.image_id));
-	if (state.color_count != 0 || !with_depth) {
-		return true;
-	}
-
-	const auto& sh_regs = ctx.GetShaderRegisters();
-	const auto& ps      = sh_ctx.GetPs();
-	return ShaderAddressValid(ps.ps_regs.data_addr) &&
-	       PixelShaderHasDepthOrCoverageSideEffects(sh_regs);
+static bool DrawHasActivePixelShader(const CommandBuffer& buffer) {
+	const auto& ctx              = buffer.GetRegisters();
+	const auto& sh_regs          = ctx.GetShaderRegisters();
+	const bool  has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
+	return ShaderAddressValid(buffer.GetShaders().GetPs().ps_regs.data_addr) &&
+	       (has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs));
 }
 
 enum class CbColorMode : uint8_t {
@@ -680,13 +723,14 @@ enum class CbColorMode : uint8_t {
 	DccDecompress      = 6,
 };
 
-static bool ConsumeMetadataColorOperation(const RenderCommandBuffer& buffer) {
+static bool ConsumeMetadataColorOperation(const CommandBuffer& buffer) {
 	const auto& ctx  = buffer.GetRegisters();
 	const auto  mode = ctx.GetColorControl().mode;
-	// These AGC CB modes run color-buffer metadata/decompression operations. The shader is a
-	// dummy vehicle for the CB, and its exported color must not be applied as a normal draw.
-	// Kyty currently stores host images as expanded Vulkan images and does not track CMASK/DCC
-	// metadata state, so the matching host operation is a no-op.
+	// These special modes run color-buffer metadata or decompression operations. The shader is a
+	// vehicle for that operation, and its exported color must not be applied as a normal draw.
+	// Kyty stores expanded Vulkan images rather than compressed guest surfaces, so no equivalent
+	// hardware pass is emitted. Tracked DCC clear state is materialized on attachment bind;
+	// future CMask/FMask support can consume its state through the same TextureCache path.
 	return mode == static_cast<uint8_t>(CbColorMode::EliminateFastClear) ||
 	       mode == static_cast<uint8_t>(CbColorMode::FmaskDecompress) ||
 	       mode == static_cast<uint8_t>(CbColorMode::DccDecompress);
@@ -707,11 +751,10 @@ struct DrawIndexBufferSource {
 };
 
 struct PreparedIndexBuffer {
-	std::shared_ptr<void> owner;
-	vk::Buffer            buffer = nullptr;
-	uint64_t              size   = 0;
-	vk::DeviceSize        offset = 0;
-	vk::IndexType         type   = vk::IndexType::eUint16;
+	vk::Buffer     buffer = nullptr;
+	uint64_t       size   = 0;
+	vk::DeviceSize offset = 0;
+	vk::IndexType  type   = vk::IndexType::eUint16;
 };
 
 static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer) {
@@ -720,10 +763,10 @@ static uint64_t VertexBufferDescriptorSize(const ShaderVertexInputBuffer& buffer
 }
 
 struct VertexBufferRange {
-	uint64_t      base_address  = 0;
-	uint64_t      requested_end = 0;
-	uint64_t      acquired_end  = 0;
-	BufferBinding binding;
+	uint64_t                     base_address  = 0;
+	uint64_t                     requested_end = 0;
+	uint64_t                     acquired_end  = 0;
+	std::pair<Buffer*, uint64_t> binding;
 
 	[[nodiscard]] uint64_t RequestedSize() const { return requested_end - base_address; }
 };
@@ -731,14 +774,12 @@ struct VertexBufferRange {
 struct PreparedVertexBuffers {
 	static constexpr uint32_t MaxBuffers = ShaderVertexInputInfo::RES_MAX;
 
-	std::array<vk::Buffer, MaxBuffers>            buffers {};
-	std::array<vk::DeviceSize, MaxBuffers>        offsets {};
-	std::array<std::shared_ptr<void>, MaxBuffers> owners {};
-	uint32_t                                      count       = 0;
-	uint32_t                                      owner_count = 0;
+	std::array<vk::Buffer, MaxBuffers>     buffers {};
+	std::array<vk::DeviceSize, MaxBuffers> offsets {};
+	uint32_t                               count = 0;
 };
 
-static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         buffer,
+static PreparedVertexBuffers AcquireVertexBuffers(CommandBuffer&               buffer,
                                                   const ShaderVertexInputInfo& vs_input_info) {
 	EXIT_IF(vs_input_info.buffers_num < 0 ||
 	        vs_input_info.buffers_num > ShaderVertexInputInfo::RES_MAX);
@@ -785,21 +826,22 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         b
 		const auto size =
 		    Libs::LibKernel::Memory::ClampRangeSize(range.base_address, range.RequestedSize());
 		range.acquired_end = range.base_address + size;
-		range.binding      = cache.ObtainBuffer(buffer, range.base_address, size);
+		range.binding      = cache.ObtainBuffer(range.base_address, size, false);
+		SetVulkanObjectNameF(
+		    buffer.GetContext().GetGraphics().device, range.binding.first->Handle(),
+		    "Kyty.VertexBufferRange[guest=0x{:016x} size=0x{:x}]", range.base_address, size);
 	}
 
 	// Rebuild slot bindings, offsetting non-empty slots into their acquired merged range.
 	PreparedVertexBuffers prepared;
-	prepared.count = static_cast<uint32_t>(vs_input_info.buffers_num);
-	std::shared_ptr<Buffer> null_owner;
-	vk::Buffer              null_buffer = nullptr;
+	prepared.count         = static_cast<uint32_t>(vs_input_info.buffers_num);
+	vk::Buffer null_buffer = nullptr;
 	for (int i = 0; i < vs_input_info.buffers_num; i++) {
 		const auto& vertex = vs_input_info.buffers[i];
 		const auto  size   = VertexBufferDescriptorSize(vertex);
 		if (size == 0) {
-			if (null_owner == nullptr) {
-				null_owner  = cache.ObtainNullBuffer();
-				null_buffer = null_owner->Handle();
+			if (null_buffer == nullptr) {
+				null_buffer = cache.GetBuffer(NULL_BUFFER_ID).Handle();
 			}
 			prepared.buffers[i] = null_buffer;
 			prepared.offsets[i] = 0;
@@ -816,24 +858,19 @@ static PreparedVertexBuffers AcquireVertexBuffers(RenderCommandBuffer&         b
 			     vertex.addr);
 		}
 
-		prepared.buffers[i] = range->binding.buffer;
-		prepared.offsets[i] = range->binding.offset + vertex.addr - range->base_address;
+		prepared.buffers[i] = range->binding.first->Handle();
+		prepared.offsets[i] = range->binding.second + vertex.addr - range->base_address;
+		SetVulkanObjectNameF(
+		    buffer.GetContext().GetGraphics().device, prepared.buffers[i],
+		    "Kyty.VertexBuffer[slot={} guest=0x{:016x} size=0x{:x} stride={} records={}]", i,
+		    vertex.addr, size, vertex.stride, vertex.num_records);
 	}
 
-	if (null_owner != nullptr) {
-		prepared.owners[prepared.owner_count++] = std::move(null_owner);
-	}
-	for (uint32_t i = 0; i < merged_count; i++) {
-		if (merged_ranges[i].binding.owner != nullptr) {
-			EXIT_IF(prepared.owner_count >= PreparedVertexBuffers::MaxBuffers);
-			prepared.owners[prepared.owner_count++] = std::move(merged_ranges[i].binding.owner);
-		}
-	}
 	return prepared;
 }
 
-static void SetDrawDebugPhase(RenderCommandBuffer& buffer, uint64_t submit_id,
-                              const DrawCallInfo& draw, uint32_t phase) {
+static void SetDrawDebugPhase(CommandBuffer& buffer, uint64_t submit_id, const DrawCallInfo& draw,
+                              uint32_t phase) {
 	EXIT_IF(draw.name == nullptr);
 
 	buffer.SetDebugInfo(static_cast<uint32_t>(draw.debug_op), submit_id, phase, draw.index_count,
@@ -881,7 +918,42 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
 	return true;
 }
 
-bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
+static bool ResolvePrimitiveRestart(const CommandBuffer& buffer, vk::PrimitiveTopology topology,
+                                    uint32_t index_type_and_size) {
+	const auto control = buffer.GetUserConfig().GetPrimitiveResetControl();
+	EXIT_NOT_IMPLEMENTED((control & ~0x3u) != 0);
+	if ((control & 0x1u) == 0) {
+		return false;
+	}
+	switch (buffer.GetUserConfig().GetPrimType()) {
+		case Prospero::PrimitiveType::kLineStrip:
+		case Prospero::PrimitiveType::kTriFan:
+		case Prospero::PrimitiveType::kTriStrip: break;
+		default: return false;
+	}
+	if (topology != vk::PrimitiveTopology::eLineStrip &&
+	    topology != vk::PrimitiveTopology::eTriangleStrip &&
+	    topology != vk::PrimitiveTopology::eTriangleFan) {
+		return false;
+	}
+
+	uint32_t index_mask = 0;
+	switch (static_cast<Prospero::IndexType>(index_type_and_size)) {
+		case Prospero::IndexType::kIndex8: index_mask = 0xffu; break;
+		case Prospero::IndexType::kIndex16: index_mask = 0xffffu; break;
+		case Prospero::IndexType::kIndex32: index_mask = 0xffffffffu; break;
+		default: EXIT("unknown index_type_and_size: %u\n", index_type_and_size);
+	}
+
+	const auto reset_index = buffer.GetRegisters().GetPrimitiveResetIndex();
+	if ((control & 0x2u) != 0 && (reset_index & ~index_mask) != 0) {
+		return false;
+	}
+	EXIT_NOT_IMPLEMENTED((reset_index & index_mask) != index_mask);
+	return true;
+}
+
+bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, CommandBuffer& buffer,
                                             const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
                                             bool log_setup_phases, DrawRenderState& state) {
@@ -909,19 +981,19 @@ bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuf
 	}
 	ResolveRenderDepthTarget(submit_id, buffer, state.depth_info);
 
+	state.ps_active       = DrawHasActivePixelShader(buffer);
 	const bool with_depth = (state.depth_info.format != vk::Format::eUndefined &&
 	                         static_cast<bool>(state.depth_info.image_id));
-	if (state.color_count == 0 && !with_depth) {
+	if (state.color_count == 0 && !with_depth && !state.ps_active) {
 		LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer,
 		                   draw.index_count, draw.flags);
 		return false;
 	}
-	state.ps_active = DrawHasActivePixelShader(buffer, state, draw);
 
 	return true;
 }
 
-static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
+static void RefreshShaders(CommandBuffer& buffer, const DrawCallInfo& draw, bool log_phases,
                            DrawRenderState& state) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ctx    = buffer.GetRegisters();
@@ -931,37 +1003,33 @@ static void RefreshShaders(RenderCommandBuffer& buffer, const DrawCallInfo& draw
 	const auto& pixel_shader_info  = sh_ctx.GetPs();
 	const auto& shader_regs        = ctx.GetShaderRegisters();
 
-	state.vs_shader     = {};
-	state.ps_shader     = {};
-	state.ps_input_info = {};
+	state.vertex_program = {};
+	state.pixel_program  = {};
+	state.ps_input_info  = {};
 	std::array<Prospero::ColorComponentMapping, RENDER_COLOR_ATTACHMENTS_MAX>
 	    target_export_mapping {};
 	for (uint32_t i = 0; i < state.color_count; i++) {
 		target_export_mapping[state.color_info[i].target_slot] = state.color_info[i].export_mapping;
 	}
-	const auto lane_mask_mode = SelectGraphicsLaneMaskMode(64u);
-
 	if (log_phases) {
-		LogDrawPhase(draw.name, "ShaderCompileInfoVS");
-	}
-	if (!ShaderCompileInfoVS(vertex_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
-	                         state.vs_shader)) {
-		EXIT("ShaderCompileInfoVS failed for draw %s\n", draw.name);
+		LogDrawPhase(draw.name, "GetVertexProgram");
 	}
+	auto& pipeline_cache = buffer.GetContext().GetPipelineCache();
+	state.vertex_program =
+	    pipeline_cache.GetVertexProgram(vertex_shader_info, shader_regs, state.vs_input_info);
 
 	if (!state.ps_active) {
 		return;
 	}
 	if (log_phases) {
-		LogDrawPhase(draw.name, "ShaderCompileInfoPS");
-	}
-	if (!ShaderCompileInfoPS(pixel_shader_info, shader_regs, lane_mask_mode, state.vs_input_info,
-	                         target_export_mapping, state.ps_input_info, state.ps_shader)) {
-		EXIT("ShaderCompileInfoPS failed for draw %s\n", draw.name);
+		LogDrawPhase(draw.name, "GetPixelProgram");
 	}
+	state.pixel_program =
+	    pipeline_cache.GetPixelProgram(pixel_shader_info, shader_regs, state.vs_input_info,
+	                                   target_export_mapping, state.ps_input_info);
 }
 
-static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, RenderCommandBuffer& buffer,
+static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, CommandBuffer& buffer,
                                                   const DrawCallInfo&          draw,
                                                   const ShaderVertexInputInfo& vs_input_info) {
 	EXIT_IF(draw.name == nullptr);
@@ -971,7 +1039,7 @@ static PreparedVertexBuffers PrepareVertexBuffers(uint64_t submit_id, RenderComm
 	return AcquireVertexBuffers(buffer, vs_input_info);
 }
 
-static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffer,
+static PreparedIndexBuffer PrepareIndexBuffer(CommandBuffer&               buffer,
                                               const DrawIndexBufferSource& source) {
 	PreparedIndexBuffer prepared;
 	if (!source.enabled) {
@@ -981,28 +1049,29 @@ static PreparedIndexBuffer PrepareIndexBuffer(RenderCommandBuffer&         buffe
 	prepared.size = source.size;
 	prepared.type = source.type;
 	if (source.host_data != nullptr) {
-		auto binding =
-		    buffer.GetContext().GetBufferCache().UploadTransient(source.host_data, source.size, 16);
-		prepared.owner  = std::move(binding.owner);
-		prepared.buffer = binding.buffer;
-		prepared.offset = binding.offset;
+		auto& stream = buffer.GetContext().GetBufferCache().GetUtilityBuffer(MemoryUsage::Stream);
+		prepared.offset = stream.Copy(source.host_data, source.size, 16);
+		prepared.buffer = stream.Handle();
 	} else {
-		auto binding =
-		    buffer.GetContext().GetBufferCache().ObtainBuffer(buffer, source.address, source.size);
-		prepared.owner  = std::move(binding.owner);
-		prepared.buffer = binding.buffer;
-		prepared.offset = binding.offset;
+		auto [buffer_ptr, offset] =
+		    buffer.GetContext().GetBufferCache().ObtainBuffer(source.address, source.size, false);
+		prepared.buffer = buffer_ptr->Handle();
+		prepared.offset = offset;
+	}
+	if (source.host_data != nullptr) {
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=transient size=0x{:x} type={}]", source.size,
+		                     static_cast<uint32_t>(source.type));
+	} else {
+		SetVulkanObjectNameF(buffer.GetContext().GetGraphics().device, prepared.buffer,
+		                     "Kyty.IndexBuffer[guest=0x{:016x} size=0x{:x} type={}]",
+		                     source.address, source.size, static_cast<uint32_t>(source.type));
 	}
 	return prepared;
 }
 
-static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                                PreparedVertexBuffers& prepared) {
-	for (uint32_t i = 0; i < prepared.owner_count; i++) {
-		if (prepared.owners[i] != nullptr) {
-			buffer.RetainResourceUntilFence(std::move(prepared.owners[i]));
-		}
-	}
+static void CommitVertexBuffers(vk::CommandBuffer            vk_buffer,
+                                const PreparedVertexBuffers& prepared) {
 	for (uint32_t i = 0; i < prepared.count; i++) {
 		EXIT_IF(prepared.buffers[i] == nullptr);
 	}
@@ -1012,19 +1081,15 @@ static void CommitVertexBuffers(RenderCommandBuffer& buffer, vk::CommandBuffer v
 	}
 }
 
-static void CommitIndexBuffer(RenderCommandBuffer& buffer, vk::CommandBuffer vk_buffer,
-                              PreparedIndexBuffer& prepared) {
+static void CommitIndexBuffer(vk::CommandBuffer vk_buffer, const PreparedIndexBuffer& prepared) {
 	if (prepared.size == 0) {
 		return;
 	}
-	if (prepared.owner != nullptr) {
-		buffer.RetainResourceUntilFence(std::move(prepared.owner));
-	}
 	EXIT_IF(prepared.buffer == nullptr);
 	vk_buffer.bindIndexBuffer(prepared.buffer, prepared.offset, prepared.type);
 }
 
-static void LogDrawStateIfNeeded(const RenderCommandBuffer& buffer, const DrawCallInfo& draw,
+static void LogDrawStateIfNeeded(const CommandBuffer& buffer, const DrawCallInfo& draw,
                                  const DrawRenderState& state, bool always_log,
                                  bool force_legacy_rect_log, uint32_t index_type_and_size,
                                  const void* index_addr) {
@@ -1092,18 +1157,18 @@ static void EmitDrawPrimitives(const HW::UserConfig& ucfg, vk::CommandBuffer vk_
 	}
 }
 
-void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, CommandBuffer& buffer,
                                          const DrawCallInfo& draw, DrawRenderState& state,
                                          vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
                                          const DrawIndexBufferSource& index_source,
-                                         bool log_pipeline_phase, bool set_bind_debug,
-                                         bool set_auto_debug) {
+                                         bool primitive_restart_enable, bool log_pipeline_phase,
+                                         bool set_bind_debug, bool set_auto_debug) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ucfg = buffer.GetUserConfig();
 
 	LogDrawPhase(draw.name, "PrepareBindings");
-	auto bindings        = PrepareGraphicsBindings(buffer, state.vs_input_info.stage,
-	                                               state.ps_input_info.stage, state.ps_active);
+	auto bindings = PrepareGraphicsBindings(state.vs_input_info.stage, state.ps_input_info.stage,
+	                                        state.ps_active);
 	auto vertex_bindings = PrepareVertexBuffers(submit_id, buffer, draw, state.vs_input_info);
 	auto index_binding   = PrepareIndexBuffer(buffer, index_source);
 	state.rendering =
@@ -1113,8 +1178,9 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 		LogDrawPhase(draw.name, "CreatePipeline");
 	}
 	auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline(
-	    state.color_info, state.color_count, state.depth_info, state.vs_input_info, buffer,
-	    &state.ps_input_info, topology, state.ps_active, state.vs_shader, state.ps_shader);
+	    std::span {state.color_info, state.color_count}, state.depth_info, state.vs_input_info, buffer,
+	    state.ps_active ? &state.ps_input_info : nullptr, topology, primitive_restart_enable,
+	    state.vertex_program, state.pixel_program);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest
@@ -1126,21 +1192,23 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	if (set_auto_debug) {
 		SetDrawDebugPhase(buffer, submit_id, draw, 0x200u);
 	}
-	CommitVertexBuffers(buffer, vk_buffer, vertex_bindings);
-	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout,
-	               bindings.vertex);
+	CommitVertexBuffers(vk_buffer, vertex_bindings);
 	if (bindings.pixel.has_value()) {
 		if (set_auto_debug) {
 			SetDrawDebugPhase(buffer, submit_id, draw, 0x300u);
 		}
-		CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline.pipeline_layout,
-		               *bindings.pixel);
 	}
-	CommitIndexBuffer(buffer, vk_buffer, index_binding);
+	std::array<PreparedBindings*, 2> descriptor_stages {&bindings.vertex, nullptr};
+	const size_t                     descriptor_stage_count = bindings.pixel.has_value() ? 2u : 1u;
+	if (bindings.pixel) {
+		descriptor_stages[1] = &*bindings.pixel;
+	}
+	CommitBindings(buffer, vk::PipelineBindPoint::eGraphics, pipeline,
+	               std::span {descriptor_stages.data(), descriptor_stage_count});
+	CommitIndexBuffer(vk_buffer, index_binding);
 
-	const auto dynamic_params =
-	    BuildGraphicsDynamicParams(buffer, state.color_info, state.color_count, state.depth_info);
-	SetDynamicParams(buffer, vk_buffer, dynamic_params);
+	SetGraphicsDynamicParams(buffer, vk_buffer, state.color_info, state.color_count,
+	                         state.depth_info);
 
 	LogDrawPhase(draw.name, "BeginRendering");
 	if (set_auto_debug) {
@@ -1173,7 +1241,7 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	}
 }
 
-void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
+void RenderExecutor::DrawIndex(uint64_t submit_id, CommandBuffer& buffer,
                                uint32_t index_type_and_size, uint32_t index_count,
                                const void* index_addr, uint32_t flags, uint32_t type,
                                uint32_t instance_count, uint32_t render_target_slice_offset,
@@ -1181,6 +1249,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ucfg   = buffer.GetUserConfig();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -1238,6 +1307,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	vk::IndexType index_type           = vk::IndexType::eUint16;
 	uint64_t      index_size           = 0;
 	bool          expand_index8_to_u16 = false;
+	const bool primitive_restart = ResolvePrimitiveRestart(buffer, topology, index_type_and_size);
 
 	switch (static_cast<Prospero::IndexType>(index_type_and_size)) {
 		case Prospero::IndexType::kIndex16:
@@ -1268,7 +1338,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 		const auto* src = static_cast<const uint8_t*>(index_addr);
 		expanded_indices.resize(index_count);
 		for (uint32_t i = 0; i < index_count; i++) {
-			expanded_indices[i] = src[i];
+			expanded_indices[i] = primitive_restart && src[i] == 0xffu ? 0xffffu : src[i];
 		}
 	}
 
@@ -1298,19 +1368,20 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	emit.indexed       = true;
 	emit.vertex_offset = vertex_offset;
 
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, true, true,
-	                    false);
+	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
+	                    primitive_restart, true, true, false);
 	ResetBindings();
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, uint32_t index_count,
+void RenderExecutor::DrawAuto(uint64_t submit_id, CommandBuffer& buffer, uint32_t index_count,
                               uint32_t flags, uint32_t render_target_slice_offset,
                               uint32_t instance_count, uint32_t first_vertex,
                               uint32_t first_instance) {
 	KYTY_PROFILER_FUNCTION();
 
 	EXIT_IF(buffer.IsInvalid());
+	m_context.GetCommandScheduler().PopPendingOperations();
 	auto& ucfg   = buffer.GetUserConfig();
 	auto& sh_ctx = buffer.GetShaders();
 
@@ -1398,11 +1469,11 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 
 	DrawIndexBufferSource index_source {};
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false,
-	                    true);
+	                    false, true);
 	ResetBindings();
 }
 
-bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,
+bool RenderExecutor::ResolveColorTargets(uint64_t submit_id, CommandBuffer& buffer,
                                          uint32_t render_target_slice_offset) {
 	const auto& hw = buffer.GetRegisters();
 	if (hw.GetColorControl().mode != 3) {

src/graphics/host_gpu/renderer/renderDraw.h

@@ -1,43 +1,12 @@
 #ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_RENDERDRAW_H_
 
-#include "graphics/host_gpu/renderer/renderTarget.h"
-
 #include <cstdint>
-#include <type_traits>
 
 namespace Libs::Graphics {
 
-struct RenderColorInfo;
 struct ShaderVertexInputInfo;
 
-#pragma pack(push, 1)
-
-struct PipelineDynamicParameters {
-	bool stencil_test_enable = false;
-	bool depth_bias_enable   = false;
-
-	float viewport_scale[3]  = {};
-	float viewport_offset[3] = {};
-	int   scissor_ltrb[4]    = {0};
-
-	float    line_width                                       = 1.0f;
-	float    depth_bias_constant_factor                       = 0.0f;
-	float    depth_bias_clamp                                 = 0.0f;
-	float    depth_bias_slope_factor                          = 0.0f;
-	uint32_t color_write_count                                = 1;
-	bool     color_write_enable[RENDER_COLOR_ATTACHMENTS_MAX] = {true, true};
-
-	PipelineStencilDynamicState stencil_front;
-	PipelineStencilDynamicState stencil_back;
-};
-
-#pragma pack(pop)
-
-static_assert(std::is_trivially_copyable_v<PipelineDynamicParameters>);
-static_assert(std::is_standard_layout_v<PipelineDynamicParameters>);
-static_assert(alignof(PipelineDynamicParameters) == 1);
-
 [[nodiscard]] int32_t ResolveVertexOffset(uint32_t                     index_offset,
                                           const ShaderVertexInputInfo& vs_input_info);
 

src/graphics/host_gpu/renderer/sync.cpp

@@ -20,9 +20,7 @@
 
 namespace Libs::Graphics::Sync {
 
-constexpr int      GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT = 0x00;
-constexpr int      GRAPHICS_EVENT_EOP                       = 0x40;
-constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY       = 100000000;
+constexpr uint64_t GRAPHICS_REFERENCE_CLOCK_FREQUENCY = 100000000;
 
 bool ScaleReferenceClock(uint64_t host_ticks, uint64_t host_frequency, uint64_t& value) {
 	if (host_frequency == 0) {
@@ -66,8 +64,9 @@ struct EndOfPipeSignal {
 	std::array<uint32_t, 4> debug_args      = {};
 	uint64_t                debug_data      = 0;
 	std::optional<uint64_t> destination;
-	EndOfPipeCompletion     completion      = EndOfPipeCompletion::None;
-	uint64_t                completion_data = 0;
+	EndOfPipeCompletion     completion         = EndOfPipeCompletion::None;
+	uint64_t                completion_data    = 0;
+	int                     interrupt_event_id = 0;
 };
 
 enum class EndOfPipeWriteSize : uint32_t { Dword = 4, Qword = 8 };
@@ -96,8 +95,10 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) {
 		case EndOfPipeCompletion::None: return;
 		case EndOfPipeCompletion::Interrupt: {
 			const auto context_id = static_cast<uint32_t>(signal.completion_data);
-			scheduler.DeferPriorityOperation(
-			    [&renderer, context_id] { renderer.TriggerEopEvent(context_id); });
+			const auto event_id   = signal.interrupt_event_id;
+			scheduler.DeferPriorityOperation([&renderer, event_id, context_id] {
+				renderer.TriggerInterrupt(event_id, context_id);
+			});
 			return;
 		}
 		case EndOfPipeCompletion::Flip: {
@@ -108,9 +109,10 @@ static void RecordEndOfPipeSignal(const EndOfPipeSignal& signal) {
 		}
 		case EndOfPipeCompletion::FlipAndInterrupt: {
 			const auto request_id = signal.completion_data;
-			scheduler.DeferPriorityOperation([&renderer, request_id] {
+			const auto event_id   = signal.interrupt_event_id;
+			scheduler.DeferPriorityOperation([&renderer, event_id, request_id] {
 				renderer.GetVideoOut().CompleteFlip(request_id);
-				renderer.TriggerEopEvent(0);
+				renderer.TriggerInterrupt(event_id, 0);
 			});
 			return;
 		}
@@ -135,7 +137,8 @@ static bool TriggersInterrupt(EndOfPipeWriteAction action) {
 
 static void RecordEndOfPipeWrite(uint64_t submit_id, CommandBuffer& buffer, uint64_t destination,
                                  uint64_t value, EndOfPipeWriteSize size,
-                                 EndOfPipeWriteAction action, uint32_t context_id = 0) {
+                                 EndOfPipeWriteAction action, int interrupt_event_id = 0,
+                                 uint32_t context_id = 0) {
 	const auto width      = static_cast<uint32_t>(size);
 	const auto value_low  = static_cast<uint32_t>(value);
 	const auto value_high = static_cast<uint32_t>(value >> 32u);
@@ -150,18 +153,12 @@ static void RecordEndOfPipeWrite(uint64_t submit_id, CommandBuffer& buffer, uint
 	    .debug_data      = destination,
 	    .destination     = destination,
 	    .completion      = interrupt ? EndOfPipeCompletion::Interrupt : EndOfPipeCompletion::None,
-	    .completion_data = context_id != 0 ? context_id : value,
+	    .completion_data = interrupt ? context_id : value,
+	    .interrupt_event_id = interrupt_event_id,
 	};
 	RecordEndOfPipeSignal(signal);
 }
 
-void TriggerAgcUserInterrupt() {
-	auto tsc    = LibKernel::KernelReadTsc();
-	auto result = LibKernel::EventQueue::KernelTriggerUserEventForAll(AGC_USER_INTERRUPT_EVENT,
-	                                                                  reinterpret_cast<void*>(tsc));
-	EXIT_NOT_IMPLEMENTED(result != OK && result != LibKernel::KERNEL_ERROR_ENOENT);
-}
-
 void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                         uint32_t value) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
@@ -220,31 +217,35 @@ void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
 }
 
 void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint64_t* dst_gpu_addr, uint64_t value,
+                                              uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                               uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
 	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::InterruptWriteBack,
-	                     context_id);
+	                     event_id, context_id);
 }
 
 void WriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint32_t* dst_gpu_addr, uint32_t value,
+                                              uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                               uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
 	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::InterruptWriteBack,
-	                     context_id);
+	                     event_id, context_id);
 }
 
 void WriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint64_t* dst_gpu_addr, uint64_t value, uint32_t context_id) {
+                                     uint64_t* dst_gpu_addr, uint64_t value, int event_id,
+                                     uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
-	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Interrupt, context_id);
+	                     EndOfPipeWriteSize::Qword, EndOfPipeWriteAction::Interrupt, event_id,
+	                     context_id);
 }
 
 void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint32_t* dst_gpu_addr, uint32_t value, uint32_t context_id) {
+                                     uint32_t* dst_gpu_addr, uint32_t value, int event_id,
+                                     uint32_t context_id) {
 	RecordEndOfPipeWrite(submit_id, buffer, reinterpret_cast<uint64_t>(dst_gpu_addr), value,
-	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, context_id);
+	                     EndOfPipeWriteSize::Dword, EndOfPipeWriteAction::Interrupt, event_id,
+	                     context_id);
 }
 
 uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int flip_mode,
@@ -270,18 +271,20 @@ uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index, int f
 void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuffer& buffer,
                                                   uint32_t* dst_gpu_addr, uint32_t value,
                                                   int handle, int index, int flip_mode,
-                                                  int64_t flip_arg, uint64_t request_id) {
+                                                  int64_t flip_arg, uint64_t request_id,
+                                                  int event_id) {
 	const auto destination = reinterpret_cast<uint64_t>(dst_gpu_addr);
 	RecordEndOfPipeSignal({
-	    .buffer          = &buffer,
-	    .submit_id       = submit_id,
-	    .debug_operation = CommandBufferDebugOp::EopWriteBackFlip,
-	    .debug_args      = {static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
-	                        static_cast<uint32_t>(flip_mode), value},
-	    .debug_data      = static_cast<uint64_t>(flip_arg),
-	    .destination     = destination,
-	    .completion      = EndOfPipeCompletion::FlipAndInterrupt,
-	    .completion_data = request_id,
+	    .buffer             = &buffer,
+	    .submit_id          = submit_id,
+	    .debug_operation    = CommandBufferDebugOp::EopWriteBackFlip,
+	    .debug_args         = {static_cast<uint32_t>(handle), static_cast<uint32_t>(index),
+	                           static_cast<uint32_t>(flip_mode), value},
+	    .debug_data         = static_cast<uint64_t>(flip_arg),
+	    .destination        = destination,
+	    .completion         = EndOfPipeCompletion::FlipAndInterrupt,
+	    .completion_data    = request_id,
+	    .interrupt_event_id = event_id,
 	});
 }
 
@@ -316,36 +319,24 @@ void WriteAtEndOfPipeOnlyFlip(uint64_t submit_id, CommandBuffer& buffer, int han
 	});
 }
 
-void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id) {
+void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, int event_id, uint32_t context_id) {
 	ValidateEndOfPipeSignal({.buffer = &buffer});
 	auto& renderer  = buffer.GetContext();
 	auto& scheduler = renderer.GetCommandScheduler();
 	EXIT_IF(!scheduler.Active() || &buffer != &scheduler.Current());
 	scheduler.DeferPriorityOperation(
-	    [&renderer, context_id] { renderer.TriggerEopEvent(context_id); });
+	    [&renderer, event_id, context_id] { renderer.TriggerInterrupt(event_id, context_id); });
 }
 
-static void EopEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) {
+static void InterruptEventResetFunc(LibKernel::EventQueue::KernelEqueueEvent* event) {
 	EXIT_IF(event == nullptr);
 	event->triggered    = false;
 	event->event.fflags = 0;
 	event->event.data   = 0;
 }
 
-static void EopEventDeleteFunc(LibKernel::EventQueue::KernelEqueue       eq,
-                               LibKernel::EventQueue::KernelEqueueEvent* event) {
-	EXIT_IF(event == nullptr);
-	EXIT_NOT_IMPLEMENTED(event->event.filter != LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS);
-	if (event->event.ident == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT ||
-	    event->event.ident == GRAPHICS_EVENT_EOP) {
-		auto* renderer = static_cast<RenderContext*>(event->filter.data);
-		EXIT_IF(renderer == nullptr);
-		renderer->DeleteEopEq(eq, static_cast<int>(event->event.ident));
-	}
-}
-
-static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
-                                void*                                     trigger_data) {
+static void InterruptEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
+                                      void*                                     trigger_data) {
 	EXIT_IF(event == nullptr);
 
 	auto triggered_event = event->event;
@@ -362,35 +353,35 @@ static void EopEventTriggerFunc(LibKernel::EventQueue::KernelEqueueEvent* event,
 int AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                void* udata) {
 	LibKernel::EventQueue::KernelEqueueEvent event;
-	event.triggered                = false;
-	event.event.ident              = static_cast<uintptr_t>(id);
-	event.event.filter             = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS;
-	event.event.udata              = udata;
-	event.event.fflags             = 0;
-	event.event.data               = id;
-	event.filter.delete_event_func = EopEventDeleteFunc;
-	event.filter.reset_func        = EopEventResetFunc;
-	event.filter.trigger_func      = EopEventTriggerFunc;
-	event.filter.data              = &renderer;
+	event.triggered           = false;
+	event.event.ident         = static_cast<uintptr_t>(id);
+	event.event.filter        = LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS;
+	event.event.udata         = udata;
+	event.event.fflags        = 0;
+	event.event.data          = id;
+	event.filter.reset_func   = InterruptEventResetFunc;
+	event.filter.trigger_func = InterruptEventTriggerFunc;
 
 	int result = LibKernel::EventQueue::KernelAddEvent(eq, event);
 
-	if (result == 0 &&
-	    (id == GRAPHICS_EVENT_QUEUED_GRAPHICS_INTERRUPT || id == GRAPHICS_EVENT_EOP)) {
-		renderer.AddEopEq(eq, id);
+	if (result == 0) {
+		renderer.AddInterruptEq(eq, id);
 	}
 
 	return result;
 }
 
-int DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id) {
+int DeleteEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id) {
 	int result = LibKernel::EventQueue::KernelDeleteEvent(
 	    eq, static_cast<uintptr_t>(id), LibKernel::EventQueue::KERNEL_EVFILT_GRAPHICS);
+	if (result == OK || result == LibKernel::KERNEL_ERROR_ENOENT) {
+		renderer.DeleteInterruptEq(eq, id);
+	}
 
 	return result;
 }
 
-void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
+void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size) {
 	const auto offset = uint64_t {dw_offset} * sizeof(uint32_t);
 	const auto size   = uint64_t {dw_size} * sizeof(uint32_t);
 	EXIT_IF(dst == nullptr || offset > gds.Size() || size > gds.Size() - offset ||

src/graphics/host_gpu/renderer/sync.h

@@ -17,8 +17,7 @@ namespace Sync {
                                            uint64_t& value);
 [[nodiscard]] uint64_t ReadReferenceClock();
 
-void TriggerAgcUserInterrupt();
-void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, uint32_t context_id);
+void TriggerEopEventAtEndOfPipe(CommandBuffer& buffer, int event_id, uint32_t context_id);
 
 void WriteAtEndOfPipe32(uint64_t submit_id, CommandBuffer& buffer, uint32_t* dst_gpu_addr,
                         uint32_t value);
@@ -35,16 +34,16 @@ void WriteAtEndOfPipeWithWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
 void WriteAtEndOfPipeWithWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
                                      uint64_t* dst_gpu_addr, uint64_t value);
 void WriteAtEndOfPipeWithInterrupt32(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint32_t* dst_gpu_addr, uint32_t value,
+                                     uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                      uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterrupt64(uint64_t submit_id, CommandBuffer& buffer,
-                                     uint64_t* dst_gpu_addr, uint64_t value,
+                                     uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                      uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterruptWriteBack32(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint32_t* dst_gpu_addr, uint32_t value,
+                                              uint32_t* dst_gpu_addr, uint32_t value, int event_id,
                                               uint32_t context_id = 0);
 void WriteAtEndOfPipeWithInterruptWriteBack64(uint64_t submit_id, CommandBuffer& buffer,
-                                              uint64_t* dst_gpu_addr, uint64_t value,
+                                              uint64_t* dst_gpu_addr, uint64_t value, int event_id,
                                               uint32_t context_id = 0);
 
 [[nodiscard]] uint64_t PrepareVideoOutFlip(CommandBuffer& buffer, int handle, int index,
@@ -57,12 +56,13 @@ void WriteAtEndOfPipeWithFlip32(uint64_t submit_id, CommandBuffer& buffer, uint3
 void WriteAtEndOfPipeWithInterruptWriteBackFlip32(uint64_t submit_id, CommandBuffer& buffer,
                                                   uint32_t* dst_gpu_addr, uint32_t value,
                                                   int handle, int index, int flip_mode,
-                                                  int64_t flip_arg, uint64_t request_id);
+                                                  int64_t flip_arg, uint64_t request_id,
+                                                  int event_id);
 
 int  AddEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id,
                 void* udata);
-int  DeleteEqEvent(LibKernel::EventQueue::KernelEqueue eq, int id);
-void ReadGds(Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
+int  DeleteEqEvent(RenderContext& renderer, LibKernel::EventQueue::KernelEqueue eq, int id);
+void ReadGds(const Buffer& gds, uint32_t* dst, uint32_t dw_offset, uint32_t dw_size);
 
 } // namespace Sync
 } // namespace Libs::Graphics

src/graphics/host_gpu/shaders/fault_buffer_process.comp

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#version 450
+#extension GL_ARB_gpu_shader_int64 : enable
+
+layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
+
+layout(std430, binding = 0) buffer input_buf {
+    uint fault_buffer[];
+};
+
+layout(std430, binding = 1) buffer output_buf {
+    uint64_t download_buffer[];
+};
+layout(std430, binding = 1) buffer output_buf32 {
+    uint download_buffer32[];
+};
+
+void main() {
+    const uint id = gl_GlobalInvocationID.x;
+    uint word = fault_buffer[id];
+    fault_buffer[id] = 0u;
+    const uint base_bit = id * 32u;
+    while (word != 0u) {
+        const uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u;
+        if (store_index >= MAX_PAGE_FAULTS) {
+            return;
+        }
+        const uint bit = findLSB(word);
+        word &= word - 1;
+        const uint page = base_bit + bit;
+        download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS;
+    }
+}

src/graphics/host_gpu/vma.cpp

@@ -72,7 +72,10 @@ bool GraphicContext::CreateAllocator() {
 	info.device           = device;
 	info.pVulkanFunctions = &functions;
 	info.vulkanApiVersion = VULKAN_TARGET_API_VERSION;
-	info.flags = memory_budget_ext_enabled ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0;
+	info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
+	if (memory_budget_ext_enabled) {
+		info.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
+	}
 
 	const auto result = static_cast<vk::Result>(vmaCreateAllocator(&info, &allocator));
 	if (result != vk::Result::eSuccess) {
@@ -199,20 +202,6 @@ void GraphicContext::CreateBuffer(uint64_t size, VulkanBuffer& buffer) {
 	VulkanTrackAllocation(buffer.memory);
 }
 
-void GraphicContext::DeleteBuffer(VulkanBuffer& buffer) {
-	KYTY_PROFILER_FUNCTION();
-	EXIT_IF(allocator == nullptr || buffer.buffer == nullptr ||
-	        buffer.memory.allocation == nullptr);
-
-	VulkanUntrackAllocation(buffer.memory);
-	vmaDestroyBuffer(allocator, buffer.buffer, buffer.memory.allocation);
-	buffer.buffer                 = nullptr;
-	buffer.memory.memory          = nullptr;
-	buffer.memory.allocation      = nullptr;
-	buffer.memory.allocation_info = {};
-	buffer.memory.offset          = 0;
-}
-
 bool GraphicContext::CreateImage(const vk::ImageCreateInfo& image_info, VulkanImage& image) {
 	KYTY_PROFILER_FUNCTION();
 	EXIT_IF(allocator == nullptr || image.image != nullptr || image.memory.allocation != nullptr);

src/graphics/host_gpu/vulkanCommon.h

@@ -10,10 +10,13 @@
 #define VMA_STATIC_VULKAN_FUNCTIONS  0
 #define VMA_DYNAMIC_VULKAN_FUNCTIONS 1
 
+#include "common/emulatorConfig.h"
 #include "graphics/guest_gpu/gpu_defs.h"
 
 #include <cstdint>
+#include <fmt/format.h>
 #include <string>
+#include <utility>
 #include <vector>
 #include <vulkan/vulkan.hpp>
 #include <vulkan/vulkan_hash.hpp>
@@ -22,11 +25,6 @@ namespace Libs::Graphics {
 
 using VulkanMemoryBarrier = vk::MemoryBarrier;
 
-template <typename Handle>
-[[nodiscard]] void* VulkanHandleToPointer(Handle handle) {
-	return reinterpret_cast<void*>(static_cast<typename Handle::CType>(handle));
-}
-
 std::string VulkanToString(vk::Result value);
 std::string VulkanToString(vk::Format value);
 std::string VulkanToString(vk::ImageLayout value);
@@ -34,6 +32,24 @@ std::string VulkanToString(vk::QueueFlags value);
 vk::Format  VulkanFormat(Prospero::BufferFormat guest_format);
 void        RequireVulkanSuccess(vk::Result result, const char* operation);
 
+template <typename Handle, typename... Args>
+void SetVulkanObjectNameF(vk::Device device, Handle handle, fmt::format_string<Args...> format,
+                          Args&&... args) {
+	if (!Config::GraphicsDebugDumpEnabled() || device == nullptr || handle == nullptr ||
+	    VULKAN_HPP_DEFAULT_DISPATCHER.vkSetDebugUtilsObjectNameEXT == nullptr) {
+		return;
+	}
+
+	const auto                      name = fmt::format(format, std::forward<Args>(args)...);
+	vk::DebugUtilsObjectNameInfoEXT info {};
+	info.sType        = vk::StructureType::eDebugUtilsObjectNameInfoEXT;
+	info.objectType   = Handle::objectType;
+	info.objectHandle = static_cast<uint64_t>(
+	    reinterpret_cast<uintptr_t>(static_cast<typename Handle::CType>(handle)));
+	info.pObjectName = name.c_str();
+	(void)device.setDebugUtilsObjectNameEXT(&info);
+}
+
 template <typename T, typename Enumerator>
 [[nodiscard]] std::vector<T> EnumerateVulkan(const char* operation, Enumerator&& enumerate) {
 	for (;;) {

src/graphics/host_gpu/vulkanInstance.h

@@ -1,93 +0,0 @@
-#ifndef EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_
-#define EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_
-
-#include "common/abi.h"
-#include "common/common.h"
-#include "common/threads.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-
-#include <map>
-#include <mutex>
-#include <tuple>
-#include <vk_mem_alloc.h>
-
-namespace Libs::Graphics {
-
-inline constexpr uint32_t VULKAN_TARGET_API_VERSION = VK_API_VERSION_1_3;
-
-struct VulkanInstance {
-	vk::Instance                       instance                          = nullptr;
-	vk::DebugUtilsMessengerEXT         debug_messenger                   = nullptr;
-	vk::PhysicalDevice                 physical_device                   = nullptr;
-	vk::PhysicalDeviceProperties       physical_device_properties        = {};
-	vk::PhysicalDeviceMemoryProperties physical_device_memory_properties = {};
-	vk::Device                         device                            = nullptr;
-	VmaAllocator                       allocator                         = nullptr;
-	bool                               memory_budget_ext_enabled         = false;
-	bool                               rt_extensions_enabled             = false;
-	bool                               subgroup_size_control_enabled     = false;
-	bool                               sample_rate_shading_enabled       = false;
-	uint32_t                           subgroup_size                     = 0;
-	uint32_t                           min_subgroup_size                 = 0;
-	uint32_t                           max_subgroup_size                 = 0;
-	vk::ShaderStageFlags               required_subgroup_size_stages     = {};
-	Common::Mutex                      queue_mutex;
-	uint32_t                           queue_family = static_cast<uint32_t>(-1);
-	vk::Queue                          queue        = nullptr;
-
-	[[nodiscard]] const vk::PhysicalDeviceProperties& GetPhysicalDeviceProperties() const {
-		return physical_device_properties;
-	}
-
-	[[nodiscard]] const vk::PhysicalDeviceMemoryProperties&
-	GetPhysicalDeviceMemoryProperties() const {
-		return physical_device_memory_properties;
-	}
-
-	[[nodiscard]] vk::FormatProperties GetFormatProperties(vk::Format format) const {
-		std::scoped_lock lock(m_format_properties_mutex);
-		auto [it, inserted] = m_format_properties.try_emplace(format);
-		if (inserted) {
-			physical_device.getFormatProperties(format, &it->second);
-		}
-		return it->second;
-	}
-
-	[[nodiscard]] vk::Result GetImageFormatProperties(vk::Format format, vk::ImageType type,
-	                                                  vk::ImageTiling            tiling,
-	                                                  vk::ImageUsageFlags        usage,
-	                                                  vk::ImageCreateFlags       flags,
-	                                                  vk::ImageFormatProperties* properties) const {
-		using Key = std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
-		                       vk::ImageCreateFlags>;
-		std::scoped_lock lock(m_image_format_properties_mutex);
-		auto [it, inserted] =
-		    m_image_format_properties.try_emplace(Key {format, type, tiling, usage, flags});
-		if (inserted) {
-			it->second.first = physical_device.getImageFormatProperties(format, type, tiling, usage,
-			                                                            flags, &it->second.second);
-		}
-		if (properties != nullptr) {
-			*properties = it->second.second;
-		}
-		return it->second.first;
-	}
-
-	[[nodiscard]] vk::DeviceSize StorageMinAlignment() const {
-		const auto alignment = physical_device_properties.limits.minStorageBufferOffsetAlignment;
-		return alignment != 0 ? alignment : 1;
-	}
-
-private:
-	mutable std::mutex                                 m_format_properties_mutex;
-	mutable std::map<vk::Format, vk::FormatProperties> m_format_properties;
-	mutable std::mutex                                 m_image_format_properties_mutex;
-	mutable std::map<std::tuple<vk::Format, vk::ImageType, vk::ImageTiling, vk::ImageUsageFlags,
-	                            vk::ImageCreateFlags>,
-	                 std::pair<vk::Result, vk::ImageFormatProperties>>
-	    m_image_format_properties;
-};
-
-} // namespace Libs::Graphics
-
-#endif // EMULATOR_SRC_GRAPHICS_HOST_GPU_VULKANINSTANCE_H_

src/graphics/presentation/renderDoc.cpp

@@ -1,15 +1,12 @@
 #include "graphics/presentation/renderDoc.h"
 
-#include "SDL_syswm.h"
-#include "SDL_version.h"
-#include "SDL_video.h"
 #include "common/logging/log.h"
 
 #include <array>
 #include <atomic>
-#include <cstdint>
-#include <cstring>
+#include <chrono>
 #include <filesystem>
+#include <renderdoc_app.h>
 #include <string>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -21,361 +18,175 @@
 #undef max
 #else
 #include <dlfcn.h>
-
-// RenderDoc uses Windows-style names in its cross-platform API.
-#define __cdecl
-using HMODULE = void*;
 #endif
 
 namespace Libs::Graphics {
 
-using RenderDocDevicePointer = void*;
-using RenderDocWindowHandle  = void*;
-
-enum RenderDocVersion {
-	eRENDERDOC_API_Version_1_4_2 = 10402,
-};
-
-enum RenderDocInputButton {
-	eRENDERDOC_Key_NonPrintable = 0x100,
-	eRENDERDOC_Key_Divide,
-	eRENDERDOC_Key_Multiply,
-	eRENDERDOC_Key_Subtract,
-	eRENDERDOC_Key_Plus,
-	eRENDERDOC_Key_F1,
-};
-
-using pRENDERDOC_SetCaptureKeys             = void(__cdecl*)(RenderDocInputButton* keys, int num);
-using pRENDERDOC_SetCaptureFilePathTemplate = void(__cdecl*)(const char* pathtemplate);
-using pRENDERDOC_GetCaptureFilePathTemplate = const char*(__cdecl*)();
-using pRENDERDOC_GetNumCaptures             = uint32_t(__cdecl*)();
-using pRENDERDOC_GetCapture = uint32_t(__cdecl*)(uint32_t idx, char* filename, uint32_t* pathlength,
-                                                 uint64_t* timestamp);
-using pRENDERDOC_UnloadCrashHandler = void(__cdecl*)();
-using pRENDERDOC_SetActiveWindow    = void(__cdecl*)(RenderDocDevicePointer device,
-                                                     RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_StartFrameCapture  = void(__cdecl*)(RenderDocDevicePointer device,
-                                                     RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_IsFrameCapturing   = uint32_t(__cdecl*)();
-using pRENDERDOC_EndFrameCapture    = uint32_t(__cdecl*)(RenderDocDevicePointer device,
-                                                         RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_GetAPI = int(__cdecl*)(RenderDocVersion version, void** out_api_pointers);
-
-struct RenderDocApi {
-	void*                                 GetAPIVersion;
-	void*                                 SetCaptureOptionU32;
-	void*                                 SetCaptureOptionF32;
-	void*                                 GetCaptureOptionU32;
-	void*                                 GetCaptureOptionF32;
-	void*                                 SetFocusToggleKeys;
-	pRENDERDOC_SetCaptureKeys             SetCaptureKeys;
-	void*                                 GetOverlayBits;
-	void*                                 MaskOverlayBits;
-	void*                                 RemoveHooks;
-	pRENDERDOC_UnloadCrashHandler         UnloadCrashHandler;
-	pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate;
-	pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate;
-	pRENDERDOC_GetNumCaptures             GetNumCaptures;
-	pRENDERDOC_GetCapture                 GetCapture;
-	void*                                 TriggerCapture;
-	void*                                 IsTargetControlConnected;
-	void*                                 LaunchReplayUI;
-	pRENDERDOC_SetActiveWindow            SetActiveWindow;
-	pRENDERDOC_StartFrameCapture          StartFrameCapture;
-	pRENDERDOC_IsFrameCapturing           IsFrameCapturing;
-	pRENDERDOC_EndFrameCapture            EndFrameCapture;
-	void*                                 TriggerMultiFrameCapture;
-	void*                                 SetCaptureFileComments;
-	void*                                 DiscardFrameCapture;
-};
-
 enum class RenderDocState : uint32_t {
 	Idle,
 	Requested,
+	Starting,
 	Capturing,
 };
 
-static RenderDocApi*               g_api             = nullptr;
-static HMODULE                     g_module          = nullptr;
-static RenderDocDevicePointer      g_device          = nullptr;
-static RenderDocWindowHandle       g_window          = nullptr;
+static RENDERDOC_API_1_6_0*        g_api             = nullptr;
 static std::atomic<RenderDocState> g_state           = RenderDocState::Idle;
-static std::atomic_bool            g_init_done       = false;
+static std::atomic_uint32_t        g_captured_flips  = 0;
 static std::atomic_bool            g_unavailable_log = false;
 
-static RenderDocDevicePointer GetRenderDocDevicePointer(vk::Instance instance) {
-	if (instance == nullptr) {
-		return nullptr;
-	}
-
-	return VulkanHandleToPointer(instance);
-}
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 static bool BindRenderDocApi(HMODULE module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(module, "RENDERDOC_GetAPI"));
-	if (get_api == nullptr) {
-		return false;
-	}
-
-	void* api = nullptr;
-	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
-		return false;
-	}
-
-	g_module = module;
-	g_api    = static_cast<RenderDocApi*>(api);
-
-	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
-	g_api->SetCaptureKeys(nullptr, 0);
-	g_api->UnloadCrashHandler();
-
-	char module_path[MAX_PATH] = {};
-	GetModuleFileNameA(module, module_path, sizeof(module_path));
-	LOGF("RenderDoc: bound API from %s\n", module_path);
-	return true;
-}
-
-static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
-	if (window == nullptr) {
-		return nullptr;
-	}
-
-	SDL_SysWMinfo info {};
-	SDL_VERSION(&info.version);
-
-	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE || info.subsystem != SDL_SYSWM_WINDOWS) {
-		return nullptr;
-	}
-
-	return info.info.win.window;
-}
-
 #else
-
-static bool BindRenderDocApi(HMODULE module) {
+static bool BindRenderDocApi(void* module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(::dlsym(module, "RENDERDOC_GetAPI"));
+#endif
 	if (get_api == nullptr) {
 		return false;
 	}
 
 	void* api = nullptr;
-	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
+	if (get_api(eRENDERDOC_API_Version_1_6_0, &api) != 1 || api == nullptr) {
 		return false;
 	}
 
-	g_module = module;
-	g_api    = static_cast<RenderDocApi*>(api);
-
-	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
+	g_api = static_cast<RENDERDOC_API_1_6_0*>(api);
 	g_api->SetCaptureKeys(nullptr, 0);
 	g_api->UnloadCrashHandler();
-
-	Dl_info info {};
-	if (::dladdr(reinterpret_cast<void*>(get_api), &info) != 0 && info.dli_fname != nullptr) {
-		LOGF("RenderDoc: bound API from %s\n", info.dli_fname);
-	} else {
-		LOGF("RenderDoc: bound API\n");
-	}
+	LOGF("RenderDoc: API 1.6.0 bound\n");
 	return true;
 }
 
-static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
-	if (window == nullptr) {
-		return nullptr;
-	}
-
-	SDL_SysWMinfo info {};
-	SDL_VERSION(&info.version);
-
-	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE) {
-		return nullptr;
-	}
-
-#if defined(SDL_VIDEO_DRIVER_X11)
-	if (info.subsystem == SDL_SYSWM_X11) {
-		// RenderDoc takes the raw xlib Window id in the pointer slot, not a Display*.
-		return reinterpret_cast<RenderDocWindowHandle>(
-		    static_cast<uintptr_t>(info.info.x11.window));
-	}
-#endif
-
-	// Wayland capture works without an active-window handle.
-	static std::atomic_bool logged = false;
-	if (!logged.exchange(true)) {
-		LOGF("RenderDoc: no native window handle for SDL subsystem %d (Wayland?); the in-app "
-		     "overlay is unavailable, but --rd captures still work\n",
-		     static_cast<int>(info.subsystem));
-	}
-	return nullptr;
-}
-
-#endif
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-
-static bool IsAvailable() {
-	return g_api != nullptr && g_device != nullptr && g_window != nullptr;
-}
-
-#else
-
-static bool IsAvailable() {
-	return g_api != nullptr && g_device != nullptr;
-}
-
-#endif
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 void RenderDocInit() {
-	bool expected = false;
-	if (!g_init_done.compare_exchange_strong(expected, true)) {
+	if (g_api != nullptr) {
 		return;
 	}
 
-	HKEY h_reg_key;
-	LONG result = RegOpenKeyExW(HKEY_LOCAL_MACHINE,
-	                            L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0,
-	                            KEY_READ, &h_reg_key);
-	if (result != ERROR_SUCCESS) {
-		return;
-	}
-	std::array<wchar_t, MAX_PATH> key_str {};
-	DWORD                         str_sz_out {key_str.size()};
-	result = RegQueryValueExW(h_reg_key, L"", 0, NULL, (LPBYTE)key_str.data(), &str_sz_out);
-	RegCloseKey(h_reg_key);
-	if (result != ERROR_SUCCESS) {
-		return;
-	}
-
-	std::filesystem::path path {key_str.cbegin(), key_str.cend()};
-	path                   = path.parent_path().append("renderdoc.dll");
-	const auto path_to_lib = path.generic_string();
-	auto*      module      = LoadLibraryA(path_to_lib.c_str());
+	auto* module = GetModuleHandleA("renderdoc.dll");
 	if (module == nullptr) {
-		return;
+		HKEY key = nullptr;
+		if (RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+		                  L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0, KEY_READ,
+		                  &key) != ERROR_SUCCESS) {
+			return;
+		}
+
+		std::array<wchar_t, MAX_PATH> path_buffer {};
+		DWORD      path_size = static_cast<DWORD>(path_buffer.size() * sizeof(wchar_t));
+		const auto result    = RegQueryValueExW(
+		    key, L"", nullptr, nullptr, reinterpret_cast<LPBYTE>(path_buffer.data()), &path_size);
+		RegCloseKey(key);
+		if (result != ERROR_SUCCESS) {
+			return;
+		}
+
+		auto path = std::filesystem::path(path_buffer.data()).parent_path() / "renderdoc.dll";
+		module    = LoadLibraryW(path.c_str());
+		if (module == nullptr) {
+			return;
+		}
 	}
 
 	if (!BindRenderDocApi(module)) {
-		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
-		FreeLibrary(module);
-		return;
+		LOGF("RenderDoc: API 1.6.0 is unavailable\n");
 	}
 }
 
 #else
 
 void RenderDocInit() {
-	bool expected = false;
-	if (!g_init_done.compare_exchange_strong(expected, true)) {
+	if (g_api != nullptr) {
 		return;
 	}
 
-	// Prefer an injected RenderDoc instance.
 	auto* module = ::dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
 	if (module == nullptr) {
 		module = ::dlopen("librenderdoc.so", RTLD_NOW);
 	}
 	if (module == nullptr) {
-		LOGF("RenderDoc: librenderdoc.so was not found; in-app capture disabled\n");
 		return;
 	}
 
 	if (!BindRenderDocApi(module)) {
-		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
+		LOGF("RenderDoc: API 1.6.0 is unavailable\n");
 		::dlclose(module);
-		return;
 	}
 }
 
 #endif
 
-void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window) {
-	if (g_api == nullptr) {
-		return;
-	}
-
-	g_device = GetRenderDocDevicePointer(instance);
-	g_window = GetRenderDocWindowHandle(window);
-
-	if (g_device == nullptr || g_window == nullptr) {
-		LOGF("RenderDoc: active Vulkan window was not registered\n");
-		return;
-	}
-
-	g_api->SetActiveWindow(g_device, g_window);
-	LOGF("RenderDoc: active Vulkan window registered\n");
-}
-
 void RenderDocRequestCapture() {
-	if (!IsAvailable()) {
+	if (g_api == nullptr) {
 		if (!g_unavailable_log.exchange(true)) {
-			LOGF("RenderDoc: capture requested, but RenderDoc is not available\n");
+			LOGF("RenderDoc: capture requested, but RenderDoc is unavailable\n");
 		}
 		return;
 	}
 
 	RenderDocState expected = RenderDocState::Idle;
 	if (g_state.compare_exchange_strong(expected, RenderDocState::Requested)) {
-		LOGF("RenderDoc: capture requested; next complete presented frame will be captured\n");
-	} else {
-		LOGF("RenderDoc: capture request ignored because a capture is already pending\n");
+		LOGF("RenderDoc: capture requested\n");
 	}
 }
 
-static void LogNewestCapture() {
-	const auto count = g_api->GetNumCaptures();
-	if (count == 0) {
-		return;
-	}
-
-	char     filename[4096] = {};
-	uint32_t path_length    = sizeof(filename);
-	uint64_t timestamp      = 0;
-
-	if (g_api->GetCapture(count - 1, filename, &path_length, &timestamp) != 0) {
-		filename[sizeof(filename) - 1] = '\0';
-		LOGF("RenderDoc: wrote capture %s\n", filename);
-	}
+bool RenderDocCaptureRequested() {
+	return g_state.load(std::memory_order_acquire) == RenderDocState::Requested;
 }
 
-void RenderDocOnPresent() {
-	if (!IsAvailable()) {
+bool RenderDocCaptureInProgress() {
+	return g_state.load(std::memory_order_acquire) == RenderDocState::Capturing;
+}
+
+void RenderDocStartCapture() {
+	RenderDocState expected = RenderDocState::Requested;
+	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Starting,
+	                                                         std::memory_order_acq_rel)) {
 		return;
 	}
 
-	switch (g_state.load()) {
-		case RenderDocState::Idle: return;
-		case RenderDocState::Requested:
-			if (g_api->IsFrameCapturing() != 0) {
-				LOGF("RenderDoc: capture request ignored because RenderDoc is already capturing\n");
-				g_state.store(RenderDocState::Idle);
-				return;
-			}
+	if (g_api->IsFrameCapturing() != 0) {
+		g_state.store(RenderDocState::Idle, std::memory_order_release);
+		LOGF("RenderDoc: capture request ignored because a capture is already active\n");
+		return;
+	}
 
-			g_api->StartFrameCapture(nullptr, nullptr);
-			if (g_api->IsFrameCapturing() == 0) {
-				LOGF("RenderDoc: StartFrameCapture returned, but RenderDoc is not capturing\n");
-				g_state.store(RenderDocState::Idle);
-				return;
-			}
-			g_state.store(RenderDocState::Capturing);
-			LOGF("RenderDoc: capture started\n");
-			return;
-		case RenderDocState::Capturing: break;
+	const auto capture_id   = std::chrono::duration_cast<std::chrono::microseconds>(
+	                              std::chrono::system_clock::now().time_since_epoch())
+	                              .count();
+	const auto capture_path = "_RenderDoc/kyty_" + std::to_string(capture_id);
+	g_api->SetCaptureFilePathTemplate(capture_path.c_str());
+	g_api->StartFrameCapture(nullptr, nullptr);
+	if (g_api->IsFrameCapturing() == 0) {
+		g_state.store(RenderDocState::Idle, std::memory_order_release);
+		LOGF("RenderDoc: capture failed to start\n");
+		return;
+	}
+	g_captured_flips.store(0, std::memory_order_release);
+	g_state.store(RenderDocState::Capturing, std::memory_order_release);
+	LOGF("RenderDoc: capture started\n");
+}
+
+void RenderDocEndCapture() {
+	if (g_api == nullptr || !RenderDocCaptureInProgress()) {
+		return;
 	}
 
 	const auto ok = g_api->EndFrameCapture(nullptr, nullptr);
-	g_state.store(RenderDocState::Idle);
+	g_state.store(RenderDocState::Idle, std::memory_order_release);
+	LOGF(ok != 0 ? "RenderDoc: capture finished\n" : "RenderDoc: capture failed\n");
+}
 
-	if (ok != 0) {
-		const auto count = g_api->GetNumCaptures();
-		LOGF("RenderDoc: capture finished, count=%u\n", count);
-		LogNewestCapture();
-	} else {
-		LOGF("RenderDoc: capture failed\n");
+void RenderDocOnGuestFlip() {
+	if (!RenderDocCaptureInProgress()) {
+		return;
+	}
+	const auto flip = g_captured_flips.fetch_add(1, std::memory_order_acq_rel) + 1;
+	LOGF("RenderDoc: captured guest flip %u/2\n", flip);
+	if (flip >= 2) {
+		RenderDocEndCapture();
 	}
 }
 

src/graphics/presentation/renderDoc.h

@@ -1,17 +1,16 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_
 
-#include "common/common.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-
-struct SDL_Window;
-
 namespace Libs::Graphics {
 
 void RenderDocInit();
-void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window);
 void RenderDocRequestCapture();
-void RenderDocOnPresent();
+void RenderDocStartCapture();
+void RenderDocEndCapture();
+void RenderDocOnGuestFlip();
+
+[[nodiscard]] bool RenderDocCaptureRequested();
+[[nodiscard]] bool RenderDocCaptureInProgress();
 
 } // namespace Libs::Graphics
 

src/graphics/presentation/videoOut.cpp

@@ -16,6 +16,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/presentation/presenter.h"
+#include "graphics/presentation/renderDoc.h"
 #include "kernel/pthread.h"
 #include "libs/errno.h"
 #include "libs/libs.h"
@@ -42,6 +43,9 @@ constexpr int      VIDEO_OUT_EVENT_PRE_VBLANK_START                     = 2;
 constexpr int      VIDEO_OUT_EVENT_SET_MODE                             = 8;
 constexpr int      VIDEO_OUT_TRUE                                       = 1;
 constexpr int      VIDEO_OUT_FALSE                                      = 0;
+constexpr int      VIDEO_OUT_BUS_TYPE_MAIN                              = 0;
+constexpr int      VIDEO_OUT_BUS_TYPE_OVERLAY                           = 1;
+constexpr int      VIDEO_OUT_BUS_TYPE_SUB                               = 2;
 constexpr int      VIDEO_OUT_FLIP_MODE_VSYNC                            = 1;
 constexpr int      VIDEO_OUT_FLIP_MODE_VSYNC_MULTI                      = 4;
 constexpr int      VIDEO_OUT_BUFFER_INDEX_BLACK                         = -2;
@@ -233,7 +237,7 @@ private:
 
 struct VideoOutDriver::Impl {
 public:
-	static constexpr int VIDEO_OUT_NUM_MAX = 2;
+	static constexpr int VIDEO_OUT_NUM_MAX = 4;
 
 	Impl(uint32_t width, uint32_t height, Graphics::Presenter& presenter)
 	    : m_renderer(presenter.Renderer()), m_presenter(presenter), m_flip_queue(presenter) {
@@ -244,7 +248,7 @@ public:
 	~Impl();
 	KYTY_CLASS_NO_COPY(Impl);
 
-	int             Open();
+	int             Open(int bus_type);
 	bool            Close(int handle);
 	VideoOutConfig* Get(int handle);
 	VideoOutConfig* Get(int handle, uint64_t& generation);
@@ -606,19 +610,11 @@ void VideoOutDriver::Impl::Init(uint32_t width, uint32_t height) {
 	}
 }
 
-int VideoOutDriver::Impl::Open() {
+int VideoOutDriver::Impl::Open(int bus_type) {
 	Common::LockGuard lock(m_mutex);
 
-	int handle = -1;
-
-	for (int i = 1; i < VIDEO_OUT_NUM_MAX; i++) {
-		if (!m_video_out_ctx[i].opened) {
-			handle = i;
-			break;
-		}
-	}
-
-	if (handle < 0) {
+	const int handle = bus_type + 1;
+	if (m_video_out_ctx[handle].opened) {
 		return -1;
 	}
 	auto&             config = m_video_out_ctx[handle];
@@ -1127,6 +1123,7 @@ bool FlipQueue::Flip(uint32_t micros) {
 	m_mutex.Unlock();
 
 	m_presenter.Present(*r.frame);
+	Graphics::RenderDocOnGuestFlip();
 
 	m_mutex.Lock();
 	if (m_requests.empty() || m_requests.front().id != r.id ||
@@ -1171,12 +1168,15 @@ KYTY_SYSV_ABI int VideoOutOpen(int user_id, int bus_type, int index, const void*
 	PRINT_NAME();
 
 	EXIT_NOT_IMPLEMENTED(user_id != 255 && user_id != 0);
-	EXIT_NOT_IMPLEMENTED(bus_type != 0);
+	if (bus_type != VIDEO_OUT_BUS_TYPE_MAIN && bus_type != VIDEO_OUT_BUS_TYPE_OVERLAY &&
+	    bus_type != VIDEO_OUT_BUS_TYPE_SUB) {
+		return VIDEO_OUT_ERROR_INVALID_VALUE;
+	}
 	EXIT_NOT_IMPLEMENTED(index != 0);
 
 	LOGF("\t param = 0x%016" PRIx64 "\n", reinterpret_cast<uint64_t>(param));
 
-	int handle = DriverState().Open();
+	int handle = DriverState().Open(bus_type);
 
 	if (handle < 0) {
 		return VIDEO_OUT_ERROR_RESOURCE_BUSY;

src/graphics/presentation/window/swapchain.cpp

@@ -32,7 +32,6 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/imeOverlay.h"
 #include "graphics/presentation/presenter.h"
-#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window/windowInternal.h"
 #include "libs/controller.h"
@@ -59,10 +58,10 @@
 namespace Libs::Graphics {
 
 struct Presenter::Frame {
-	VulkanImage                    image;
-	std::unique_ptr<CommandBuffer> present_commands;
-	bool                           busy         = false;
-	bool                           reusing_last = false;
+	VulkanImage image;
+	uint64_t    present_tick = 0;
+	bool        busy         = false;
+	bool        reusing_last = false;
 
 	void Configure(GraphicContext& graphics, vk::Extent2D extent, vk::Format format);
 	void Transit(vk::CommandBuffer command, vk::ImageLayout layout, vk::AccessFlags2 access);
@@ -72,12 +71,11 @@ struct Presenter::Frame {
 
 class FramePool final {
 public:
-	explicit FramePool(WindowContext& window): m_window(window) {}
+	FramePool(WindowContext& window, CommandScheduler& scheduler)
+	    : m_window(window), m_scheduler(scheduler) {}
 	~FramePool() {
+		m_scheduler.Wait(m_scheduler.CurrentTick() - 1);
 		for (auto& frame: m_frames) {
-			if (frame->present_commands != nullptr) {
-				frame->present_commands->WaitForFenceOnly();
-			}
 			if (frame->image.image != nullptr) {
 				m_window.graphic_ctx.DeleteImage(frame->image);
 			}
@@ -189,15 +187,10 @@ public:
 	}
 
 private:
-	static void WaitForFrame(Presenter::Frame& frame) {
-		// The producer only waits here. Reset stays on the presentation thread that owns the
-		// allocating Vulkan command pool.
-		if (frame.present_commands != nullptr) {
-			frame.present_commands->WaitForFenceOnly();
-		}
-	}
+	void WaitForFrame(Presenter::Frame& frame) { m_scheduler.Wait(frame.present_tick); }
 
 	WindowContext&                                 m_window;
+	CommandScheduler&                              m_scheduler;
 	Common::Mutex                                  m_mutex;
 	Common::CondVar                                m_available;
 	std::vector<std::unique_ptr<Presenter::Frame>> m_frames;
@@ -331,7 +324,7 @@ public:
 	[[nodiscard]] Status AcquireNextImage();
 	[[nodiscard]] bool   PrepareImeOverlay();
 	void RecordPresentCommands(CommandBuffer& command, VulkanImage& source, bool draw_ime_overlay);
-	void Submit(CommandBuffer& command);
+	uint64_t             Submit(CommandScheduler& scheduler);
 	[[nodiscard]] Status Present();
 
 	[[nodiscard]] uint32_t ImageCount() const noexcept {
@@ -359,7 +352,7 @@ private:
 struct Presenter::Impl {
 	explicit Impl(WindowContext& owner)
 	    : renderer(*owner.render_context), window(owner), swapchain(owner),
-	      present_scheduler(renderer, owner.graphic_ctx), frames(owner) {
+	      present_scheduler(renderer, owner.graphic_ctx), frames(owner, present_scheduler) {
 		EXIT_IF(owner.render_context == nullptr);
 		swapchain.Create();
 		frames.Initialize(swapchain.ImageCount(), swapchain.Format());
@@ -629,7 +622,6 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 	}
 	EXIT_IF(m_image_index >= m_images.size());
 	auto vk_command = command.Handle();
-	command.Begin();
 
 	vk::ImageMemoryBarrier to_transfer {};
 	to_transfer.sType                           = vk::StructureType::eImageMemoryBarrier;
@@ -700,15 +692,14 @@ void Swapchain::RecordPresentCommands(CommandBuffer& command, VulkanImage& sourc
 		                           vk::DependencyFlagBits::eByRegion, 0, nullptr, 0, nullptr, 1,
 		                           &to_present);
 	}
-	command.End();
 }
 
-void Swapchain::Submit(CommandBuffer& command) {
+uint64_t Swapchain::Submit(CommandScheduler& scheduler) {
 	EXIT_IF(m_frame_index >= m_image_acquired.size() || m_image_index >= m_render_complete.size());
 	SubmitInfo submit;
 	submit.AddWait(m_image_acquired[m_frame_index], 1, vk::PipelineStageFlagBits::eTransfer);
 	submit.AddSignal(m_render_complete[m_image_index]);
-	command.Execute(submit);
+	return scheduler.Submit(submit);
 }
 
 Swapchain::Status Swapchain::Present() {
@@ -783,15 +774,9 @@ Presenter::Frame& Presenter::PrepareBlankFrame(uint32_t width, uint32_t height,
 		EXIT_IF(producer->IsInvalid());
 		frame->Clear(*producer, clear);
 	} else {
-		if (frame->present_commands == nullptr) {
-			frame->present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
-		}
-		auto& command = *frame->present_commands;
-		command.WaitForFenceAndReset();
-		command.Begin();
+		auto& command = m_impl->present_scheduler.BeginCommand();
 		frame->Clear(command, clear);
-		command.End();
-		command.Execute();
+		frame->present_tick = m_impl->present_scheduler.Submit();
 	}
 	return *frame;
 }
@@ -849,16 +834,12 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			m_impl->RecoverSwapchain(status);
 			continue;
 		}
-		if (frame.present_commands == nullptr) {
-			frame.present_commands = std::make_unique<CommandBuffer>(m_impl->present_scheduler);
-		}
 		{
 			Common::LockGuard render_lock(m_impl->renderer.GetMutex());
-			frame.present_commands->WaitForFenceAndReset();
-			auto&      command          = *frame.present_commands;
-			const bool draw_ime_overlay = ime_visual.active && swapchain.PrepareImeOverlay();
+			auto&             command          = m_impl->present_scheduler.BeginCommand();
+			const bool        draw_ime_overlay = ime_visual.active && swapchain.PrepareImeOverlay();
 			swapchain.RecordPresentCommands(command, frame.image, draw_ime_overlay);
-			swapchain.Submit(command);
+			frame.present_tick = swapchain.Submit(m_impl->present_scheduler);
 		}
 		status = swapchain.Present();
 		if (status != Swapchain::Status::Success) {
@@ -866,7 +847,6 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			continue;
 		}
 
-		RenderDocOnPresent();
 		m_impl->presented_ime_revision.store(ime_visual.revision, std::memory_order_release);
 		window.UpdateTitle();
 		m_impl->frames.Release(&frame, true);

src/graphics/presentation/window/vulkanWindow.cpp

@@ -32,7 +32,6 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/imeOverlay.h"
 #include "graphics/presentation/presenter.h"
-#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/presentation/window/windowInternal.h"
@@ -200,7 +199,15 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 
 		vk::PhysicalDeviceVulkan12Features features12 {};
 		features12.sType = vk::StructureType::ePhysicalDeviceVulkan12Features;
+#if defined(__APPLE__)
 		features12.pNext = &depth_clip_control;
+#else
+		vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
+		fragment_barycentric.sType =
+		    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+		fragment_barycentric.pNext = &depth_clip_control;
+		features12.pNext           = &fragment_barycentric;
+#endif
 		features13.pNext = &features12;
 
 		device_features2.sType = vk::StructureType::ePhysicalDeviceFeatures2;
@@ -232,6 +239,12 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			skip_device = true;
 #endif
 		}
+#if !defined(__APPLE__)
+		if (fragment_barycentric.fragmentShaderBarycentric != VK_TRUE) {
+			LOGF("fragmentShaderBarycentric is not supported\n");
+			skip_device = true;
+		}
+#endif
 
 		if (features12.samplerMirrorClampToEdge != VK_TRUE) {
 			LOGF("samplerMirrorClampToEdge is not supported\n");
@@ -263,6 +276,22 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			LOGF("depthBiasClamp is not supported\n");
 			skip_device = true;
 		}
+		if (device_features2.features.shaderClipDistance != VK_TRUE) {
+			LOGF("shaderClipDistance is not supported\n");
+			skip_device = true;
+		}
+		if (device_features2.features.shaderCullDistance != VK_TRUE) {
+			LOGF("shaderCullDistance is not supported\n");
+			skip_device = true;
+		}
+		if (device_features2.features.largePoints != VK_TRUE) {
+			LOGF("largePoints is not supported\n");
+			skip_device = true;
+		}
+		if (features12.shaderOutputLayer != VK_TRUE) {
+			LOGF("shaderOutputLayer is not supported\n");
+			skip_device = true;
+		}
 
 		if (device_features2.features.fragmentStoresAndAtomics != VK_TRUE) {
 			LOGF("fragmentStoresAndAtomics is not supported\n");
@@ -287,10 +316,6 @@ static void VulkanFindPhysicalDevice(vk::Instance instance, vk::SurfaceKHR surfa
 			LOGF("shaderStorageImageWriteWithoutFormat is not supported\n");
 			skip_device = true;
 		}
-		if (device_features2.features.shaderStorageImageReadWithoutFormat != VK_TRUE) {
-			LOGF("shaderStorageImageReadWithoutFormat is not supported\n");
-			skip_device = true;
-		}
 
 		if (device_features2.features.shaderImageGatherExtended != VK_TRUE) {
 			LOGF("shaderImageGatherExtended is not supported\n");
@@ -468,14 +493,19 @@ static void VulkanInitSubgroupSizeControl(vk::PhysicalDevice physical_device,
 	graphics.min_subgroup_size             = subgroup_size_control.minSubgroupSize;
 	graphics.max_subgroup_size             = subgroup_size_control.maxSubgroupSize;
 	graphics.required_subgroup_size_stages = subgroup_size_control.requiredSubgroupSizeStages;
-	graphics.subgroup_size_control_enabled = features13.subgroupSizeControl == VK_TRUE &&
-	                                         subgroup_size_control.minSubgroupSize <= 64 &&
-	                                         subgroup_size_control.maxSubgroupSize >= 64;
+	graphics.compute_subgroup_size_control_enabled =
+	    features13.subgroupSizeControl == VK_TRUE &&
+	    (graphics.required_subgroup_size_stages & vk::ShaderStageFlagBits::eCompute) &&
+	    subgroup_size_control.minSubgroupSize <= 64 &&
+	    subgroup_size_control.maxSubgroupSize >= 64;
+	graphics.compute_wave64_supported =
+	    graphics.subgroup_size == 64u || graphics.compute_subgroup_size_control_enabled;
 
-	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s\n",
+	LOGF("Vulkan subgroup: default=%u min=%u max=%u stages=0x%08x size_control=%s wave64=%s\n",
 	     graphics.subgroup_size, graphics.min_subgroup_size, graphics.max_subgroup_size,
 	     static_cast<vk::ShaderStageFlags::MaskType>(graphics.required_subgroup_size_stages),
-	     graphics.subgroup_size_control_enabled ? "true" : "false");
+	     graphics.compute_subgroup_size_control_enabled ? "true" : "false",
+	     graphics.compute_wave64_supported ? "true" : "false");
 }
 
 static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const VulkanExtensions& r,
@@ -519,10 +549,6 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	features12.pNext                    = &depth_clip_control;
 	features12.samplerMirrorClampToEdge = VK_TRUE;
 
-	vk::PhysicalDeviceSubgroupSizeControlFeatures subgroup_size_control {};
-	subgroup_size_control.sType = vk::StructureType::ePhysicalDeviceSubgroupSizeControlFeatures;
-	subgroup_size_control.pNext = &features12;
-
 	vk::PhysicalDeviceVulkan13Features supported_features13 {};
 	supported_features13.sType = vk::StructureType::ePhysicalDeviceVulkan13Features;
 
@@ -531,6 +557,14 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	supported_features12.pNext = nullptr;
 	supported_features13.pNext = &supported_features12;
 
+#if !defined(__APPLE__)
+	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR supported_fragment_barycentric {};
+	supported_fragment_barycentric.sType =
+	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+	supported_fragment_barycentric.pNext = nullptr;
+	supported_features12.pNext           = &supported_fragment_barycentric;
+#endif
+
 	const auto robustness2_ext_enabled =
 	    HasExtension(device_extensions, VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
 
@@ -538,7 +572,11 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	supported_robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
 	supported_robustness2.pNext = nullptr;
 	if (robustness2_ext_enabled) {
+#if defined(__APPLE__)
 		supported_features12.pNext = &supported_robustness2;
+#else
+		supported_fragment_barycentric.pNext = &supported_robustness2;
+#endif
 	}
 
 	vk::PhysicalDeviceFeatures2 supported_features2 {};
@@ -553,7 +591,19 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	                     supported_features13.synchronization2 != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.sampleRateShading != VK_TRUE);
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderClipDistance != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderCullDistance != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.largePoints != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.shaderInt64 != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features2.features.vertexPipelineStoresAndAtomics != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features12.shaderOutputLayer != VK_TRUE);
+	EXIT_NOT_IMPLEMENTED(supported_features12.bufferDeviceAddress != VK_TRUE);
+#if !defined(__APPLE__)
+	EXIT_NOT_IMPLEMENTED(supported_fragment_barycentric.fragmentShaderBarycentric != VK_TRUE);
+#endif
 	features12.timelineSemaphore = VK_TRUE;
+	features12.shaderOutputLayer = VK_TRUE;
+	features12.bufferDeviceAddress = VK_TRUE;
 
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
@@ -563,49 +613,61 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	device_features.depthBounds = VK_TRUE; // unsupported by MoltenVK
 #endif
 	device_features.shaderStorageImageWriteWithoutFormat = VK_TRUE;
-	device_features.shaderStorageImageReadWithoutFormat  = VK_TRUE;
 	device_features.shaderImageGatherExtended            = VK_TRUE;
 	device_features.independentBlend                     = VK_TRUE;
 	device_features.tessellationShader                   = VK_TRUE;
 	device_features.sampleRateShading                    = VK_TRUE;
 	device_features.depthBiasClamp                       = VK_TRUE;
+	device_features.shaderClipDistance                   = VK_TRUE;
+	device_features.shaderCullDistance                   = VK_TRUE;
+	device_features.largePoints                          = VK_TRUE;
+	device_features.vertexPipelineStoresAndAtomics       = VK_TRUE;
 	graphics.sample_rate_shading_enabled                 = true;
-	device_features.vertexPipelineStoresAndAtomics =
-	    supported_features2.features.vertexPipelineStoresAndAtomics;
-
-	if (graphics.subgroup_size_control_enabled &&
-	    supported_features13.subgroupSizeControl == VK_TRUE) {
-		subgroup_size_control.subgroupSizeControl = VK_TRUE;
-	}
-
-	const auto* base_feature_chain = (subgroup_size_control.subgroupSizeControl == VK_TRUE
-	                                      ? static_cast<const void*>(&subgroup_size_control)
-	                                      : static_cast<const void*>(&features12));
+	device_features.shaderInt64 = VK_TRUE;
 
 	vk::PhysicalDeviceRobustness2FeaturesEXT robustness2 {};
 	robustness2.sType = vk::StructureType::ePhysicalDeviceRobustness2FeaturesEXT;
-	robustness2.pNext = const_cast<void*>(base_feature_chain);
+#if defined(__APPLE__)
+	robustness2.pNext = &features12;
+#else
+	vk::PhysicalDeviceFragmentShaderBarycentricFeaturesKHR fragment_barycentric {};
+	fragment_barycentric.sType =
+	    vk::StructureType::ePhysicalDeviceFragmentShaderBarycentricFeaturesKHR;
+	fragment_barycentric.pNext                     = &features12;
+	fragment_barycentric.fragmentShaderBarycentric = VK_TRUE;
+	robustness2.pNext                              = &fragment_barycentric;
+#endif
 	if (robustness2_ext_enabled) {
 		robustness2.robustBufferAccess2 = supported_robustness2.robustBufferAccess2;
 		robustness2.robustImageAccess2  = supported_robustness2.robustImageAccess2;
 		robustness2.nullDescriptor      = supported_robustness2.nullDescriptor;
 	}
 
+	const bool subgroup_size_control_enabled =
+	    graphics.compute_subgroup_size_control_enabled &&
+	    supported_features13.subgroupSizeControl == VK_TRUE;
+
 	auto features13 = required_features13;
-	features13.pNext =
-	    robustness2_ext_enabled ? &robustness2 : const_cast<void*>(base_feature_chain);
-	features13.robustImageAccess = supported_features13.robustImageAccess;
+#if defined(__APPLE__)
+	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
+	                                           : static_cast<void*>(&features12);
+#else
+	features13.pNext = robustness2_ext_enabled ? static_cast<void*>(&robustness2)
+	                                           : static_cast<void*>(&fragment_barycentric);
+#endif
+	features13.robustImageAccess   = supported_features13.robustImageAccess;
+	features13.subgroupSizeControl = subgroup_size_control_enabled ? VK_TRUE : VK_FALSE;
 
 	LOGF("Vulkan robustness: robustImageAccess=%s robustImageAccess2=%s\n",
 	     features13.robustImageAccess == VK_TRUE ? "true" : "false",
 	     robustness2_ext_enabled && robustness2.robustImageAccess2 == VK_TRUE ? "true" : "false");
 
 	vk::DeviceCreateInfo create_info {};
-	create_info.sType                = vk::StructureType::eDeviceCreateInfo;
-	create_info.pNext                = &features13;
-	create_info.flags                = {};
-	create_info.pQueueCreateInfos    = &queue_create_info;
-	create_info.queueCreateInfoCount = 1;
+	create_info.sType                   = vk::StructureType::eDeviceCreateInfo;
+	create_info.pNext                   = &features13;
+	create_info.flags                   = {};
+	create_info.pQueueCreateInfos       = &queue_create_info;
+	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
 	create_info.ppEnabledExtensionNames = device_extensions.data();
 	create_info.pEnabledFeatures        = &device_features;
@@ -834,23 +896,26 @@ void WindowContext::CreateVulkan() {
 	app_info.engineVersion      = 1;
 	app_info.apiVersion         = VULKAN_TARGET_API_VERSION; // NOLINT
 
-	vk::ValidationFeatureEnableEXT enabled_features[] = {
+	if (Config::SpirvDebugPrintfEnabled() && Config::GpuAssistedValidationEnabled()) {
+		EXIT("--spirv-debug-printf and --gpu-assisted-validation are mutually exclusive\n");
+	}
+
+	vk::ValidationFeatureEnableEXT enabled_features[3]    = {};
+	uint32_t                       enabled_features_count = 0;
 #ifdef KYTY_ENABLE_BEST_PRACTICES
-	    vk::ValidationFeatureEnableEXT::eBestPractices,
+	enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eBestPractices;
 #endif
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
-	    vk::ValidationFeatureEnableEXT::eDebugPrintf,
-#endif
-	};
-
-	uint32_t enabled_features_count =
-	    sizeof(enabled_features) / sizeof(vk::ValidationFeatureEnableEXT);
-
-#ifdef KYTY_ENABLE_DEBUG_PRINTF
-	if (!Config::SpirvDebugPrintfEnabled()) {
-		enabled_features_count--;
+	if (Config::SpirvDebugPrintfEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eDebugPrintf;
 	}
 #endif
+	if (Config::GpuAssistedValidationEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eGpuAssisted;
+		LOGF("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::printf("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::fflush(stdout);
+	}
 
 	vk::ValidationFeaturesEXT validation_features {};
 	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
@@ -923,6 +988,7 @@ void WindowContext::CreateVulkan() {
 #else
 	device_extensions.push_back(VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME);
 	device_extensions.push_back(VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
+	device_extensions.push_back(VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME);
 #endif
 
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
@@ -940,7 +1006,15 @@ void WindowContext::CreateVulkan() {
 		EXIT("Could not find suitable device");
 	}
 
-	graphic_ctx.physical_device.getProperties(&graphic_ctx.physical_device_properties);
+	vk::PhysicalDevicePushDescriptorProperties push_descriptor_properties {};
+	vk::PhysicalDeviceProperties2              physical_device_properties {};
+	push_descriptor_properties.sType = vk::StructureType::ePhysicalDevicePushDescriptorProperties;
+	push_descriptor_properties.pNext = nullptr;
+	physical_device_properties.sType = vk::StructureType::ePhysicalDeviceProperties2;
+	physical_device_properties.pNext = &push_descriptor_properties;
+	graphic_ctx.physical_device.getProperties2(&physical_device_properties);
+	graphic_ctx.physical_device_properties = physical_device_properties.properties;
+	graphic_ctx.max_push_descriptors       = push_descriptor_properties.maxPushDescriptors;
 	graphic_ctx.physical_device.getMemoryProperties(&graphic_ctx.physical_device_memory_properties);
 	const auto& device_properties = graphic_ctx.GetPhysicalDeviceProperties();
 
@@ -986,7 +1060,6 @@ void WindowContext::CreateVulkan() {
 	render_context = std::make_unique<RenderContext>(graphic_ctx);
 	LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
 	presenter = std::make_unique<Presenter>(*this);
-	RenderDocSetActiveWindow(graphic_ctx.instance, window);
 }
 
 void WindowContext::RefreshSurfaceCapabilities() {

src/graphics/shader/recompiler/ExecMask.cpp

@@ -1,51 +0,0 @@
-#include "graphics/shader/recompiler/ExecMask.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Exec {
-
-Mask FullMask(uint32_t wave_size) {
-	return {0xffffffffu, wave_size == 64u ? 0xffffffffu : 0u};
-}
-
-Mask ApplyWaveSize(Mask mask, uint32_t wave_size) {
-	const auto full = FullMask(wave_size);
-	mask.low &= full.low;
-	mask.high &= full.high;
-	return mask;
-}
-
-Mask ReadExec(const State& state) {
-	return ApplyWaveSize(state.exec, state.wave_size);
-}
-
-void WriteExec(State& state, Mask mask) {
-	state.exec = ApplyWaveSize(mask, state.wave_size);
-}
-
-void AndExec(State& state, Mask mask) {
-	WriteExec(state, {state.exec.low & mask.low, state.exec.high & mask.high});
-}
-
-void OrExec(State& state, Mask mask) {
-	WriteExec(state, {state.exec.low | mask.low, state.exec.high | mask.high});
-}
-
-void XorExec(State& state, Mask mask) {
-	WriteExec(state, {state.exec.low ^ mask.low, state.exec.high ^ mask.high});
-}
-
-void InvertExec(State& state) {
-	const auto full = FullMask(state.wave_size);
-	WriteExec(state, {~state.exec.low & full.low, ~state.exec.high & full.high});
-}
-
-bool IsExecZero(const State& state) {
-	const auto exec = ReadExec(state);
-	return exec.low == 0u && exec.high == 0u;
-}
-
-bool IsVccZero(const State& state) {
-	const auto vcc = ApplyWaveSize(state.vcc, state.wave_size);
-	return vcc.low == 0u && vcc.high == 0u;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Exec

src/graphics/shader/recompiler/ExecMask.h

@@ -1,34 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_
-
-#include "common/common.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Exec {
-
-struct Mask {
-	uint32_t low  = 0;
-	uint32_t high = 0;
-};
-
-struct State {
-	uint32_t wave_size = 64;
-	Mask     exec;
-	Mask     vcc;
-	bool     scc = false;
-};
-
-Mask FullMask(uint32_t wave_size);
-Mask ApplyWaveSize(Mask mask, uint32_t wave_size);
-
-Mask ReadExec(const State& state);
-void WriteExec(State& state, Mask mask);
-void AndExec(State& state, Mask mask);
-void OrExec(State& state, Mask mask);
-void XorExec(State& state, Mask mask);
-void InvertExec(State& state);
-bool IsExecZero(const State& state);
-bool IsVccZero(const State& state);
-
-} // namespace Libs::Graphics::ShaderRecompiler::Exec
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_ */

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -2,18 +2,21 @@
 
 #include "common/assert.h"
 #include "common/logging/log.h"
-#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
-#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
-#include "graphics/shader/recompiler/ir/ReadLaneElimination.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
-#include "graphics/shader/recompiler/ir/ResourceTracking.h"
-#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/translate/Translate.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h"
-#include "graphics/shader/recompiler/ir/SrtPatcher.h"
-#include "graphics/shader/recompiler/ir/SrtWalker.h"
+#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
+#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
+#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
+#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
+#include "graphics/shader/recompiler/ir/passes/SharedMemoryBarrier.h"
+#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
+#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
 
 #include <algorithm>
 #include <array>
@@ -43,6 +46,8 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
 	std::string dump = "CFG:\n";
 	dump += CFG::GraphToString(cfg);
 	dump += "\nIR:\n";
+	dump += fmt::format("mode={} scratch_dwords={}\n",
+	                    ir.dispatcher_fallback ? "dispatcher" : "structured", ir.scratch_dwords);
 	dump += IR::ProgramToString(ir);
 	return dump;
 }
@@ -56,76 +61,22 @@ const char* StageName(ShaderType stage) {
 	}
 }
 
-std::string FormatCfgFailure(const CFG::Graph& cfg, const CompileOptions& options,
-                             const std::string& reason) {
-	const auto block_id = cfg.failure_block != UINT32_MAX ? cfg.failure_block : cfg.entry_block;
-	return CFG::FormatBlockDiagnostic(cfg, block_id, StageName(options.stage), reason);
-}
-
-bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
-	switch (inst.op) {
-		case IR::Opcode::SLoadDword:
-		case IR::Opcode::SBufferLoadDword:
-		case IR::Opcode::BufferLoadUbyte:
-		case IR::Opcode::BufferLoadSbyte:
-		case IR::Opcode::BufferLoadUshort:
-		case IR::Opcode::BufferLoadSshort:
-		case IR::Opcode::BufferLoadDword:
-		case IR::Opcode::BufferStoreByte:
-		case IR::Opcode::BufferStoreShort:
-		case IR::Opcode::BufferStoreDword:
-		case IR::Opcode::AtomicSwapU32:
-		case IR::Opcode::AtomicAddU32:
-		case IR::Opcode::AtomicSubU32:
-		case IR::Opcode::AtomicSMinI32:
-		case IR::Opcode::AtomicUMinU32:
-		case IR::Opcode::AtomicSMaxI32:
-		case IR::Opcode::AtomicUMaxU32:
-		case IR::Opcode::AtomicAndU32:
-		case IR::Opcode::AtomicOrU32:
-		case IR::Opcode::AtomicXorU32:
-		case IR::Opcode::AtomicFMinF32:
-		case IR::Opcode::AtomicFMaxF32:
-		case IR::Opcode::FlatLoadUbyte:
-		case IR::Opcode::FlatLoadSbyte:
-		case IR::Opcode::FlatLoadUshort:
-		case IR::Opcode::FlatLoadSshort:
-		case IR::Opcode::FlatLoadDword:
-		case IR::Opcode::FlatStoreByte:
-		case IR::Opcode::FlatStoreShort:
-		case IR::Opcode::FlatStoreDword:
-		case IR::Opcode::DsMinF32:
-		case IR::Opcode::DsMaxF32:
-		case IR::Opcode::DsWriteByte:
-		case IR::Opcode::DsWriteShort:
-		case IR::Opcode::DsWriteB32:
-		case IR::Opcode::DsWriteAddtidB32:
-		case IR::Opcode::DsAppend:
-		case IR::Opcode::DsConsume:
-		case IR::Opcode::ImageStore:
-		case IR::Opcode::Export: return true;
-		default: return false;
-	}
-}
-
-bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* reason) {
-	for (const auto& block: ir.blocks) {
-		if (!block.terminator.loop_header) {
-			continue;
-		}
-		for (const auto& inst: block.instructions) {
-			if (!InstructionMaySplitSpirvBlock(inst)) {
-				continue;
-			}
-			if (reason != nullptr) {
-				*reason = fmt::format("loop header block {} contains an instruction whose SPIR-V "
-				                      "lowering emits internal control flow: {}",
-				                      block.id, IR::InstructionToString(inst).c_str());
-			}
-			return true;
-		}
-	}
-	return false;
+void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase,
+                           const std::string& reason) {
+	const auto* block        = cfg.FindBlock(cfg.failure_block);
+	const auto  start        = block != nullptr ? block->start_pc : UINT32_MAX;
+	const auto  end          = block != nullptr ? block->end_pc : UINT32_MAX;
+	const auto  predecessors = block != nullptr ? block->predecessors.size() : 0u;
+	const auto  successors   = block != nullptr ? block->successors.size() : 0u;
+	LOGF("%s CFG dispatcher fallback: stage=%s hash=0x%016" PRIx64
+	     " phase=%s failure=%s block=%" PRIu32 " pc=0x%08" PRIx32 "..0x%08" PRIx32 " preds=%" PRIu64
+	     " succs=%" PRIu64 " blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64
+	     " reason=%s\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash, phase,
+	     CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
+	     static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
+	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
+	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
 }
 
 enum class EmbeddedFetchValueType {
@@ -165,17 +116,8 @@ void ClearEmbeddedFetchVectorLanes(EmbeddedFetchVectorLanes* lanes, uint32_t reg
 	lanes->erase(first, last);
 }
 
-struct EmbeddedFetchLoad {
-	uint32_t              pc         = 0;
-	int                   attrib_id  = -1;
-	uint32_t              components = 0;
-	std::vector<uint32_t> prolog_loads;
-};
-
-struct EmbeddedFetchData {
-	std::vector<EmbeddedFetchLoad> loads;
-	int32_t                        vertex_offset_sgpr = -1;
-};
+using EmbeddedFetchLoad = Frontend::EmbeddedFetchLoad;
+using EmbeddedFetchData = Frontend::EmbeddedFetchPlan;
 
 bool IsDecodedSgpr(const Decoder::Operand& op) {
 	return op.kind == Decoder::OperandKind::Sgpr || op.kind == Decoder::OperandKind::VccLo ||
@@ -199,19 +141,19 @@ uint32_t DecodedDstSize(const Decoder::Instruction& inst) {
 }
 
 uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) {
-	return inst.opcode == Decoder::Opcode::VMadU64U32 ? 2u : DecodedDstSize(inst);
+	return inst.opcode == Decoder::Opcode::V_MAD_U64_U32 ? 2u : DecodedDstSize(inst);
 }
 
 bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) {
 	switch (opcode) {
-		case Decoder::Opcode::SSetpcB64:
-		case Decoder::Opcode::SBranch:
-		case Decoder::Opcode::SCbranchScc0:
-		case Decoder::Opcode::SCbranchScc1:
-		case Decoder::Opcode::SCbranchVccz:
-		case Decoder::Opcode::SCbranchVccnz:
-		case Decoder::Opcode::SCbranchExecz:
-		case Decoder::Opcode::SCbranchExecnz: return true;
+		case Decoder::Opcode::S_SETPC_B64:
+		case Decoder::Opcode::S_BRANCH:
+		case Decoder::Opcode::S_CBRANCH_SCC0:
+		case Decoder::Opcode::S_CBRANCH_SCC1:
+		case Decoder::Opcode::S_CBRANCH_VCCZ:
+		case Decoder::Opcode::S_CBRANCH_VCCNZ:
+		case Decoder::Opcode::S_CBRANCH_EXECZ:
+		case Decoder::Opcode::S_CBRANCH_EXECNZ: return true;
 		default: return false;
 	}
 }
@@ -260,32 +202,32 @@ bool TryDecodedSmemOffset(const std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
 
 bool IsEmbeddedFetchSLoad(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::SLoadDword:
-		case Decoder::Opcode::SLoadDwordx2:
-		case Decoder::Opcode::SLoadDwordx4:
-		case Decoder::Opcode::SLoadDwordx8:
-		case Decoder::Opcode::SLoadDwordx16: return true;
+		case Decoder::Opcode::S_LOAD_DWORD:
+		case Decoder::Opcode::S_LOAD_DWORDX2:
+		case Decoder::Opcode::S_LOAD_DWORDX4:
+		case Decoder::Opcode::S_LOAD_DWORDX8:
+		case Decoder::Opcode::S_LOAD_DWORDX16: return true;
 		default: return false;
 	}
 }
 
 bool IsEmbeddedFetchBufferLoad(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::BufferLoadFormatX:
-		case Decoder::Opcode::BufferLoadFormatXy:
-		case Decoder::Opcode::BufferLoadFormatXyz:
-		case Decoder::Opcode::BufferLoadFormatXyzw: return true;
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_X:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: return true;
 		default: return false;
 	}
 }
 
 bool IsEmbeddedFetchAttribPropagationAlu(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::SBfeU32:
-		case Decoder::Opcode::SAndB32:
-		case Decoder::Opcode::SAddI32:
-		case Decoder::Opcode::SAddU32:
-		case Decoder::Opcode::SLshlB32: return true;
+		case Decoder::Opcode::S_BFE_U32:
+		case Decoder::Opcode::S_AND_B32:
+		case Decoder::Opcode::S_ADD_I32:
+		case Decoder::Opcode::S_ADD_U32:
+		case Decoder::Opcode::S_LSHL_B32: return true;
 		default: return false;
 	}
 }
@@ -294,23 +236,11 @@ int BufferTableAttribFromOffset(uint32_t raw_offset, int dword) {
 	return static_cast<int>((raw_offset + static_cast<uint32_t>(dword) * 4u) / 16u);
 }
 
-void AppendUniquePcs(std::vector<uint32_t>& dst, const std::vector<uint32_t>& src) {
-	dst.reserve(dst.size() + src.size());
-	for (auto pc: src) {
-		if (std::find(dst.begin(), dst.end(), pc) == dst.end()) {
-			dst.push_back(pc);
-		}
-	}
-}
-
 EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded,
                                             const ShaderVertexInputInfo* input_info,
                                             uint32_t user_data_base, uint32_t user_data_count,
                                             uint32_t wave_size) {
 	EmbeddedFetchData data;
-	if (input_info == nullptr || !input_info->fetch_embedded) {
-		return data;
-	}
 	data.loads.reserve(input_info->resources_num);
 	int32_t offset_candidate = -1;
 	bool    offset_conflict  = false;
@@ -348,10 +278,11 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 		uint32_t   sad_zero = 0;
 		const bool vertex_offset_add =
 		    vertex_index_accumulator && IsDecodedSgpr(inst.src0) &&
-		    ((inst.opcode == Decoder::Opcode::VAddI32 && IsDecodedVgpr(inst.src1) &&
+		    ((inst.opcode == Decoder::Opcode::V_ADD_I32 && IsDecodedVgpr(inst.src1) &&
 		      inst.src1.reg == inst.dst.reg) ||
-		     (user_data_base == 8 && inst.dst.reg == 5 && inst.opcode == Decoder::Opcode::VSadU32 &&
-		      IsDecodedVgpr(inst.src2) && inst.src2.reg == inst.dst.reg &&
+		     (user_data_base == 8 && inst.dst.reg == 5 &&
+		      inst.opcode == Decoder::Opcode::V_SAD_U32 && IsDecodedVgpr(inst.src2) &&
+		      inst.src2.reg == inst.dst.reg &&
 		      TryDecodedOperandConstant(sgprs, inst.src1, sad_zero) && sad_zero == 0));
 		if (data.loads.empty() && vertex_offset_add) {
 			const auto reg = DecodedSgprReg(inst.src0);
@@ -364,7 +295,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 			}
 		}
 		switch (inst.opcode) {
-			case Decoder::Opcode::VWritelaneB32: {
+			case Decoder::Opcode::V_WRITELANE_B32: {
 				uint32_t lane = 0;
 				if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
 					vgprs[inst.dst.reg] = {};
@@ -380,7 +311,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 				}
 				break;
 			}
-			case Decoder::Opcode::VReadlaneB32: {
+			case Decoder::Opcode::V_READLANE_B32: {
 				uint32_t lane = 0;
 				if (track_vector_lanes && IsDecodedSgpr(inst.dst) &&
 				    DecodedSgprReg(inst.dst) < sgprs.size() && IsDecodedVgpr(inst.src0) &&
@@ -394,7 +325,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 				}
 				break;
 			}
-			case Decoder::Opcode::SMovB32:
+			case Decoder::Opcode::S_MOV_B32:
 				if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
 				    DecodedSgprReg(inst.src0) < sgprs.size()) {
 					sgprs[DecodedSgprReg(inst.dst)] = sgprs[DecodedSgprReg(inst.src0)];
@@ -410,7 +341,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 					}
 				}
 				break;
-			case Decoder::Opcode::SMovkI32:
+			case Decoder::Opcode::S_MOVK_I32:
 				if (IsDecodedSgpr(inst.dst)) {
 					auto& dst = sgprs[DecodedSgprReg(inst.dst)];
 					dst.type  = EmbeddedFetchValueType::Constant;
@@ -471,7 +402,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 					} else {
 						ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
 					}
-				} else if (inst.opcode == Decoder::Opcode::VCndmaskB32) {
+				} else if (inst.opcode == Decoder::Opcode::V_CNDMASK_B32) {
 					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
 						ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
 					}
@@ -493,11 +424,11 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 							auto& dst = sgprs[DecodedSgprReg(inst.dst)];
 							dst.type  = EmbeddedFetchValueType::Constant;
 							switch (inst.opcode) {
-								case Decoder::Opcode::SAndB32: dst.value = src0 & src1; break;
-								case Decoder::Opcode::SLshlB32:
+								case Decoder::Opcode::S_AND_B32: dst.value = src0 & src1; break;
+								case Decoder::Opcode::S_LSHL_B32:
 									dst.value = src0 << (src1 & 31u);
 									break;
-								case Decoder::Opcode::SBfeU32:
+								case Decoder::Opcode::S_BFE_U32:
 									dst.value = src0 >> (src1 & 31u);
 									break;
 								default: dst.value = src0 + src1; break;
@@ -526,9 +457,9 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 				}
 				break;
 		}
-		if (inst.opcode == Decoder::Opcode::VMovreldB32) {
+		if (inst.opcode == Decoder::Opcode::V_MOVRELD_B32) {
 			vector_lanes.clear();
-		} else if (inst.opcode != Decoder::Opcode::VWritelaneB32 && IsDecodedVgpr(inst.dst)) {
+		} else if (inst.opcode != Decoder::Opcode::V_WRITELANE_B32 && IsDecodedVgpr(inst.dst)) {
 			for (uint32_t i = 0; i < EmbeddedFetchDstSize(inst) && inst.dst.reg + i < vgprs.size();
 			     i++) {
 				ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg + i);
@@ -539,104 +470,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 	return data;
 }
 
-const EmbeddedFetchLoad* FindEmbeddedFetchLoad(const std::vector<EmbeddedFetchLoad>& loads,
-                                               uint32_t                              pc) {
-	for (const auto& load: loads) {
-		if (load.pc == pc) {
-			return &load;
-		}
-	}
-	return nullptr;
-}
-
-bool EmbeddedFetchPcInList(const std::vector<uint32_t>& pcs, uint32_t pc) {
-	return std::find(pcs.begin(), pcs.end(), pc) != pcs.end();
-}
-
-bool IsIrFetchPrologLoad(const IR::Instruction& inst) {
-	return inst.op == IR::Opcode::SLoadDword || inst.op == IR::Opcode::SBufferLoadDword;
-}
-
-int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
-                                 const EmbeddedFetchLoad&     load) {
-	if (input_info == nullptr) {
-		return -1;
-	}
-	if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num &&
-	    input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) {
-		return load.attrib_id;
-	}
-	for (int i = 0; i < input_info->resources_num; i++) {
-		const auto& dst = input_info->resources_dst[i];
-		if (dst.attr_id == load.attrib_id &&
-		    load.components <= static_cast<uint32_t>(std::max(dst.registers_num, 1))) {
-			return i;
-		}
-	}
-	for (int i = 0; i < input_info->resources_num; i++) {
-		if (input_info->resources_dst[i].attr_id == load.attrib_id) {
-			return i;
-		}
-	}
-	return -1;
-}
-
-uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputInfo* input_info,
-                                      const std::vector<EmbeddedFetchLoad>& loads) {
-	if (input_info == nullptr || loads.empty()) {
-		return 0;
-	}
-
-	std::vector<uint32_t> prolog_pcs;
-	prolog_pcs.reserve(loads.size());
-	for (const auto& load: loads) {
-		AppendUniquePcs(prolog_pcs, load.prolog_loads);
-	}
-
-	auto*    mutable_input_info = const_cast<ShaderVertexInputInfo*>(input_info);
-	uint32_t rewritten          = 0;
-	for (auto& block: ir.blocks) {
-		for (auto& inst: block.instructions) {
-			if (IsIrFetchPrologLoad(inst) && EmbeddedFetchPcInList(prolog_pcs, inst.pc)) {
-				auto pc = inst.pc;
-				inst    = {};
-				inst.pc = pc;
-				inst.op = IR::Opcode::ControlNop;
-				continue;
-			}
-
-			if (inst.op != IR::Opcode::BufferLoadDword) {
-				continue;
-			}
-			const auto* load = FindEmbeddedFetchLoad(loads, inst.pc);
-			if (load == nullptr || inst.memory.component_index >= load->components) {
-				continue;
-			}
-
-			const auto resource_id = ResolveEmbeddedFetchResource(input_info, *load);
-			if (resource_id < 0 || resource_id >= input_info->resources_num) {
-				LOGF("ShaderRecompiler VS embedded fetch remap failed: pc=0x%08" PRIx32
-				     " attrib=%d resources=%d\n",
-				     inst.pc, load->attrib_id, input_info->resources_num);
-				continue;
-			}
-
-			inst.op              = IR::Opcode::LoadInputF32;
-			inst.input_info.attr = static_cast<uint32_t>(resource_id);
-			inst.input_info.chan = inst.memory.component_index;
-			inst.memory          = {};
-			inst.src_count       = 0;
-
-			mutable_input_info->resource_fetch_components[resource_id] =
-			    std::max(mutable_input_info->resource_fetch_components[resource_id],
-			             static_cast<int>(inst.input_info.chan + 1u));
-			rewritten++;
-		}
-	}
-
-	return rewritten;
-}
-
 } // namespace
 
 bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
@@ -697,27 +530,25 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	bool        dispatcher_fallback = false;
 	std::string dispatcher_reason;
 	if (cfg.irreducible) {
-		dispatcher_fallback   = true;
-		dispatcher_reason     = cfg.unsupported_reason;
-		const auto diagnostic = FormatCfgFailure(cfg, options, cfg.unsupported_reason);
-		LOGF("%s irreducible CFG detected: %s\n", GetDumpLabel(options), diagnostic.c_str());
+		dispatcher_fallback = true;
+		dispatcher_reason   = cfg.unsupported_reason;
+		LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
 	} else {
 		std::string structure_error;
 		const auto  unstructured_cfg = cfg;
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
 		if (!CFG::Structurize(cfg, &structure_error)) {
-			const auto diagnostic = FormatCfgFailure(cfg, options, structure_error);
-			LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str());
 			dispatcher_fallback      = true;
 			dispatcher_reason        = structure_error;
 			const auto failure_kind  = cfg.failure_kind;
 			const auto failure_block = cfg.failure_block;
-			cfg                      = unstructured_cfg;
-			cfg.unsupported          = true;
-			cfg.failure_kind         = failure_kind;
-			cfg.failure_block        = failure_block;
-			cfg.unsupported_reason   = structure_error;
+			LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
+			cfg                    = unstructured_cfg;
+			cfg.unsupported        = true;
+			cfg.failure_kind       = failure_kind;
+			cfg.failure_block      = failure_block;
+			cfg.unsupported_reason = structure_error;
 		} else {
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(cfg.blocks.size()));
@@ -730,35 +561,70 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	}
 
 	IR::Program ir;
-	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
-	     StageName(options.stage), options.shader_hash);
-	if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, ir, error)) {
+	const ShaderVertexInputInfo*  vertex  = nullptr;
+	const ShaderPixelInputInfo*   pixel   = nullptr;
+	const ShaderComputeInputInfo* compute = nullptr;
+	switch (options.stage) {
+		case ShaderType::Vertex: vertex = options.input_info.vertex; break;
+		case ShaderType::Pixel: pixel = options.input_info.pixel; break;
+		case ShaderType::Compute: compute = options.input_info.compute; break;
+		default: break;
+	}
+	EmbeddedFetchData embedded_fetch;
+	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
+		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, options.user_data_base,
+		                                           options.user_data_count, options.wave_size);
+		if (!embedded_fetch.loads.empty()) {
+			LOGF("%s embedded vertex fetch plan: detected=%" PRIu64 "\n", GetDumpLabel(options),
+			     static_cast<uint64_t>(embedded_fetch.loads.size()));
+		}
+	}
+	Frontend::TranslateOptions translate_options {
+	    .stage               = options.stage,
+	    .wave_size           = options.wave_size,
+	    .shader_hash         = options.shader_hash,
+	    .user_data_base      = options.user_data_base,
+	    .user_data_count     = options.user_data_count,
+	    .scratch_dwords      = options.scratch_dwords,
+	    .dispatcher_fallback = dispatcher_fallback,
+	    .cfg_failure_kind    = cfg.failure_kind,
+	    .fallback_reason     = dispatcher_reason.empty() ? cfg.unsupported_reason
+	                                                    : dispatcher_reason,
+	    .vertex              = vertex,
+	    .pixel               = pixel,
+	    .compute             = compute,
+	    .embedded_fetch      = embedded_fetch.loads.empty() ? nullptr : &embedded_fetch,
+	};
+	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
+	if (!Frontend::TranslateProgram(decoded, cfg, translate_options, ir, error)) {
 		return false;
 	}
-	ir.lane_mask_mode  = options.lane_mask_mode;
-	ir.shader_hash     = options.shader_hash;
-	ir.user_data_base  = options.user_data_base;
-	ir.user_data_count = options.user_data_count;
-	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
+	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR TranslateProgram blocks=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
-	EmbeddedFetchData embedded_fetch;
-	if (options.stage == ShaderType::Vertex && options.vertex_input_info != nullptr &&
-	    options.vertex_input_info->fetch_embedded) {
-		embedded_fetch =
-		    DetectEmbeddedVertexFetch(decoded, options.vertex_input_info, ir.user_data_base,
-		                              ir.user_data_count, options.wave_size);
-		auto rewritten =
-		    RewriteEmbeddedVertexFetches(ir, options.vertex_input_info, embedded_fetch.loads);
-		if (rewritten > 0 || !embedded_fetch.loads.empty()) {
-			LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n",
-			     GetDumpLabel(options), static_cast<uint64_t>(embedded_fetch.loads.size()),
-			     rewritten);
-		}
+	IR::RewriteToSsa(ir.blocks);
+	IR::ConstantPropagationPass(ir.blocks);
+	IR::ResolveControlFlowIdentities(ir);
+	IR::RemoveIdentities(ir.blocks);
+	IR::EliminateDeadCode(ir.blocks);
+	const auto read_lane_stats = IR::EliminateReadLane(ir, ir.wave_size);
+	if (read_lane_stats.rewritten_reads != 0) {
+		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
+		     read_lane_stats.rewritten_reads);
+		IR::ConstantPropagationPass(ir.blocks);
+		IR::ResolveControlFlowIdentities(ir);
+		IR::RemoveIdentities(ir.blocks);
+		IR::EliminateDeadCode(ir.blocks);
 	}
-	if (!IR::BuildScalarProvenance(ir, error)) {
-		return false;
+	if (options.stage == ShaderType::Compute) {
+		const auto lds_barriers =
+		    IR::InsertSharedMemoryBarriers(ir, ir.wave_size, *compute);
+		if (lds_barriers.inserted_barriers != 0) {
+			LOGF("%s wave64 LDS synchronization: barriers=%" PRIu32 "\n", GetDumpLabel(options),
+			     lds_barriers.inserted_barriers);
+		}
 	}
 	std::string srt_error;
 	if (!IR::BuildSrtPlan(ir, &srt_error)) {
@@ -768,29 +634,14 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		}
 		return false;
 	}
-	ir.dispatcher_fallback = dispatcher_fallback;
-	if (!dispatcher_reason.empty()) {
-		ir.fallback_reason = dispatcher_reason;
-	}
-
-	if (!IR::PatchSrtReads(ir, error) || !IR::TrackResources(ir, error)) {
+	IR::EliminateDeadCode(ir.blocks);
+	if (!IR::TrackResources(ir, error)) {
 		return false;
 	}
+	IR::EliminateDeadCode(ir.blocks);
 	if (options.stage == ShaderType::Vertex) {
 		ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
 	}
-	if (!dispatcher_fallback) {
-		std::string emitter_reason;
-		if (NeedsDispatcherForStructuredLoopHeader(ir, &emitter_reason)) {
-			dispatcher_fallback    = true;
-			dispatcher_reason      = emitter_reason;
-			ir.dispatcher_fallback = true;
-			ir.cfg_failure_kind    = CFG::FailureKind::StructuredControlFlow;
-			ir.fallback_reason     = emitter_reason;
-			LOGF("%s structured CFG emitter fallback: %s\n", GetDumpLabel(options),
-			     emitter_reason.c_str());
-		}
-	}
 
 	IR::ResourceSnapshot resources;
 	if (options.resource_snapshot != nullptr) {
@@ -809,11 +660,11 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		runtime.shader_base = options.shader_base != 0 ? options.shader_base
 		                                               : reinterpret_cast<uint64_t>(code.data());
 		runtime.read_memory = options.read_memory;
+		runtime.read_specialization_memory = options.read_specialization_memory;
 		if (runtime.read_memory == nullptr && options.user_data == nullptr) {
 			runtime.read_memory = ReadZeroMemory;
 		}
 		runtime.userdata         = options.read_memory_data;
-		runtime.flat_memory_base = options.flat_memory_base;
 		if (!IR::MaterializeResources(ir, runtime, resources, error)) {
 			return false;
 		}
@@ -822,29 +673,18 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		return false;
 	}
 
-	ShaderVertexInputInfo  default_vertex {};
-	ShaderPixelInputInfo   default_pixel {};
-	ShaderComputeInputInfo default_compute {};
-	IR::ShaderInfoOptions  info_options;
-	info_options.vertex =
-	    options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex;
-	info_options.pixel =
-	    options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel;
-	info_options.compute =
-	    options.compute_input_info != nullptr ? options.compute_input_info : &default_compute;
+	IR::ShaderInfoOptions info_options;
+	info_options.vertex  = vertex;
+	info_options.pixel   = pixel;
+	info_options.compute = compute;
 	if (!IR::CollectShaderInfo(ir, info_options, error)) {
 		return false;
 	}
-	IR::BindingLayoutOptions layout_options;
-	layout_options.descriptor_set       = options.descriptor_set;
-	layout_options.push_constant_offset = options.push_constant_offset;
-	if (!IR::AllocateBindings(ir, layout_options, error)) {
+	if (!IR::AllocateBindings(ir, options.push_constant_offset, error)) {
 		return false;
 	}
-	const auto read_lane_stats = IR::EliminateReadLane(ir);
-	if (read_lane_stats.rewritten_reads != 0) {
-		LOGF("%s read-lane elimination: reads=%" PRIu32 " shadow_writes=%" PRIu32 "\n",
-		     GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.shadow_writes);
+	if (!Spirv::AnalyzeProgramRequirements(ir, error)) {
+		return false;
 	}
 	std::string ir_dump;
 	if (options.dump_ir) {
@@ -858,8 +698,8 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	std::string           emit_error;
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
-	if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info,
-	                        options.compute_input_info, spirv, &emit_error)) {
+	if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error)) {
+		LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str());
 		if (dispatcher_fallback && error != nullptr) {
 			*error = fmt::format("dispatcher fallback failed after {}: {}",
 			                     dispatcher_reason.c_str(), emit_error.c_str());
@@ -873,10 +713,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(spirv.size()), phase_ms());
-	if (dispatcher_fallback) {
-		LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str());
-	}
-
 	result.spirv     = std::move(spirv);
 	result.program   = std::move(ir);
 	result.resources = std::move(resources);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -3,36 +3,32 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 
-#include <optional>
 #include <span>
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler {
 
 struct CompileOptions {
-	ShaderType                    stage           = ShaderType::Compute;
-	ShaderLaneMaskMode            lane_mask_mode  = ShaderLaneMaskMode::NativeWave;
-	uint32_t                      wave_size       = 64;
-	uint32_t                      user_data_base  = 0;
-	uint32_t                      user_data_count = 64;
-	uint64_t                      shader_hash     = 0;
-	uint64_t                      shader_base     = 0;
-	std::optional<uint64_t>       flat_memory_base;
-	uint32_t                      descriptor_set       = 0;
-	uint32_t                      push_constant_offset = 0;
-	bool                          dump_ir              = true;
-	bool                          early_dump           = false;
-	const char*                   dump_label           = nullptr;
-	const uint32_t*               user_data            = nullptr;
-	IR::SrtMemoryReader           read_memory          = nullptr;
-	void*                         read_memory_data     = nullptr;
-	const IR::ResourceSnapshot*   resource_snapshot    = nullptr;
-	const ShaderVertexInputInfo*  vertex_input_info    = nullptr;
-	const ShaderPixelInputInfo*   pixel_input_info     = nullptr;
-	const ShaderComputeInputInfo* compute_input_info   = nullptr;
+	ShaderType                  stage           = ShaderType::Compute;
+	uint32_t                    wave_size       = 64;
+	uint32_t                    user_data_base  = 0;
+	uint32_t                    user_data_count = 64;
+	uint32_t                    scratch_dwords  = 0;
+	uint64_t                    shader_hash     = 0;
+	uint64_t                    shader_base     = 0;
+	uint32_t                    push_constant_offset      = 0;
+	bool                        dump_ir                    = true;
+	bool                        early_dump                 = false;
+	const char*                 dump_label                 = nullptr;
+	const uint32_t*             user_data                  = nullptr;
+	IR::SrtMemoryReader         read_memory                = nullptr;
+	IR::SrtMemoryReader         read_specialization_memory = nullptr;
+	void*                       read_memory_data           = nullptr;
+	const IR::ResourceSnapshot* resource_snapshot          = nullptr;
+	ShaderStageInputInfo        input_info;
 };
 
 struct CompileResult {

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -0,0 +1,270 @@
+#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
+
+#include "common/debug.h"
+
+#include <cstring>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv {
+
+static constexpr size_t InitialSpirvSectionReserve         = 4096;
+static constexpr size_t InitialSpirvFunctionSectionReserve = 32768;
+
+static void AppendInstructionWords(std::vector<uint32_t>& section, const uint32_t* words,
+                                   size_t words_num) {
+	if (words_num == 0) {
+		return;
+	}
+	const auto opcode     = words[0];
+	const auto word_count = static_cast<uint32_t>(words_num);
+	section.push_back((word_count << 16u) | opcode);
+	section.insert(section.end(), words + 1, words + words_num);
+}
+
+Builder::Builder(uint32_t version): m_version(version) {
+	m_debug.reserve(InitialSpirvSectionReserve);
+	m_annotations.reserve(InitialSpirvSectionReserve);
+	m_declarations.reserve(InitialSpirvSectionReserve);
+	m_functions.reserve(InitialSpirvFunctionSectionReserve);
+}
+
+uint32_t Builder::AllocateId() {
+	return m_next_id++;
+}
+
+void Builder::RequireCapability(uint32_t capability) {
+	if (m_required_capabilities.insert(capability).second) {
+		AddCapability({capability});
+	}
+}
+
+void Builder::RequireExtension(const char* name) {
+	if (m_required_extensions.emplace(name).second) {
+		AddExtension(name);
+	}
+}
+
+uint32_t Builder::Import(const char* name) {
+	if (const auto it = m_import_ids.find(name); it != m_import_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_import_ids.emplace(name, id);
+	AddExtInstImport(id, name);
+	return id;
+}
+
+uint32_t Builder::Type(uint32_t opcode, std::initializer_list<uint32_t> operands) {
+	return Type(opcode, std::vector<uint32_t>(operands));
+}
+
+uint32_t Builder::Type(uint32_t opcode, const std::vector<uint32_t>& operands) {
+	std::vector<uint32_t> key;
+	key.reserve(operands.size() + 2u);
+	key.push_back(opcode);
+	key.push_back(static_cast<uint32_t>(operands.size()));
+	key.insert(key.end(), operands.begin(), operands.end());
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	return id;
+}
+
+uint32_t Builder::DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+                                std::initializer_list<TypeAnnotation> annotations) {
+	if (annotations.size() == 0) {
+		return Type(opcode, operands);
+	}
+	std::vector<uint32_t> key {opcode, static_cast<uint32_t>(operands.size())};
+	key.insert(key.end(), operands.begin(), operands.end());
+	key.push_back(static_cast<uint32_t>(annotations.size()));
+	for (const auto& annotation: annotations) {
+		key.push_back(annotation.opcode);
+		key.push_back(static_cast<uint32_t>(annotation.operands.size()));
+		key.insert(key.end(), annotation.operands.begin(), annotation.operands.end());
+	}
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	for (const auto& annotation: annotations) {
+		words = {annotation.opcode, id};
+		words.insert(words.end(), annotation.operands.begin(), annotation.operands.end());
+		AppendInstructionWords(m_annotations, words.data(), words.size());
+	}
+	return id;
+}
+
+uint32_t Builder::Constant(uint32_t opcode, uint32_t type,
+                           std::initializer_list<uint32_t> operands) {
+	return Constant(opcode, type, std::vector<uint32_t>(operands));
+}
+
+uint32_t Builder::Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands) {
+	std::vector<uint32_t> key;
+	key.reserve(operands.size() + 2u);
+	key.push_back(opcode);
+	key.push_back(type);
+	key.insert(key.end(), operands.begin(), operands.end());
+	if (const auto it = m_declaration_ids.find(key); it != m_declaration_ids.end()) {
+		return it->second;
+	}
+	const auto id = AllocateId();
+	m_declaration_ids.emplace(std::move(key), id);
+	std::vector<uint32_t> words {opcode, type, id};
+	words.insert(words.end(), operands.begin(), operands.end());
+	AppendInstructionWords(m_declarations, words.data(), words.size());
+	return id;
+}
+
+uint32_t Builder::DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class) {
+	const auto id = AllocateId();
+	DefineGlobalVariable(id, pointer_type, storage_class);
+	return id;
+}
+
+void Builder::DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class) {
+	AddType({59u, pointer_type, id, storage_class});
+}
+
+void Builder::AppendString(std::vector<uint32_t>& words, const char* text) {
+	const auto len        = text != nullptr ? std::strlen(text) : 0;
+	const auto word_count = (len + 1u + 3u) / 4u;
+	for (size_t i = 0; i < word_count; i++) {
+		uint32_t word = 0;
+		for (size_t byte = 0; byte < 4; byte++) {
+			const auto index = i * 4u + byte;
+			if (index < len) {
+				word |= static_cast<uint32_t>(static_cast<unsigned char>(text[index]))
+				        << (byte * 8u);
+			}
+		}
+		words.push_back(word);
+	}
+}
+
+void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+                                const std::vector<uint32_t>& operands) {
+	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
+	section.push_back((word_count << 16u) | opcode);
+	section.insert(section.end(), operands.begin(), operands.end());
+}
+
+void Builder::AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+                                std::initializer_list<uint32_t> operands) {
+	const uint32_t word_count = static_cast<uint32_t>(operands.size() + 1u);
+	section.push_back((word_count << 16u) | opcode);
+	section.insert(section.end(), operands.begin(), operands.end());
+}
+
+void Builder::AddCapability(std::initializer_list<uint32_t> operands) {
+	AppendInstruction(m_capabilities, 17u, operands);
+}
+
+void Builder::AddExtension(const char* name) {
+	std::vector<uint32_t> operands;
+	AppendString(operands, name);
+	AppendInstruction(m_extensions, 10u, operands);
+}
+
+void Builder::AddExtInstImport(uint32_t id, const char* name) {
+	std::vector<uint32_t> operands = {id};
+	AppendString(operands, name);
+	AppendInstruction(m_ext_inst_imports, 11u, operands);
+}
+
+void Builder::AddMemoryModel(std::initializer_list<uint32_t> operands) {
+	AppendInstruction(m_memory_model, 14u, operands);
+}
+
+void Builder::AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
+                            const std::vector<uint32_t>& interfaces) {
+	std::vector<uint32_t> operands = {execution_model, entry_point};
+	AppendString(operands, name);
+	operands.insert(operands.end(), interfaces.begin(), interfaces.end());
+	AppendInstruction(m_entry_points, 15u, operands);
+}
+
+void Builder::AddExecutionMode(std::initializer_list<uint32_t> operands) {
+	AppendInstruction(m_execution_modes, 16u, operands);
+}
+
+void Builder::AddName(uint32_t target, const char* name) {
+	std::vector<uint32_t> operands = {target};
+	AppendString(operands, name);
+	AppendInstruction(m_debug, 5u, operands);
+}
+
+void Builder::AddAnnotation(std::initializer_list<uint32_t> words) {
+	AppendInstructionWords(m_annotations, words.begin(), words.size());
+}
+
+void Builder::AddType(std::initializer_list<uint32_t> words) {
+	AppendInstructionWords(m_declarations, words.begin(), words.size());
+}
+
+void Builder::AddFunction(std::initializer_list<uint32_t> words) {
+	AppendInstructionWords(m_functions, words.begin(), words.size());
+}
+
+void Builder::AddFunction(const std::vector<uint32_t>& words) {
+	AppendInstructionWords(m_functions, words.data(), words.size());
+}
+
+DeferredPhi Builder::AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count) {
+	std::vector<uint32_t> words {245u, type, result};
+	words.resize(words.size() + incoming_count * 2u);
+	const DeferredPhi phi {m_functions.size(), incoming_count};
+	AddFunction(words);
+	m_unpatched_phi_incomings += incoming_count;
+	return phi;
+}
+
+void Builder::PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent) {
+	EXIT_IF(incoming >= phi.incoming_count || value == 0 || parent == 0);
+	const auto value_word  = phi.word_offset + 3u + incoming * 2u;
+	const auto parent_word = value_word + 1u;
+	EXIT_IF(m_functions.at(value_word) != 0 || m_functions.at(parent_word) != 0);
+	m_functions[value_word]  = value;
+	m_functions[parent_word] = parent;
+	m_unpatched_phi_incomings--;
+}
+
+std::vector<uint32_t> Builder::Build() const {
+	EXIT_IF(m_unpatched_phi_incomings != 0);
+
+	std::vector<uint32_t> module;
+	module.reserve(5u + m_capabilities.size() + m_extensions.size() + m_ext_inst_imports.size() +
+	               m_memory_model.size() + m_entry_points.size() + m_execution_modes.size() +
+	               m_debug.size() + m_annotations.size() + m_declarations.size() +
+	               m_functions.size());
+
+	module.push_back(0x07230203u);
+	module.push_back(m_version);
+	module.push_back(0u);
+	module.push_back(m_next_id);
+	module.push_back(0u);
+
+	module.insert(module.end(), m_capabilities.begin(), m_capabilities.end());
+	module.insert(module.end(), m_extensions.begin(), m_extensions.end());
+	module.insert(module.end(), m_ext_inst_imports.begin(), m_ext_inst_imports.end());
+	module.insert(module.end(), m_memory_model.begin(), m_memory_model.end());
+	module.insert(module.end(), m_entry_points.begin(), m_entry_points.end());
+	module.insert(module.end(), m_execution_modes.begin(), m_execution_modes.end());
+	module.insert(module.end(), m_debug.begin(), m_debug.end());
+	module.insert(module.end(), m_annotations.begin(), m_annotations.end());
+	module.insert(module.end(), m_declarations.begin(), m_declarations.end());
+	module.insert(module.end(), m_functions.begin(), m_functions.end());
+
+	return module;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.h

@@ -0,0 +1,89 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_
+
+#include "common/common.h"
+
+#include <initializer_list>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv {
+
+struct TypeAnnotation {
+	uint32_t              opcode = 0;
+	std::vector<uint32_t> operands;
+};
+
+struct DeferredPhi {
+	size_t word_offset    = 0;
+	size_t incoming_count = 0;
+};
+
+class Builder {
+public:
+	explicit Builder(uint32_t version = 0x00010300u);
+	~Builder() = default;
+	KYTY_CLASS_DEFAULT_COPY(Builder);
+
+	uint32_t AllocateId();
+	void     RequireCapability(uint32_t capability);
+	void     RequireExtension(const char* name);
+	uint32_t Import(const char* name);
+	uint32_t Type(uint32_t opcode, std::initializer_list<uint32_t> operands = {});
+	uint32_t Type(uint32_t opcode, const std::vector<uint32_t>& operands);
+	uint32_t DecoratedType(uint32_t opcode, std::initializer_list<uint32_t> operands,
+	                       std::initializer_list<TypeAnnotation> annotations);
+	uint32_t Constant(uint32_t opcode, uint32_t type,
+	                  std::initializer_list<uint32_t> operands = {});
+	uint32_t Constant(uint32_t opcode, uint32_t type, const std::vector<uint32_t>& operands);
+	uint32_t DefineGlobalVariable(uint32_t pointer_type, uint32_t storage_class);
+	void     DefineGlobalVariable(uint32_t id, uint32_t pointer_type, uint32_t storage_class);
+
+	void        AddMemoryModel(std::initializer_list<uint32_t> operands);
+	void        AddEntryPoint(uint32_t execution_model, uint32_t entry_point, const char* name,
+	                          const std::vector<uint32_t>& interfaces);
+	void        AddExecutionMode(std::initializer_list<uint32_t> operands);
+	void        AddName(uint32_t target, const char* name);
+	void        AddAnnotation(std::initializer_list<uint32_t> words);
+	void        AddFunction(std::initializer_list<uint32_t> words);
+	void        AddFunction(const std::vector<uint32_t>& words);
+	DeferredPhi AddDeferredPhi(uint32_t type, uint32_t result, size_t incoming_count);
+	void        PatchDeferredPhi(DeferredPhi phi, size_t incoming, uint32_t value, uint32_t parent);
+
+	[[nodiscard]] std::vector<uint32_t> Build() const;
+
+private:
+	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+	                              const std::vector<uint32_t>& operands);
+	static void AppendInstruction(std::vector<uint32_t>& section, uint32_t opcode,
+	                              std::initializer_list<uint32_t> operands);
+	static void AppendString(std::vector<uint32_t>& words, const char* text);
+	void        AddCapability(std::initializer_list<uint32_t> operands);
+	void        AddExtension(const char* name);
+	void        AddExtInstImport(uint32_t id, const char* name);
+	void        AddType(std::initializer_list<uint32_t> words);
+
+	uint32_t                                  m_next_id = 1;
+	uint32_t                                  m_version = 0;
+	std::vector<uint32_t>                     m_capabilities;
+	std::vector<uint32_t>                     m_extensions;
+	std::vector<uint32_t>                     m_ext_inst_imports;
+	std::vector<uint32_t>                     m_memory_model;
+	std::vector<uint32_t>                     m_entry_points;
+	std::vector<uint32_t>                     m_execution_modes;
+	std::vector<uint32_t>                     m_debug;
+	std::vector<uint32_t>                     m_annotations;
+	std::vector<uint32_t>                     m_declarations;
+	std::vector<uint32_t>                     m_functions;
+	std::set<uint32_t>                        m_required_capabilities;
+	std::set<std::string>                     m_required_extensions;
+	std::map<std::string, uint32_t>           m_import_ids;
+	std::map<std::vector<uint32_t>, uint32_t> m_declaration_ids;
+	size_t                                    m_unpatched_phi_incomings = 0;
+};
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVBUILDER_H_ */

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -0,0 +1,352 @@
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/shader.h"
+
+#include <algorithm>
+#include <array>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv {
+
+using ShaderError::Fail;
+
+namespace {
+
+bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
+	using Kind                                             = IR::DescriptorBindingKind;
+	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
+	std::array<std::vector<uint32_t>, KindCount> expected;
+	std::array<bool, KindCount>                  present {};
+	const auto                                   Dense = [](size_t size) {
+		std::vector<uint32_t> values(size);
+		for (uint32_t i = 0; i < values.size(); i++) {
+			values[i] = i;
+		}
+		return values;
+	};
+	auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
+		const auto index = static_cast<size_t>(kind);
+		present[index]   = true;
+		expected[index]  = std::move(resources);
+	};
+	if (!program.info.buffers.empty()) {
+		Expect(Kind::Buffers, Dense(program.info.buffers.size()));
+	}
+	for (uint32_t i = 0; i < program.info.images.size(); i++) {
+		const auto kind = IR::DescriptorBindingForImage(program.info.images[i]);
+		if (!kind.has_value()) {
+			return Fail(error, "native shader plan has an invalid image class");
+		}
+		present[static_cast<size_t>(*kind)] = true;
+		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage;
+		const auto count   = dynamic ? program.info.images[i].mip_count : 1u;
+		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
+			return Fail(error, "native shader plan has an invalid image mip descriptor count");
+		}
+		expected[static_cast<size_t>(*kind)].insert(expected[static_cast<size_t>(*kind)].end(),
+		                                            count, i);
+	}
+	if (!program.info.samplers.empty()) {
+		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
+	}
+	bool uses_gds = false;
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			if (IR::SharedAccessOf(inst.GetOpcode()) == IR::SharedAccess::None) {
+				continue;
+			}
+			const auto index = inst.Flags<IR::MemoryFlags>().index;
+			if (index >= program.memory_info.size()) {
+				return Fail(error, "shared operation has invalid memory metadata");
+			}
+			const auto kind = program.memory_info[index].kind;
+			if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+				return Fail(error, "shared operation has invalid resource kind");
+			}
+			uses_gds |= kind == IR::ResourceKind::Gds;
+		}
+	}
+	if (uses_gds) {
+		Expect(Kind::Gds);
+	}
+	if (program.info.uses_dma) {
+		Expect(Kind::BdaPagetable);
+		Expect(Kind::FaultBuffer);
+	}
+	const bool uses_flattened_runtime =
+	    !program.srt_reads.empty() ||
+	     std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
+		     return image.indirect_mapping_capacity != 0u;
+	     });
+	if (uses_flattened_runtime) {
+		Expect(Kind::FlattenedSrt);
+	}
+	if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
+		Expect(Kind::UserData);
+	}
+
+	std::array<bool, KindCount> seen {};
+	for (const auto& binding: program.bindings.descriptors) {
+		const auto kind = static_cast<size_t>(binding.kind);
+		if (kind >= KindCount || seen[kind] || !present[kind] ||
+		    binding.resources != expected[kind]) {
+			return Fail(error, "native descriptor groups do not match shader topology");
+		}
+		seen[kind] = true;
+	}
+	for (size_t i = 0; i < KindCount; i++) {
+		if (present[i] != seen[i]) {
+			return Fail(error, "native shader plan is missing a required descriptor group");
+		}
+	}
+	const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
+	if (program.bindings.push_constant_offset % sizeof(uint32_t) != 0 ||
+	    program.bindings.push_constant_offset + program.bindings.push_constant_size >
+	        IR::NativePushConstantSize ||
+	    program.bindings.memory_offset_dword != program.bindings.user_data_registers.size() ||
+	    program.bindings.memory_offset_count != program.info.buffers.size() ||
+	    (!has_user_storage &&
+	     program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
+	    (has_user_storage && program.bindings.push_constant_size != 0) ||
+	    !std::is_sorted(program.bindings.user_data_registers.begin(),
+	                    program.bindings.user_data_registers.end()) ||
+	    std::adjacent_find(program.bindings.user_data_registers.begin(),
+	                       program.bindings.user_data_registers.end()) !=
+	        program.bindings.user_data_registers.end()) {
+		return Fail(error, "native user-data layout is inconsistent");
+	}
+
+	const auto planning_only_handle = [&](const IR::Inst& handle) {
+		return !handle.Uses().empty() &&
+		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
+			       const auto op = use.user->GetOpcode();
+			       if (op != IR::ValueOpcode::LoadAddressU32 &&
+			           op != IR::ValueOpcode::ReadConstBuffer) {
+				       return false;
+			       }
+			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
+			       return index < program.memory_info.size() &&
+			              program.memory_info[index].planning_only;
+		       });
+	};
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			const auto dense = inst.Flags<uint32_t>();
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::GetBufferResource:
+					if (planning_only_handle(inst)) {
+						break;
+					}
+					if (dense >= program.info.buffers.size()) {
+						return Fail(error, "typed buffer handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::GetAddressResource:
+					if (planning_only_handle(inst)) {
+						break;
+					}
+					if (inst.NumArgs() != 2 || !program.info.uses_dma) {
+						return Fail(error, "typed address handle has invalid DMA metadata");
+					}
+					break;
+				case IR::ValueOpcode::GetScratchResource:
+					if (inst.NumArgs() != 0 || program.scratch_dwords == 0) {
+						return Fail(error, "typed scratch handle has invalid shader metadata");
+					}
+					break;
+				case IR::ValueOpcode::GetImageResource:
+					if (dense >= program.info.images.size()) {
+						return Fail(error, "typed image handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::GetSamplerResource:
+					if (dense >= program.info.samplers.size()) {
+						return Fail(error, "typed sampler handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::ReadConst: {
+					const auto slot = inst.Arg(1).Resolve();
+					if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
+					    slot.U32() >= program.srt_reads.size()) {
+						return Fail(error, "flattened SRT read has an invalid dense slot");
+					}
+					break;
+				}
+				default: break;
+			}
+		}
+	}
+	return true;
+}
+
+} // namespace
+
+bool AnalyzeProgramRequirements(IR::Program& program, std::string* error) {
+	program.spirv_requirements.reset();
+	IR::SpirvRequirements requirements {};
+	const auto MarkBallot = [&] { requirements.subgroup_ballot = true; };
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			const auto address_access = IR::AddressOpcodeInfoOf(inst.GetOpcode()).access;
+			if (address_access != IR::AddressAccess::None) {
+				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
+				if (memory_index >= program.memory_info.size()) {
+					return Fail(error, "address operation has invalid memory metadata");
+				}
+				if (program.memory_info[memory_index].kind == IR::ResourceKind::Scratch) {
+					if (program.scratch_dwords == 0) {
+						return Fail(error, "scratch operation has no per-thread storage");
+					}
+					requirements.function_scratch = true;
+				} else if (address_access == IR::AddressAccess::Write) {
+					return Fail(error, "writable FLAT/GLOBAL addresses require GPU ownership tracking");
+				}
+			}
+			if (IR::BufferAccessOf(inst.GetOpcode()) != IR::BufferAccess::None) {
+				const auto memory_index = inst.Flags<IR::MemoryFlags>().index;
+				if (memory_index >= program.memory_info.size()) {
+					return Fail(error, "buffer operation has invalid memory metadata");
+				}
+				const auto& memory = program.memory_info[memory_index];
+				if (memory.kind == IR::ResourceKind::Buffer) {
+					if (memory.resource >= program.info.buffers.size()) {
+						return Fail(error, "buffer operation has invalid resource metadata");
+					}
+					if ((program.info.buffers[memory.resource].packed_stride & (1u << 20u)) != 0u) {
+						if (program.stage != ShaderType::Compute) {
+							return Fail(error, "buffer ADD_TID is only valid for compute shaders");
+						}
+						requirements.subgroup_local_invocation_id = true;
+					}
+				}
+			}
+			const auto shared_access = IR::SharedAccessOf(inst.GetOpcode());
+			if (shared_access != IR::SharedAccess::None) {
+				const auto index = inst.Flags<IR::MemoryFlags>().index;
+				if (index >= program.memory_info.size()) {
+					return Fail(error, "shared operation has invalid memory metadata");
+				}
+				const auto kind = program.memory_info[index].kind;
+				if (kind != IR::ResourceKind::Lds && kind != IR::ResourceKind::Gds) {
+					return Fail(error, "shared operation has invalid resource kind");
+				}
+				if (program.stage != ShaderType::Compute && kind == IR::ResourceKind::Lds) {
+					requirements.function_lds = true;
+				}
+				if (shared_access == IR::SharedAccess::Append ||
+				    shared_access == IR::SharedAccess::Consume) {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+				}
+			}
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::Ballot: MarkBallot(); break;
+				case IR::ValueOpcode::DppMoveU32:
+				case IR::ValueOpcode::ReadFirstLane:
+				case IR::ValueOpcode::ReadLane: {
+					MarkBallot();
+					requirements.subgroup_shuffle = true;
+					if (inst.GetOpcode() == IR::ValueOpcode::DppMoveU32) {
+						requirements.subgroup_local_invocation_id = true;
+					}
+					break;
+				}
+				case IR::ValueOpcode::DppUpdateU32:
+				case IR::ValueOpcode::WqmMask:
+				case IR::ValueOpcode::WriteLane: {
+					MarkBallot();
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::Permlane16U32: {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::SwizzleU32: {
+					MarkBallot();
+					requirements.subgroup_shuffle             = true;
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				}
+				case IR::ValueOpcode::LaneId:
+					requirements.subgroup_local_invocation_id = true;
+					break;
+				case IR::ValueOpcode::ImageQueryLod: requirements.compute_derivatives = true; break;
+				case IR::ValueOpcode::ImageGatherRaw:
+					requirements.image_gather_extended = true;
+					break;
+				case IR::ValueOpcode::SetAttribute: {
+					const auto index = inst.Flags<IR::ExportFlags>().index;
+					if (index >= program.export_info.size()) {
+						return Fail(error, "attribute export has invalid metadata");
+					}
+					if (program.stage == ShaderType::Pixel &&
+					    program.export_info[index].vm) {
+						requirements.pixel_valid_mask = true;
+					}
+					break;
+				}
+				default: break;
+			}
+		}
+	}
+	program.spirv_requirements.emplace(requirements);
+	return true;
+}
+
+bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
+                 ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv,
+                 std::string* error) {
+	using namespace Emitter;
+
+	if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
+	    program.stage != ShaderType::Pixel) {
+		SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
+		return false;
+	}
+	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
+	    !program.shader_info_complete || !program.binding_layout_complete ||
+	    !program.spirv_requirements.has_value()) {
+		SetError(error, "SPIR-V emitter requires a fully planned native shader program");
+		return false;
+	}
+	if (!IR::ValidateResourceSnapshot(program, resources, error)) {
+		return false;
+	}
+	if (!IR::ValidateResourceSpecialization(program, resources, error)) {
+		return false;
+	}
+	if (!ValidateNativeProgram(program, error)) {
+		return false;
+	}
+	if (!IR::ValidateProgram(program, true, error)) {
+		return false;
+	}
+	EmitterState state(program, resources, input_info);
+	state.stage     = program.stage;
+	state.wave_size = program.wave_size;
+	state.inputs.reserve(program.info.inputs.size());
+	state.outputs.reserve(program.info.outputs.size());
+	state.interface_variables.reserve(program.info.inputs.size() + program.info.outputs.size());
+	CopyProgramInputsAndOutputs(state, program);
+	AllocateInputVariables(state);
+	AllocateOutputVariables(state);
+	DefineModule(state);
+	if (!EmitProgram(state, program, error)) {
+		return false;
+	}
+
+	auto binary = state.builder.Build();
+	if (binary.empty()) {
+		SetError(error, "SPIR-V builder returned an empty module");
+		return false;
+	}
+	spirv = std::move(binary);
+	return true;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -3,19 +3,16 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
 #include <vector>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv {
 
-bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
+bool AnalyzeProgramRequirements(IR::Program& program, std::string* error);
 
 bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
-                 const ShaderVertexInputInfo*  vertex_input_info,
-                 const ShaderPixelInputInfo*   pixel_input_info,
-                 const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
-                 std::string* error);
+                 ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -0,0 +1,690 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <array>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+struct Pair {
+	uint32_t low  = 0;
+	uint32_t high = 0;
+};
+
+uint32_t NewUnary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, value});
+	return result;
+}
+
+uint32_t NewBinary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
+                   uint32_t rhs) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, lhs, rhs});
+	return result;
+}
+
+uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
+                   uint32_t false_value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
+	return result;
+}
+
+Pair ExtractPair(EmitterState& state, uint32_t value) {
+	Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.low, value, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), result.high, value, 1});
+	return result;
+}
+
+uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeConstruct, TypeU64(state), result, low, high});
+	return result;
+}
+
+uint32_t CompareEqual64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                        bool not_equal) {
+	const auto compare = NewBinary(state, not_equal ? OpINotEqual : OpIEqual,
+	                               TypeBoolVector(state, 2), lhs_value, rhs_value);
+	return NewUnary(state, not_equal ? OpAny : OpAll, TypeBool(state), compare);
+}
+
+uint32_t CompareOrdered64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                          uint32_t high_compare, uint32_t low_compare) {
+	const auto lhs         = ExtractPair(state, lhs_value);
+	const auto rhs         = ExtractPair(state, rhs_value);
+	const auto high_equal  = NewBinary(state, OpIEqual, TypeBool(state), lhs.high, rhs.high);
+	const auto high_result = NewBinary(state, high_compare, TypeBool(state), lhs.high, rhs.high);
+	const auto low_result  = NewBinary(state, low_compare, TypeBool(state), lhs.low, rhs.low);
+	const auto low_path = NewBinary(state, OpLogicalAnd, TypeBool(state), high_equal, low_result);
+	return NewBinary(state, OpLogicalOr, TypeBool(state), high_result, low_path);
+}
+
+uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
+	const auto lhs      = ExtractPair(state, lhs_value);
+	const auto rhs      = ExtractPair(state, rhs_value);
+	const auto low_pair = state.builder.AllocateId();
+	const auto low      = state.builder.AllocateId();
+	const auto carry    = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAddCarry, TypeU32Pair(state), low_pair, lhs.low, rhs.low});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, low_pair, 0});
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), carry, low_pair, 1});
+	const auto high0 = NewBinary(state, OpIAdd, TypeU32(state), lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpIAdd, TypeU32(state), high0, carry);
+	return MakePair(state, low, high);
+}
+
+uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
+	const auto lhs    = ExtractPair(state, lhs_value);
+	const auto rhs    = ExtractPair(state, rhs_value);
+	const auto low    = NewBinary(state, OpISub, TypeU32(state), lhs.low, rhs.low);
+	const auto borrow = NewBinary(state, OpULessThan, TypeBool(state), lhs.low, rhs.low);
+	const auto borrow_u32 =
+	    NewSelect(state, TypeU32(state), borrow, ConstantU32(state, 1), ConstantU32(state, 0));
+	const auto high0 = NewBinary(state, OpISub, TypeU32(state), lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpISub, TypeU32(state), high0, borrow_u32);
+	return MakePair(state, low, high);
+}
+
+uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
+	const auto operand_type = signed_value ? TypeI32(state) : TypeU32(state);
+	const auto pair_type    = signed_value ? TypeI32Pair(state) : TypeU32Pair(state);
+	uint32_t   lhs_operand  = lhs;
+	uint32_t   rhs_operand  = rhs;
+	if (signed_value) {
+		lhs_operand = NewUnary(state, OpBitcast, TypeI32(state), lhs);
+		rhs_operand = NewUnary(state, OpBitcast, TypeI32(state), rhs);
+	}
+	const auto extended = state.builder.AllocateId();
+	state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended,
+	                           lhs_operand, rhs_operand});
+	const auto high = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1});
+	return signed_value ? NewUnary(state, OpBitcast, TypeU32(state), high) : high;
+}
+
+uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
+	const auto lhs   = ExtractPair(state, lhs_value);
+	const auto rhs   = ExtractPair(state, rhs_value);
+	const auto low   = NewBinary(state, OpIMul, TypeU32(state), lhs.low, rhs.low);
+	const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
+	const auto high1 = NewBinary(state, OpIMul, TypeU32(state), lhs.low, rhs.high);
+	const auto high2 = NewBinary(state, OpIMul, TypeU32(state), lhs.high, rhs.low);
+	return MakePair(state, low,
+	                NewBinary(state, OpIAdd, TypeU32(state),
+	                          NewBinary(state, OpIAdd, TypeU32(state), high0, high1), high2));
+}
+
+uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
+	const auto pair = ExtractPair(state, value);
+	uint32_t   low  = 0;
+	uint32_t   high = 0;
+	if (opcode == OpShiftLeftLogical) {
+		EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high);
+	} else if (opcode == OpShiftRightLogical) {
+		EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
+	} else {
+		const auto amount =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), shift, ConstantU32(state, 63));
+		const auto word_shift =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state), amount, ConstantU32(state, 31));
+		const auto at_least_32 =
+		    NewBinary(state, OpUGreaterThanEqual, TypeBool(state), amount, ConstantU32(state, 32));
+		const auto nonzero =
+		    NewBinary(state, OpINotEqual, TypeBool(state), amount, ConstantU32(state, 0));
+		const auto high_shifted =
+		    NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high, word_shift);
+		const auto low_shifted =
+		    NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low, word_shift);
+		const auto carry_count =
+		    NewBinary(state, OpBitwiseAnd, TypeU32(state),
+		              NewBinary(state, OpISub, TypeU32(state), ConstantU32(state, 32), word_shift),
+		              ConstantU32(state, 31));
+		const auto carry =
+		    NewSelect(state, TypeU32(state), nonzero,
+		              NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high, carry_count),
+		              ConstantU32(state, 0));
+		const auto low_below_32 = NewBinary(state, OpBitwiseOr, TypeU32(state), low_shifted, carry);
+		const auto negative =
+		    NewBinary(state, OpSLessThan, TypeBool(state), pair.high, ConstantU32(state, 0));
+		const auto sign_fill = NewSelect(state, TypeU32(state), negative,
+		                                 ConstantU32(state, 0xffffffffu), ConstantU32(state, 0));
+		return MakePair(state,
+		                NewSelect(state, TypeU32(state), at_least_32, high_shifted, low_below_32),
+		                NewSelect(state, TypeU32(state), at_least_32, sign_fill, high_shifted));
+	}
+	return MakePair(state, low, high);
+}
+
+uint32_t EmitConstantShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
+	shift &= 63u;
+	if (shift == 0u) {
+		return value;
+	}
+	const auto pair = ExtractPair(state, value);
+	if (opcode == OpShiftLeftLogical) {
+		if (shift < 32u) {
+			const auto low  = NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.low,
+			                            ConstantU32(state, shift));
+			const auto high = NewBinary(state, OpBitwiseOr, TypeU32(state),
+			                            NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+			                                      pair.high, ConstantU32(state, shift)),
+			                            NewBinary(state, OpShiftRightLogical, TypeU32(state),
+			                                      pair.low, ConstantU32(state, 32u - shift)));
+			return MakePair(state, low, high);
+		}
+		return MakePair(state, ConstantU32(state, 0),
+		                shift == 32u ? pair.low
+		                             : NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+		                                         pair.low, ConstantU32(state, shift - 32u)));
+	}
+	if (shift < 32u) {
+		const auto low = NewBinary(state, OpBitwiseOr, TypeU32(state),
+		                           NewBinary(state, OpShiftRightLogical, TypeU32(state), pair.low,
+		                                     ConstantU32(state, shift)),
+		                           NewBinary(state, OpShiftLeftLogical, TypeU32(state), pair.high,
+		                                     ConstantU32(state, 32u - shift)));
+		const auto high =
+		    NewBinary(state, opcode, TypeU32(state), pair.high, ConstantU32(state, shift));
+		return MakePair(state, low, high);
+	}
+	const auto high = opcode == OpShiftRightArithmetic
+	                      ? NewBinary(state, OpShiftRightArithmetic, TypeU32(state), pair.high,
+	                                  ConstantU32(state, 31u))
+	                      : ConstantU32(state, 0);
+	const auto low  = shift == 32u ? pair.high
+	                               : NewBinary(state, opcode, TypeU32(state), pair.high,
+	                                           ConstantU32(state, shift - 32u));
+	return MakePair(state, low, high);
+}
+
+uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
+	const auto pair   = ExtractPair(state, value);
+	const auto high_i = state.builder.AllocateId();
+	const auto low_i  = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, TypeI32(state), high_i, GlslStd450(state), GlslFindUMsb, pair.high});
+	state.builder.AddFunction(
+	    {OpExtInst, TypeI32(state), low_i, GlslStd450(state), GlslFindUMsb, pair.low});
+	const auto high = NewUnary(state, OpBitcast, TypeU32(state), high_i);
+	const auto low  = NewUnary(state, OpBitcast, TypeU32(state), low_i);
+	const auto high_nonzero =
+	    NewBinary(state, OpINotEqual, TypeBool(state), pair.high, ConstantU32(state, 0));
+	return NewSelect(state, TypeU32(state), high_nonzero,
+	                 NewBinary(state, OpIAdd, TypeU32(state), high, ConstantU32(state, 32)), low);
+}
+
+uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
+                     bool max_value) {
+	const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
+	                             : EmitMinMaxU32Value(state, a, b, max_value);
+	return signed_value ? EmitMinMaxI32Value(state, ab, c, max_value)
+	                    : EmitMinMaxU32Value(state, ab, c, max_value);
+}
+
+uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
+	const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
+	const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
+	const auto ab      = NewBinary(state, OpIAdd, TypeU32(state), a, b);
+	const auto abc     = NewBinary(state, OpIAdd, TypeU32(state), ab, c);
+	return NewBinary(state, OpISub, TypeU32(state),
+	                 NewBinary(state, OpISub, TypeU32(state), abc, minimum), maximum);
+}
+
+uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
+	return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
+}
+
+uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
+	const auto min_ab   = EmitMinMaxF32Value(state, a, b, false);
+	const auto min3     = EmitMinMaxF32Value(state, min_ab, c, false);
+	const auto max_ab   = EmitMinMaxF32Value(state, a, b, true);
+	const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
+	const auto median   = EmitMinMaxF32Value(state, min_ab, high_min, true);
+	const auto nan_ab   = NewBinary(state, OpLogicalOr, TypeBool(state),
+	                                EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
+	const auto any_nan =
+	    NewBinary(state, OpLogicalOr, TypeBool(state), nan_ab, EmitClassifyF32(state, c).nan);
+	return NewSelect(state, TypeF32(state), any_nan, min3, median);
+}
+
+uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
+                 std::initializer_list<uint32_t> args) {
+	const auto            result = state.builder.AllocateId();
+	std::vector<uint32_t> words {OpExtInst, type, result, GlslStd450(state), opcode};
+	words.insert(words.end(), args.begin(), args.end());
+	state.builder.AddFunction(words);
+	return result;
+}
+
+uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
+	const auto trunc         = EmitTruncF32Value(state, src);
+	const auto converted_raw = state.builder.AllocateId();
+	if (signed_value) {
+		const auto converted_i = state.builder.AllocateId();
+		state.builder.AddFunction({OpConvertFToS, TypeI32(state), converted_i, trunc});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), converted_raw, converted_i});
+	} else {
+		state.builder.AddFunction({OpConvertFToU, TypeU32(state), converted_raw, trunc});
+	}
+	const auto nan = EmitClassifyF32(state, src).nan;
+	if (signed_value) {
+		const auto below = NewBinary(state, OpFOrdLessThanEqual, TypeBool(state), src,
+		                             ConstantF32(state, 0xcf000000u));
+		const auto above = NewBinary(state, OpFOrdGreaterThanEqual, TypeBool(state), src,
+		                             ConstantF32(state, 0x4f000000u));
+		const auto high =
+		    NewSelect(state, TypeU32(state), above, ConstantU32(state, 0x7fffffffu), converted_raw);
+		const auto low =
+		    NewSelect(state, TypeU32(state), below, ConstantU32(state, 0x80000000u), high);
+		return NewSelect(state, TypeU32(state), nan, ConstantU32(state, 0), low);
+	}
+	const auto below =
+	    NewBinary(state, OpFOrdLessThanEqual, TypeBool(state), src, ConstantF32(state, 0));
+	const auto above = NewBinary(state, OpFOrdGreaterThanEqual, TypeBool(state), src,
+	                             ConstantF32(state, 0x4f800000u));
+	const auto zero  = NewBinary(state, OpLogicalOr, TypeBool(state), nan, below);
+	const auto high =
+	    NewSelect(state, TypeU32(state), above, ConstantU32(state, 0xffffffffu), converted_raw);
+	return NewSelect(state, TypeU32(state), zero, ConstantU32(state, 0), high);
+}
+
+uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
+	return EmitExt(state, TypeU32(state), GlslPackHalf2x16, {src});
+}
+
+uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) {
+	const auto pair   = EmitExt(state, TypeF32Vector(state, 2), GlslUnpackHalf2x16, {bits});
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), result, pair, 0});
+	return result;
+}
+
+} // namespace
+
+bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state = ctx.state;
+	const auto op    = inst.GetOpcode();
+	const auto unary = [&](uint32_t spirv, IR::Type type) {
+		ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0)});
+		return true;
+	};
+	const auto binary = [&](uint32_t spirv, IR::Type type) {
+		ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+		return true;
+	};
+	const auto ext_unary = [&](uint32_t ext) {
+		ctx.Define(inst, EmitExt(state, TypeF32(state), ext, {ctx.Arg(inst, 0)}));
+		return true;
+	};
+	switch (op) {
+		case IR::ValueOpcode::BitCastU16F16:
+		case IR::ValueOpcode::BitCastF16U16:
+		case IR::ValueOpcode::ConvertU32U16:
+		case IR::ValueOpcode::ConvertU32U8:
+		case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
+		case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
+		case IR::ValueOpcode::ConvertU16U32:
+			ctx.Emit(inst, OpBitwiseAnd, IR::Type::U16,
+			         {ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)});
+			return true;
+		case IR::ValueOpcode::ConvertU8U32:
+			ctx.Emit(inst, OpBitwiseAnd, IR::Type::U8,
+			         {ctx.Arg(inst, 0), ConstantU32(state, 0xffu)});
+			return true;
+		case IR::ValueOpcode::ConvertF16F32: {
+			const auto pair = state.builder.AllocateId();
+			state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 2), pair,
+			                           ctx.Arg(inst, 0), ConstantF32(state, 0)});
+			ctx.Define(inst, EmitPackHalf(state, pair));
+			return true;
+		}
+		case IR::ValueOpcode::ConvertF32F16:
+			ctx.Define(inst, EmitUnpackHalf(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::ConvertS32F32:
+			ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), true));
+			return true;
+		case IR::ValueOpcode::ConvertU32F32:
+			ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
+			return true;
+		case IR::ValueOpcode::ConvertF32S32: {
+			const auto signed_value = NewUnary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 0));
+			ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value});
+			return true;
+		}
+		case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
+		case IR::ValueOpcode::CompositeConstructU64:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U64,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::CompositeConstructU32x2:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::CompositeConstructU32x3:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x3,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::CompositeConstructF32x2:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::CompositeConstructU32x4:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
+			return true;
+		case IR::ValueOpcode::CompositeExtractU64:
+		case IR::ValueOpcode::CompositeExtractU32x2:
+		case IR::ValueOpcode::CompositeExtractU32x3:
+		case IR::ValueOpcode::CompositeExtractU32x4:
+			ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
+			         {ctx.Arg(inst, 0), inst.Arg(1).U32()});
+			return true;
+		case IR::ValueOpcode::PackHalf2x16:
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
+			return true;
+		case IR::ValueOpcode::PackSnorm2x16:
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
+			return true;
+		case IR::ValueOpcode::PackUnorm2x16:
+			ctx.Define(inst, EmitExt(state, TypeU32(state), GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
+			return true;
+		case IR::ValueOpcode::PackFloat2x16Rtz: {
+			const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
+			const auto high =
+			    NewBinary(state, OpShiftLeftLogical, TypeU32(state),
+			              EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
+			ctx.Define(inst, NewBinary(state, OpBitwiseOr, TypeU32(state), low, high));
+			return true;
+		}
+		case IR::ValueOpcode::FPAbs32:
+			ctx.Define(inst, EmitFAbsValue(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FPNeg32:
+			ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FPSaturate32:
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFClamp,
+			                         {ctx.Arg(inst, 0), ConstantF32(state, 0),
+			                          ConstantF32(state, 0x3f800000u)}));
+			return true;
+		case IR::ValueOpcode::BitFieldInsert:
+			ctx.Emit(inst, OpBitFieldInsert, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
+			return true;
+		case IR::ValueOpcode::BitFieldUExtract:
+			ctx.Emit(inst, OpBitFieldUExtract, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::BitFieldSExtract:
+			ctx.Emit(inst, OpBitFieldSExtract, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::SelectU1:
+			ctx.Emit(inst, OpSelect, IR::Type::U1,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::SelectU32:
+			ctx.Emit(inst, OpSelect, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::SelectF32:
+			ctx.Emit(inst, OpSelect, IR::Type::F32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::IAdd32: return binary(OpIAdd, IR::Type::U32);
+		case IR::ValueOpcode::ISub32: return binary(OpISub, IR::Type::U32);
+		case IR::ValueOpcode::IMul32: return binary(OpIMul, IR::Type::U32);
+		case IR::ValueOpcode::UDiv32: return binary(OpUDiv, IR::Type::U32);
+		case IR::ValueOpcode::IAdd64:
+			ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::ISub64:
+			ctx.Define(inst, EmitSub64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::IMul64:
+			ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::IAddCarry32:
+			ctx.Emit(inst, OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::SMulHi:
+			ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::UMulHi:
+			ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::IAbs32: {
+			const auto value = ctx.Arg(inst, 0);
+			const auto neg   = NewUnary(state, OpSNegate, TypeU32(state), value);
+			const auto negative =
+			    NewBinary(state, OpSLessThan, TypeBool(state), value, ConstantU32(state, 0));
+			ctx.Define(inst, NewSelect(state, TypeU32(state), negative, neg, value));
+			return true;
+		}
+		case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32);
+		case IR::ValueOpcode::ShiftRightLogical32:
+			return binary(OpShiftRightLogical, IR::Type::U32);
+		case IR::ValueOpcode::ShiftRightArithmetic32:
+			return binary(OpShiftRightArithmetic, IR::Type::U32);
+		case IR::ValueOpcode::ShiftLeftLogical64:
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
+			                     ? EmitConstantShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::ShiftRightLogical64:
+			ctx.Define(inst, inst.Arg(1).Resolve().IsImmediate()
+			                     ? EmitConstantShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                           inst.Arg(1).Resolve().U32())
+			                     : EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0),
+			                                   ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::ShiftRightArithmetic64:
+			ctx.Define(inst,
+			           inst.Arg(1).Resolve().IsImmediate()
+			               ? EmitConstantShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			                                     inst.Arg(1).Resolve().U32())
+			               : EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			                             ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseOr32: return binary(OpBitwiseOr, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseAnd64:
+			ctx.Define(inst, NewBinary(state, OpBitwiseAnd, TypeU64(state), ctx.Arg(inst, 0),
+			                           ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
+		case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
+		case IR::ValueOpcode::BitCount64: {
+			const auto pair =
+			    ExtractPair(state, NewUnary(state, OpBitCount, TypeU64(state), ctx.Arg(inst, 0)));
+			ctx.Define(inst, NewBinary(state, OpIAdd, TypeU32(state), pair.low, pair.high));
+			return true;
+		}
+		case IR::ValueOpcode::FindILsb32: {
+			const auto value = EmitExt(state, TypeI32(state), GlslFindILsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, TypeU32(state), value));
+			return true;
+		}
+		case IR::ValueOpcode::FindUMsb32: {
+			const auto value = EmitExt(state, TypeI32(state), GlslFindUMsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, TypeU32(state), value));
+			return true;
+		}
+		case IR::ValueOpcode::FindUMsb64:
+			ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::SMin32:
+			ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::SMax32:
+			ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::UMin32:
+			ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::UMax32:
+			ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::SMinTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), true, false));
+			return true;
+		case IR::ValueOpcode::SMaxTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), true, true));
+			return true;
+		case IR::ValueOpcode::UMinTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), false, false));
+			return true;
+		case IR::ValueOpcode::UMaxTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), false, true));
+			return true;
+		case IR::ValueOpcode::SMedTri32:
+			ctx.Define(inst,
+			           EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), true));
+			return true;
+		case IR::ValueOpcode::UMedTri32:
+			ctx.Define(
+			    inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false));
+			return true;
+		case IR::ValueOpcode::SLessThan32: return binary(OpSLessThan, IR::Type::U1);
+		case IR::ValueOpcode::ULessThan32: return binary(OpULessThan, IR::Type::U1);
+		case IR::ValueOpcode::IEqual32: return binary(OpIEqual, IR::Type::U1);
+		case IR::ValueOpcode::SLessThanEqual32: return binary(OpSLessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::ULessThanEqual32: return binary(OpULessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::SGreaterThan32: return binary(OpSGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::UGreaterThan32: return binary(OpUGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::INotEqual32: return binary(OpINotEqual, IR::Type::U1);
+		case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::IEqual64:
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::INotEqual64:
+			ctx.Define(inst, CompareEqual64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::ULessThan64:
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpULessThan, OpULessThan));
+			return true;
+		case IR::ValueOpcode::SLessThan64:
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpSLessThan, OpULessThan));
+			return true;
+		case IR::ValueOpcode::UGreaterThan64:
+			ctx.Define(inst, CompareOrdered64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                                  OpUGreaterThan, OpUGreaterThan));
+			return true;
+		case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
+		case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
+		case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);
+		case IR::ValueOpcode::LogicalNot: return unary(OpLogicalNot, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdEqual32: return binary(OpFOrdEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordEqual32: return binary(OpFUnordEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdNotEqual32: return binary(OpFOrdNotEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordNotEqual32: return binary(OpFUnordNotEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdLessThan32: return binary(OpFOrdLessThan, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordLessThan32: return binary(OpFUnordLessThan, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdGreaterThan32: return binary(OpFOrdGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordGreaterThan32:
+			return binary(OpFUnordGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdLessThanEqual32:
+			return binary(OpFOrdLessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordLessThanEqual32:
+			return binary(OpFUnordLessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdGreaterThanEqual32:
+			return binary(OpFOrdGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordGreaterThanEqual32:
+			return binary(OpFUnordGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPIsNan32:
+			ctx.Emit(inst, OpFUnordNotEqual, IR::Type::U1, {ctx.Arg(inst, 0), ctx.Arg(inst, 0)});
+			return true;
+		case IR::ValueOpcode::FPCmpClass32:
+			ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::FPAdd32: return binary(OpFAdd, IR::Type::F32);
+		case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32);
+		case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32);
+		case IR::ValueOpcode::FPFma32:
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslFma,
+			                         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
+			return true;
+		case IR::ValueOpcode::FPMin32:
+			ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::FPMax32:
+			ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::FPMinTri32:
+			ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                              ctx.Arg(inst, 2), false));
+			return true;
+		case IR::ValueOpcode::FPMaxTri32:
+			ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                              ctx.Arg(inst, 2), true));
+			return true;
+		case IR::ValueOpcode::FPMedTri32:
+			ctx.Define(inst,
+			           EmitFMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)));
+			return true;
+		case IR::ValueOpcode::FPRecip32: {
+			const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
+			ctx.Define(inst, NewBinary(state, OpFDiv, TypeF32(state),
+			                           ConstantF32(state, 0x3f800000u), source));
+			return true;
+		}
+		case IR::ValueOpcode::FPRecipIFlag32:
+			// Integer-to-float inputs used by IFLAG cannot be denormal.
+			ctx.Define(inst, NewBinary(state, OpFDiv, TypeF32(state),
+			                           ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FPRecipSqrt32:
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslInverseSqrt,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPSqrt:
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslSqrt,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPSin:
+		case IR::ValueOpcode::FPCos: {
+			auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
+			source =
+			    NewBinary(state, OpFMul, TypeF32(state), source, ConstantF32(state, 0x40c90fdbu));
+			ctx.Define(inst, EmitExt(state, TypeF32(state),
+			                         op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source}));
+			return true;
+		}
+		case IR::ValueOpcode::FPExp2:
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslExp2,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPLog2:
+			ctx.Define(inst, EmitExt(state, TypeF32(state), GlslLog2,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPLdexp: {
+			const auto exponent = NewUnary(state, OpBitcast, TypeI32(state), ctx.Arg(inst, 1));
+			ctx.Define(inst,
+			           EmitExt(state, TypeF32(state), GlslLdexp, {ctx.Arg(inst, 0), exponent}));
+			return true;
+		}
+		case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven);
+		case IR::ValueOpcode::FPFloor32: return ext_unary(GlslFloor);
+		case IR::ValueOpcode::FPCeil32: return ext_unary(GlslCeil);
+		case IR::ValueOpcode::FPTrunc32: return ext_unary(GlslTrunc);
+		case IR::ValueOpcode::FPFract32: return ext_unary(GlslFract);
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -0,0 +1,393 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
+                            uint32_t false_value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, true_value, false_value});
+	return ret;
+}
+
+uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
+
+void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
+                                   uint32_t& out_low, uint32_t& out_high) {
+	const auto amount           = EmitAndConstant(state, shift, 63u);
+	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
+	const auto less32           = state.builder.AllocateId();
+	const auto non_zero         = state.builder.AllocateId();
+	const auto carry_count_base = state.builder.AllocateId();
+	const auto carry_count      = state.builder.AllocateId();
+	const auto low_part         = state.builder.AllocateId();
+	const auto high_part        = state.builder.AllocateId();
+	const auto carry_part       = state.builder.AllocateId();
+	const auto carry            = state.builder.AllocateId();
+	const auto high_less        = state.builder.AllocateId();
+	const auto high_ge          = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
+	state.builder.AddFunction(
+	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), carry_part, low, carry_count});
+	state.builder.AddFunction(
+	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), high_less, high_part, carry});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), high_ge, low, dword_shift});
+	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
+	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
+}
+
+void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
+                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
+	const auto amount           = EmitAndConstant(state, shift, 63u);
+	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
+	const auto less32           = state.builder.AllocateId();
+	const auto non_zero         = state.builder.AllocateId();
+	const auto carry_count_base = state.builder.AllocateId();
+	const auto carry_count      = state.builder.AllocateId();
+	const auto low_part         = state.builder.AllocateId();
+	const auto high_part        = state.builder.AllocateId();
+	const auto carry_part       = state.builder.AllocateId();
+	const auto carry            = state.builder.AllocateId();
+	const auto low_less         = state.builder.AllocateId();
+	const auto low_ge           = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpULessThan, TypeBool(state), less32, amount, ConstantU32(state, 32)});
+	state.builder.AddFunction(
+	    {OpINotEqual, TypeBool(state), non_zero, amount, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpISub, TypeU32(state), carry_count_base, ConstantU32(state, 32), dword_shift});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, TypeU32(state), carry_part, high, carry_count});
+	state.builder.AddFunction(
+	    {OpSelect, TypeU32(state), carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), low_less, low_part, carry});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU32(state), low_ge, high, dword_shift});
+	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
+	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
+}
+
+uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, value, ConstantU32(state, mask)});
+	return ret;
+}
+
+uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), ret, value, ConstantU32(state, shift)});
+	return ret;
+}
+
+uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+                                uint32_t constant) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
+	return ret;
+}
+
+uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpISub, TypeU32(state), ret, ConstantU32(state, constant), value});
+	return ret;
+}
+
+uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, f32});
+
+	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
+	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
+	const auto mant = EmitAndConstant(state, bits, 0x007fffffu);
+
+	const auto half_exp       = state.builder.AllocateId();
+	const auto normal_exp     = state.builder.AllocateId();
+	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
+	const auto normal_payload = state.builder.AllocateId();
+	const auto normal         = state.builder.AllocateId();
+	state.builder.AddFunction({OpISub, TypeU32(state), half_exp, exp, ConstantU32(state, 112)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, TypeU32(state), normal_exp, half_exp, ConstantU32(state, 10)});
+	state.builder.AddFunction(
+	    {OpBitwiseOr, TypeU32(state), normal_payload, normal_exp, normal_mant});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), normal, sign, normal_payload});
+
+	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
+	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
+	const auto exp_lt_103       = EmitCompareU32Constant(state, OpULessThan, exp, 103);
+	const auto exp_gt_112       = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112);
+	const auto sub_shift_low =
+	    EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
+	const auto sub_shift =
+	    EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low);
+	const auto sub_mant  = state.builder.AllocateId();
+	const auto subnormal = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), sub_mant, mant_with_hidden, sub_shift});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), subnormal, sign, sub_mant});
+
+	const auto nan_payload =
+	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
+	const auto nan =
+	    EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
+	const auto inf        = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
+	const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
+	const auto mant_zero  = EmitCompareU32Constant(state, OpIEqual, mant, 0);
+	const auto special    = EmitSelectValueU32(state, mant_zero, inf, nan);
+
+	const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112);
+	const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143);
+	const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255);
+	const auto finite0    = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
+	const auto finite1    = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
+	const auto finite2    = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
+	return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu);
+}
+
+uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto cond = state.builder.AllocateId();
+	const auto ret  = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {max_value ? OpUGreaterThan : OpULessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto cond = state.builder.AllocateId();
+	const auto ret  = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {max_value ? OpSGreaterThan : OpSLessThan, TypeBool(state), cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, TypeU32(state), ret, cond, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
+	return ret;
+}
+
+uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
+	return ret;
+}
+
+uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalOr, TypeBool(state), ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalNot, TypeBool(state), ret, value});
+	return ret;
+}
+
+F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
+	F32Class cls;
+	cls.bits                 = EmitBitcastF32ToU32(state, value);
+	const auto abs_bits      = EmitAndConstant(state, cls.bits, 0x7fffffffu);
+	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
+	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
+	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
+	cls.nan  = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
+	cls.zero = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	return cls;
+}
+
+uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
+                               uint32_t class_match) {
+	const auto selected =
+	    EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
+	return EmitLogicalAndBool(state, selected, class_match);
+}
+
+uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
+	const auto bits          = EmitBitcastF32ToU32(state, value);
+	const auto sign_bits     = EmitAndConstant(state, bits, 0x80000000u);
+	const auto abs_bits      = EmitAndConstant(state, bits, 0x7fffffffu);
+	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
+	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
+	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
+
+	const auto sign             = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0);
+	const auto positive         = EmitLogicalNotBool(state, sign);
+	const auto exponent_zero    = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0);
+	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
+	const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
+	const auto mantissa_zero    = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0);
+	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
+	const auto quiet            = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0);
+
+	const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
+	const auto snan       = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
+	const auto qnan       = EmitLogicalAndBool(state, nan_common, quiet);
+	const auto inf        = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
+	const auto normal     = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
+	const auto denorm     = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
+	const auto zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+
+	uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
+	match          = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
+	match          = EmitLogicalOrBool(
+	    state, match, EmitClassMaskBitMatch(state, mask, 2, EmitLogicalAndBool(state, inf, sign)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 3, EmitLogicalAndBool(state, normal, sign)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 4, EmitLogicalAndBool(state, denorm, sign)));
+	match = EmitLogicalOrBool(
+	    state, match, EmitClassMaskBitMatch(state, mask, 5, EmitLogicalAndBool(state, zero, sign)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 6, EmitLogicalAndBool(state, zero, positive)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 7, EmitLogicalAndBool(state, denorm, positive)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 8, EmitLogicalAndBool(state, normal, positive)));
+	return EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive)));
+}
+
+uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto lhs_class = EmitClassifyF32(state, lhs);
+	const auto rhs_class = EmitClassifyF32(state, rhs);
+
+	const auto numeric_cond = state.builder.AllocateId();
+	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, TypeBool(state),
+	                           numeric_cond, lhs, rhs});
+	const auto ordered_bits =
+	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
+
+	const auto both_zero    = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero);
+	const auto zero_bits    = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits)
+	                                    : EmitOrU32(state, lhs_class.bits, rhs_class.bits);
+	const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits);
+
+	const auto rhs_nan_bits =
+	    EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits);
+	const auto result_bits = EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
+	return EmitBitcastU32ToF32(state, result_bits);
+}
+
+uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslTrunc, value});
+	return ret;
+}
+
+uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
+	const auto bits      = state.builder.AllocateId();
+	const auto abs_bits  = state.builder.AllocateId();
+	const auto sign_bits = state.builder.AllocateId();
+	const auto non_zero  = state.builder.AllocateId();
+	const auto subnormal = state.builder.AllocateId();
+	const auto flush     = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto ret       = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), sign_bits, bits, ConstantU32(state, 0x80000000u)});
+	state.builder.AddFunction(
+	    {OpINotEqual, TypeBool(state), non_zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpULessThan, TypeBool(state), subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), flush, non_zero, subnormal});
+	state.builder.AddFunction({OpSelect, TypeU32(state), selected, flush, sign_bits, bits});
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, selected});
+	return ret;
+}
+
+uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero) {
+	const auto fract        = state.builder.AllocateId();
+	const auto bits         = state.builder.AllocateId();
+	const auto abs_bits     = state.builder.AllocateId();
+	const auto large        = state.builder.AllocateId();
+	const auto finite       = state.builder.AllocateId();
+	const auto large_finite = state.builder.AllocateId();
+	const auto reduced      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, TypeF32(state), fract, GlslStd450(state), GlslFract, src});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, src});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	state.builder.AddFunction(
+	    {OpUGreaterThanEqual, TypeBool(state), large, abs_bits, ConstantU32(state, 0x4b000000u)});
+	state.builder.AddFunction(
+	    {OpULessThan, TypeBool(state), finite, abs_bits, ConstantU32(state, 0x7f800000u)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), large_finite, large, finite});
+	state.builder.AddFunction(
+	    {OpSelect, TypeF32(state), reduced, large_finite, ConstantF32(state, 0), fract});
+	if (!preserve_signed_zero) {
+		return reduced;
+	}
+	const auto zero = state.builder.AllocateId();
+	const auto ret  = state.builder.AllocateId();
+	state.builder.AddFunction({OpIEqual, TypeBool(state), zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpSelect, TypeF32(state), ret, zero, src, reduced});
+	return ret;
+}
+
+uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpFNegate, TypeF32(state), ret, value});
+	return ret;
+}
+
+uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpExtInst, TypeF32(state), ret, GlslStd450(state), GlslFAbs, value});
+	return ret;
+}
+
+uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
+	const auto unpacked = state.builder.AllocateId();
+	const auto ret      = state.builder.AllocateId();
+	state.builder.AddFunction({OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state),
+	                           GlslUnpackHalf2x16, bits});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), ret, unpacked, 0});
+	return ret;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -0,0 +1,343 @@
+#include "common/assert.h"
+#include "graphics/guest_gpu/gpu_defs.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
+	if (state.stage != ShaderType::Pixel) {
+		return attr;
+	}
+	return ShaderPixelParameterMappedLocation(*state.input_info.pixel, attr);
+}
+
+uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
+	std::array<uint32_t, 32> active_inputs {};
+	uint32_t                 active_count = 0;
+	for (const auto& input: state.inputs) {
+		if (input.kind == IR::StageInputKind::Parameter) {
+			active_inputs[active_count++] = input.location;
+		}
+	}
+	return state.stage == ShaderType::Pixel
+	           ? ShaderPixelParameterLocation(*state.input_info.pixel,
+	                                          {active_inputs.data(), active_count}, attr)
+	           : attr;
+}
+
+bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
+	return state.stage == ShaderType::Pixel &&
+	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
+}
+
+bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr) {
+	return state.stage == ShaderType::Pixel &&
+	       ShaderPixelParameterIsCustom(*state.input_info.pixel, attr);
+}
+
+void SetError(std::string* error, const char* message) {
+	if (error != nullptr) {
+		*error = message;
+	}
+}
+
+bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
+               uint32_t index) {
+	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
+		return binding.kind == kind && binding.index == index;
+	});
+}
+
+void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program) {
+	for (const auto& input: program.info.inputs) {
+		state.inputs.push_back({input.kind, input.location, input.component_count, 0,
+		                        input.debug_name, input.per_vertex});
+	}
+	for (const auto& output: program.info.outputs) {
+		if (HasOutput(state.outputs, output.kind, output.index)) {
+			continue;
+		}
+		state.outputs.push_back({output.kind, output.index, output.location, 0, output.debug_name});
+	}
+}
+
+uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp) {
+	if (exp.kind == IR::ExportTargetKind::Position && exp.index == 0) {
+		return state.per_vertex_variable;
+	}
+	if (exp.kind == IR::ExportTargetKind::MrtZ) {
+		return state.depth_variable;
+	}
+	for (const auto& binding: state.outputs) {
+		const auto expected_kind = exp.kind == IR::ExportTargetKind::Mrt
+		                               ? IR::StageOutputKind::Mrt
+		                               : IR::StageOutputKind::Parameter;
+		if (binding.kind == expected_kind && binding.index == exp.index) {
+			return binding.variable_id;
+		}
+	}
+	return 0;
+}
+
+uint32_t          ConstantU32(EmitterState& state, uint32_t value);
+[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState&       state,
+                                               IR::DescriptorBindingKind kind, uint32_t resource,
+                                               const char* reason) {
+	EXIT("shader binding resolution failed during SPIR-V emit: hash=0x%016" PRIx64
+	     " stage=%u resource=%" PRIu32 " binding_kind=%u reason=%s\n",
+	     state.program.shader_hash, static_cast<unsigned>(state.stage), resource,
+	     static_cast<unsigned>(kind), reason);
+	std::abort();
+}
+
+DescriptorResourceBinding ResourceForDescriptor(const EmitterState&       state,
+                                                IR::DescriptorBindingKind kind, uint32_t resource) {
+	const auto* descriptor = IR::FindBinding(state.program.bindings, kind);
+	if (descriptor == nullptr) {
+		ExitDescriptorBindingFailure(state, kind, resource, "descriptor group was not allocated");
+	}
+	const auto found =
+	    std::find(descriptor->resources.begin(), descriptor->resources.end(), resource);
+	if (found == descriptor->resources.end()) {
+		ExitDescriptorBindingFailure(state, kind, resource,
+		                             "resource is absent from descriptor group");
+	}
+	return {descriptor, static_cast<uint32_t>(found - descriptor->resources.begin())};
+}
+
+uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
+                                  uint32_t variable_id, uint32_t array_index,
+                                  IR::DescriptorBindingKind kind, uint32_t resource,
+                                  const char* variable_name) {
+	if (variable_id == 0) {
+		ExitDescriptorBindingFailure(state, kind, resource, variable_name);
+	}
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpAccessChain, result_ptr_type, pointer, variable_id, ConstantU32(state, array_index)});
+	return pointer;
+}
+
+ImageViewKind ImageViewKindFromDimension(Decoder::ImageDimension dimension) {
+	switch (dimension) {
+		case Decoder::ImageDimension::Dim1D: return ImageViewKind::Dim1D;
+		case Decoder::ImageDimension::Dim1DArray: return ImageViewKind::Dim1DArray;
+		case Decoder::ImageDimension::Dim2DArray: return ImageViewKind::Dim2DArray;
+		case Decoder::ImageDimension::Dim3D: return ImageViewKind::Dim3D;
+		case Decoder::ImageDimension::Dim2DMsaa: return ImageViewKind::Dim2DMsaa;
+		case Decoder::ImageDimension::Dim2DMsaaArray: return ImageViewKind::Dim2DMsaaArray;
+		default: return ImageViewKind::Dim2D;
+	}
+}
+
+ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
+                                   uint32_t use_pc) {
+	(void)state;
+	(void)use_pc;
+	return ImageViewKindFromDimension(mem.image_dimension);
+}
+
+ImageViewKind StorageImageViewKind(const IR::MemoryInfo& mem) {
+	return ImageViewKindFromDimension(mem.image_dimension);
+}
+
+uint32_t ImageViewCoordinateComponents(ImageViewKind view) {
+	switch (view) {
+		case ImageViewKind::Dim1D: return 1u;
+		case ImageViewKind::Dim1DArray:
+		case ImageViewKind::Dim2D: return 2u;
+		case ImageViewKind::Dim2DArray:
+		case ImageViewKind::Dim2DMsaaArray:
+		case ImageViewKind::Dim3D: return 3u;
+		case ImageViewKind::Dim2DMsaa: return 2u;
+		default: return 0u;
+	}
+}
+
+uint32_t ImageViewSpatialComponents(ImageViewKind view) {
+	switch (view) {
+		case ImageViewKind::Dim1D:
+		case ImageViewKind::Dim1DArray: return 1u;
+		case ImageViewKind::Dim2D:
+		case ImageViewKind::Dim2DArray:
+		case ImageViewKind::Dim2DMsaa:
+		case ImageViewKind::Dim2DMsaaArray: return 2u;
+		case ImageViewKind::Dim3D: return 3u;
+		default: return 0u;
+	}
+}
+
+uint32_t ImageViewImageType(EmitterState& state, ImageViewKind view, bool integer) {
+	const auto component = integer ? TypeU32(state) : TypeF32(state);
+	return state.builder.Type(OpTypeImage,
+	                          {component, ImageSpirvDimension(view), 0, ImageSpirvArrayed(view),
+	                           ImageSpirvMultisampled(view), 1, ImageFormatUnknown});
+}
+
+uint32_t ImageViewSampledImageType(EmitterState& state, ImageViewKind view, bool integer) {
+	return state.builder.Type(OpTypeSampledImage, {ImageViewImageType(state, view, integer)});
+}
+
+uint32_t ImageViewSizeType(EmitterState& state, ImageViewKind view) {
+	switch (ImageViewCoordinateComponents(view)) {
+		case 1u: return TypeU32(state);
+		case 2u: return TypeU32Vector(state, 2);
+		case 3u: return TypeU32Vector(state, 3);
+		default: return 0;
+	}
+}
+
+uint32_t StorageImageType(EmitterState& state, StorageImageClass image_class, ImageViewKind view) {
+	uint32_t component = TypeF32(state);
+	uint32_t format    = ImageFormatUnknown;
+	if (image_class == StorageImageClass::FormatlessUint) {
+		component = TypeU32(state);
+	} else if (image_class == StorageImageClass::AtomicUint) {
+		component = TypeU32(state);
+		format    = ImageFormatR32ui;
+	}
+	return state.builder.Type(OpTypeImage, {component, ImageSpirvDimension(view), 0,
+	                                        ImageSpirvArrayed(view), 0, 2, format});
+}
+
+uint32_t StorageImagePointerType(EmitterState& state, StorageImageClass image_class,
+                                 ImageViewKind view) {
+	return state.builder.Type(
+	    OpTypePointer, {StorageClassUniformConstant, StorageImageType(state, image_class, view)});
+}
+
+StorageImageClass StorageImageClassForResource(const EmitterState& state, uint32_t resource) {
+	const auto& image      = state.program.info.images.at(resource);
+	const bool  uint_image = image.kind == IR::ResourceKind::StorageImageUint;
+	EXIT_IF(image.kind != IR::ResourceKind::StorageImage && !uint_image);
+	EXIT_IF(image.atomic && !uint_image);
+	return StorageImageClassFor(uint_image, image.atomic);
+}
+
+uint32_t StorageImageVariable(const EmitterState& state, StorageImageClass image_class,
+                              ImageViewKind view) {
+	return state.storage_image_variables[StorageImageIndex(image_class, view)];
+}
+
+uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                                    ImageViewKind view) {
+	(void)use_pc;
+	const bool integer      = mem.kind == IR::ResourceKind::ImageUint;
+	const auto kind         = SampledBindingKind(integer, view);
+	const auto binding      = ResourceForDescriptor(state, kind, mem.resource);
+	const auto variable     = state.sampled_image_variables[SampledImageIndex(integer, view)];
+	const auto pointer_type = state.builder.Type(
+	    OpTypePointer, {StorageClassUniformConstant, ImageViewImageType(state, view, integer)});
+	const auto pointer =
+	    DescriptorElementPointer(state, pointer_type, variable, binding.array_index, kind,
+	                             mem.resource, "sampled image descriptor array was not emitted");
+	const auto image = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, ImageViewImageType(state, view, integer), image, pointer});
+	return image;
+}
+
+uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc) {
+	(void)use_pc;
+	const auto binding = ResourceForDescriptor(state, IR::DescriptorBindingKind::Samplers, sampler);
+	const auto sampler_type = state.builder.Type(OpTypeSampler);
+	const auto pointer_type =
+	    state.builder.Type(OpTypePointer, {StorageClassUniformConstant, sampler_type});
+	const auto pointer = DescriptorElementPointer(
+	    state, pointer_type, state.sampler_variable, binding.array_index,
+	    IR::DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted");
+	const auto sampler_id = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, sampler_type, sampler_id, pointer});
+	return sampler_id;
+}
+
+uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                          ImageViewKind view) {
+	const auto image   = LoadSampledImageDescriptor(state, mem, use_pc, view);
+	const auto sampler = LoadSamplerDescriptor(state, mem.sampler, use_pc);
+	if (image == 0 || sampler == 0) {
+		ExitDescriptorBindingFailure(
+		    state, SampledBindingKind(mem.kind == IR::ResourceKind::ImageUint, view), mem.resource,
+		    "sampled image or sampler descriptor load failed");
+	}
+	const auto sampled_image = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpSampledImage,
+	     ImageViewSampledImageType(state, view, mem.kind == IR::ResourceKind::ImageUint),
+	     sampled_image, image, sampler});
+	return sampled_image;
+}
+
+uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                          ImageViewKind view, uint32_t image_resource) {
+	auto selected     = mem;
+	selected.resource = image_resource;
+	return MakeSampledImage(state, selected, use_pc, view);
+}
+
+uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, ImageViewKind view) {
+	const auto image_class = StorageImageClassForResource(state, resource);
+	const auto kind        = StorageBindingKind(image_class, view);
+	const auto binding  = ResourceForDescriptor(state, kind, resource);
+	const auto ptr_type    = StorageImagePointerType(state, image_class, view);
+	const auto variable    = StorageImageVariable(state, image_class, view);
+	return DescriptorElementPointer(state, ptr_type, variable, binding.array_index, kind, resource,
+	                                "storage image descriptor array was not emitted");
+}
+
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, ImageViewKind view,
+                           uint32_t mip_lod, uint32_t coord, uint32_t texel) {
+	const auto image_class = StorageImageClassForResource(state, resource);
+	if (image_class != StorageImageClass::AtomicUint) {
+		state.builder.RequireCapability(CapabilityStorageImageWriteWithoutFormat);
+	}
+	const auto  kind    = StorageBindingKind(image_class, view);
+	const auto  binding = ResourceForDescriptor(state, kind, resource);
+	const auto& image   = state.program.info.images.at(resource);
+	const auto  LoadAt  = [&](uint32_t array_index) {
+		const auto pointer = DescriptorElementPointer(
+		    state, StorageImagePointerType(state, image_class, view),
+		    StorageImageVariable(state, image_class, view), array_index, kind, resource,
+		    "storage image descriptor array was not emitted");
+		const auto descriptor = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpLoad, StorageImageType(state, image_class, view), descriptor, pointer});
+		return descriptor;
+	};
+	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
+		state.builder.AddFunction({OpImageWrite, LoadAt(binding.array_index), coord, texel});
+		return;
+	}
+	if (image.mip_count == 0u) {
+		ExitDescriptorBindingFailure(state, kind, resource,
+		                             "dynamic storage image has no mip descriptors");
+	}
+
+	const auto            merge_label = state.builder.AllocateId();
+	std::vector<uint32_t> labels(image.mip_count);
+	std::vector<uint32_t> words {OpSwitch, mip_lod, merge_label};
+	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
+		labels[mip] = state.builder.AllocateId();
+		words.push_back(mip);
+		words.push_back(labels[mip]);
+	}
+	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction(words);
+	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
+		EmitLabel(state, labels[mip]);
+		state.builder.AddFunction({OpImageWrite, LoadAt(binding.array_index + mip), coord, texel});
+		state.builder.AddFunction({OpBranch, merge_label});
+	}
+	EmitLabel(state, merge_label);
+}
+
+uint32_t ExecutionModelForStage(ShaderType stage) {
+	switch (stage) {
+		case ShaderType::Vertex: return ExecutionModelVertex;
+		case ShaderType::Pixel: return ExecutionModelFragment;
+		default: return ExecutionModelGLCompute;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -0,0 +1,662 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+bool UserDataDwordIndex(const EmitterState& state, IR::ScalarReg reg, uint32_t& dword_index) {
+	const auto register_index = IR::RegIndex(reg);
+	const auto& registers = state.program.bindings.user_data_registers;
+	const auto  found     = std::lower_bound(registers.begin(), registers.end(), register_index);
+	if (found == registers.end() || *found != register_index) {
+		return false;
+	}
+	dword_index = static_cast<uint32_t>(found - registers.begin());
+	return true;
+}
+
+uint32_t EmitWqmWordU32(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), shifted_one, value, ConstantU32(state, 1)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_one, value, shifted_one});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), shifted_two, merged_one, ConstantU32(state, 2)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), quad_bits, merged_two, ConstantU32(state, 0x11111111u)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU32(state), result, quad_bits, ConstantU32(state, 0x0fu)});
+	return result;
+}
+
+uint32_t EmitWqmU64(EmitterState& state, uint32_t value) {
+	const auto shifted_one = state.builder.AllocateId();
+	const auto merged_one  = state.builder.AllocateId();
+	const auto shifted_two = state.builder.AllocateId();
+	const auto merged_two  = state.builder.AllocateId();
+	const auto quad_bits   = state.builder.AllocateId();
+	const auto result      = state.builder.AllocateId();
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_one, value,
+	                           ConstantU64(state, 0x0000000100000001ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_one, value, shifted_one});
+	state.builder.AddFunction({OpShiftRightLogical, TypeU64(state), shifted_two, merged_one,
+	                           ConstantU64(state, 0x0000000200000002ull)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU64(state), merged_two, merged_one, shifted_two});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU64(state), quad_bits, merged_two,
+	                           ConstantU64(state, 0x1111111111111111ull)});
+	state.builder.AddFunction(
+	    {OpIMul, TypeU64(state), result, quad_bits, ConstantU64(state, 0x0000000f0000000full)});
+	return result;
+}
+
+uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
+	auto& state = ctx.state;
+	if (kind == IR::StageInputKind::LocalInvocationIndex) {
+		return EmitLocalInvocationIndex(state);
+	}
+	const auto variable = InputVariableForKind(state, kind);
+	if (variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	if (kind == IR::StageInputKind::FrontFacing) {
+		const auto value = state.builder.AllocateId();
+		const auto bits  = state.builder.AllocateId();
+		state.builder.AddFunction({OpLoad, TypeBool(state), value, variable});
+		state.builder.AddFunction(
+		    {OpSelect, TypeU32(state), bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) {
+		const auto value = state.builder.AllocateId();
+		const auto bits  = state.builder.AllocateId();
+		state.builder.AddFunction({OpLoad, TypeI32(state), value, variable});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::FragCoord) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		const auto bits    = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain,
+		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		                           variable, ConstantU32(state, component)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::BaryCoordSmooth ||
+	    kind == IR::StageInputKind::BaryCoordNoPerspective) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		const auto bits    = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain,
+		                           TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		                           variable, ConstantU32(state, component + 1u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+		return bits;
+	}
+	return EmitInputComponentU32(state, kind, component);
+}
+
+uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
+	auto&      state  = ctx.state;
+	const auto ballot = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
+	                           ConstantU32(state, ScopeSubgroup), active});
+	const auto low = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, ballot, 0});
+	const auto wqm_low  = EmitWqmWordU32(state, low);
+	const auto lane     = EmitSubgroupLocalInvocationId(state);
+	uint32_t   mask     = wqm_low;
+	uint32_t   bit_lane = lane;
+	if (state.wave_size == 64u) {
+		const auto high = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, ballot, 1});
+		const auto wqm_high = EmitWqmWordU32(state, high);
+		const auto upper    = state.builder.AllocateId();
+		mask                = state.builder.AllocateId();
+		bit_lane            = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, TypeU32(state), mask, upper, wqm_high, wqm_low});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, TypeU32(state), bit_lane, lane, ConstantU32(state, 31)});
+	}
+	const auto bit    = state.builder.AllocateId();
+	const auto hit    = state.builder.AllocateId();
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), bit_lane});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, TypeBool(state), result, hit, ConstantU32(state, 0)});
+	return result;
+}
+
+uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
+                               uint32_t exec) {
+	auto&      state      = ctx.state;
+	const auto lane       = EmitSubgroupLocalInvocationId(state);
+	const auto bank_shift = state.builder.AllocateId();
+	const auto row_shift  = state.builder.AllocateId();
+	const auto bank       = state.builder.AllocateId();
+	const auto row        = state.builder.AllocateId();
+	const auto bank_bit   = state.builder.AllocateId();
+	const auto row_bit    = state.builder.AllocateId();
+	const auto bank_hit   = state.builder.AllocateId();
+	const auto row_hit    = state.builder.AllocateId();
+	const auto bank_ok    = state.builder.AllocateId();
+	const auto row_ok     = state.builder.AllocateId();
+	const auto masks_ok   = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), bank_shift, lane, ConstantU32(state, 2)});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), row_shift, lane, ConstantU32(state, 4)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), bank, bank_shift, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), row, row_shift, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, TypeU32(state), bank_bit, ConstantU32(state, 1), bank});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, TypeU32(state), row_bit, ConstantU32(state, 1), row});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), row_hit, ConstantU32(state, flags.row_mask), row_bit});
+	state.builder.AddFunction(
+	    {OpINotEqual, TypeBool(state), bank_ok, bank_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpINotEqual, TypeBool(state), row_ok, row_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), masks_ok, bank_ok, row_ok});
+	uint32_t writable = masks_ok;
+	if (!flags.bound_control) {
+		const auto target  = EmitDppTargetLane(state, flags.control);
+		const auto bounded = state.builder.AllocateId();
+		state.builder.AddFunction({OpLogicalAnd, TypeBool(state), bounded, writable, target.valid});
+		writable = bounded;
+	}
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), result, exec, writable});
+	return result;
+}
+
+uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
+	auto&       state = ctx.state;
+	const auto* input = InputBindingForParameter(state, attr);
+	if (input == nullptr || input->variable_id == 0) {
+		return ConstantU32(state, 0);
+	}
+	if (state.stage == ShaderType::Vertex) {
+		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
+	}
+	const auto load_per_vertex = [&](uint32_t vertex) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		return value;
+	};
+	if (input->per_vertex) {
+		const auto barycentric_kind = state.input_info.pixel->ps_no_perspective
+		                                  ? IR::StageInputKind::BaryCoordNoPerspective
+		                                  : IR::StageInputKind::BaryCoordSmooth;
+		const auto barycentric      = InputVariableForKind(state, barycentric_kind);
+		uint32_t   sum              = 0;
+		for (uint32_t vertex = 0; vertex < 3u; vertex++) {
+			const auto pointer = state.builder.AllocateId();
+			const auto weight  = state.builder.AllocateId();
+			const auto product = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain,
+			                           TypePointer(state, StorageClassInput, TypeF32(state)),
+			                           pointer, barycentric, ConstantU32(state, vertex)});
+			state.builder.AddFunction({OpLoad, TypeF32(state), weight, pointer});
+			state.builder.AddFunction(
+			    {OpFMul, TypeF32(state), product, load_per_vertex(vertex), weight});
+			if (vertex == 0u) {
+				sum = product;
+			} else {
+				const auto next = state.builder.AllocateId();
+				state.builder.AddFunction({OpFAdd, TypeF32(state), next, sum, product});
+				sum = next;
+			}
+		}
+		const auto bits = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeU32(state), bits, sum});
+		return bits;
+	}
+	const auto vector    = state.builder.AllocateId();
+	const auto component = state.builder.AllocateId();
+	const auto bits      = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, TypeF32Vector(state, 4), vector, input->variable_id});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, vector, chan & 3u});
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, component});
+	return bits;
+}
+
+uint32_t EmitInterpolationParameter(ValueEmitContext& ctx, uint32_t attr, uint32_t chan,
+                                    uint32_t mode) {
+	auto&       state = ctx.state;
+	const auto* input = InputBindingForParameter(state, attr);
+	if (!input->per_vertex) {
+		return EmitAttribute(ctx, attr, chan);
+	}
+	const auto load_vertex = [&](uint32_t vertex) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, TypePointer(state, StorageClassInput, TypeF32(state)), pointer,
+		     input->variable_id, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)});
+		state.builder.AddFunction({OpLoad, TypeF32(state), value, pointer});
+		return value;
+	};
+
+	const auto selected_vertex = (mode + 1u) % 3u;
+	uint32_t   value           = load_vertex(selected_vertex);
+	if (!PixelParameterIsCustom(state, attr) && mode < 2u) {
+		const auto delta = state.builder.AllocateId();
+		state.builder.AddFunction({OpFSub, TypeF32(state), delta, value, load_vertex(0)});
+		value = delta;
+	}
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, value});
+	return bits;
+}
+
+uint32_t MrtOutputMode(const EmitterState& state, const IR::ExportInfo& exp) {
+	if (state.stage != ShaderType::Pixel || exp.kind != IR::ExportTargetKind::Mrt ||
+	    exp.index >= std::size(state.input_info.pixel->target_output_mode)) {
+		return 0;
+	}
+	return state.input_info.pixel->target_output_mode[exp.index];
+}
+
+uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
+	const auto value = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(
+	    {OpCompositeExtract, TypeU32(ctx.state), value, vector, component});
+	return value;
+}
+
+uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp,
+                      bool uint_output) {
+	auto& state = ctx.state;
+	if (exp.compr && !uint_output) {
+		const auto unpack =
+		    MrtOutputMode(state, exp) == 5u ? GlslUnpackUnorm2x16 : GlslUnpackHalf2x16;
+		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
+		                   ConstantF32(state, 0x3f800000u)};
+		for (uint32_t pair = 0; pair < 2u; pair++) {
+			if ((exp.en & (3u << (pair * 2u))) == 0u) {
+				continue;
+			}
+			const auto packed   = state.builder.AllocateId();
+			const auto unpacked = state.builder.AllocateId();
+			state.builder.AddFunction({OpCompositeExtract, TypeU32(state), packed, data, pair});
+			state.builder.AddFunction(
+			    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), unpack, packed});
+			for (uint32_t lane = 0; lane < 2u; lane++) {
+				const auto component = pair * 2u + lane;
+				if (((exp.en >> component) & 1u) != 0u) {
+					f32[component] = state.builder.AllocateId();
+					state.builder.AddFunction(
+					    {OpCompositeExtract, TypeF32(state), f32[component], unpacked, lane});
+				}
+			}
+		}
+		const auto vector = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0],
+		                           f32[1], f32[2], f32[3]});
+		return vector;
+	}
+	uint32_t raw[4] = {
+	    ConstantU32(state, 0),
+	    ConstantU32(state, 0),
+	    ConstantU32(state, 0),
+	    ConstantU32(state, uint_output ? 1u : 0x3f800000u),
+	};
+	if (exp.compr) {
+		for (uint32_t pair = 0; pair < 2u; pair++) {
+			if ((exp.en & (3u << (pair * 2u))) == 0u) {
+				continue;
+			}
+			const auto packed = ExportRawComponent(ctx, data, pair);
+			for (uint32_t lane = 0; lane < 2u; lane++) {
+				const auto component = pair * 2u + lane;
+				if (((exp.en >> component) & 1u) == 0u) {
+					continue;
+				}
+				raw[component] = state.builder.AllocateId();
+				state.builder.AddFunction({OpBitFieldUExtract, TypeU32(state), raw[component],
+				                           packed, ConstantU32(state, lane * 16u),
+				                           ConstantU32(state, 16)});
+			}
+		}
+	} else {
+		for (uint32_t component = 0; component < 4u; component++) {
+			if (((exp.en >> component) & 1u) != 0u) {
+				raw[component] = ExportRawComponent(ctx, data, component);
+			}
+		}
+	}
+	if (uint_output) {
+		const auto vector = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), vector, raw[0],
+		                           raw[1], raw[2], raw[3]});
+		return vector;
+	}
+	uint32_t f32[4] {};
+	for (uint32_t component = 0; component < 4u; component++) {
+		f32[component] = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeF32(state), f32[component], raw[component]});
+	}
+	const auto vector = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpCompositeConstruct, TypeF32Vector(state, 4), vector, f32[0], f32[1], f32[2], f32[3]});
+	return vector;
+}
+
+void EmitAuxPositionExport(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp) {
+	auto& state = ctx.state;
+	for (uint32_t component = 0; component < 4; component++) {
+		if ((exp.en & (1u << component)) == 0) {
+			continue;
+		}
+		const auto output = IR::DecodePositionExportComponent(
+		    state.input_info.vertex->pa_cl_vs_out_cntl, exp.index, component);
+		if (output.layer) {
+			const auto raw   = ExportRawComponent(ctx, data, component);
+			const auto layer = state.builder.AllocateId();
+			state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), layer, raw,
+			                           ConstantU32(state, 0x7ffu)});
+			state.builder.AddFunction({OpStore, state.layer_variable, layer});
+			continue;
+		}
+		if (!output.point_size && output.clip_distance == UINT32_MAX &&
+		    output.cull_distance == UINT32_MAX) {
+			continue;
+		}
+
+		const auto raw = ExportRawComponent(ctx, data, component);
+		const auto f32 = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+		if (output.point_size) {
+			state.builder.AddFunction({OpStore, state.point_size_variable, f32});
+			continue;
+		}
+		auto StoreDistance = [&](uint32_t variable, uint32_t index) {
+			if (index == UINT32_MAX) {
+				return;
+			}
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32(state)), pointer,
+			     variable, ConstantU32(state, index)});
+			state.builder.AddFunction({OpStore, pointer, f32});
+		};
+		StoreDistance(state.clip_distance_variable, output.clip_distance);
+		StoreDistance(state.cull_distance_variable, output.cull_distance);
+	}
+}
+
+void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&       state = ctx.state;
+	const auto& exp   = ctx.Export(inst);
+	const auto  exec  = ctx.Arg(inst, 1);
+	if (state.stage == ShaderType::Pixel && exp.vm && state.requirements.pixel_valid_mask &&
+	    state.pixel_valid_mask_variable != 0) {
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpSelect, TypeU32(state), value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
+		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
+	}
+	if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
+	    exp.en == 0u) {
+		return;
+	}
+	EmitIfCondition(state, exec, [&]() {
+		const auto data = ctx.Arg(inst, 0);
+		if (exp.kind == IR::ExportTargetKind::Position && exp.index != 0) {
+			EmitAuxPositionExport(ctx, data, exp);
+			return;
+		}
+		if (exp.kind == IR::ExportTargetKind::MrtZ) {
+			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
+				const auto raw = ExportRawComponent(ctx, data, 0);
+				const auto f32 = state.builder.AllocateId();
+				state.builder.AddFunction({OpBitcast, TypeF32(state), f32, raw});
+				state.builder.AddFunction({OpStore, state.depth_variable, f32});
+			}
+			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
+				const auto raw     = ExportRawComponent(ctx, data, 2);
+				const auto value   = state.builder.AllocateId();
+				const auto pointer = state.builder.AllocateId();
+				state.builder.AddFunction({OpBitcast, TypeI32(state), value, raw});
+				state.builder.AddFunction(
+				    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeI32(state)), pointer,
+				     state.sample_mask_variable, ConstantU32(state, 0)});
+				state.builder.AddFunction({OpStore, pointer, value});
+			}
+			return;
+		}
+		const auto variable = OutputVariableForExport(state, exp);
+		if (variable == 0) {
+			return;
+		}
+		const bool uint_output = MrtOutputMode(state, exp) == 7u;
+		const auto vector_type = uint_output ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4);
+		auto       value       = ExportVector(ctx, data, exp, uint_output);
+		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
+			const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];
+			if (!mapping.IsIdentity()) {
+				const auto mapped = state.builder.AllocateId();
+				state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
+				                           mapping.Map(0), mapping.Map(1), mapping.Map(2),
+				                           mapping.Map(3)});
+				value = mapped;
+			}
+		}
+		if (exp.kind == IR::ExportTargetKind::Position) {
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpAccessChain, TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4)),
+			     pointer, variable, ConstantU32(state, 0)});
+			state.builder.AddFunction({OpStore, pointer, value});
+		} else {
+			state.builder.AddFunction({OpStore, variable, value});
+		}
+	});
+}
+
+} // namespace
+
+bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& state = ctx.state;
+	switch (inst.GetOpcode()) {
+		case IR::ValueOpcode::Identity: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
+		case IR::ValueOpcode::Void:
+		case IR::ValueOpcode::Reference:
+		case IR::ValueOpcode::ReferenceU32:
+		case IR::ValueOpcode::ControlNop:
+		case IR::ValueOpcode::Waitcnt:
+		case IR::ValueOpcode::Sendmsg:
+		case IR::ValueOpcode::TtraceData:
+		case IR::ValueOpcode::InstPrefetch: return true;
+		case IR::ValueOpcode::Barrier: {
+			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
+			state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
+			                           ConstantU32(state, ScopeWorkgroup),
+			                           ConstantU32(state, semantics)});
+			return true;
+		}
+		case IR::ValueOpcode::GetUserData: {
+			const auto reg   = inst.Arg(0).ScalarRegister();
+			uint32_t   dword = 0;
+			if (!UserDataDwordIndex(state, reg, dword)) {
+				ctx.Define(inst, ConstantU32(state, 0));
+			} else {
+				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
+			}
+			return true;
+		}
+		case IR::ValueOpcode::GetBuiltin:
+			ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
+			                                inst.Arg(1).U32()));
+			return true;
+		case IR::ValueOpcode::UndefU1:
+		case IR::ValueOpcode::UndefU8:
+		case IR::ValueOpcode::UndefU16:
+		case IR::ValueOpcode::UndefU32:
+		case IR::ValueOpcode::UndefU64:
+			state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
+			return true;
+		case IR::ValueOpcode::DppMoveU32: {
+			const auto flags    = inst.Flags<IR::DppMoveFlags>();
+			const auto target   = EmitDppTargetLane(state, flags.control);
+			const auto shuffled = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
+			                           target.lane});
+			if (flags.fetch_inactive) {
+				ctx.Define(inst, shuffled);
+				return true;
+			}
+			const auto ballot = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
+			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
+			const auto can_fetch     = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpLogicalAnd, TypeBool(state), can_fetch, target.valid, source_active});
+			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
+			return true;
+		}
+		case IR::ValueOpcode::DppUpdateU32: {
+			const auto flags = inst.Flags<IR::DppMoveFlags>();
+			const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
+			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		}
+		case IR::ValueOpcode::WqmU64:
+			ctx.Define(inst, EmitWqmU64(ctx.state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::WqmMask:
+			ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::LaneId:
+			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
+			return true;
+		case IR::ValueOpcode::Ballot:
+			ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
+			return true;
+		case IR::ValueOpcode::ReadFirstLane: {
+			const auto ballot = state.builder.AllocateId();
+			const auto lane   = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), ballot,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
+			state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, TypeU32(state), lane,
+			                           ConstantU32(state, ScopeSubgroup), ballot});
+			ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
+			return true;
+		}
+		case IR::ValueOpcode::ReadLane:
+			ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::WriteLane: {
+			const auto hit = state.builder.AllocateId();
+			state.builder.AddFunction({OpIEqual, TypeBool(state), hit,
+			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
+			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
+			return true;
+		}
+		case IR::ValueOpcode::Permlane16U32: {
+			const auto flags     = inst.Flags<IR::PermlaneFlags>();
+			const auto subid     = EmitSubgroupLocalInvocationId(state);
+			const auto row       = state.builder.AllocateId();
+			const auto row_value = state.builder.AllocateId();
+			const auto lane      = state.builder.AllocateId();
+			const auto lane8     = state.builder.AllocateId();
+			const auto shift     = state.builder.AllocateId();
+			const auto upper     = state.builder.AllocateId();
+			const auto selected  = state.builder.AllocateId();
+			const auto shifted   = state.builder.AllocateId();
+			const auto index     = state.builder.AllocateId();
+			const auto target    = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+			if (flags.x16) {
+				state.builder.AddFunction(
+				    {OpBitwiseXor, TypeU32(state), row_value, row, ConstantU32(state, 16)});
+			} else {
+				state.builder.AddFunction({OpCopyObject, TypeU32(state), row_value, row});
+			}
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, TypeU32(state), lane8, lane, ConstantU32(state, 7)});
+			state.builder.AddFunction(
+			    {OpShiftLeftLogical, TypeU32(state), shift, lane8, ConstantU32(state, 2)});
+			state.builder.AddFunction(
+			    {OpUGreaterThanEqual, TypeBool(state), upper, lane, ConstantU32(state, 8)});
+			state.builder.AddFunction(
+			    {OpSelect, TypeU32(state), selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
+			state.builder.AddFunction(
+			    {OpShiftRightLogical, TypeU32(state), shifted, selected, shift});
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, TypeU32(state), index, shifted, ConstantU32(state, 15)});
+			state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row_value, index});
+			const auto shuffled = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformShuffle, TypeU32(state), shuffled,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
+			                           target});
+			uint32_t result = shuffled;
+			if (!flags.fetch_inactive) {
+				const auto source_exec = state.builder.AllocateId();
+				state.builder.AddFunction({OpGroupNonUniformShuffle, TypeBool(state), source_exec,
+				                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
+				                           target});
+				result = state.builder.AllocateId();
+				state.builder.AddFunction({OpSelect, TypeU32(state), result, source_exec, shuffled,
+				                           ConstantU32(state, 0)});
+			}
+			ctx.Define(inst, result);
+			return true;
+		}
+		case IR::ValueOpcode::GetAttribute:
+			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
+			return true;
+		case IR::ValueOpcode::GetInterpolationParameter:
+			ctx.Define(inst, EmitInterpolationParameter(ctx, inst.Arg(0).U32(), inst.Arg(1).U32(),
+			                                            inst.Arg(2).U32()));
+			return true;
+		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
+		case IR::ValueOpcode::GetShaderBase:
+			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
+			// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
+			ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
+			return true;
+		case IR::ValueOpcode::GetSrtResource:
+		case IR::ValueOpcode::GetBufferResource:
+		case IR::ValueOpcode::GetAddressResource:
+		case IR::ValueOpcode::GetScratchResource:
+		case IR::ValueOpcode::GetImageResource:
+		case IR::ValueOpcode::GetSamplerResource:
+		case IR::ValueOpcode::MakeImageAddress: return true;
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -0,0 +1,244 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t EmitTrueBool(EmitterState& state) {
+	return ConstantBool(state, true);
+}
+
+DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
+	const auto quad_base = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto shift     = state.builder.AllocateId();
+	const auto selected0 = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), quad_base, subid, ConstantU32(state, 0xfffffffcu)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, TypeU32(state), shift, lane, ConstantU32(state, 1)});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, TypeU32(state), selected0, ConstantU32(state, control), shift});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), selected, selected0, ConstantU32(state, 3)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, quad_base, selected});
+	return {target, EmitTrueBool(state)};
+}
+
+DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
+                                        bool left) {
+	const auto row          = state.builder.AllocateId();
+	const auto lane         = state.builder.AllocateId();
+	const auto lane_shifted = state.builder.AllocateId();
+	const auto target       = state.builder.AllocateId();
+	const auto valid        = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	if (left) {
+		state.builder.AddFunction(
+		    {OpIAdd, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
+		state.builder.AddFunction(
+		    {OpULessThan, TypeBool(state), valid, lane, ConstantU32(state, 16u - amount)});
+	} else {
+		state.builder.AddFunction(
+		    {OpISub, TypeU32(state), lane_shifted, lane, ConstantU32(state, amount)});
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, TypeBool(state), valid, lane, ConstantU32(state, amount)});
+	}
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, lane_shifted});
+	return {target, valid};
+}
+
+DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
+                                              uint32_t amount) {
+	const auto row      = state.builder.AllocateId();
+	const auto lane     = state.builder.AllocateId();
+	const auto in_high  = state.builder.AllocateId();
+	const auto minus    = state.builder.AllocateId();
+	const auto plus     = state.builder.AllocateId();
+	const auto selected = state.builder.AllocateId();
+	const auto target   = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 15)});
+	state.builder.AddFunction(
+	    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction({OpISub, TypeU32(state), minus, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction(
+	    {OpIAdd, TypeU32(state), plus, lane, ConstantU32(state, 16u - amount)});
+	state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, minus, plus});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, row, selected});
+	return {target, EmitTrueBool(state)};
+}
+
+DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
+	const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
+	const auto lane_mask = half_row ? 7u : 15u;
+	const auto base      = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto mirrored  = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, base_mask)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, lane_mask)});
+	state.builder.AddFunction(
+	    {OpISub, TypeU32(state), mirrored, ConstantU32(state, lane_mask), lane});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, mirrored});
+	return {target, EmitTrueBool(state)};
+}
+
+DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
+	const auto subid = EmitSubgroupLocalInvocationId(state);
+	if (control <= 0xffu) {
+		return EmitDppQuadPermTargetLane(state, subid, control);
+	}
+	if (control >= 0x101u && control <= 0x10fu) {
+		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
+	}
+	if (control >= 0x111u && control <= 0x11fu) {
+		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
+	}
+	if (control >= 0x121u && control <= 0x12fu) {
+		return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
+	}
+	if (control == 0x140u) {
+		return EmitDppMirrorTargetLane(state, subid, false);
+	}
+	if (control == 0x141u) {
+		return EmitDppMirrorTargetLane(state, subid, true);
+	}
+	return {subid, EmitTrueBool(state)};
+}
+
+uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
+	if (state.subgroup_local_invocation_id_variable == 0) {
+		EXIT("SubgroupLocalInvocationId was not declared before SPIR-V function emission\n");
+	}
+	const auto value = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpLoad, TypeU32(state), value, state.subgroup_local_invocation_id_variable});
+	return value;
+}
+
+uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
+	for (const auto& input: state.inputs) {
+		if (input.kind == kind) {
+			return input.variable_id;
+		}
+	}
+	return 0;
+}
+
+const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
+	for (const auto& input: state.inputs) {
+		if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
+			return &input;
+		}
+	}
+	return nullptr;
+}
+
+uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
+	const auto variable = InputVariableForKind(state, kind);
+	if (variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	const auto pointer = state.builder.AllocateId();
+	const auto value   = state.builder.AllocateId();
+	state.builder.AddFunction({OpAccessChain, TypePointer(state, StorageClassInput, TypeU32(state)),
+	                           pointer, variable, ConstantU32(state, component)});
+	state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+	return value;
+}
+
+uint32_t EmitLocalInvocationIndex(EmitterState& state) {
+	const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
+	if (variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	const auto value = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, TypeU32(state), value, variable});
+	return value;
+}
+
+uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
+                                        uint32_t component) {
+	if ((component & 3u) != 3u) {
+		return ConstantU32(state, 0);
+	}
+	return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
+}
+
+uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
+                                         uint32_t component) {
+	const auto count = VertexParameterComponentCount(state, input);
+	const auto kind  = VertexParameterScalarKind(state, input.location);
+	if (component >= count) {
+		return VertexInputDefaultComponentU32(state, kind, component);
+	}
+
+	const auto scalar_type = VertexParameterScalarType(state, kind);
+	uint32_t   raw         = state.builder.AllocateId();
+	if (count == 1u) {
+		state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
+	} else {
+		const auto pointer_type = VertexParameterScalarPointerType(state, kind);
+		const auto pointer      = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
+		                           ConstantU32(state, component)});
+		state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
+	}
+
+	if (kind == VertexInputScalarKind::Uint) {
+		return raw;
+	}
+
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), bits, raw});
+	return bits;
+}
+
+uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
+	const auto active_ballot = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformBallot, TypeU32Vector(state, 4), active_ballot,
+	                           ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
+	return EmitBallotLaneActiveBool(state, active_ballot, lane);
+}
+uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
+	const auto low = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, TypeU32(state), low, active_ballot, 0});
+	uint32_t mask = low;
+	if (state.wave_size == 64u) {
+		const auto high     = state.builder.AllocateId();
+		const auto in_high  = state.builder.AllocateId();
+		const auto selected = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeExtract, TypeU32(state), high, active_ballot, 1});
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, TypeBool(state), in_high, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, TypeU32(state), selected, in_high, high, low});
+		mask = selected;
+	}
+
+	const auto lane_low = state.builder.AllocateId();
+	const auto bit      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), lane_low, lane, ConstantU32(state, 31)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, TypeU32(state), bit, ConstantU32(state, 1), lane_low});
+
+	const auto hit      = state.builder.AllocateId();
+	const auto active   = state.builder.AllocateId();
+	const auto in_range = state.builder.AllocateId();
+	const auto ret      = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, TypeBool(state), active, hit, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpULessThan, TypeBool(state), in_range, lane, ConstantU32(state, state.wave_size)});
+	state.builder.AddFunction({OpLogicalAnd, TypeBool(state), ret, active, in_range});
+	return ret;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -0,0 +1,818 @@
+#include "graphics/guest_gpu/gpu_format.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+#include <bit>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
+	uint32_t index = 0;
+	for (uint32_t i = 0; i < component; i++) {
+		index += (dmask >> i) & 1u;
+	}
+	return index;
+}
+
+uint32_t DmaskComponent(uint32_t dmask, uint32_t index) {
+	for (uint32_t component = 0; component < 4u; component++) {
+		if (((dmask >> component) & 1u) != 0u && index-- == 0u) return component;
+	}
+	return 0u;
+}
+
+uint32_t ImageGatherComponent(uint32_t dmask) {
+	switch (dmask) {
+		case 0x2u: return 1;
+		case 0x4u: return 2;
+		case 0x8u: return 3;
+		default: return 0;
+	}
+}
+
+uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, value});
+	return result;
+}
+
+uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, lhs, rhs});
+	return result;
+}
+
+bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {
+	return (mem.image_sample_flags & flag) != 0u;
+}
+
+bool UsesA16(const IR::MemoryInfo& mem) {
+	return HasFlag(mem, Decoder::ImageSampleFlagA16);
+}
+
+bool ComponentUsesA16(const IR::MemoryInfo& mem, uint32_t component) {
+	if (!UsesA16(mem)) return false;
+	uint32_t cursor = 0;
+	if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) {
+		if (component == cursor) return false;
+		cursor++;
+	}
+	if (HasFlag(mem, Decoder::ImageSampleFlagBias)) {
+		if (component == cursor) return true;
+		cursor++;
+	}
+	if (HasFlag(mem, Decoder::ImageSampleFlagCompare) && component == cursor) return false;
+	return true;
+}
+
+uint32_t HalfComponent(const IR::MemoryInfo& mem, uint32_t component) {
+	if (!UsesA16(mem)) return component * 2u;
+	uint32_t half = 0;
+	for (uint32_t index = 0; index < component; index++) {
+		const auto width = ComponentUsesA16(mem, index) ? 1u : 2u;
+		if (width == 2u && (half & 1u) != 0u) half++;
+		half += width;
+	}
+	if (!ComponentUsesA16(mem, component) && (half & 1u) != 0u) half++;
+	return half;
+}
+
+uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                    uint32_t component) {
+	const auto half   = HalfComponent(mem, component);
+	const auto packed = half / 2u;
+	if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0);
+	auto value = ctx.Def(address.Arg(packed));
+	if (ComponentUsesA16(mem, component)) {
+		if ((half & 1u) != 0u) {
+			value = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), value,
+			               ConstantU32(ctx.state, 16));
+		}
+		value = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), value,
+		               ConstantU32(ctx.state, 0xffffu));
+	}
+	return value;
+}
+
+uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                    uint32_t component) {
+	const auto value = AddressU32(ctx, mem, address, component);
+	return ComponentUsesA16(mem, component)
+	           ? EmitF16BitsToF32(ctx.state, value)
+	           : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), value);
+}
+
+ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) {
+	ImageSampleLayout layout;
+	uint32_t          cursor = 0;
+	if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
+		layout.grad_x = cursor;
+		cursor += ImageViewSpatialComponents(view);
+		layout.grad_y = cursor;
+		cursor += ImageViewSpatialComponents(view);
+	}
+	layout.coord = cursor;
+	cursor += ImageViewCoordinateComponents(view);
+	if (HasFlag(mem, Decoder::ImageSampleFlagLod)) layout.lod = cursor++;
+	return layout;
+}
+
+uint32_t ZeroF32(EmitterState& state) {
+	return ConstantF32(state, 0);
+}
+
+uint32_t CubeAxis(EmitterState& state, uint32_t value) {
+	return Binary(state, OpFSub, TypeF32(state), value, ConstantF32(state, 0x3f800000u));
+}
+
+uint32_t CubeLayer(EmitterState& state, uint32_t value) {
+	const auto guest = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertFToU, TypeU32(state), guest, value});
+	const auto padding =
+	    Binary(state, OpShiftLeftLogical, TypeU32(state),
+	           Binary(state, OpShiftRightLogical, TypeU32(state), guest, ConstantU32(state, 3)),
+	           ConstantU32(state, 1));
+	const auto host   = Binary(state, OpISub, TypeU32(state), guest, padding);
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertUToF, TypeF32(state), result, host});
+	return result;
+}
+
+uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                  uint32_t first, uint32_t components) {
+	auto x = AddressF32(ctx, mem, address, first);
+	if (components == 1u) return x;
+	auto y = mem.image_address_components > first + 1u ? AddressF32(ctx, mem, address, first + 1u)
+	                                                   : ZeroF32(ctx.state);
+	if (mem.image_cube) {
+		x = CubeAxis(ctx.state, x);
+		y = CubeAxis(ctx.state, y);
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	if (components == 3u) {
+		auto z = mem.image_address_components > first + 2u
+		             ? AddressF32(ctx, mem, address, first + 2u)
+		             : ZeroF32(ctx.state);
+		if (mem.image_cube) z = CubeLayer(ctx.state, z);
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 3), result, x, y, z});
+	} else {
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), result, x, y});
+	}
+	return result;
+}
+
+uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                  ImageViewKind view) {
+	const auto components = ImageViewCoordinateComponents(view);
+	const auto x          = AddressU32(ctx, mem, address, 0);
+	if (components == 1u) return x;
+	const auto y      = mem.image_address_components > 1u ? AddressU32(ctx, mem, address, 1)
+	                                                      : ConstantU32(ctx.state, 0);
+	const auto result = ctx.state.builder.AllocateId();
+	if (components == 3u) {
+		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
+		                                                 : ConstantU32(ctx.state, 0);
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z});
+	} else {
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y});
+	}
+	return result;
+}
+
+uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                ImageViewKind view) {
+	const auto component = ImageViewCoordinateComponents(view);
+	return mem.image_has_mip && mem.image_address_components > component
+	           ? AddressU32(ctx, mem, address, component)
+	           : ConstantU32(ctx.state, 0);
+}
+
+uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
+	return Unary(ctx.state, OpBitcast, TypeU32(ctx.state), value);
+}
+
+uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref,
+                      const IR::MemoryInfo& mem, bool gather = false) {
+	if (mem.data_bits == 16u) {
+		uint32_t   packed[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+		                        ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+		const auto scalar    = [&](uint32_t index) {
+			if (dref) return value;
+			const auto component = gather ? index : DmaskComponent(mem.dmask, index);
+			const auto result    = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction({OpCompositeExtract,
+			                               integer ? TypeU32(ctx.state) : TypeF32(ctx.state),
+			                               result, value, component});
+			return result;
+		};
+		for (uint32_t word = 0; word < mem.data_dwords; word++) {
+			const auto low_index  = word * 2u;
+			const auto high_index = low_index + 1u;
+			const auto low        = scalar(low_index);
+			const auto high = high_index < mem.component_count
+			                      ? scalar(high_index)
+			                      : (integer ? ConstantU32(ctx.state, 0) : ZeroF32(ctx.state));
+			if (integer) {
+				const auto mask = ConstantU32(ctx.state, 0xffffu);
+				packed[word] =
+				    Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state),
+				           Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), low, mask),
+				           Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state),
+				                  Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), high, mask),
+				                  ConstantU32(ctx.state, 16u)));
+			} else {
+				const auto pair = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction(
+				    {OpCompositeConstruct, TypeF32Vector(ctx.state, 2), pair, low, high});
+				packed[word] = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction({OpExtInst, TypeU32(ctx.state), packed[word],
+				                               GlslStd450(ctx.state), GlslPackHalf2x16, pair});
+			}
+		}
+		const auto result = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+		                               packed[0], packed[1], packed[2], packed[3]});
+		return result;
+	}
+	uint32_t component[4] {};
+	for (uint32_t index = 0; index < 4u; index++) {
+		if (dref) {
+			component[index] = index == 0u ? FloatBits(ctx, value) : ConstantU32(ctx.state, 0);
+			continue;
+		}
+		const auto scalar = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpCompositeExtract,
+		                               integer ? TypeU32(ctx.state) : TypeF32(ctx.state), scalar,
+		                               value, index});
+		component[index] = integer ? scalar : FloatBits(ctx, scalar);
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               component[0], component[1], component[2], component[3]});
+	return result;
+}
+
+uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                         const IR::Inst& address) {
+	const auto pc    = inst.Flags<IR::MemoryFlags>().pc;
+	const auto view  = SampledImageViewKind(ctx.state, mem, pc);
+	const auto image = LoadSampledImageDescriptor(ctx.state, mem, pc, view);
+	const auto size  = ctx.state.builder.AllocateId();
+	if (ImageSpirvMultisampled(view) != 0u) {
+		ctx.state.builder.AddFunction(
+		    {OpImageQuerySize, ImageViewSizeType(ctx.state, view), size, image});
+	} else {
+		ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, view),
+		                               size, image, AddressU32(ctx, mem, address, 0)});
+	}
+	const auto components = ImageViewCoordinateComponents(view);
+	uint32_t   result[4]  = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+	                         ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	if (components == 1u) {
+		result[0] = size;
+	} else {
+		for (uint32_t index = 0; index < components; index++) {
+			result[index] = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpCompositeExtract, TypeU32(ctx.state), result[index], size, index});
+		}
+	}
+	if (ImageSpirvMultisampled(view) == 0u) {
+		result[3] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpImageQueryLevels, TypeU32(ctx.state), result[3], image});
+	}
+	const auto vector = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), vector,
+	                               result[0], result[1], result[2], result[3]});
+	return vector;
+}
+
+uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                      const ImageSampleLayout& layout, ImageViewKind view) {
+	const auto components = ImageViewSpatialComponents(view);
+	const auto zero       = ConstantI32(ctx.state, 0);
+	if (layout.offset == NoImageComponent || mem.image_address_components <= layout.offset) {
+		if (components == 1u) return zero;
+		const auto result = ctx.state.builder.AllocateId();
+		if (components == 3u) {
+			ctx.state.builder.AddFunction(
+			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result, zero, zero, zero});
+		} else {
+			ctx.state.builder.AddFunction(
+			    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, zero, zero});
+		}
+		return result;
+	}
+	const auto packed    = Unary(ctx.state, OpBitcast, TypeI32(ctx.state),
+	                             AddressU32(ctx, mem, address, layout.offset));
+	uint32_t   values[3] = {zero, zero, zero};
+	for (uint32_t index = 0; index < components; index++) {
+		values[index] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpBitFieldSExtract, TypeI32(ctx.state), values[index],
+		                               packed, ConstantU32(ctx.state, index * 8u),
+		                               ConstantU32(ctx.state, 6)});
+	}
+	if (components == 1u) return values[0];
+	const auto result = ctx.state.builder.AllocateId();
+	if (components == 3u) {
+		ctx.state.builder.AddFunction({OpCompositeConstruct, TypeI32Vector(ctx.state, 3), result,
+		                               values[0], values[1], values[2]});
+	} else {
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, TypeI32Vector(ctx.state, 2), result, values[0], values[1]});
+	}
+	return result;
+}
+
+uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) {
+	const auto components   = ImageViewSpatialComponents(view);
+	const auto count        = ConstantU32(state, 4);
+	const auto element_type = components == 1u ? TypeI32(state) : TypeI32Vector(state, 2);
+	const auto array_type   = state.builder.Type(OpTypeArray, {element_type, count});
+	uint32_t   offsets[4] {};
+	for (uint32_t index = 0; index < 4u; index++) {
+		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
+		if (components == 1u) {
+			offsets[index] = x;
+		} else {
+			offsets[index] = state.builder.Constant(OpConstantComposite, TypeI32Vector(state, 2),
+			                                        {x, ConstantI32(state, 0)});
+		}
+	}
+	return state.builder.Constant(OpConstantComposite, array_type,
+	                              {offsets[0], offsets[1], offsets[2], offsets[3]});
+}
+
+uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
+	for (uint32_t source = 0; source < 4u; source++) {
+		if (((swizzle >> (source * 3u)) & 7u) == 4u + component) return source;
+	}
+	return UINT32_MAX;
+}
+
+Format::BufferFormatInfo ImageConversionFormat(const EmitterState&   state,
+                                               const IR::MemoryInfo& mem) {
+	const auto format = state.program.info.images[mem.resource].conversion_format;
+	if (format == Prospero::BufferFormat::kInvalid) return {};
+	const auto info = Format::GetFormatInfo(format);
+	EXIT_IF(!Prospero::IsSampledTextureFormat(format) ||
+	        !Prospero::IsUintTextureFormat(format) ||
+	        Prospero::RemapTextureFormat(format) == format ||
+	        info.type != Format::ComponentType::Uint || !info.packed_bitfield ||
+	        info.byte_size != sizeof(uint32_t) || info.component_count == 0u ||
+	        info.component_count > 4u);
+	return info;
+}
+
+uint32_t UnpackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+
+	const auto packed = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeExtract, TypeU32(ctx.state), packed, texel, 0u});
+	uint32_t components[4] = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+	                          ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	for (uint32_t component = 0; component < info.component_count; component++) {
+		components[component] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state),
+		                               components[component], packed,
+		                               ConstantU32(ctx.state, info.component_bit_offset[component]),
+		                               ConstantU32(ctx.state, info.component_bits[component])});
+	}
+	for (uint32_t component = info.component_count; component < 4u; component++) {
+		components[component] = components[component % info.component_count];
+	}
+
+	const auto swizzle = ctx.state.program.info.images[mem.resource].shader_swizzle;
+	uint32_t   selected[4] {};
+	for (uint32_t component = 0; component < 4u; component++) {
+		const auto selector = (swizzle >> (component * 3u)) & 7u;
+		if (selector == 1u) {
+			selected[component] = ConstantU32(ctx.state, 1u);
+		} else if (selector >= 4u) {
+			selected[component] = components[selector - 4u];
+		} else {
+			selected[component] = ConstantU32(ctx.state, 0u);
+		}
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               selected[0], selected[1], selected[2], selected[3]});
+	return result;
+}
+
+uint32_t UnpackImageGather(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t gathered) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return gathered;
+
+	const auto component = ImageGatherComponent(mem.dmask);
+	const auto selector =
+	    (ctx.state.program.info.images[mem.resource].shader_swizzle >> (component * 3u)) & 7u;
+	if (selector < 4u) {
+		const auto value = ConstantU32(ctx.state, selector == 1u ? 1u : 0u);
+		return ctx.state.builder.Constant(OpConstantComposite, TypeU32Vector(ctx.state, 4),
+		                                  {value, value, value, value});
+	}
+
+	uint32_t values[4] {};
+	for (uint32_t lane = 0; lane < 4u; lane++) {
+		const auto packed = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpCompositeExtract, TypeU32(ctx.state), packed, gathered, lane});
+		values[lane]        = ctx.state.builder.AllocateId();
+		const auto physical = (selector - 4u) % info.component_count;
+		ctx.state.builder.AddFunction({OpBitFieldUExtract, TypeU32(ctx.state), values[lane], packed,
+		                               ConstantU32(ctx.state, info.component_bit_offset[physical]),
+		                               ConstantU32(ctx.state, info.component_bits[physical])});
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result,
+	                               values[0], values[1], values[2], values[3]});
+	return result;
+}
+
+uint32_t PackImageTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t texel) {
+	const auto info = ImageConversionFormat(ctx.state, mem);
+	if (info.format == Prospero::BufferFormat::kInvalid) return texel;
+
+	auto packed = ConstantU32(ctx.state, 0u);
+	for (uint32_t component = 0; component < info.component_count; component++) {
+		const auto value = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpCompositeExtract, TypeU32(ctx.state), value, texel, component});
+		const auto maximum =
+		    ConstantU32(ctx.state, info.component_bits[component] == 32u
+		                               ? UINT32_MAX
+		                               : (1u << info.component_bits[component]) - 1u);
+		const auto within  = Binary(ctx.state, OpULessThan, TypeBool(ctx.state), value, maximum);
+		const auto clamped = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpSelect, TypeU32(ctx.state), clamped, within, value, maximum});
+		const auto shifted =
+		    info.component_bit_offset[component] == 0u
+		        ? clamped
+		        : Binary(ctx.state, OpShiftLeftLogical, TypeU32(ctx.state), clamped,
+		                 ConstantU32(ctx.state, info.component_bit_offset[component]));
+		packed = Binary(ctx.state, OpBitwiseOr, TypeU32(ctx.state), packed, shifted);
+	}
+	const auto zero   = ConstantU32(ctx.state, 0u);
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(
+	    {OpCompositeConstruct, TypeU32Vector(ctx.state, 4), result, packed, zero, zero, zero});
+	return result;
+}
+
+uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) {
+	const auto swizzle = ctx.state.program.info.images[mem.resource].shader_swizzle;
+	uint32_t   values[4] {};
+	const auto dmask = mem.dmask != 0u ? mem.dmask : 1u;
+	for (uint32_t component = 0; component < 4u; component++) {
+		const auto source = InverseSwizzle(swizzle, component);
+		uint32_t   raw    = ConstantU32(ctx.state, 0);
+		if (source < 4u && ((dmask >> source) & 1u) != 0u) {
+			const auto packed_index = DmaskComponentIndex(dmask, source);
+			raw                     = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpCompositeExtract, TypeU32(ctx.state), raw, data,
+			     mem.data_bits == 16u ? packed_index / 2u : packed_index});
+			if (mem.data_bits == 16u) {
+				if ((packed_index & 1u) != 0u) {
+					raw = Binary(ctx.state, OpShiftRightLogical, TypeU32(ctx.state), raw,
+					             ConstantU32(ctx.state, 16u));
+				}
+				raw = Binary(ctx.state, OpBitwiseAnd, TypeU32(ctx.state), raw,
+				             ConstantU32(ctx.state, 0xffffu));
+			}
+		}
+		values[component] = integer ? raw
+		                    : mem.data_bits == 16u
+		                        ? EmitF16BitsToF32(ctx.state, raw)
+		                        : Unary(ctx.state, OpBitcast, TypeF32(ctx.state), raw);
+	}
+	const auto texel = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(
+	    {OpCompositeConstruct, integer ? TypeU32Vector(ctx.state, 4) : TypeF32Vector(ctx.state, 4),
+	     texel, values[0], values[1], values[2], values[3]});
+	return PackImageTexel(ctx, mem, texel);
+}
+
+uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
+	switch (opcode) {
+		case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
+		case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
+		default: return 0;
+	}
+}
+
+} // namespace
+
+bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto op         = inst.GetOpcode();
+	const auto image_info = IR::ImageOpcodeInfoOf(op);
+	if (image_info.access == IR::ImageAccess::None) {
+		return false;
+	}
+	auto&       state     = ctx.state;
+	const auto& mem       = ctx.Memory(inst);
+	const auto  pc        = inst.Flags<IR::MemoryFlags>().pc;
+	const auto  image_arg = inst.Arg(0);
+	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
+	const auto* address = ctx.ImageAddress(inst.Arg(image_info.needs_sampler ? 2 : 1));
+	if (address == nullptr) return true;
+	if (op == IR::ValueOpcode::ImageQueryDimensions) {
+		state.builder.RequireCapability(CapabilityImageQuery);
+		ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address));
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageQueryLod) {
+		state.builder.RequireCapability(CapabilityImageQuery);
+		const auto view    = SampledImageViewKind(state, mem, pc);
+		const auto sampled = MakeSampledImage(state, mem, pc, view);
+		const auto lod     = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpImageQueryLod, TypeF32Vector(state, 2), lod, sampled,
+		     CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))});
+		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
+		                      ConstantU32(state, 0)};
+		for (uint32_t index = 0; index < 2u; index++) {
+			const auto component = state.builder.AllocateId();
+			state.builder.AddFunction({OpCompositeExtract, TypeF32(state), component, lod, index});
+			values[index] = FloatBits(ctx, component);
+		}
+		const auto result = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0],
+		                           values[1], values[2], values[3]});
+		ctx.Define(inst, result);
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageRead) {
+		const auto view      = SampledImageViewKind(state, mem, pc);
+		const bool integer   = mem.kind == IR::ResourceKind::ImageUint;
+		const auto condition = ctx.Arg(inst, 2);
+		ctx.Define(
+		    inst,
+		    EmitValueOrDefaultIfCondition(
+		        state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4),
+		        [&]() {
+			        const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
+			        const auto color = state.builder.AllocateId();
+			        const auto coord = CoordU32(ctx, mem, *address, view);
+			        if (ImageSpirvMultisampled(view) != 0u) {
+				        state.builder.AddFunction(
+				            {OpImageFetch,
+				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
+				             image, coord, ImageOperandsSampleMask,
+				             AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))});
+			        } else {
+				        state.builder.AddFunction(
+				            {OpImageFetch,
+				             integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4), color,
+				             image, coord, ImageOperandsLodMask, LodU32(ctx, mem, *address, view)});
+			        }
+			        return ResultVector(ctx, UnpackImageTexel(ctx, mem, color), integer, false,
+			                            mem);
+		        }));
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageWrite) {
+		const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint;
+		const auto view       = StorageImageViewKind(mem);
+		EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
+			const auto mip_lod =
+			    state.program.info.images[mem.resource].mip_mode == IR::ImageMipMode::DynamicStorage
+			        ? LodU32(ctx, mem, *address, view)
+			        : 0u;
+			const auto coord = CoordU32(ctx, mem, *address, view);
+			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
+			EmitStorageImageWrite(state, mem.resource, view, mip_lod, coord, texel);
+		});
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
+		const auto view    = SampledImageViewKind(state, mem, pc);
+		const auto layout  = Layout(mem, view);
+		const bool integer = mem.kind == IR::ResourceKind::ImageUint;
+		const bool dref    = HasFlag(mem, Decoder::ImageSampleFlagCompare);
+		if (dref && state.program.info.images[mem.resource].conversion_format !=
+		                Prospero::BufferFormat::kInvalid) {
+			ctx.Fail(inst, "uses depth comparison with a packed integer image");
+			return true;
+		}
+		const auto coord =
+		    CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view));
+		if (op == IR::ValueOpcode::ImageGatherRaw) {
+			const auto            sampled = MakeSampledImage(state, mem, pc, view);
+			const auto            sample  = state.builder.AllocateId();
+			std::vector<uint32_t> words =
+			    dref ? std::vector<uint32_t> {OpImageDrefGather,
+			                                  TypeF32Vector(state, 4),
+			                                  sample,
+			                                  sampled,
+			                                  coord,
+			                                  layout.dref != NoImageComponent
+			                                      ? AddressF32(ctx, mem, *address, layout.dref)
+			                                      : ZeroF32(state)}
+			         : std::vector<uint32_t> {
+			               OpImageGather,
+			               integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4),
+			               sample,
+			               sampled,
+			               coord,
+			               ConstantU32(state, ImageConversionFormat(state, mem).format ==
+			                                          Prospero::BufferFormat::kInvalid
+			                                      ? ImageGatherComponent(mem.dmask)
+			                                      : 0u)};
+			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
+				words.push_back(ImageOperandsConstOffsetsMask);
+				words.push_back(HorizontalOffsets(state, view));
+			} else if (layout.offset != NoImageComponent) {
+				words.push_back(ImageOperandsOffsetMask);
+				words.push_back(PackedOffset(ctx, mem, *address, layout, view));
+			}
+			state.builder.AddFunction(words);
+			ctx.Define(inst, ResultVector(ctx, UnpackImageGather(ctx, mem, sample),
+			                              integer && !dref, false, mem, true));
+			return true;
+		}
+		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
+		                          HasFlag(mem, Decoder::ImageSampleFlagLod) ||
+		                          HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
+		                          state.stage != ShaderType::Pixel;
+		const auto opcode = explicit_lod
+		                        ? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod)
+		                        : (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod);
+		const auto result_type =
+		    dref ? TypeF32(state) : (integer ? TypeU32Vector(state, 4) : TypeF32Vector(state, 4));
+		const auto dref_value =
+		    dref ? (layout.dref != NoImageComponent ? AddressF32(ctx, mem, *address, layout.dref)
+		                                            : ZeroF32(state))
+		         : 0u;
+		uint32_t              operand_mask = 0;
+		std::vector<uint32_t> operands;
+		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
+			operand_mask |= ImageOperandsGradMask;
+			operands.push_back(
+			    CoordF32(ctx, mem, *address, layout.grad_x, ImageViewSpatialComponents(view)));
+			operands.push_back(
+			    CoordF32(ctx, mem, *address, layout.grad_y, ImageViewSpatialComponents(view)));
+		} else if (explicit_lod) {
+			operand_mask |= ImageOperandsLodMask;
+			operands.push_back(HasFlag(mem, Decoder::ImageSampleFlagLod) &&
+			                           layout.lod != NoImageComponent
+			                       ? AddressF32(ctx, mem, *address, layout.lod)
+			                       : ZeroF32(state));
+		} else if (layout.bias != NoImageComponent) {
+			operand_mask |= ImageOperandsBiasMask;
+			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
+		}
+		const auto EmitSample = [&](uint32_t resource) {
+			const auto            sampled = MakeSampledImage(state, mem, pc, view, resource);
+			const auto            sample  = state.builder.AllocateId();
+			std::vector<uint32_t> words {opcode, result_type, sample, sampled, coord};
+			if (dref) {
+				words.push_back(dref_value);
+			}
+			if (operand_mask != 0u) {
+				words.push_back(operand_mask);
+				words.insert(words.end(), operands.begin(), operands.end());
+			}
+			state.builder.AddFunction(words);
+			return sample;
+		};
+		const auto& image = state.program.info.images[mem.resource];
+		if (image.indirect_root != mem.resource) {
+			const auto sample = EmitSample(mem.resource);
+			ctx.Define(inst, ResultVector(ctx, dref ? sample : UnpackImageTexel(ctx, mem, sample),
+			                              integer, dref, mem));
+			return true;
+		}
+		const auto* handle = image_arg.ResolveInstruction();
+		const auto* source = image.source < ctx.program.descriptor_sources.size()
+		                         ? &ctx.program.descriptor_sources[image.source]
+		                         : nullptr;
+		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
+		    source->indirect_image->key_arg >= handle->NumArgs()) {
+			ctx.Fail(inst, "has invalid indirect image key provenance");
+			return true;
+		}
+		const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg));
+		if (state.flattened_srt_variable == 0 || image.indirect_mapping_capacity == 0u ||
+		    image.indirect_resources.size() < 2u) {
+			ctx.Fail(inst, "has no indirect image runtime mapping");
+			return true;
+		}
+		const auto LoadMapping = [&](uint32_t index) {
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state),
+			                           pointer, state.flattened_srt_variable, ConstantU32(state, 0),
+			                           index});
+			const auto value = state.builder.AllocateId();
+			state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+			return value;
+		};
+		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
+		auto       low      = ConstantU32(state, 0u);
+		auto       high     = LoadMapping(mapping);
+		auto       selected = ConstantU32(state, 0u);
+		for (uint32_t iteration = 0; iteration < std::bit_width(image.indirect_mapping_capacity);
+		     iteration++) {
+			const auto active = Binary(state, OpULessThan, TypeBool(state), low, high);
+			const auto mid =
+			    Binary(state, OpShiftRightLogical, TypeU32(state),
+			           Binary(state, OpIAdd, TypeU32(state), low, high), ConstantU32(state, 1u));
+			const auto probe = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, TypeU32(state), probe, active, mid, ConstantU32(state, 0u)});
+			const auto entry      = Binary(state, OpIAdd, TypeU32(state), mapping,
+			                               Binary(state, OpIAdd, TypeU32(state),
+			                                      Binary(state, OpShiftLeftLogical, TypeU32(state),
+			                                             probe, ConstantU32(state, 1u)),
+			                                      ConstantU32(state, 1u)));
+			const auto mapped_key = LoadMapping(entry);
+			const auto candidate =
+			    LoadMapping(Binary(state, OpIAdd, TypeU32(state), entry, ConstantU32(state, 1u)));
+			const auto equal         = Binary(state, OpIEqual, TypeBool(state), mapped_key, key);
+			const auto match         = Binary(state, OpLogicalAnd, TypeBool(state), active, equal);
+			const auto next_selected = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, TypeU32(state), next_selected, match, candidate, selected});
+			selected              = next_selected;
+			const auto less       = Binary(state, OpULessThan, TypeBool(state), mapped_key, key);
+			const auto take_upper = Binary(state, OpLogicalAnd, TypeBool(state), active, less);
+			const auto take_lower = Binary(state, OpLogicalAnd, TypeBool(state), active,
+			                               Unary(state, OpLogicalNot, TypeBool(state), less));
+			const auto next_low   = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, TypeU32(state), next_low, take_upper,
+			     Binary(state, OpIAdd, TypeU32(state), mid, ConstantU32(state, 1u)), low});
+			low                  = next_low;
+			const auto next_high = state.builder.AllocateId();
+			state.builder.AddFunction({OpSelect, TypeU32(state), next_high, take_lower, mid, high});
+			high = next_high;
+		}
+		const auto            default_label = state.builder.AllocateId();
+		const auto            merge_label   = state.builder.AllocateId();
+		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
+		std::vector<uint32_t> switch_words {OpSwitch, selected, default_label};
+		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
+			labels[candidate - 1u] = state.builder.AllocateId();
+			switch_words.push_back(candidate);
+			switch_words.push_back(labels[candidate - 1u]);
+		}
+		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+		state.builder.AddFunction(switch_words);
+		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
+		EmitLabel(state, default_label);
+		phi_words.push_back(EmitSample(image.indirect_resources[0]));
+		phi_words.push_back(default_label);
+		state.builder.AddFunction({OpBranch, merge_label});
+		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
+			EmitLabel(state, labels[candidate - 1u]);
+			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
+			phi_words.push_back(labels[candidate - 1u]);
+			state.builder.AddFunction({OpBranch, merge_label});
+		}
+		EmitLabel(state, merge_label);
+		state.builder.AddFunction(phi_words);
+		ctx.Define(inst,
+		           ResultVector(ctx, dref ? phi_words[2] : UnpackImageTexel(ctx, mem, phi_words[2]),
+		                        integer, dref, mem));
+		return true;
+	}
+	const auto atomic_opcode = ImageAtomicOpcode(op);
+	if (atomic_opcode != 0u) {
+		const auto view = StorageImageViewKind(mem);
+		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
+			           const auto pointer = state.builder.AllocateId();
+			           const auto pointer_type =
+			               state.builder.Type(OpTypePointer, {StorageClassImage, TypeU32(state)});
+			           state.builder.AddFunction(
+			               {OpImageTexelPointer, pointer_type, pointer,
+			                StorageImageDescriptorPointer(state, mem.resource, view),
+			                CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)});
+			           const auto old = state.builder.AllocateId();
+			           state.builder.AddFunction({atomic_opcode, TypeU32(state), old, pointer,
+			                                      ConstantU32(state, ScopeDevice),
+			                                      ConstantU32(state, MemorySemanticsNone),
+			                                      ctx.Arg(inst, 2)});
+			           EmitDeviceAtomicMemoryBarrier(state);
+			           return old;
+		           }));
+		return true;
+	}
+	return false;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h

@@ -0,0 +1,915 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_
+
+#include "common/common.h"
+#include "common/stringUtils.h"
+#include "graphics/shader/recompiler/BufferFormat.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+
+#include <algorithm>
+#include <array>
+#include <cinttypes>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <iterator>
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+enum : uint32_t {
+	ExecutionModelVertex                     = 0,
+	ExecutionModelFragment                   = 4,
+	ExecutionModelGLCompute                  = 5,
+	ExecutionModeOriginUpperLeft             = 7,
+	ExecutionModeEarlyFragmentTests          = 9,
+	ExecutionModeDepthReplacing              = 12,
+	ExecutionModeLocalSize                   = 17,
+	ExecutionModeSignedZeroInfNanPreserve    = 4461,
+	ExecutionModeDerivativeGroupQuadsKHR     = 5289,
+	AddressingModelLogical                   = 0,
+	AddressingModelPhysicalStorageBuffer64   = 5348,
+	MemoryModelGLSL450                       = 1,
+	CapabilityShader                         = 1,
+	CapabilityInt64                          = 11,
+	CapabilityImageGatherExtended            = 25,
+	CapabilityClipDistance                   = 32,
+	CapabilityCullDistance                   = 33,
+	CapabilitySampled1D                      = 43,
+	CapabilityImage1D                        = 44,
+	CapabilityImageQuery                     = 50,
+	CapabilityStorageImageWriteWithoutFormat = 56,
+	CapabilityGroupNonUniform                = 61,
+	CapabilityGroupNonUniformBallot          = 64,
+	CapabilityGroupNonUniformShuffle         = 65,
+	CapabilitySignedZeroInfNanPreserve       = 4466,
+	CapabilityShaderViewportIndexLayerEXT    = 5254,
+	CapabilityFragmentBarycentricKHR         = 5284,
+	CapabilityComputeDerivativeGroupQuadsKHR = 5288,
+	CapabilityPhysicalStorageBufferAddresses = 5347,
+	StorageClassUniformConstant              = 0,
+	StorageClassInput                        = 1,
+	StorageClassOutput                       = 3,
+	StorageClassWorkgroup                    = 4,
+	StorageClassFunction                     = 7,
+	StorageClassPushConstant                 = 9,
+	StorageClassImage                        = 11,
+	StorageClassStorageBuffer                = 12,
+	StorageClassPhysicalStorageBuffer        = 5349,
+	FunctionControlNone                      = 0,
+	SelectionControlNone                     = 0,
+	LoopControlNone                          = 0,
+};
+
+enum : uint32_t {
+	DecorationBlock         = 2,
+	DecorationBuiltIn       = 11,
+	DecorationNoPerspective = 13,
+	DecorationFlat          = 14,
+	DecorationLocation      = 30,
+	DecorationArrayStride   = 6,
+	DecorationBinding       = 33,
+	DecorationDescriptorSet = 34,
+	DecorationOffset        = 35,
+	DecorationPerVertexKHR  = 5285,
+};
+
+enum : uint32_t {
+	BuiltInPosition                  = 0,
+	BuiltInPointSize                 = 1,
+	BuiltInClipDistance              = 3,
+	BuiltInCullDistance              = 4,
+	BuiltInLayer                     = 9,
+	BuiltInFragCoord                 = 15,
+	BuiltInFrontFacing               = 17,
+	BuiltInSampleMask                = 20,
+	BuiltInFragDepth                 = 22,
+	BuiltInWorkgroupId               = 26,
+	BuiltInLocalInvocationId         = 27,
+	BuiltInGlobalInvocationId        = 28,
+	BuiltInLocalInvocationIndex      = 29,
+	BuiltInSubgroupLocalInvocationId = 41,
+	BuiltInVertexIndex               = 42,
+	BuiltInInstanceIndex             = 43,
+	BuiltInBaryCoordKHR              = 5286,
+	BuiltInBaryCoordNoPerspKHR       = 5287,
+};
+
+enum : uint32_t {
+	Dim1D              = 0,
+	Dim3D              = 2,
+	Dim2D              = 1,
+	ImageFormatUnknown = 0,
+	ImageFormatRgba32f = 1,
+	ImageFormatR32ui   = 33,
+};
+
+enum : uint32_t {
+	ImageOperandsBiasMask         = 0x00000001u,
+	ImageOperandsLodMask          = 0x00000002u,
+	ImageOperandsGradMask         = 0x00000004u,
+	ImageOperandsOffsetMask       = 0x00000010u,
+	ImageOperandsConstOffsetsMask = 0x00000020u,
+	ImageOperandsSampleMask       = 0x00000040u,
+};
+
+enum : uint32_t {
+	ScopeDevice                    = 1,
+	ScopeWorkgroup                 = 2,
+	ScopeSubgroup                  = 3,
+	MemorySemanticsNone            = 0,
+	MemorySemanticsAcquireRelease  = 0x00000008u,
+	MemorySemanticsUniformMemory   = 0x00000040u,
+	MemorySemanticsWorkgroupMemory = 0x00000100u,
+};
+
+enum : uint32_t {
+	OpExtInst                      = 12,
+	OpTypeVoid                     = 19,
+	OpTypeBool                     = 20,
+	OpTypeInt                      = 21,
+	OpTypeFloat                    = 22,
+	OpTypeVector                   = 23,
+	OpTypeImage                    = 25,
+	OpTypeSampler                  = 26,
+	OpTypeSampledImage             = 27,
+	OpTypeArray                    = 28,
+	OpTypeRuntimeArray             = 29,
+	OpTypeStruct                   = 30,
+	OpTypePointer                  = 32,
+	OpTypeFunction                 = 33,
+	OpConstantTrue                 = 41,
+	OpConstantFalse                = 42,
+	OpConstant                     = 43,
+	OpConstantComposite            = 44,
+	OpUndef                        = 1,
+	OpFunction                     = 54,
+	OpFunctionParameter            = 55,
+	OpFunctionEnd                  = 56,
+	OpFunctionCall                 = 57,
+	OpVariable                     = 59,
+	OpImageTexelPointer            = 60,
+	OpLoad                         = 61,
+	OpStore                        = 62,
+	OpAccessChain                  = 65,
+	OpArrayLength                  = 68,
+	OpDecorate                     = 71,
+	OpMemberDecorate               = 72,
+	OpVectorShuffle                = 79,
+	OpCompositeConstruct           = 80,
+	OpCompositeExtract             = 81,
+	OpCopyObject                   = 83,
+	OpSampledImage                 = 86,
+	OpImageSampleImplicitLod       = 87,
+	OpImageSampleExplicitLod       = 88,
+	OpImageSampleDrefImplicitLod   = 89,
+	OpImageSampleDrefExplicitLod   = 90,
+	OpImageFetch                   = 95,
+	OpImageGather                  = 96,
+	OpImageDrefGather              = 97,
+	OpImageWrite                   = 99,
+	OpImageQuerySizeLod            = 103,
+	OpImageQueryLod                = 105,
+	OpImageQuerySize               = 104,
+	OpImageQueryLevels             = 106,
+	OpConvertFToU                  = 109,
+	OpConvertFToS                  = 110,
+	OpConvertSToF                  = 111,
+	OpConvertUToF                  = 112,
+	OpUConvert                     = 113,
+	OpConvertUToPtr                = 120,
+	OpBitcast                      = 124,
+	OpSNegate                      = 126,
+	OpFNegate                      = 127,
+	OpIAdd                         = 128,
+	OpFAdd                         = 129,
+	OpISub                         = 130,
+	OpFSub                         = 131,
+	OpIMul                         = 132,
+	OpFMul                         = 133,
+	OpUDiv                         = 134,
+	OpFDiv                         = 136,
+	OpIAddCarry                    = 149,
+	OpUMulExtended                 = 151,
+	OpSMulExtended                 = 152,
+	OpAny                          = 154,
+	OpAll                          = 155,
+	OpLogicalNotEqual              = 165,
+	OpLogicalOr                    = 166,
+	OpLogicalAnd                   = 167,
+	OpLogicalNot                   = 168,
+	OpSelect                       = 169,
+	OpIEqual                       = 170,
+	OpINotEqual                    = 171,
+	OpUGreaterThan                 = 172,
+	OpSGreaterThan                 = 173,
+	OpUGreaterThanEqual            = 174,
+	OpSGreaterThanEqual            = 175,
+	OpULessThan                    = 176,
+	OpSLessThan                    = 177,
+	OpULessThanEqual               = 178,
+	OpSLessThanEqual               = 179,
+	OpFOrdEqual                    = 180,
+	OpFUnordEqual                  = 181,
+	OpFOrdNotEqual                 = 182,
+	OpFUnordNotEqual               = 183,
+	OpFOrdLessThan                 = 184,
+	OpFUnordLessThan               = 185,
+	OpFOrdGreaterThan              = 186,
+	OpFUnordGreaterThan            = 187,
+	OpFOrdLessThanEqual            = 188,
+	OpFUnordLessThanEqual          = 189,
+	OpFOrdGreaterThanEqual         = 190,
+	OpFUnordGreaterThanEqual       = 191,
+	OpShiftRightLogical            = 194,
+	OpShiftRightArithmetic         = 195,
+	OpShiftLeftLogical             = 196,
+	OpBitwiseOr                    = 197,
+	OpBitwiseXor                   = 198,
+	OpBitwiseAnd                   = 199,
+	OpNot                          = 200,
+	OpBitFieldInsert               = 201,
+	OpBitFieldSExtract             = 202,
+	OpBitFieldUExtract             = 203,
+	OpBitReverse                   = 204,
+	OpBitCount                     = 205,
+	OpControlBarrier               = 224,
+	OpMemoryBarrier                = 225,
+	OpAtomicLoad                   = 227,
+	OpAtomicExchange               = 229,
+	OpAtomicCompareExchange        = 230,
+	OpAtomicIAdd                   = 234,
+	OpAtomicISub                   = 235,
+	OpAtomicSMin                   = 236,
+	OpAtomicUMin                   = 237,
+	OpAtomicSMax                   = 238,
+	OpAtomicUMax                   = 239,
+	OpAtomicAnd                    = 240,
+	OpAtomicOr                     = 241,
+	OpAtomicXor                    = 242,
+	OpPhi                          = 245,
+	OpLoopMerge                    = 246,
+	OpSelectionMerge               = 247,
+	OpLabel                        = 248,
+	OpBranch                       = 249,
+	OpBranchConditional            = 250,
+	OpSwitch                       = 251,
+	OpKill                         = 252,
+	OpReturn                       = 253,
+	OpReturnValue                  = 254,
+	OpGroupNonUniformBallot        = 339,
+	OpGroupNonUniformBallotFindLSB = 343,
+	OpGroupNonUniformShuffle       = 345,
+};
+
+enum : uint32_t {
+	MemoryAccessAlignedMask = 0x2,
+};
+
+enum : uint32_t {
+	GlslRoundEven       = 2,
+	GlslTrunc           = 3,
+	GlslFAbs            = 4,
+	GlslFloor           = 8,
+	GlslCeil            = 9,
+	GlslFract           = 10,
+	GlslSin             = 13,
+	GlslCos             = 14,
+	GlslExp2            = 29,
+	GlslLog2            = 30,
+	GlslSqrt            = 31,
+	GlslInverseSqrt     = 32,
+	GlslFMin            = 37,
+	GlslFMax            = 40,
+	GlslFClamp          = 43,
+	GlslLdexp           = 53,
+	GlslFma             = 50,
+	GlslPackSnorm2x16   = 56,
+	GlslPackUnorm2x16   = 57,
+	GlslPackHalf2x16    = 58,
+	GlslUnpackUnorm2x16 = 61,
+	GlslUnpackHalf2x16  = 62,
+	GlslFindILsb        = 73,
+	GlslFindUMsb        = 75,
+};
+
+struct InputBinding {
+	IR::StageInputKind kind            = IR::StageInputKind::VertexIndex;
+	uint32_t           location        = 0;
+	uint32_t           component_count = 1;
+	uint32_t           variable_id     = 0;
+	std::string        debug_name;
+	bool               per_vertex = false;
+};
+
+struct OutputBinding {
+	IR::StageOutputKind kind        = IR::StageOutputKind::Parameter;
+	uint32_t            index       = 0;
+	uint32_t            location    = 0;
+	uint32_t            variable_id = 0;
+	std::string         debug_name;
+};
+
+struct DescriptorResourceBinding {
+	const IR::DescriptorBinding* descriptor  = nullptr;
+	uint32_t                     array_index = 0;
+};
+
+enum class ImageViewKind {
+	Dim1D,
+	Dim1DArray,
+	Dim2D,
+	Dim2DArray,
+	Dim3D,
+	Dim2DMsaa,
+	Dim2DMsaaArray,
+	Count,
+};
+
+constexpr uint32_t SampledImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Count);
+constexpr uint32_t StorageImageViewKindCount = static_cast<uint32_t>(ImageViewKind::Dim2DMsaa);
+
+enum class StorageImageClass : uint32_t {
+	FormatlessFloat,
+	FormatlessUint,
+	AtomicUint,
+	Count,
+};
+
+struct EmitterState {
+	EmitterState(const IR::Program& program_, const IR::ResourceSnapshot& resources_,
+	             ShaderStageInputInfo input_info_)
+	    : program(program_), resources(resources_), input_info(input_info_),
+	      requirements(*program_.spirv_requirements) {}
+
+	Builder                      builder;
+	const IR::Program&           program;
+	const IR::ResourceSnapshot&  resources;
+	ShaderStageInputInfo         input_info;
+	const IR::SpirvRequirements& requirements;
+	ShaderType                   stage                   = ShaderType::Unknown;
+	uint32_t                     wave_size               = 64;
+	uint32_t                     storage_buffer_variable = 0;
+	std::array<uint32_t, IR::ShaderInfo::MaxBuffers>      memory_byte_offsets {};
+	uint32_t                                             bda_pagetable_variable = 0;
+	uint32_t                                             fault_buffer_variable  = 0;
+	uint32_t                                             bda_pointer_function   = 0;
+	uint32_t                                             gds_variable            = 0;
+	uint32_t                                             gds_length              = 0;
+	uint32_t                                             push_constant_variable  = 0;
+	uint32_t                                             vsharp_storage_variable = 0;
+	uint32_t                                             flattened_srt_variable  = 0;
+	uint32_t                                             lds_variable            = 0;
+	uint32_t                                             scratch_variable        = 0;
+	std::array<uint32_t, 2u * SampledImageViewKindCount> sampled_image_variables {};
+	std::array<uint32_t,
+	           static_cast<uint32_t>(StorageImageClass::Count) * StorageImageViewKindCount>
+	                           storage_image_variables {};
+	uint32_t                   sampler_variable                      = 0;
+	uint32_t                   main_func                             = 0;
+	uint32_t                   entry_label                           = 0;
+	uint32_t                   current_label                         = 0;
+	uint32_t                   pixel_valid_mask_variable             = 0;
+	uint32_t                   subgroup_local_invocation_id_variable = 0;
+	uint32_t                   per_vertex_variable                   = 0;
+	uint32_t                   point_size_variable                   = 0;
+	uint32_t                   clip_distance_variable                = 0;
+	uint32_t                   cull_distance_variable                = 0;
+	uint32_t                   layer_variable                        = 0;
+	uint32_t                   clip_distance_count                   = 0;
+	uint32_t                   cull_distance_count                   = 0;
+	uint32_t                   depth_variable                        = 0;
+	uint32_t                   sample_mask_variable                  = 0;
+	std::vector<InputBinding>  inputs;
+	std::vector<OutputBinding> outputs;
+	std::vector<uint32_t>      interface_variables;
+};
+
+uint32_t TypeVoid(EmitterState& state);
+uint32_t TypeBool(EmitterState& state);
+uint32_t TypeBoolVector(EmitterState& state, uint32_t components);
+uint32_t TypeU32(EmitterState& state);
+uint32_t TypeU64(EmitterState& state);
+uint32_t TypeDeviceAddress(EmitterState& state);
+uint32_t TypeU32Pair(EmitterState& state);
+uint32_t TypeI32(EmitterState& state);
+uint32_t TypeI32Pair(EmitterState& state);
+uint32_t TypeF32(EmitterState& state);
+uint32_t TypeU32Vector(EmitterState& state, uint32_t components);
+
+uint32_t TypeU32Composite(EmitterState& state, uint32_t components);
+uint32_t TypeI32Vector(EmitterState& state, uint32_t components);
+uint32_t TypeF32Vector(EmitterState& state, uint32_t components);
+uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee);
+uint32_t TypeFunction(EmitterState& state);
+uint32_t TypeStorageBufferPointer(EmitterState& state);
+uint32_t TypeStorageBufferElementPointer(EmitterState& state);
+uint32_t TypeDeviceAddressStoragePointer(EmitterState& state);
+uint32_t TypePhysicalU32Pointer(EmitterState& state);
+uint32_t TypePushConstantElementPointer(EmitterState& state);
+uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords);
+uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class);
+
+inline void EmitLabel(EmitterState& state, uint32_t label) {
+	state.current_label = label;
+	state.builder.AddFunction({OpLabel, label});
+}
+
+struct ValueEmitContext {
+	ValueEmitContext(EmitterState& state_, const IR::Program& program_)
+	    : state(state_), program(program_) {}
+
+	uint32_t              Def(IR::Value value);
+	uint32_t              Arg(const IR::Inst& inst, size_t index);
+	uint32_t              Result(const IR::Inst& inst);
+	uint32_t              TypeId(IR::Type type) const;
+	uint32_t              Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
+	                           std::initializer_list<uint32_t> args);
+	uint32_t              Define(const IR::Inst& inst, uint32_t value);
+	uint32_t              ResourceIndex(IR::Value value, IR::ValueOpcode opcode);
+	const IR::Inst*       ImageAddress(IR::Value value);
+	const IR::MemoryInfo& Memory(const IR::Inst& inst) const;
+	const IR::ExportInfo& Export(const IR::Inst& inst) const;
+	uint32_t              Label(const IR::Block* block) const;
+	void                  Fail(const IR::Inst& inst, const char* reason);
+
+	EmitterState&                                                      state;
+	const IR::Program&                                            program;
+	std::unordered_map<const IR::Inst*, uint32_t>                      definitions;
+	std::unordered_map<const IR::Block*, uint32_t>                     labels;
+	const std::unordered_map<const IR::Inst*, uint32_t>*               dispatcher_spills = nullptr;
+	std::unordered_map<const IR::Inst*, std::pair<uint32_t, uint32_t>> dispatcher_block_loads;
+	const IR::Block*                                                   current_block = nullptr;
+	uint32_t                                                           scratch_u32_variable = 0;
+	bool                                                               failed               = false;
+	std::string                                                        error;
+};
+
+enum class VertexInputScalarKind { Float, Sint, Uint };
+
+constexpr uint32_t NoImageComponent = 0xffffffffu;
+
+struct DppTargetLane {
+	uint32_t lane  = 0;
+	uint32_t valid = 0;
+};
+
+struct ImageSampleLayout {
+	uint32_t offset = NoImageComponent;
+	uint32_t dref   = NoImageComponent;
+	uint32_t bias   = NoImageComponent;
+	uint32_t coord  = 0;
+	uint32_t lod    = NoImageComponent;
+	uint32_t grad_x = NoImageComponent;
+	uint32_t grad_y = NoImageComponent;
+};
+
+constexpr uint32_t SampledImageIndex(bool integer, ImageViewKind view) {
+	return static_cast<uint32_t>(view) + (integer ? SampledImageViewKindCount : 0u);
+}
+
+constexpr StorageImageClass StorageImageClassFor(bool uint_image, bool atomic) {
+	if (atomic) {
+		return StorageImageClass::AtomicUint;
+	}
+	if (uint_image) {
+		return StorageImageClass::FormatlessUint;
+	}
+	return StorageImageClass::FormatlessFloat;
+}
+
+constexpr uint32_t StorageImageIndex(StorageImageClass image_class, ImageViewKind view) {
+	return static_cast<uint32_t>(view) +
+	       static_cast<uint32_t>(image_class) * StorageImageViewKindCount;
+}
+
+constexpr IR::DescriptorBindingKind SampledBindingKind(bool integer, ImageViewKind view) {
+	if (integer) {
+		switch (view) {
+			case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::SampledUint1D;
+			case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::SampledUint1DArray;
+			case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::SampledUint2D;
+			case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::SampledUint2DArray;
+			case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::SampledUint3D;
+			case ImageViewKind::Dim2DMsaa: return IR::DescriptorBindingKind::SampledUint2DMsaa;
+			case ImageViewKind::Dim2DMsaaArray:
+				return IR::DescriptorBindingKind::SampledUint2DMsaaArray;
+			default: break;
+		}
+	}
+	switch (view) {
+		case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::Sampled1D;
+		case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::Sampled1DArray;
+		case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::Sampled2D;
+		case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Sampled2DArray;
+		case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Sampled3D;
+		case ImageViewKind::Dim2DMsaa: return IR::DescriptorBindingKind::Sampled2DMsaa;
+		case ImageViewKind::Dim2DMsaaArray: return IR::DescriptorBindingKind::Sampled2DMsaaArray;
+		default: break;
+	}
+	return IR::DescriptorBindingKind::Count;
+}
+
+constexpr IR::DescriptorBindingKind StorageBindingKind(StorageImageClass image_class,
+                                                       ImageViewKind     view) {
+	switch (image_class) {
+		case StorageImageClass::FormatlessFloat:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::Storage1D;
+				case ImageViewKind::Dim1DArray: return IR::DescriptorBindingKind::Storage1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::Storage2D;
+				case ImageViewKind::Dim2DArray: return IR::DescriptorBindingKind::Storage2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::Storage3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::FormatlessUint:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageUint1D;
+				case ImageViewKind::Dim1DArray:
+					return IR::DescriptorBindingKind::StorageUint1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageUint2D;
+				case ImageViewKind::Dim2DArray:
+					return IR::DescriptorBindingKind::StorageUint2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageUint3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::AtomicUint:
+			switch (view) {
+				case ImageViewKind::Dim1D: return IR::DescriptorBindingKind::StorageAtomic1D;
+				case ImageViewKind::Dim1DArray:
+					return IR::DescriptorBindingKind::StorageAtomic1DArray;
+				case ImageViewKind::Dim2D: return IR::DescriptorBindingKind::StorageAtomic2D;
+				case ImageViewKind::Dim2DArray:
+					return IR::DescriptorBindingKind::StorageAtomic2DArray;
+				case ImageViewKind::Dim3D: return IR::DescriptorBindingKind::StorageAtomic3D;
+				default: break;
+			}
+			break;
+		case StorageImageClass::Count: break;
+	}
+	return IR::DescriptorBindingKind::Count;
+}
+
+constexpr uint32_t ImageSpirvDimension(ImageViewKind view) {
+	switch (view) {
+		case ImageViewKind::Dim1D:
+		case ImageViewKind::Dim1DArray: return Dim1D;
+		case ImageViewKind::Dim2D:
+		case ImageViewKind::Dim2DArray:
+		case ImageViewKind::Dim2DMsaa:
+		case ImageViewKind::Dim2DMsaaArray:
+		case ImageViewKind::Count: return Dim2D;
+		case ImageViewKind::Dim3D: return Dim3D;
+	}
+	return Dim2D;
+}
+
+constexpr uint32_t ImageSpirvArrayed(ImageViewKind view) {
+	return view == ImageViewKind::Dim1DArray || view == ImageViewKind::Dim2DArray ||
+	               view == ImageViewKind::Dim2DMsaaArray
+	           ? 1u
+	           : 0u;
+}
+
+constexpr uint32_t ImageSpirvMultisampled(ImageViewKind view) {
+	return view == ImageViewKind::Dim2DMsaa || view == ImageViewKind::Dim2DMsaaArray ? 1u : 0u;
+}
+
+struct F32Class {
+	uint32_t bits = 0;
+	uint32_t nan  = 0;
+	uint32_t zero = 0;
+};
+
+uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr);
+
+uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr);
+
+bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr);
+
+bool PixelParameterIsCustom(const EmitterState& state, uint32_t attr);
+
+VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location);
+
+uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input);
+
+uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind);
+
+uint32_t VertexParameterScalarPointerType(EmitterState& state, VertexInputScalarKind kind);
+
+uint32_t VertexParameterVectorOrScalarType(EmitterState& state, VertexInputScalarKind kind,
+                                           uint32_t components);
+
+uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
+                                         uint32_t components);
+
+void SetError(std::string* error, const char* message);
+
+bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind, uint32_t index);
+
+void CopyProgramInputsAndOutputs(EmitterState& state, const IR::Program& program);
+
+uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo& exp);
+
+uint32_t ConstantU32(EmitterState& state, uint32_t value);
+
+uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
+
+[[noreturn]] void ExitDescriptorBindingFailure(const EmitterState&       state,
+                                               IR::DescriptorBindingKind kind, uint32_t resource,
+                                               const char* reason);
+
+DescriptorResourceBinding ResourceForDescriptor(const EmitterState&       state,
+                                                IR::DescriptorBindingKind kind, uint32_t resource);
+
+uint32_t DescriptorElementPointer(EmitterState& state, uint32_t result_ptr_type,
+                                  uint32_t variable_id, uint32_t array_index,
+                                  IR::DescriptorBindingKind kind, uint32_t resource,
+                                  const char* variable_name);
+
+ImageViewKind SampledImageViewKind(const EmitterState& state, const IR::MemoryInfo& mem,
+                                   uint32_t use_pc);
+
+uint32_t ImageViewCoordinateComponents(ImageViewKind view);
+
+uint32_t ImageViewSpatialComponents(ImageViewKind view);
+
+uint32_t ImageViewImageType(EmitterState& state, ImageViewKind view, bool integer);
+
+uint32_t ImageViewSampledImageType(EmitterState& state, ImageViewKind view, bool integer);
+
+uint32_t ImageViewSizeType(EmitterState& state, ImageViewKind view);
+
+uint32_t StorageImageType(EmitterState& state, StorageImageClass image_class, ImageViewKind view);
+
+uint32_t StorageImagePointerType(EmitterState& state, StorageImageClass image_class,
+                                 ImageViewKind view);
+
+uint32_t LoadSampledImageDescriptor(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                                    ImageViewKind view);
+
+uint32_t LoadSamplerDescriptor(EmitterState& state, uint32_t sampler, uint32_t use_pc);
+
+uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                          ImageViewKind view);
+uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                          ImageViewKind view, uint32_t image_resource);
+
+ImageViewKind StorageImageViewKind(const IR::MemoryInfo& mem);
+
+uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, ImageViewKind view);
+
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, ImageViewKind view,
+                           uint32_t mip_lod, uint32_t coord, uint32_t texel);
+
+uint32_t ExecutionModelForStage(ShaderType stage);
+
+uint32_t ConstantU32(EmitterState& state, uint32_t value);
+
+uint32_t ConstantI32(EmitterState& state, int32_t value);
+
+uint32_t ConstantF32(EmitterState& state, uint32_t bits);
+
+uint32_t FloatBits(float value);
+
+uint32_t ConstantF32Value(EmitterState& state, float value);
+
+uint32_t ConstantBool(EmitterState& state, bool value);
+
+uint32_t ConstantU64(EmitterState& state, uint64_t value);
+
+uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components);
+
+uint32_t GlslStd450(EmitterState& state);
+
+void AllocateInputVariables(EmitterState& state);
+
+void AllocateOutputVariables(EmitterState& state);
+
+uint32_t BuiltInForInput(IR::StageInputKind kind);
+
+void AddInputAnnotationsAndNames(EmitterState& state);
+
+void AddOutputAnnotationsAndNames(EmitterState& state);
+
+void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name,
+                        IR::DescriptorBindingKind kind);
+
+void AddDescriptorAnnotationsAndNames(EmitterState& state);
+
+void DefineModule(EmitterState& state);
+
+uint32_t EmitTrueBool(EmitterState& state);
+
+DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
+
+DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
+                                        bool left);
+
+DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid, uint32_t amount);
+
+DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row);
+
+DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control);
+
+uint32_t EmitSubgroupLocalInvocationId(EmitterState& state);
+
+uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind);
+
+const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location);
+
+uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
+                                         uint32_t component);
+
+uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component);
+
+uint32_t EmitLocalInvocationIndex(EmitterState& state);
+
+uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t ballot, uint32_t lane);
+
+uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane);
+
+uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+
+uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs);
+
+uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index);
+
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem);
+
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem);
+
+void EmitMemoryOffsets(EmitterState& state);
+
+uint32_t LdsDwordCount(const EmitterState& state);
+
+struct MemoryResourceAccess {
+	IR::ResourceKind kind             = IR::ResourceKind::None;
+	uint32_t         object_pointer   = 0;
+	uint32_t         length           = 0;
+	uint32_t         index_offset     = 0;
+	bool             add_index_offset = false;
+};
+
+MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem);
+
+uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
+                                uint32_t raw_index);
+
+uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
+                                   uint32_t index);
+
+uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                  uint32_t index);
+
+uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value);
+
+uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value);
+
+uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value);
+
+uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+                                       uint32_t constant);
+
+uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
+                              uint32_t false_value);
+
+bool IsSignedFormatComponent(Format::ComponentType type);
+
+uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw);
+
+uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits);
+
+uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
+                                  uint32_t component, uint32_t raw);
+
+void EmitDeviceAtomicMemoryBarrier(EmitterState& state);
+
+uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control);
+
+uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
+                            uint32_t false_value);
+
+void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
+                                   uint32_t& out_low, uint32_t& out_high);
+
+void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
+                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high);
+
+uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
+
+uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift);
+
+uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+
+uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+                                uint32_t constant);
+
+uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value);
+
+uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32);
+
+uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
+
+uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
+
+uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value);
+
+uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value);
+
+uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs);
+
+uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
+
+uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs);
+
+uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value);
+
+F32Class EmitClassifyF32(EmitterState& state, uint32_t value);
+
+uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
+                               uint32_t class_match);
+
+uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask);
+
+uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value);
+
+uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value);
+
+uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value);
+
+uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero);
+
+uint32_t EmitFNegateValue(EmitterState& state, uint32_t value);
+
+uint32_t EmitFAbsValue(EmitterState& state, uint32_t value);
+
+uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits);
+
+bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst);
+
+bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst);
+
+bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst);
+
+bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst);
+
+bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error);
+
+void DefineGetBdaPointer(EmitterState& state);
+
+// These templates accept local lambdas from several emitter translation units.
+template <typename Fn>
+void EmitIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
+	const auto then_label  = state.builder.AllocateId();
+	const auto merge_label = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction({OpBranchConditional, condition, then_label, merge_label});
+	EmitLabel(state, then_label);
+	fn();
+	state.builder.AddFunction({OpBranch, merge_label});
+	EmitLabel(state, merge_label);
+}
+
+template <typename Fn>
+uint32_t EmitValueOrDefaultIfCondition(EmitterState& state, uint32_t condition, uint32_t type,
+                                       uint32_t default_value, Fn&& fn) {
+	const auto then_label  = state.builder.AllocateId();
+	const auto then_exit   = state.builder.AllocateId();
+	const auto else_label  = state.builder.AllocateId();
+	const auto merge_label = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction({OpBranchConditional, condition, then_label, else_label});
+	EmitLabel(state, then_label);
+	const auto then_value = fn();
+	state.builder.AddFunction({OpBranch, then_exit});
+	EmitLabel(state, then_exit);
+	state.builder.AddFunction({OpBranch, merge_label});
+	EmitLabel(state, else_label);
+	state.builder.AddFunction({OpBranch, merge_label});
+	EmitLabel(state, merge_label);
+	const auto value = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpPhi, type, value, then_value, then_exit, default_value, else_label});
+	return value;
+}
+
+template <typename Fn>
+uint32_t EmitValueOrZeroIfCondition(EmitterState& state, uint32_t condition, Fn&& fn) {
+	return EmitValueOrDefaultIfCondition(state, condition, TypeU32(state), ConstantU32(state, 0),
+	                                     std::forward<Fn>(fn));
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SPIRVEMITTER_INTERNAL_H_ */

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -0,0 +1,362 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
+	if (state.push_constant_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, TypePushConstantElementPointer(state), pointer,
+		                           state.push_constant_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		return value;
+	}
+	if (state.vsharp_storage_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+		                           state.vsharp_storage_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, TypeU32(state), value, pointer});
+		return value;
+	}
+	return ConstantU32(state, 0);
+}
+
+uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAdd, TypeU32(state), ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, TypeU32(state), ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem) {
+	if (mem.resource >= state.program.info.buffers.size()) {
+		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
+		                             "buffer specialization is missing");
+	}
+	return state.program.info.buffers[mem.resource].packed_stride;
+}
+
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem) {
+	if (mem.resource >= state.program.info.buffers.size()) {
+		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
+		                             "buffer specialization is missing");
+	}
+	return state.program.info.buffers[mem.resource].descriptor_format;
+}
+
+void EmitMemoryOffsets(EmitterState& state) {
+	for (uint32_t i = 0; i < state.program.bindings.memory_offset_count; i++) {
+		const auto word =
+		    EmitShaderDataDwordLoad(state, state.program.bindings.memory_offset_dword + i / 4u);
+		const auto shift             = ConstantU32(state, (i % 4u) * 8u);
+		state.memory_byte_offsets[i] = EmitBinaryU32(
+		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    ConstantU32(state, 0xffu));
+	}
+}
+
+uint32_t LdsDwordCount(const EmitterState& state) {
+	return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
+}
+
+static void EnsureLdsStorage(EmitterState& state) {
+	if (state.lds_variable != 0) {
+		return;
+	}
+	if (state.stage != ShaderType::Compute) {
+		EXIT("function LDS was not prepared before SPIR-V function emission\n");
+	}
+	state.lds_variable = state.builder.DefineGlobalVariable(
+	    TypeU32ArrayPointer(state, StorageClassWorkgroup, LdsDwordCount(state)),
+	    StorageClassWorkgroup);
+	state.builder.AddName(state.lds_variable, "lds_dwords");
+}
+
+MemoryResourceAccess PrepareMemoryResourceAccess(EmitterState& state, const IR::MemoryInfo& mem) {
+	MemoryResourceAccess access {.kind = mem.kind};
+	switch (mem.kind) {
+		case IR::ResourceKind::Lds:
+			EnsureLdsStorage(state);
+			access.object_pointer = state.lds_variable;
+			access.length         = ConstantU32(state, LdsDwordCount(state));
+			return access;
+		case IR::ResourceKind::Gds:
+			if (state.gds_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Gds, mem.resource,
+				                             "GDS binding was not emitted");
+			}
+			access.object_pointer = state.gds_variable;
+			if (state.gds_length == 0) {
+				EXIT("GDS length was not prepared at function entry\n");
+			}
+			access.length = state.gds_length;
+			return access;
+		case IR::ResourceKind::Scratch:
+			if (state.scratch_variable == 0) {
+				EXIT("scratch storage was not prepared before SPIR-V function emission\n");
+			}
+			access.object_pointer = state.scratch_variable;
+			access.length         = ConstantU32(state, state.program.scratch_dwords);
+			return access;
+		case IR::ResourceKind::ScalarAddress:
+		case IR::ResourceKind::Flat:
+		case IR::ResourceKind::Global:
+			EXIT("physical address memory must use the BDA emitter\n");
+		case IR::ResourceKind::ScalarBuffer:
+		case IR::ResourceKind::Buffer: {
+			if (state.storage_buffer_variable == 0) {
+				ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers,
+				                             mem.resource,
+				                             "storage buffer descriptor array was not emitted");
+			}
+			const auto binding =
+			    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
+			access.object_pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, TypeStorageBufferPointer(state),
+			                           access.object_pointer, state.storage_buffer_variable,
+			                           ConstantU32(state, binding.array_index)});
+			access.index_offset     = EmitBinaryU32(state, OpShiftRightLogical,
+			                                        state.memory_byte_offsets[binding.array_index],
+			                                        ConstantU32(state, 2u));
+			access.add_index_offset = true;
+			break;
+		}
+		default: EXIT("unsupported memory resource kind: %u\n", static_cast<unsigned>(mem.kind));
+	}
+	access.length = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpArrayLength, TypeU32(state), access.length, access.object_pointer, 0});
+	return access;
+}
+
+uint32_t EmitMemoryElementIndex(EmitterState& state, const MemoryResourceAccess& access,
+                                uint32_t raw_index) {
+	return access.add_index_offset ? EmitAddU32(state, raw_index, access.index_offset) : raw_index;
+}
+
+uint32_t EmitMemoryElementInBounds(EmitterState& state, const MemoryResourceAccess& access,
+                                   uint32_t index) {
+	const auto in_bounds = state.builder.AllocateId();
+	state.builder.AddFunction({OpULessThan, TypeBool(state), in_bounds, index, access.length});
+	return in_bounds;
+}
+
+uint32_t EmitMemoryElementPointer(EmitterState& state, const MemoryResourceAccess& access,
+                                  uint32_t index) {
+	const auto pointer = state.builder.AllocateId();
+	if (access.kind == IR::ResourceKind::Lds || access.kind == IR::ResourceKind::Scratch) {
+		const auto storage_class = access.kind == IR::ResourceKind::Scratch ? StorageClassFunction
+		                           : state.stage == ShaderType::Compute     ? StorageClassWorkgroup
+		                                                                    : StorageClassFunction;
+		state.builder.AddFunction({OpAccessChain, TypeU32ElementPointer(state, storage_class),
+		                           pointer, access.object_pointer, index});
+	} else {
+		state.builder.AddFunction({OpAccessChain, TypeStorageBufferElementPointer(state), pointer,
+		                           access.object_pointer, ConstantU32(state, 0), index});
+	}
+	return pointer;
+}
+
+uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeU32(state), ret, value});
+	return ret;
+}
+
+uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeF32(state), ret, value});
+	return ret;
+}
+
+uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, TypeI32(state), ret, value});
+	return ret;
+}
+
+uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+                                       uint32_t constant) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, TypeBool(state), ret, value, ConstantU32(state, constant)});
+	return ret;
+}
+
+uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
+                              uint32_t false_value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelect, TypeF32(state), ret, condition, true_value, false_value});
+	return ret;
+}
+
+bool IsSignedFormatComponent(Format::ComponentType type) {
+	return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm ||
+	       type == Format::ComponentType::Sscaled;
+}
+
+uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) {
+	const auto unpacked = state.builder.AllocateId();
+	const auto value    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, TypeF32Vector(state, 2), unpacked, GlslStd450(state), GlslUnpackHalf2x16, raw});
+	state.builder.AddFunction({OpCompositeExtract, TypeF32(state), value, unpacked, 0});
+	return EmitTBufferBitcastF32ToU32(state, value);
+}
+
+uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
+	const auto mantissa_bits = bits == 11u ? 6u : 5u;
+	const auto mantissa_mask = (1u << mantissa_bits) - 1u;
+	const auto mantissa =
+	    EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
+	const auto exponent = EmitBinaryU32(
+	    state, OpBitwiseAnd,
+	    EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
+	    ConstantU32(state, 0x1fu));
+
+	const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
+	const auto exponent_bits =
+	    EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
+	const auto mantissa_bits_32 =
+	    EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits));
+	const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32);
+	const auto normal      = EmitTBufferBitcastU32ToF32(state, normal_bits);
+
+	const auto special_bits =
+	    EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
+	const auto special = EmitTBufferBitcastU32ToF32(state, special_bits);
+
+	const auto mantissa_f32 = state.builder.AllocateId();
+	const auto subnormal    = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertUToF, TypeF32(state), mantissa_f32, mantissa});
+	state.builder.AddFunction(
+	    {OpFMul, TypeF32(state), subnormal, mantissa_f32,
+	     ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
+
+	const auto zero_exp    = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0);
+	const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31);
+	const auto finite      = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
+	const auto result      = EmitTBufferSelectF32(state, special_exp, special, finite);
+	return EmitTBufferBitcastF32ToU32(state, result);
+}
+
+uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
+                                  uint32_t component, uint32_t raw) {
+	const auto bits = info.component_bits[component];
+	switch (info.type) {
+		case Format::ComponentType::Uint:
+		case Format::ComponentType::Sint: return raw;
+		case Format::ComponentType::Uscaled: {
+			const auto value = state.builder.AllocateId();
+			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
+			return EmitTBufferBitcastF32ToU32(state, value);
+		}
+		case Format::ComponentType::Sscaled: {
+			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
+			const auto value      = state.builder.AllocateId();
+			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
+			return EmitTBufferBitcastF32ToU32(state, value);
+		}
+		case Format::ComponentType::Unorm: {
+			const auto value      = state.builder.AllocateId();
+			const auto normalized = state.builder.AllocateId();
+			const auto max_value  = static_cast<float>((1u << bits) - 1u);
+			state.builder.AddFunction({OpConvertUToF, TypeF32(state), value, raw});
+			state.builder.AddFunction(
+			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
+			return EmitTBufferBitcastF32ToU32(state, normalized);
+		}
+		case Format::ComponentType::Snorm: {
+			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
+			const auto value      = state.builder.AllocateId();
+			const auto normalized = state.builder.AllocateId();
+			const auto clamped    = state.builder.AllocateId();
+			const auto max_value  = static_cast<float>((1u << (bits - 1u)) - 1u);
+			state.builder.AddFunction({OpConvertSToF, TypeF32(state), value, signed_raw});
+			state.builder.AddFunction(
+			    {OpFDiv, TypeF32(state), normalized, value, ConstantF32Value(state, max_value)});
+			state.builder.AddFunction({OpExtInst, TypeF32(state), clamped, GlslStd450(state),
+			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			return EmitTBufferBitcastF32ToU32(state, clamped);
+		}
+		case Format::ComponentType::Float:
+			if (bits == 32u) {
+				return raw;
+			}
+			if (bits == 16u) {
+				return EmitHalfToF32Bits(state, raw);
+			}
+			return EmitUFloatToF32Bits(state, raw, bits);
+		default: return raw;
+	}
+}
+
+void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
+	const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory;
+	state.builder.AddFunction(
+	    {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
+}
+
+uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
+	if ((control & 0xc000u) == 0xc000u) {
+		const uint32_t mask         = control & 0x1fu;
+		const uint32_t rotate       = (control >> 5u) & 0x1fu;
+		const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate;
+		const auto     lane         = EmitAndConstant(state, subid, 31);
+		const auto     rotated_sum  = EmitAddU32(state, lane, ConstantU32(state, rotate_delta));
+		const auto     rotated      = EmitAndConstant(state, rotated_sum, 31);
+		const auto     kept         = EmitAndConstant(state, lane, mask);
+		const auto     moved        = EmitAndConstant(state, rotated, (~mask) & 31u);
+		const auto     combined     = EmitOrU32(state, kept, moved);
+		const auto     base         = EmitAndConstant(state, subid, 0xffffffe0u);
+		return EmitOrU32(state, base, combined);
+	}
+
+	if ((control & 0x8000u) != 0) {
+		const auto lane2  = state.builder.AllocateId();
+		const auto shift  = state.builder.AllocateId();
+		const auto perm0  = state.builder.AllocateId();
+		const auto perm   = state.builder.AllocateId();
+		const auto base   = state.builder.AllocateId();
+		const auto target = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, TypeU32(state), lane2, subid, ConstantU32(state, 3)});
+		state.builder.AddFunction(
+		    {OpShiftLeftLogical, TypeU32(state), shift, lane2, ConstantU32(state, 1)});
+		state.builder.AddFunction(
+		    {OpShiftRightLogical, TypeU32(state), perm0, ConstantU32(state, control), shift});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, TypeU32(state), perm, perm0, ConstantU32(state, 3)});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xfffffffcu)});
+		state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, perm});
+		return target;
+	}
+
+	const auto lane   = state.builder.AllocateId();
+	const auto masked = state.builder.AllocateId();
+	const auto ored   = state.builder.AllocateId();
+	const auto xored  = state.builder.AllocateId();
+	const auto base   = state.builder.AllocateId();
+	const auto target = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, TypeU32(state), lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), masked, lane, ConstantU32(state, control & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseOr, TypeU32(state), ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseXor, TypeU32(state), xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, TypeU32(state), base, subid, ConstantU32(state, 0xffffffe0u)});
+	state.builder.AddFunction({OpBitwiseOr, TypeU32(state), target, base, xored});
+	return target;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -0,0 +1,858 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+const IR::DescriptorBinding* DescriptorBinding(const EmitterState&       state,
+                                               IR::DescriptorBindingKind kind);
+uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind);
+
+uint32_t TypeVoid(EmitterState& state) {
+	return state.builder.Type(OpTypeVoid);
+}
+
+uint32_t TypeBool(EmitterState& state) {
+	return state.builder.Type(OpTypeBool);
+}
+
+uint32_t TypeBoolVector(EmitterState& state, uint32_t components) {
+	return state.builder.Type(OpTypeVector, {TypeBool(state), components});
+}
+
+uint32_t TypeU32(EmitterState& state) {
+	return state.builder.Type(OpTypeInt, {32, 0});
+}
+
+uint32_t TypeU64(EmitterState& state) {
+	return TypeU32Vector(state, 2);
+}
+
+uint32_t TypeDeviceAddress(EmitterState& state) {
+	return state.builder.Type(OpTypeInt, {64, 0});
+}
+
+uint32_t TypeU32Pair(EmitterState& state) {
+	const auto element = TypeU32(state);
+	return state.builder.Type(OpTypeStruct, {element, element});
+}
+
+uint32_t TypeI32(EmitterState& state) {
+	return state.builder.Type(OpTypeInt, {32, 1});
+}
+
+uint32_t TypeI32Pair(EmitterState& state) {
+	const auto element = TypeI32(state);
+	return state.builder.Type(OpTypeStruct, {element, element});
+}
+
+uint32_t TypeF32(EmitterState& state) {
+	return state.builder.Type(OpTypeFloat, {32});
+}
+
+uint32_t TypeU32Vector(EmitterState& state, uint32_t components) {
+	return state.builder.Type(OpTypeVector, {TypeU32(state), components});
+}
+
+uint32_t TypeU32Composite(EmitterState& state, uint32_t components) {
+	EXIT_IF(components < 2u || components > 4u);
+	return components == 2u ? TypeU32Pair(state) : TypeU32Vector(state, components);
+}
+
+uint32_t TypeI32Vector(EmitterState& state, uint32_t components) {
+	return state.builder.Type(OpTypeVector, {TypeI32(state), components});
+}
+
+uint32_t TypeF32Vector(EmitterState& state, uint32_t components) {
+	return state.builder.Type(OpTypeVector, {TypeF32(state), components});
+}
+
+uint32_t TypePointer(EmitterState& state, uint32_t storage_class, uint32_t pointee) {
+	return state.builder.Type(OpTypePointer, {storage_class, pointee});
+}
+
+uint32_t TypeFunction(EmitterState& state) {
+	return state.builder.Type(OpTypeFunction, {TypeVoid(state)});
+}
+
+uint32_t StorageRuntimeArrayType(EmitterState& state) {
+	return state.builder.DecoratedType(OpTypeRuntimeArray, {TypeU32(state)},
+	                                   {{OpDecorate, {DecorationArrayStride, sizeof(uint32_t)}}});
+}
+
+uint32_t StorageBufferType(EmitterState& state) {
+	return state.builder.DecoratedType(
+	    OpTypeStruct, {StorageRuntimeArrayType(state)},
+	    {{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}});
+}
+
+uint32_t TypeStorageBufferPointer(EmitterState& state) {
+	return TypePointer(state, StorageClassStorageBuffer, StorageBufferType(state));
+}
+
+uint32_t TypeStorageBufferElementPointer(EmitterState& state) {
+	return TypePointer(state, StorageClassStorageBuffer, TypeU32(state));
+}
+
+uint32_t TypeDeviceAddressStoragePointer(EmitterState& state) {
+	return TypePointer(state, StorageClassStorageBuffer, TypeDeviceAddress(state));
+}
+
+uint32_t TypePhysicalU32Pointer(EmitterState& state) {
+	return TypePointer(state, StorageClassPhysicalStorageBuffer, TypeU32(state));
+}
+
+uint32_t TypePushConstantElementPointer(EmitterState& state) {
+	return TypePointer(state, StorageClassPushConstant, TypeU32(state));
+}
+
+uint32_t TypeU32ArrayPointer(EmitterState& state, uint32_t storage_class, uint32_t dwords) {
+	const auto count = ConstantU32(state, std::max(dwords, 1u));
+	const auto array = state.builder.Type(OpTypeArray, {TypeU32(state), count});
+	return TypePointer(state, storage_class, array);
+}
+
+uint32_t TypeU32ElementPointer(EmitterState& state, uint32_t storage_class) {
+	return TypePointer(state, storage_class, TypeU32(state));
+}
+
+namespace {
+
+uint32_t PushConstantArrayType(EmitterState& state) {
+	const auto count =
+	    ConstantU32(state, state.program.bindings.push_constant_size / sizeof(uint32_t));
+	return state.builder.DecoratedType(OpTypeArray, {TypeU32(state), count},
+	                                   {{OpDecorate, {DecorationArrayStride, sizeof(uint32_t)}}});
+}
+
+uint32_t PushConstantBlockType(EmitterState& state) {
+	return state.builder.DecoratedType(
+	    OpTypeStruct, {PushConstantArrayType(state)},
+	    {{OpMemberDecorate, {0, DecorationOffset, state.program.bindings.push_constant_offset}},
+	     {OpDecorate, {DecorationBlock}}});
+}
+
+uint32_t PerVertexType(EmitterState& state) {
+	return state.builder.DecoratedType(OpTypeStruct, {TypeF32Vector(state, 4)},
+	                                   {{OpMemberDecorate, {0, DecorationBuiltIn, BuiltInPosition}},
+	                                    {OpDecorate, {DecorationBlock}}});
+}
+
+uint32_t SampleMaskArrayType(EmitterState& state) {
+	return state.builder.Type(OpTypeArray, {TypeI32(state), ConstantU32(state, 1)});
+}
+
+uint32_t BdaPagetableType(EmitterState& state) {
+	const auto array = state.builder.DecoratedType(
+	    OpTypeRuntimeArray, {TypeDeviceAddress(state)},
+	    {{OpDecorate, {DecorationArrayStride, sizeof(uint64_t)}}});
+	return state.builder.DecoratedType(
+	    OpTypeStruct, {array},
+	    {{OpMemberDecorate, {0, DecorationOffset, 0}}, {OpDecorate, {DecorationBlock}}});
+}
+
+uint32_t F32ArrayType(EmitterState& state, uint32_t count) {
+	return state.builder.Type(OpTypeArray, {TypeF32(state), ConstantU32(state, count)});
+}
+
+void DefineDescriptorVariables(EmitterState& state) {
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::Buffers) != nullptr) {
+		const auto count =
+		    ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Buffers));
+		const auto array_type = state.builder.Type(OpTypeArray, {StorageBufferType(state), count});
+		const auto pointer_type = TypePointer(state, StorageClassStorageBuffer, array_type);
+		state.storage_buffer_variable =
+		    state.builder.DefineGlobalVariable(pointer_type, StorageClassStorageBuffer);
+	}
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::BdaPagetable) != nullptr) {
+		state.bda_pagetable_variable = state.builder.DefineGlobalVariable(
+		    TypePointer(state, StorageClassStorageBuffer, BdaPagetableType(state)),
+		    StorageClassStorageBuffer);
+	}
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::FaultBuffer) != nullptr) {
+		state.fault_buffer_variable = state.builder.DefineGlobalVariable(
+		    TypeStorageBufferPointer(state), StorageClassStorageBuffer);
+	}
+	if (state.program.bindings.push_constant_size != 0) {
+		const auto pointer_type =
+		    TypePointer(state, StorageClassPushConstant, PushConstantBlockType(state));
+		state.push_constant_variable =
+		    state.builder.DefineGlobalVariable(pointer_type, StorageClassPushConstant);
+	}
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::UserData) != nullptr) {
+		state.vsharp_storage_variable = state.builder.DefineGlobalVariable(
+		    TypeStorageBufferPointer(state), StorageClassStorageBuffer);
+	}
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::FlattenedSrt) != nullptr) {
+		state.flattened_srt_variable = state.builder.DefineGlobalVariable(
+		    TypeStorageBufferPointer(state), StorageClassStorageBuffer);
+	}
+	for (uint32_t i = 0; i < state.sampled_image_variables.size(); i++) {
+		const auto view    = static_cast<ImageViewKind>(i % SampledImageViewKindCount);
+		const bool integer = i >= SampledImageViewKindCount;
+		const auto kind    = SampledBindingKind(integer, view);
+		if (DescriptorBinding(state, kind) == nullptr) {
+			continue;
+		}
+		const auto count        = ConstantU32(state, DescriptorCount(state, kind));
+		const auto image_type   = ImageViewImageType(state, view, integer);
+		const auto array_type   = state.builder.Type(OpTypeArray, {image_type, count});
+		const auto pointer_type = TypePointer(state, StorageClassUniformConstant, array_type);
+		state.sampled_image_variables[i] =
+		    state.builder.DefineGlobalVariable(pointer_type, StorageClassUniformConstant);
+		if (view == ImageViewKind::Dim1D || view == ImageViewKind::Dim1DArray) {
+			state.builder.RequireCapability(CapabilitySampled1D);
+		}
+	}
+	for (uint32_t i = 0; i < state.storage_image_variables.size(); i++) {
+		const auto view        = static_cast<ImageViewKind>(i % StorageImageViewKindCount);
+		const auto image_class = static_cast<StorageImageClass>(i / StorageImageViewKindCount);
+		const auto kind        = StorageBindingKind(image_class, view);
+		if (DescriptorBinding(state, kind) == nullptr) {
+			continue;
+		}
+		const auto count        = ConstantU32(state, DescriptorCount(state, kind));
+		const auto image_type   = StorageImageType(state, image_class, view);
+		const auto array_type   = state.builder.Type(OpTypeArray, {image_type, count});
+		const auto pointer_type = TypePointer(state, StorageClassUniformConstant, array_type);
+		state.storage_image_variables[i] =
+		    state.builder.DefineGlobalVariable(pointer_type, StorageClassUniformConstant);
+		if (view == ImageViewKind::Dim1D || view == ImageViewKind::Dim1DArray) {
+			state.builder.RequireCapability(CapabilityImage1D);
+		}
+	}
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::Samplers) != nullptr) {
+		const auto sampler_type = state.builder.Type(OpTypeSampler);
+		const auto count =
+		    ConstantU32(state, DescriptorCount(state, IR::DescriptorBindingKind::Samplers));
+		const auto array_type   = state.builder.Type(OpTypeArray, {sampler_type, count});
+		const auto pointer_type = TypePointer(state, StorageClassUniformConstant, array_type);
+		state.sampler_variable =
+		    state.builder.DefineGlobalVariable(pointer_type, StorageClassUniformConstant);
+	}
+	if (DescriptorBinding(state, IR::DescriptorBindingKind::Gds) != nullptr) {
+		state.gds_variable = state.builder.DefineGlobalVariable(TypeStorageBufferPointer(state),
+		                                                        StorageClassStorageBuffer);
+	}
+}
+
+} // namespace
+
+const IR::DescriptorBinding* DescriptorBinding(const EmitterState&       state,
+                                               IR::DescriptorBindingKind kind) {
+	return IR::FindBinding(state.program.bindings, kind);
+}
+
+uint32_t DescriptorCount(const EmitterState& state, IR::DescriptorBindingKind kind) {
+	const auto* binding = DescriptorBinding(state, kind);
+	return binding != nullptr ? static_cast<uint32_t>(binding->resources.size()) : 0;
+}
+
+uint32_t ConstantU32(EmitterState& state, uint32_t value) {
+	return state.builder.Constant(OpConstant, TypeU32(state), {value});
+}
+
+uint32_t ConstantI32(EmitterState& state, int32_t value) {
+	return state.builder.Constant(OpConstant, TypeI32(state), {static_cast<uint32_t>(value)});
+}
+
+uint32_t ConstantF32(EmitterState& state, uint32_t bits) {
+	return state.builder.Constant(OpConstant, TypeF32(state), {bits});
+}
+
+uint32_t FloatBits(float value) {
+	uint32_t bits = 0;
+	std::memcpy(&bits, &value, sizeof(bits));
+	return bits;
+}
+
+uint32_t ConstantF32Value(EmitterState& state, float value) {
+	return ConstantF32(state, FloatBits(value));
+}
+
+uint32_t ConstantBool(EmitterState& state, bool value) {
+	return state.builder.Constant(value ? OpConstantTrue : OpConstantFalse, TypeBool(state));
+}
+
+uint32_t ConstantU64(EmitterState& state, uint64_t value) {
+	return state.builder.Constant(OpConstantComposite, TypeU64(state),
+	                              {ConstantU32(state, static_cast<uint32_t>(value)),
+	                               ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
+}
+
+uint32_t ConstantU32CompositeZero(EmitterState& state, uint32_t components) {
+	EXIT_IF(components < 2u || components > 4u);
+	const auto            zero = ConstantU32(state, 0);
+	std::vector<uint32_t> values(components, zero);
+	return state.builder.Constant(OpConstantComposite, TypeU32Composite(state, components), values);
+}
+
+uint32_t GlslStd450(EmitterState& state) {
+	return state.builder.Import("GLSL.std.450");
+}
+
+VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
+	if (state.stage != ShaderType::Vertex || location >= ShaderVertexInputInfo::RES_MAX ||
+	    location >= static_cast<uint32_t>(state.input_info.vertex->resources_num)) {
+		return VertexInputScalarKind::Float;
+	}
+
+	switch (state.input_info.vertex->resources[location].Format()) {
+		case Prospero::BufferFormat::k8UInt:
+		case Prospero::BufferFormat::k16UInt:
+		case Prospero::BufferFormat::k8_8UInt:
+		case Prospero::BufferFormat::k32UInt:
+		case Prospero::BufferFormat::k16_16UInt:
+		case Prospero::BufferFormat::k8_8_8_8UInt:
+		case Prospero::BufferFormat::k32_32UInt:
+		case Prospero::BufferFormat::k16_16_16_16UInt:
+		case Prospero::BufferFormat::k32_32_32UInt:
+		case Prospero::BufferFormat::k32_32_32_32UInt: return VertexInputScalarKind::Uint;
+		case Prospero::BufferFormat::k8SInt:
+		case Prospero::BufferFormat::k16SInt:
+		case Prospero::BufferFormat::k8_8SInt:
+		case Prospero::BufferFormat::k32SInt:
+		case Prospero::BufferFormat::k16_16SInt:
+		case Prospero::BufferFormat::k8_8_8_8SInt:
+		case Prospero::BufferFormat::k32_32SInt:
+		case Prospero::BufferFormat::k16_16_16_16SInt:
+		case Prospero::BufferFormat::k32_32_32SInt:
+		case Prospero::BufferFormat::k32_32_32_32SInt: return VertexInputScalarKind::Sint;
+		default: return VertexInputScalarKind::Float;
+	}
+}
+
+uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) {
+	uint32_t count = input.component_count;
+	if (state.stage == ShaderType::Vertex && input.location < ShaderVertexInputInfo::RES_MAX &&
+	    input.location < static_cast<uint32_t>(state.input_info.vertex->resources_num) &&
+	    state.input_info.vertex->resources_dst[input.location].registers_num > 0) {
+		count = static_cast<uint32_t>(
+		    state.input_info.vertex->resources_dst[input.location].registers_num);
+	}
+	return std::clamp(count, 1u, 4u);
+}
+
+uint32_t VertexParameterScalarType(EmitterState& state, VertexInputScalarKind kind) {
+	switch (kind) {
+		case VertexInputScalarKind::Sint: return TypeI32(state);
+		case VertexInputScalarKind::Uint: return TypeU32(state);
+		case VertexInputScalarKind::Float:
+		default: return TypeF32(state);
+	}
+}
+
+uint32_t VertexParameterScalarPointerType(EmitterState& state, VertexInputScalarKind kind) {
+	switch (kind) {
+		case VertexInputScalarKind::Sint:
+			return TypePointer(state, StorageClassInput, TypeI32(state));
+		case VertexInputScalarKind::Uint:
+			return TypePointer(state, StorageClassInput, TypeU32(state));
+		case VertexInputScalarKind::Float:
+		default: return TypePointer(state, StorageClassInput, TypeF32(state));
+	}
+}
+
+uint32_t VertexParameterVectorOrScalarType(EmitterState& state, VertexInputScalarKind kind,
+                                           uint32_t components) {
+	switch (kind) {
+		case VertexInputScalarKind::Sint:
+			switch (components) {
+				case 1: return TypeI32(state);
+				case 2: return TypeI32Vector(state, 2);
+				case 3: return TypeI32Vector(state, 3);
+				default: return TypeI32Vector(state, 4);
+			}
+		case VertexInputScalarKind::Uint:
+			switch (components) {
+				case 1: return TypeU32(state);
+				case 2: return TypeU32Vector(state, 2);
+				case 3: return TypeU32Vector(state, 3);
+				default: return TypeU32Vector(state, 4);
+			}
+		case VertexInputScalarKind::Float:
+		default:
+			switch (components) {
+				case 1: return TypeF32(state);
+				case 2: return TypeF32Vector(state, 2);
+				case 3: return TypeF32Vector(state, 3);
+				default: return TypeF32Vector(state, 4);
+			}
+	}
+}
+
+uint32_t VertexParameterInputPointerType(EmitterState& state, VertexInputScalarKind kind,
+                                         uint32_t components) {
+	switch (kind) {
+		case VertexInputScalarKind::Sint:
+			switch (components) {
+				case 1: return TypePointer(state, StorageClassInput, TypeI32(state));
+				case 2: return TypePointer(state, StorageClassInput, TypeI32Vector(state, 2));
+				case 3: return TypePointer(state, StorageClassInput, TypeI32Vector(state, 3));
+				default: return TypePointer(state, StorageClassInput, TypeI32Vector(state, 4));
+			}
+		case VertexInputScalarKind::Uint:
+			switch (components) {
+				case 1: return TypePointer(state, StorageClassInput, TypeU32(state));
+				case 2: return TypePointer(state, StorageClassInput, TypeU32Vector(state, 2));
+				case 3: return TypePointer(state, StorageClassInput, TypeU32Vector(state, 3));
+				default: return TypePointer(state, StorageClassInput, TypeU32Vector(state, 4));
+			}
+		case VertexInputScalarKind::Float:
+		default:
+			switch (components) {
+				case 1: return TypePointer(state, StorageClassInput, TypeF32(state));
+				case 2: return TypePointer(state, StorageClassInput, TypeF32Vector(state, 2));
+				case 3: return TypePointer(state, StorageClassInput, TypeF32Vector(state, 3));
+				default: return TypePointer(state, StorageClassInput, TypeF32Vector(state, 4));
+			}
+	}
+}
+
+static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) {
+	return state.stage == ShaderType::Pixel &&
+	       index < std::size(state.input_info.pixel->target_output_mode) &&
+	       state.input_info.pixel->target_output_mode[index] == 7u;
+}
+
+void AllocateInputVariables(EmitterState& state) {
+	for (auto& binding: state.inputs) {
+		binding.variable_id = state.builder.AllocateId();
+		state.interface_variables.push_back(binding.variable_id);
+	}
+	if (state.requirements.subgroup_local_invocation_id) {
+		state.subgroup_local_invocation_id_variable = state.builder.AllocateId();
+		state.interface_variables.push_back(state.subgroup_local_invocation_id_variable);
+	}
+}
+
+static uint32_t AllocateInterfaceVariable(EmitterState& state) {
+	const auto variable = state.builder.AllocateId();
+	state.interface_variables.push_back(variable);
+	return variable;
+}
+
+static uint32_t AllocateSharedOutputVariable(EmitterState& state, uint32_t& variable) {
+	if (variable == 0) {
+		variable = AllocateInterfaceVariable(state);
+	}
+	return variable;
+}
+
+void AllocateOutputVariables(EmitterState& state) {
+	for (auto& binding: state.outputs) {
+		switch (binding.kind) {
+			case IR::StageOutputKind::Position:
+				binding.variable_id =
+				    AllocateSharedOutputVariable(state, state.per_vertex_variable);
+				break;
+			case IR::StageOutputKind::PointSize:
+				binding.variable_id =
+				    AllocateSharedOutputVariable(state, state.point_size_variable);
+				break;
+			case IR::StageOutputKind::ClipDistance:
+				binding.variable_id =
+				    AllocateSharedOutputVariable(state, state.clip_distance_variable);
+				state.clip_distance_count = std::max(state.clip_distance_count, binding.index + 1);
+				break;
+			case IR::StageOutputKind::CullDistance:
+				binding.variable_id =
+				    AllocateSharedOutputVariable(state, state.cull_distance_variable);
+				state.cull_distance_count = std::max(state.cull_distance_count, binding.index + 1);
+				break;
+			case IR::StageOutputKind::Layer:
+				binding.variable_id = AllocateSharedOutputVariable(state, state.layer_variable);
+				break;
+			case IR::StageOutputKind::Depth:
+				binding.variable_id = AllocateSharedOutputVariable(state, state.depth_variable);
+				break;
+			case IR::StageOutputKind::SampleMask:
+				binding.variable_id =
+				    AllocateSharedOutputVariable(state, state.sample_mask_variable);
+				break;
+			case IR::StageOutputKind::Parameter:
+			case IR::StageOutputKind::Mrt:
+				binding.variable_id = AllocateInterfaceVariable(state);
+				break;
+		}
+	}
+}
+
+uint32_t BuiltInForInput(IR::StageInputKind kind) {
+	switch (kind) {
+		case IR::StageInputKind::VertexIndex: return BuiltInVertexIndex;
+		case IR::StageInputKind::InstanceIndex: return BuiltInInstanceIndex;
+		case IR::StageInputKind::FragCoord: return BuiltInFragCoord;
+		case IR::StageInputKind::FrontFacing: return BuiltInFrontFacing;
+		case IR::StageInputKind::BaryCoordSmooth: return BuiltInBaryCoordKHR;
+		case IR::StageInputKind::BaryCoordNoPerspective: return BuiltInBaryCoordNoPerspKHR;
+		case IR::StageInputKind::WorkgroupId: return BuiltInWorkgroupId;
+		case IR::StageInputKind::LocalInvocationId: return BuiltInLocalInvocationId;
+		case IR::StageInputKind::LocalInvocationIndex: return BuiltInLocalInvocationIndex;
+		case IR::StageInputKind::GlobalInvocationId: return BuiltInGlobalInvocationId;
+		default: return UINT32_MAX;
+	}
+}
+
+void AddInputAnnotationsAndNames(EmitterState& state) {
+	if (state.subgroup_local_invocation_id_variable != 0) {
+		state.builder.AddName(state.subgroup_local_invocation_id_variable,
+		                      "gl_SubgroupInvocationID");
+		state.builder.AddAnnotation({OpDecorate, state.subgroup_local_invocation_id_variable,
+		                             DecorationBuiltIn, BuiltInSubgroupLocalInvocationId});
+		if (state.stage == ShaderType::Pixel) {
+			state.builder.AddAnnotation(
+			    {OpDecorate, state.subgroup_local_invocation_id_variable, DecorationFlat});
+		}
+	}
+	for (const auto& input: state.inputs) {
+		state.builder.AddName(input.variable_id, input.debug_name.c_str());
+		if (input.kind == IR::StageInputKind::Parameter) {
+			const auto flat = PixelParameterIsFlat(state, input.location);
+			if (input.per_vertex) {
+				state.builder.AddAnnotation(
+				    {OpDecorate, input.variable_id, DecorationPerVertexKHR});
+			} else if (flat) {
+				state.builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
+			}
+			if (state.stage == ShaderType::Pixel && state.input_info.pixel->ps_no_perspective &&
+			    !flat && !input.per_vertex) {
+				state.builder.AddAnnotation(
+				    {OpDecorate, input.variable_id, DecorationNoPerspective});
+			}
+			const auto location = PixelParameterLocation(state, input.location);
+			state.builder.AddAnnotation(
+			    {OpDecorate, input.variable_id, DecorationLocation, location});
+			continue;
+		}
+		const auto builtin = BuiltInForInput(input.kind);
+		if (builtin != UINT32_MAX) {
+			state.builder.AddAnnotation(
+			    {OpDecorate, input.variable_id, DecorationBuiltIn, builtin});
+		}
+	}
+}
+
+void AddOutputAnnotationsAndNames(EmitterState& state) {
+	if (state.per_vertex_variable != 0) {
+		state.builder.AddName(PerVertexType(state), "gl_PerVertex");
+		state.builder.AddName(state.per_vertex_variable, "outPerVertex");
+	}
+	auto AddBuiltIn = [&](uint32_t variable, const char* name, uint32_t builtin) {
+		if (variable != 0) {
+			state.builder.AddName(variable, name);
+			state.builder.AddAnnotation({OpDecorate, variable, DecorationBuiltIn, builtin});
+		}
+	};
+	AddBuiltIn(state.point_size_variable, "gl_PointSize", BuiltInPointSize);
+	AddBuiltIn(state.clip_distance_variable, "gl_ClipDistance", BuiltInClipDistance);
+	AddBuiltIn(state.cull_distance_variable, "gl_CullDistance", BuiltInCullDistance);
+	AddBuiltIn(state.layer_variable, "gl_Layer", BuiltInLayer);
+	if (state.depth_variable != 0) {
+		state.builder.AddName(state.depth_variable, "gl_FragDepth");
+		state.builder.AddAnnotation(
+		    {OpDecorate, state.depth_variable, DecorationBuiltIn, BuiltInFragDepth});
+	}
+	if (state.sample_mask_variable != 0) {
+		state.builder.AddName(state.sample_mask_variable, "gl_SampleMask");
+		state.builder.AddAnnotation(
+		    {OpDecorate, state.sample_mask_variable, DecorationBuiltIn, BuiltInSampleMask});
+	}
+	for (const auto& binding: state.outputs) {
+		if (binding.kind == IR::StageOutputKind::Parameter ||
+		    binding.kind == IR::StageOutputKind::Mrt) {
+			state.builder.AddName(binding.variable_id, binding.debug_name.c_str());
+			state.builder.AddAnnotation(
+			    {OpDecorate, binding.variable_id, DecorationLocation, binding.location});
+		}
+	}
+}
+
+void DecorateDescriptor(EmitterState& state, uint32_t variable, const char* name,
+                        IR::DescriptorBindingKind kind) {
+	if (variable == 0) {
+		return;
+	}
+	state.builder.AddName(variable, name);
+	state.builder.AddAnnotation({OpDecorate, variable, DecorationDescriptorSet, 0});
+	state.builder.AddAnnotation(
+	    {OpDecorate, variable, DecorationBinding, IR::NativeBinding(state.program.stage, kind)});
+}
+
+void AddDescriptorAnnotationsAndNames(EmitterState& state) {
+	auto Decorate = [&](uint32_t variable, const char* name, IR::DescriptorBindingKind kind) {
+		DecorateDescriptor(state, variable, name, kind);
+	};
+	if (state.storage_buffer_variable != 0) {
+		Decorate(state.storage_buffer_variable, "buffers", IR::DescriptorBindingKind::Buffers);
+	}
+	if (state.bda_pagetable_variable != 0) {
+		Decorate(state.bda_pagetable_variable, "bda_pagetable",
+		         IR::DescriptorBindingKind::BdaPagetable);
+	}
+	if (state.fault_buffer_variable != 0) {
+		Decorate(state.fault_buffer_variable, "fault_buffer",
+		         IR::DescriptorBindingKind::FaultBuffer);
+	}
+	constexpr const char* SampledNames[] = {"sampled_1d",
+	                                        "sampled_1d_array",
+	                                        "sampled_2d",
+	                                        "sampled_2d_array",
+	                                        "sampled_3d",
+	                                        "sampled_2d_msaa",
+	                                        "sampled_2d_msaa_array",
+	                                        "sampled_uint_1d",
+	                                        "sampled_uint_1d_array",
+	                                        "sampled_uint_2d",
+	                                        "sampled_uint_2d_array",
+	                                        "sampled_uint_3d",
+	                                        "sampled_uint_2d_msaa",
+	                                        "sampled_uint_2d_msaa_array"};
+	for (uint32_t i = 0; i < state.sampled_image_variables.size(); i++) {
+		const auto view = static_cast<ImageViewKind>(i % SampledImageViewKindCount);
+		Decorate(state.sampled_image_variables[i], SampledNames[i],
+		         SampledBindingKind(i >= SampledImageViewKindCount, view));
+	}
+	constexpr const char* StorageNames[] = {"storage_1d",
+	                                        "storage_1d_array",
+	                                        "storage_2d",
+	                                        "storage_2d_array",
+	                                        "storage_3d",
+	                                        "storage_uint_1d",
+	                                        "storage_uint_1d_array",
+	                                        "storage_uint_2d",
+	                                        "storage_uint_2d_array",
+	                                        "storage_uint_3d",
+	                                        "storage_atomic_1d",
+	                                        "storage_atomic_1d_array",
+	                                        "storage_atomic_2d",
+	                                        "storage_atomic_2d_array",
+	                                        "storage_atomic_3d"};
+	for (uint32_t i = 0; i < state.storage_image_variables.size(); i++) {
+		const auto view        = static_cast<ImageViewKind>(i % StorageImageViewKindCount);
+		const auto image_class = static_cast<StorageImageClass>(i / StorageImageViewKindCount);
+		Decorate(state.storage_image_variables[i], StorageNames[i],
+		         StorageBindingKind(image_class, view));
+	}
+	if (state.sampler_variable != 0) {
+		Decorate(state.sampler_variable, "samplers", IR::DescriptorBindingKind::Samplers);
+	}
+	if (state.gds_variable != 0) {
+		Decorate(state.gds_variable, "gds", IR::DescriptorBindingKind::Gds);
+	}
+	if (state.flattened_srt_variable != 0) {
+		Decorate(state.flattened_srt_variable, "flattened_srt",
+		         IR::DescriptorBindingKind::FlattenedSrt);
+	}
+}
+
+void AddVsharpAnnotationsAndNames(EmitterState& state) {
+	if (state.push_constant_variable != 0) {
+		state.builder.AddName(PushConstantBlockType(state), "BufferResource");
+		state.builder.AddName(state.push_constant_variable, "vsharp");
+	}
+	if (state.vsharp_storage_variable != 0) {
+		DecorateDescriptor(state, state.vsharp_storage_variable, "user_data",
+		                   IR::DescriptorBindingKind::UserData);
+	}
+}
+
+void DefineModule(EmitterState& state) {
+	DefineDescriptorVariables(state);
+	if (state.requirements.function_lds) {
+		state.lds_variable = state.builder.AllocateId();
+	}
+	if (state.requirements.function_scratch) {
+		state.scratch_variable = state.builder.AllocateId();
+	}
+	state.main_func   = state.builder.AllocateId();
+	state.entry_label = state.builder.AllocateId();
+
+	state.builder.RequireCapability(CapabilityShader);
+	state.builder.RequireCapability(CapabilitySignedZeroInfNanPreserve);
+	if (state.program.info.uses_dma) {
+		state.builder.RequireCapability(CapabilityInt64);
+		state.builder.RequireCapability(CapabilityPhysicalStorageBufferAddresses);
+		state.builder.RequireExtension("SPV_KHR_physical_storage_buffer");
+	}
+	if (state.clip_distance_variable != 0) {
+		state.builder.RequireCapability(CapabilityClipDistance);
+	}
+	if (state.cull_distance_variable != 0) {
+		state.builder.RequireCapability(CapabilityCullDistance);
+	}
+	if (state.layer_variable != 0) {
+		state.builder.RequireCapability(CapabilityShaderViewportIndexLayerEXT);
+		state.builder.RequireExtension("SPV_EXT_shader_viewport_index_layer");
+	}
+	if (state.requirements.image_gather_extended) {
+		state.builder.RequireCapability(CapabilityImageGatherExtended);
+	}
+	if (state.requirements.subgroup_ballot || state.requirements.subgroup_shuffle ||
+	    state.requirements.subgroup_local_invocation_id) {
+		state.builder.RequireCapability(CapabilityGroupNonUniform);
+	}
+	if (state.requirements.subgroup_ballot) {
+		state.builder.RequireCapability(CapabilityGroupNonUniformBallot);
+	}
+	if (state.requirements.subgroup_shuffle) {
+		state.builder.RequireCapability(CapabilityGroupNonUniformShuffle);
+	}
+	if (state.requirements.compute_derivatives && state.stage == ShaderType::Compute) {
+		state.builder.RequireCapability(CapabilityComputeDerivativeGroupQuadsKHR);
+		state.builder.RequireExtension("SPV_KHR_compute_shader_derivatives");
+	}
+	const bool fragment_barycentric =
+	    state.stage == ShaderType::Pixel &&
+	    std::any_of(state.inputs.begin(), state.inputs.end(), [](const InputBinding& input) {
+		    return input.per_vertex || input.kind == IR::StageInputKind::BaryCoordSmooth ||
+		           input.kind == IR::StageInputKind::BaryCoordNoPerspective;
+	    });
+	if (fragment_barycentric) {
+		state.builder.RequireCapability(CapabilityFragmentBarycentricKHR);
+		state.builder.RequireExtension("SPV_KHR_fragment_shader_barycentric");
+	}
+	state.builder.RequireExtension("SPV_KHR_float_controls");
+	state.builder.AddMemoryModel(
+	    {state.program.info.uses_dma ? AddressingModelPhysicalStorageBuffer64
+	                                 : AddressingModelLogical,
+	     MemoryModelGLSL450});
+	state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
+	                            state.interface_variables);
+	// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
+	// contract prevents host compilers from treating synthesized IEEE values as finite.
+	state.builder.AddExecutionMode({state.main_func, ExecutionModeSignedZeroInfNanPreserve, 32u});
+	if (state.stage == ShaderType::Compute) {
+		uint32_t    local_x = state.requirements.compute_derivatives ? 2u : 1u;
+		uint32_t    local_y = state.requirements.compute_derivatives ? 2u : 1u;
+		uint32_t    local_z = 1u;
+		const auto* cs      = state.input_info.compute;
+		local_x             = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
+		local_y             = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
+		local_z             = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
+		state.builder.AddExecutionMode(
+		    {state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
+	}
+	if (state.stage == ShaderType::Pixel) {
+		state.builder.AddExecutionMode({state.main_func, ExecutionModeOriginUpperLeft});
+		if (state.depth_variable != 0) {
+			state.builder.AddExecutionMode({state.main_func, ExecutionModeDepthReplacing});
+		}
+		if (state.input_info.pixel->ps_early_z && !state.input_info.pixel->ps_pixel_kill_enable &&
+		    !state.input_info.pixel->ps_depth_export_enable &&
+		    !state.input_info.pixel->ps_sample_mask_export_enable) {
+			state.builder.AddExecutionMode({state.main_func, ExecutionModeEarlyFragmentTests});
+		}
+	}
+	if (state.requirements.compute_derivatives && state.stage == ShaderType::Compute) {
+		state.builder.AddExecutionMode({state.main_func, ExecutionModeDerivativeGroupQuadsKHR});
+	}
+	state.builder.AddName(state.main_func, "main");
+	if (state.requirements.function_lds) {
+		state.builder.AddName(state.lds_variable, "lds_dwords");
+	}
+	if (state.requirements.function_scratch) {
+		state.builder.AddName(state.scratch_variable, "scratch_dwords");
+	}
+	AddInputAnnotationsAndNames(state);
+	AddOutputAnnotationsAndNames(state);
+	AddDescriptorAnnotationsAndNames(state);
+	AddVsharpAnnotationsAndNames(state);
+
+	if (state.subgroup_local_invocation_id_variable != 0) {
+		state.builder.DefineGlobalVariable(state.subgroup_local_invocation_id_variable,
+		                                   TypePointer(state, StorageClassInput, TypeU32(state)),
+		                                   StorageClassInput);
+	}
+	for (const auto& input: state.inputs) {
+		uint32_t ptr_type = TypePointer(state, StorageClassInput, TypeU32(state));
+		switch (input.kind) {
+			case IR::StageInputKind::VertexIndex:
+			case IR::StageInputKind::InstanceIndex:
+				ptr_type = TypePointer(state, StorageClassInput, TypeI32(state));
+				break;
+			case IR::StageInputKind::WorkgroupId:
+			case IR::StageInputKind::LocalInvocationId:
+			case IR::StageInputKind::GlobalInvocationId:
+				ptr_type = TypePointer(state, StorageClassInput, TypeU32Vector(state, 3));
+				break;
+			case IR::StageInputKind::FragCoord:
+				ptr_type = TypePointer(state, StorageClassInput, TypeF32Vector(state, 4));
+				break;
+			case IR::StageInputKind::BaryCoordSmooth:
+			case IR::StageInputKind::BaryCoordNoPerspective:
+				ptr_type = TypePointer(state, StorageClassInput, TypeF32Vector(state, 3));
+				break;
+			case IR::StageInputKind::FrontFacing:
+				ptr_type = TypePointer(state, StorageClassInput, TypeBool(state));
+				break;
+			case IR::StageInputKind::Parameter:
+				if (state.stage == ShaderType::Vertex) {
+					const auto kind       = VertexParameterScalarKind(state, input.location);
+					const auto components = VertexParameterComponentCount(state, input);
+					ptr_type = VertexParameterInputPointerType(state, kind, components);
+				} else if (input.per_vertex) {
+					const auto array_type = state.builder.Type(
+					    OpTypeArray, {TypeF32Vector(state, 4), ConstantU32(state, 3)});
+					ptr_type = TypePointer(state, StorageClassInput, array_type);
+				} else {
+					ptr_type = TypePointer(state, StorageClassInput, TypeF32Vector(state, 4));
+				}
+				break;
+			default: break;
+		}
+		state.builder.DefineGlobalVariable(input.variable_id, ptr_type, StorageClassInput);
+	}
+	if (state.per_vertex_variable != 0) {
+		state.builder.DefineGlobalVariable(
+		    state.per_vertex_variable, TypePointer(state, StorageClassOutput, PerVertexType(state)),
+		    StorageClassOutput);
+	}
+	if (state.point_size_variable != 0) {
+		state.builder.DefineGlobalVariable(
+		    state.point_size_variable, TypePointer(state, StorageClassOutput, TypeF32(state)),
+		    StorageClassOutput);
+	}
+	if (state.clip_distance_variable != 0) {
+		state.builder.DefineGlobalVariable(
+		    state.clip_distance_variable,
+		    TypePointer(state, StorageClassOutput,
+		                F32ArrayType(state, state.clip_distance_count)),
+		    StorageClassOutput);
+	}
+	if (state.cull_distance_variable != 0) {
+		state.builder.DefineGlobalVariable(
+		    state.cull_distance_variable,
+		    TypePointer(state, StorageClassOutput,
+		                F32ArrayType(state, state.cull_distance_count)),
+		    StorageClassOutput);
+	}
+	if (state.layer_variable != 0) {
+		state.builder.DefineGlobalVariable(
+		    state.layer_variable, TypePointer(state, StorageClassOutput, TypeU32(state)),
+		    StorageClassOutput);
+	}
+	for (const auto& binding: state.outputs) {
+		if (binding.kind == IR::StageOutputKind::Parameter ||
+		    binding.kind == IR::StageOutputKind::Mrt) {
+			const auto pointer_type =
+			    binding.kind == IR::StageOutputKind::Mrt && MrtUsesUintOutput(state, binding.index)
+			        ? TypePointer(state, StorageClassOutput, TypeU32Vector(state, 4))
+			        : TypePointer(state, StorageClassOutput, TypeF32Vector(state, 4));
+			state.builder.DefineGlobalVariable(binding.variable_id, pointer_type,
+			                                   StorageClassOutput);
+		}
+	}
+	if (state.depth_variable != 0) {
+		state.builder.DefineGlobalVariable(state.depth_variable,
+		                                   TypePointer(state, StorageClassOutput, TypeF32(state)),
+		                                   StorageClassOutput);
+	}
+	if (state.sample_mask_variable != 0) {
+		state.builder.DefineGlobalVariable(
+		    state.sample_mask_variable,
+		    TypePointer(state, StorageClassOutput, SampleMaskArrayType(state)), StorageClassOutput);
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -0,0 +1,640 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+#include <bit>
+#include <fmt/format.h>
+#include <functional>
+#include <optional>
+#include <unordered_set>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
+	const auto kill_label  = state.builder.AllocateId();
+	const auto merge_label = state.builder.AllocateId();
+	const auto inactive    = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalNot, TypeBool(state), inactive, active});
+	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
+	EmitLabel(state, kill_label);
+	state.builder.AddFunction({OpKill});
+	EmitLabel(state, merge_label);
+}
+
+void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
+	if (state.pixel_valid_mask_variable == 0) {
+		return;
+	}
+
+	const auto mask_value = state.builder.AllocateId();
+	const auto active     = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpLoad, TypeU32(state), mask_value, state.pixel_valid_mask_variable});
+	state.builder.AddFunction(
+	    {OpINotEqual, TypeBool(state), active, mask_value, ConstantU32(state, 0)});
+	EmitKillIfBoolFalse(state, active);
+}
+
+uint32_t SpillPointerType(ValueEmitContext& ctx, IR::Type type) {
+	const auto value_type = ctx.TypeId(type);
+	return value_type == 0 ? 0 : TypePointer(ctx.state, StorageClassFunction, value_type);
+}
+
+struct DeferredPhiPatch {
+	DeferredPhi     phi;
+	const IR::Inst* instruction = nullptr;
+};
+
+struct StructuredFunctionState {
+	std::unordered_map<const IR::Block*, uint32_t> block_exit_labels;
+	std::vector<DeferredPhiPatch>                  deferred_phis;
+};
+
+struct DispatcherFunctionState {
+	std::unordered_map<const IR::Inst*, uint32_t> spills;
+	uint32_t                                      header_label       = 0;
+	uint32_t                                      select_label       = 0;
+	uint32_t                                      after_switch_label = 0;
+	uint32_t                                      continue_label     = 0;
+	uint32_t                                      merge_label        = 0;
+};
+
+void StoreDispatcherPhiEdge(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                            const IR::Block* from, const IR::Block* to) {
+	if (to == nullptr) {
+		return;
+	}
+	for (const auto& phi: *to) {
+		if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
+			break;
+		}
+		for (size_t index = 0; index < phi.NumArgs(); index++) {
+			if (phi.PhiBlock(index) == from) {
+				ctx.state.builder.AddFunction(
+				    {OpStore, dispatcher.spills.at(&phi), ctx.Def(phi.Arg(index))});
+				break;
+			}
+		}
+	}
+}
+
+const IR::Block* TargetBlock(const IR::Program& program, uint32_t id) {
+	const auto found = std::ranges::find_if(
+	    program.block_info, [&](const IR::BlockInfo& info) { return info.id == id; });
+	if (found == program.block_info.end()) {
+		return nullptr;
+	}
+	return program.blocks[static_cast<size_t>(found - program.block_info.begin())];
+}
+
+void EmitReturn(ValueEmitContext& ctx) {
+	EmitKillIfPixelValidMaskInactive(ctx.state);
+	ctx.state.builder.AddFunction({OpReturn});
+}
+
+void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
+                              const IR::BlockInfo& info) {
+	const auto& term       = info.terminator;
+	const auto  emit_merge = [&]() {
+		if (term.loop_header) {
+			const auto* merge = TargetBlock(ctx.program, term.merge_block);
+			const auto* cont  = TargetBlock(ctx.program, term.continue_block);
+			if (merge != nullptr && cont != nullptr) {
+				ctx.state.builder.AddFunction(
+				    {OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone});
+			}
+		} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
+		           term.merge_block != UINT32_MAX) {
+			if (const auto* merge = TargetBlock(ctx.program, term.merge_block); merge != nullptr) {
+				ctx.state.builder.AddFunction(
+				    {OpSelectionMerge, ctx.Label(merge), SelectionControlNone});
+			}
+		}
+	};
+
+	switch (term.kind) {
+		case CFG::TerminatorKind::Branch: {
+			const auto* target = TargetBlock(ctx.program, term.true_block);
+			if (target == nullptr) {
+				EmitReturn(ctx);
+				return;
+			}
+			emit_merge();
+			ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
+			return;
+		}
+		case CFG::TerminatorKind::ConditionalBranch: {
+			const auto* true_block  = TargetBlock(ctx.program, term.true_block);
+			const auto* false_block = TargetBlock(ctx.program, term.false_block);
+			if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) {
+				EmitReturn(ctx);
+				return;
+			}
+			const auto condition = ctx.Def(info.condition);
+			emit_merge();
+			ctx.state.builder.AddFunction(
+			    {OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
+			return;
+		}
+		default: EmitReturn(ctx); return;
+	}
+}
+
+void EmitDispatcherTarget(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                          const IR::Block* from, uint32_t target) {
+	const auto* block = TargetBlock(ctx.program, target);
+	if (block != nullptr) {
+		StoreDispatcherPhiEdge(ctx, dispatcher, from, block);
+	}
+}
+
+uint32_t EmitDispatcherNextPc(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                              const IR::Block* block, const IR::BlockInfo& info) {
+	const auto& term = info.terminator;
+	switch (term.kind) {
+		case CFG::TerminatorKind::Branch:
+			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
+			return ConstantU32(ctx.state, term.true_block);
+		case CFG::TerminatorKind::ConditionalBranch: {
+			EmitDispatcherTarget(ctx, dispatcher, block, term.true_block);
+			EmitDispatcherTarget(ctx, dispatcher, block, term.false_block);
+			const auto selected = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), selected,
+			                               ctx.Def(info.condition),
+			                               ConstantU32(ctx.state, term.true_block),
+			                               ConstantU32(ctx.state, term.false_block)});
+			return selected;
+		}
+		case CFG::TerminatorKind::IndirectBranch: {
+			for (const auto target: term.indirect_targets) {
+				EmitDispatcherTarget(ctx, dispatcher, block, target);
+			}
+			uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
+			if (!info.indirect_target.IsEmpty()) {
+				const auto  selector = ctx.Def(info.indirect_target);
+				const auto& values   = term.indirect_selector_code != UINT32_MAX
+				                           ? term.indirect_selector_values
+				                           : term.indirect_target_pcs;
+				const auto& targets  = term.indirect_selector_code != UINT32_MAX
+				                           ? term.indirect_selector_targets
+				                           : term.indirect_targets;
+				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
+					const auto match = ctx.state.builder.AllocateId();
+					const auto next  = ctx.state.builder.AllocateId();
+					ctx.state.builder.AddFunction({OpIEqual, TypeBool(ctx.state), match, selector,
+					                               ConstantU32(ctx.state, values[index])});
+					ctx.state.builder.AddFunction({OpSelect, TypeU32(ctx.state), next, match,
+					                               ConstantU32(ctx.state, targets[index]),
+					                               selected});
+					selected = next;
+				}
+			}
+			return selected;
+		}
+		default: return ConstantU32(ctx.state, UINT32_MAX);
+	}
+}
+
+bool EmitDirectInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
+	if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
+	    EmitValueImage(ctx, inst)) {
+		return true;
+	}
+	ctx.Fail(inst, "has no direct SPIR-V emitter");
+	return false;
+}
+
+bool EmitStructuredInstruction(ValueEmitContext& ctx, StructuredFunctionState& structured,
+                               const IR::Inst& inst) {
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		const auto type = ctx.TypeId(inst.GetType());
+		if (type == 0 || inst.NumArgs() == 0) {
+			ctx.Fail(inst, "has no native SPIR-V representation");
+			return false;
+		}
+		for (size_t index = 0; index < inst.NumArgs(); index++) {
+			const auto* predecessor = inst.PhiBlock(index);
+			if (predecessor == nullptr || !ctx.labels.contains(predecessor)) {
+				ctx.Fail(inst, "has a predecessor outside the structured function");
+				return false;
+			}
+		}
+		structured.deferred_phis.push_back(
+		    {ctx.state.builder.AddDeferredPhi(type, ctx.Result(inst), inst.NumArgs()), &inst});
+		return true;
+	}
+	return EmitDirectInstruction(ctx, inst);
+}
+
+bool EmitDispatcherInstruction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher,
+                               const IR::Inst& inst) {
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		const auto type = ctx.TypeId(inst.GetType());
+		if (type == 0) {
+			ctx.Fail(inst, "cannot be loaded by the dispatcher");
+			return false;
+		}
+		ctx.state.builder.AddFunction(
+		    {OpLoad, type, ctx.Result(inst), dispatcher.spills.at(&inst)});
+		return true;
+	}
+	if (!EmitDirectInstruction(ctx, inst)) {
+		return false;
+	}
+	if (const auto found = dispatcher.spills.find(&inst); found != dispatcher.spills.end()) {
+		ctx.state.builder.AddFunction(
+		    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
+	}
+	return true;
+}
+
+template <typename EmitInstruction>
+void EmitBlock(ValueEmitContext& ctx, const IR::Block* block, EmitInstruction&& emit_instruction) {
+	ctx.current_block = block;
+	EmitLabel(ctx.state, ctx.Label(block));
+	bool emitted_non_phi = false;
+	for (const auto& inst: *block) {
+		if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+			if (emitted_non_phi) {
+				ctx.Fail(inst, "appears after a non-Phi instruction");
+				return;
+			}
+		} else {
+			emitted_non_phi = true;
+		}
+		if (!emit_instruction(inst)) {
+			return;
+		}
+	}
+}
+
+void PatchStructuredPhis(ValueEmitContext& ctx, StructuredFunctionState& structured) {
+	for (const auto& deferred: structured.deferred_phis) {
+		for (size_t index = 0; index < deferred.instruction->NumArgs(); index++) {
+			const auto* predecessor = deferred.instruction->PhiBlock(index);
+			const auto  found       = structured.block_exit_labels.find(predecessor);
+			if (found == structured.block_exit_labels.end()) {
+				ctx.failed = true;
+				ctx.error  = "typed Phi predecessor was not emitted";
+				return;
+			}
+			ctx.state.builder.PatchDeferredPhi(
+			    deferred.phi, index, ctx.Def(deferred.instruction->Arg(index)), found->second);
+			if (ctx.failed) {
+				return;
+			}
+		}
+	}
+}
+
+void EmitStructuredFunction(ValueEmitContext& ctx) {
+	StructuredFunctionState structured;
+	ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
+	for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
+		const auto* block = ctx.program.blocks[index];
+		EmitBlock(ctx, block, [&](const IR::Inst& inst) {
+			return EmitStructuredInstruction(ctx, structured, inst);
+		});
+		if (ctx.failed) {
+			break;
+		}
+		structured.block_exit_labels.emplace(block, ctx.state.current_label);
+		EmitStructuredTerminator(ctx, block, ctx.program.block_info[index]);
+	}
+	if (!ctx.failed) {
+		PatchStructuredPhis(ctx, structured);
+	}
+}
+
+void EmitDispatcherFunction(ValueEmitContext& ctx, const DispatcherFunctionState& dispatcher) {
+	auto&       state = ctx.state;
+	const auto* entry = ctx.program.blocks.front();
+	state.builder.AddFunction({OpBranch, ctx.Label(entry)});
+	EmitBlock(ctx, entry, [&](const IR::Inst& inst) {
+		return EmitDispatcherInstruction(ctx, dispatcher, inst);
+	});
+	if (ctx.failed) {
+		return;
+	}
+	const auto initial_pc =
+	    EmitDispatcherNextPc(ctx, dispatcher, entry, ctx.program.block_info.front());
+	const auto initial_parent = state.current_label;
+	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+
+	EmitLabel(state, dispatcher.header_label);
+	const auto pc      = state.builder.AllocateId();
+	const auto next_pc = state.builder.AllocateId();
+	state.builder.AddFunction({OpPhi, TypeU32(state), pc, initial_pc, initial_parent, next_pc,
+	                           dispatcher.continue_label});
+	const auto done = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpIEqual, TypeBool(state), done, pc, ConstantU32(ctx.state, UINT32_MAX)});
+	state.builder.AddFunction(
+	    {OpLoopMerge, dispatcher.merge_label, dispatcher.continue_label, LoopControlNone});
+	state.builder.AddFunction(
+	    {OpBranchConditional, done, dispatcher.merge_label, dispatcher.select_label});
+
+	EmitLabel(state, dispatcher.select_label);
+	state.builder.AddFunction(
+	    {OpSelectionMerge, dispatcher.after_switch_label, SelectionControlNone});
+	std::vector<uint32_t> words {OpSwitch, pc, dispatcher.after_switch_label};
+	for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
+		words.push_back(ctx.program.block_info[index].id);
+		words.push_back(ctx.Label(ctx.program.blocks[index]));
+	}
+	state.builder.AddFunction(words);
+	std::vector<uint32_t> next_pc_words {OpPhi, TypeU32(state), next_pc,
+	                                     ConstantU32(state, UINT32_MAX), dispatcher.select_label};
+
+	for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) {
+		EmitBlock(ctx, ctx.program.blocks[index], [&](const IR::Inst& inst) {
+			return EmitDispatcherInstruction(ctx, dispatcher, inst);
+		});
+		if (ctx.failed) {
+			break;
+		}
+		const auto selected = EmitDispatcherNextPc(ctx, dispatcher, ctx.program.blocks[index],
+		                                           ctx.program.block_info[index]);
+		next_pc_words.push_back(selected);
+		next_pc_words.push_back(state.current_label);
+		state.builder.AddFunction({OpBranch, dispatcher.after_switch_label});
+	}
+	if (ctx.failed) {
+		return;
+	}
+	EmitLabel(state, dispatcher.after_switch_label);
+	state.builder.AddFunction(next_pc_words);
+	state.builder.AddFunction({OpBranch, dispatcher.continue_label});
+	EmitLabel(state, dispatcher.continue_label);
+	state.builder.AddFunction({OpBranch, dispatcher.header_label});
+	EmitLabel(state, dispatcher.merge_label);
+	EmitReturn(ctx);
+}
+
+} // namespace
+
+uint32_t ValueEmitContext::TypeId(IR::Type type) const {
+	switch (type) {
+		case IR::Type::U1: return TypeBool(state);
+		case IR::Type::U8:
+		case IR::Type::U16:
+		case IR::Type::U32:
+		case IR::Type::F16: return TypeU32(state);
+		case IR::Type::U64: return TypeU64(state);
+		case IR::Type::U32x2: return TypeU32Pair(state);
+		case IR::Type::F32: return TypeF32(state);
+		case IR::Type::U32x3: return TypeU32Vector(state, 3);
+		case IR::Type::U32x4: return TypeU32Vector(state, 4);
+		case IR::Type::F32x2: return TypeF32Vector(state, 2);
+		default: return 0;
+	}
+}
+
+uint32_t ValueEmitContext::Def(IR::Value value) {
+	value = value.Resolve();
+	if (value.IsImmediate()) {
+		switch (value.GetType()) {
+			case IR::Type::U1: return ConstantBool(state, value.U1());
+			case IR::Type::U8: return ConstantU32(state, value.U8());
+			case IR::Type::U16: return ConstantU32(state, value.U16());
+			case IR::Type::U32: return ConstantU32(state, value.U32());
+			case IR::Type::U64: return ConstantU64(state, value.U64());
+			case IR::Type::F16: return ConstantU32(state, value.F16Bits());
+			case IR::Type::F32:
+				return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
+			default: break;
+		}
+	}
+	const auto* inst = value.ResolveInstruction();
+	if (inst == nullptr) {
+		failed = true;
+		error  = "direct SPIR-V emitter received a non-value argument";
+		return ConstantU32(state, 0);
+	}
+	if (dispatcher_spills != nullptr && current_block != nullptr &&
+	    inst->Parent() != current_block) {
+		if (const auto found = dispatcher_spills->find(inst); found != dispatcher_spills->end()) {
+			if (const auto loaded = dispatcher_block_loads.find(inst);
+			    loaded != dispatcher_block_loads.end() &&
+			    loaded->second.first == state.current_label) {
+				return loaded->second.second;
+			}
+			const auto id = state.builder.AllocateId();
+			state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
+			dispatcher_block_loads.insert_or_assign(inst, std::pair {state.current_label, id});
+			return id;
+		}
+	}
+	if (const auto found = definitions.find(inst); found != definitions.end()) {
+		return found->second;
+	}
+	const auto id = state.builder.AllocateId();
+	definitions.emplace(inst, id);
+	return id;
+}
+
+uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
+	return Def(inst.Arg(index));
+}
+
+uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
+	if (const auto found = definitions.find(&inst); found != definitions.end()) {
+		return found->second;
+	}
+	const auto id = state.builder.AllocateId();
+	definitions.emplace(&inst, id);
+	return id;
+}
+
+uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
+                                std::initializer_list<uint32_t> args) {
+	std::vector<uint32_t> words {opcode, TypeId(type), Result(inst)};
+	words.insert(words.end(), args.begin(), args.end());
+	state.builder.AddFunction(words);
+	return Result(inst);
+}
+
+uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
+	if (const auto found = definitions.find(&inst); found != definitions.end()) {
+		if (found->second != value) {
+			state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value});
+		}
+		return found->second;
+	}
+	definitions.emplace(&inst, value);
+	return value;
+}
+
+uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) {
+	const auto* inst = value.ResolveInstruction();
+	if (inst == nullptr || inst->GetOpcode() != opcode) {
+		failed = true;
+		error  = "typed resource handle has the wrong producer";
+		return 0;
+	}
+	return inst->Flags<uint32_t>();
+}
+
+const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) {
+	const auto* inst = value.ResolveInstruction();
+	if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) {
+		failed = true;
+		error  = "typed image address was not constructed by MakeImageAddress";
+		return nullptr;
+	}
+	return inst;
+}
+
+const IR::MemoryInfo& ValueEmitContext::Memory(const IR::Inst& inst) const {
+	return program.memory_info.at(inst.Flags<IR::MemoryFlags>().index);
+}
+
+const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const {
+	return program.export_info.at(inst.Flags<IR::ExportFlags>().index);
+}
+
+uint32_t ValueEmitContext::Label(const IR::Block* block) const {
+	return labels.at(block);
+}
+
+void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
+	failed = true;
+	error  = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
+}
+
+bool EmitProgram(EmitterState& state, const IR::Program& program, std::string* error) {
+	ValueEmitContext                       ctx(state, program);
+	std::optional<DispatcherFunctionState> dispatcher;
+	if (state.stage == ShaderType::Pixel && state.requirements.pixel_valid_mask) {
+		state.pixel_valid_mask_variable = state.builder.AllocateId();
+		state.builder.AddName(state.pixel_valid_mask_variable, "pixel_valid_mask_active");
+	}
+	for (const auto* block: program.blocks) {
+		ctx.labels.emplace(block, state.builder.AllocateId());
+	}
+	if (state.program.dispatcher_fallback) {
+		auto& dispatch = dispatcher.emplace();
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
+					continue;
+				}
+				if (SpillPointerType(ctx, inst.GetType()) == 0) {
+					ctx.Fail(inst, "cannot be stored by the dispatcher");
+					break;
+				}
+				dispatch.spills.emplace(&inst, state.builder.AllocateId());
+			}
+		}
+		const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
+			value                  = value.Resolve();
+			const auto* definition = value.TryInstruction();
+			if (definition == nullptr || definition->Parent() == consumer ||
+			    definition->Parent() == program.blocks.front()) {
+				return;
+			}
+			if (SpillPointerType(ctx, definition->GetType()) == 0) {
+				ctx.Fail(*definition, "cannot be stored by the dispatcher");
+				return;
+			}
+			if (!dispatch.spills.contains(definition)) {
+				dispatch.spills.emplace(definition, state.builder.AllocateId());
+			}
+		};
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				for (size_t index = 0; index < inst.NumArgs(); index++) {
+					const auto* consumer =
+					    inst.GetOpcode() == IR::ValueOpcode::Phi ? inst.PhiBlock(index) : block;
+					mark_cross_block(inst.Arg(index), consumer);
+				}
+			}
+		}
+		for (size_t index = 0; index < program.blocks.size(); index++) {
+			mark_cross_block(program.block_info[index].condition, program.blocks[index]);
+			mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
+		}
+		dispatch.header_label       = state.builder.AllocateId();
+		dispatch.select_label       = state.builder.AllocateId();
+		dispatch.after_switch_label = state.builder.AllocateId();
+		dispatch.continue_label     = state.builder.AllocateId();
+		dispatch.merge_label        = state.builder.AllocateId();
+		ctx.dispatcher_spills       = &dispatch.spills;
+	}
+	if (ctx.failed) {
+		SetError(error, ctx.error.c_str());
+		return false;
+	}
+	DefineGetBdaPointer(state);
+	for (const auto* block: program.blocks) {
+		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
+			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32;
+		    })) {
+			ctx.scratch_u32_variable = state.builder.AllocateId();
+			break;
+		}
+	}
+	state.builder.AddFunction(
+	    {OpFunction, TypeVoid(state), state.main_func, FunctionControlNone, TypeFunction(state)});
+	EmitLabel(state, state.entry_label);
+	if (state.requirements.function_lds) {
+		state.builder.AddFunction(
+		    {OpVariable, TypeU32ArrayPointer(state, StorageClassFunction, LdsDwordCount(state)),
+		     state.lds_variable, StorageClassFunction});
+	}
+	if (state.requirements.function_scratch) {
+		state.builder.AddFunction(
+		    {OpVariable,
+		     TypeU32ArrayPointer(state, StorageClassFunction, state.program.scratch_dwords),
+		     state.scratch_variable, StorageClassFunction});
+	}
+	if (state.pixel_valid_mask_variable != 0) {
+		state.builder.AddFunction({OpVariable,
+		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
+		                           state.pixel_valid_mask_variable, StorageClassFunction});
+	}
+	if (state.program.dispatcher_fallback) {
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				if (const auto found = dispatcher->spills.find(&inst);
+				    found != dispatcher->spills.end()) {
+					state.builder.AddFunction({OpVariable, SpillPointerType(ctx, inst.GetType()),
+					                           found->second, StorageClassFunction});
+				}
+			}
+		}
+	}
+	if (ctx.scratch_u32_variable != 0) {
+		state.builder.AddFunction({OpVariable,
+		                           TypePointer(state, StorageClassFunction, TypeU32(state)),
+		                           ctx.scratch_u32_variable, StorageClassFunction});
+	}
+	if (state.gds_variable != 0) {
+		state.gds_length = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpArrayLength, TypeU32(state), state.gds_length, state.gds_variable, 0});
+	}
+	if (state.pixel_valid_mask_variable != 0) {
+		state.builder.AddFunction(
+		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
+	}
+	EmitMemoryOffsets(state);
+	if (program.blocks.empty()) {
+		EmitReturn(ctx);
+	} else if (state.program.dispatcher_fallback) {
+		EmitDispatcherFunction(ctx, *dispatcher);
+	} else {
+		EmitStructuredFunction(ctx);
+	}
+	state.builder.AddFunction({OpFunctionEnd});
+	if (ctx.failed) {
+		SetError(error, ctx.error.c_str());
+		return false;
+	}
+	return true;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/decompiler/MemoryOps.cpp

@@ -1,477 +0,0 @@
-#include "graphics/shader/recompiler/decompiler/MemoryOps.h"
-
-#include <fmt/format.h>
-#include <iterator>
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-namespace {
-
-struct MemoryOpcodeInfo {
-	uint32_t opcode      = 0;
-	Opcode   decoded     = Opcode::Unsupported;
-	uint32_t data_dwords = 1;
-	uint32_t data_bits   = 32;
-	bool     data_signed = false;
-	bool     typed       = false;
-	bool     formatted   = false;
-};
-
-constexpr MemoryOpcodeInfo SMEM_OPS[] = {
-    {0x00u, Opcode::SLoadDword, 1, 32},         {0x01u, Opcode::SLoadDwordx2, 2, 32},
-    {0x02u, Opcode::SLoadDwordx4, 4, 32},       {0x03u, Opcode::SLoadDwordx8, 8, 32},
-    {0x04u, Opcode::SLoadDwordx16, 16, 32},     {0x08u, Opcode::SBufferLoadDword, 1, 32},
-    {0x09u, Opcode::SBufferLoadDwordx2, 2, 32}, {0x0au, Opcode::SBufferLoadDwordx4, 4, 32},
-    {0x0bu, Opcode::SBufferLoadDwordx8, 8, 32}, {0x0cu, Opcode::SBufferLoadDwordx16, 16, 32},
-};
-
-constexpr MemoryOpcodeInfo MUBUF_OPS[] = {
-    {0x00u, Opcode::BufferLoadFormatX, 1, 32, false, false, true},
-    {0x01u, Opcode::BufferLoadFormatXy, 2, 32, false, false, true},
-    {0x02u, Opcode::BufferLoadFormatXyz, 3, 32, false, false, true},
-    {0x03u, Opcode::BufferLoadFormatXyzw, 4, 32, false, false, true},
-    {0x04u, Opcode::BufferStoreFormatX, 1, 32, false, false, true},
-    {0x05u, Opcode::BufferStoreFormatXy, 2, 32, false, false, true},
-    {0x06u, Opcode::BufferStoreFormatXyz, 3, 32, false, false, true},
-    {0x07u, Opcode::BufferStoreFormatXyzw, 4, 32, false, false, true},
-    {0x08u, Opcode::BufferLoadUbyte, 1, 8},
-    {0x09u, Opcode::BufferLoadSbyte, 1, 8, true},
-    {0x0au, Opcode::BufferLoadUshort, 1, 16},
-    {0x0bu, Opcode::BufferLoadSshort, 1, 16, true},
-    {0x0cu, Opcode::BufferLoadDword, 1, 32},
-    {0x0du, Opcode::BufferLoadDwordx2, 2, 32},
-    {0x0eu, Opcode::BufferLoadDwordx4, 4, 32},
-    {0x0fu, Opcode::BufferLoadDwordx3, 3, 32},
-    {0x18u, Opcode::BufferStoreByte, 1, 8},
-    {0x1au, Opcode::BufferStoreShort, 1, 16},
-    {0x1cu, Opcode::BufferStoreDword, 1, 32},
-    {0x1du, Opcode::BufferStoreDwordx2, 2, 32},
-    {0x1eu, Opcode::BufferStoreDwordx4, 4, 32},
-    {0x1fu, Opcode::BufferStoreDwordx3, 3, 32},
-    {0x30u, Opcode::BufferAtomicSwap, 1, 32},
-    {0x32u, Opcode::BufferAtomicAdd, 1, 32},
-    {0x33u, Opcode::BufferAtomicSub, 1, 32},
-    {0x35u, Opcode::BufferAtomicSMin, 1, 32},
-    {0x36u, Opcode::BufferAtomicUMin, 1, 32},
-    {0x37u, Opcode::BufferAtomicSMax, 1, 32},
-    {0x38u, Opcode::BufferAtomicUMax, 1, 32},
-    {0x39u, Opcode::BufferAtomicAnd, 1, 32},
-    {0x3au, Opcode::BufferAtomicOr, 1, 32},
-    {0x3bu, Opcode::BufferAtomicXor, 1, 32},
-    {0x3fu, Opcode::BufferAtomicFMin, 1, 32},
-    {0x40u, Opcode::BufferAtomicFMax, 1, 32},
-};
-
-constexpr MemoryOpcodeInfo MTBUF_OPS[] = {
-    {0x00u, Opcode::TBufferLoadFormatX, 1, 32, false, true, true},
-    {0x01u, Opcode::TBufferLoadFormatXy, 2, 32, false, true, true},
-    {0x02u, Opcode::TBufferLoadFormatXyz, 3, 32, false, true, true},
-    {0x03u, Opcode::TBufferLoadFormatXyzw, 4, 32, false, true, true},
-    {0x04u, Opcode::TBufferStoreFormatX, 1, 32, false, true, true},
-    {0x05u, Opcode::TBufferStoreFormatXy, 2, 32, false, true, true},
-    {0x06u, Opcode::TBufferStoreFormatXyz, 3, 32, false, true, true},
-    {0x07u, Opcode::TBufferStoreFormatXyzw, 4, 32, false, true, true},
-};
-
-constexpr MemoryOpcodeInfo FLAT_OPS[] = {
-    {0x08u, Opcode::FlatLoadUbyte, 1, 8},     {0x09u, Opcode::FlatLoadSbyte, 1, 8, true},
-    {0x0au, Opcode::FlatLoadUshort, 1, 16},   {0x0bu, Opcode::FlatLoadSshort, 1, 16, true},
-    {0x0cu, Opcode::FlatLoadDword, 1, 32},    {0x0du, Opcode::FlatLoadDwordx2, 2, 32},
-    {0x0eu, Opcode::FlatLoadDwordx4, 4, 32},  {0x0fu, Opcode::FlatLoadDwordx3, 3, 32},
-    {0x18u, Opcode::FlatStoreByte, 1, 8},     {0x1au, Opcode::FlatStoreShort, 1, 16},
-    {0x1cu, Opcode::FlatStoreDword, 1, 32},   {0x1du, Opcode::FlatStoreDwordx2, 2, 32},
-    {0x1eu, Opcode::FlatStoreDwordx4, 4, 32}, {0x1fu, Opcode::FlatStoreDwordx3, 3, 32},
-};
-
-constexpr MemoryOpcodeInfo DS_OPS[] = {
-    {0x00u, Opcode::DsAddU32, 1, 32},           {0x01u, Opcode::DsSubU32, 1, 32},
-    {0x05u, Opcode::DsMinI32, 1, 32},           {0x06u, Opcode::DsMaxI32, 1, 32},
-    {0x07u, Opcode::DsMinU32, 1, 32},           {0x08u, Opcode::DsMaxU32, 1, 32},
-    {0x09u, Opcode::DsAndB32, 1, 32},           {0x0au, Opcode::DsOrB32, 1, 32},
-    {0x0bu, Opcode::DsXorB32, 1, 32},           {0x0du, Opcode::DsWriteB32, 1, 32},
-    {0x0eu, Opcode::DsWrite2B32, 2, 32},        {0x0fu, Opcode::DsWrite2St64B32, 2, 32},
-    {0x12u, Opcode::DsMinF32, 1, 32},           {0x13u, Opcode::DsMaxF32, 1, 32},
-    {0x1eu, Opcode::DsWriteByte, 1, 8},         {0x1fu, Opcode::DsWriteShort, 1, 16},
-    {0x20u, Opcode::DsAddRtnU32, 1, 32},        {0x21u, Opcode::DsSubRtnU32, 1, 32},
-    {0x25u, Opcode::DsMinRtnI32, 1, 32},        {0x26u, Opcode::DsMaxRtnI32, 1, 32},
-    {0x27u, Opcode::DsMinRtnU32, 1, 32},        {0x28u, Opcode::DsMaxRtnU32, 1, 32},
-    {0x29u, Opcode::DsAndRtnB32, 1, 32},        {0x2au, Opcode::DsOrRtnB32, 1, 32},
-    {0x2bu, Opcode::DsXorRtnB32, 1, 32},        {0x2du, Opcode::DsWrxchgRtnB32, 1, 32},
-    {0x35u, Opcode::DsSwizzleB32, 1, 32},       {0x36u, Opcode::DsReadB32, 1, 32},
-    {0x37u, Opcode::DsRead2B32, 2, 32},         {0x38u, Opcode::DsRead2B32, 2, 32},
-    {0x39u, Opcode::DsReadSbyte, 1, 8, true},   {0x3au, Opcode::DsReadUbyte, 1, 8},
-    {0x3bu, Opcode::DsReadSshort, 1, 16, true}, {0x3cu, Opcode::DsReadUshort, 1, 16},
-    {0x3du, Opcode::DsConsume, 1, 32},          {0x3eu, Opcode::DsAppend, 1, 32},
-    {0x4du, Opcode::DsWriteB64, 2, 32},         {0x4eu, Opcode::DsWrite2B64, 4, 32},
-    {0x4fu, Opcode::DsWrite2St64B64, 4, 32},    {0x76u, Opcode::DsReadB64, 2, 32},
-    {0x77u, Opcode::DsRead2B64, 4, 32},         {0x78u, Opcode::DsRead2St64B64, 4, 32},
-    {0xb0u, Opcode::DsWriteAddtidB32, 1, 32},   {0xb1u, Opcode::DsReadAddtidB32, 1, 32},
-    {0xdeu, Opcode::DsWriteB96, 3, 32},         {0xdfu, Opcode::DsWriteB128, 4, 32},
-    {0xfeu, Opcode::DsReadB96, 3, 32},          {0xffu, Opcode::DsReadB128, 4, 32},
-};
-
-const MemoryOpcodeInfo* LookupMemoryOpcode(const MemoryOpcodeInfo* ops, uint32_t count,
-                                           uint32_t opcode) {
-	for (uint32_t i = 0; i < count; i++) {
-		if (ops[i].opcode == opcode) {
-			return &ops[i];
-		}
-	}
-	return nullptr;
-}
-
-uint32_t SignExtendU32(uint32_t value, uint32_t bits) {
-	if (bits == 0u || bits >= 32u) {
-		return value;
-	}
-	const uint32_t sign = 1u << (bits - 1u);
-	return (value ^ sign) - sign;
-}
-
-void MarkMemoryUnsupported(Instruction& inst, Family family, uint32_t opcode, const char* reason) {
-	inst.family    = family;
-	inst.opcode_id = opcode;
-	inst.opcode    = Opcode::Unsupported;
-	SetUnsupported(inst, family, opcode, reason);
-}
-
-void ApplyMemoryInfo(Instruction& inst, const MemoryOpcodeInfo* info) {
-	if (info == nullptr) {
-		inst.opcode = Opcode::Unsupported;
-		return;
-	}
-	inst.opcode      = info->decoded;
-	inst.data_dwords = info->data_dwords;
-	inst.data_bits   = info->data_bits;
-	inst.data_signed = info->data_signed;
-	inst.typed       = info->typed;
-	inst.formatted   = info->formatted;
-}
-
-bool IsDsWriteOpcode(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::DsWriteByte:
-		case Opcode::DsWriteShort:
-		case Opcode::DsWrite2B32:
-		case Opcode::DsWrite2St64B32:
-		case Opcode::DsWrite2B64:
-		case Opcode::DsWrite2St64B64:
-		case Opcode::DsWriteB32:
-		case Opcode::DsWriteB64:
-		case Opcode::DsWriteB96:
-		case Opcode::DsWriteB128: return true;
-		default: return false;
-	}
-}
-
-bool IsDsAtomicOpcode(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::DsAddU32:
-		case Opcode::DsAddRtnU32:
-		case Opcode::DsSubU32:
-		case Opcode::DsSubRtnU32:
-		case Opcode::DsMinI32:
-		case Opcode::DsMinRtnI32:
-		case Opcode::DsMaxI32:
-		case Opcode::DsMaxRtnI32:
-		case Opcode::DsMinU32:
-		case Opcode::DsMinRtnU32:
-		case Opcode::DsMaxU32:
-		case Opcode::DsMaxRtnU32:
-		case Opcode::DsAndB32:
-		case Opcode::DsAndRtnB32:
-		case Opcode::DsOrB32:
-		case Opcode::DsOrRtnB32:
-		case Opcode::DsXorB32:
-		case Opcode::DsXorRtnB32:
-		case Opcode::DsWrxchgRtnB32: return true;
-		default: return false;
-	}
-}
-
-uint32_t DsSourceCount(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::DsWrite2B32:
-		case Opcode::DsWrite2St64B32:
-		case Opcode::DsWrite2B64:
-		case Opcode::DsWrite2St64B64:
-		case Opcode::DsMinF32:
-		case Opcode::DsMaxF32: return 3u;
-		case Opcode::DsReadAddtidB32:
-		case Opcode::DsConsume:
-		case Opcode::DsAppend: return 0u;
-		default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u;
-	}
-}
-
-bool IsFlatStoreOpcode(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::FlatStoreByte:
-		case Opcode::FlatStoreShort:
-		case Opcode::FlatStoreDword:
-		case Opcode::FlatStoreDwordx2:
-		case Opcode::FlatStoreDwordx3:
-		case Opcode::FlatStoreDwordx4: return true;
-		default: return false;
-	}
-}
-
-} // namespace
-
-bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated SMEM instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = (word0 >> 18u) & 0xffu;
-	const uint32_t sdst    = (word0 >> 6u) & 0x7fu;
-	const uint32_t sbase   = word0 & 0x3fu;
-	const uint32_t soffset = (word1 >> 25u) & 0x7fu;
-
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = SignExtendU32(word1 & 0x1fffffu, 21u);
-	inst.glc        = ((word0 >> 16u) & 1u) != 0;
-	inst.family     = Family::SMEM;
-	inst.opcode_id  = opcode;
-	const auto* info =
-	    LookupMemoryOpcode(SMEM_OPS, static_cast<uint32_t>(std::size(SMEM_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
-	}
-
-	DecodeScalarDestination(sdst, pc, inst.dst, nullptr);
-	// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
-	// pair index; their descriptor operand consumes four SGPRs from that base.
-	DecodeScalarSource(sbase * 2u, pc, inst.src0, nullptr);
-	DecodeScalarSource(soffset, pc, inst.src1, nullptr);
-	inst.src_count = 2;
-	return true;
-}
-
-bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated MUBUF instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = ((word0 >> 18u) & 0x7fu) | (((word0 >> 25u) & 1u) << 7u);
-	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
-	const uint32_t vaddr   = word1 & 0xffu;
-	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
-	const uint32_t soffset = (word1 >> 24u) & 0xffu;
-
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = word0 & 0xfffu;
-	inst.idxen      = ((word0 >> 13u) & 1u) != 0;
-	inst.offen      = ((word0 >> 12u) & 1u) != 0;
-	inst.glc        = ((word0 >> 14u) & 1u) != 0;
-	inst.slc        = ((word1 >> 22u) & 1u) != 0;
-	inst.family     = Family::MUBUF;
-	inst.opcode_id  = opcode;
-	const auto* info =
-	    LookupMemoryOpcode(MUBUF_OPS, static_cast<uint32_t>(std::size(MUBUF_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented");
-	}
-
-	DecodeVectorGpr(vdata, inst.dst, nullptr);
-	DecodeVectorGpr(vaddr, inst.src0, nullptr);
-	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
-	DecodeScalarSource(soffset, pc, inst.src2, nullptr);
-	inst.src_count = 3;
-	return true;
-}
-
-bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated MTBUF instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = ((word0 >> 16u) & 0x7u) | (((word1 >> 21u) & 1u) << 3u);
-	const uint32_t dfmt    = (word0 >> 19u) & 0xfu;
-	const uint32_t nfmt    = (word0 >> 23u) & 0x7u;
-	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
-	const uint32_t vaddr   = word1 & 0xffu;
-	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
-	const uint32_t soffset = (word1 >> 24u) & 0xffu;
-
-	inst.pc            = pc;
-	inst.word          = word0;
-	inst.word_count    = 2;
-	inst.offset        = word0 & 0xfffu;
-	inst.idxen         = ((word0 >> 13u) & 1u) != 0;
-	inst.offen         = ((word0 >> 12u) & 1u) != 0;
-	inst.glc           = ((word0 >> 14u) & 1u) != 0;
-	inst.slc           = ((word1 >> 22u) & 1u) != 0;
-	inst.family        = Family::MTBUF;
-	inst.opcode_id     = opcode;
-	inst.data_format   = dfmt;
-	inst.number_format = nfmt;
-	const auto* info =
-	    LookupMemoryOpcode(MTBUF_OPS, static_cast<uint32_t>(std::size(MTBUF_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::MTBUF, opcode, "MTBUF opcode is not implemented");
-	}
-
-	DecodeVectorGpr(vdata, inst.dst, nullptr);
-	DecodeVectorGpr(vaddr, inst.src0, nullptr);
-	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
-	DecodeScalarSource(soffset, pc, inst.src2, nullptr);
-	inst.src_count = 3;
-	return true;
-}
-
-bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated FLAT instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0  = code[word_index];
-	const uint32_t word1  = code[word_index + 1u];
-	const uint32_t offset = word0 & 0xfffu;
-	const uint32_t dlc    = (word0 >> 12u) & 1u;
-	const uint32_t lds    = (word0 >> 13u) & 1u;
-	const uint32_t seg    = (word0 >> 14u) & 0x3u;
-	const uint32_t opcode = (word0 >> 18u) & 0x7fu;
-	const uint32_t vdst   = (word1 >> 24u) & 0xffu;
-	const uint32_t saddr  = (word1 >> 16u) & 0x7fu;
-	const uint32_t data   = (word1 >> 8u) & 0xffu;
-	const uint32_t addr   = word1 & 0xffu;
-
-	inst.pc             = pc;
-	inst.word           = word0;
-	inst.word_count     = 2;
-	inst.offset         = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
-	inst.glc            = ((word0 >> 16u) & 1u) != 0;
-	inst.slc            = ((word0 >> 17u) & 1u) != 0;
-	inst.family         = Family::FLAT;
-	inst.opcode_id      = opcode;
-	inst.memory_segment = seg;
-	const auto* info =
-	    LookupMemoryOpcode(FLAT_OPS, static_cast<uint32_t>(std::size(FLAT_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-
-	if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) {
-		SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
-		return true;
-	}
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::FLAT, opcode, "FLAT opcode is not implemented");
-		return true;
-	}
-
-	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst, nullptr);
-	DecodeVectorGpr(addr, inst.src0, nullptr);
-	inst.src_count = 1;
-	if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
-		DecodeVectorGpr(addr + 1u, inst.src1, nullptr);
-		inst.src_count = 2;
-	} else {
-		DecodeScalarSource(saddr, pc, inst.src1, nullptr);
-		inst.src_count = 2;
-	}
-	return true;
-}
-
-bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-              std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated DS instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = (word0 >> 18u) & 0xffu;
-	const uint32_t offset0 = word0 & 0xffu;
-	const uint32_t offset1 = (word0 >> 8u) & 0xffu;
-	const uint32_t vdst    = (word1 >> 24u) & 0xffu;
-	const uint32_t data1   = (word1 >> 16u) & 0xffu;
-	const uint32_t data0   = (word1 >> 8u) & 0xffu;
-	const uint32_t addr    = word1 & 0xffu;
-
-	inst.pc          = pc;
-	inst.word        = word0;
-	inst.word_count  = 2;
-	inst.offset      = offset0 | (offset1 << 8u);
-	inst.gds         = ((word0 >> 17u) & 1u) != 0u;
-	inst.family      = Family::DS;
-	inst.opcode_id   = opcode;
-	const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
-	}
-	if (inst.opcode == Opcode::DsSwizzleB32 && inst.offset >= 0xe000u) {
-		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
-	}
-	if (inst.gds &&
-	    (inst.opcode == Opcode::DsSwizzleB32 || inst.opcode == Opcode::DsWriteAddtidB32 ||
-	     inst.opcode == Opcode::DsReadAddtidB32)) {
-		SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
-	}
-	if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "DS write addtid data1 operand is not implemented");
-	}
-	if (inst.opcode == Opcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "DS read addtid data operands are not implemented");
-	}
-	if (inst.opcode == Opcode::DsWrite2B32 || opcode == 0x37u) {
-		inst.offset           = offset0 * 4u;
-		inst.secondary_offset = offset1 * 4u;
-	} else if (inst.opcode == Opcode::DsWrite2St64B32 || opcode == 0x38u) {
-		inst.offset           = offset0 * 256u;
-		inst.secondary_offset = offset1 * 256u;
-	} else if (inst.opcode == Opcode::DsWrite2B64 || inst.opcode == Opcode::DsRead2B64) {
-		inst.offset           = offset0 * 8u;
-		inst.secondary_offset = offset1 * 8u;
-	} else if (inst.opcode == Opcode::DsWrite2St64B64 || inst.opcode == Opcode::DsRead2St64B64) {
-		inst.offset           = offset0 * 512u;
-		inst.secondary_offset = offset1 * 512u;
-	}
-
-	DecodeVectorGpr(vdst, inst.dst, nullptr);
-	DecodeVectorGpr(addr, inst.src0, nullptr);
-	DecodeVectorGpr(data0, inst.src1, nullptr);
-	DecodeVectorGpr(data1, inst.src2, nullptr);
-	inst.src_count = DsSourceCount(inst.opcode);
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/decompiler/MemoryOps.h

@@ -1,21 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
-
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-
-bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error);
-bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error);
-bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error);
-bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error);
-bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-              std::string* error);
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ */

src/graphics/shader/recompiler/decompiler/OpcodeTable.h

@@ -1,45 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
-
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
-
-#include <cstddef>
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder::Detail {
-
-struct OpcodeMap {
-	uint32_t encoding = 0;
-	Opcode   decoded  = Opcode::Unknown;
-};
-
-template <typename Entry, size_t N>
-constexpr const Entry* FindOpcode(const Entry (&table)[N], uint32_t encoding) {
-	for (const auto& entry: table) {
-		if (entry.encoding == encoding) {
-			return &entry;
-		}
-	}
-	return nullptr;
-}
-
-template <typename Entry, size_t N>
-constexpr Opcode LookupOpcode(const Entry (&table)[N], uint32_t encoding) {
-	const auto* entry = FindOpcode(table, encoding);
-	return entry != nullptr ? entry->decoded : Opcode::Unsupported;
-}
-
-template <typename Entry, size_t N>
-constexpr bool HasUniqueEncodings(const Entry (&table)[N]) {
-	for (size_t i = 0; i < N; i++) {
-		for (size_t j = i + 1; j < N; j++) {
-			if (table[i].encoding == table[j].encoding) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder::Detail
-
-#endif // EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_

src/graphics/shader/recompiler/decompiler/ScalarAluOps.cpp

@@ -1,277 +0,0 @@
-#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h"
-
-#include "graphics/shader/recompiler/decompiler/OpcodeTable.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-namespace {
-
-using Detail::OpcodeMap;
-
-constexpr OpcodeMap SOP2_OPS[] = {
-    {0x00u, Opcode::SAddU32},       {0x01u, Opcode::SSubU32},       {0x02u, Opcode::SAddI32},
-    {0x03u, Opcode::SSubI32},       {0x04u, Opcode::SAddcU32},      {0x05u, Opcode::SSubbU32},
-    {0x06u, Opcode::SMinI32},       {0x07u, Opcode::SMinU32},       {0x08u, Opcode::SMaxI32},
-    {0x09u, Opcode::SMaxU32},       {0x0au, Opcode::SCselectB32},   {0x0bu, Opcode::SCselectB64},
-    {0x0eu, Opcode::SAndB32},       {0x0fu, Opcode::SAndB64},       {0x10u, Opcode::SOrB32},
-    {0x11u, Opcode::SOrB64},        {0x12u, Opcode::SXorB32},       {0x13u, Opcode::SXorB64},
-    {0x14u, Opcode::SAndn2B32},     {0x15u, Opcode::SAndn2B64},     {0x16u, Opcode::SOrn2B32},
-    {0x17u, Opcode::SOrn2B64},      {0x18u, Opcode::SNandB32},      {0x19u, Opcode::SNandB64},
-    {0x1au, Opcode::SNorB32},       {0x1bu, Opcode::SNorB64},       {0x1cu, Opcode::SXnorB32},
-    {0x1du, Opcode::SXnorB64},      {0x1eu, Opcode::SLshlB32},      {0x1fu, Opcode::SLshlB64},
-    {0x20u, Opcode::SLshrB32},      {0x21u, Opcode::SLshrB64},      {0x22u, Opcode::SAshrI32},
-    {0x24u, Opcode::SBfmB32},       {0x25u, Opcode::SBfmB64},       {0x26u, Opcode::SMulI32},
-    {0x27u, Opcode::SBfeU32},       {0x29u, Opcode::SBfeU64},       {0x2eu, Opcode::SLshl1AddU32},
-    {0x2fu, Opcode::SLshl2AddU32},  {0x30u, Opcode::SLshl3AddU32},  {0x31u, Opcode::SLshl4AddU32},
-    {0x32u, Opcode::SPackLlB32B16}, {0x33u, Opcode::SPackLhB32B16}, {0x34u, Opcode::SPackHhB32B16},
-    {0x35u, Opcode::SMulHiU32},
-};
-
-constexpr OpcodeMap SOP1_OPS[] = {
-    {0x03u, Opcode::SMovB32},
-    {0x04u, Opcode::SMovB64},
-    {0x07u, Opcode::SNotB32},
-    {0x08u, Opcode::SNotB64},
-    {0x0au, Opcode::SWqmB64},
-    {0x0bu, Opcode::SBrevB32},
-    {0x0fu, Opcode::SBcnt1I32B32},
-    {0x10u, Opcode::SBcnt1I32B64},
-    {0x13u, Opcode::SFf1I32B32},
-    {0x16u, Opcode::SFlbitI32B64},
-    {0x1bu, Opcode::SBitset0B32},
-    {0x1du, Opcode::SBitset1B32},
-    {0x1fu, Opcode::SGetpcB64},
-    {0x20u, Opcode::SSetpcB64},
-    {0x24u, Opcode::SAndSaveexecB64},
-    {0x28u, Opcode::SOrn2SaveexecB64},
-    {0x34u, Opcode::SAbsI32},
-    {0x37u, Opcode::SAndn1SaveexecB64},
-    {0x3bu, Opcode::SBitreplicateB64B32},
-    {0x3cu, Opcode::SAndSaveexecB32},
-    {0x44u, Opcode::SAndn1SaveexecB32},
-};
-
-constexpr OpcodeMap SOPC_OPS[] = {
-    {0x00u, Opcode::SCmpEqI32},   {0x01u, Opcode::SCmpLgI32},   {0x02u, Opcode::SCmpGtI32},
-    {0x03u, Opcode::SCmpGeI32},   {0x04u, Opcode::SCmpLtI32},   {0x05u, Opcode::SCmpLeI32},
-    {0x06u, Opcode::SCmpEqU32},   {0x07u, Opcode::SCmpLgU32},   {0x08u, Opcode::SCmpGtU32},
-    {0x09u, Opcode::SCmpGeU32},   {0x0au, Opcode::SCmpLtU32},   {0x0bu, Opcode::SCmpLeU32},
-    {0x0cu, Opcode::SBitcmp0B32}, {0x0du, Opcode::SBitcmp1B32}, {0x12u, Opcode::SCmpEqU64},
-    {0x13u, Opcode::SCmpLgU64},
-};
-
-constexpr OpcodeMap SOPK_OPS[] = {
-    {0x00u, Opcode::SMovkI32},   {0x03u, Opcode::SCmpEqI32}, {0x04u, Opcode::SCmpLgI32},
-    {0x05u, Opcode::SCmpGtI32},  {0x06u, Opcode::SCmpGeI32}, {0x07u, Opcode::SCmpLtI32},
-    {0x08u, Opcode::SCmpLeI32},  {0x09u, Opcode::SCmpEqU32}, {0x0au, Opcode::SCmpLgU32},
-    {0x0bu, Opcode::SCmpGtU32},  {0x0cu, Opcode::SCmpGeU32}, {0x0du, Opcode::SCmpLtU32},
-    {0x0eu, Opcode::SCmpLeU32},  {0x0fu, Opcode::SAddI32},   {0x10u, Opcode::SMulkI32},
-    {0x13u, Opcode::SSetregB32}, {0x17u, Opcode::SWaitcnt},  {0x18u, Opcode::SWaitcnt},
-    {0x19u, Opcode::SWaitcnt},   {0x1au, Opcode::SWaitcnt},
-};
-
-constexpr OpcodeMap SOPP_OPS[] = {
-    {0x00u, Opcode::SNop},          {0x01u, Opcode::SEndpgm},       {0x02u, Opcode::SBranch},
-    {0x04u, Opcode::SCbranchScc0},  {0x05u, Opcode::SCbranchScc1},  {0x06u, Opcode::SCbranchVccz},
-    {0x07u, Opcode::SCbranchVccnz}, {0x08u, Opcode::SCbranchExecz}, {0x09u, Opcode::SCbranchExecnz},
-    {0x0au, Opcode::SBarrier},      {0x0cu, Opcode::SWaitcnt},      {0x0eu, Opcode::SSleep},
-    {0x10u, Opcode::SSendmsg},      {0x16u, Opcode::STtraceData},   {0x20u, Opcode::SInstPrefetch},
-};
-static_assert(Detail::HasUniqueEncodings(SOP1_OPS));
-static_assert(Detail::HasUniqueEncodings(SOP2_OPS));
-static_assert(Detail::HasUniqueEncodings(SOPK_OPS));
-static_assert(Detail::HasUniqueEncodings(SOPC_OPS));
-static_assert(Detail::HasUniqueEncodings(SOPP_OPS));
-
-bool DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                         Instruction& inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) {
-	if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
-	    !DecodeScalarSource(ssrc1, pc, inst.src1, error)) {
-		return false;
-	}
-	inst.src_count = 2;
-	return ReadLiteralOperands(code, word_index, inst, error);
-}
-
-} // namespace
-
-bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 8u) & 0xffu;
-	const uint32_t ssrc0  = word & 0xffu;
-	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-
-	inst.pc        = pc;
-	inst.word      = word;
-	inst.family    = Family::SOP1;
-	inst.opcode_id = opcode;
-	inst.opcode    = Detail::LookupOpcode(SOP1_OPS, opcode);
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented");
-		return true;
-	}
-
-	switch (inst.opcode) {
-		case Opcode::SGetpcB64:
-			inst.src_count = 0;
-			return DecodeScalarDestination(sdst, pc, inst.dst, error);
-		case Opcode::SSetpcB64:
-			inst.src_count = 1;
-			inst.dst.kind  = OperandKind::Null;
-			if (!DecodeScalarSource(ssrc0, pc, inst.src0, error)) {
-				return false;
-			}
-			return ReadLiteralOperands(code, word_index, inst, error);
-		default: break;
-	}
-
-	if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
-	    !DecodeScalarDestination(sdst, pc, inst.dst, error)) {
-		return false;
-	}
-	inst.src_count = 1;
-	return ReadLiteralOperands(code, word_index, inst, error);
-}
-
-bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 23u) & 0x7fu;
-	const uint32_t ssrc1  = (word >> 8u) & 0xffu;
-	const uint32_t ssrc0  = word & 0xffu;
-	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-
-	inst.pc        = pc;
-	inst.word      = word;
-	inst.family    = Family::SOP2;
-	inst.opcode_id = opcode;
-	inst.opcode    = Detail::LookupOpcode(SOP2_OPS, opcode);
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented");
-		return true;
-	}
-
-	if (!DecodeScalarDestination(sdst, pc, inst.dst, error)) {
-		return false;
-	}
-	return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
-}
-
-bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 23u) & 0x1fu;
-	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-	const auto     imm    = static_cast<int16_t>(word & 0xffffu);
-
-	(void)error;
-
-	inst.pc              = pc;
-	inst.word            = word;
-	inst.family          = Family::SOPK;
-	inst.opcode_id       = opcode;
-	inst.opcode          = Detail::LookupOpcode(SOPK_OPS, opcode);
-	inst.src0.kind       = OperandKind::IntegerInlineConstant;
-	inst.src0.signed_val = imm;
-	inst.src0.value      = static_cast<uint32_t>(imm);
-	inst.src_count       = 1;
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented");
-		return true;
-	}
-
-	switch (inst.opcode) {
-		case Opcode::SMovkI32: return DecodeScalarDestination(sdst, pc, inst.dst, error);
-		case Opcode::SWaitcnt: {
-			const uint32_t waitcnt = word & 0xffffu;
-			inst.dst.kind          = OperandKind::Null;
-			inst.src0.signed_val   = static_cast<int32_t>(waitcnt);
-			inst.src0.value        = waitcnt;
-			inst.src_count         = 1;
-			return true;
-		}
-		case Opcode::SSetregB32:
-			inst.dst.kind        = OperandKind::Null;
-			inst.src1.kind       = OperandKind::LiteralConstant;
-			inst.src1.value      = word & 0xffffu;
-			inst.src1.signed_val = static_cast<int32_t>(imm);
-			inst.src_count       = 2;
-			return DecodeScalarSource(sdst, pc, inst.src0, error);
-		default: break;
-	}
-
-	inst.src1 = inst.src0;
-	if (!DecodeScalarSource(sdst, pc, inst.src0, error)) {
-		return false;
-	}
-	if (inst.opcode == Opcode::SAddI32 || inst.opcode == Opcode::SMulkI32) {
-		inst.src_count = 2;
-		return DecodeScalarDestination(sdst, pc, inst.dst, error);
-	}
-
-	inst.dst.kind  = OperandKind::Scc;
-	inst.src_count = 2;
-	return true;
-}
-
-bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t ssrc1  = (word >> 8u) & 0xffu;
-	const uint32_t ssrc0  = word & 0xffu;
-	const uint32_t opcode = (word >> 16u) & 0x7fu;
-
-	inst.pc        = pc;
-	inst.word      = word;
-	inst.family    = Family::SOPC;
-	inst.opcode_id = opcode;
-	inst.opcode    = Detail::LookupOpcode(SOPC_OPS, opcode);
-	inst.dst.kind  = OperandKind::Scc;
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented");
-		return true;
-	}
-
-	return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
-}
-
-bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 16u) & 0x7fu;
-	const uint32_t simm   = word & 0xffffu;
-
-	(void)error;
-
-	inst.pc              = pc;
-	inst.word            = word;
-	inst.family          = Family::SOPP;
-	inst.opcode_id       = opcode;
-	inst.opcode          = Detail::LookupOpcode(SOPP_OPS, opcode);
-	inst.src0.kind       = OperandKind::LiteralConstant;
-	inst.src0.value      = simm;
-	inst.src0.signed_val = static_cast<int16_t>(simm);
-	inst.src_count = (inst.opcode == Opcode::SNop || inst.opcode == Opcode::SWaitcnt ||
-	                  inst.opcode == Opcode::SSleep || inst.opcode == Opcode::SSendmsg ||
-	                  inst.opcode == Opcode::STtraceData || inst.opcode == Opcode::SInstPrefetch)
-	                     ? 1
-	                     : 0;
-	inst.branch_offset = static_cast<int32_t>(static_cast<int16_t>(simm)) * 4;
-	inst.branch_target = pc + 4u + static_cast<uint32_t>(inst.branch_offset);
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOPP, opcode, "SOPP control-flow opcode is not implemented");
-	}
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder