graphics: add --gpu-assisted-validation and report fatal VkResults (#268)

* graphics: report the VkResult when a submit or fence wait fails

* graphics: add a --gpu-assisted-validation flag

* graphics: imply Vulkan validation for GPU-assisted validation

---------

Co-authored-by: nmzik <Nmzik@mail.ru>

3rdparty/renderdoc/renderdoc_app.h

@@ -0,0 +1,875 @@
+/******************************************************************************
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015-2026 Baldur Karlsson
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ ******************************************************************************/
+
+#pragma once
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Documentation for the API is available at https://renderdoc.org/docs/in_application_api.html
+//
+
+#if !defined(RENDERDOC_NO_STDINT)
+#include <stdint.h>
+#endif
+
+#if defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER)
+#define RENDERDOC_CC __cdecl
+#elif defined(__linux__) || defined(__FreeBSD__) || defined(__sun__) || defined(__OpenBSD__)
+#define RENDERDOC_CC
+#elif defined(__APPLE__)
+#define RENDERDOC_CC
+#else
+#error "Unknown platform"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// Constants not used directly in below API
+
+// This is a GUID/magic value used for when applications pass a path where shader debug
+// information can be found to match up with a stripped shader.
+// the define can be used like so: const GUID RENDERDOC_ShaderDebugMagicValue =
+// RENDERDOC_ShaderDebugMagicValue_value
+#define RENDERDOC_ShaderDebugMagicValue_struct                                \
+  {                                                                           \
+    0xeab25520, 0x6670, 0x4865, 0x84, 0x29, 0x6c, 0x8, 0x51, 0x54, 0x00, 0xff \
+  }
+
+// as an alternative when you want a byte array (assuming x86 endianness):
+#define RENDERDOC_ShaderDebugMagicValue_bytearray                                                 \
+  {                                                                                               \
+    0x20, 0x55, 0xb2, 0xea, 0x70, 0x66, 0x65, 0x48, 0x84, 0x29, 0x6c, 0x8, 0x51, 0x54, 0x00, 0xff \
+  }
+
+// truncated version when only a uint64_t is available (e.g. Vulkan tags):
+#define RENDERDOC_ShaderDebugMagicValue_truncated 0x48656670eab25520ULL
+
+// this is a magic value for vulkan user tags to indicate which dispatchable API objects are which
+// for object annotations
+#define RENDERDOC_APIObjectAnnotationHelper 0xfbb3b337b664d0adULL
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// RenderDoc capture options
+//
+
+typedef enum RENDERDOC_CaptureOption
+{
+  // Allow the application to enable vsync
+  //
+  // Default - enabled
+  //
+  // 1 - The application can enable or disable vsync at will
+  // 0 - vsync is force disabled
+  eRENDERDOC_Option_AllowVSync = 0,
+
+  // Allow the application to enable fullscreen
+  //
+  // Default - enabled
+  //
+  // 1 - The application can enable or disable fullscreen at will
+  // 0 - fullscreen is force disabled
+  eRENDERDOC_Option_AllowFullscreen = 1,
+
+  // Record API debugging events and messages
+  //
+  // Default - disabled
+  //
+  // 1 - Enable built-in API debugging features and records the results into
+  //     the capture, which is matched up with events on replay
+  // 0 - no API debugging is forcibly enabled
+  eRENDERDOC_Option_APIValidation = 2,
+  eRENDERDOC_Option_DebugDeviceMode = 2,    // deprecated name of this enum
+
+  // Capture CPU callstacks for API events
+  //
+  // Default - disabled
+  //
+  // 1 - Enables capturing of callstacks
+  // 0 - no callstacks are captured
+  eRENDERDOC_Option_CaptureCallstacks = 3,
+
+  // When capturing CPU callstacks, only capture them from actions.
+  // This option does nothing without the above option being enabled
+  //
+  // Default - disabled
+  //
+  // 1 - Only captures callstacks for actions.
+  //     Ignored if CaptureCallstacks is disabled
+  // 0 - Callstacks, if enabled, are captured for every event.
+  eRENDERDOC_Option_CaptureCallstacksOnlyDraws = 4,
+  eRENDERDOC_Option_CaptureCallstacksOnlyActions = 4,
+
+  // Specify a delay in seconds to wait for a debugger to attach, after
+  // creating or injecting into a process, before continuing to allow it to run.
+  //
+  // 0 indicates no delay, and the process will run immediately after injection
+  //
+  // Default - 0 seconds
+  //
+  eRENDERDOC_Option_DelayForDebugger = 5,
+
+  // Verify buffer access. This includes checking the memory returned by a Map() call to
+  // detect any out-of-bounds modification, as well as initialising buffers with undefined contents
+  // to a marker value to catch use of uninitialised memory.
+  //
+  // NOTE: This option is only valid for OpenGL and D3D11. Explicit APIs such as D3D12 and Vulkan do
+  // not do the same kind of interception & checking and undefined contents are really undefined.
+  //
+  // Default - disabled
+  //
+  // 1 - Verify buffer access
+  // 0 - No verification is performed, and overwriting bounds may cause crashes or corruption in
+  //     RenderDoc.
+  eRENDERDOC_Option_VerifyBufferAccess = 6,
+
+  // The old name for eRENDERDOC_Option_VerifyBufferAccess was eRENDERDOC_Option_VerifyMapWrites.
+  // This option now controls the filling of uninitialised buffers with 0xdddddddd which was
+  // previously always enabled
+  eRENDERDOC_Option_VerifyMapWrites = eRENDERDOC_Option_VerifyBufferAccess,
+
+  // Hooks any system API calls that create child processes, and injects
+  // RenderDoc into them recursively with the same options.
+  //
+  // Default - disabled
+  //
+  // 1 - Hooks into spawned child processes
+  // 0 - Child processes are not hooked by RenderDoc
+  eRENDERDOC_Option_HookIntoChildren = 7,
+
+  // By default RenderDoc only includes resources in the final capture necessary
+  // for that frame, this allows you to override that behaviour.
+  //
+  // Default - disabled
+  //
+  // 1 - all live resources at the time of capture are included in the capture
+  //     and available for inspection
+  // 0 - only the resources referenced by the captured frame are included
+  eRENDERDOC_Option_RefAllResources = 8,
+
+  // **NOTE**: As of RenderDoc v1.1 this option has been deprecated. Setting or
+  // getting it will be ignored, to allow compatibility with older versions.
+  // In v1.1 the option acts as if it's always enabled.
+  //
+  // By default RenderDoc skips saving initial states for resources where the
+  // previous contents don't appear to be used, assuming that writes before
+  // reads indicate previous contents aren't used.
+  //
+  // Default - disabled
+  //
+  // 1 - initial contents at the start of each captured frame are saved, even if
+  //     they are later overwritten or cleared before being used.
+  // 0 - unless a read is detected, initial contents will not be saved and will
+  //     appear as black or empty data.
+  eRENDERDOC_Option_SaveAllInitials = 9,
+
+  // In APIs that allow for the recording of command lists to be replayed later,
+  // RenderDoc may choose to not capture command lists before a frame capture is
+  // triggered, to reduce overheads. This means any command lists recorded once
+  // and replayed many times will not be available and may cause a failure to
+  // capture.
+  //
+  // NOTE: This is only true for APIs where multithreading is difficult or
+  // discouraged. Newer APIs like Vulkan and D3D12 will ignore this option
+  // and always capture all command lists since the API is heavily oriented
+  // around it and the overheads have been reduced by API design.
+  //
+  // 1 - All command lists are captured from the start of the application
+  // 0 - Command lists are only captured if their recording begins during
+  //     the period when a frame capture is in progress.
+  eRENDERDOC_Option_CaptureAllCmdLists = 10,
+
+  // Mute API debugging output when the API validation mode option is enabled
+  //
+  // Default - enabled
+  //
+  // 1 - Mute any API debug messages from being displayed or passed through
+  // 0 - API debugging is displayed as normal
+  eRENDERDOC_Option_DebugOutputMute = 11,
+
+  // Option to allow vendor extensions to be used even when they may be
+  // incompatible with RenderDoc and cause corrupted replays or crashes.
+  //
+  // Default - inactive
+  //
+  // No values are documented, this option should only be used when absolutely
+  // necessary as directed by a RenderDoc developer.
+  eRENDERDOC_Option_AllowUnsupportedVendorExtensions = 12,
+
+  // Define a soft memory limit which some APIs may aim to keep overhead under where
+  // possible. Anything above this limit will where possible be saved directly to disk during
+  // capture.
+  // This will cause increased disk space use (which may cause a capture to fail if disk space is
+  // exhausted) as well as slower capture times.
+  //
+  // Not all memory allocations may be deferred like this so it is not a guarantee of a memory
+  // limit.
+  //
+  // Units are in MBs, suggested values would range from 200MB to 1000MB.
+  //
+  // Default - 0 Megabytes
+  eRENDERDOC_Option_SoftMemoryLimit = 13,
+} RENDERDOC_CaptureOption;
+
+// Sets an option that controls how RenderDoc behaves on capture.
+//
+// Returns 1 if the option and value are valid
+// Returns 0 if either is invalid and the option is unchanged
+typedef int(RENDERDOC_CC *pRENDERDOC_SetCaptureOptionU32)(RENDERDOC_CaptureOption opt, uint32_t val);
+typedef int(RENDERDOC_CC *pRENDERDOC_SetCaptureOptionF32)(RENDERDOC_CaptureOption opt, float val);
+
+// Gets the current value of an option as a uint32_t
+//
+// If the option is invalid, 0xffffffff is returned
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetCaptureOptionU32)(RENDERDOC_CaptureOption opt);
+
+// Gets the current value of an option as a float
+//
+// If the option is invalid, -FLT_MAX is returned
+typedef float(RENDERDOC_CC *pRENDERDOC_GetCaptureOptionF32)(RENDERDOC_CaptureOption opt);
+
+typedef enum RENDERDOC_InputButton
+{
+  // '0' - '9' matches ASCII values
+  eRENDERDOC_Key_0 = 0x30,
+  eRENDERDOC_Key_1 = 0x31,
+  eRENDERDOC_Key_2 = 0x32,
+  eRENDERDOC_Key_3 = 0x33,
+  eRENDERDOC_Key_4 = 0x34,
+  eRENDERDOC_Key_5 = 0x35,
+  eRENDERDOC_Key_6 = 0x36,
+  eRENDERDOC_Key_7 = 0x37,
+  eRENDERDOC_Key_8 = 0x38,
+  eRENDERDOC_Key_9 = 0x39,
+
+  // 'A' - 'Z' matches ASCII values
+  eRENDERDOC_Key_A = 0x41,
+  eRENDERDOC_Key_B = 0x42,
+  eRENDERDOC_Key_C = 0x43,
+  eRENDERDOC_Key_D = 0x44,
+  eRENDERDOC_Key_E = 0x45,
+  eRENDERDOC_Key_F = 0x46,
+  eRENDERDOC_Key_G = 0x47,
+  eRENDERDOC_Key_H = 0x48,
+  eRENDERDOC_Key_I = 0x49,
+  eRENDERDOC_Key_J = 0x4A,
+  eRENDERDOC_Key_K = 0x4B,
+  eRENDERDOC_Key_L = 0x4C,
+  eRENDERDOC_Key_M = 0x4D,
+  eRENDERDOC_Key_N = 0x4E,
+  eRENDERDOC_Key_O = 0x4F,
+  eRENDERDOC_Key_P = 0x50,
+  eRENDERDOC_Key_Q = 0x51,
+  eRENDERDOC_Key_R = 0x52,
+  eRENDERDOC_Key_S = 0x53,
+  eRENDERDOC_Key_T = 0x54,
+  eRENDERDOC_Key_U = 0x55,
+  eRENDERDOC_Key_V = 0x56,
+  eRENDERDOC_Key_W = 0x57,
+  eRENDERDOC_Key_X = 0x58,
+  eRENDERDOC_Key_Y = 0x59,
+  eRENDERDOC_Key_Z = 0x5A,
+
+  // leave the rest of the ASCII range free
+  // in case we want to use it later
+  eRENDERDOC_Key_NonPrintable = 0x100,
+
+  eRENDERDOC_Key_Divide,
+  eRENDERDOC_Key_Multiply,
+  eRENDERDOC_Key_Subtract,
+  eRENDERDOC_Key_Plus,
+
+  eRENDERDOC_Key_F1,
+  eRENDERDOC_Key_F2,
+  eRENDERDOC_Key_F3,
+  eRENDERDOC_Key_F4,
+  eRENDERDOC_Key_F5,
+  eRENDERDOC_Key_F6,
+  eRENDERDOC_Key_F7,
+  eRENDERDOC_Key_F8,
+  eRENDERDOC_Key_F9,
+  eRENDERDOC_Key_F10,
+  eRENDERDOC_Key_F11,
+  eRENDERDOC_Key_F12,
+
+  eRENDERDOC_Key_Home,
+  eRENDERDOC_Key_End,
+  eRENDERDOC_Key_Insert,
+  eRENDERDOC_Key_Delete,
+  eRENDERDOC_Key_PageUp,
+  eRENDERDOC_Key_PageDn,
+
+  eRENDERDOC_Key_Backspace,
+  eRENDERDOC_Key_Tab,
+  eRENDERDOC_Key_PrtScrn,
+  eRENDERDOC_Key_Pause,
+
+  eRENDERDOC_Key_Max,
+} RENDERDOC_InputButton;
+
+// Sets which key or keys can be used to toggle focus between multiple windows
+//
+// If keys is NULL or num is 0, toggle keys will be disabled
+typedef void(RENDERDOC_CC *pRENDERDOC_SetFocusToggleKeys)(RENDERDOC_InputButton *keys, int num);
+
+// Sets which key or keys can be used to capture the next frame
+//
+// If keys is NULL or num is 0, captures keys will be disabled
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureKeys)(RENDERDOC_InputButton *keys, int num);
+
+typedef enum RENDERDOC_OverlayBits
+{
+  // This single bit controls whether the overlay is enabled or disabled globally
+  eRENDERDOC_Overlay_Enabled = 0x1,
+
+  // Show the average framerate over several seconds as well as min/max
+  eRENDERDOC_Overlay_FrameRate = 0x2,
+
+  // Show the current frame number
+  eRENDERDOC_Overlay_FrameNumber = 0x4,
+
+  // Show a list of recent captures, and how many captures have been made
+  eRENDERDOC_Overlay_CaptureList = 0x8,
+
+  // Default values for the overlay mask
+  eRENDERDOC_Overlay_Default = (eRENDERDOC_Overlay_Enabled | eRENDERDOC_Overlay_FrameRate |
+                                eRENDERDOC_Overlay_FrameNumber | eRENDERDOC_Overlay_CaptureList),
+
+  // Enable all bits
+  eRENDERDOC_Overlay_All = 0x7ffffff,
+
+  // Disable all bits
+  eRENDERDOC_Overlay_None = 0,
+} RENDERDOC_OverlayBits;
+
+// returns the overlay bits that have been set
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetOverlayBits)(void);
+// sets the overlay bits with an and & or mask
+typedef void(RENDERDOC_CC *pRENDERDOC_MaskOverlayBits)(uint32_t And, uint32_t Or);
+
+// this function will attempt to remove RenderDoc's hooks in the application.
+//
+// Note: that this can only work correctly if done immediately after
+// the module is loaded, before any API work happens. RenderDoc will remove its
+// injected hooks and shut down. Behaviour is undefined if this is called
+// after any API functions have been called, and there is still no guarantee of
+// success.
+typedef void(RENDERDOC_CC *pRENDERDOC_RemoveHooks)(void);
+
+// DEPRECATED: compatibility for code compiled against pre-1.4.1 headers.
+typedef pRENDERDOC_RemoveHooks pRENDERDOC_Shutdown;
+
+// This function will unload RenderDoc's crash handler.
+//
+// If you use your own crash handler and don't want RenderDoc's handler to
+// intercede, you can call this function to unload it and any unhandled
+// exceptions will pass to the next handler.
+typedef void(RENDERDOC_CC *pRENDERDOC_UnloadCrashHandler)(void);
+
+// Sets the capture file path template
+//
+// pathtemplate is a UTF-8 string that gives a template for how captures will be named
+// and where they will be saved.
+//
+// Any extension is stripped off the path, and captures are saved in the directory
+// specified, and named with the filename and the frame number appended. If the
+// directory does not exist it will be created, including any parent directories.
+//
+// If pathtemplate is NULL, the template will remain unchanged
+//
+// Example:
+//
+// SetCaptureFilePathTemplate("my_captures/example");
+//
+// Capture #1 -> my_captures/example_frame123.rdc
+// Capture #2 -> my_captures/example_frame456.rdc
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureFilePathTemplate)(const char *pathtemplate);
+
+// returns the current capture path template, see SetCaptureFileTemplate above, as a UTF-8 string
+typedef const char *(RENDERDOC_CC *pRENDERDOC_GetCaptureFilePathTemplate)(void);
+
+// DEPRECATED: compatibility for code compiled against pre-1.1.2 headers.
+typedef pRENDERDOC_SetCaptureFilePathTemplate pRENDERDOC_SetLogFilePathTemplate;
+typedef pRENDERDOC_GetCaptureFilePathTemplate pRENDERDOC_GetLogFilePathTemplate;
+
+// returns the number of captures that have been made
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetNumCaptures)(void);
+
+// This function returns the details of a capture, by index. New captures are added
+// to the end of the list.
+//
+// filename will be filled with the absolute path to the capture file, as a UTF-8 string
+// pathlength will be written with the length in bytes of the filename string
+// timestamp will be written with the time of the capture, in seconds since the Unix epoch
+//
+// Any of the parameters can be NULL and they'll be skipped.
+//
+// The function will return 1 if the capture index is valid, or 0 if the index is invalid
+// If the index is invalid, the values will be unchanged
+//
+// Note: when captures are deleted in the UI they will remain in this list, so the
+// capture path may not exist anymore.
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetCapture)(uint32_t idx, char *filename,
+                                                      uint32_t *pathlength, uint64_t *timestamp);
+
+// Sets the comments associated with a capture file. These comments are displayed in the
+// UI program when opening.
+//
+// filePath should be a path to the capture file to add comments to. If set to NULL or ""
+// the most recent capture file created made will be used instead.
+// comments should be a NULL-terminated UTF-8 string to add as comments.
+//
+// Any existing comments will be overwritten.
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureFileComments)(const char *filePath,
+                                                              const char *comments);
+
+// returns 1 if the RenderDoc UI is connected to this application, 0 otherwise
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_IsTargetControlConnected)(void);
+
+// DEPRECATED: compatibility for code compiled against pre-1.1.1 headers.
+// This was renamed to IsTargetControlConnected in API 1.1.1, the old typedef is kept here for
+// backwards compatibility with old code, it is castable either way since it's ABI compatible
+// as the same function pointer type.
+typedef pRENDERDOC_IsTargetControlConnected pRENDERDOC_IsRemoteAccessConnected;
+
+// This function will launch the Replay UI associated with the RenderDoc library injected
+// into the running application.
+//
+// if connectTargetControl is 1, the Replay UI will be launched with a command line parameter
+// to connect to this application
+// cmdline is the rest of the command line, as a UTF-8 string. E.g. a captures to open
+// if cmdline is NULL, the command line will be empty.
+//
+// returns the PID of the replay UI if successful, 0 if not successful.
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_LaunchReplayUI)(uint32_t connectTargetControl,
+                                                          const char *cmdline);
+
+// RenderDoc can return a higher version than requested if it's backwards compatible,
+// this function returns the actual version returned. If a parameter is NULL, it will be
+// ignored and the others will be filled out.
+typedef void(RENDERDOC_CC *pRENDERDOC_GetAPIVersion)(int *major, int *minor, int *patch);
+
+// Requests that the replay UI show itself (if hidden or not the current top window). This can be
+// used in conjunction with IsTargetControlConnected and LaunchReplayUI to intelligently handle
+// showing the UI after making a capture.
+//
+// This will return 1 if the request was successfully passed on, though it's not guaranteed that
+// the UI will be on top in all cases depending on OS rules. It will return 0 if there is no current
+// target control connection to make such a request, or if there was another error
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_ShowReplayUI)(void);
+
+//////////////////////////////////////////////////////////////////////////
+// Capturing functions
+//
+
+// A device pointer is a pointer to the API's root handle.
+//
+// This would be an ID3D11Device, HGLRC/GLXContext, ID3D12Device, etc
+typedef void *RENDERDOC_DevicePointer;
+
+// A window handle is the OS's native window handle
+//
+// This would be an HWND, GLXDrawable, etc
+typedef void *RENDERDOC_WindowHandle;
+
+// A helper macro for Vulkan, where the device handle cannot be used directly.
+//
+// Passing the VkInstance to this macro will return the RENDERDOC_DevicePointer to use.
+//
+// Specifically, the value needed is the dispatch table pointer, which sits as the first
+// pointer-sized object in the memory pointed to by the VkInstance. Thus we cast to a void** and
+// indirect once.
+#define RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(inst) (*((void **)(inst)))
+
+// This sets the RenderDoc in-app overlay in the API/window pair as 'active' and it will
+// respond to keypresses. Neither parameter can be NULL
+typedef void(RENDERDOC_CC *pRENDERDOC_SetActiveWindow)(RENDERDOC_DevicePointer device,
+                                                       RENDERDOC_WindowHandle wndHandle);
+
+// capture the next frame on whichever window and API is currently considered active
+typedef void(RENDERDOC_CC *pRENDERDOC_TriggerCapture)(void);
+
+// capture the next N frames on whichever window and API is currently considered active
+typedef void(RENDERDOC_CC *pRENDERDOC_TriggerMultiFrameCapture)(uint32_t numFrames);
+
+// When choosing either a device pointer or a window handle to capture, you can pass NULL.
+// Passing NULL specifies a 'wildcard' match against anything. This allows you to specify
+// any API rendering to a specific window, or a specific API instance rendering to any window,
+// or in the simplest case of one window and one API, you can just pass NULL for both.
+//
+// In either case, if there are two or more possible matching (device,window) pairs it
+// is undefined which one will be captured.
+//
+// Note: for headless rendering you can pass NULL for the window handle and either specify
+// a device pointer or leave it NULL as above.
+
+// Immediately starts capturing API calls on the specified device pointer and window handle.
+//
+// If there is no matching thing to capture (e.g. no supported API has been initialised),
+// this will do nothing.
+//
+// The results are undefined (including crashes) if two captures are started overlapping,
+// even on separate devices and/oror windows.
+typedef void(RENDERDOC_CC *pRENDERDOC_StartFrameCapture)(RENDERDOC_DevicePointer device,
+                                                         RENDERDOC_WindowHandle wndHandle);
+
+// Returns whether or not a frame capture is currently ongoing anywhere.
+//
+// This will return 1 if a capture is ongoing, and 0 if there is no capture running
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_IsFrameCapturing)(void);
+
+// Ends capturing immediately.
+//
+// This will return 1 if the capture succeeded, and 0 if there was an error capturing.
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_EndFrameCapture)(RENDERDOC_DevicePointer device,
+                                                           RENDERDOC_WindowHandle wndHandle);
+
+// Ends capturing immediately and discard any data stored without saving to disk.
+//
+// This will return 1 if the capture was discarded, and 0 if there was an error or no capture
+// was in progress
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_DiscardFrameCapture)(RENDERDOC_DevicePointer device,
+                                                               RENDERDOC_WindowHandle wndHandle);
+
+// Only valid to be called between a call to StartFrameCapture and EndFrameCapture. Gives a custom
+// title to the capture produced which will be displayed in the UI.
+//
+// If multiple captures are ongoing, this title will be applied to the first capture to end after
+// this call. The second capture to end will have no title, unless this function is called again.
+//
+// Calling this function has no effect if no capture is currently running, and if it is called
+// multiple times only the last title will be used.
+typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureTitle)(const char *title);
+
+// Annotations API:
+//
+// These functions allow you to specify annotations either on a per-command level, or a per-object
+// level.
+//
+// Basic types of annotations are supported, as well as vector versions and references to API objects.
+//
+// The annotations are stored as keys, with the key being a dot-separated path allowing arbitrary
+// nesting and user organisation. The keys are sorted in human order so `foo.2.bar` will be displayed
+// before `foo.10.bar` to allow creation of arrays if desired.
+//
+// Deleting an annotation can be done by assigning an empty value to it.
+
+// the type of an annotation value, or Empty to delete an annotation
+typedef enum RENDERDOC_AnnotationType
+{
+  eRENDERDOC_Empty,
+  eRENDERDOC_Bool,
+  eRENDERDOC_Int32,
+  eRENDERDOC_UInt32,
+  eRENDERDOC_Int64,
+  eRENDERDOC_UInt64,
+  eRENDERDOC_Float,
+  eRENDERDOC_Double,
+  eRENDERDOC_String,
+  eRENDERDOC_APIObject,
+  eRENDERDOC_AnnotationMax = 0x7FFFFFFF,
+} RENDERDOC_AnnotationType;
+
+// a union with vector annotation value data
+typedef union RENDERDOC_AnnotationVectorValue
+{
+  bool boolean[4];
+  int32_t int32[4];
+  int64_t int64[4];
+  uint32_t uint32[4];
+  uint64_t uint64[4];
+  float float32[4];
+  double float64[4];
+} RENDERDOC_AnnotationVectorValue;
+
+// a union with scalar annotation value data
+typedef union RENDERDOC_AnnotationValue
+{
+  bool boolean;
+  int32_t int32;
+  int64_t int64;
+  uint32_t uint32;
+  uint64_t uint64;
+  float float32;
+  double float64;
+
+  RENDERDOC_AnnotationVectorValue vector;
+
+  const char *string;
+  void *apiObject;
+} RENDERDOC_AnnotationValue;
+
+// a struct for specifying a GL object, as we don't have pointers we can use so instead we specify a
+// pointer to this struct giving both the type and the name
+typedef struct RENDERDOC_GLResourceReference
+{
+  // this is the same GLenum identifier as passed to glObjectLabel
+  uint32_t identifier;
+  uint32_t name;
+} GLResourceReference;
+
+// simple C++ helpers to avoid the need for a temporary objects for value passing and GL object specification
+#ifdef __cplusplus
+struct RDGLObjectHelper
+{
+  RENDERDOC_GLResourceReference gl;
+
+  RDGLObjectHelper(uint32_t identifier, uint32_t name)
+  {
+    gl.identifier = identifier;
+    gl.name = name;
+  }
+
+  operator RENDERDOC_GLResourceReference *() { return &gl; }
+};
+
+struct RDAnnotationHelper
+{
+  RENDERDOC_AnnotationValue val;
+
+  RDAnnotationHelper(bool b) { val.boolean = b; }
+  RDAnnotationHelper(int32_t i) { val.int32 = i; }
+  RDAnnotationHelper(int64_t i) { val.int64 = i; }
+  RDAnnotationHelper(uint32_t i) { val.uint32 = i; }
+  RDAnnotationHelper(uint64_t i) { val.uint64 = i; }
+  RDAnnotationHelper(float f) { val.float32 = f; }
+  RDAnnotationHelper(double d) { val.float64 = d; }
+  RDAnnotationHelper(const char *s) { val.string = s; }
+
+  operator RENDERDOC_AnnotationValue *() { return &val; }
+};
+#endif
+
+// The device is specified in the same way as other API calls that take a RENDERDOC_DevicePointer
+// to specify the device.
+//
+// The object or queue/commandbuffer will depend on the graphics API in question.
+//
+// Return value:
+// 0 - The annotation was applied successfully.
+// 1 - The device is unknown/invalid
+// 2 - The device is valid but the annotation is not supported for API-specific reasons, such as an
+//     unrecognised or invalid object or queue/commandbuffer
+// 3 - The call is ill-formed or invalid e.g. empty is specified with a value pointer, or non-empty
+//     is specified with a NULL value pointer
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_SetObjectAnnotation)(RENDERDOC_DevicePointer device,
+                                                               void *object, const char *key,
+                                                               RENDERDOC_AnnotationType valueType,
+                                                               uint32_t valueVectorWidth,
+                                                               const RENDERDOC_AnnotationValue *value);
+
+typedef uint32_t(RENDERDOC_CC *pRENDERDOC_SetCommandAnnotation)(
+    RENDERDOC_DevicePointer device, void *queueOrCommandBuffer, const char *key,
+    RENDERDOC_AnnotationType valueType, uint32_t valueVectorWidth,
+    const RENDERDOC_AnnotationValue *value);
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// RenderDoc API versions
+//
+
+// RenderDoc uses semantic versioning (http://semver.org/).
+//
+// MAJOR version is incremented when incompatible API changes happen.
+// MINOR version is incremented when functionality is added in a backwards-compatible manner.
+// PATCH version is incremented when backwards-compatible bug fixes happen.
+//
+// Note that this means the API returned can be higher than the one you might have requested.
+// e.g. if you are running against a newer RenderDoc that supports 1.0.1, it will be returned
+// instead of 1.0.0. You can check this with the GetAPIVersion entry point
+typedef enum RENDERDOC_Version
+{
+  eRENDERDOC_API_Version_1_0_0 = 10000,    // RENDERDOC_API_1_0_0 = 1 00 00
+  eRENDERDOC_API_Version_1_0_1 = 10001,    // RENDERDOC_API_1_0_1 = 1 00 01
+  eRENDERDOC_API_Version_1_0_2 = 10002,    // RENDERDOC_API_1_0_2 = 1 00 02
+  eRENDERDOC_API_Version_1_1_0 = 10100,    // RENDERDOC_API_1_1_0 = 1 01 00
+  eRENDERDOC_API_Version_1_1_1 = 10101,    // RENDERDOC_API_1_1_1 = 1 01 01
+  eRENDERDOC_API_Version_1_1_2 = 10102,    // RENDERDOC_API_1_1_2 = 1 01 02
+  eRENDERDOC_API_Version_1_2_0 = 10200,    // RENDERDOC_API_1_2_0 = 1 02 00
+  eRENDERDOC_API_Version_1_3_0 = 10300,    // RENDERDOC_API_1_3_0 = 1 03 00
+  eRENDERDOC_API_Version_1_4_0 = 10400,    // RENDERDOC_API_1_4_0 = 1 04 00
+  eRENDERDOC_API_Version_1_4_1 = 10401,    // RENDERDOC_API_1_4_1 = 1 04 01
+  eRENDERDOC_API_Version_1_4_2 = 10402,    // RENDERDOC_API_1_4_2 = 1 04 02
+  eRENDERDOC_API_Version_1_5_0 = 10500,    // RENDERDOC_API_1_5_0 = 1 05 00
+  eRENDERDOC_API_Version_1_6_0 = 10600,    // RENDERDOC_API_1_6_0 = 1 06 00
+  eRENDERDOC_API_Version_1_7_0 = 10700,    // RENDERDOC_API_1_7_0 = 1 07 00
+} RENDERDOC_Version;
+
+// API version changelog:
+//
+// 1.0.0 - initial release
+// 1.0.1 - Bugfix: IsFrameCapturing() was returning false for captures that were triggered
+//         by keypress or TriggerCapture, instead of Start/EndFrameCapture.
+// 1.0.2 - Refactor: Renamed eRENDERDOC_Option_DebugDeviceMode to eRENDERDOC_Option_APIValidation
+// 1.1.0 - Add feature: TriggerMultiFrameCapture(). Backwards compatible with 1.0.x since the new
+//         function pointer is added to the end of the struct, the original layout is identical
+// 1.1.1 - Refactor: Renamed remote access to target control (to better disambiguate from remote
+//         replay/remote server concept in replay UI)
+// 1.1.2 - Refactor: Renamed "log file" in function names to just capture, to clarify that these
+//         are captures and not debug logging files. This is the first API version in the v1.0
+//         branch.
+// 1.2.0 - Added feature: SetCaptureFileComments() to add comments to a capture file that will be
+//         displayed in the UI program on load.
+// 1.3.0 - Added feature: New capture option eRENDERDOC_Option_AllowUnsupportedVendorExtensions
+//         which allows users to opt-in to allowing unsupported vendor extensions to function.
+//         Should be used at the user's own risk.
+//         Refactor: Renamed eRENDERDOC_Option_VerifyMapWrites to
+//         eRENDERDOC_Option_VerifyBufferAccess, which now also controls initialisation to
+//         0xdddddddd of uninitialised buffer contents.
+// 1.4.0 - Added feature: DiscardFrameCapture() to discard a frame capture in progress and stop
+//         capturing without saving anything to disk.
+// 1.4.1 - Refactor: Renamed Shutdown to RemoveHooks to better clarify what is happening
+// 1.4.2 - Refactor: Renamed 'draws' to 'actions' in callstack capture option.
+// 1.5.0 - Added feature: ShowReplayUI() to request that the replay UI show itself if connected
+// 1.6.0 - Added feature: SetCaptureTitle() which can be used to set a title for a
+//         capture made with StartFrameCapture() or EndFrameCapture()
+// 1.7.0 - Added feature: SetObjectAnnotation() / SetCommandAnnotation() for adding rich
+//         annotations to objects and command streams
+
+typedef struct RENDERDOC_API_1_7_0
+{
+  pRENDERDOC_GetAPIVersion GetAPIVersion;
+
+  pRENDERDOC_SetCaptureOptionU32 SetCaptureOptionU32;
+  pRENDERDOC_SetCaptureOptionF32 SetCaptureOptionF32;
+
+  pRENDERDOC_GetCaptureOptionU32 GetCaptureOptionU32;
+  pRENDERDOC_GetCaptureOptionF32 GetCaptureOptionF32;
+
+  pRENDERDOC_SetFocusToggleKeys SetFocusToggleKeys;
+  pRENDERDOC_SetCaptureKeys SetCaptureKeys;
+
+  pRENDERDOC_GetOverlayBits GetOverlayBits;
+  pRENDERDOC_MaskOverlayBits MaskOverlayBits;
+
+  // Shutdown was renamed to RemoveHooks in 1.4.1.
+  // These unions allow old code to continue compiling without changes
+  union
+  {
+    pRENDERDOC_Shutdown Shutdown;
+    pRENDERDOC_RemoveHooks RemoveHooks;
+  };
+  pRENDERDOC_UnloadCrashHandler UnloadCrashHandler;
+
+  // Get/SetLogFilePathTemplate was renamed to Get/SetCaptureFilePathTemplate in 1.1.2.
+  // These unions allow old code to continue compiling without changes
+  union
+  {
+    // deprecated name
+    pRENDERDOC_SetLogFilePathTemplate SetLogFilePathTemplate;
+    // current name
+    pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate;
+  };
+  union
+  {
+    // deprecated name
+    pRENDERDOC_GetLogFilePathTemplate GetLogFilePathTemplate;
+    // current name
+    pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate;
+  };
+
+  pRENDERDOC_GetNumCaptures GetNumCaptures;
+  pRENDERDOC_GetCapture GetCapture;
+
+  pRENDERDOC_TriggerCapture TriggerCapture;
+
+  // IsRemoteAccessConnected was renamed to IsTargetControlConnected in 1.1.1.
+  // This union allows old code to continue compiling without changes
+  union
+  {
+    // deprecated name
+    pRENDERDOC_IsRemoteAccessConnected IsRemoteAccessConnected;
+    // current name
+    pRENDERDOC_IsTargetControlConnected IsTargetControlConnected;
+  };
+  pRENDERDOC_LaunchReplayUI LaunchReplayUI;
+
+  pRENDERDOC_SetActiveWindow SetActiveWindow;
+
+  pRENDERDOC_StartFrameCapture StartFrameCapture;
+  pRENDERDOC_IsFrameCapturing IsFrameCapturing;
+  pRENDERDOC_EndFrameCapture EndFrameCapture;
+
+  // new function in 1.1.0
+  pRENDERDOC_TriggerMultiFrameCapture TriggerMultiFrameCapture;
+
+  // new function in 1.2.0
+  pRENDERDOC_SetCaptureFileComments SetCaptureFileComments;
+
+  // new function in 1.4.0
+  pRENDERDOC_DiscardFrameCapture DiscardFrameCapture;
+
+  // new function in 1.5.0
+  pRENDERDOC_ShowReplayUI ShowReplayUI;
+
+  // new function in 1.6.0
+  pRENDERDOC_SetCaptureTitle SetCaptureTitle;
+
+  // new functions in 1.7.0
+  pRENDERDOC_SetObjectAnnotation SetObjectAnnotation;
+  pRENDERDOC_SetCommandAnnotation SetCommandAnnotation;
+} RENDERDOC_API_1_7_0;
+
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_1;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_2;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_1;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_2;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_2_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_3_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_1;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_2;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_5_0;
+typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_6_0;
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+// RenderDoc API entry point
+//
+// This entry point can be obtained via GetProcAddress/dlsym if RenderDoc is available.
+//
+// The name is the same as the typedef - "RENDERDOC_GetAPI"
+//
+// This function is not thread safe, and should not be called on multiple threads at once.
+// Ideally, call this once as early as possible in your application's startup, before doing
+// any API work, since some configuration functionality etc has to be done also before
+// initialising any APIs.
+//
+// Parameters:
+//   version is a single value from the RENDERDOC_Version above.
+//
+//   outAPIPointers will be filled out with a pointer to the corresponding struct of function
+//   pointers.
+//
+// Returns:
+//   1 - if the outAPIPointers has been filled with a pointer to the API struct requested
+//   0 - if the requested version is not supported or the arguments are invalid.
+//
+typedef int(RENDERDOC_CC *pRENDERDOC_GetAPI)(RENDERDOC_Version version, void **outAPIPointers);
+
+#ifdef __cplusplus
+}    // extern "C"
+#endif

CMakeLists.txt

@@ -203,14 +203,21 @@ file(GLOB kyty_emulator_src CONFIGURE_DEPENDS
 	"${KYTY_SOURCE_DIR}/graphics/shader/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/*.h"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/cfg/*.h"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/decompiler/*.h"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/emitter/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/cfg/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/cfg/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/decode/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/decode/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/translate/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/frontend/translate/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/backend/spirv/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/backend/spirv/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.h"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/opcodes/*.inc"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/*.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/presentation/*.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/presentation/*.h"
 	"${KYTY_SOURCE_DIR}/graphics/presentation/window/*.cpp"
@@ -322,6 +329,7 @@ set(inc_headers
 	${KYTY_THIRD_PARTY_DIR}/fmt/include
 	${KYTY_THIRD_PARTY_DIR}/xxhash
 	${KYTY_THIRD_PARTY_DIR}/cpuinfo/include
+	${KYTY_THIRD_PARTY_DIR}/renderdoc
 	${KYTY_THIRD_PARTY_DIR}/stb
 )
 
@@ -367,9 +375,8 @@ add_kyty_full_emulator_test(shader_cfg_tests "${KYTY_TESTS_DIR}/shaderCfgTests.c
 add_executable(scalar_provenance_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ScalarProvenanceTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ReadLaneElimination.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ReadLaneElimination.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 )
 target_link_libraries(scalar_provenance_tests fmt::fmt)
 target_include_directories(scalar_provenance_tests PRIVATE ${inc_headers})
@@ -413,9 +420,8 @@ add_executable(shader_stage_runtime_tests EXCLUDE_FROM_ALL
 	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/shader/shaderStageRuntime.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
 )
 target_link_libraries(shader_stage_runtime_tests fmt::fmt)
 target_include_directories(shader_stage_runtime_tests PRIVATE ${inc_headers})
@@ -424,13 +430,11 @@ add_executable(resource_tracking_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/ResourceTrackingTests.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/guest_gpu/gpu_format.cpp"
 	"${KYTY_SOURCE_DIR}/graphics/host_gpu/hostMemory.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ScalarProvenance.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtWalker.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/SrtPatcher.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceTracking.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ResourceMaterialization.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/ShaderInfoCollection.cpp"
-	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/BindingLayout.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/SrtWalker.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceTracking.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ResourceMaterialization.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/ShaderInfoCollection.cpp"
+	"${KYTY_SOURCE_DIR}/graphics/shader/recompiler/ir/passes/BindingLayout.cpp"
 )
 target_link_libraries(resource_tracking_tests fmt::fmt)
 target_include_directories(resource_tracking_tests PRIVATE ${inc_headers})
@@ -451,6 +455,9 @@ add_executable(event_queue_lifetime_tests EXCLUDE_FROM_ALL
 target_link_libraries(event_queue_lifetime_tests common fmt::fmt)
 target_include_directories(event_queue_lifetime_tests PRIVATE ${inc_headers})
 
+add_kyty_full_emulator_test(kernel_file_system_tests
+	"${KYTY_TESTS_DIR}/KernelFileSystemTests.cpp")
+
 add_executable(sync_on_address_tests EXCLUDE_FROM_ALL
 	"${KYTY_TESTS_DIR}/SyncOnAddressTests.cpp"
 	"${KYTY_SOURCE_DIR}/kernel/syncOnAddress.cpp"
@@ -511,9 +518,12 @@ if(BUILD_TESTING)
 	add_test(NAME shader_stage_runtime COMMAND $<TARGET_FILE:shader_stage_runtime_tests>)
 	add_test(NAME resource_tracking COMMAND $<TARGET_FILE:resource_tracking_tests>)
 	add_test(NAME event_queue_lifetime COMMAND $<TARGET_FILE:event_queue_lifetime_tests>)
+	add_test(NAME kernel_file_system COMMAND $<TARGET_FILE:kernel_file_system_tests>)
 	add_test(NAME sync_on_address COMMAND $<TARGET_FILE:sync_on_address_tests>)
 	add_test(NAME audio_out2_port COMMAND $<TARGET_FILE:audio_out2_port_tests>)
 	add_test(NAME shader_recompiler_compute COMMAND $<TARGET_FILE:shader_recompiler_compute_tests>)
+	add_test(NAME shader_recompiler_alignbyte
+		COMMAND $<TARGET_FILE:shader_recompiler_compute_tests> --alignbyte-only)
 	add_test(NAME virtual_memory_allocation
 		COMMAND $<TARGET_FILE:virtual_memory_allocation_tests>)
 	add_test(NAME guest_red_zone_patcher
@@ -562,9 +572,11 @@ if(BUILD_TESTING)
 		shader_stage_runtime_tests
 		resource_tracking_tests
 		event_queue_lifetime_tests
+		kernel_file_system_tests
 		sync_on_address_tests
 		shader_recompiler_compute_tests
 		virtual_memory_allocation_tests
+		audio_out2_port_tests
 	)
 endif()
 

src/common/emulatorConfig.cpp

@@ -93,6 +93,10 @@ bool SpirvDebugPrintfEnabled() {
 	return g_config->spirv_debug_printf_enabled;
 }
 
+bool GpuAssistedValidationEnabled() {
+	return g_config->gpu_assisted_validation_enabled && g_config->vulkan_validation_enabled;
+}
+
 bool RenderDocEnabled() {
 	return g_config->renderdoc_enabled;
 }
@@ -101,6 +105,10 @@ bool ReadbackLinearImagesEnabled() {
 	return g_config->readback_linear_images;
 }
 
+bool PlayGoHackEnabled() {
+	return g_config->playgo_hack_enabled;
+}
+
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled() {
 	return g_config->red_zone_protection_enabled;

src/common/emulatorConfig.h

@@ -49,8 +49,10 @@ struct ConfigOptions {
 	std::filesystem::path  printf_output_file          = "_kyty.txt";
 	ProfilerDirection      profiler_direction          = ProfilerDirection::None;
 	bool                   spirv_debug_printf_enabled  = false;
+	bool                   gpu_assisted_validation_enabled = false;
 	bool                   renderdoc_enabled           = false;
 	bool                   readback_linear_images      = false;
+	bool                   playgo_hack_enabled         = false;
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 	bool red_zone_protection_enabled = false;
 #endif
@@ -83,8 +85,11 @@ ProfilerDirection GetProfilerDirection();
 
 bool SpirvDebugPrintfEnabled();
 
+bool GpuAssistedValidationEnabled();
+
 bool RenderDocEnabled();
 bool ReadbackLinearImagesEnabled();
+bool PlayGoHackEnabled();
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 bool RedZoneProtectionEnabled();
 #endif

src/common/hostException.cpp

@@ -2,7 +2,6 @@
 
 #include <atomic>
 #include <cstdio>
-#include <cstdlib>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 #include <windows.h> // IWYU pragma: keep
@@ -28,54 +27,14 @@ namespace Common::HostException {
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
-static thread_local bool    g_in_exception_filter = false;
 
 static_assert(decltype(g_handler)::is_always_lock_free);
 static_assert(decltype(g_install_state)::is_always_lock_free);
-
-[[noreturn]] static void FailFast(const char* reason) noexcept {
-	std::fputs("HostException fail-fast: ", stderr);
-	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
-	std::fputc('\n', stderr);
-	std::fflush(stderr);
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	TerminateProcess(GetCurrentProcess(), static_cast<UINT>(EXCEPTION_NONCONTINUABLE_EXCEPTION));
-#endif
-	std::_Exit(321);
-}
-
-class FilterScope final {
-public:
-	FilterScope() noexcept {
-		if (g_in_exception_filter) {
-			FailFast("nested exception while resolving a host fault");
-		}
-		g_in_exception_filter = true;
-	}
-
-	~FilterScope() { g_in_exception_filter = false; }
-
-	KYTY_CLASS_NO_COPY(FilterScope);
-};
-
-static Handler LoadInstalledHandler() noexcept {
-	if (g_install_state.load(std::memory_order_acquire) == 0) {
-		FailFast("host exception handler is not installed");
-	}
-
-	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
-	return handler;
-}
 #endif
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
-static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
-	FilterScope filter_scope;
-
+static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) noexcept {
 	auto* exception_record = exception->ExceptionRecord;
 
 	if (exception_record->ExceptionCode == DBG_PRINTEXCEPTION_C ||
@@ -105,12 +64,6 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	} else if (exception_record->ExceptionCode == EXCEPTION_ILLEGAL_INSTRUCTION) {
 		info.type = ExceptionType::IllegalInstruction;
 	} else {
-		printf("Unhandled win exception: code=0x%08" PRIx32 ", addr=0x%016" PRIx64
-		       ", rip=0x%016" PRIx64 ", rsp=0x%016" PRIx64 ", rbp=0x%016" PRIx64 "\n",
-		       static_cast<uint32_t>(exception_record->ExceptionCode),
-		       reinterpret_cast<uint64_t>(exception_record->ExceptionAddress),
-		       exception->ContextRecord->Rip, exception->ContextRecord->Rsp,
-		       exception->ContextRecord->Rbp);
 		return EXCEPTION_CONTINUE_SEARCH;
 	}
 
@@ -131,28 +84,21 @@ static LONG WINAPI ExceptionFilter(PEXCEPTION_POINTERS exception) {
 	info.r14 = exception->ContextRecord->R14;
 	info.r15 = exception->ContextRecord->R15;
 
-	const auto handler = LoadInstalledHandler();
-
-	return handler(info) ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler != nullptr && handler(info)) {
+		return EXCEPTION_CONTINUE_EXECUTION;
+	}
+	return EXCEPTION_CONTINUE_SEARCH;
 }
 
 #elif defined(__APPLE__)
 
 static std::atomic<Handler> g_handler {nullptr};
 static std::atomic_uint32_t g_install_state {0};
-static thread_local bool    g_in_exception_filter = false;
 
 static_assert(decltype(g_handler)::is_always_lock_free);
 static_assert(decltype(g_install_state)::is_always_lock_free);
 
-[[noreturn]] static void FailFast(const char* reason) noexcept {
-	std::fputs("HostException fail-fast: ", stderr);
-	std::fputs(reason != nullptr ? reason : "unspecified", stderr);
-	std::fputc('\n', stderr);
-	std::fflush(stderr);
-	std::_Exit(321);
-}
-
 // Translate the x86-64 page-fault error code (mcontext __es.__err) into an access type.
 // bit 1 (0x2) = write, bit 4 (0x10) = instruction fetch, otherwise a read.
 static AccessViolationType DecodeAccess(uint64_t err) {
@@ -170,11 +116,6 @@ static AccessViolationType DecodeAccess(uint64_t err) {
 // re-executing the faulting instruction against the now-fixed protection. An unresolved
 // fault restores the default disposition so the retry terminates the process.
 static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
-	if (g_in_exception_filter) {
-		FailFast("nested exception while resolving a host fault");
-	}
-	g_in_exception_filter = true;
-
 	auto*       uc = static_cast<ucontext_t*>(uctx);
 	const auto* mc = uc->uc_mcontext;
 	const auto& ss = mc->__ss;
@@ -210,14 +151,7 @@ static void SignalHandler(int sig, siginfo_t* si, void* uctx) {
 	info.r15 = ss.__r15;
 
 	const auto handler = g_handler.load(std::memory_order_acquire);
-	if (handler == nullptr) {
-		FailFast("host exception callback is null");
-	}
-
-	const bool resolved   = handler(info);
-	g_in_exception_filter = false;
-
-	if (resolved) {
+	if (handler != nullptr && handler(info)) {
 		return; // retry the faulting instruction against the fixed mapping
 	}
 
@@ -244,8 +178,6 @@ static void ChainToDefault(int signal_number) noexcept {
 }
 
 static void SignalHandler(int signal_number, siginfo_t* signal_info, void* native_context) {
-	FilterScope filter_scope;
-
 	auto* context = static_cast<ucontext_t*>(native_context);
 	auto* gregs   = context->uc_mcontext.gregs;
 
@@ -289,9 +221,8 @@ static void SignalHandler(int signal_number, siginfo_t* signal_info, void* nativ
 	info.r14 = static_cast<uint64_t>(gregs[REG_R14]);
 	info.r15 = static_cast<uint64_t>(gregs[REG_R15]);
 
-	const auto handler = LoadInstalledHandler();
-
-	if (handler(info)) {
+	const auto handler = g_handler.load(std::memory_order_acquire);
+	if (handler != nullptr && handler(info)) {
 		return;
 	}
 
@@ -313,7 +244,7 @@ bool InstallHandler(Handler handler) {
 	g_handler.store(handler, std::memory_order_release);
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-	if (AddVectoredExceptionHandler(1, ExceptionFilter) == nullptr) {
+	if (AddVectoredExceptionHandler(0, ExceptionFilter) == nullptr) {
 		g_handler.store(nullptr, std::memory_order_release);
 		g_install_state.store(0, std::memory_order_release);
 		printf("AddVectoredExceptionHandler() failed\n");

src/graphics/guest_gpu/command_processor/commandProcessor.h

@@ -112,6 +112,7 @@ public:
 	void SetCeComplete(bool complete) { m_ce_complete = complete; }
 	void WaitCe();
 	void WaitDeDiff(uint32_t diff);
+	void WaitForRewind(bool valid);
 	void IncrementDe();
 	void IncrementCe();
 

src/graphics/guest_gpu/command_processor/pm4Dispatch.cpp

@@ -224,6 +224,7 @@ constexpr auto MakeOpcodeDispatchTable() {
 	g_cp_op_func[Pm4::IT_EVENT_WRITE_EOS]           = CpOpEventWriteEos;
 	g_cp_op_func[Pm4::IT_DMA_DATA]                  = CpOpDmaData;
 	g_cp_op_func[Pm4::IT_ACQUIRE_MEM]               = CpOpAcquireMem;
+	g_cp_op_func[Pm4::IT_REWIND]                    = CpOpRewind;
 	g_cp_op_func[Pm4::IT_SET_CONTEXT_REG]           = CpOpSetContextReg;
 	g_cp_op_func[Pm4::IT_SET_SH_REG]                = CpOpSetShaderReg;
 	g_cp_op_func[Pm4::IT_SET_UCONFIG_REG]           = CpOpSetUconfigReg;

src/graphics/guest_gpu/command_processor/pm4Dispatch.h

@@ -133,6 +133,7 @@ uint32_t CpOpDrawIndirectMulti(CommandProcessor&, uint32_t, const uint32_t*, uin
 uint32_t CpOpDrawIndex(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDrawIndexOffset(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpPfpSyncMe(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
+uint32_t CpOpRewind(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpIndexType(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpNumInstances(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);
 uint32_t CpOpDrawIndexAuto(CommandProcessor&, uint32_t, const uint32_t*, uint32_t, uint32_t);

src/graphics/guest_gpu/command_processor/pm4Handlers.cpp

@@ -1627,16 +1627,10 @@ KYTY_HW_UC_PARSER(HwUcSetIaMultiVgtParam) {
 	return num_values;
 }
 
-static void HwIgnoreMultiPrimIbReset([[maybe_unused]] uint32_t value) {}
-
 KYTY_HW_UC_PARSER(HwUcSetMultiPrimIbReset) {
-	auto num_values = KYTY_PM4_LEN(cmd_id) - 2u;
-
-	for (uint32_t i = 0; i < num_values; i++) {
-		HwIgnoreMultiPrimIbReset(buffer[i]);
-	}
-
-	return num_values;
+	EXIT_NOT_IMPLEMENTED(cmd_offset != Pm4::GE_MULTI_PRIM_IB_RESET_EN);
+	cp.GetUcfg().SetPrimitiveResetControl(buffer[0]);
+	return 1;
 }
 
 static void HwUcIgnoreBorderColorTableAddr([[maybe_unused]] uint32_t cmd_offset,
@@ -1836,6 +1830,17 @@ KYTY_CP_OP_PARSER(CpOpPfpSyncMe) {
 	return 1;
 }
 
+KYTY_CP_OP_PARSER(CpOpRewind) {
+	KYTY_PROFILER_FUNCTION();
+
+	EXIT_NOT_IMPLEMENTED(cmd_id != 0xc0005900);
+	EXIT_NOT_IMPLEMENTED((buffer[0] & ~0x81000000u) != 0);
+
+	cp.WaitForRewind((buffer[0] & 0x80000000u) != 0);
+
+	return 1;
+}
+
 KYTY_CP_OP_PARSER(CpOpSetPredication) {
 	KYTY_PROFILER_FUNCTION();
 
@@ -3546,7 +3551,7 @@ void GraphicsInitJmpTablesCxIndirect() {
 		cp.GetCtx().SetReuseOff(value);
 	};
 	g_hw_ctx_indirect_func[Pm4::VGT_MULTI_PRIM_IB_RESET_INDX] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
-		HwIgnoreMultiPrimIbReset(value);
+		cp.GetCtx().SetPrimitiveResetIndex(value);
 	};
 	g_hw_ctx_indirect_func[Pm4::VGT_TESS_DISTRIBUTION] = [](KYTY_HW_CTX_INDIRECT_ARGS) {
 		cp.GetCtx().SetTessDistribution(value);
@@ -4424,7 +4429,7 @@ void GraphicsInitJmpTablesUcIndirect() {
 		HwUcIgnoreIaMultiVgtParam(value);
 	};
 	g_hw_uc_indirect_func[Pm4::GE_MULTI_PRIM_IB_RESET_EN] = [](KYTY_HW_UC_INDIRECT_ARGS) {
-		HwIgnoreMultiPrimIbReset(value);
+		cp.GetUcfg().SetPrimitiveResetControl(value);
 	};
 	g_hw_uc_indirect_func[Pm4::TA_CS_BC_BASE_ADDR] = [](KYTY_HW_UC_INDIRECT_ARGS) {
 		HwUcIgnoreBorderColorTableAddr(cmd_offset, value);

src/graphics/guest_gpu/graphicsRun.cpp

@@ -13,6 +13,7 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/renderer/sync.h"
+#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/shader/shader.h"
@@ -36,7 +37,7 @@ static thread_local CommandProcessor* g_current_processor = nullptr;
 static thread_local Pm4Execution*     g_current_execution = nullptr;
 static thread_local bool              g_gpu_mutex_owned   = false;
 static thread_local bool              g_gpu_thread        = false;
-static thread_local GpuState*         g_gpu_state         = nullptr;
+static thread_local GuestGpu*         g_gpu_state         = nullptr;
 
 class GpuMutexLock final {
 public:
@@ -59,115 +60,30 @@ private:
 	Common::Mutex& m_mutex;
 };
 
-struct OwnedCmdBuffer {
-	OwnedCmdBuffer() = default;
-	explicit OwnedCmdBuffer(const uint32_t* data, uint32_t count) {
-		EXIT_IF(data == nullptr && count != 0);
-		if (count != 0) {
-			m_words.assign(data, data + count);
-		}
-	}
-
-	[[nodiscard]] bool      Empty() const noexcept { return m_words.empty(); }
-	[[nodiscard]] uint32_t  Size() const noexcept { return static_cast<uint32_t>(m_words.size()); }
-	[[nodiscard]] uint32_t* Data() noexcept { return m_words.data(); }
-
-private:
-	std::vector<uint32_t> m_words;
-};
-
-class GpuState {
-public:
-	static constexpr uint32_t ComputePipeCount     = 7;
-	static constexpr uint32_t QueuesPerComputePipe = 8;
-	static constexpr uint32_t ComputeQueueCount    = ComputePipeCount * QueuesPerComputePipe;
-	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
-
-	explicit GpuState(RenderContext& renderer): m_renderer(renderer) {
-		EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-		m_gfx_cp = std::make_unique<CommandProcessor>(renderer);
-		m_thread = std::jthread(ThreadRun, this);
-	}
-	~GpuState();
-
-	KYTY_CLASS_NO_COPY(GpuState);
-
-	void Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
-	            uint32_t num_const_dw, bool trigger_agc_interrupt_on_done);
-	void SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
-	                   bool trigger_agc_interrupt_on_done);
-	void SubmitFlipPreparation(uint64_t request_id);
-	void Done();
-	void Shutdown();
-	[[nodiscard]] bool        IsStopping();
-	void                      SendCommand(Common::UniqueFunction<void>&& command);
-	void                      SendCommandSync(Common::UniqueFunction<void>&& command);
-	int                       GetFrameNum();
-	[[nodiscard]] static bool IsGpuThread() noexcept { return g_gpu_thread; }
-
-private:
-	enum class SubmissionType { Graphics, Compute, FlipPreparation };
-
-	struct Submission {
-		SubmissionType type     = SubmissionType::Graphics;
-		uint32_t       queue_id = 0;
-		OwnedCmdBuffer commands;
-		OwnedCmdBuffer constant_commands;
-		Pm4Execution   command_execution;
-		Pm4Execution   constant_execution;
-		bool           trigger_agc_interrupt_on_done = false;
-		bool           reset_processor               = false;
-		bool           started                       = false;
-		bool           command_complete              = false;
-		bool           constant_complete             = false;
-		bool           blocked                       = false;
-		uint64_t       flip_request_id               = 0;
-	};
-
-	void              Enqueue(Submission submission);
-	void              WaitForIdle();
-	void              ProcessCommands();
-	bool              Process(Submission& submission);
-	static void       ThreadRun(void* data);
-	CommandProcessor& GetProcessor(uint32_t queue_id);
-
-	RenderContext&                                 m_renderer;
-	Common::Mutex                                  m_submission_mutex;
-	Common::Mutex                                  m_queue_mutex;
-	std::mutex                                     m_shutdown_mutex;
-	Common::CondVar                                m_work_available;
-	Common::CondVar                                m_idle;
-	std::array<std::deque<Submission>, QueueCount> m_queues;
-	std::deque<Common::UniqueFunction<void>>       m_commands;
-	std::atomic_uint32_t                           m_pending_commands {0};
-	uint32_t                                       m_next_queue        = 0;
-	uint32_t                                       m_submission_count  = 0;
-	bool                                           m_processing        = false;
-	bool                                           m_graphics_done     = true;
-	bool                                           m_accepting         = true;
-	bool                                           m_stopping          = false;
-	bool                                           m_shutdown_complete = false;
-
-	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
-	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;
-
-	uint64_t        m_submit_id = 0;
-	std::atomic_int m_done_num  = 0;
-	std::jthread    m_thread;
-
-	friend class CommandProcessor;
-};
-
 static bool GraphicsRunDebugDumpEnabled() {
 	return Config::GraphicsDebugDumpEnabled() &&
 	       Config::GetPrintfDirection() != Config::OutputDirection::Silent;
 }
 
-GpuState::~GpuState() {
+GuestGpu::OwnedCmdBuffer::OwnedCmdBuffer(const uint32_t* data, uint32_t count) {
+	EXIT_IF(data == nullptr && count != 0);
+	if (count != 0) {
+		m_words.assign(data, data + count);
+	}
+}
+
+GuestGpu::GuestGpu(RenderContext& renderer): m_renderer(renderer) {
+	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
+	GraphicsInitJmpTables();
+	m_gfx_cp = std::make_unique<CommandProcessor>(renderer);
+	m_thread = std::jthread(ThreadRun, this);
+}
+
+GuestGpu::~GuestGpu() {
 	Shutdown();
 }
 
-void GpuState::Shutdown() {
+void GuestGpu::Shutdown() {
 	std::lock_guard shutdown_lock(m_shutdown_mutex);
 	if (m_shutdown_complete) {
 		return;
@@ -184,12 +100,12 @@ void GpuState::Shutdown() {
 	m_shutdown_complete = true;
 }
 
-bool GpuState::IsStopping() {
+bool GuestGpu::IsStopping() {
 	Common::LockGuard lock(m_queue_mutex);
 	return m_stopping;
 }
 
-void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
+void GuestGpu::SendCommand(Common::UniqueFunction<void>&& command) {
 	EXIT_IF(!command);
 	if (IsGpuThread()) {
 		command();
@@ -202,7 +118,7 @@ void GpuState::SendCommand(Common::UniqueFunction<void>&& command) {
 	m_work_available.Signal();
 }
 
-void GpuState::ProcessCommands() {
+void GuestGpu::ProcessCommands() {
 	EXIT_IF(!IsGpuThread());
 	while (m_pending_commands.load(std::memory_order_acquire) != 0) {
 		Common::UniqueFunction<void> command;
@@ -217,7 +133,7 @@ void GpuState::ProcessCommands() {
 	}
 }
 
-void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
+void GuestGpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	EXIT_IF(!command);
 	if (IsGpuThread()) {
 		command();
@@ -231,8 +147,12 @@ void GpuState::SendCommandSync(Common::UniqueFunction<void>&& command) {
 	done.acquire();
 }
 
-void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
+void GuestGpu::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t* cmd_const_buffer,
                       uint32_t num_const_dw, bool trigger_agc_interrupt_on_done) {
+	if (num_draw_dw == 0) {
+		return;
+	}
+	EXIT_IF(cmd_draw_buffer == nullptr);
 	GpuMutexLock lock(m_submission_mutex);
 	Submission   submission;
 	submission.type                          = SubmissionType::Graphics;
@@ -245,8 +165,9 @@ void GpuState::Submit(uint32_t* cmd_draw_buffer, uint32_t num_draw_dw, uint32_t*
 	Enqueue(std::move(submission));
 }
 
-void GpuState::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
+void GuestGpu::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_dw,
                              bool trigger_agc_interrupt_on_done) {
+	EXIT_IF(cmd_buffer == nullptr || num_dw == 0);
 	GpuMutexLock lock(m_submission_mutex);
 
 	constexpr uint32_t compute_queue_base = 0x20u;
@@ -262,7 +183,7 @@ void GpuState::SubmitCompute(uint32_t queue, uint32_t* cmd_buffer, uint32_t num_
 	Enqueue(std::move(submission));
 }
 
-void GpuState::SubmitFlipPreparation(uint64_t request_id) {
+void GuestGpu::SubmitFlipPreparation(uint64_t request_id) {
 	GpuMutexLock lock(m_submission_mutex);
 	Submission   submission;
 	submission.type            = SubmissionType::FlipPreparation;
@@ -273,20 +194,26 @@ void GpuState::SubmitFlipPreparation(uint64_t request_id) {
 	Enqueue(std::move(submission));
 }
 
-void GpuState::Done() {
+void GuestGpu::Done() {
 	GpuMutexLock lock(m_submission_mutex);
 	if (!IsGpuThread()) {
 		WaitForIdle();
 	}
+	if (RenderDocCaptureInProgress()) {
+		SendCommandSync([this] {
+			Common::LockGuard render_lock(m_renderer.GetMutex());
+			RenderDocEndCapture();
+		});
+	}
 	m_graphics_done = true;
 	m_done_num++;
 }
 
-int GpuState::GetFrameNum() {
+int GuestGpu::GetFrameNum() const {
 	return m_done_num;
 }
 
-CommandProcessor& GpuState::GetProcessor(uint32_t queue_id) {
+CommandProcessor& GuestGpu::GetProcessor(uint32_t queue_id) {
 	EXIT_IF(queue_id >= QueueCount);
 	if (queue_id == 0) {
 		return *m_gfx_cp;
@@ -374,6 +301,12 @@ void CommandProcessor::WaitDeDiff(uint32_t diff) {
 	}
 }
 
+void CommandProcessor::WaitForRewind(bool valid) {
+	if (!valid) {
+		SuspendPm4();
+	}
+}
+
 void CommandProcessor::IncrementDe() {
 	m_de_count++;
 }
@@ -489,6 +422,13 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 			default: return false;
 		}
 	};
+	if (dst_sel == 2) {
+		// kNowhere discards the GL2 prefetch destination without a guest-visible write.
+		if (src_sel != 3) {
+			EXIT("unsupported dmaData nowhere source selector 0x%02" PRIx8 "\n", src_sel);
+		}
+		return;
+	}
 	bool dst_gds = false;
 	if (!decode_gds(dst_sel, dst_gds)) {
 		EXIT("unsupported dmaData destination selector 0x%02" PRIx8 "\n", dst_sel);
@@ -511,7 +451,7 @@ void CommandProcessor::DmaData(uint8_t engine, uint8_t dst_sel, uint8_t dst_cach
 	                        dst_gds, src_gds);
 }
 
-void GpuState::Enqueue(Submission submission) {
+void GuestGpu::Enqueue(Submission submission) {
 	EXIT_IF(submission.queue_id >= QueueCount);
 	Common::LockGuard lock(m_queue_mutex);
 	EXIT_IF(!m_accepting);
@@ -520,15 +460,15 @@ void GpuState::Enqueue(Submission submission) {
 	m_work_available.Signal();
 }
 
-void GpuState::WaitForIdle() {
+void GuestGpu::WaitForIdle() {
 	Common::LockGuard lock(m_queue_mutex);
 	while (m_processing || !m_commands.empty() || m_submission_count != 0) {
 		m_idle.Wait(&m_queue_mutex);
 	}
 }
 
-void GpuState::ThreadRun(void* data) {
-	auto* gpu = static_cast<GpuState*>(data);
+void GuestGpu::ThreadRun(void* data) {
+	auto* gpu = static_cast<GuestGpu*>(data);
 	EXIT_IF(gpu == nullptr);
 	KYTY_PROFILER_THREAD("Thread_Gpu");
 	g_gpu_thread = true;
@@ -624,9 +564,13 @@ void GpuState::ThreadRun(void* data) {
 	}
 }
 
-bool GpuState::Process(Submission& submission) {
-	auto&      cp          = GetProcessor(submission.queue_id);
+bool GuestGpu::Process(Submission& submission) {
 	const bool first_slice = !submission.started;
+	if (first_slice && RenderDocCaptureRequested()) {
+		Common::LockGuard render_lock(m_renderer.GetMutex());
+		RenderDocStartCapture();
+	}
+	auto& cp = GetProcessor(submission.queue_id);
 
 	if (first_slice && submission.reset_processor) {
 		cp.Reset();
@@ -1662,57 +1606,8 @@ void CommandProcessor::SynchronizeGpu() {
 	GetScheduler().FinishCurrent();
 }
 
-Gpu::Gpu(RenderContext& renderer) {
-	EXIT_NOT_IMPLEMENTED(!Common::Thread::IsMainThread());
-	GraphicsInitJmpTables();
-	m_state = std::make_unique<GpuState>(renderer);
-}
-
-Gpu::~Gpu() = default;
-
-void Gpu::Shutdown() {
-	m_state->Shutdown();
-}
-
-bool Gpu::IsStopping() {
-	return m_state->IsStopping();
-}
-
-void Gpu::SendCommand(Common::UniqueFunction<void>&& command) {
-	m_state->SendCommand(std::move(command));
-}
-
-void Gpu::SendCommandSync(Common::UniqueFunction<void>&& command) {
-	m_state->SendCommandSync(std::move(command));
-}
-
-void Gpu::Submit(uint32_t* draw_commands, uint32_t draw_size_dw, uint32_t* constant_commands,
-                 uint32_t constant_size_dw, bool trigger_agc_interrupt_on_done) {
-	EXIT_IF(draw_commands == nullptr || draw_size_dw == 0);
-	m_state->Submit(draw_commands, draw_size_dw, constant_commands, constant_size_dw,
-	                trigger_agc_interrupt_on_done);
-}
-
-void Gpu::SubmitCompute(uint32_t queue, uint32_t* commands, uint32_t size_dw,
-                        bool trigger_agc_interrupt_on_done) {
-	EXIT_IF(commands == nullptr || size_dw == 0);
-	m_state->SubmitCompute(queue, commands, size_dw, trigger_agc_interrupt_on_done);
-}
-
-void Gpu::SubmitFlipPreparation(uint64_t request_id) {
-	m_state->SubmitFlipPreparation(request_id);
-}
-
-void Gpu::Done() {
-	m_state->Done();
-}
-
-int Gpu::GetFrameNum() const {
-	return m_state->GetFrameNum();
-}
-
-bool Gpu::IsGpuThread() noexcept {
-	return GpuState::IsGpuThread();
+bool GuestGpu::IsGpuThread() noexcept {
+	return g_gpu_thread;
 }
 
 } // namespace Libs::Graphics

src/graphics/guest_gpu/graphicsRun.h

@@ -3,20 +3,28 @@
 
 #include "common/abi.h"
 #include "common/common.h"
+#include "common/threads.h"
 #include "common/uniqueFunction.h"
+#include "graphics/guest_gpu/command_processor/commandProcessor.h"
 
+#include <array>
+#include <atomic>
+#include <cstdint>
+#include <deque>
 #include <memory>
+#include <mutex>
+#include <thread>
+#include <vector>
 
 namespace Libs::Graphics {
 
-class GpuState;
 class RenderContext;
 
-class Gpu final {
+class GuestGpu final {
 public:
-	explicit Gpu(RenderContext& renderer);
-	~Gpu();
-	KYTY_CLASS_NO_COPY(Gpu);
+	explicit GuestGpu(RenderContext& renderer);
+	~GuestGpu();
+	KYTY_CLASS_NO_COPY(GuestGpu);
 
 	void               Shutdown();
 	[[nodiscard]] bool IsStopping();
@@ -34,7 +42,76 @@ public:
 	[[nodiscard]] static bool IsGpuThread() noexcept;
 
 private:
-	std::unique_ptr<GpuState> m_state;
+	static constexpr uint32_t ComputePipeCount     = 7;
+	static constexpr uint32_t QueuesPerComputePipe = 8;
+	static constexpr uint32_t ComputeQueueCount    = ComputePipeCount * QueuesPerComputePipe;
+	static constexpr uint32_t QueueCount           = 1 + ComputeQueueCount;
+
+	class OwnedCmdBuffer {
+	public:
+		OwnedCmdBuffer() = default;
+		explicit OwnedCmdBuffer(const uint32_t* data, uint32_t count);
+
+		[[nodiscard]] bool     Empty() const noexcept { return m_words.empty(); }
+		[[nodiscard]] uint32_t Size() const noexcept {
+			return static_cast<uint32_t>(m_words.size());
+		}
+		[[nodiscard]] uint32_t* Data() noexcept { return m_words.data(); }
+
+	private:
+		std::vector<uint32_t> m_words;
+	};
+
+	enum class SubmissionType { Graphics, Compute, FlipPreparation };
+
+	struct Submission {
+		SubmissionType type     = SubmissionType::Graphics;
+		uint32_t       queue_id = 0;
+		OwnedCmdBuffer commands;
+		OwnedCmdBuffer constant_commands;
+		Pm4Execution   command_execution;
+		Pm4Execution   constant_execution;
+		bool           trigger_agc_interrupt_on_done = false;
+		bool           reset_processor               = false;
+		bool           started                       = false;
+		bool           command_complete              = false;
+		bool           constant_complete             = false;
+		bool           blocked                       = false;
+		uint64_t       flip_request_id               = 0;
+	};
+
+	void              Enqueue(Submission submission);
+	void              WaitForIdle();
+	void              ProcessCommands();
+	bool              Process(Submission& submission);
+	static void       ThreadRun(void* data);
+	CommandProcessor& GetProcessor(uint32_t queue_id);
+
+	RenderContext&                                 m_renderer;
+	Common::Mutex                                  m_submission_mutex;
+	Common::Mutex                                  m_queue_mutex;
+	std::mutex                                     m_shutdown_mutex;
+	Common::CondVar                                m_work_available;
+	Common::CondVar                                m_idle;
+	std::array<std::deque<Submission>, QueueCount> m_queues;
+	std::deque<Common::UniqueFunction<void>>       m_commands;
+	std::atomic_uint32_t                           m_pending_commands {0};
+	uint32_t                                       m_next_queue        = 0;
+	uint32_t                                       m_submission_count  = 0;
+	bool                                           m_processing        = false;
+	bool                                           m_graphics_done     = true;
+	bool                                           m_accepting         = true;
+	bool                                           m_stopping          = false;
+	bool                                           m_shutdown_complete = false;
+
+	std::unique_ptr<CommandProcessor>                                m_gfx_cp;
+	std::array<std::unique_ptr<CommandProcessor>, ComputeQueueCount> m_compute_cp;
+
+	uint64_t        m_submit_id = 0;
+	std::atomic_int m_done_num  = 0;
+	std::jthread    m_thread;
+
+	friend class CommandProcessor;
 };
 } // namespace Libs::Graphics
 

src/graphics/guest_gpu/hardwareContext.h

@@ -295,10 +295,7 @@ struct ClipControl {
 	bool    linear_attribute_clip_enable        = false;
 	bool    force_viewport_index_from_vs_enable = false;
 
-	[[nodiscard]] bool IsZClipModeRepresentable() const {
-		return min_z_clip_disable == max_z_clip_disable;
-	}
-	[[nodiscard]] bool IsZClipEnabled() const { return !min_z_clip_disable; }
+	[[nodiscard]] bool IsZClipEnabled() const { return !min_z_clip_disable && !max_z_clip_disable; }
 };
 
 struct DepthControl {
@@ -941,8 +938,10 @@ public:
 	[[nodiscard]] uint8_t GetStencilClearValue() const { return m_stencil_clear_value; }
 	void SetStencilClearValue(uint8_t clear_value) { m_stencil_clear_value = clear_value; }
 
-	[[nodiscard]] float GetLineWidth() const { return m_line_width; }
-	void                SetLineWidth(float width) { m_line_width = width; }
+	[[nodiscard]] float    GetLineWidth() const { return m_line_width; }
+	void                   SetLineWidth(float width) { m_line_width = width; }
+	[[nodiscard]] uint32_t GetPrimitiveResetIndex() const { return m_primitive_reset_index; }
+	void SetPrimitiveResetIndex(uint32_t index) { m_primitive_reset_index = index; }
 
 	[[nodiscard]] const ShaderRegisters& GetShaderRegisters() const { return m_sh_regs; }
 
@@ -987,7 +986,8 @@ public:
 	void SetScShaderControl(uint32_t value) { m_sh_regs.m_paScShaderControl = value; }
 
 private:
-	float m_line_width = 1.0f;
+	float    m_line_width            = 1.0f;
+	uint32_t m_primitive_reset_index = 0xffffffffu;
 
 	BlendControl    m_blend_control[8];
 	BlendColor      m_blend_color;
@@ -1035,6 +1035,8 @@ public:
 	[[nodiscard]] uint32_t GetIndexOffset() const { return m_index_offset; }
 	void                   SetObjectId(uint32_t object_id) { m_object_id = object_id; }
 	[[nodiscard]] uint32_t GetObjectId() const { return m_object_id; }
+	void SetPrimitiveResetControl(uint32_t control) { m_primitive_reset_control = control; }
+	[[nodiscard]] uint32_t GetPrimitiveResetControl() const { return m_primitive_reset_control; }
 
 	[[nodiscard]] const GeControl& GetGeControl() const { return m_ge_cntl; }
 	void                           SetGeControl(const GeControl& control) { m_ge_cntl = control; }
@@ -1049,9 +1051,10 @@ public:
 	void SetGdsOaAddress(uint32_t value) { m_gds_oa.counters[m_gds_oa.GetIndex()].address = value; }
 
 private:
-	Prospero::PrimitiveType m_prim_type    = Prospero::PrimitiveType::kNone;
-	uint32_t                m_index_offset = 0;
-	uint32_t                m_object_id    = 0;
+	Prospero::PrimitiveType m_prim_type               = Prospero::PrimitiveType::kNone;
+	uint32_t                m_index_offset            = 0;
+	uint32_t                m_object_id               = 0;
+	uint32_t                m_primitive_reset_control = 0;
 
 	GeControl    m_ge_cntl;
 	GeUserVgprEn m_ge_user_vgpr_en;

src/graphics/guest_gpu/tile.cpp

@@ -1351,6 +1351,7 @@ uint32_t TileGetTexturePitch(Prospero::BufferFormat format, uint32_t width,
 			}
 			break;
 		}
+		case Prospero::TileMode::kStandard4KB:
 		case Prospero::TileMode::kStandard64KB:
 		case Prospero::TileMode::kPrt:
 		case Prospero::TileMode::kDepth:

src/graphics/host_gpu/renderer/cache/bufferCache.cpp

@@ -257,11 +257,7 @@ void BufferCache::InvalidateMemory(uint64_t vaddr, uint64_t size) {
 }
 
 void BufferCache::ReadMemory(uint64_t vaddr, uint64_t size, bool is_write) {
-	if (Gpu::IsGpuThread()) {
-		ReadMemoryOnGpu(vaddr, size, is_write);
-		return;
-	}
-	if (CommandScheduler::InDeferredOperation()) {
+	if (!GuestGpu::IsGpuThread() && CommandScheduler::InDeferredOperation()) {
 		EXIT("unsupported buffer readback from an asynchronous GPU completion, "
 		     "addr=0x%016" PRIx64 " size=0x%016" PRIx64 "\n",
 		     vaddr, size);
@@ -405,17 +401,14 @@ BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command
                                                           uint64_t size) {
 	const auto begin = AlignDown(vaddr);
 	const auto end   = AlignUp(vaddr + size);
-	auto       it    = m_buffers.upper_bound(vaddr);
-	if (it != m_buffers.begin()) {
-		auto previous = std::prev(it);
-		if (previous->second->buffer->IsInBounds(vaddr, size)) {
-			it = previous;
+	const auto next  = m_buffers.upper_bound(vaddr);
+	if (next != m_buffers.begin()) {
+		auto owner = std::prev(next);
+		if (owner->second->buffer->IsInBounds(vaddr, size)) {
+			owner->second->tick_accessed_last = m_gc_tick;
+			return *owner->second;
 		}
 	}
-	if (it != m_buffers.end() && it->second->buffer->IsInBounds(vaddr, size)) {
-		it->second->tick_accessed_last = m_gc_tick;
-		return *it->second;
-	}
 
 	CacheRange merged {.address = begin, .size = end - begin};
 	using Iterator = decltype(m_buffers.begin());
@@ -435,21 +428,6 @@ BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command
 			overlaps.push_back(candidate);
 		}
 	}
-	for (const auto overlap: overlaps) {
-		auto&                                      old = *overlap->second;
-		std::vector<std::pair<uint64_t, uint64_t>> uploads;
-		m_memory_tracker.ForEachUploadRange(
-		    old.vaddr, old.size, false,
-		    [&](uint64_t address, uint64_t bytes) noexcept {
-			    uploads.emplace_back(address, bytes);
-		    },
-		    [&]() noexcept {
-			    for (const auto& [address, bytes]: uploads) {
-				    Upload(command, *old.buffer, old.buffer->Offset(address),
-				           reinterpret_cast<const void*>(address), bytes);
-			    }
-		    });
-	}
 
 	auto cached                = std::make_unique<CachedBuffer>();
 	cached->vaddr              = merged.address;
@@ -473,6 +451,24 @@ BufferCache::CachedBuffer& BufferCache::GetOrCreateBuffer(CommandBuffer& command
 	return *m_buffers.emplace(cached->vaddr, std::move(cached)).first->second;
 }
 
+bool BufferCache::SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
+                                    uint64_t size, bool is_written, bool is_texel_buffer) {
+	std::vector<std::pair<uint64_t, uint64_t>> uploads;
+	m_memory_tracker.ForEachUploadRange(
+	    vaddr, size, is_written,
+	    [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); },
+	    [&]() noexcept {
+		    for (const auto& [address, bytes]: uploads) {
+			    Upload(command, buffer, buffer.Offset(address),
+			           reinterpret_cast<const void*>(address), bytes);
+		    }
+	    });
+	if (is_texel_buffer && !is_written) {
+		return SynchronizeBufferFromImage(buffer, vaddr, size);
+	}
+	return false;
+}
+
 BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr, uint64_t size,
                                         bool is_written, bool is_read, bool is_formatted) {
 	if (command.IsInvalid() || command.IsExecute()) {
@@ -504,25 +500,14 @@ BufferBinding BufferCache::ObtainBuffer(CommandBuffer& command, uint64_t vaddr,
 		(void)m_texture_cache.InvalidateMemoryFromGPU(vaddr, size, true);
 	}
 
-	auto&                                      cached = GetOrCreateBuffer(command, vaddr, size);
-	std::vector<std::pair<uint64_t, uint64_t>> uploads;
-	m_memory_tracker.ForEachUploadRange(
-	    vaddr, size, is_written,
-	    [&](uint64_t address, uint64_t bytes) noexcept { uploads.emplace_back(address, bytes); },
-	    [&]() noexcept {
-		    for (const auto& [address, bytes]: uploads) {
-			    Upload(command, *cached.buffer, cached.buffer->Offset(address),
-			           reinterpret_cast<const void*>(address), bytes);
-		    }
-	    });
+	auto& cached = GetOrCreateBuffer(command, vaddr, size);
+	(void)SynchronizeBuffer(command, *cached.buffer, vaddr, size, is_written,
+	                        is_formatted && is_read);
 	if (is_written) {
 		m_gpu_modified_ranges.Add(vaddr, size);
 	}
 	auto     owner  = cached.buffer;
 	uint64_t offset = cached.buffer->Offset(vaddr);
-	if (is_formatted && is_read && !is_written) {
-		(void)SynchronizeBufferFromImage(*owner, vaddr, size);
-	}
 	return {owner, owner->Handle(), offset};
 }
 
@@ -542,13 +527,13 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 	    size > TRACKER_ADDRESS_SIZE - vaddr) {
 		EXIT("BufferCache: invalid image source\n");
 	}
-	auto find_owner = [&]() {
+	auto find_owner = [&]() -> CachedBuffer* {
 		auto owner = m_buffers.upper_bound(vaddr);
 		if (owner == m_buffers.begin()) {
-			return m_buffers.end();
+			return nullptr;
 		}
 		--owner;
-		return owner->second->buffer->IsInBounds(vaddr, size) ? owner : m_buffers.end();
+		return owner->second->buffer->IsInBounds(vaddr, size) ? owner->second.get() : nullptr;
 	};
 
 	{
@@ -559,85 +544,47 @@ ImageBufferSource BufferCache::ObtainBufferForImage(uint64_t vaddr, uint64_t siz
 		m_memory_tracker.ValidateGpuDirtyOwnership(m_gpu_modified_ranges, vaddr, size,
 		                                           "image source");
 
-		auto owner = find_owner();
-		if (has_dirty_buffer_source && owner == m_buffers.end()) {
-			CacheRange merged {.address = AlignDown(vaddr),
-			                   .size    = AlignUp(vaddr + size) - AlignDown(vaddr)};
-			using Iterator = decltype(m_buffers.begin());
-			std::vector<Iterator> overlaps;
-			auto                  first = m_buffers.lower_bound(merged.address);
-			if (first != m_buffers.begin()) {
-				auto previous = std::prev(first);
-				if (ResolveOverlap(merged, {previous->second->vaddr, previous->second->size})) {
-					first = previous;
-				}
-			}
-			for (auto candidate = first; candidate != m_buffers.end(); ++candidate) {
-				if (candidate->first >= merged.address + merged.size) {
-					break;
-				}
-				if (ResolveOverlap(merged, {candidate->second->vaddr, candidate->second->size})) {
-					overlaps.push_back(candidate);
-				}
-			}
-			if (overlaps.empty()) {
+		auto* owner = find_owner();
+		if (has_dirty_buffer_source && owner == nullptr) {
+			if (!IsRegionRegistered(vaddr, size)) {
 				EXIT("BufferCache: GPU-dirty image source has no native buffer\n");
 			}
-
-			auto cached                = std::make_unique<CachedBuffer>();
-			cached->vaddr              = merged.address;
-			cached->size               = merged.size;
-			cached->tick_accessed_last = m_gc_tick;
-			cached->buffer =
-			    std::make_shared<Buffer>(m_graphics, m_scheduler, MemoryUsage::DeviceLocal,
-			                             merged.address, AllFlags, merged.size);
-			for (const auto overlap: overlaps) {
-				const auto& old = *overlap->second;
-				cached->buffer->CopyFrom(m_scheduler.Current(), *old.buffer, 0,
-				                         old.vaddr - cached->vaddr, old.size);
-				m_scheduler.Current().RetainResourceUntilFence(old.buffer);
-			}
-			for (const auto overlap: overlaps) {
-				if (overlap->second->size > m_total_used_memory) {
-					EXIT("BufferCache: allocation accounting underflow\n");
-				}
-				m_total_used_memory -= overlap->second->size;
-				m_buffers.erase(overlap);
-			}
-			m_total_used_memory += cached->size;
-			owner = m_buffers.emplace(cached->vaddr, std::move(cached)).first;
-			if (!owner->second->buffer->IsInBounds(vaddr, size)) {
+			auto& cached        = GetOrCreateBuffer(m_scheduler.Current(), vaddr, size);
+			owner               = &cached;
+			const auto resolved = m_buffers.find(cached.vaddr);
+			if (resolved == m_buffers.end() || resolved->second.get() != &cached ||
+			    !cached.buffer->IsInBounds(vaddr, size)) {
 				EXIT("BufferCache: merged image source does not contain the requested range\n");
 			}
 		}
-		if (owner != m_buffers.end() && !cpu_modified &&
-		    (!gpu_modified || has_dirty_buffer_source)) {
-			owner->second->tick_accessed_last = m_gc_tick;
-			return {owner->second->buffer.get(), owner->second->buffer->Offset(vaddr)};
+		if (owner != nullptr && !cpu_modified && (!gpu_modified || has_dirty_buffer_source)) {
+			owner->tick_accessed_last = m_gc_tick;
+			return {owner->buffer.get(), owner->buffer->Offset(vaddr)};
 		}
-		if (has_dirty_buffer_source && owner == m_buffers.end()) {
+		if (has_dirty_buffer_source && owner == nullptr) {
 			EXIT("BufferCache: GPU-dirty image source could not resolve its native owner\n");
 		}
 	}
 
 	auto [staging, stage_offset] = m_staging_buffer.Map(size, 16);
-	if (staging == nullptr || !Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size)) {
+	if (staging == nullptr || (!Libs::LibKernel::Memory::TryReadBacking(vaddr, staging, size) &&
+	                           !Libs::LibKernel::Memory::TryReadPrtBacking(vaddr, staging, size))) {
 		EXIT("BufferCache: failed to read mapped guest image backing\n");
 	}
 	m_staging_buffer.Commit();
 
 	const auto dirty                   = m_gpu_modified_ranges.Intersections(vaddr, size);
 	const bool has_dirty_buffer_source = !dirty.empty();
-	auto       owner                   = find_owner();
-	if (has_dirty_buffer_source && owner == m_buffers.end()) {
+	auto*      owner                   = find_owner();
+	if (has_dirty_buffer_source && owner == nullptr) {
 		EXIT("BufferCache: GPU-dirty image source lost its native owner\n");
 	}
-	if (owner == m_buffers.end() ||
+	if (owner == nullptr ||
 	    (m_memory_tracker.IsRegionGpuModified(vaddr, size) && !has_dirty_buffer_source)) {
 		return {&m_staging_buffer, stage_offset};
 	}
 
-	auto& cached              = *owner->second;
+	auto& cached              = *owner;
 	cached.tick_accessed_last = m_gc_tick;
 	std::vector<std::pair<uint64_t, uint64_t>> uploads;
 	m_memory_tracker.ForEachUploadRange(
@@ -724,10 +671,11 @@ void BufferCache::CopyBuffer(uint64_t dst_vaddr, uint64_t src_vaddr, uint64_t si
 	if ((dst_memory && dst_vaddr == 0) || (src_memory && src_vaddr == 0) || size == 0 ||
 	    ((dst_gds || src_gds) && ((dst_vaddr | src_vaddr | size) & 3u) != 0) ||
 	    size > UINT64_MAX - dst_vaddr || size > UINT64_MAX - src_vaddr || (dst_gds && src_gds) ||
-	    (dst_gds == src_gds && src_vaddr < dst_vaddr + size && dst_vaddr < src_vaddr + size) ||
 	    (dst_gds && (dst_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - dst_vaddr)) ||
 	    (src_gds && (src_vaddr > m_gds_buffer.Size() || size > m_gds_buffer.Size() - src_vaddr))) {
-		EXIT("BufferCache: invalid or overlapping copy range\n");
+		EXIT("BufferCache: invalid copy range, src=0x%016" PRIx64 " dst=0x%016" PRIx64
+		     " size=0x%016" PRIx64 " src_gds=%d dst_gds=%d\n",
+		     src_vaddr, dst_vaddr, size, static_cast<int>(src_gds), static_cast<int>(dst_gds));
 	}
 	if (src_memory || dst_memory) {
 		const auto src_region =

src/graphics/host_gpu/renderer/cache/bufferCache.h

@@ -89,6 +89,8 @@ private:
 	            const void* source, uint64_t size);
 	[[nodiscard]] CachedBuffer& GetOrCreateBuffer(CommandBuffer& command, uint64_t vaddr,
 	                                              uint64_t size);
+	[[nodiscard]] bool SynchronizeBuffer(CommandBuffer& command, Buffer& buffer, uint64_t vaddr,
+	                                     uint64_t size, bool is_written, bool is_texel_buffer);
 	[[nodiscard]] bool SynchronizeBufferFromImage(Buffer& buffer, uint64_t vaddr, uint64_t size);
 	[[nodiscard]] std::vector<DownloadRange> RecordDownloads(std::span<const DownloadCopy> copies);
 	void PublishDownloads(std::span<const DownloadRange> downloads);

src/graphics/host_gpu/renderer/cache/gpuResourceManager.h

@@ -13,7 +13,7 @@
 namespace Libs::Graphics {
 
 class CommandScheduler;
-class Gpu;
+class GuestGpu;
 
 class GpuResourceManager {
 public:
@@ -23,7 +23,7 @@ public:
 
 	[[nodiscard]] BufferCache&  GetBufferCache() { return m_buffer_cache; }
 	[[nodiscard]] TextureCache& GetTextureCache() { return m_texture_cache; }
-	void                        SetGpu(Gpu* gpu) noexcept { m_gpu = gpu; }
+	void                        SetGpu(GuestGpu* gpu) noexcept { m_gpu = gpu; }
 
 	[[nodiscard]] bool HandleFault(PageFaultAccess access, uint64_t fault_vaddr) noexcept;
 	[[nodiscard]] bool InvalidateMemory(uint64_t vaddr, uint64_t size);
@@ -39,7 +39,7 @@ private:
 	TextureCache              m_texture_cache;
 	mutable std::shared_mutex m_mapped_ranges_mutex;
 	RangeSet                  m_mapped_ranges;
-	Gpu*                      m_gpu = nullptr;
+	GuestGpu*                 m_gpu = nullptr;
 };
 
 } // namespace Libs::Graphics

src/graphics/host_gpu/renderer/cache/textureCache.cpp

@@ -22,7 +22,7 @@
 #include <cstring>
 #include <limits>
 #include <mutex>
-#include <set>
+#include <span>
 #include <tuple>
 #include <vulkan/vulkan_format_traits.hpp>
 
@@ -274,25 +274,9 @@ void TextureCache::DeleteImage(ImageId id) {
 	}
 }
 
-void TextureCache::DeleteImages(std::span<const ImageId> ids,
-                                std::optional<ImageId>   native_source) {
-	std::set<std::pair<uint32_t, uint32_t>> unique;
-	for (const auto id: ids) {
-		if (!id || !unique.emplace(id.index, id.generation).second) {
-			continue;
-		}
-		auto owner = ResolveOwner(id);
-		if (owner == nullptr) {
-			continue;
-		}
-		if (native_source == id) {
-			ClearGpuModified(id);
-		} else if (owner->IsGpuModified()) {
-			DownloadImage(id);
-			ClearGpuModified(id);
-		}
-		DeleteImage(id);
-	}
+void TextureCache::FreeImage(ImageId id) {
+	ClearGpuModified(id);
+	DeleteImage(id);
 }
 
 void TextureCache::RetainImage(CommandBuffer& command, ImageId id) {
@@ -316,6 +300,14 @@ void TextureCache::TouchImage(Image& image) {
 	}
 }
 
+void TextureCache::MarkAsMaybeDirty(ImageId id, Image& image) {
+	image.MarkMaybeCpuDirty();
+	if (image.NeedsMaybeCpuHash()) {
+		image.SetMaybeCpuHash(image.HashGuestEdges());
+	}
+	UntrackImage(id);
+}
+
 void TextureCache::TrackImage(ImageId id) {
 	auto& image = ResolveImage(id);
 	if (!image.registered) {
@@ -399,11 +391,7 @@ void TextureCache::UntrackImageHead(ImageId id) {
 	const auto size    = address - begin;
 	image.track_addr   = address;
 	if (image.track_addr == image.track_addr_end) {
-		image.MarkMaybeCpuDirty();
-		if (image.NeedsMaybeCpuHash()) {
-			image.SetMaybeCpuHash(image.HashGuestEdges());
-		}
-		UntrackImage(id);
+		MarkAsMaybeDirty(id, image);
 	}
 	if (size != 0) {
 		m_page_manager.UpdatePageWatchers<false>(begin, size);
@@ -420,11 +408,7 @@ void TextureCache::UntrackImageTail(ImageId id) {
 	const auto size      = end - address;
 	image.track_addr_end = address;
 	if (image.track_addr == image.track_addr_end) {
-		image.MarkMaybeCpuDirty();
-		if (image.NeedsMaybeCpuHash()) {
-			image.SetMaybeCpuHash(image.HashGuestEdges());
-		}
-		UntrackImage(id);
+		MarkAsMaybeDirty(id, image);
 	}
 	if (size != 0) {
 		m_page_manager.UpdatePageWatchers<false>(address, size);
@@ -724,7 +708,6 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 	if (cached.binding.is_bound || cached.binding.is_target) {
 		cached.binding.needs_rebind = true;
 	}
-	bool copied = false;
 	if (cached.backing.samples == replacement.backing.samples) {
 		const bool copy_supported =
 		    cached.backing.samples == 1 || cached.backing.format == replacement.backing.format ||
@@ -732,7 +715,6 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 		     ImageViewOps::FormatsCompatible(cached.backing.format, replacement.backing.format));
 		if (copy_supported) {
 			CopyImage(replacement_id, cached_id);
-			copied = true;
 		} else {
 			LOGF_COLOR(Log::Color::BrightYellow,
 			           "TextureCache: unsupported cross-format multisample depth copy\n");
@@ -749,18 +731,12 @@ ImageId TextureCache::ResolveDepthOverlap(const ImageInfo& requested, BindingTyp
 		RetainImage(command, cached_id);
 		RetainImage(command, replacement_id);
 		CommitGpuWrite(replacement);
-		copied = true;
 	} else {
 		LOGF_COLOR(Log::Color::BrightYellow,
 		           "TextureCache: unsupported unequal-sample depth overlap copy (%u -> %u)\n",
 		           cached.backing.samples, replacement.backing.samples);
 	}
-	if (copied) {
-		DeleteImages(std::array {cached_id}, cached_id);
-	} else {
-		ClearGpuModified(cached_id);
-		DeleteImage(cached_id);
-	}
+	FreeImage(cached_id);
 	return replacement_id;
 }
 
@@ -782,7 +758,7 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		if (requested.BlockExtent() != cached.info.BlockExtent() ||
 		    requested_block != cached_block) {
 			if (safe_to_delete) {
-				DeleteImages(std::array {cached_id}, cached_id);
+				FreeImage(cached_id);
 			}
 			return {merged_id};
 		}
@@ -799,7 +775,7 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 		}
 		if (requested.tile_mode != cached.info.tile_mode) {
 			if (safe_to_delete) {
-				DeleteImages(std::array {cached_id}, cached_id);
+				FreeImage(cached_id);
 			}
 			return {merged_id};
 		}
@@ -835,7 +811,7 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 			}
 		}
 		if (safe_to_delete) {
-			DeleteImages(std::array {cached_id}, cached_id);
+			FreeImage(cached_id);
 		}
 		return {};
 	}
@@ -849,13 +825,13 @@ TextureCache::OverlapResult TextureCache::ResolveOverlap(const ImageInfo& reques
 				if (merged_id) {
 					ResolveImage(merged_id).binding.is_target = true;
 				}
-				DeleteImages(std::array {cached_id}, cached_id);
+				FreeImage(cached_id);
 				return {merged_id};
 			}
 			if (merged_id) {
 				CopyImageMip(merged_id, cached_id, static_cast<uint32_t>(mip),
 				             static_cast<uint32_t>(layer));
-				DeleteImages(std::array {cached_id}, cached_id);
+				FreeImage(cached_id);
 			}
 		}
 	}
@@ -874,7 +850,7 @@ ImageId TextureCache::ExpandImage(const ImageInfo& info, ImageId source_id) {
 	InitializeImage(expanded_id,
 	                ImageDesc {.info = info, .view_info = {}, .type = UploadBinding(source)});
 	CopyImage(expanded_id, source_id);
-	DeleteImages(std::array {source_id}, source_id);
+	FreeImage(source_id);
 	return expanded_id;
 }
 
@@ -924,7 +900,7 @@ TextureCache::BuildColorTransfer(const Image& image, BindingType binding,
 					EXIT("TextureCache: invalid color-attachment upload\n");
 				}
 				format           = ImageOps::RenderTargetTransferFormat(info.bytes_per_block);
-				allow_depth_tile = false;
+				allow_depth_tile = true;
 				plan.swap_bgra16 = info.bgra16;
 				owner            = "RenderTarget";
 				break;
@@ -950,7 +926,7 @@ TextureCache::BuildColorTransfer(const Image& image, BindingType binding,
 		format           = binding == BindingType::RenderTarget
 		                       ? ImageOps::RenderTargetTransferFormat(info.bytes_per_block)
 		                       : info.guest_format;
-		allow_depth_tile = binding == BindingType::Storage;
+		allow_depth_tile = binding == BindingType::Storage || binding == BindingType::RenderTarget;
 		plan.swap_bgra16 = info.bgra16;
 	}
 
@@ -1192,11 +1168,8 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		    FindImagesInRegion(desc.info.data.address, desc.info.data.size, false);
 
 		for (const auto id: candidates) {
-			const auto owner = ResolveOwner(id);
-			if (owner == nullptr) {
-				continue;
-			}
-			if (SameBacking(owner->info, desc.info, exact_format)) {
+			const auto& image = ResolveImage(id);
+			if (SameBacking(image.info, desc.info, exact_format)) {
 				result = id;
 			}
 		}
@@ -1205,12 +1178,8 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 		int32_t view_layer = -1;
 		if (!result) {
 			for (const auto candidate: candidates) {
-				view_mip         = -1;
-				view_layer       = -1;
-				const auto owner = ResolveOwner(candidate);
-				if (owner == nullptr) {
-					continue;
-				}
+				view_mip                = -1;
+				view_layer              = -1;
 				const auto& merged_info = result ? ResolveImage(result).info : desc.info;
 				const auto  overlap     = ResolveOverlap(merged_info, desc.type, candidate, result);
 				if (overlap.image) {
@@ -1226,7 +1195,8 @@ ImageId TextureCache::FindImage(ImageDesc& desc, bool exact_format) {
 			if (exact_format && resolved.info.pixel_format != desc.info.pixel_format) {
 				result = {};
 			} else if (resolved.info.resources < desc.info.resources) {
-				result = ExpandImage(desc.info, result);
+				FreeImage(result);
+				result = {};
 			}
 		}
 		if (!result) {
@@ -1788,17 +1758,17 @@ bool TextureCache::InvalidateMemoryFromGPU(uint64_t address, uint64_t size,
 	CacheLock lock(*this, m_lock);
 	bool      found = false;
 	for (const auto id: FindImagesInRegion(address, size, true)) {
-		auto owner = ResolveOwner(id);
-		if (owner == nullptr || owner->depth_id || !owner->Overlaps(address, size)) {
+		auto& image = ResolveImage(id);
+		if (image.depth_id || !image.Overlaps(address, size)) {
 			continue;
 		}
-		if (owner->IsGpuModified()) {
+		if (image.IsGpuModified()) {
 			if (!formatted_buffer_write) {
 				EXIT("TextureCache: buffer write aliases GPU-modified image\n");
 			}
-			ClearGpuModified(id);
+			image.ClearGpuModified();
 		}
-		owner->MarkBufferModified();
+		image.MarkBufferModified();
 		found = true;
 	}
 	return found;
@@ -1811,13 +1781,13 @@ TextureCache::RegionInfo TextureCache::QueryRegion(uint64_t address, uint64_t si
 	}
 	CacheLock lock(*this, m_lock);
 	for (const auto id: FindImagesInRegion(address, size, true)) {
-		auto owner = ResolveOwner(id);
-		if (owner == nullptr || owner->depth_id || !owner->Overlaps(address, size, true)) {
+		const auto& image = ResolveImage(id);
+		if (image.depth_id) {
 			continue;
 		}
 		result.image_pages = true;
-		result.image_bytes |= owner->Overlaps(address, size);
-		result.gpu_image_bytes |= owner->GpuOverlaps(address, size);
+		result.image_bytes |= image.Overlaps(address, size);
+		result.gpu_image_bytes |= image.GpuOverlaps(address, size);
 	}
 	return result;
 }
@@ -1842,11 +1812,7 @@ void TextureCache::InvalidateCpuAliases(uint64_t address, uint64_t size) {
 		} else if (image_begin < page_begin) {
 			UntrackImageTail(id);
 		} else {
-			owner->MarkMaybeCpuDirty();
-			if (owner->NeedsMaybeCpuHash()) {
-				owner->SetMaybeCpuHash(owner->HashGuestEdges());
-			}
-			UntrackImage(id);
+			MarkAsMaybeDirty(id, *owner);
 		}
 	}
 }

src/graphics/host_gpu/renderer/cache/textureCache.h

@@ -13,9 +13,7 @@
 #include <compare>
 #include <map>
 #include <memory>
-#include <optional>
 #include <set>
-#include <span>
 #include <utility>
 #include <vector>
 
@@ -110,17 +108,18 @@ private:
 	void                                 RegisterImage(ImageId id);
 	void                                 UnregisterImage(ImageId id);
 	void                                 DeleteImage(ImageId id);
-	void DeleteImages(std::span<const ImageId> ids, std::optional<ImageId> native_source = {});
-	void RetainImage(CommandBuffer& command, ImageId id);
-	void TouchImage(Image& image);
-	void TrackImage(ImageId id);
-	void TrackImageHead(ImageId id);
-	void TrackImageTail(ImageId id);
-	void UntrackImage(ImageId id);
-	void UntrackImageHead(ImageId id);
-	void UntrackImageTail(ImageId id);
-	void TrackImageDownload(ImageId id);
-	void TrackImageDownloadLocked(ImageId id, Image& image);
+	void                                 FreeImage(ImageId id);
+	void                                 RetainImage(CommandBuffer& command, ImageId id);
+	void                                 TouchImage(Image& image);
+	void                                 TrackImage(ImageId id);
+	void                                 TrackImageHead(ImageId id);
+	void                                 TrackImageTail(ImageId id);
+	void                                 UntrackImage(ImageId id);
+	void                                 UntrackImageHead(ImageId id);
+	void                                 UntrackImageTail(ImageId id);
+	void                                 MarkAsMaybeDirty(ImageId id, Image& image);
+	void                                 TrackImageDownload(ImageId id);
+	void                                 TrackImageDownloadLocked(ImageId id, Image& image);
 	[[nodiscard]] static bool SameBacking(const ImageInfo& cached, const ImageInfo& requested,
 	                                      bool exact_format);
 	[[nodiscard]] static BindingType UploadBinding(const Image& image);

src/graphics/host_gpu/renderer/colorRenderTarget.cpp

@@ -130,12 +130,17 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (volume && samples != 1) {
 		EXIT("multisampled 3D render targets are unsupported\n");
 	}
-	const uint32_t depth      = volume ? rt.attrib3.depth + 1u : 1u;
-	const bool     standard64 = rt.attrib3.tile_mode == Prospero::TileMode::kStandard64KB;
+	const uint32_t depth        = volume ? rt.attrib3.depth + 1u : 1u;
+	const bool     standard4    = rt.attrib3.tile_mode == Prospero::TileMode::kStandard4KB;
+	const bool     standard64   = rt.attrib3.tile_mode == Prospero::TileMode::kStandard64KB;
+	const bool     depth_tile   = rt.attrib3.tile_mode == Prospero::TileMode::kDepth;
+	const bool     texture_tile = standard4 || standard64 || depth_tile;
 
 	switch (rt.attrib3.tile_mode) {
 		case Prospero::TileMode::kLinear:
+		case Prospero::TileMode::kStandard4KB:
 		case Prospero::TileMode::kStandard64KB:
+		case Prospero::TileMode::kDepth:
 		case Prospero::TileMode::kRenderTarget:
 			tile = !RenderIsColorTileModeLinear(rt.attrib3.tile_mode);
 			break;
@@ -147,6 +152,9 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	if (samples > 1 && (!tile || levels != 1)) {
 		EXIT("multisampled render targets require a single-mip tiled surface\n");
 	}
+	if (texture_tile && samples != 1) {
+		EXIT("texture-tiled color render targets do not support multisampling\n");
+	}
 
 	width  = rt.attrib2.width + 1;
 	height = rt.attrib2.height + 1;
@@ -157,31 +165,38 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 		EXIT("render-target format has no valid element size\n");
 	}
 	const auto transfer_format = ImageOps::RenderTargetTransferFormat(bytes_per_element);
-	if (standard64 &&
-	    (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 || levels != 1 ||
-	     rt.view.current_mip_level != 0 || view.base_layer != 0 || view.image_layers != 1 ||
-	     samples != 1 || bytes_per_element != 4 || rt.pitch.pitch_div8_minus1 != 0 ||
-	     (rt.base.addr & 0xffffu) != 0 || rt.info.fmask_compression_enable ||
-	     rt.info.fmask_data_compression_disable || rt.info.fmask_one_frag_mode ||
-	     rt.info.cmask_fast_clear_enable || rt.info.dcc_compression_enable ||
-	     rt.info.cmask_is_linear != 0 || rt.info.cmask_addr_type != 0 || rt.info.alt_tile_mode ||
-	     rt.cmask.addr != 0 || rt.fmask.addr != 0 || rt.dcc_addr.addr != 0 ||
-	     rt.dcc.data_write_on_dcc_clear_to_reg)) {
-		EXIT("unsupported Standard64KB render target: addr=0x%016" PRIx64
+	TileTextureBlockLayout texture_tile_layout {};
+	if (texture_tile &&
+	    (!TileGetTextureBlockLayout(transfer_format, rt.attrib3.tile_mode, volume,
+	                                texture_tile_layout) ||
+	     rt.pitch.pitch_div8_minus1 != 0 ||
+	     (rt.base.addr & (texture_tile_layout.block.block_size - 1u)) != 0 ||
+	     rt.info.fmask_compression_enable || rt.info.fmask_data_compression_disable ||
+	     rt.info.fmask_one_frag_mode || rt.info.cmask_fast_clear_enable ||
+	     rt.info.dcc_compression_enable || rt.info.cmask_is_linear != 0 ||
+	     rt.info.cmask_addr_type != 0 || rt.info.alt_tile_mode || rt.cmask.addr != 0 ||
+	     rt.fmask.addr != 0 || rt.dcc_addr.addr != 0 || rt.dcc.data_write_on_dcc_clear_to_reg)) {
+		EXIT("unsupported texture-tiled render target: addr=0x%016" PRIx64 " tile=%u"
 		     " dimension=%u depth=%u levels=%u layer=%u/%u samples=%u fragments=%u bpe=%u"
 		     " cmask=0x%016" PRIx64 " fmask=0x%016" PRIx64 " dcc=0x%016" PRIx64 "\n",
-		     rt.base.addr, rt.attrib3.dimension, rt.attrib3.depth, levels, view.base_layer,
-		     view.image_layers, rt.attrib.num_samples, rt.attrib.num_fragments, bytes_per_element,
-		     rt.cmask.addr, rt.fmask.addr, rt.dcc_addr.addr);
+		     rt.base.addr, static_cast<uint32_t>(rt.attrib3.tile_mode), rt.attrib3.dimension,
+		     rt.attrib3.depth, levels, view.base_layer, view.image_layers, rt.attrib.num_samples,
+		     rt.attrib.num_fragments, bytes_per_element, rt.cmask.addr, rt.fmask.addr,
+		     rt.dcc_addr.addr);
+	}
+	if ((standard64 || depth_tile) && (rt.attrib3.dimension != 1 || rt.attrib3.depth != 0 ||
+	                                   view.base_layer != 0 || view.image_layers != 1)) {
+		EXIT("unsupported 64KB texture-tiled render-target view: dimension=%u depth=%u"
+		     " layer=%u/%u\n",
+		     rt.attrib3.dimension, rt.attrib3.depth, view.base_layer, view.image_layers);
 	}
 	if (rt.pitch.pitch_div8_minus1 != 0) {
 		pitch = (rt.pitch.pitch_div8_minus1 + 1u) << 3u;
 	} else if (tile) {
 		if (volume) {
 			pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
-		} else if (standard64) {
-			pitch =
-			    TileGetTexturePitch(Prospero::BufferFormat::k32Float, width, rt.attrib3.tile_mode);
+		} else if (texture_tile) {
+			pitch = TileGetTexturePitch(transfer_format, width, rt.attrib3.tile_mode);
 		} else {
 			pitch = TileGetRenderTargetPitch(width, bytes_per_element, rt.attrib.num_fragments);
 		}
@@ -214,10 +229,10 @@ void RenderExecutor::ResolveRenderColorTarget(uint64_t submit_id, RenderCommandB
 	} else if (tile) {
 		TileSizeAlign layout {};
 		bool          valid_layout = false;
-		if (standard64) {
-			TileGetTextureSize(Prospero::BufferFormat::k32Float, width, height, levels,
-			                   rt.attrib3.tile_mode, &layout, mip_sizes, mip_padded);
-			valid_layout = layout.size != 0 && layout.align == 65536;
+		if (texture_tile) {
+			TileGetTextureSize(transfer_format, width, height, levels, rt.attrib3.tile_mode,
+			                   &layout, mip_sizes, mip_padded);
+			valid_layout = layout.size != 0 && layout.align == texture_tile_layout.block.block_size;
 		} else {
 			valid_layout =
 			    levels == 1 ? TileGetRenderTargetSize(width, height, pitch, bytes_per_element,

src/graphics/host_gpu/renderer/context.cpp

@@ -17,10 +17,29 @@
 
 #include <algorithm>
 #include <bit>
+#include <cstdio>
 #include <cstring>
 #include <memory>
 namespace Libs::Graphics {
 
+namespace {
+
+void ReportVulkanFatal(const char* what, vk::Result result, uint32_t slot, uint64_t submit_seq,
+                       uint32_t debug_op, uint64_t debug_submit, uint32_t arg0, uint32_t arg1,
+                       uint32_t arg2, uint32_t arg3, uint64_t arg4) {
+	LOGF("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	     " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	     what, VulkanToString(result).c_str(), static_cast<int>(result), slot, submit_seq, debug_op,
+	     debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::printf("%s failed: %s (%d), slot=%u submit_seq=%" PRIu64 " debug_op=%u debug_submit=%" PRIu64
+	            " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
+	            what, VulkanToString(result).c_str(), static_cast<int>(result), slot, submit_seq,
+	            debug_op, debug_submit, arg0, arg1, arg2, arg3, arg4);
+	std::fflush(stdout);
+}
+
+} // namespace
+
 FenceResourceRetainer::~FenceResourceRetainer() {
 	if (!m_resources.empty()) {
 		EXIT("fence resource retainer destroyed before release\n");
@@ -169,8 +188,9 @@ void CommandBuffer::Execute(const SubmitInfo& submit) {
 
 	auto result = graphics.device.resetFences(1, &fence);
 	if (result != vk::Result::eSuccess) {
-		LOGF("vkResetFences failed before submit: %s (%d)\n", VulkanToString(result).c_str(),
-		     static_cast<int>(result));
+		ReportVulkanFatal("vkResetFences (before submit)", result, m_slot->id, m_submit_seq,
+		                  m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2,
+		                  m_debug_arg3, m_debug_arg4);
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 
@@ -192,11 +212,9 @@ void CommandBuffer::Execute(const SubmitInfo& submit) {
 	m_fence_waited = false;
 
 	if (result != vk::Result::eSuccess) {
-		LOGF("vkQueueSubmit failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
+		ReportVulkanFatal("vkQueueSubmit", result, m_slot->id, m_submit_seq, m_debug_op,
+		                  m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
+		                  m_debug_arg4);
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 }
@@ -213,11 +231,9 @@ void CommandBuffer::WaitForFenceOnly() {
 	auto device = m_graphics.device;
 	auto result = device.waitForFences(1, &m_slot->fence, VK_TRUE, UINT64_MAX);
 	if (result != vk::Result::eSuccess) {
-		LOGF("vkWaitForFences failed: %s (%d), slot=%u submit_seq=%" PRIu64
-		     " debug_op=%u debug_submit=%" PRIu64 " args=%u,%u,%u,%u,0x%016" PRIx64 "\n",
-		     VulkanToString(result).c_str(), static_cast<int>(result), m_slot->id, m_submit_seq,
-		     m_debug_op, m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
-		     m_debug_arg4);
+		ReportVulkanFatal("vkWaitForFences", result, m_slot->id, m_submit_seq, m_debug_op,
+		                  m_debug_submit_id, m_debug_arg0, m_debug_arg1, m_debug_arg2, m_debug_arg3,
+		                  m_debug_arg4);
 	}
 	EXIT_NOT_IMPLEMENTED(result != vk::Result::eSuccess);
 	m_fence_waited = true;

src/graphics/host_gpu/renderer/debug.cpp

@@ -71,14 +71,16 @@ void uc_print(const char* func, const HW::UserConfig& uc) {
 	LOGF("\t GetPrimType()         = 0x%08" PRIx32 "\n"
 	     "\t GetIndexOffset()      = 0x%08" PRIx32 "\n"
 	     "\t GetObjectId()         = 0x%08" PRIx32 "\n"
+	     "\t primitive_reset       = 0x%08" PRIx32 "\n"
 	     "\t primitive_group_size  = 0x%04" PRIx16 "\n"
 	     "\t vertex_group_size     = 0x%04" PRIx16 "\n"
 	     "\t en_user_vgpr1         = %s\n"
 	     "\t en_user_vgpr2         = %s\n"
 	     "\t en_user_vgpr3         = %s\n",
 	     static_cast<uint32_t>(uc.GetPrimType()), uc.GetIndexOffset(), uc.GetObjectId(),
-	     ge_cntl.primitive_group_size, ge_cntl.vertex_group_size, user_en.vgpr1 ? "true" : "false",
-	     user_en.vgpr2 ? "true" : "false", user_en.vgpr3 ? "true" : "false");
+	     uc.GetPrimitiveResetControl(), ge_cntl.primitive_group_size, ge_cntl.vertex_group_size,
+	     user_en.vgpr1 ? "true" : "false", user_en.vgpr2 ? "true" : "false",
+	     user_en.vgpr3 ? "true" : "false");
 }
 
 void uc_check(const HW::UserConfig& uc) {
@@ -615,8 +617,7 @@ static void ClipCheck(const HW::ClipControl& c) {
 	// dx_linear_attr_clip_enable preserves linear (noperspective) attributes at clip-generated
 	// vertices, which Vulkan provides as part of clipping and interpolation.
 	EXIT_NOT_IMPLEMENTED(c.user_clip_planes != 0 || c.user_clip_plane_mode != 0 ||
-	                     c.vertex_kill_any || !c.IsZClipModeRepresentable() ||
-	                     c.user_clip_plane_negate_y || c.clip_disable ||
+	                     c.vertex_kill_any || c.user_clip_plane_negate_y || c.clip_disable ||
 	                     c.user_clip_plane_cull_only || c.cull_on_clipping_error_disable ||
 	                     c.force_viewport_index_from_vs_enable);
 }
@@ -640,17 +641,6 @@ static void RcPrint(const char* func, const HW::RenderControl& c) {
 static void RcCheck(const HW::RenderControl& c) {
 	// EXIT_NOT_IMPLEMENTED(c.depth_clear_enable != false);
 	// EXIT_NOT_IMPLEMENTED(c.stencil_clear_enable != false);
-	if ((c.resummarize_enable || c.copy_centroid || c.copy_sample != 0)) {
-		static std::atomic<uint32_t> log_count {0};
-		if (log_count.fetch_add(1) < 16) {
-			LOGF("\t warning: unsupported PS5 render-control metadata state, continuing: "
-			     "resummarize=%s copy_centroid=%s copy_sample=%" PRIu8 "\n",
-			     c.resummarize_enable ? "true" : "false", c.copy_centroid ? "true" : "false",
-			     c.copy_sample);
-		}
-		return;
-	}
-	EXIT_NOT_IMPLEMENTED(c.resummarize_enable != false);
 	// EXIT_NOT_IMPLEMENTED(c.stencil_compress_disable != false);
 	// EXIT_NOT_IMPLEMENTED(c.depth_compress_disable != false);
 	EXIT_NOT_IMPLEMENTED(c.copy_centroid != false);
@@ -1265,9 +1255,10 @@ void hw_print(const RenderCommandBuffer& buffer) {
 		     "\t GetRenderTargetMask()   = 0x%08" PRIx32 "\n"
 		     "\t GetDepthClearValue()    = %f\n"
 		     "\t GetStencilClearValue()  = %" PRIu8 "\n"
-		     "\t GetLineWidth()          = %f\n",
+		     "\t GetLineWidth()          = %f\n"
+		     "\t primitive_reset_index   = 0x%08" PRIx32 "\n",
 		     hw.GetRenderTargetMask(), hw.GetDepthClearValue(), hw.GetStencilClearValue(),
-		     hw.GetLineWidth());
+		     hw.GetLineWidth(), hw.GetPrimitiveResetIndex());
 
 		LOGF("%s", Common::Concat(rt_print("RenderTraget:", rt), "").c_str());
 

src/graphics/host_gpu/renderer/depthRenderTarget.cpp

@@ -166,19 +166,18 @@ void RenderExecutor::ResolveRenderDepthTarget(uint64_t submit_id, RenderCommandB
 			DepthFatal("invalid depth view: base=%u last=%u", z.depth_view.slice_start,
 			           z.depth_view.slice_max);
 	}
-	if (rc.resummarize_enable || rc.copy_centroid || rc.copy_sample != 0 ||
-	    z.z_info.expclear_enabled || z.stencil_info.expclear_enabled ||
-	    z.z_info.partially_resident || z.stencil_info.partially_resident ||
-	    z.z_info.max_mip_level != 0 || z.depth_view.current_mip_level != 0 ||
-	    z.depth_info.addr5_swizzle_mask != 0 || z.depth_info.array_mode != 0 ||
-	    z.depth_info.pipe_config != 0 || z.depth_info.bank_width != 0 ||
-	    z.depth_info.bank_height != 0 || z.depth_info.macro_tile_aspect != 0 ||
-	    z.depth_info.num_banks != 0 || z.htile_surface.linear != 0 ||
-	    z.htile_surface.full_cache != 0 || z.htile_surface.htile_uses_preload_win != 0 ||
-	    z.htile_surface.preload != 0 || z.htile_surface.prefetch_width != 0 ||
-	    z.htile_surface.prefetch_height != 0 || z.htile_surface.dst_outside_zero_to_one != 0 ||
-	    z.z_read_base_addr == 0 || z.z_write_base_addr != z.z_read_base_addr ||
-	    (z.z_read_base_addr & 0xffffu) != 0 ||
+	if (rc.copy_centroid || rc.copy_sample != 0 || z.z_info.expclear_enabled ||
+	    z.stencil_info.expclear_enabled || z.z_info.partially_resident ||
+	    z.stencil_info.partially_resident || z.z_info.max_mip_level != 0 ||
+	    z.depth_view.current_mip_level != 0 || z.depth_info.addr5_swizzle_mask != 0 ||
+	    z.depth_info.array_mode != 0 || z.depth_info.pipe_config != 0 ||
+	    z.depth_info.bank_width != 0 || z.depth_info.bank_height != 0 ||
+	    z.depth_info.macro_tile_aspect != 0 || z.depth_info.num_banks != 0 ||
+	    z.htile_surface.linear != 0 || z.htile_surface.full_cache != 0 ||
+	    z.htile_surface.htile_uses_preload_win != 0 || z.htile_surface.preload != 0 ||
+	    z.htile_surface.prefetch_width != 0 || z.htile_surface.prefetch_height != 0 ||
+	    z.htile_surface.dst_outside_zero_to_one != 0 || z.z_read_base_addr == 0 ||
+	    z.z_write_base_addr != z.z_read_base_addr || (z.z_read_base_addr & 0xffffu) != 0 ||
 	    dc.zfunc > static_cast<uint8_t>(vk::CompareOp::eAlways)) {
 		DepthFatal("unsupported depth register state");
 	}

src/graphics/host_gpu/renderer/image/image.h

@@ -55,6 +55,7 @@ struct ImageBinding {
 	bool is_target     = false;
 	bool needs_rebind  = false;
 	bool force_general = false;
+	bool shader_write  = false;
 };
 
 class Image final {

src/graphics/host_gpu/renderer/image/imageInfo.h

@@ -212,7 +212,6 @@ struct DepthFormatPolicy {
 	Prospero::DepthFormat     depth_format;
 	Prospero::BufferFormat    guest_format;
 	uint32_t                  bytes_per_element;
-	vk::Format                sampled_view_format;
 	vk::Format                depth_attachment_format;
 	std::array<vk::Format, 3> stencil_attachment_formats;
 };
@@ -221,13 +220,11 @@ inline constexpr std::array<DepthFormatPolicy, 2> DEPTH_FORMAT_POLICIES {{
     {Prospero::DepthFormat::kZ16,
      Prospero::BufferFormat::k16UNorm,
      2,
-     vk::Format::eR16Unorm,
      vk::Format::eD16Unorm,
      {vk::Format::eD16UnormS8Uint, vk::Format::eD24UnormS8Uint, vk::Format::eD32SfloatS8Uint}},
     {Prospero::DepthFormat::kZ32F,
      Prospero::BufferFormat::k32Float,
      4,
-     vk::Format::eR32Sfloat,
      vk::Format::eD32Sfloat,
      {vk::Format::eD32SfloatS8Uint, vk::Format::eUndefined, vk::Format::eUndefined}},
 }};
@@ -317,25 +314,6 @@ inline bool ImageInfo::IsDepth() const noexcept {
 	return std::bit_cast<uint32_t>(static_cast<float>(value) / 65535.0f);
 }
 
-[[nodiscard]] inline constexpr bool
-IsSupportedSampledDepthFormat(vk::Format image_format, Prospero::BufferFormat guest_format,
-                              vk::Format view_format) noexcept {
-	const auto* policy = FindGuestDepthFormatPolicy(guest_format);
-	return policy != nullptr && view_format == policy->sampled_view_format &&
-	       (image_format == policy->depth_attachment_format ||
-	        IsStencilAttachmentFormat(*policy, image_format));
-}
-
-[[nodiscard]] inline constexpr bool IsSupportedSampledDepthFormat(vk::Format image_format,
-                                                                  vk::Format view_format) noexcept {
-	for (const auto& policy: DEPTH_FORMAT_POLICIES) {
-		if (IsSupportedSampledDepthFormat(image_format, policy.guest_format, view_format)) {
-			return true;
-		}
-	}
-	return false;
-}
-
 [[nodiscard]] inline constexpr bool IsSupportedDepthTargetFormat(const ImageInfo& info) {
 	const auto* policy = FindGuestDepthFormatPolicy(info.guest_format);
 	return policy != nullptr && info.bytes_per_block == policy->bytes_per_element &&

src/graphics/host_gpu/renderer/image/imageView.cpp

@@ -36,17 +36,6 @@ namespace {
 	}
 }
 
-[[nodiscard]] bool IsDepthViewFormat(vk::Format format) {
-	switch (format) {
-		case vk::Format::eD16Unorm:
-		case vk::Format::eR16Unorm:
-		case vk::Format::eD32Sfloat:
-		case vk::Format::eR32Sfloat:
-		case vk::Format::eR32Uint: return true;
-		default: return false;
-	}
-}
-
 [[nodiscard]] bool IsValidViewType(const VulkanImage& image, const ImageViewInfo& info) {
 	switch (image.image_type) {
 		case vk::ImageType::e1D:
@@ -327,13 +316,13 @@ vk::ImageView Image::FindView(const ImageViewInfo& view_info) {
 	const auto& image      = backing;
 	auto        normalized = view_info;
 	const bool  is_storage = static_cast<bool>(normalized.usage & vk::ImageUsageFlagBits::eStorage);
-	normalized.aspect      = FullAspectMask(image.format);
-	if (normalized.aspect & vk::ImageAspectFlagBits::eDepth &&
-	    IsDepthViewFormat(normalized.format)) {
+	const auto  image_aspect = FullAspectMask(image.format);
+	if (image_aspect & vk::ImageAspectFlagBits::eDepth &&
+	    ImageViewOps::IsFormatDepthCompatible(normalized.format)) {
 		normalized.format = image.format;
 		normalized.aspect = vk::ImageAspectFlagBits::eDepth;
 	}
-	if (normalized.aspect & vk::ImageAspectFlagBits::eStencil &&
+	if (image_aspect & vk::ImageAspectFlagBits::eStencil &&
 	    IsStencilViewFormat(normalized.format)) {
 		normalized.format = image.format;
 		normalized.aspect = vk::ImageAspectFlagBits::eStencil;

src/graphics/host_gpu/renderer/image/imageView.h

@@ -12,8 +12,25 @@ namespace ImageViewOps {
 
 [[nodiscard]] vk::ImageAspectFlags DepthAspectMask(vk::Format format);
 [[nodiscard]] bool                 FormatsCompatible(vk::Format base, vk::Format view) noexcept;
+
+[[nodiscard]] inline bool IsFormatDepthCompatible(vk::Format format) noexcept {
+	switch (format) {
+		case vk::Format::eD32Sfloat:
+		case vk::Format::eR32Sfloat:
+		case vk::Format::eR32Uint:
+		case vk::Format::eD16Unorm:
+		case vk::Format::eR16Unorm: return true;
+		default: return false;
+	}
+}
 } // namespace ImageViewOps
 
+[[nodiscard]] inline bool IsSupportedSampledDepthFormat(vk::Format image_format,
+                                                        vk::Format view_format) noexcept {
+	return DepthAspectTransferFormat(image_format) != vk::Format::eUndefined &&
+	       ImageViewOps::IsFormatDepthCompatible(view_format);
+}
+
 [[nodiscard]] inline bool IsValidImageSwizzle(uint32_t swizzle) noexcept {
 	if ((swizzle & ~0xfffu) != 0) {
 		return false;
@@ -86,8 +103,14 @@ SelectSampledDepthView(vk::Format image_format, vk::Format view_format, uint32_t
 
 [[nodiscard]] inline bool
 IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
-	return resource.kind == ShaderRecompiler::IR::ResourceKind::Image &&
-	       (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
+	if (resource.kind != ShaderRecompiler::IR::ResourceKind::Image &&
+	    resource.kind != ShaderRecompiler::IR::ResourceKind::ImageUint) {
+		return false;
+	}
+	if (resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint && resource.depth_compare) {
+		return false;
+	}
+	return (resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaa ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DMsaaArray) &&
@@ -95,14 +118,6 @@ IsSupportedSampledDepthResource(const ShaderRecompiler::IR::ImageResource& resou
 	       !resource.written && !resource.atomic;
 }
 
-[[nodiscard]] inline bool
-IsSupportedSampledDepthUintResource(const ShaderRecompiler::IR::ImageResource& resource) noexcept {
-	return resource.kind == ShaderRecompiler::IR::ResourceKind::ImageUint &&
-	       resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D &&
-	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.read &&
-	       !resource.written && !resource.atomic && !resource.depth_compare;
-}
-
 inline void ValidateStorageColorView(vk::Format image_format, vk::Format view_format,
                                      uint32_t swizzle) noexcept {
 	if (!ImageViewOps::FormatsCompatible(image_format, view_format) ||
@@ -120,7 +135,11 @@ IsSupportedStorageImageResource(const ShaderRecompiler::IR::ImageResource& resou
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim3D ||
 	        resource.dimension == ShaderRecompiler::Decoder::ImageDimension::Dim2DArray) &&
-	       resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None && resource.written &&
+	       ((resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::None &&
+	         resource.mip_count == 1u) ||
+	        (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage &&
+	         resource.mip_count != 0u)) &&
+	       resource.written &&
 	       (!resource.atomic ||
 	        (resource.kind == ShaderRecompiler::IR::ResourceKind::StorageImageUint &&
 	         resource.read)) &&

src/graphics/host_gpu/renderer/pipeline/descriptorCache.cpp

@@ -94,10 +94,14 @@ vk::DescriptorBufferInfo BufferInfo(const BufferView& view) {
 
 } // namespace
 
-vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture) {
-	EXIT_IF(!texture.image_id || texture.image_view == nullptr ||
-	        texture.layout == vk::ImageLayout::eUndefined);
-	return {nullptr, texture.image_view, texture.layout};
+vk::DescriptorImageInfo DescriptorCache::MakeImageInfo(const TextureBinding& texture,
+                                                       uint32_t              element) {
+	const auto view =
+	    texture.mip_views.empty()
+	        ? (element == 0u ? texture.image_view : vk::ImageView {})
+	        : (element < texture.mip_views.size() ? texture.mip_views[element] : vk::ImageView {});
+	EXIT_IF(!texture.image_id || view == nullptr || texture.layout == vk::ImageLayout::eUndefined);
+	return {nullptr, view, texture.layout};
 }
 
 DescriptorCache::~DescriptorCache() {
@@ -229,14 +233,17 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage
 	auto* set = Allocate(stage, program);
 	EXIT_NOT_IMPLEMENTED(set == nullptr);
 
-	const auto descriptor_count = program.info.buffers.size() + program.info.images.size() +
-	                              program.info.samplers.size() + program.info.addresses.size() + 3u;
+	size_t descriptor_count = 0;
+	for (const auto& binding: program.bindings.descriptors) {
+		descriptor_count += DescriptorCount(binding);
+	}
 	std::vector<vk::DescriptorBufferInfo> buffer_infos;
 	std::vector<vk::DescriptorImageInfo>  image_infos;
 	std::vector<vk::WriteDescriptorSet>   writes;
 	buffer_infos.reserve(descriptor_count);
 	image_infos.reserve(descriptor_count);
 	writes.reserve(program.bindings.descriptors.size());
+	std::vector<uint32_t> image_occurrences(data.images.size());
 
 	for (const auto& binding: program.bindings.descriptors) {
 		vk::WriteDescriptorSet write {};
@@ -273,7 +280,7 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage
 			default: {
 				for (const auto resource: binding.resources) {
 					const auto& texture = data.images.at(resource);
-					image_infos.push_back(MakeImageInfo(texture));
+					image_infos.push_back(MakeImageInfo(texture, image_occurrences.at(resource)++));
 				}
 				break;
 			}
@@ -286,6 +293,12 @@ VulkanDescriptorSet& DescriptorCache::GetDescriptor(Stage
 		}
 		writes.push_back(write);
 	}
+	for (uint32_t i = 0; i < data.images.size(); i++) {
+		const auto expected = data.images[i].mip_views.empty()
+		                          ? 1u
+		                          : static_cast<uint32_t>(data.images[i].mip_views.size());
+		EXIT_IF(image_occurrences[i] != expected);
+	}
 	m_graphics.device.updateDescriptorSets(static_cast<uint32_t>(writes.size()), writes.data(), 0,
 	                                       nullptr);
 	return *set;

src/graphics/host_gpu/renderer/pipeline/descriptorCache.h

@@ -47,10 +47,11 @@ public:
 	enum class Stage { Unknown, Vertex, Pixel, Compute };
 
 	struct TextureBinding {
-		ImageId                 image_id;
-		vk::ImageView           image_view = nullptr;
-		TextureCache::ImageDesc desc;
-		vk::ImageLayout         layout = vk::ImageLayout::eUndefined;
+		ImageId                    image_id;
+		vk::ImageView              image_view = nullptr;
+		TextureCache::ImageDesc    desc;
+		vk::ImageLayout            layout = vk::ImageLayout::eUndefined;
+		std::vector<vk::ImageView> mip_views;
 	};
 
 	struct NativeDescriptors {
@@ -94,7 +95,8 @@ private:
 		int                next_free_pool = -1;
 	};
 
-	static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture);
+	static vk::DescriptorImageInfo MakeImageInfo(const TextureBinding& texture,
+	                                             uint32_t              element = 0);
 	void                           CreatePool();
 	VulkanDescriptorSet* Allocate(Stage stage, const ShaderRecompiler::IR::Program& program);
 	vk::DescriptorSetLayout

src/graphics/host_gpu/renderer/pipeline/descriptors.cpp

@@ -23,10 +23,11 @@
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vma.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
+#include "kernel/memory.h"
 
 #include <algorithm>
 #include <atomic>
@@ -91,11 +92,12 @@ static BufferView NativeStorageBuffer(RenderContext& context, CommandBuffer& com
 	if (stride != 0 && records > UINT64_MAX / stride) {
 		EXIT("storage buffer descriptor footprint overflow\n");
 	}
-	const auto size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
-	if (address == 0 || size == 0) {
+	const auto requested_size = stride != 0 ? static_cast<uint64_t>(stride) * records : records;
+	if (address == 0 || requested_size == 0) {
 		BindNullStorageBuffer(context, result);
 		return result;
 	}
+	const auto  size      = Libs::LibKernel::Memory::ClampRangeSize(address, requested_size);
 	const auto& graphics  = context.GetGraphics();
 	const auto  alignment = graphics.StorageMinAlignment();
 	if (alignment == 0 ||
@@ -314,8 +316,7 @@ static void ValidateDepthTargetBinding(const ShaderRecompiler::IR::ImageResource
 	const bool encoding_ok =
 	    image != nullptr && IsSupportedDepthTextureEncoding(descriptor, *image);
 	const bool format_ok =
-	    image != nullptr &&
-	    IsSupportedSampledDepthFormat(image->info.pixel_format, descriptor.Format(), view_format);
+	    image != nullptr && IsSupportedSampledDepthFormat(image->info.pixel_format, view_format);
 	if (resource_ok && descriptor_ok && encoding_ok && format_ok && size != 0) {
 		return;
 	}
@@ -403,10 +404,14 @@ static bool IsSupportedStorageTextureDescriptor(const ShaderRecompiler::IR::Imag
 	const bool supported_swizzle =
 	    IsValidImageSwizzle(swizzle) &&
 	    (swizzle == DstSel(4, 5, 6, 7) || !resource.read || resource.atomic);
+	const auto view_last_level =
+	    resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage
+	        ? descriptor.LastLevel()
+	        : std::min(descriptor.LastLevel(), descriptor.MaxMip());
 	return (is_1d || is_1d_array || is_2d || is_2d_array || is_3d) && supported_tile &&
-	       descriptor.BaseLevel() == descriptor.LastLevel() &&
-	       descriptor.LastLevel() <= descriptor.MaxMip() && descriptor.MinLod() == 0 &&
-	       supported_swizzle && descriptor.BCSwizzle() == 0 && !descriptor.MsaaDepth();
+	       descriptor.BaseLevel() <= view_last_level && view_last_level <= descriptor.MaxMip() &&
+	       descriptor.MinLod() == 0 && supported_swizzle && descriptor.BCSwizzle() == 0 &&
+	       !descriptor.MsaaDepth();
 }
 
 static bool IsSupportedStorageTextureEncoding(const ShaderTextureResource& descriptor) {
@@ -627,11 +632,15 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	const auto type         = TextureType(descriptor);
 	const bool multisampled = IsMultisampledTexture(type);
 	const auto levels       = multisampled ? 1u : static_cast<uint32_t>(descriptor.MaxMip()) + 1u;
-	const auto tile         = descriptor.TileMode();
-	const bool depth_tile   = tile == Prospero::TileMode::kDepth;
-	const bool msaa_tile    = depth_tile || tile == Prospero::TileMode::kRenderTarget;
-	const bool msaa_array   = type == Prospero::ImageType::kColor2DMsaaArray;
-	if ((!multisampled && (base_level > last_level || last_level >= levels)) ||
+	const bool dynamic_storage =
+	    storage && resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage;
+	const auto view_last_level =
+	    !multisampled && !dynamic_storage ? std::min(last_level, descriptor.MaxMip()) : last_level;
+	const auto tile       = descriptor.TileMode();
+	const bool depth_tile = tile == Prospero::TileMode::kDepth;
+	const bool msaa_tile  = depth_tile || tile == Prospero::TileMode::kRenderTarget;
+	const bool msaa_array = type == Prospero::ImageType::kColor2DMsaaArray;
+	if ((!multisampled && (base_level > view_last_level || view_last_level >= levels)) ||
 	    (multisampled &&
 	     (base_level != 0 || last_level == 0 || last_level > 3 ||
 	      descriptor.MaxMip() != last_level || !msaa_tile ||
@@ -650,7 +659,7 @@ RenderExecutor::ResolveTexture(const ShaderRecompiler::IR::ImageResource&   reso
 	}
 	const auto samples = multisampled ? 1u << last_level : 1u;
 	const auto view_levels =
-	    multisampled ? 1u : static_cast<uint32_t>(last_level - base_level) + 1u;
+	    multisampled ? 1u : static_cast<uint32_t>(view_last_level - base_level) + 1u;
 	const auto depth  = static_cast<uint32_t>(descriptor.Depth()) + 1u;
 	const auto format = descriptor.Format();
 	const bool sampled_numeric_class =
@@ -784,9 +793,10 @@ void RenderExecutor::BindImage(ImageId id, bool storage) {
 		return;
 	}
 	if (image.binding.is_bound) {
-		image.binding.force_general |= storage;
+		image.binding.force_general |= image.binding.shader_write != storage;
 	}
 	image.binding.is_bound = true;
+	image.binding.shader_write |= storage;
 	TrackImageBinding(id);
 }
 
@@ -902,8 +912,27 @@ void RenderExecutor::RebindImages(CommandBuffer&                     buffer,
 			BindImage(images[i].image_id,
 			          images[i].desc.type == TextureCache::BindingType::Storage);
 		}
-		auto& binding      = images[i];
-		binding.image_view = texture_cache.FindTexture(binding.image_id, binding.desc);
+		auto& binding = images[i];
+		binding.mip_views.clear();
+		const auto& resource = program.info.images[i];
+		if (resource.mip_mode == ShaderRecompiler::IR::ImageMipMode::DynamicStorage) {
+			EXIT_IF(resource.mip_count == 0u ||
+			        resource.mip_count != binding.desc.view_info.level_count);
+			binding.mip_views.reserve(resource.mip_count);
+			for (uint32_t mip = 0; mip < resource.mip_count; mip++) {
+				auto desc = binding.desc;
+				desc.view_info.base_level += mip;
+				desc.view_info.level_count = 1;
+				binding.mip_views.push_back(texture_cache.FindTexture(binding.image_id, desc));
+			}
+			binding.image_view = binding.mip_views.front();
+		} else {
+			auto desc = binding.desc;
+			if (desc.type == TextureCache::BindingType::Storage) {
+				desc.view_info.level_count = 1;
+			}
+			binding.image_view = texture_cache.FindTexture(binding.image_id, desc);
+		}
 		auto&      image   = texture_cache.GetImage(binding.image_id);
 		const bool storage = binding.desc.type == TextureCache::BindingType::Storage;
 		image.usage.storage |= storage;
@@ -992,8 +1021,11 @@ void RenderExecutor::CommitBindings(CommandBuffer&                     buffer,
 			              range, vk_buffer);
 		} else if ((image.binding.force_general || image.binding.is_target) &&
 		           !image.info.IsDepth()) {
+			const vk::AccessFlags2 storage_access = image.binding.shader_write
+			                                            ? vk::AccessFlagBits2::eShaderWrite
+			                                            : vk::AccessFlags2 {};
 			image.Transit(vk::ImageLayout::eGeneral,
-			              vk::AccessFlagBits2::eShaderRead |
+			              vk::AccessFlagBits2::eShaderRead | storage_access |
 			                  vk::AccessFlagBits2::eColorAttachmentRead |
 			                  vk::AccessFlagBits2::eColorAttachmentWrite,
 			              {}, vk_buffer);

src/graphics/host_gpu/renderer/pipeline/pipelineCache.cpp

@@ -55,8 +55,9 @@ bool PipelineStaticParameters::operator==(const PipelineStaticParameters& other)
 PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
     RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
     ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
-    ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology, bool ps_active,
-    std::span<const uint32_t> vs_spirv, std::span<const uint32_t> ps_spirv) {
+    ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
+    bool primitive_restart_enable, bool ps_active, std::span<const uint32_t> vs_spirv,
+    std::span<const uint32_t> ps_spirv) {
 	KYTY_PROFILER_BLOCK("PipelineCache::CreatePipeline(Gfx)", profiler::colors::DeepOrangeA200);
 
 	EXIT_IF(colors == nullptr);
@@ -120,6 +121,12 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 			     depth.samples);
 		}
 	}
+	if (color_count == 0 && !with_depth) {
+		attachment_samples = render_sample_count(ctx.GetAaConfig().msaa_num_samples);
+		EXIT_IF(!static_cast<bool>(
+		    m_graphics.GetPhysicalDeviceProperties().limits.framebufferNoAttachmentsSampleCounts &
+		    vulkan_sample_count(attachment_samples)));
+	}
 	EXIT_IF(attachment_samples == 0 ||
 	        vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {});
 
@@ -130,12 +137,12 @@ PipelineCache::GraphicsPipeline& PipelineCache::CreateGraphicsPipeline(
 		}
 	}
 
-	const auto& clip_control = ctx.GetClipControl();
-	EXIT_NOT_IMPLEMENTED(!clip_control.IsZClipModeRepresentable());
-	static_params.negative_one_to_one = !clip_control.dx_clip_space;
-	static_params.depth_clip_enable   = clip_control.IsZClipEnabled();
-	static_params.topology            = topology;
-	static_params.samples             = attachment_samples;
+	const auto& clip_control               = ctx.GetClipControl();
+	static_params.negative_one_to_one      = !clip_control.dx_clip_space;
+	static_params.depth_clip_enable        = clip_control.IsZClipEnabled();
+	static_params.topology                 = topology;
+	static_params.primitive_restart_enable = primitive_restart_enable;
+	static_params.samples                  = attachment_samples;
 	static_params.sample_shading_enable =
 	    ps_active && attachment_samples > 1 && ps_input_info->ps_sample_shading;
 	if (static_params.sample_shading_enable && !m_graphics.sample_rate_shading_enabled) {

src/graphics/host_gpu/renderer/pipeline/pipelineCache.h

@@ -38,6 +38,7 @@ struct PipelineStaticParameters {
 	bool                       depth_clip_enable        = true;
 	int                        scissor_ltrb[4]          = {0};
 	vk::PrimitiveTopology      topology                 = vk::PrimitiveTopology::ePointList;
+	bool                       primitive_restart_enable = false;
 	uint32_t                   samples                  = 1;
 	bool                       sample_shading_enable    = false;
 	bool                       with_depth               = false;
@@ -79,9 +80,9 @@ static_assert(std::is_standard_layout_v<PipelineStaticParameters>);
 static_assert(alignof(PipelineStaticParameters) == 1);
 static_assert(sizeof(PipelineStaticParameters) ==
               sizeof(float[3]) + sizeof(float[3]) + sizeof(bool) * 2 + sizeof(int[4]) +
-                  sizeof(vk::PrimitiveTopology) + sizeof(uint32_t) + sizeof(bool) * 4 +
-                  sizeof(vk::CompareOp) + sizeof(bool) + sizeof(float) * 2 + sizeof(bool) +
-                  sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) +
+                  sizeof(vk::PrimitiveTopology) + sizeof(bool) + sizeof(uint32_t) +
+                  sizeof(bool) * 4 + sizeof(vk::CompareOp) + sizeof(bool) + sizeof(float) * 2 +
+                  sizeof(bool) + sizeof(PipelineStencilStaticState) * 2 + sizeof(uint32_t) +
                   sizeof(uint32_t[RENDER_COLOR_ATTACHMENTS_MAX]) + sizeof(bool) * 3 +
                   sizeof(uint8_t[RENDER_COLOR_ATTACHMENTS_MAX]) * 6 +
                   sizeof(bool[RENDER_COLOR_ATTACHMENTS_MAX]) * 3 + sizeof(float) * 4);
@@ -122,8 +123,8 @@ public:
 	CreateGraphicsPipeline(RenderColorInfo* colors, uint32_t color_count, RenderDepthInfo& depth,
 	                       ShaderVertexInputInfo& vs_input_info, RenderCommandBuffer& command,
 	                       ShaderPixelInputInfo* ps_input_info, vk::PrimitiveTopology topology,
-	                       bool ps_active, std::span<const uint32_t> vs_spirv,
-	                       std::span<const uint32_t> ps_spirv);
+	                       bool primitive_restart_enable, bool ps_active,
+	                       std::span<const uint32_t> vs_spirv, std::span<const uint32_t> ps_spirv);
 	ComputePipeline& CreateComputePipeline(ShaderComputeInputInfo&      input_info,
 	                                       const HW::ComputeShaderInfo& cs_regs,
 	                                       std::span<const uint32_t>    cs_spirv);

src/graphics/host_gpu/renderer/pipeline/shaderResourceBarrier.h

@@ -2,7 +2,7 @@
 #define EMULATOR_SRC_GRAPHICS_HOST_GPU_RENDERER_SHADERRESOURCEBARRIER_H_
 
 #include "graphics/host_gpu/graphicContext.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
 #include <vector>
 

src/graphics/host_gpu/renderer/pipeline/shaderSubgroup.cpp

@@ -1,6 +1,6 @@
 #include "graphics/host_gpu/renderer/pipeline/shaderSubgroup.h"
 
-#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
 
 namespace Libs::Graphics {
 

src/graphics/host_gpu/renderer/pipeline/shaders.cpp

@@ -726,7 +726,8 @@ void CreatePipelineInternal(
 	input_assembly.pNext    = nullptr;
 	input_assembly.flags    = {};
 	input_assembly.topology = static_params.topology;
-	input_assembly.primitiveRestartEnable = VK_FALSE;
+	input_assembly.primitiveRestartEnable =
+	    static_params.primitive_restart_enable ? VK_TRUE : VK_FALSE;
 
 	vk::Viewport viewport {};
 	viewport.x        = static_params.viewport_offset[0] - static_params.viewport_scale[0];

src/graphics/host_gpu/renderer/render.h

@@ -212,11 +212,12 @@ private:
 	                                          const DrawCallInfo& draw,
 	                                          uint32_t            render_target_slice_offset,
 	                                          bool log_setup_phases, DrawRenderState& state);
-	void ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
-	                         const DrawCallInfo& draw, DrawRenderState& state,
-	                         vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
-	                         const DrawIndexBufferSource& index_source, bool log_pipeline_phase,
-	                         bool set_bind_debug, bool set_auto_debug);
+	void               ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer& buffer,
+	                                       const DrawCallInfo& draw, DrawRenderState& state,
+	                                       vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
+	                                       const DrawIndexBufferSource& index_source,
+	                                       bool primitive_restart_enable, bool log_pipeline_phase,
+	                                       bool set_bind_debug, bool set_auto_debug);
 	[[nodiscard]] RenderState AcquireRenderTargets(CommandBuffer& buffer, RenderColorInfo* colors,
 	                                               uint32_t color_count, RenderDepthInfo& depth);
 	[[nodiscard]] bool        ResolveColorTargets(uint64_t submit_id, RenderCommandBuffer& buffer,

src/graphics/host_gpu/renderer/renderCompute.cpp

@@ -19,8 +19,8 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
 #include "kernel/pthread.h"
@@ -261,9 +261,10 @@ void RenderExecutor::DispatchDirect(uint64_t submit_id, RenderCommandBuffer& buf
 			     r.Base40(), static_cast<uint32_t>(r.Type()), static_cast<uint32_t>(r.Format()),
 			     static_cast<uint32_t>(r.Width5()) + 1u, static_cast<uint32_t>(r.Height5()) + 1u,
 			     static_cast<uint32_t>(r.Depth()) + 1u,
-			     std::max<uint32_t>(static_cast<uint32_t>(r.LastLevel()),
-			                        static_cast<uint32_t>(r.MaxMip())) +
-			         1u,
+			     r.Type() == Prospero::ImageType::kColor2DMsaa ||
+			             r.Type() == Prospero::ImageType::kColor2DMsaaArray
+			         ? 1u
+			         : static_cast<uint32_t>(r.MaxMip()) + 1u,
 			     static_cast<uint32_t>(r.TileMode()));
 		}
 		for (uint32_t i = 0; i < program.info.samplers.size(); i++) {

src/graphics/host_gpu/renderer/renderContext.cpp

@@ -26,7 +26,7 @@ RenderContext::~RenderContext() {
 void RenderContext::InitializeGpu(VideoOut::VideoOutDriver* video_out) {
 	EXIT_IF(m_gpu != nullptr);
 	m_video_out = video_out;
-	m_gpu       = std::make_unique<Gpu>(*this);
+	m_gpu       = std::make_unique<GuestGpu>(*this);
 	m_gpu_resources.SetGpu(m_gpu.get());
 }
 
@@ -45,7 +45,7 @@ void RenderContext::ShutdownGpu() {
 	}
 }
 
-Gpu& RenderContext::GetGpu() const {
+GuestGpu& RenderContext::GetGpu() const {
 	EXIT_IF(m_gpu == nullptr);
 	return *m_gpu;
 }

src/graphics/host_gpu/renderer/renderContext.h

@@ -24,7 +24,7 @@ class VideoOutDriver;
 namespace Libs::Graphics {
 
 constexpr int AGC_USER_INTERRUPT_EVENT = 0x1800;
-class Gpu;
+class GuestGpu;
 
 class RenderContext {
 public:
@@ -35,7 +35,7 @@ public:
 	[[nodiscard]] GraphicContext&           GetGraphics() const noexcept { return m_graphics; }
 	void                                    InitializeGpu(VideoOut::VideoOutDriver* video_out);
 	void                                    ShutdownGpu();
-	[[nodiscard]] Gpu&                      GetGpu() const;
+	[[nodiscard]] GuestGpu&                 GetGpu() const;
 	[[nodiscard]] VideoOut::VideoOutDriver& GetVideoOut() const;
 
 	Common::Mutex&      GetMutex() { return m_mutex; }
@@ -67,7 +67,7 @@ private:
 	PipelineCache             m_pipeline_cache;
 	SamplerCache              m_sampler_cache;
 	GpuResourceManager        m_gpu_resources;
-	std::unique_ptr<Gpu>      m_gpu;
+	std::unique_ptr<GuestGpu> m_gpu;
 	VideoOut::VideoOutDriver* m_video_out = nullptr;
 
 	Common::Mutex                  m_eop_mutex;

src/graphics/host_gpu/renderer/renderDraw.cpp

@@ -22,8 +22,8 @@
 #include "graphics/host_gpu/renderer/render.h"
 #include "graphics/host_gpu/renderer/renderContext.h"
 #include "graphics/host_gpu/vulkanCommon.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 #include "kernel/eventQueue.h"
 #include "kernel/memory.h"
@@ -35,9 +35,11 @@
 #include <atomic>
 #include <bit>
 #include <cmath>
+#include <cstdio>
 #include <cstring>
 #include <limits>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <span>
 #include <unordered_map>
@@ -338,6 +340,9 @@ static PipelineDynamicParameters BuildGraphicsDynamicParams(const RenderCommandB
 		framebuffer_extent = colors[0].extent;
 	} else if (depth.image_id) {
 		framebuffer_extent = {depth.width, depth.height};
+	} else {
+		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
+		framebuffer_extent = {limits.maxFramebufferWidth, limits.maxFramebufferHeight};
 	}
 
 	const auto final_scissor = calc_final_scissor(vp, ctx.GetScanModeControl(), framebuffer_extent);
@@ -503,6 +508,12 @@ static bool ShouldSkipGeShader(const RenderCommandBuffer& buffer) {
 	    sh_regs.m_geMaxOutputPerSubgroup > 0x00000040;
 
 	if (unsupported_stage_mask || unsupported_gs_stage || ge_group_size || ge_shader_regs) {
+		static std::once_flag warning_once;
+		std::call_once(warning_once, [] {
+			std::printf("Warning: game uses unsupported graphics pipelines; some draw calls were "
+			            "skipped.\n");
+		});
+
 		const auto log_id = g_shader_stage_log_count.fetch_add(1);
 		if (log_id < 32) {
 			LOGF("Skipping unsupported GE shader draw: stages=0x%08" PRIx32
@@ -640,8 +651,12 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 		attachment.has_stencil    = static_cast<bool>(aspects & vk::ImageAspectFlagBits::eStencil);
 		attachment.stencil_clear  = depth.stencil_clear_enable;
 	}
-	if (attachment_samples == 0 ||
-	    vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
+	if (color_count == 0 && !depth.image_id) {
+		const auto& limits = buffer.GetGraphics().GetPhysicalDeviceProperties().limits;
+		state.width        = limits.maxFramebufferWidth;
+		state.height       = limits.maxFramebufferHeight;
+	} else if (attachment_samples == 0 ||
+	           vulkan_sample_count(attachment_samples) == vk::SampleCountFlagBits {}) {
 		EXIT("render state has no valid attachments\n");
 	}
 	if (state.num_layers == std::numeric_limits<uint32_t>::max()) {
@@ -653,22 +668,11 @@ RenderState RenderExecutor::AcquireRenderTargets(CommandBuffer& buffer, RenderCo
 	return state;
 }
 
-static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer,
-                                     const DrawRenderState& state, const DrawCallInfo& draw) {
-	EXIT_IF(draw.name == nullptr);
-	const auto& ctx    = buffer.GetRegisters();
-	const auto& sh_ctx = buffer.GetShaders();
-
-	const bool with_depth = (state.depth_info.format != vk::Format::eUndefined &&
-	                         static_cast<bool>(state.depth_info.image_id));
-	if (state.color_count != 0 || !with_depth) {
-		return true;
-	}
-
-	const auto& sh_regs = ctx.GetShaderRegisters();
-	const auto& ps      = sh_ctx.GetPs();
-	return ShaderAddressValid(ps.ps_regs.data_addr) &&
-	       PixelShaderHasDepthOrCoverageSideEffects(sh_regs);
+static bool DrawHasActivePixelShader(const RenderCommandBuffer& buffer) {
+	const auto& ctx              = buffer.GetRegisters();
+	const auto& sh_regs          = ctx.GetShaderRegisters();
+	const bool  has_color_output = (ctx.GetRenderTargetMask() & sh_regs.m_cbShaderMask) != 0;
+	return has_color_output || PixelShaderHasDepthOrCoverageSideEffects(sh_regs);
 }
 
 enum class CbColorMode : uint8_t {
@@ -881,6 +885,41 @@ static bool GetDrawTopology(const HW::UserConfig& ucfg, bool auto_draw,
 	return true;
 }
 
+static bool ResolvePrimitiveRestart(const RenderCommandBuffer& buffer,
+                                    vk::PrimitiveTopology topology, uint32_t index_type_and_size) {
+	const auto control = buffer.GetUserConfig().GetPrimitiveResetControl();
+	EXIT_NOT_IMPLEMENTED((control & ~0x3u) != 0);
+	if ((control & 0x1u) == 0) {
+		return false;
+	}
+	switch (buffer.GetUserConfig().GetPrimType()) {
+		case Prospero::PrimitiveType::kLineStrip:
+		case Prospero::PrimitiveType::kTriFan:
+		case Prospero::PrimitiveType::kTriStrip: break;
+		default: return false;
+	}
+	if (topology != vk::PrimitiveTopology::eLineStrip &&
+	    topology != vk::PrimitiveTopology::eTriangleStrip &&
+	    topology != vk::PrimitiveTopology::eTriangleFan) {
+		return false;
+	}
+
+	uint32_t index_mask = 0;
+	switch (static_cast<Prospero::IndexType>(index_type_and_size)) {
+		case Prospero::IndexType::kIndex8: index_mask = 0xffu; break;
+		case Prospero::IndexType::kIndex16: index_mask = 0xffffu; break;
+		case Prospero::IndexType::kIndex32: index_mask = 0xffffffffu; break;
+		default: EXIT("unknown index_type_and_size: %u\n", index_type_and_size);
+	}
+
+	const auto reset_index = buffer.GetRegisters().GetPrimitiveResetIndex();
+	if ((control & 0x2u) != 0 && (reset_index & ~index_mask) != 0) {
+		return false;
+	}
+	EXIT_NOT_IMPLEMENTED((reset_index & index_mask) != index_mask);
+	return true;
+}
+
 bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuffer& buffer,
                                             const DrawCallInfo& draw,
                                             uint32_t            render_target_slice_offset,
@@ -909,14 +948,14 @@ bool RenderExecutor::PrepareDrawRenderState(uint64_t submit_id, RenderCommandBuf
 	}
 	ResolveRenderDepthTarget(submit_id, buffer, state.depth_info);
 
+	state.ps_active = DrawHasActivePixelShader(buffer);
 	const bool with_depth = (state.depth_info.format != vk::Format::eUndefined &&
 	                         static_cast<bool>(state.depth_info.image_id));
-	if (state.color_count == 0 && !with_depth) {
+	if (state.color_count == 0 && !with_depth && !state.ps_active) {
 		LogFramebufferSkip(draw.name, state.color_info[0], state.depth_info, buffer,
 		                   draw.index_count, draw.flags);
 		return false;
 	}
-	state.ps_active = DrawHasActivePixelShader(buffer, state, draw);
 
 	return true;
 }
@@ -1096,8 +1135,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
                                          const DrawCallInfo& draw, DrawRenderState& state,
                                          vk::PrimitiveTopology topology, const DrawEmitInfo& emit,
                                          const DrawIndexBufferSource& index_source,
-                                         bool log_pipeline_phase, bool set_bind_debug,
-                                         bool set_auto_debug) {
+                                         bool primitive_restart_enable, bool log_pipeline_phase,
+                                         bool set_bind_debug, bool set_auto_debug) {
 	EXIT_IF(draw.name == nullptr);
 	auto& ucfg = buffer.GetUserConfig();
 
@@ -1114,7 +1153,8 @@ void RenderExecutor::ExecutePreparedDraw(uint64_t submit_id, RenderCommandBuffer
 	}
 	auto& pipeline = m_context.GetPipelineCache().CreateGraphicsPipeline(
 	    state.color_info, state.color_count, state.depth_info, state.vs_input_info, buffer,
-	    &state.ps_input_info, topology, state.ps_active, state.vs_shader, state.ps_shader);
+	    &state.ps_input_info, topology, primitive_restart_enable, state.ps_active, state.vs_shader,
+	    state.ps_shader);
 
 	// Resource preparation above may synchronously finish and restart the scheduler. From this
 	// point onward, every operation targets the current command buffer and cannot touch guest
@@ -1238,6 +1278,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	vk::IndexType index_type           = vk::IndexType::eUint16;
 	uint64_t      index_size           = 0;
 	bool          expand_index8_to_u16 = false;
+	const bool primitive_restart = ResolvePrimitiveRestart(buffer, topology, index_type_and_size);
 
 	switch (static_cast<Prospero::IndexType>(index_type_and_size)) {
 		case Prospero::IndexType::kIndex16:
@@ -1268,7 +1309,7 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 		const auto* src = static_cast<const uint8_t*>(index_addr);
 		expanded_indices.resize(index_count);
 		for (uint32_t i = 0; i < index_count; i++) {
-			expanded_indices[i] = src[i];
+			expanded_indices[i] = primitive_restart && src[i] == 0xffu ? 0xffffu : src[i];
 		}
 	}
 
@@ -1298,8 +1339,8 @@ void RenderExecutor::DrawIndex(uint64_t submit_id, RenderCommandBuffer& buffer,
 	emit.indexed       = true;
 	emit.vertex_offset = vertex_offset;
 
-	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, true, true,
-	                    false);
+	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source,
+	                    primitive_restart, true, true, false);
 	ResetBindings();
 }
 
@@ -1398,7 +1439,7 @@ void RenderExecutor::DrawAuto(uint64_t submit_id, RenderCommandBuffer& buffer, u
 
 	DrawIndexBufferSource index_source {};
 	ExecutePreparedDraw(submit_id, buffer, draw, state, topology, emit, index_source, false, false,
-	                    true);
+	                    false, true);
 	ResetBindings();
 }
 

src/graphics/host_gpu/vulkanCommon.h

@@ -22,11 +22,6 @@ namespace Libs::Graphics {
 
 using VulkanMemoryBarrier = vk::MemoryBarrier;
 
-template <typename Handle>
-[[nodiscard]] void* VulkanHandleToPointer(Handle handle) {
-	return reinterpret_cast<void*>(static_cast<typename Handle::CType>(handle));
-}
-
 std::string VulkanToString(vk::Result value);
 std::string VulkanToString(vk::Format value);
 std::string VulkanToString(vk::ImageLayout value);

src/graphics/presentation/renderDoc.cpp

@@ -1,15 +1,12 @@
 #include "graphics/presentation/renderDoc.h"
 
-#include "SDL_syswm.h"
-#include "SDL_version.h"
-#include "SDL_video.h"
 #include "common/logging/log.h"
 
 #include <array>
 #include <atomic>
-#include <cstdint>
-#include <cstring>
+#include <chrono>
 #include <filesystem>
+#include <renderdoc_app.h>
 #include <string>
 
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
@@ -21,362 +18,161 @@
 #undef max
 #else
 #include <dlfcn.h>
-
-// RenderDoc uses Windows-style names in its cross-platform API.
-#define __cdecl
-using HMODULE = void*;
 #endif
 
 namespace Libs::Graphics {
 
-using RenderDocDevicePointer = void*;
-using RenderDocWindowHandle  = void*;
-
-enum RenderDocVersion {
-	eRENDERDOC_API_Version_1_4_2 = 10402,
-};
-
-enum RenderDocInputButton {
-	eRENDERDOC_Key_NonPrintable = 0x100,
-	eRENDERDOC_Key_Divide,
-	eRENDERDOC_Key_Multiply,
-	eRENDERDOC_Key_Subtract,
-	eRENDERDOC_Key_Plus,
-	eRENDERDOC_Key_F1,
-};
-
-using pRENDERDOC_SetCaptureKeys             = void(__cdecl*)(RenderDocInputButton* keys, int num);
-using pRENDERDOC_SetCaptureFilePathTemplate = void(__cdecl*)(const char* pathtemplate);
-using pRENDERDOC_GetCaptureFilePathTemplate = const char*(__cdecl*)();
-using pRENDERDOC_GetNumCaptures             = uint32_t(__cdecl*)();
-using pRENDERDOC_GetCapture = uint32_t(__cdecl*)(uint32_t idx, char* filename, uint32_t* pathlength,
-                                                 uint64_t* timestamp);
-using pRENDERDOC_UnloadCrashHandler = void(__cdecl*)();
-using pRENDERDOC_SetActiveWindow    = void(__cdecl*)(RenderDocDevicePointer device,
-                                                     RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_StartFrameCapture  = void(__cdecl*)(RenderDocDevicePointer device,
-                                                     RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_IsFrameCapturing   = uint32_t(__cdecl*)();
-using pRENDERDOC_EndFrameCapture    = uint32_t(__cdecl*)(RenderDocDevicePointer device,
-                                                         RenderDocWindowHandle  wndHandle);
-using pRENDERDOC_GetAPI = int(__cdecl*)(RenderDocVersion version, void** out_api_pointers);
-
-struct RenderDocApi {
-	void*                                 GetAPIVersion;
-	void*                                 SetCaptureOptionU32;
-	void*                                 SetCaptureOptionF32;
-	void*                                 GetCaptureOptionU32;
-	void*                                 GetCaptureOptionF32;
-	void*                                 SetFocusToggleKeys;
-	pRENDERDOC_SetCaptureKeys             SetCaptureKeys;
-	void*                                 GetOverlayBits;
-	void*                                 MaskOverlayBits;
-	void*                                 RemoveHooks;
-	pRENDERDOC_UnloadCrashHandler         UnloadCrashHandler;
-	pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate;
-	pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate;
-	pRENDERDOC_GetNumCaptures             GetNumCaptures;
-	pRENDERDOC_GetCapture                 GetCapture;
-	void*                                 TriggerCapture;
-	void*                                 IsTargetControlConnected;
-	void*                                 LaunchReplayUI;
-	pRENDERDOC_SetActiveWindow            SetActiveWindow;
-	pRENDERDOC_StartFrameCapture          StartFrameCapture;
-	pRENDERDOC_IsFrameCapturing           IsFrameCapturing;
-	pRENDERDOC_EndFrameCapture            EndFrameCapture;
-	void*                                 TriggerMultiFrameCapture;
-	void*                                 SetCaptureFileComments;
-	void*                                 DiscardFrameCapture;
-};
-
 enum class RenderDocState : uint32_t {
 	Idle,
 	Requested,
 	Capturing,
 };
 
-static RenderDocApi*               g_api             = nullptr;
-static HMODULE                     g_module          = nullptr;
-static RenderDocDevicePointer      g_device          = nullptr;
-static RenderDocWindowHandle       g_window          = nullptr;
+static RENDERDOC_API_1_6_0*        g_api             = nullptr;
 static std::atomic<RenderDocState> g_state           = RenderDocState::Idle;
-static std::atomic_bool            g_init_done       = false;
 static std::atomic_bool            g_unavailable_log = false;
 
-static RenderDocDevicePointer GetRenderDocDevicePointer(vk::Instance instance) {
-	if (instance == nullptr) {
-		return nullptr;
-	}
-
-	return VulkanHandleToPointer(instance);
-}
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 static bool BindRenderDocApi(HMODULE module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(module, "RENDERDOC_GetAPI"));
-	if (get_api == nullptr) {
-		return false;
-	}
-
-	void* api = nullptr;
-	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
-		return false;
-	}
-
-	g_module = module;
-	g_api    = static_cast<RenderDocApi*>(api);
-
-	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
-	g_api->SetCaptureKeys(nullptr, 0);
-	g_api->UnloadCrashHandler();
-
-	char module_path[MAX_PATH] = {};
-	GetModuleFileNameA(module, module_path, sizeof(module_path));
-	LOGF("RenderDoc: bound API from %s\n", module_path);
-	return true;
-}
-
-static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
-	if (window == nullptr) {
-		return nullptr;
-	}
-
-	SDL_SysWMinfo info {};
-	SDL_VERSION(&info.version);
-
-	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE || info.subsystem != SDL_SYSWM_WINDOWS) {
-		return nullptr;
-	}
-
-	return info.info.win.window;
-}
-
 #else
-
-static bool BindRenderDocApi(HMODULE module) {
+static bool BindRenderDocApi(void* module) {
 	auto* get_api = reinterpret_cast<pRENDERDOC_GetAPI>(::dlsym(module, "RENDERDOC_GetAPI"));
+#endif
 	if (get_api == nullptr) {
 		return false;
 	}
 
 	void* api = nullptr;
-	if (get_api(eRENDERDOC_API_Version_1_4_2, &api) == 0 || api == nullptr) {
+	if (get_api(eRENDERDOC_API_Version_1_6_0, &api) != 1 || api == nullptr) {
 		return false;
 	}
 
-	g_module = module;
-	g_api    = static_cast<RenderDocApi*>(api);
-
-	g_api->SetCaptureFilePathTemplate("_RenderDoc/kyty");
+	g_api = static_cast<RENDERDOC_API_1_6_0*>(api);
 	g_api->SetCaptureKeys(nullptr, 0);
 	g_api->UnloadCrashHandler();
-
-	Dl_info info {};
-	if (::dladdr(reinterpret_cast<void*>(get_api), &info) != 0 && info.dli_fname != nullptr) {
-		LOGF("RenderDoc: bound API from %s\n", info.dli_fname);
-	} else {
-		LOGF("RenderDoc: bound API\n");
-	}
+	LOGF("RenderDoc: API 1.6.0 bound\n");
 	return true;
 }
 
-static RenderDocWindowHandle GetRenderDocWindowHandle(SDL_Window* window) {
-	if (window == nullptr) {
-		return nullptr;
-	}
-
-	SDL_SysWMinfo info {};
-	SDL_VERSION(&info.version);
-
-	if (SDL_GetWindowWMInfo(window, &info) != SDL_TRUE) {
-		return nullptr;
-	}
-
-#if defined(SDL_VIDEO_DRIVER_X11)
-	if (info.subsystem == SDL_SYSWM_X11) {
-		// RenderDoc takes the raw xlib Window id in the pointer slot, not a Display*.
-		return reinterpret_cast<RenderDocWindowHandle>(
-		    static_cast<uintptr_t>(info.info.x11.window));
-	}
-#endif
-
-	// Wayland capture works without an active-window handle.
-	static std::atomic_bool logged = false;
-	if (!logged.exchange(true)) {
-		LOGF("RenderDoc: no native window handle for SDL subsystem %d (Wayland?); the in-app "
-		     "overlay is unavailable, but --rd captures still work\n",
-		     static_cast<int>(info.subsystem));
-	}
-	return nullptr;
-}
-
-#endif
-
-#if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
-
-static bool IsAvailable() {
-	return g_api != nullptr && g_device != nullptr && g_window != nullptr;
-}
-
-#else
-
-static bool IsAvailable() {
-	return g_api != nullptr && g_device != nullptr;
-}
-
-#endif
-
 #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS
 
 void RenderDocInit() {
-	bool expected = false;
-	if (!g_init_done.compare_exchange_strong(expected, true)) {
+	if (g_api != nullptr) {
 		return;
 	}
 
-	HKEY h_reg_key;
-	LONG result = RegOpenKeyExW(HKEY_LOCAL_MACHINE,
-	                            L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0,
-	                            KEY_READ, &h_reg_key);
-	if (result != ERROR_SUCCESS) {
-		return;
-	}
-	std::array<wchar_t, MAX_PATH> key_str {};
-	DWORD                         str_sz_out {key_str.size()};
-	result = RegQueryValueExW(h_reg_key, L"", 0, NULL, (LPBYTE)key_str.data(), &str_sz_out);
-	RegCloseKey(h_reg_key);
-	if (result != ERROR_SUCCESS) {
-		return;
-	}
-
-	std::filesystem::path path {key_str.cbegin(), key_str.cend()};
-	path                   = path.parent_path().append("renderdoc.dll");
-	const auto path_to_lib = path.generic_string();
-	auto*      module      = LoadLibraryA(path_to_lib.c_str());
+	auto* module = GetModuleHandleA("renderdoc.dll");
 	if (module == nullptr) {
-		return;
+		HKEY key = nullptr;
+		if (RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+		                  L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0, KEY_READ,
+		                  &key) != ERROR_SUCCESS) {
+			return;
+		}
+
+		std::array<wchar_t, MAX_PATH> path_buffer {};
+		DWORD      path_size = static_cast<DWORD>(path_buffer.size() * sizeof(wchar_t));
+		const auto result    = RegQueryValueExW(
+		    key, L"", nullptr, nullptr, reinterpret_cast<LPBYTE>(path_buffer.data()), &path_size);
+		RegCloseKey(key);
+		if (result != ERROR_SUCCESS) {
+			return;
+		}
+
+		auto path = std::filesystem::path(path_buffer.data()).parent_path() / "renderdoc.dll";
+		module    = LoadLibraryW(path.c_str());
+		if (module == nullptr) {
+			return;
+		}
 	}
 
 	if (!BindRenderDocApi(module)) {
-		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
-		FreeLibrary(module);
-		return;
+		LOGF("RenderDoc: API 1.6.0 is unavailable\n");
 	}
 }
 
 #else
 
 void RenderDocInit() {
-	bool expected = false;
-	if (!g_init_done.compare_exchange_strong(expected, true)) {
+	if (g_api != nullptr) {
 		return;
 	}
 
-	// Prefer an injected RenderDoc instance.
 	auto* module = ::dlopen("librenderdoc.so", RTLD_NOW | RTLD_NOLOAD);
 	if (module == nullptr) {
 		module = ::dlopen("librenderdoc.so", RTLD_NOW);
 	}
 	if (module == nullptr) {
-		LOGF("RenderDoc: librenderdoc.so was not found; in-app capture disabled\n");
 		return;
 	}
 
 	if (!BindRenderDocApi(module)) {
-		LOGF("RenderDoc: API 1.4.2 is not available; in-app capture disabled\n");
+		LOGF("RenderDoc: API 1.6.0 is unavailable\n");
 		::dlclose(module);
-		return;
 	}
 }
 
 #endif
 
-void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window) {
-	if (g_api == nullptr) {
-		return;
-	}
-
-	g_device = GetRenderDocDevicePointer(instance);
-	g_window = GetRenderDocWindowHandle(window);
-
-	if (g_device == nullptr || g_window == nullptr) {
-		LOGF("RenderDoc: active Vulkan window was not registered\n");
-		return;
-	}
-
-	g_api->SetActiveWindow(g_device, g_window);
-	LOGF("RenderDoc: active Vulkan window registered\n");
-}
-
 void RenderDocRequestCapture() {
-	if (!IsAvailable()) {
+	if (g_api == nullptr) {
 		if (!g_unavailable_log.exchange(true)) {
-			LOGF("RenderDoc: capture requested, but RenderDoc is not available\n");
+			LOGF("RenderDoc: capture requested, but RenderDoc is unavailable\n");
 		}
 		return;
 	}
 
 	RenderDocState expected = RenderDocState::Idle;
 	if (g_state.compare_exchange_strong(expected, RenderDocState::Requested)) {
-		LOGF("RenderDoc: capture requested; next complete presented frame will be captured\n");
-	} else {
-		LOGF("RenderDoc: capture request ignored because a capture is already pending\n");
+		LOGF("RenderDoc: capture requested\n");
 	}
 }
 
-static void LogNewestCapture() {
-	const auto count = g_api->GetNumCaptures();
-	if (count == 0) {
-		return;
-	}
-
-	char     filename[4096] = {};
-	uint32_t path_length    = sizeof(filename);
-	uint64_t timestamp      = 0;
-
-	if (g_api->GetCapture(count - 1, filename, &path_length, &timestamp) != 0) {
-		filename[sizeof(filename) - 1] = '\0';
-		LOGF("RenderDoc: wrote capture %s\n", filename);
-	}
+bool RenderDocCaptureRequested() {
+	return g_state.load(std::memory_order_acquire) == RenderDocState::Requested;
 }
 
-void RenderDocOnPresent() {
-	if (!IsAvailable()) {
+bool RenderDocCaptureInProgress() {
+	return g_state.load(std::memory_order_acquire) == RenderDocState::Capturing;
+}
+
+void RenderDocStartCapture() {
+	RenderDocState expected = RenderDocState::Requested;
+	if (g_api == nullptr || !g_state.compare_exchange_strong(expected, RenderDocState::Capturing,
+	                                                         std::memory_order_acq_rel)) {
 		return;
 	}
 
-	switch (g_state.load()) {
-		case RenderDocState::Idle: return;
-		case RenderDocState::Requested:
-			if (g_api->IsFrameCapturing() != 0) {
-				LOGF("RenderDoc: capture request ignored because RenderDoc is already capturing\n");
-				g_state.store(RenderDocState::Idle);
-				return;
-			}
+	if (g_api->IsFrameCapturing() != 0) {
+		g_state.store(RenderDocState::Idle, std::memory_order_release);
+		LOGF("RenderDoc: capture request ignored because a capture is already active\n");
+		return;
+	}
 
-			g_api->StartFrameCapture(nullptr, nullptr);
-			if (g_api->IsFrameCapturing() == 0) {
-				LOGF("RenderDoc: StartFrameCapture returned, but RenderDoc is not capturing\n");
-				g_state.store(RenderDocState::Idle);
-				return;
-			}
-			g_state.store(RenderDocState::Capturing);
-			LOGF("RenderDoc: capture started\n");
-			return;
-		case RenderDocState::Capturing: break;
+	const auto capture_id   = std::chrono::duration_cast<std::chrono::microseconds>(
+	                              std::chrono::system_clock::now().time_since_epoch())
+	                              .count();
+	const auto capture_path = "_RenderDoc/kyty_" + std::to_string(capture_id);
+	g_api->SetCaptureFilePathTemplate(capture_path.c_str());
+	g_api->StartFrameCapture(nullptr, nullptr);
+	if (g_api->IsFrameCapturing() == 0) {
+		g_state.store(RenderDocState::Idle, std::memory_order_release);
+		LOGF("RenderDoc: capture failed to start\n");
+		return;
+	}
+	LOGF("RenderDoc: capture started\n");
+}
+
+void RenderDocEndCapture() {
+	if (g_api == nullptr || !RenderDocCaptureInProgress()) {
+		return;
 	}
 
 	const auto ok = g_api->EndFrameCapture(nullptr, nullptr);
-	g_state.store(RenderDocState::Idle);
-
-	if (ok != 0) {
-		const auto count = g_api->GetNumCaptures();
-		LOGF("RenderDoc: capture finished, count=%u\n", count);
-		LogNewestCapture();
-	} else {
-		LOGF("RenderDoc: capture failed\n");
-	}
+	g_state.store(RenderDocState::Idle, std::memory_order_release);
+	LOGF(ok != 0 ? "RenderDoc: capture finished\n" : "RenderDoc: capture failed\n");
 }
 
 } // namespace Libs::Graphics

src/graphics/presentation/renderDoc.h

@@ -1,17 +1,15 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_RENDERDOC_H_
 
-#include "common/common.h"
-#include "graphics/host_gpu/vulkanCommon.h"
-
-struct SDL_Window;
-
 namespace Libs::Graphics {
 
 void RenderDocInit();
-void RenderDocSetActiveWindow(vk::Instance instance, SDL_Window* window);
 void RenderDocRequestCapture();
-void RenderDocOnPresent();
+void RenderDocStartCapture();
+void RenderDocEndCapture();
+
+[[nodiscard]] bool RenderDocCaptureRequested();
+[[nodiscard]] bool RenderDocCaptureInProgress();
 
 } // namespace Libs::Graphics
 

src/graphics/presentation/window/swapchain.cpp

@@ -32,7 +32,6 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/imeOverlay.h"
 #include "graphics/presentation/presenter.h"
-#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window/windowInternal.h"
 #include "libs/controller.h"
@@ -866,7 +865,6 @@ void Presenter::Present(Frame& frame, bool reuse) {
 			continue;
 		}
 
-		RenderDocOnPresent();
 		m_impl->presented_ime_revision.store(ime_visual.revision, std::memory_order_release);
 		window.UpdateTitle();
 		m_impl->frames.Release(&frame, true);

src/graphics/presentation/window/vulkanWindow.cpp

@@ -32,7 +32,6 @@
 #include "graphics/host_gpu/vulkanCommon.h"
 #include "graphics/presentation/imeOverlay.h"
 #include "graphics/presentation/presenter.h"
-#include "graphics/presentation/renderDoc.h"
 #include "graphics/presentation/videoOut.h"
 #include "graphics/presentation/window.h"
 #include "graphics/presentation/window/windowInternal.h"
@@ -555,6 +554,14 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	EXIT_NOT_IMPLEMENTED(supported_features2.features.depthBiasClamp != VK_TRUE);
 	features12.timelineSemaphore = VK_TRUE;
 
+	if (Config::GpuAssistedValidationEnabled()) {
+		if (supported_features12.bufferDeviceAddress != VK_TRUE) {
+			EXIT("--gpu-assisted-validation requires bufferDeviceAddress, unsupported by this "
+			     "device\n");
+		}
+		features12.bufferDeviceAddress = VK_TRUE;
+	}
+
 	vk::PhysicalDeviceFeatures device_features {};
 	device_features.fragmentStoresAndAtomics = VK_TRUE;
 	device_features.samplerAnisotropy        = VK_TRUE;
@@ -572,6 +579,9 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	graphics.sample_rate_shading_enabled                 = true;
 	device_features.vertexPipelineStoresAndAtomics =
 	    supported_features2.features.vertexPipelineStoresAndAtomics;
+	if (Config::GpuAssistedValidationEnabled()) {
+		device_features.shaderInt64 = supported_features2.features.shaderInt64;
+	}
 
 	if (graphics.subgroup_size_control_enabled &&
 	    supported_features13.subgroupSizeControl == VK_TRUE) {
@@ -601,11 +611,11 @@ static vk::Device VulkanCreateDevice(vk::PhysicalDevice physical_device, const V
 	     robustness2_ext_enabled && robustness2.robustImageAccess2 == VK_TRUE ? "true" : "false");
 
 	vk::DeviceCreateInfo create_info {};
-	create_info.sType                = vk::StructureType::eDeviceCreateInfo;
-	create_info.pNext                = &features13;
-	create_info.flags                = {};
-	create_info.pQueueCreateInfos    = &queue_create_info;
-	create_info.queueCreateInfoCount = 1;
+	create_info.sType                   = vk::StructureType::eDeviceCreateInfo;
+	create_info.pNext                   = &features13;
+	create_info.flags                   = {};
+	create_info.pQueueCreateInfos       = &queue_create_info;
+	create_info.queueCreateInfoCount    = 1;
 	create_info.enabledExtensionCount   = static_cast<uint32_t>(device_extensions.size());
 	create_info.ppEnabledExtensionNames = device_extensions.data();
 	create_info.pEnabledFeatures        = &device_features;
@@ -834,23 +844,26 @@ void WindowContext::CreateVulkan() {
 	app_info.engineVersion      = 1;
 	app_info.apiVersion         = VULKAN_TARGET_API_VERSION; // NOLINT
 
-	vk::ValidationFeatureEnableEXT enabled_features[] = {
+	if (Config::SpirvDebugPrintfEnabled() && Config::GpuAssistedValidationEnabled()) {
+		EXIT("--spirv-debug-printf and --gpu-assisted-validation are mutually exclusive\n");
+	}
+
+	vk::ValidationFeatureEnableEXT enabled_features[3] = {};
+	uint32_t                       enabled_features_count = 0;
 #ifdef KYTY_ENABLE_BEST_PRACTICES
-	    vk::ValidationFeatureEnableEXT::eBestPractices,
+	enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eBestPractices;
 #endif
 #ifdef KYTY_ENABLE_DEBUG_PRINTF
-	    vk::ValidationFeatureEnableEXT::eDebugPrintf,
-#endif
-	};
-
-	uint32_t enabled_features_count =
-	    sizeof(enabled_features) / sizeof(vk::ValidationFeatureEnableEXT);
-
-#ifdef KYTY_ENABLE_DEBUG_PRINTF
-	if (!Config::SpirvDebugPrintfEnabled()) {
-		enabled_features_count--;
+	if (Config::SpirvDebugPrintfEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eDebugPrintf;
 	}
 #endif
+	if (Config::GpuAssistedValidationEnabled()) {
+		enabled_features[enabled_features_count++] = vk::ValidationFeatureEnableEXT::eGpuAssisted;
+		LOGF("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::printf("Vulkan GPU-assisted validation is enabled; expect a large slowdown\n");
+		std::fflush(stdout);
+	}
 
 	vk::ValidationFeaturesEXT validation_features {};
 	validation_features.sType                          = vk::StructureType::eValidationFeaturesEXT;
@@ -986,7 +999,6 @@ void WindowContext::CreateVulkan() {
 	render_context = std::make_unique<RenderContext>(graphic_ctx);
 	LibKernel::Memory::InstallGpuResources(&render_context->GetGpuResources());
 	presenter = std::make_unique<Presenter>(*this);
-	RenderDocSetActiveWindow(graphic_ctx.instance, window);
 }
 
 void WindowContext::RefreshSurfaceCapabilities() {

src/graphics/shader/recompiler/ExecMask.cpp

@@ -1,51 +0,0 @@
-#include "graphics/shader/recompiler/ExecMask.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Exec {
-
-Mask FullMask(uint32_t wave_size) {
-	return {0xffffffffu, wave_size == 64u ? 0xffffffffu : 0u};
-}
-
-Mask ApplyWaveSize(Mask mask, uint32_t wave_size) {
-	const auto full = FullMask(wave_size);
-	mask.low &= full.low;
-	mask.high &= full.high;
-	return mask;
-}
-
-Mask ReadExec(const State& state) {
-	return ApplyWaveSize(state.exec, state.wave_size);
-}
-
-void WriteExec(State& state, Mask mask) {
-	state.exec = ApplyWaveSize(mask, state.wave_size);
-}
-
-void AndExec(State& state, Mask mask) {
-	WriteExec(state, {state.exec.low & mask.low, state.exec.high & mask.high});
-}
-
-void OrExec(State& state, Mask mask) {
-	WriteExec(state, {state.exec.low | mask.low, state.exec.high | mask.high});
-}
-
-void XorExec(State& state, Mask mask) {
-	WriteExec(state, {state.exec.low ^ mask.low, state.exec.high ^ mask.high});
-}
-
-void InvertExec(State& state) {
-	const auto full = FullMask(state.wave_size);
-	WriteExec(state, {~state.exec.low & full.low, ~state.exec.high & full.high});
-}
-
-bool IsExecZero(const State& state) {
-	const auto exec = ReadExec(state);
-	return exec.low == 0u && exec.high == 0u;
-}
-
-bool IsVccZero(const State& state) {
-	const auto vcc = ApplyWaveSize(state.vcc, state.wave_size);
-	return vcc.low == 0u && vcc.high == 0u;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Exec

src/graphics/shader/recompiler/ExecMask.h

@@ -1,34 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_
-
-#include "common/common.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Exec {
-
-struct Mask {
-	uint32_t low  = 0;
-	uint32_t high = 0;
-};
-
-struct State {
-	uint32_t wave_size = 64;
-	Mask     exec;
-	Mask     vcc;
-	bool     scc = false;
-};
-
-Mask FullMask(uint32_t wave_size);
-Mask ApplyWaveSize(Mask mask, uint32_t wave_size);
-
-Mask ReadExec(const State& state);
-void WriteExec(State& state, Mask mask);
-void AndExec(State& state, Mask mask);
-void OrExec(State& state, Mask mask);
-void XorExec(State& state, Mask mask);
-void InvertExec(State& state);
-bool IsExecZero(const State& state);
-bool IsVccZero(const State& state);
-
-} // namespace Libs::Graphics::ShaderRecompiler::Exec
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXECMASK_H_ */

src/graphics/shader/recompiler/ShaderRecompiler.cpp

@@ -2,18 +2,21 @@
 
 #include "common/assert.h"
 #include "common/logging/log.h"
-#include "graphics/shader/recompiler/cfg/ShaderCFG.h"
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
-#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
-#include "graphics/shader/recompiler/ir/BindingLayout.h"
-#include "graphics/shader/recompiler/ir/ReadLaneElimination.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
-#include "graphics/shader/recompiler/ir/ResourceTracking.h"
-#include "graphics/shader/recompiler/ir/ScalarProvenance.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+#include "graphics/shader/recompiler/frontend/cfg/ShaderCFG.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/translate/Translate.h"
 #include "graphics/shader/recompiler/ir/ShaderIR.h"
-#include "graphics/shader/recompiler/ir/ShaderInfoCollection.h"
-#include "graphics/shader/recompiler/ir/SrtPatcher.h"
-#include "graphics/shader/recompiler/ir/SrtWalker.h"
+#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/recompiler/ir/passes/BindingLayout.h"
+#include "graphics/shader/recompiler/ir/passes/ConstantPropagation.h"
+#include "graphics/shader/recompiler/ir/passes/DeadCodeElimination.h"
+#include "graphics/shader/recompiler/ir/passes/ReadLaneElimination.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceTracking.h"
+#include "graphics/shader/recompiler/ir/passes/ShaderInfoCollection.h"
+#include "graphics/shader/recompiler/ir/passes/SrtWalker.h"
+#include "graphics/shader/recompiler/ir/passes/SsaRewrite.h"
 
 #include <algorithm>
 #include <array>
@@ -43,7 +46,7 @@ std::string MakeIrDump(const CFG::Graph& cfg, const IR::Program& ir) {
 	std::string dump = "CFG:\n";
 	dump += CFG::GraphToString(cfg);
 	dump += "\nIR:\n";
-	dump += IR::ProgramToString(ir);
+	dump += ir.values != nullptr ? IR::ValueProgramToString(*ir.values) : IR::ProgramToString(ir);
 	return dump;
 }
 
@@ -56,10 +59,22 @@ const char* StageName(ShaderType stage) {
 	}
 }
 
-std::string FormatCfgFailure(const CFG::Graph& cfg, const CompileOptions& options,
-                             const std::string& reason) {
-	const auto block_id = cfg.failure_block != UINT32_MAX ? cfg.failure_block : cfg.entry_block;
-	return CFG::FormatBlockDiagnostic(cfg, block_id, StageName(options.stage), reason);
+void LogDispatcherFallback(const CompileOptions& options, const CFG::Graph& cfg, const char* phase,
+                           const std::string& reason) {
+	const auto* block        = cfg.FindBlock(cfg.failure_block);
+	const auto  start        = block != nullptr ? block->start_pc : UINT32_MAX;
+	const auto  end          = block != nullptr ? block->end_pc : UINT32_MAX;
+	const auto  predecessors = block != nullptr ? block->predecessors.size() : 0u;
+	const auto  successors   = block != nullptr ? block->successors.size() : 0u;
+	LOGF("%s CFG dispatcher fallback: stage=%s hash=0x%016" PRIx64
+	     " phase=%s failure=%s block=%" PRIu32 " pc=0x%08" PRIx32 "..0x%08" PRIx32 " preds=%" PRIu64
+	     " succs=%" PRIu64 " blocks=%" PRIu64 " loops=%" PRIu64 " back_edges=%" PRIu64
+	     " reason=%s\n",
+	     GetDumpLabel(options), StageName(options.stage), options.shader_hash, phase,
+	     CFG::FailureKindToString(cfg.failure_kind).c_str(), cfg.failure_block, start, end,
+	     static_cast<uint64_t>(predecessors), static_cast<uint64_t>(successors),
+	     static_cast<uint64_t>(cfg.blocks.size()), static_cast<uint64_t>(cfg.natural_loops.size()),
+	     static_cast<uint64_t>(cfg.back_edges.size()), reason.c_str());
 }
 
 bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
@@ -94,21 +109,31 @@ bool InstructionMaySplitSpirvBlock(const IR::Instruction& inst) {
 		case IR::Opcode::FlatStoreByte:
 		case IR::Opcode::FlatStoreShort:
 		case IR::Opcode::FlatStoreDword:
+		case IR::Opcode::DsReadUbyte:
+		case IR::Opcode::DsReadSbyte:
+		case IR::Opcode::DsReadUshort:
+		case IR::Opcode::DsReadSshort:
+		case IR::Opcode::DsReadB32:
 		case IR::Opcode::DsMinF32:
 		case IR::Opcode::DsMaxF32:
 		case IR::Opcode::DsWriteByte:
 		case IR::Opcode::DsWriteShort:
 		case IR::Opcode::DsWriteB32:
 		case IR::Opcode::DsWriteAddtidB32:
+		case IR::Opcode::DsReadAddtidB32:
 		case IR::Opcode::DsAppend:
 		case IR::Opcode::DsConsume:
+		case IR::Opcode::ImageLoad:
 		case IR::Opcode::ImageStore:
+		case IR::Opcode::ImageSample:
+		case IR::Opcode::ImageGather4:
 		case IR::Opcode::Export: return true;
 		default: return false;
 	}
 }
 
-bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string* reason) {
+bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, uint32_t* failure_block,
+                                            std::string* reason) {
 	for (const auto& block: ir.blocks) {
 		if (!block.terminator.loop_header) {
 			continue;
@@ -122,6 +147,9 @@ bool NeedsDispatcherForStructuredLoopHeader(const IR::Program& ir, std::string*
 				                      "lowering emits internal control flow: {}",
 				                      block.id, IR::InstructionToString(inst).c_str());
 			}
+			if (failure_block != nullptr) {
+				*failure_block = block.id;
+			}
 			return true;
 		}
 	}
@@ -199,19 +227,19 @@ uint32_t DecodedDstSize(const Decoder::Instruction& inst) {
 }
 
 uint32_t EmbeddedFetchDstSize(const Decoder::Instruction& inst) {
-	return inst.opcode == Decoder::Opcode::VMadU64U32 ? 2u : DecodedDstSize(inst);
+	return inst.opcode == Decoder::Opcode::V_MAD_U64_U32 ? 2u : DecodedDstSize(inst);
 }
 
 bool EmbeddedFetchHasBranch(Decoder::Opcode opcode) {
 	switch (opcode) {
-		case Decoder::Opcode::SSetpcB64:
-		case Decoder::Opcode::SBranch:
-		case Decoder::Opcode::SCbranchScc0:
-		case Decoder::Opcode::SCbranchScc1:
-		case Decoder::Opcode::SCbranchVccz:
-		case Decoder::Opcode::SCbranchVccnz:
-		case Decoder::Opcode::SCbranchExecz:
-		case Decoder::Opcode::SCbranchExecnz: return true;
+		case Decoder::Opcode::S_SETPC_B64:
+		case Decoder::Opcode::S_BRANCH:
+		case Decoder::Opcode::S_CBRANCH_SCC0:
+		case Decoder::Opcode::S_CBRANCH_SCC1:
+		case Decoder::Opcode::S_CBRANCH_VCCZ:
+		case Decoder::Opcode::S_CBRANCH_VCCNZ:
+		case Decoder::Opcode::S_CBRANCH_EXECZ:
+		case Decoder::Opcode::S_CBRANCH_EXECNZ: return true;
 		default: return false;
 	}
 }
@@ -260,32 +288,32 @@ bool TryDecodedSmemOffset(const std::array<EmbeddedFetchSgprInfo, 108>& sgprs,
 
 bool IsEmbeddedFetchSLoad(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::SLoadDword:
-		case Decoder::Opcode::SLoadDwordx2:
-		case Decoder::Opcode::SLoadDwordx4:
-		case Decoder::Opcode::SLoadDwordx8:
-		case Decoder::Opcode::SLoadDwordx16: return true;
+		case Decoder::Opcode::S_LOAD_DWORD:
+		case Decoder::Opcode::S_LOAD_DWORDX2:
+		case Decoder::Opcode::S_LOAD_DWORDX4:
+		case Decoder::Opcode::S_LOAD_DWORDX8:
+		case Decoder::Opcode::S_LOAD_DWORDX16: return true;
 		default: return false;
 	}
 }
 
 bool IsEmbeddedFetchBufferLoad(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::BufferLoadFormatX:
-		case Decoder::Opcode::BufferLoadFormatXy:
-		case Decoder::Opcode::BufferLoadFormatXyz:
-		case Decoder::Opcode::BufferLoadFormatXyzw: return true;
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_X:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XY:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZ:
+		case Decoder::Opcode::BUFFER_LOAD_FORMAT_XYZW: return true;
 		default: return false;
 	}
 }
 
 bool IsEmbeddedFetchAttribPropagationAlu(const Decoder::Instruction& inst) {
 	switch (inst.opcode) {
-		case Decoder::Opcode::SBfeU32:
-		case Decoder::Opcode::SAndB32:
-		case Decoder::Opcode::SAddI32:
-		case Decoder::Opcode::SAddU32:
-		case Decoder::Opcode::SLshlB32: return true;
+		case Decoder::Opcode::S_BFE_U32:
+		case Decoder::Opcode::S_AND_B32:
+		case Decoder::Opcode::S_ADD_I32:
+		case Decoder::Opcode::S_ADD_U32:
+		case Decoder::Opcode::S_LSHL_B32: return true;
 		default: return false;
 	}
 }
@@ -308,9 +336,6 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
                                             uint32_t user_data_base, uint32_t user_data_count,
                                             uint32_t wave_size) {
 	EmbeddedFetchData data;
-	if (input_info == nullptr || !input_info->fetch_embedded) {
-		return data;
-	}
 	data.loads.reserve(input_info->resources_num);
 	int32_t offset_candidate = -1;
 	bool    offset_conflict  = false;
@@ -348,10 +373,11 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 		uint32_t   sad_zero = 0;
 		const bool vertex_offset_add =
 		    vertex_index_accumulator && IsDecodedSgpr(inst.src0) &&
-		    ((inst.opcode == Decoder::Opcode::VAddI32 && IsDecodedVgpr(inst.src1) &&
+		    ((inst.opcode == Decoder::Opcode::V_ADD_I32 && IsDecodedVgpr(inst.src1) &&
 		      inst.src1.reg == inst.dst.reg) ||
-		     (user_data_base == 8 && inst.dst.reg == 5 && inst.opcode == Decoder::Opcode::VSadU32 &&
-		      IsDecodedVgpr(inst.src2) && inst.src2.reg == inst.dst.reg &&
+		     (user_data_base == 8 && inst.dst.reg == 5 &&
+		      inst.opcode == Decoder::Opcode::V_SAD_U32 && IsDecodedVgpr(inst.src2) &&
+		      inst.src2.reg == inst.dst.reg &&
 		      TryDecodedOperandConstant(sgprs, inst.src1, sad_zero) && sad_zero == 0));
 		if (data.loads.empty() && vertex_offset_add) {
 			const auto reg = DecodedSgprReg(inst.src0);
@@ -364,7 +390,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 			}
 		}
 		switch (inst.opcode) {
-			case Decoder::Opcode::VWritelaneB32: {
+			case Decoder::Opcode::V_WRITELANE_B32: {
 				uint32_t lane = 0;
 				if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
 					vgprs[inst.dst.reg] = {};
@@ -380,7 +406,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 				}
 				break;
 			}
-			case Decoder::Opcode::VReadlaneB32: {
+			case Decoder::Opcode::V_READLANE_B32: {
 				uint32_t lane = 0;
 				if (track_vector_lanes && IsDecodedSgpr(inst.dst) &&
 				    DecodedSgprReg(inst.dst) < sgprs.size() && IsDecodedVgpr(inst.src0) &&
@@ -394,7 +420,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 				}
 				break;
 			}
-			case Decoder::Opcode::SMovB32:
+			case Decoder::Opcode::S_MOV_B32:
 				if (IsDecodedSgpr(inst.dst) && IsDecodedSgpr(inst.src0) &&
 				    DecodedSgprReg(inst.src0) < sgprs.size()) {
 					sgprs[DecodedSgprReg(inst.dst)] = sgprs[DecodedSgprReg(inst.src0)];
@@ -410,7 +436,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 					}
 				}
 				break;
-			case Decoder::Opcode::SMovkI32:
+			case Decoder::Opcode::S_MOVK_I32:
 				if (IsDecodedSgpr(inst.dst)) {
 					auto& dst = sgprs[DecodedSgprReg(inst.dst)];
 					dst.type  = EmbeddedFetchValueType::Constant;
@@ -471,7 +497,7 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 					} else {
 						ClearEmbeddedFetchSgprs(sgprs, inst.dst, DecodedDstSize(inst));
 					}
-				} else if (inst.opcode == Decoder::Opcode::VCndmaskB32) {
+				} else if (inst.opcode == Decoder::Opcode::V_CNDMASK_B32) {
 					if (IsDecodedVgpr(inst.dst) && inst.dst.reg < vgprs.size()) {
 						ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg);
 					}
@@ -493,11 +519,11 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 							auto& dst = sgprs[DecodedSgprReg(inst.dst)];
 							dst.type  = EmbeddedFetchValueType::Constant;
 							switch (inst.opcode) {
-								case Decoder::Opcode::SAndB32: dst.value = src0 & src1; break;
-								case Decoder::Opcode::SLshlB32:
+								case Decoder::Opcode::S_AND_B32: dst.value = src0 & src1; break;
+								case Decoder::Opcode::S_LSHL_B32:
 									dst.value = src0 << (src1 & 31u);
 									break;
-								case Decoder::Opcode::SBfeU32:
+								case Decoder::Opcode::S_BFE_U32:
 									dst.value = src0 >> (src1 & 31u);
 									break;
 								default: dst.value = src0 + src1; break;
@@ -526,9 +552,9 @@ EmbeddedFetchData DetectEmbeddedVertexFetch(const Decoder::Program&      decoded
 				}
 				break;
 		}
-		if (inst.opcode == Decoder::Opcode::VMovreldB32) {
+		if (inst.opcode == Decoder::Opcode::V_MOVRELD_B32) {
 			vector_lanes.clear();
-		} else if (inst.opcode != Decoder::Opcode::VWritelaneB32 && IsDecodedVgpr(inst.dst)) {
+		} else if (inst.opcode != Decoder::Opcode::V_WRITELANE_B32 && IsDecodedVgpr(inst.dst)) {
 			for (uint32_t i = 0; i < EmbeddedFetchDstSize(inst) && inst.dst.reg + i < vgprs.size();
 			     i++) {
 				ClearEmbeddedFetchVectorLanes(&vector_lanes, inst.dst.reg + i);
@@ -559,9 +585,6 @@ bool IsIrFetchPrologLoad(const IR::Instruction& inst) {
 
 int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
                                  const EmbeddedFetchLoad&     load) {
-	if (input_info == nullptr) {
-		return -1;
-	}
 	if (load.attrib_id >= 0 && load.attrib_id < input_info->resources_num &&
 	    input_info->resources_dst[load.attrib_id].attr_id == load.attrib_id) {
 		return load.attrib_id;
@@ -583,7 +606,7 @@ int ResolveEmbeddedFetchResource(const ShaderVertexInputInfo* input_info,
 
 uint32_t RewriteEmbeddedVertexFetches(IR::Program& ir, const ShaderVertexInputInfo* input_info,
                                       const std::vector<EmbeddedFetchLoad>& loads) {
-	if (input_info == nullptr || loads.empty()) {
+	if (loads.empty()) {
 		return 0;
 	}
 
@@ -697,27 +720,25 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	bool        dispatcher_fallback = false;
 	std::string dispatcher_reason;
 	if (cfg.irreducible) {
-		dispatcher_fallback   = true;
-		dispatcher_reason     = cfg.unsupported_reason;
-		const auto diagnostic = FormatCfgFailure(cfg, options, cfg.unsupported_reason);
-		LOGF("%s irreducible CFG detected: %s\n", GetDumpLabel(options), diagnostic.c_str());
+		dispatcher_fallback = true;
+		dispatcher_reason   = cfg.unsupported_reason;
+		LogDispatcherFallback(options, cfg, "build", dispatcher_reason);
 	} else {
 		std::string structure_error;
 		const auto  unstructured_cfg = cfg;
 		LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " CFG Structurize\n",
 		     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
 		if (!CFG::Structurize(cfg, &structure_error)) {
-			const auto diagnostic = FormatCfgFailure(cfg, options, structure_error);
-			LOGF("%s structured CFG bug/failure: %s\n", GetDumpLabel(options), diagnostic.c_str());
 			dispatcher_fallback      = true;
 			dispatcher_reason        = structure_error;
 			const auto failure_kind  = cfg.failure_kind;
 			const auto failure_block = cfg.failure_block;
-			cfg                      = unstructured_cfg;
-			cfg.unsupported          = true;
-			cfg.failure_kind         = failure_kind;
-			cfg.failure_block        = failure_block;
-			cfg.unsupported_reason   = structure_error;
+			LogDispatcherFallback(options, cfg, "structurize", dispatcher_reason);
+			cfg                    = unstructured_cfg;
+			cfg.unsupported        = true;
+			cfg.failure_kind       = failure_kind;
+			cfg.failure_block      = failure_block;
+			cfg.unsupported_reason = structure_error;
 		} else {
 			LOGF("%s structured CFG success: blocks=%" PRIu64 "\n", GetDumpLabel(options),
 			     static_cast<uint64_t>(cfg.blocks.size()));
@@ -732,34 +753,134 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	IR::Program ir;
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " IR LowerProgram\n", GetDumpLabel(options),
 	     StageName(options.stage), options.shader_hash);
-	if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, ir, error)) {
+	const auto lower_program = [&](IR::Program& output) {
+		output = {};
+		if (!IR::LowerProgram(decoded, cfg, options.stage, options.wave_size, output, error)) {
+			return false;
+		}
+		output.lane_mask_mode  = options.lane_mask_mode;
+		output.shader_hash     = options.shader_hash;
+		output.user_data_base  = options.user_data_base;
+		output.user_data_count = options.user_data_count;
+		return true;
+	};
+	if (!lower_program(ir)) {
 		return false;
 	}
-	ir.lane_mask_mode  = options.lane_mask_mode;
-	ir.shader_hash     = options.shader_hash;
-	ir.user_data_base  = options.user_data_base;
-	ir.user_data_count = options.user_data_count;
+
+	if (!dispatcher_fallback) {
+		const auto select_dispatcher = [&](uint32_t failure_block, const std::string& reason,
+		                                   const CFG::Graph& diagnostic_cfg) {
+			dispatcher_fallback    = true;
+			dispatcher_reason      = reason;
+			cfg.failure_kind       = CFG::FailureKind::StructuredControlFlow;
+			cfg.failure_block      = failure_block;
+			cfg.unsupported_reason = reason;
+			ir.cfg_failure_kind    = CFG::FailureKind::StructuredControlFlow;
+			ir.fallback_reason     = reason;
+			LogDispatcherFallback(options, diagnostic_cfg, "emit", reason);
+		};
+		const auto retry_budget = cfg.natural_loops.size();
+		for (size_t retry = 0;; retry++) {
+			std::string emitter_reason;
+			uint32_t    failure_block = UINT32_MAX;
+			if (!NeedsDispatcherForStructuredLoopHeader(ir, &failure_block, &emitter_reason)) {
+				break;
+			}
+			if (retry >= retry_budget) {
+				select_dispatcher(failure_block,
+				                  emitter_reason + "; no-clone loop-header isolation exhausted",
+				                  cfg);
+				break;
+			}
+
+			const auto* source_block = cfg.FindBlock(failure_block);
+			const auto  source_pc = source_block != nullptr ? source_block->start_pc : UINT32_MAX;
+			const auto  previous_cfg  = cfg;
+			const auto  blocks_before = cfg.blocks.size();
+			std::string isolation_error;
+			if (!CFG::IsolateLoopHeader(cfg, failure_block, &isolation_error)) {
+				const auto failed_cfg = cfg;
+				cfg                   = previous_cfg;
+				select_dispatcher(failure_block,
+				                  fmt::format("{}; no-clone loop-header isolation failed: {}",
+				                              emitter_reason, isolation_error),
+				                  failed_cfg);
+				break;
+			}
+
+			IR::Program isolated_ir;
+			if (!lower_program(isolated_ir)) {
+				const auto lower_error = error != nullptr ? *error : "IR lowering failed";
+				if (error != nullptr) {
+					error->clear();
+				}
+				cfg = previous_cfg;
+				select_dispatcher(failure_block,
+				                  fmt::format("{}; isolated loop-header lowering failed: {}",
+				                              emitter_reason, lower_error),
+				                  cfg);
+				break;
+			}
+			ir = std::move(isolated_ir);
+			LOGF("%s CFG isolated semantic loop header: stage=%s hash=0x%016" PRIx64
+			     " source_block=%" PRIu32 " pc=0x%08" PRIx32 " blocks=%" PRIu64 "->%" PRIu64
+			     " reason=%s\n",
+			     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
+			     failure_block, source_pc, static_cast<uint64_t>(blocks_before),
+			     static_cast<uint64_t>(cfg.blocks.size()), emitter_reason.c_str());
+		}
+	}
 	LOGF("%s phase end: stage=%s hash=0x%016" PRIx64 " IR LowerProgram blocks=%" PRIu64
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(ir.blocks.size()), phase_ms());
+	const ShaderVertexInputInfo*  vertex  = nullptr;
+	const ShaderPixelInputInfo*   pixel   = nullptr;
+	const ShaderComputeInputInfo* compute = nullptr;
+	switch (options.stage) {
+		case ShaderType::Vertex: vertex = options.input_info.vertex; break;
+		case ShaderType::Pixel: pixel = options.input_info.pixel; break;
+		case ShaderType::Compute: compute = options.input_info.compute; break;
+		default: break;
+	}
 	EmbeddedFetchData embedded_fetch;
-	if (options.stage == ShaderType::Vertex && options.vertex_input_info != nullptr &&
-	    options.vertex_input_info->fetch_embedded) {
-		embedded_fetch =
-		    DetectEmbeddedVertexFetch(decoded, options.vertex_input_info, ir.user_data_base,
-		                              ir.user_data_count, options.wave_size);
-		auto rewritten =
-		    RewriteEmbeddedVertexFetches(ir, options.vertex_input_info, embedded_fetch.loads);
+	if (options.stage == ShaderType::Vertex && vertex->fetch_embedded) {
+		embedded_fetch = DetectEmbeddedVertexFetch(decoded, vertex, ir.user_data_base,
+		                                           ir.user_data_count, options.wave_size);
+		auto rewritten = RewriteEmbeddedVertexFetches(ir, vertex, embedded_fetch.loads);
 		if (rewritten > 0 || !embedded_fetch.loads.empty()) {
 			LOGF("%s embedded vertex fetch rewrite: detected=%" PRIu64 " rewritten=%" PRIu32 "\n",
 			     GetDumpLabel(options), static_cast<uint64_t>(embedded_fetch.loads.size()),
 			     rewritten);
 		}
 	}
-	if (!IR::BuildScalarProvenance(ir, error)) {
+	ir.dispatcher_fallback = dispatcher_fallback;
+	if (!dispatcher_reason.empty()) {
+		ir.fallback_reason = dispatcher_reason;
+	}
+
+	ir.values = std::make_shared<IR::ValueProgram>();
+	if (!Frontend::TranslateProgram(ir, *ir.values, vertex, pixel, compute, error)) {
 		return false;
 	}
+	ir.blocks.clear();
+	IR::RewriteToSsa(ir.values->blocks);
+	IR::ConstantPropagationPass(ir.values->blocks);
+	for (auto& info: ir.values->block_info) {
+		info.condition       = info.condition.Resolve();
+		info.indirect_target = info.indirect_target.Resolve();
+	}
+	IR::RemoveIdentities(ir.values->blocks);
+	IR::EliminateDeadCode(ir.values->blocks);
+	const auto read_lane_stats = IR::EliminateReadLane(*ir.values, ir.wave_size);
+	if (read_lane_stats.rewritten_reads != 0) {
+		LOGF("%s read-lane elimination: reads=%" PRIu32 "\n", GetDumpLabel(options),
+		     read_lane_stats.rewritten_reads);
+		IR::ConstantPropagationPass(ir.values->blocks);
+		IR::RemoveIdentities(ir.values->blocks);
+		IR::EliminateDeadCode(ir.values->blocks);
+	}
 	std::string srt_error;
 	if (!IR::BuildSrtPlan(ir, &srt_error)) {
 		LOGF("%s SRT planning failed: %s\n", GetDumpLabel(options), srt_error.c_str());
@@ -768,29 +889,14 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		}
 		return false;
 	}
-	ir.dispatcher_fallback = dispatcher_fallback;
-	if (!dispatcher_reason.empty()) {
-		ir.fallback_reason = dispatcher_reason;
-	}
-
-	if (!IR::PatchSrtReads(ir, error) || !IR::TrackResources(ir, error)) {
+	IR::EliminateDeadCode(ir.values->blocks);
+	if (!IR::TrackResources(ir, error)) {
 		return false;
 	}
+	IR::EliminateDeadCode(ir.values->blocks);
 	if (options.stage == ShaderType::Vertex) {
 		ir.info.vertex_offset_sgpr = embedded_fetch.vertex_offset_sgpr;
 	}
-	if (!dispatcher_fallback) {
-		std::string emitter_reason;
-		if (NeedsDispatcherForStructuredLoopHeader(ir, &emitter_reason)) {
-			dispatcher_fallback    = true;
-			dispatcher_reason      = emitter_reason;
-			ir.dispatcher_fallback = true;
-			ir.cfg_failure_kind    = CFG::FailureKind::StructuredControlFlow;
-			ir.fallback_reason     = emitter_reason;
-			LOGF("%s structured CFG emitter fallback: %s\n", GetDumpLabel(options),
-			     emitter_reason.c_str());
-		}
-	}
 
 	IR::ResourceSnapshot resources;
 	if (options.resource_snapshot != nullptr) {
@@ -809,6 +915,7 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		runtime.shader_base = options.shader_base != 0 ? options.shader_base
 		                                               : reinterpret_cast<uint64_t>(code.data());
 		runtime.read_memory = options.read_memory;
+		runtime.read_specialization_memory = options.read_specialization_memory;
 		if (runtime.read_memory == nullptr && options.user_data == nullptr) {
 			runtime.read_memory = ReadZeroMemory;
 		}
@@ -822,16 +929,10 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 		return false;
 	}
 
-	ShaderVertexInputInfo  default_vertex {};
-	ShaderPixelInputInfo   default_pixel {};
-	ShaderComputeInputInfo default_compute {};
-	IR::ShaderInfoOptions  info_options;
-	info_options.vertex =
-	    options.vertex_input_info != nullptr ? options.vertex_input_info : &default_vertex;
-	info_options.pixel =
-	    options.pixel_input_info != nullptr ? options.pixel_input_info : &default_pixel;
-	info_options.compute =
-	    options.compute_input_info != nullptr ? options.compute_input_info : &default_compute;
+	IR::ShaderInfoOptions info_options;
+	info_options.vertex  = vertex;
+	info_options.pixel   = pixel;
+	info_options.compute = compute;
 	if (!IR::CollectShaderInfo(ir, info_options, error)) {
 		return false;
 	}
@@ -841,11 +942,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	if (!IR::AllocateBindings(ir, layout_options, error)) {
 		return false;
 	}
-	const auto read_lane_stats = IR::EliminateReadLane(ir);
-	if (read_lane_stats.rewritten_reads != 0) {
-		LOGF("%s read-lane elimination: reads=%" PRIu32 " shadow_writes=%" PRIu32 "\n",
-		     GetDumpLabel(options), read_lane_stats.rewritten_reads, read_lane_stats.shadow_writes);
-	}
 	std::string ir_dump;
 	if (options.dump_ir) {
 		ir_dump = MakeIrDump(cfg, ir);
@@ -858,8 +954,8 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	std::string           emit_error;
 	LOGF("%s phase begin: stage=%s hash=0x%016" PRIx64 " SPIR-V EmitProgram\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash);
-	if (!Spirv::EmitProgram(ir, resources, options.vertex_input_info, options.pixel_input_info,
-	                        options.compute_input_info, spirv, &emit_error)) {
+	if (!Spirv::EmitProgram(ir, resources, options.input_info, spirv, &emit_error)) {
+		LOGF("%s typed SPIR-V emit failed: %s\n", GetDumpLabel(options), emit_error.c_str());
 		if (dispatcher_fallback && error != nullptr) {
 			*error = fmt::format("dispatcher fallback failed after {}: {}",
 			                     dispatcher_reason.c_str(), emit_error.c_str());
@@ -873,10 +969,6 @@ bool TryRecompile(std::span<const uint32_t> code, const CompileOptions& options,
 	     " elapsed_ms=%" PRIu64 "\n",
 	     GetDumpLabel(options), StageName(options.stage), options.shader_hash,
 	     static_cast<uint64_t>(spirv.size()), phase_ms());
-	if (dispatcher_fallback) {
-		LOGF("%s dispatcher fallback used: %s\n", GetDumpLabel(options), dispatcher_reason.c_str());
-	}
-
 	result.spirv     = std::move(spirv);
 	result.program   = std::move(ir);
 	result.resources = std::move(resources);

src/graphics/shader/recompiler/ShaderRecompiler.h

@@ -3,7 +3,7 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 #include "graphics/shader/shader.h"
 
 #include <optional>
@@ -13,26 +13,25 @@
 namespace Libs::Graphics::ShaderRecompiler {
 
 struct CompileOptions {
-	ShaderType                    stage           = ShaderType::Compute;
-	ShaderLaneMaskMode            lane_mask_mode  = ShaderLaneMaskMode::NativeWave;
-	uint32_t                      wave_size       = 64;
-	uint32_t                      user_data_base  = 0;
-	uint32_t                      user_data_count = 64;
-	uint64_t                      shader_hash     = 0;
-	uint64_t                      shader_base     = 0;
-	std::optional<uint64_t>       flat_memory_base;
-	uint32_t                      descriptor_set       = 0;
-	uint32_t                      push_constant_offset = 0;
-	bool                          dump_ir              = true;
-	bool                          early_dump           = false;
-	const char*                   dump_label           = nullptr;
-	const uint32_t*               user_data            = nullptr;
-	IR::SrtMemoryReader           read_memory          = nullptr;
-	void*                         read_memory_data     = nullptr;
-	const IR::ResourceSnapshot*   resource_snapshot    = nullptr;
-	const ShaderVertexInputInfo*  vertex_input_info    = nullptr;
-	const ShaderPixelInputInfo*   pixel_input_info     = nullptr;
-	const ShaderComputeInputInfo* compute_input_info   = nullptr;
+	ShaderType                  stage           = ShaderType::Compute;
+	ShaderLaneMaskMode          lane_mask_mode  = ShaderLaneMaskMode::NativeWave;
+	uint32_t                    wave_size       = 64;
+	uint32_t                    user_data_base  = 0;
+	uint32_t                    user_data_count = 64;
+	uint64_t                    shader_hash     = 0;
+	uint64_t                    shader_base     = 0;
+	std::optional<uint64_t>     flat_memory_base;
+	uint32_t                    descriptor_set             = 0;
+	uint32_t                    push_constant_offset       = 0;
+	bool                        dump_ir                    = true;
+	bool                        early_dump                 = false;
+	const char*                 dump_label                 = nullptr;
+	const uint32_t*             user_data                  = nullptr;
+	IR::SrtMemoryReader         read_memory                = nullptr;
+	IR::SrtMemoryReader         read_specialization_memory = nullptr;
+	void*                       read_memory_data           = nullptr;
+	const IR::ResourceSnapshot* resource_snapshot          = nullptr;
+	ShaderStageInputInfo        input_info;
 };
 
 struct CompileResult {

src/graphics/shader/recompiler/backend/spirv/SpirvBuilder.cpp

@@ -1,4 +1,4 @@
-#include "graphics/shader/recompiler/emitter/SpirvBuilder.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvBuilder.h"
 
 #include <cstdio>
 #include <cstring>

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.cpp

@@ -0,0 +1,392 @@
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/ValueProgram.h"
+#include "graphics/shader/shader.h"
+
+#include <algorithm>
+#include <array>
+#include <cinttypes>
+#include <cstdio>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv {
+
+using ShaderError::Fail;
+
+namespace {
+
+bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
+	using Kind = IR::DescriptorBindingKind;
+	using Dim  = Decoder::ImageDimension;
+	if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
+		const bool integer = image.kind == IR::ResourceKind::ImageUint;
+		switch (image.dimension) {
+			case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
+			case Dim::Dim1DArray:
+				kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
+				return true;
+			case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
+			case Dim::Dim2DMsaa:
+				kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
+				return true;
+			case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
+			case Dim::Dim2DArray:
+				kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
+				return true;
+			case Dim::Dim2DMsaaArray:
+				kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
+				return true;
+			case Dim::Unknown: return false;
+		}
+	}
+	const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
+	if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
+		return false;
+	}
+	switch (image.dimension) {
+		case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
+		case Dim::Dim1DArray:
+			kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
+			return true;
+		case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
+		case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
+		case Dim::Dim2DArray:
+			kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
+			return true;
+		case Dim::Dim2DMsaa:
+		case Dim::Dim2DMsaaArray: return false;
+		case Dim::Unknown: return false;
+	}
+	return false;
+}
+
+bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
+	using Kind                                             = IR::DescriptorBindingKind;
+	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
+	std::array<std::vector<uint32_t>, KindCount> expected;
+	std::array<bool, KindCount>                  present {};
+	const auto                                   Dense = [](size_t size) {
+		std::vector<uint32_t> values(size);
+		for (uint32_t i = 0; i < values.size(); i++) {
+			values[i] = i;
+		}
+		return values;
+	};
+	auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
+		const auto index = static_cast<size_t>(kind);
+		present[index]   = true;
+		expected[index]  = std::move(resources);
+	};
+	if (!program.info.buffers.empty()) {
+		Expect(Kind::Buffers, Dense(program.info.buffers.size()));
+	}
+	for (uint32_t i = 0; i < program.info.images.size(); i++) {
+		Kind kind;
+		if (!ImageBinding(program.info.images[i], kind)) {
+			return Fail(error, "native shader plan has an invalid image class");
+		}
+		present[static_cast<size_t>(kind)] = true;
+		const auto dynamic = program.info.images[i].mip_mode == IR::ImageMipMode::DynamicStorage;
+		const auto count   = dynamic ? program.info.images[i].mip_count : 1u;
+		if (count == 0u || (!dynamic && program.info.images[i].mip_count != 1u)) {
+			return Fail(error, "native shader plan has an invalid image mip descriptor count");
+		}
+		expected[static_cast<size_t>(kind)].insert(expected[static_cast<size_t>(kind)].end(), count,
+		                                           i);
+	}
+	if (!program.info.samplers.empty()) {
+		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
+	}
+	const auto uses_gds = program.values != nullptr &&
+	                      std::ranges::any_of(program.values->blocks, [](const IR::Block* block) {
+		                      return std::ranges::any_of(*block, [](const IR::Inst& inst) {
+			                      return inst.GetOpcode() == IR::ValueOpcode::GetGdsResource;
+		                      });
+	                      });
+	if (uses_gds) {
+		Expect(Kind::Gds);
+	}
+	if (!program.info.addresses.empty()) {
+		Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
+	}
+	const bool uses_flattened_runtime =
+	    program.values != nullptr &&
+	    (!program.values->srt_reads.empty() ||
+	     std::ranges::any_of(program.info.images, [](const IR::ImageResource& image) {
+		     return image.indirect_mapping_capacity != 0u;
+	     }));
+	if (uses_flattened_runtime) {
+		Expect(Kind::FlattenedSrt);
+	}
+	if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
+		Expect(Kind::UserData);
+	}
+
+	std::array<bool, KindCount> seen {};
+	for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
+		const auto& binding = program.bindings.descriptors[i];
+		const auto  kind    = static_cast<size_t>(binding.kind);
+		if (kind >= KindCount || seen[kind] || !present[kind] ||
+		    binding.resources != expected[kind]) {
+			return Fail(error, "native descriptor groups do not match shader topology");
+		}
+		for (uint32_t j = 0; j < i; j++) {
+			if (program.bindings.descriptors[j].binding == binding.binding) {
+				return Fail(error, "native descriptor binding numbers are not unique");
+			}
+		}
+		seen[kind] = true;
+	}
+	for (size_t i = 0; i < KindCount; i++) {
+		if (present[i] != seen[i]) {
+			return Fail(error, "native shader plan is missing a required descriptor group");
+		}
+	}
+	const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
+	if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
+	    program.bindings.buffer_offset_count != program.info.buffers.size() ||
+	    (!has_user_storage &&
+	     program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
+	    (has_user_storage && program.bindings.push_constant_size != 0) ||
+	    !std::is_sorted(program.bindings.user_data_registers.begin(),
+	                    program.bindings.user_data_registers.end()) ||
+	    std::adjacent_find(program.bindings.user_data_registers.begin(),
+	                       program.bindings.user_data_registers.end()) !=
+	        program.bindings.user_data_registers.end()) {
+		return Fail(error, "native user-data layout is inconsistent");
+	}
+
+	if (program.values == nullptr) {
+		return Fail(error, "native shader plan has no typed SSA");
+	}
+	const auto planning_only_handle = [&](const IR::Inst& handle) {
+		return !handle.Uses().empty() &&
+		       std::ranges::all_of(handle.Uses(), [&](const IR::Use& use) {
+			       const auto op = use.user->GetOpcode();
+			       if (op != IR::ValueOpcode::LoadAddressU32 &&
+			           op != IR::ValueOpcode::ReadConstBuffer) {
+				       return false;
+			       }
+			       const auto index = use.user->Flags<IR::MemoryFlags>().index;
+			       return index < program.values->memory_info.size() &&
+			              program.values->memory_info[index].planning_only;
+		       });
+	};
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			const auto dense = inst.Flags<uint32_t>();
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::GetBufferResource:
+					if (planning_only_handle(inst)) {
+						break;
+					}
+					if (dense >= program.info.buffers.size()) {
+						return Fail(error, "typed buffer handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::GetAddressResource:
+					if (planning_only_handle(inst)) {
+						break;
+					}
+					if (dense >= program.info.addresses.size()) {
+						return Fail(error, "typed address handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::GetImageResource:
+					if (dense >= program.info.images.size()) {
+						return Fail(error, "typed image handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::GetSamplerResource:
+					if (dense >= program.info.samplers.size()) {
+						return Fail(error, "typed sampler handle has an invalid dense resource");
+					}
+					break;
+				case IR::ValueOpcode::ReadConst: {
+					const auto slot = inst.Arg(1).Resolve();
+					if (!slot.IsImmediate() || slot.GetType() != IR::Type::U32 ||
+					    slot.U32() >= program.values->srt_reads.size()) {
+						return Fail(error, "flattened SRT read has an invalid dense slot");
+					}
+					break;
+				}
+				default: break;
+			}
+		}
+	}
+	return true;
+}
+
+} // namespace
+
+static constexpr size_t InitialEmitterVectorReserve = 4096;
+
+static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
+                                  size_t size) {
+	if (size <= InitialEmitterVectorReserve) {
+		return;
+	}
+	std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
+	            " stage=%u vector=%s size=%zu reserve=%zu\n",
+	            program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
+	            InitialEmitterVectorReserve);
+}
+
+void CollectValueRegisterMirrors(const IR::ValueProgram&                program,
+                                 std::vector<Emitter::RegisterBinding>& registers) {
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::SetScalarRegister:
+				case IR::ValueOpcode::SetThreadBitScalarRegister:
+					if (block != program.blocks.front()) {
+						Emitter::CollectRegister(
+						    registers,
+						    {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())});
+					}
+					break;
+				case IR::ValueOpcode::SetVectorRegister:
+					Emitter::CollectRegister(
+					    registers,
+					    {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())});
+					break;
+				case IR::ValueOpcode::SetM0:
+					if (block != program.blocks.front()) {
+						Emitter::CollectRegister(registers, {IR::RegisterFile::M0, 0});
+					}
+					break;
+				case IR::ValueOpcode::SetExecLo:
+				case IR::ValueOpcode::SetExecHi:
+					if (block != program.blocks.front()) {
+						Emitter::CollectRegister(
+						    registers, {IR::RegisterFile::Exec,
+						                inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u});
+					}
+					break;
+				case IR::ValueOpcode::SetVccLo:
+				case IR::ValueOpcode::SetVccHi:
+					if (block != program.blocks.front()) {
+						Emitter::CollectRegister(
+						    registers, {IR::RegisterFile::Vcc,
+						                inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u});
+					}
+					break;
+				default: break;
+			}
+		}
+	}
+}
+
+bool ValueProgramHasOpcode(const IR::ValueProgram& program, IR::ValueOpcode opcode) {
+	return std::ranges::any_of(program.blocks, [opcode](const IR::Block* block) {
+		return std::ranges::any_of(
+		    *block, [opcode](const IR::Inst& inst) { return inst.GetOpcode() == opcode; });
+	});
+}
+
+bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
+	if (program.values == nullptr) {
+		return false;
+	}
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::Ballot:
+				case IR::ValueOpcode::DppMoveU32:
+				case IR::ValueOpcode::DppUpdateU32:
+				case IR::ValueOpcode::WqmMask:
+				case IR::ValueOpcode::ReadFirstLane:
+				case IR::ValueOpcode::ReadLane:
+				case IR::ValueOpcode::WriteLane:
+				case IR::ValueOpcode::Permlane16U32:
+				case IR::ValueOpcode::SwizzleU32:
+				case IR::ValueOpcode::DataAppend:
+				case IR::ValueOpcode::DataConsume:
+				case IR::ValueOpcode::GdsDataAppend:
+				case IR::ValueOpcode::GdsDataConsume: return true;
+				default: break;
+			}
+		}
+	}
+	return false;
+}
+
+bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
+                 ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv,
+                 std::string* error) {
+	using namespace Emitter;
+
+	if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
+	    program.stage != ShaderType::Pixel) {
+		SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
+		return false;
+	}
+	if (!program.srt_plan_complete || !program.resource_tracking_complete ||
+	    !program.shader_info_complete || !program.binding_layout_complete) {
+		SetError(error, "SPIR-V emitter requires a fully planned native shader program");
+		return false;
+	}
+	if (!IR::ValidateResourceSnapshot(program, resources, error)) {
+		return false;
+	}
+	if (!IR::ValidateResourceSpecialization(program, resources, error)) {
+		return false;
+	}
+	if (!ValidateNativeProgram(program, error)) {
+		return false;
+	}
+	if (program.values == nullptr) {
+		SetError(error, "SPIR-V emitter requires planned typed SSA");
+		return false;
+	}
+	const auto& value_program = *program.values;
+	if (!IR::ValidateValueProgram(value_program, true, error)) {
+		return false;
+	}
+	EmitterState state(program, resources, input_info);
+	state.stage                = program.stage;
+	state.wave_size            = program.wave_size;
+	state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
+	state.exact_subgroup_operations =
+	    !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
+	state.registers.reserve(InitialEmitterVectorReserve);
+	state.inputs.reserve(InitialEmitterVectorReserve);
+	state.outputs.reserve(InitialEmitterVectorReserve);
+	state.interface_variables.reserve(InitialEmitterVectorReserve);
+	state.reachable_blocks.reserve(InitialEmitterVectorReserve);
+	CollectValueRegisterMirrors(value_program, state.registers);
+	CopyProgramInputsAndOutputs(state, program);
+	state.needs_subgroup_ballot  = ProgramNeedsSubgroupBallot(program) ||
+	                               (!state.per_invocation_masks &&
+	                                ValueProgramHasOpcode(value_program, IR::ValueOpcode::Ballot));
+	state.needs_subgroup_shuffle = ProgramNeedsSubgroupShuffle(program);
+	state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
+	state.needs_compute_derivatives          = ProgramNeedsComputeDerivatives(program);
+	state.needs_image_gather_extended        = ProgramNeedsImageGatherExtended(program);
+	state.needs_function_lds                 = ProgramNeedsFunctionLds(program);
+	state.needs_pixel_valid_mask             = ProgramNeedsPixelValidMask(program);
+	AllocateInputVariables(state);
+	AllocateOutputVariables(state);
+	AllocateDescriptorVariables(state);
+	EmitHeaderAndTypes(state);
+	AllocateRegisterVariables(state);
+	if (!EmitValueProgram(state, value_program, error)) {
+		return false;
+	}
+
+	ReportReserveExceeded(program, "registers", state.registers.size());
+	ReportReserveExceeded(program, "inputs", state.inputs.size());
+	ReportReserveExceeded(program, "outputs", state.outputs.size());
+	ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
+	ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
+
+	auto binary = state.builder.Build();
+	if (binary.empty()) {
+		SetError(error, "SPIR-V builder returned an empty module");
+		return false;
+	}
+	spirv = std::move(binary);
+	return true;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/backend/spirv/SpirvEmitter.h

@@ -3,7 +3,7 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/ir/ResourceMaterialization.h"
+#include "graphics/shader/recompiler/ir/passes/ResourceMaterialization.h"
 
 #include <vector>
 
@@ -12,10 +12,7 @@ namespace Libs::Graphics::ShaderRecompiler::Spirv {
 bool ProgramRequiresExactSubgroupSize(const IR::Program& program);
 
 bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
-                 const ShaderVertexInputInfo*  vertex_input_info,
-                 const ShaderPixelInputInfo*   pixel_input_info,
-                 const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
-                 std::string* error);
+                 ShaderStageInputInfo input_info, std::vector<uint32_t>& spirv, std::string* error);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Spirv
 

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAlu.cpp

@@ -0,0 +1,677 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <array>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+struct Pair {
+	uint32_t low  = 0;
+	uint32_t high = 0;
+};
+
+uint32_t NewUnary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, value});
+	return result;
+}
+
+uint32_t NewBinary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs,
+                   uint32_t rhs) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, lhs, rhs});
+	return result;
+}
+
+uint32_t NewSelect(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
+                   uint32_t false_value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
+	return result;
+}
+
+Pair ExtractPair(EmitterState& state, uint32_t value) {
+	Pair result {state.builder.AllocateId(), state.builder.AllocateId()};
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.low, value, 0});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, result.high, value, 1});
+	return result;
+}
+
+uint32_t MakePair(EmitterState& state, uint32_t low, uint32_t high) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeConstruct, state.uint_pair_type, result, low, high});
+	return result;
+}
+
+uint32_t Compare64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value,
+                   uint32_t high_compare, uint32_t low_compare, bool equal_compare = false,
+                   bool not_equal = false) {
+	const auto lhs        = ExtractPair(state, lhs_value);
+	const auto rhs        = ExtractPair(state, rhs_value);
+	const auto high_equal = NewBinary(state, OpIEqual, state.bool_type, lhs.high, rhs.high);
+	const auto low_equal  = NewBinary(state, OpIEqual, state.bool_type, lhs.low, rhs.low);
+	if (equal_compare) {
+		return NewBinary(
+		    state, not_equal ? OpLogicalOr : OpLogicalAnd, state.bool_type,
+		    not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, high_equal) : high_equal,
+		    not_equal ? NewUnary(state, OpLogicalNot, state.bool_type, low_equal) : low_equal);
+	}
+	const auto high_result = NewBinary(state, high_compare, state.bool_type, lhs.high, rhs.high);
+	const auto low_result  = NewBinary(state, low_compare, state.bool_type, lhs.low, rhs.low);
+	const auto low_path = NewBinary(state, OpLogicalAnd, state.bool_type, high_equal, low_result);
+	return NewBinary(state, OpLogicalOr, state.bool_type, high_result, low_path);
+}
+
+uint32_t EmitAdd64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
+	const auto lhs   = ExtractPair(state, lhs_value);
+	const auto rhs   = ExtractPair(state, rhs_value);
+	const auto carry = EmitAddCarryValues(state, lhs.low, rhs.low, ConstantU32(state, 0));
+	const auto high0 = NewBinary(state, OpIAdd, state.uint_type, lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpIAdd, state.uint_type, high0, carry.carry);
+	return MakePair(state, carry.sum, high);
+}
+
+uint32_t EmitSub64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
+	const auto lhs    = ExtractPair(state, lhs_value);
+	const auto rhs    = ExtractPair(state, rhs_value);
+	const auto low    = NewBinary(state, OpISub, state.uint_type, lhs.low, rhs.low);
+	const auto borrow = NewBinary(state, OpULessThan, state.bool_type, lhs.low, rhs.low);
+	const auto borrow_u32 =
+	    NewSelect(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0));
+	const auto high0 = NewBinary(state, OpISub, state.uint_type, lhs.high, rhs.high);
+	const auto high  = NewBinary(state, OpISub, state.uint_type, high0, borrow_u32);
+	return MakePair(state, low, high);
+}
+
+uint32_t EmitMulHigh(EmitterState& state, uint32_t lhs, uint32_t rhs, bool signed_value) {
+	const auto operand_type = signed_value ? state.int_type : state.uint_type;
+	const auto pair_type    = signed_value ? state.int_pair_type : state.uint_pair_type;
+	uint32_t   lhs_operand  = lhs;
+	uint32_t   rhs_operand  = rhs;
+	if (signed_value) {
+		lhs_operand = NewUnary(state, OpBitcast, state.int_type, lhs);
+		rhs_operand = NewUnary(state, OpBitcast, state.int_type, rhs);
+	}
+	const auto extended = state.builder.AllocateId();
+	state.builder.AddFunction({signed_value ? OpSMulExtended : OpUMulExtended, pair_type, extended,
+	                           lhs_operand, rhs_operand});
+	const auto high = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, operand_type, high, extended, 1});
+	return signed_value ? NewUnary(state, OpBitcast, state.uint_type, high) : high;
+}
+
+uint32_t EmitMul64(EmitterState& state, uint32_t lhs_value, uint32_t rhs_value) {
+	const auto lhs   = ExtractPair(state, lhs_value);
+	const auto rhs   = ExtractPair(state, rhs_value);
+	const auto low   = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.low);
+	const auto high0 = EmitMulHigh(state, lhs.low, rhs.low, false);
+	const auto high1 = NewBinary(state, OpIMul, state.uint_type, lhs.low, rhs.high);
+	const auto high2 = NewBinary(state, OpIMul, state.uint_type, lhs.high, rhs.low);
+	return MakePair(state, low,
+	                NewBinary(state, OpIAdd, state.uint_type,
+	                          NewBinary(state, OpIAdd, state.uint_type, high0, high1), high2));
+}
+
+uint32_t EmitBitwise64(EmitterState& state, uint32_t opcode, uint32_t lhs_value,
+                       uint32_t rhs_value) {
+	const auto lhs = ExtractPair(state, lhs_value);
+	const auto rhs = ExtractPair(state, rhs_value);
+	return MakePair(state, NewBinary(state, opcode, state.uint_type, lhs.low, rhs.low),
+	                NewBinary(state, opcode, state.uint_type, lhs.high, rhs.high));
+}
+
+uint32_t EmitShift64(EmitterState& state, uint32_t opcode, uint32_t value, uint32_t shift) {
+	const auto pair = ExtractPair(state, value);
+	uint32_t   low  = 0;
+	uint32_t   high = 0;
+	if (opcode == OpShiftLeftLogical) {
+		EmitShiftLeftLogicalU64Values(state, pair.low, pair.high, shift, low, high);
+	} else if (opcode == OpShiftRightLogical) {
+		EmitShiftRightLogicalU64Values(state, pair.low, pair.high, shift, low, high);
+	} else {
+		const auto logical      = EmitShift64(state, OpShiftRightLogical, value, shift);
+		const auto logical_pair = ExtractPair(state, logical);
+		const auto sign =
+		    NewBinary(state, OpSLessThan, state.bool_type, pair.high, ConstantU32(state, 0));
+		const auto inverse =
+		    NewBinary(state, OpISub, state.uint_type, ConstantU32(state, 64), shift);
+		const auto fill = NewBinary(state, OpShiftLeftLogical, state.uint_type,
+		                            ConstantU32(state, 0xffffffffu), inverse);
+		const auto high_shift =
+		    NewBinary(state, OpISub, state.uint_type, shift, ConstantU32(state, 32));
+		const auto high_fill = NewSelect(
+		    state, state.uint_type,
+		    NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift, ConstantU32(state, 32)),
+		    ConstantU32(state, 0xffffffffu), fill);
+		const auto fill_low =
+		    NewSelect(state, state.uint_type, sign,
+		              NewSelect(state, state.uint_type,
+		                        NewBinary(state, OpUGreaterThanEqual, state.bool_type, shift,
+		                                  ConstantU32(state, 32)),
+		                        NewBinary(state, OpShiftLeftLogical, state.uint_type,
+		                                  ConstantU32(state, 0xffffffffu), high_shift),
+		                        ConstantU32(state, 0)),
+		              ConstantU32(state, 0));
+		const auto fill_high =
+		    NewSelect(state, state.uint_type, sign, high_fill, ConstantU32(state, 0));
+		return MakePair(
+		    state, NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.low, fill_low),
+		    NewBinary(state, OpBitwiseOr, state.uint_type, logical_pair.high, fill_high));
+	}
+	return MakePair(state, low, high);
+}
+
+uint32_t EmitFindMsb64(EmitterState& state, uint32_t value) {
+	const auto pair   = ExtractPair(state, value);
+	const auto high_i = state.builder.AllocateId();
+	const auto low_i  = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, state.int_type, high_i, state.glsl_std450, GlslFindUMsb, pair.high});
+	state.builder.AddFunction(
+	    {OpExtInst, state.int_type, low_i, state.glsl_std450, GlslFindUMsb, pair.low});
+	const auto high = NewUnary(state, OpBitcast, state.uint_type, high_i);
+	const auto low  = NewUnary(state, OpBitcast, state.uint_type, low_i);
+	const auto high_nonzero =
+	    NewBinary(state, OpINotEqual, state.bool_type, pair.high, ConstantU32(state, 0));
+	return NewSelect(state, state.uint_type, high_nonzero,
+	                 NewBinary(state, OpIAdd, state.uint_type, high, ConstantU32(state, 32)), low);
+}
+
+uint32_t EmitMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value,
+                     bool max_value) {
+	const auto ab = signed_value ? EmitMinMaxI32Value(state, a, b, max_value)
+	                             : EmitMinMaxU32Value(state, a, b, max_value);
+	return signed_value ? EmitMinMaxI32Value(state, ab, c, max_value)
+	                    : EmitMinMaxU32Value(state, ab, c, max_value);
+}
+
+uint32_t EmitMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool signed_value) {
+	const auto minimum = EmitMinMax3(state, a, b, c, signed_value, false);
+	const auto maximum = EmitMinMax3(state, a, b, c, signed_value, true);
+	const auto ab      = NewBinary(state, OpIAdd, state.uint_type, a, b);
+	const auto abc     = NewBinary(state, OpIAdd, state.uint_type, ab, c);
+	return NewBinary(state, OpISub, state.uint_type,
+	                 NewBinary(state, OpISub, state.uint_type, abc, minimum), maximum);
+}
+
+uint32_t EmitFMinMax3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c, bool max_value) {
+	return EmitMinMaxF32Value(state, EmitMinMaxF32Value(state, a, b, max_value), c, max_value);
+}
+
+uint32_t EmitFMed3(EmitterState& state, uint32_t a, uint32_t b, uint32_t c) {
+	const auto min_ab   = EmitMinMaxF32Value(state, a, b, false);
+	const auto min3     = EmitMinMaxF32Value(state, min_ab, c, false);
+	const auto max_ab   = EmitMinMaxF32Value(state, a, b, true);
+	const auto high_min = EmitMinMaxF32Value(state, max_ab, c, false);
+	const auto median   = EmitMinMaxF32Value(state, min_ab, high_min, true);
+	const auto nan_ab   = NewBinary(state, OpLogicalOr, state.bool_type,
+	                                EmitClassifyF32(state, a).nan, EmitClassifyF32(state, b).nan);
+	const auto any_nan =
+	    NewBinary(state, OpLogicalOr, state.bool_type, nan_ab, EmitClassifyF32(state, c).nan);
+	return NewSelect(state, state.float_type, any_nan, min3, median);
+}
+
+uint32_t EmitExt(EmitterState& state, uint32_t type, uint32_t opcode,
+                 std::initializer_list<uint32_t> args) {
+	const auto            result = state.builder.AllocateId();
+	std::vector<uint32_t> words {OpExtInst, type, result, state.glsl_std450, opcode};
+	words.insert(words.end(), args.begin(), args.end());
+	state.builder.AddFunction(words);
+	return result;
+}
+
+uint32_t EmitF32ToU32(EmitterState& state, uint32_t src, bool signed_value) {
+	const auto trunc         = EmitTruncF32Value(state, src);
+	const auto converted_raw = state.builder.AllocateId();
+	if (signed_value) {
+		const auto converted_i = state.builder.AllocateId();
+		state.builder.AddFunction({OpConvertFToS, state.int_type, converted_i, trunc});
+		state.builder.AddFunction({OpBitcast, state.uint_type, converted_raw, converted_i});
+	} else {
+		state.builder.AddFunction({OpConvertFToU, state.uint_type, converted_raw, trunc});
+	}
+	const auto nan = EmitClassifyF32(state, src).nan;
+	if (signed_value) {
+		const auto below = NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src,
+		                             ConstantF32(state, 0xcf000000u));
+		const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
+		                             ConstantF32(state, 0x4f000000u));
+		const auto high  = NewSelect(state, state.uint_type, above, ConstantU32(state, 0x7fffffffu),
+		                             converted_raw);
+		const auto low =
+		    NewSelect(state, state.uint_type, below, ConstantU32(state, 0x80000000u), high);
+		return NewSelect(state, state.uint_type, nan, ConstantU32(state, 0), low);
+	}
+	const auto below =
+	    NewBinary(state, OpFOrdLessThanEqual, state.bool_type, src, ConstantF32(state, 0));
+	const auto above = NewBinary(state, OpFOrdGreaterThanEqual, state.bool_type, src,
+	                             ConstantF32(state, 0x4f800000u));
+	const auto zero  = NewBinary(state, OpLogicalOr, state.bool_type, nan, below);
+	const auto high =
+	    NewSelect(state, state.uint_type, above, ConstantU32(state, 0xffffffffu), converted_raw);
+	return NewSelect(state, state.uint_type, zero, ConstantU32(state, 0), high);
+}
+
+uint32_t EmitPackHalf(EmitterState& state, uint32_t src) {
+	return EmitExt(state, state.uint_type, GlslPackHalf2x16, {src});
+}
+
+uint32_t EmitUnpackHalf(EmitterState& state, uint32_t bits) {
+	const auto pair   = EmitExt(state, state.vec2_float_type, GlslUnpackHalf2x16, {bits});
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, state.float_type, result, pair, 0});
+	return result;
+}
+
+} // namespace
+
+bool EmitValueAlu(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state = ctx.state;
+	const auto op    = inst.GetOpcode();
+	const auto unary = [&](uint32_t spirv, IR::Type type) {
+		ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0)});
+		return true;
+	};
+	const auto binary = [&](uint32_t spirv, IR::Type type) {
+		ctx.Emit(inst, spirv, type, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+		return true;
+	};
+	const auto ext_unary = [&](uint32_t ext) {
+		ctx.Define(inst, EmitExt(state, state.float_type, ext, {ctx.Arg(inst, 0)}));
+		return true;
+	};
+	switch (op) {
+		case IR::ValueOpcode::BitCastU16F16:
+		case IR::ValueOpcode::BitCastF16U16:
+		case IR::ValueOpcode::ConvertU32U16:
+		case IR::ValueOpcode::ConvertU32U8:
+		case IR::ValueOpcode::PackUint2x32:
+		case IR::ValueOpcode::UnpackUint2x32: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
+		case IR::ValueOpcode::BitCastU32F32: return unary(OpBitcast, IR::Type::U32);
+		case IR::ValueOpcode::BitCastF32U32: return unary(OpBitcast, IR::Type::F32);
+		case IR::ValueOpcode::ConvertU16U32:
+			ctx.Emit(inst, OpBitwiseAnd, IR::Type::U16,
+			         {ctx.Arg(inst, 0), ConstantU32(state, 0xffffu)});
+			return true;
+		case IR::ValueOpcode::ConvertU8U32:
+			ctx.Emit(inst, OpBitwiseAnd, IR::Type::U8,
+			         {ctx.Arg(inst, 0), ConstantU32(state, 0xffu)});
+			return true;
+		case IR::ValueOpcode::ConvertF16F32: {
+			const auto pair = state.builder.AllocateId();
+			state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, pair,
+			                           ctx.Arg(inst, 0), ConstantF32(state, 0)});
+			ctx.Define(inst, EmitPackHalf(state, pair));
+			return true;
+		}
+		case IR::ValueOpcode::ConvertF32F16:
+			ctx.Define(inst, EmitUnpackHalf(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::ConvertS32F32:
+			ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), true));
+			return true;
+		case IR::ValueOpcode::ConvertU32F32:
+			ctx.Define(inst, EmitF32ToU32(state, ctx.Arg(inst, 0), false));
+			return true;
+		case IR::ValueOpcode::ConvertF32S32: {
+			const auto signed_value = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0));
+			ctx.Emit(inst, OpConvertSToF, IR::Type::F32, {signed_value});
+			return true;
+		}
+		case IR::ValueOpcode::ConvertF32U32: return unary(OpConvertUToF, IR::Type::F32);
+		case IR::ValueOpcode::CompositeConstructU32x2:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x2,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::CompositeConstructF32x2:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::F32x2,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::CompositeConstructU32x4:
+			ctx.Emit(inst, OpCompositeConstruct, IR::Type::U32x4,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
+			return true;
+		case IR::ValueOpcode::CompositeExtractU32x2:
+		case IR::ValueOpcode::CompositeExtractU32x4:
+			ctx.Emit(inst, OpCompositeExtract, IR::Type::U32,
+			         {ctx.Arg(inst, 0), inst.Arg(1).U32()});
+			return true;
+		case IR::ValueOpcode::PackHalf2x16:
+			ctx.Define(inst, EmitExt(state, state.uint_type, GlslPackHalf2x16, {ctx.Arg(inst, 0)}));
+			return true;
+		case IR::ValueOpcode::PackSnorm2x16:
+			ctx.Define(inst,
+			           EmitExt(state, state.uint_type, GlslPackSnorm2x16, {ctx.Arg(inst, 0)}));
+			return true;
+		case IR::ValueOpcode::PackUnorm2x16:
+			ctx.Define(inst,
+			           EmitExt(state, state.uint_type, GlslPackUnorm2x16, {ctx.Arg(inst, 0)}));
+			return true;
+		case IR::ValueOpcode::PackFloat2x16Rtz: {
+			const auto low = EmitF32ToF16RtzBits(state, ctx.Arg(inst, 0));
+			const auto high =
+			    NewBinary(state, OpShiftLeftLogical, state.uint_type,
+			              EmitF32ToF16RtzBits(state, ctx.Arg(inst, 1)), ConstantU32(state, 16));
+			ctx.Define(inst, NewBinary(state, OpBitwiseOr, state.uint_type, low, high));
+			return true;
+		}
+		case IR::ValueOpcode::FPAbs32:
+			ctx.Define(inst, EmitFAbsValue(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FPNeg32:
+			ctx.Define(inst, EmitFNegateValue(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FPSaturate32:
+			ctx.Define(inst, EmitExt(state, state.float_type, GlslFClamp,
+			                         {ctx.Arg(inst, 0), ConstantF32(state, 0),
+			                          ConstantF32(state, 0x3f800000u)}));
+			return true;
+		case IR::ValueOpcode::BitFieldInsert:
+			ctx.Emit(inst, OpBitFieldInsert, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)});
+			return true;
+		case IR::ValueOpcode::BitFieldUExtract:
+			ctx.Emit(inst, OpBitFieldUExtract, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::BitFieldSExtract:
+			ctx.Emit(inst, OpBitFieldSExtract, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::SelectU1:
+			ctx.Emit(inst, OpSelect, IR::Type::U1,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::SelectU32:
+			ctx.Emit(inst, OpSelect, IR::Type::U32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::SelectF32:
+			ctx.Emit(inst, OpSelect, IR::Type::F32,
+			         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)});
+			return true;
+		case IR::ValueOpcode::IAdd32: return binary(OpIAdd, IR::Type::U32);
+		case IR::ValueOpcode::ISub32: return binary(OpISub, IR::Type::U32);
+		case IR::ValueOpcode::IMul32: return binary(OpIMul, IR::Type::U32);
+		case IR::ValueOpcode::UDiv32: return binary(OpUDiv, IR::Type::U32);
+		case IR::ValueOpcode::IAdd64:
+			ctx.Define(inst, EmitAdd64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::ISub64:
+			ctx.Define(inst, EmitSub64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::IMul64:
+			ctx.Define(inst, EmitMul64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::IAddCarry32:
+			ctx.Emit(inst, OpIAddCarry, IR::Type::U32x2, {ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::SMulHi:
+			ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::UMulHi:
+			ctx.Define(inst, EmitMulHigh(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::IAbs32: {
+			const auto value = ctx.Arg(inst, 0);
+			const auto neg   = NewUnary(state, OpSNegate, state.uint_type, value);
+			const auto negative =
+			    NewBinary(state, OpSLessThan, state.bool_type, value, ConstantU32(state, 0));
+			ctx.Define(inst, NewSelect(state, state.uint_type, negative, neg, value));
+			return true;
+		}
+		case IR::ValueOpcode::ShiftLeftLogical32: return binary(OpShiftLeftLogical, IR::Type::U32);
+		case IR::ValueOpcode::ShiftRightLogical32:
+			return binary(OpShiftRightLogical, IR::Type::U32);
+		case IR::ValueOpcode::ShiftRightArithmetic32:
+			return binary(OpShiftRightArithmetic, IR::Type::U32);
+		case IR::ValueOpcode::ShiftLeftLogical64:
+			ctx.Define(inst,
+			           EmitShift64(state, OpShiftLeftLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::ShiftRightLogical64:
+			ctx.Define(inst,
+			           EmitShift64(state, OpShiftRightLogical, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::ShiftRightArithmetic64:
+			ctx.Define(inst, EmitShift64(state, OpShiftRightArithmetic, ctx.Arg(inst, 0),
+			                             ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::BitwiseAnd32: return binary(OpBitwiseAnd, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseOr32: return binary(OpBitwiseOr, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseXor32: return binary(OpBitwiseXor, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseNot32: return unary(OpNot, IR::Type::U32);
+		case IR::ValueOpcode::BitwiseAnd64:
+			ctx.Define(inst,
+			           EmitBitwise64(state, OpBitwiseAnd, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::BitwiseOr64:
+			ctx.Define(inst, EmitBitwise64(state, OpBitwiseOr, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::BitReverse32: return unary(OpBitReverse, IR::Type::U32);
+		case IR::ValueOpcode::BitCount32: return unary(OpBitCount, IR::Type::U32);
+		case IR::ValueOpcode::BitCount64: {
+			const auto pair = ExtractPair(state, ctx.Arg(inst, 0));
+			ctx.Define(inst, NewBinary(state, OpIAdd, state.uint_type,
+			                           NewUnary(state, OpBitCount, state.uint_type, pair.low),
+			                           NewUnary(state, OpBitCount, state.uint_type, pair.high)));
+			return true;
+		}
+		case IR::ValueOpcode::FindILsb32: {
+			const auto value = EmitExt(state, state.int_type, GlslFindILsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			return true;
+		}
+		case IR::ValueOpcode::FindUMsb32: {
+			const auto value = EmitExt(state, state.int_type, GlslFindUMsb, {ctx.Arg(inst, 0)});
+			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			return true;
+		}
+		case IR::ValueOpcode::FindUMsb64:
+			ctx.Define(inst, EmitFindMsb64(state, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FindSMsb32: {
+			const auto value =
+			    EmitExt(state, state.int_type, 74u,
+			            {NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 0))});
+			ctx.Define(inst, NewUnary(state, OpBitcast, state.uint_type, value));
+			return true;
+		}
+		case IR::ValueOpcode::SMin32:
+			ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::SMax32:
+			ctx.Define(inst, EmitMinMaxI32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::UMin32:
+			ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::UMax32:
+			ctx.Define(inst, EmitMinMaxU32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::SMinTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), true, false));
+			return true;
+		case IR::ValueOpcode::SMaxTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), true, true));
+			return true;
+		case IR::ValueOpcode::UMinTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), false, false));
+			return true;
+		case IR::ValueOpcode::UMaxTri32:
+			ctx.Define(inst, EmitMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                             ctx.Arg(inst, 2), false, true));
+			return true;
+		case IR::ValueOpcode::SMedTri32:
+			ctx.Define(inst,
+			           EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), true));
+			return true;
+		case IR::ValueOpcode::UMedTri32:
+			ctx.Define(
+			    inst, EmitMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2), false));
+			return true;
+		case IR::ValueOpcode::SLessThan32: return binary(OpSLessThan, IR::Type::U1);
+		case IR::ValueOpcode::ULessThan32: return binary(OpULessThan, IR::Type::U1);
+		case IR::ValueOpcode::IEqual32: return binary(OpIEqual, IR::Type::U1);
+		case IR::ValueOpcode::SLessThanEqual32: return binary(OpSLessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::ULessThanEqual32: return binary(OpULessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::SGreaterThan32: return binary(OpSGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::UGreaterThan32: return binary(OpUGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::INotEqual32: return binary(OpINotEqual, IR::Type::U1);
+		case IR::ValueOpcode::SGreaterThanEqual32: return binary(OpSGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::UGreaterThanEqual32: return binary(OpUGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::IEqual64:
+			ctx.Define(inst,
+			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, false));
+			return true;
+		case IR::ValueOpcode::INotEqual64:
+			ctx.Define(inst,
+			           Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), 0, 0, true, true));
+			return true;
+		case IR::ValueOpcode::ULessThan64:
+			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan,
+			                           OpULessThan));
+			return true;
+		case IR::ValueOpcode::SLessThan64:
+			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan,
+			                           OpULessThan));
+			return true;
+		case IR::ValueOpcode::UGreaterThan64:
+			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpUGreaterThan,
+			                           OpUGreaterThan));
+			return true;
+		case IR::ValueOpcode::SGreaterThan64:
+			ctx.Define(inst, Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSGreaterThan,
+			                           OpUGreaterThan));
+			return true;
+		case IR::ValueOpcode::ULessThanEqual64: {
+			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                               OpUGreaterThan, OpUGreaterThan);
+			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
+			return true;
+		}
+		case IR::ValueOpcode::SLessThanEqual64: {
+			const auto greater = Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                               OpSGreaterThan, OpUGreaterThan);
+			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, greater));
+			return true;
+		}
+		case IR::ValueOpcode::UGreaterThanEqual64: {
+			const auto less =
+			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpULessThan, OpULessThan);
+			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
+			return true;
+		}
+		case IR::ValueOpcode::SGreaterThanEqual64: {
+			const auto less =
+			    Compare64(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), OpSLessThan, OpULessThan);
+			ctx.Define(inst, NewUnary(state, OpLogicalNot, state.bool_type, less));
+			return true;
+		}
+		case IR::ValueOpcode::LogicalOr: return binary(OpLogicalOr, IR::Type::U1);
+		case IR::ValueOpcode::LogicalAnd: return binary(OpLogicalAnd, IR::Type::U1);
+		case IR::ValueOpcode::LogicalXor: return binary(OpLogicalNotEqual, IR::Type::U1);
+		case IR::ValueOpcode::LogicalNot: return unary(OpLogicalNot, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdEqual32: return binary(OpFOrdEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordEqual32: return binary(OpFUnordEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdNotEqual32: return binary(OpFOrdNotEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordNotEqual32: return binary(OpFUnordNotEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdLessThan32: return binary(OpFOrdLessThan, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordLessThan32: return binary(OpFUnordLessThan, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdGreaterThan32: return binary(OpFOrdGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordGreaterThan32:
+			return binary(OpFUnordGreaterThan, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdLessThanEqual32:
+			return binary(OpFOrdLessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordLessThanEqual32:
+			return binary(OpFUnordLessThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPOrdGreaterThanEqual32:
+			return binary(OpFOrdGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPUnordGreaterThanEqual32:
+			return binary(OpFUnordGreaterThanEqual, IR::Type::U1);
+		case IR::ValueOpcode::FPIsNan32:
+			ctx.Emit(inst, OpFUnordNotEqual, IR::Type::U1, {ctx.Arg(inst, 0), ctx.Arg(inst, 0)});
+			return true;
+		case IR::ValueOpcode::FPCmpClass32:
+			ctx.Define(inst, EmitClassMaskF32(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::FPAdd32: return binary(OpFAdd, IR::Type::F32);
+		case IR::ValueOpcode::FPSub32: return binary(OpFSub, IR::Type::F32);
+		case IR::ValueOpcode::FPMul32: return binary(OpFMul, IR::Type::F32);
+		case IR::ValueOpcode::FPFma32:
+			ctx.Define(inst, EmitExt(state, state.float_type, GlslFma,
+			                         {ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)}));
+			return true;
+		case IR::ValueOpcode::FPMin32:
+			ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), false));
+			return true;
+		case IR::ValueOpcode::FPMax32:
+			ctx.Define(inst, EmitMinMaxF32Value(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), true));
+			return true;
+		case IR::ValueOpcode::FPMinTri32:
+			ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                              ctx.Arg(inst, 2), false));
+			return true;
+		case IR::ValueOpcode::FPMaxTri32:
+			ctx.Define(inst, EmitFMinMax3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1),
+			                              ctx.Arg(inst, 2), true));
+			return true;
+		case IR::ValueOpcode::FPMedTri32:
+			ctx.Define(inst,
+			           EmitFMed3(state, ctx.Arg(inst, 0), ctx.Arg(inst, 1), ctx.Arg(inst, 2)));
+			return true;
+		case IR::ValueOpcode::FPRecip32: {
+			const auto source = EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0));
+			ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
+			                           ConstantF32(state, 0x3f800000u), source));
+			return true;
+		}
+		case IR::ValueOpcode::FPRecipIFlag32:
+			// Integer-to-float inputs used by IFLAG cannot be denormal.
+			ctx.Define(inst, NewBinary(state, OpFDiv, state.float_type,
+			                           ConstantF32(state, 0x3f800000u), ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::FPRecipSqrt32:
+			ctx.Define(inst, EmitExt(state, state.float_type, GlslInverseSqrt,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPSqrt:
+			ctx.Define(inst, EmitExt(state, state.float_type, GlslSqrt,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPSin:
+		case IR::ValueOpcode::FPCos: {
+			auto source = EmitTrigCycleF32(state, ctx.Arg(inst, 0), op == IR::ValueOpcode::FPSin);
+			source =
+			    NewBinary(state, OpFMul, state.float_type, source, ConstantF32(state, 0x40c90fdbu));
+			ctx.Define(inst, EmitExt(state, state.float_type,
+			                         op == IR::ValueOpcode::FPSin ? GlslSin : GlslCos, {source}));
+			return true;
+		}
+		case IR::ValueOpcode::FPExp2:
+			ctx.Define(inst, EmitExt(state, state.float_type, GlslExp2,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPLog2:
+			ctx.Define(inst, EmitExt(state, state.float_type, GlslLog2,
+			                         {EmitFlushF32DenormToSignedZero(state, ctx.Arg(inst, 0))}));
+			return true;
+		case IR::ValueOpcode::FPLdexp: {
+			const auto exponent = NewUnary(state, OpBitcast, state.int_type, ctx.Arg(inst, 1));
+			ctx.Define(inst,
+			           EmitExt(state, state.float_type, GlslLdexp, {ctx.Arg(inst, 0), exponent}));
+			return true;
+		}
+		case IR::ValueOpcode::FPRoundEven32: return ext_unary(GlslRoundEven);
+		case IR::ValueOpcode::FPFloor32: return ext_unary(GlslFloor);
+		case IR::ValueOpcode::FPCeil32: return ext_unary(GlslCeil);
+		case IR::ValueOpcode::FPTrunc32: return ext_unary(GlslTrunc);
+		case IR::ValueOpcode::FPFract32: return ext_unary(GlslFract);
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAluHelpers.cpp

@@ -0,0 +1,423 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t EmitSelectValueU32(EmitterState& state, uint32_t cond, uint32_t true_value,
+                            uint32_t false_value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, true_value, false_value});
+	return ret;
+}
+
+uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask);
+
+void EmitShiftLeftLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high, uint32_t shift,
+                                   uint32_t& out_low, uint32_t& out_high) {
+	const auto amount           = EmitAndConstant(state, shift, 63u);
+	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
+	const auto less32           = state.builder.AllocateId();
+	const auto non_zero         = state.builder.AllocateId();
+	const auto carry_count_base = state.builder.AllocateId();
+	const auto carry_count      = state.builder.AllocateId();
+	const auto low_part         = state.builder.AllocateId();
+	const auto high_part        = state.builder.AllocateId();
+	const auto carry_part       = state.builder.AllocateId();
+	const auto carry            = state.builder.AllocateId();
+	const auto high_less        = state.builder.AllocateId();
+	const auto high_ge          = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, carry_part, low, carry_count});
+	state.builder.AddFunction(
+	    {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, high_less, high_part, carry});
+	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, high_ge, low, dword_shift});
+	out_low  = EmitSelectValueU32(state, less32, low_part, ConstantU32(state, 0));
+	out_high = EmitSelectValueU32(state, less32, high_less, high_ge);
+}
+
+void EmitShiftRightLogicalU64Values(EmitterState& state, uint32_t low, uint32_t high,
+                                    uint32_t shift, uint32_t& out_low, uint32_t& out_high) {
+	const auto amount           = EmitAndConstant(state, shift, 63u);
+	const auto dword_shift      = EmitAndConstant(state, amount, 31u);
+	const auto less32           = state.builder.AllocateId();
+	const auto non_zero         = state.builder.AllocateId();
+	const auto carry_count_base = state.builder.AllocateId();
+	const auto carry_count      = state.builder.AllocateId();
+	const auto low_part         = state.builder.AllocateId();
+	const auto high_part        = state.builder.AllocateId();
+	const auto carry_part       = state.builder.AllocateId();
+	const auto carry            = state.builder.AllocateId();
+	const auto low_less         = state.builder.AllocateId();
+	const auto low_ge           = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpULessThan, state.bool_type, less32, amount, ConstantU32(state, 32)});
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, non_zero, amount, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpISub, state.uint_type, carry_count_base, ConstantU32(state, 32), dword_shift});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, carry_count, carry_count_base, ConstantU32(state, 31)});
+	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_part, low, dword_shift});
+	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, high_part, high, dword_shift});
+	state.builder.AddFunction({OpShiftLeftLogical, state.uint_type, carry_part, high, carry_count});
+	state.builder.AddFunction(
+	    {OpSelect, state.uint_type, carry, non_zero, carry_part, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, low_less, low_part, carry});
+	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, low_ge, high, dword_shift});
+	out_low  = EmitSelectValueU32(state, less32, low_less, low_ge);
+	out_high = EmitSelectValueU32(state, less32, high_part, ConstantU32(state, 0));
+}
+
+uint32_t EmitAndConstant(EmitterState& state, uint32_t value, uint32_t mask) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, ret, value, ConstantU32(state, mask)});
+	return ret;
+}
+
+AddCarryResult EmitAddCarryValues(EmitterState& state, uint32_t lhs, uint32_t rhs,
+                                  uint32_t carry_in) {
+	const auto pair0  = state.builder.AllocateId();
+	const auto sum0   = state.builder.AllocateId();
+	const auto carry0 = state.builder.AllocateId();
+	const auto pair1  = state.builder.AllocateId();
+	const auto sum1   = state.builder.AllocateId();
+	const auto carry1 = state.builder.AllocateId();
+	const auto carry  = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair0, lhs, rhs});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum0, pair0, 0});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry0, pair0, 1});
+	state.builder.AddFunction({OpIAddCarry, state.uint_pair_type, pair1, sum0, carry_in});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, sum1, pair1, 0});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, carry1, pair1, 1});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, carry, carry0, carry1});
+	return {sum1, carry};
+}
+
+uint32_t EmitShiftRightConstant(EmitterState& state, uint32_t value, uint32_t shift) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, ret, value, ConstantU32(state, shift)});
+	return ret;
+}
+
+uint32_t EmitOrU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+                                uint32_t constant) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	return ret;
+}
+
+uint32_t EmitSubConstantMinusU32(EmitterState& state, uint32_t constant, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpISub, state.uint_type, ret, ConstantU32(state, constant), value});
+	return ret;
+}
+
+uint32_t EmitF32ToF16RtzBits(EmitterState& state, uint32_t f32) {
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.uint_type, bits, f32});
+
+	const auto sign = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 16), 0x8000u);
+	const auto exp  = EmitAndConstant(state, EmitShiftRightConstant(state, bits, 23), 0xffu);
+	const auto mant = EmitAndConstant(state, bits, 0x007fffffu);
+
+	const auto half_exp       = state.builder.AllocateId();
+	const auto normal_exp     = state.builder.AllocateId();
+	const auto normal_mant    = EmitShiftRightConstant(state, mant, 13);
+	const auto normal_payload = state.builder.AllocateId();
+	const auto normal         = state.builder.AllocateId();
+	state.builder.AddFunction({OpISub, state.uint_type, half_exp, exp, ConstantU32(state, 112)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, normal_exp, half_exp, ConstantU32(state, 10)});
+	state.builder.AddFunction(
+	    {OpBitwiseOr, state.uint_type, normal_payload, normal_exp, normal_mant});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, normal, sign, normal_payload});
+
+	const auto mant_with_hidden = EmitOrU32(state, mant, ConstantU32(state, 0x00800000u));
+	const auto raw_sub_shift    = EmitSubConstantMinusU32(state, 126, exp);
+	const auto exp_lt_103       = EmitCompareU32Constant(state, OpULessThan, exp, 103);
+	const auto exp_gt_112       = EmitCompareU32Constant(state, OpUGreaterThan, exp, 112);
+	const auto sub_shift_low =
+	    EmitSelectValueU32(state, exp_lt_103, ConstantU32(state, 31), raw_sub_shift);
+	const auto sub_shift =
+	    EmitSelectValueU32(state, exp_gt_112, ConstantU32(state, 14), sub_shift_low);
+	const auto sub_mant  = state.builder.AllocateId();
+	const auto subnormal = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, sub_mant, mant_with_hidden, sub_shift});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, subnormal, sign, sub_mant});
+
+	const auto nan_payload =
+	    EmitOrU32(state, EmitShiftRightConstant(state, mant, 13), ConstantU32(state, 0x0200u));
+	const auto nan =
+	    EmitOrU32(state, sign, EmitOrU32(state, ConstantU32(state, 0x7c00u), nan_payload));
+	const auto inf        = EmitOrU32(state, sign, ConstantU32(state, 0x7c00u));
+	const auto max_finite = EmitOrU32(state, sign, ConstantU32(state, 0x7bffu));
+	const auto mant_zero  = EmitCompareU32Constant(state, OpIEqual, mant, 0);
+	const auto special    = EmitSelectValueU32(state, mant_zero, inf, nan);
+
+	const auto exp_le_112 = EmitCompareU32Constant(state, OpULessThanEqual, exp, 112);
+	const auto exp_ge_143 = EmitCompareU32Constant(state, OpUGreaterThanEqual, exp, 143);
+	const auto exp_eq_255 = EmitCompareU32Constant(state, OpIEqual, exp, 255);
+	const auto finite0    = EmitSelectValueU32(state, exp_le_112, subnormal, normal);
+	const auto finite1    = EmitSelectValueU32(state, exp_lt_103, sign, finite0);
+	const auto finite2    = EmitSelectValueU32(state, exp_ge_143, max_finite, finite1);
+	return EmitAndConstant(state, EmitSelectValueU32(state, exp_eq_255, special, finite2), 0xffffu);
+}
+
+uint32_t EmitMinMaxU32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto cond = state.builder.AllocateId();
+	const auto ret  = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {max_value ? OpUGreaterThan : OpULessThan, state.bool_type, cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitMinMaxI32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto cond = state.builder.AllocateId();
+	const auto ret  = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {max_value ? OpSGreaterThan : OpSLessThan, state.bool_type, cond, lhs, rhs});
+	state.builder.AddFunction({OpSelect, state.uint_type, ret, cond, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitBitcastF32ToU32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
+	return ret;
+}
+
+uint32_t EmitBitcastU32ToF32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
+	return ret;
+}
+
+uint32_t EmitAndU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitLogicalAndBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitLogicalOrBool(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalOr, state.bool_type, ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitLogicalNotBool(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalNot, state.bool_type, ret, value});
+	return ret;
+}
+
+F32Class EmitClassifyF32(EmitterState& state, uint32_t value) {
+	F32Class cls;
+	cls.bits                 = EmitBitcastF32ToU32(state, value);
+	const auto abs_bits      = EmitAndConstant(state, cls.bits, 0x7fffffffu);
+	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
+	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
+	const auto exponent_max  = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto mantissa_nonzero = EmitCompareU32Constant(state, OpINotEqual, mantissa_bits, 0);
+	const auto quiet_bit_clear =
+	    EmitCompareU32Constant(state, OpIEqual, EmitAndConstant(state, cls.bits, 0x00400000u), 0);
+	cls.nan        = EmitLogicalAndBool(state, exponent_max, mantissa_nonzero);
+	cls.snan       = EmitLogicalAndBool(state, cls.nan, quiet_bit_clear);
+	cls.zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+	cls.quiet_bits = EmitOrU32(state, cls.bits, ConstantU32(state, 0x00400000u));
+	return cls;
+}
+
+uint32_t EmitClassMaskBitMatch(EmitterState& state, uint32_t mask, uint32_t bit,
+                               uint32_t class_match) {
+	const auto selected =
+	    EmitCompareU32Constant(state, OpINotEqual, EmitAndConstant(state, mask, 1u << bit), 0);
+	return EmitLogicalAndBool(state, selected, class_match);
+}
+
+uint32_t EmitClassMaskF32(EmitterState& state, uint32_t value, uint32_t mask) {
+	const auto bits          = EmitBitcastF32ToU32(state, value);
+	const auto sign_bits     = EmitAndConstant(state, bits, 0x80000000u);
+	const auto abs_bits      = EmitAndConstant(state, bits, 0x7fffffffu);
+	const auto exponent_bits = EmitAndConstant(state, abs_bits, 0x7f800000u);
+	const auto mantissa_bits = EmitAndConstant(state, abs_bits, 0x007fffffu);
+	const auto quiet_bits    = EmitAndConstant(state, mantissa_bits, 0x00400000u);
+
+	const auto sign             = EmitCompareU32Constant(state, OpINotEqual, sign_bits, 0);
+	const auto positive         = EmitLogicalNotBool(state, sign);
+	const auto exponent_zero    = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0);
+	const auto exponent_nonzero = EmitLogicalNotBool(state, exponent_zero);
+	const auto exponent_inf = EmitCompareU32Constant(state, OpIEqual, exponent_bits, 0x7f800000u);
+	const auto finite_exponent  = EmitLogicalNotBool(state, exponent_inf);
+	const auto mantissa_zero    = EmitCompareU32Constant(state, OpIEqual, mantissa_bits, 0);
+	const auto mantissa_nonzero = EmitLogicalNotBool(state, mantissa_zero);
+	const auto quiet            = EmitCompareU32Constant(state, OpINotEqual, quiet_bits, 0);
+
+	const auto nan_common = EmitLogicalAndBool(state, exponent_inf, mantissa_nonzero);
+	const auto snan       = EmitLogicalAndBool(state, nan_common, EmitLogicalNotBool(state, quiet));
+	const auto qnan       = EmitLogicalAndBool(state, nan_common, quiet);
+	const auto inf        = EmitLogicalAndBool(state, exponent_inf, mantissa_zero);
+	const auto normal     = EmitLogicalAndBool(state, exponent_nonzero, finite_exponent);
+	const auto denorm     = EmitLogicalAndBool(state, exponent_zero, mantissa_nonzero);
+	const auto zero       = EmitCompareU32Constant(state, OpIEqual, abs_bits, 0);
+
+	uint32_t match = EmitClassMaskBitMatch(state, mask, 0, snan);
+	match          = EmitLogicalOrBool(state, match, EmitClassMaskBitMatch(state, mask, 1, qnan));
+	match          = EmitLogicalOrBool(
+	    state, match, EmitClassMaskBitMatch(state, mask, 2, EmitLogicalAndBool(state, inf, sign)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 3, EmitLogicalAndBool(state, normal, sign)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 4, EmitLogicalAndBool(state, denorm, sign)));
+	match = EmitLogicalOrBool(
+	    state, match, EmitClassMaskBitMatch(state, mask, 5, EmitLogicalAndBool(state, zero, sign)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 6, EmitLogicalAndBool(state, zero, positive)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 7, EmitLogicalAndBool(state, denorm, positive)));
+	match = EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 8, EmitLogicalAndBool(state, normal, positive)));
+	return EmitLogicalOrBool(
+	    state, match,
+	    EmitClassMaskBitMatch(state, mask, 9, EmitLogicalAndBool(state, inf, positive)));
+}
+
+uint32_t EmitMinMaxF32Value(EmitterState& state, uint32_t lhs, uint32_t rhs, bool max_value) {
+	const auto lhs_class = EmitClassifyF32(state, lhs);
+	const auto rhs_class = EmitClassifyF32(state, rhs);
+
+	const auto numeric_cond = state.builder.AllocateId();
+	state.builder.AddFunction({max_value ? OpFOrdGreaterThanEqual : OpFOrdLessThan, state.bool_type,
+	                           numeric_cond, lhs, rhs});
+	const auto ordered_bits =
+	    EmitSelectValueU32(state, numeric_cond, lhs_class.bits, rhs_class.bits);
+
+	const auto both_zero    = EmitLogicalAndBool(state, lhs_class.zero, rhs_class.zero);
+	const auto zero_bits    = max_value ? EmitAndU32(state, lhs_class.bits, rhs_class.bits)
+	                                    : EmitOrU32(state, lhs_class.bits, rhs_class.bits);
+	const auto numeric_bits = EmitSelectValueU32(state, both_zero, zero_bits, ordered_bits);
+
+	const auto rhs_nan_bits =
+	    EmitSelectValueU32(state, rhs_class.nan, lhs_class.bits, numeric_bits);
+	const auto non_snan_bits =
+	    EmitSelectValueU32(state, lhs_class.nan, rhs_class.bits, rhs_nan_bits);
+	const auto rhs_snan_bits =
+	    EmitSelectValueU32(state, rhs_class.snan, rhs_class.quiet_bits, non_snan_bits);
+	const auto result_bits =
+	    EmitSelectValueU32(state, lhs_class.snan, lhs_class.quiet_bits, rhs_snan_bits);
+	return EmitBitcastU32ToF32(state, result_bits);
+}
+
+uint32_t EmitTruncF32Value(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, state.float_type, ret, state.glsl_std450, GlslTrunc, value});
+	return ret;
+}
+
+uint32_t EmitFlushF32DenormToSignedZero(EmitterState& state, uint32_t value) {
+	const auto bits      = state.builder.AllocateId();
+	const auto abs_bits  = state.builder.AllocateId();
+	const auto sign_bits = state.builder.AllocateId();
+	const auto non_zero  = state.builder.AllocateId();
+	const auto subnormal = state.builder.AllocateId();
+	const auto flush     = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto ret       = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, sign_bits, bits, ConstantU32(state, 0x80000000u)});
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, non_zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpULessThan, state.bool_type, subnormal, abs_bits, ConstantU32(state, 0x00800000u)});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, flush, non_zero, subnormal});
+	state.builder.AddFunction({OpSelect, state.uint_type, selected, flush, sign_bits, bits});
+	state.builder.AddFunction({OpBitcast, state.float_type, ret, selected});
+	return ret;
+}
+
+uint32_t EmitTrigCycleF32(EmitterState& state, uint32_t src, bool preserve_signed_zero) {
+	const auto fract        = state.builder.AllocateId();
+	const auto bits         = state.builder.AllocateId();
+	const auto abs_bits     = state.builder.AllocateId();
+	const auto large        = state.builder.AllocateId();
+	const auto finite       = state.builder.AllocateId();
+	const auto large_finite = state.builder.AllocateId();
+	const auto reduced      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, state.float_type, fract, state.glsl_std450, GlslFract, src});
+	state.builder.AddFunction({OpBitcast, state.uint_type, bits, src});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, abs_bits, bits, ConstantU32(state, 0x7fffffffu)});
+	state.builder.AddFunction(
+	    {OpUGreaterThanEqual, state.bool_type, large, abs_bits, ConstantU32(state, 0x4b000000u)});
+	state.builder.AddFunction(
+	    {OpULessThan, state.bool_type, finite, abs_bits, ConstantU32(state, 0x7f800000u)});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, large_finite, large, finite});
+	state.builder.AddFunction(
+	    {OpSelect, state.float_type, reduced, large_finite, ConstantF32(state, 0), fract});
+	if (!preserve_signed_zero) {
+		return reduced;
+	}
+	const auto zero = state.builder.AllocateId();
+	const auto ret  = state.builder.AllocateId();
+	state.builder.AddFunction({OpIEqual, state.bool_type, zero, abs_bits, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpSelect, state.float_type, ret, zero, src, reduced});
+	return ret;
+}
+
+uint32_t EmitFNegateValue(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpFNegate, state.float_type, ret, value});
+	return ret;
+}
+
+uint32_t EmitFAbsValue(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, state.float_type, ret, state.glsl_std450, GlslFAbs, value});
+	return ret;
+}
+
+uint32_t EmitF16BitsToF32(EmitterState& state, uint32_t bits) {
+	const auto unpacked = state.builder.AllocateId();
+	const auto ret      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, bits});
+	state.builder.AddFunction({OpCompositeExtract, state.float_type, ret, unpacked, 0});
+	return ret;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterAnalysis.cpp

@@ -1,16 +1,16 @@
 #include "common/assert.h"
 #include "graphics/guest_gpu/gpu_defs.h"
-#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/SpirvEmitter.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/ir/ValueProgram.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
 uint32_t PixelParameterMappedLocation(const EmitterState& state, uint32_t attr) {
-	const auto* ps = state.pixel_input_info;
-	if (state.stage != ShaderType::Pixel || ps == nullptr) {
+	if (state.stage != ShaderType::Pixel) {
 		return attr;
 	}
-	return ShaderPixelParameterMappedLocation(*ps, attr);
+	return ShaderPixelParameterMappedLocation(*state.input_info.pixel, attr);
 }
 
 uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
@@ -21,16 +21,15 @@ uint32_t PixelParameterLocation(const EmitterState& state, uint32_t attr) {
 			active_inputs[active_count++] = input.location;
 		}
 	}
-	return state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr
-	           ? ShaderPixelParameterLocation(*state.pixel_input_info,
+	return state.stage == ShaderType::Pixel
+	           ? ShaderPixelParameterLocation(*state.input_info.pixel,
 	                                          {active_inputs.data(), active_count}, attr)
 	           : attr;
 }
 
 bool PixelParameterIsFlat(const EmitterState& state, uint32_t attr) {
-	const auto* ps = state.pixel_input_info;
-	return state.stage == ShaderType::Pixel && ps != nullptr &&
-	       ShaderPixelParameterIsFlat(*ps, attr);
+	return state.stage == ShaderType::Pixel &&
+	       ShaderPixelParameterIsFlat(*state.input_info.pixel, attr);
 }
 
 void SetError(std::string* error, const char* message) {
@@ -47,196 +46,10 @@ void CollectRegister(std::vector<RegisterBinding>& registers, IR::Register reg)
 	registers.push_back({reg, 0});
 }
 
-IR::Operand MakeRegisterOperand(IR::RegisterFile file, uint32_t index) {
-	IR::Operand operand;
-	operand.kind = IR::OperandKind::Register;
-	operand.reg  = {file, index};
-	return operand;
-}
-
-IR::Register SccRegister() {
-	return {IR::RegisterFile::Scc, static_cast<uint32_t>(Decoder::OperandKind::Scc)};
-}
-
-IR::Operand SccOperand() {
-	return MakeRegisterOperand(IR::RegisterFile::Scc,
-	                           static_cast<uint32_t>(Decoder::OperandKind::Scc));
-}
-
 bool IsInactiveWave32ExecHigh(const EmitterState& state, IR::Register reg) {
 	return state.wave_size == 32u && reg.file == IR::RegisterFile::Exec && reg.index == 1u;
 }
 
-bool IsMaskRegisterFile(IR::RegisterFile file) {
-	return file == IR::RegisterFile::Exec || file == IR::RegisterFile::Vcc;
-}
-
-bool IsSccOperand(const IR::Operand& operand) {
-	return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Scc;
-}
-
-void CollectMaskStateRegisters(std::vector<RegisterBinding>& registers) {
-	CollectRegister(registers, {IR::RegisterFile::Exec, 0});
-	CollectRegister(registers, {IR::RegisterFile::Exec, 1});
-	CollectRegister(registers, {IR::RegisterFile::Vcc, 0});
-	CollectRegister(registers, {IR::RegisterFile::Vcc, 1});
-	CollectRegister(registers, SccRegister());
-}
-
-void CollectSequentialRegisters(std::vector<RegisterBinding>& registers, const IR::Operand& base,
-                                uint32_t count) {
-	if (base.kind != IR::OperandKind::Register) {
-		return;
-	}
-	for (uint32_t i = 0; i < count; i++) {
-		auto reg = base.reg;
-		reg.index += i;
-		CollectRegister(registers, reg);
-	}
-}
-
-uint32_t MaxCollectedVectorRegisterEnd(const std::vector<RegisterBinding>& registers) {
-	uint32_t max_end = 0;
-	for (const auto& binding: registers) {
-		if (binding.reg.file == IR::RegisterFile::Vector) {
-			max_end = std::max(max_end, binding.reg.index + 1u);
-		}
-	}
-	return std::min(max_end, 256u);
-}
-
-void CollectMoveRelSourceRegisters(const IR::Program&            program,
-                                   std::vector<RegisterBinding>& registers) {
-	const auto max_vector_end = MaxCollectedVectorRegisterEnd(registers);
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.op != IR::Opcode::MoveRelSourceU32 ||
-			    inst.src[0].kind != IR::OperandKind::Register ||
-			    inst.src[0].reg.file != IR::RegisterFile::Vector) {
-				continue;
-			}
-			const auto base_index = inst.src[0].reg.index;
-			const auto limit      = std::min(std::max(max_vector_end, base_index + 1u), 256u);
-			CollectSequentialRegisters(registers, inst.src[0], limit - base_index);
-		}
-	}
-}
-
-bool IsPairDwordOpcode(IR::Opcode op) {
-	switch (op) {
-		case IR::Opcode::MoveU64:
-		case IR::Opcode::WqmB64:
-		case IR::Opcode::SaveexecB64:
-		case IR::Opcode::BitwiseAndU64:
-		case IR::Opcode::BitwiseAndNotU64:
-		case IR::Opcode::BitwiseOrU64:
-		case IR::Opcode::BitwiseOrNotU64:
-		case IR::Opcode::BitwiseXorU64:
-		case IR::Opcode::BitwiseNandU64:
-		case IR::Opcode::BitwiseNorU64:
-		case IR::Opcode::BitwiseXnorU64:
-		case IR::Opcode::BitwiseNotU64:
-		case IR::Opcode::BitFieldMaskU64:
-		case IR::Opcode::BitFieldExtractU64:
-		case IR::Opcode::BitReplicateB64B32:
-		case IR::Opcode::ShiftLeftLogicalU64:
-		case IR::Opcode::ShiftRightLogicalU64:
-		case IR::Opcode::SelectU64: return true;
-		default: return false;
-	}
-}
-
-uint32_t PairDwordSourceCount(IR::Opcode op, uint32_t src_count) {
-	switch (op) {
-		case IR::Opcode::BitFieldMaskU64: return 0;
-		case IR::Opcode::BitReplicateB64B32: return 0;
-		case IR::Opcode::BitFieldExtractU64: return 1;
-		case IR::Opcode::ShiftLeftLogicalU64:
-		case IR::Opcode::ShiftRightLogicalU64:
-		case IR::Opcode::BitwiseNotU64:
-		case IR::Opcode::SaveexecB64:
-		case IR::Opcode::WqmB64:
-		case IR::Opcode::MoveU64: return std::min<uint32_t>(src_count, 1u);
-		default: return src_count;
-	}
-}
-
-void CollectRegisters(const IR::Program& program, std::vector<RegisterBinding>& registers) {
-	CollectMaskStateRegisters(registers);
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.dst.kind == IR::OperandKind::Register) {
-				CollectRegister(registers, inst.dst.reg);
-			}
-			if (inst.dst2.kind == IR::OperandKind::Register) {
-				CollectRegister(registers, inst.dst2.reg);
-			}
-			for (uint32_t i = 0; i < inst.src_count; i++) {
-				if (inst.src[i].kind == IR::OperandKind::Register) {
-					CollectRegister(registers, inst.src[i].reg);
-				}
-			}
-			if (IsPairDwordOpcode(inst.op)) {
-				CollectSequentialRegisters(registers, inst.dst, 2);
-				const uint32_t first_pair_src = inst.op == IR::Opcode::SelectU64 ? 1u : 0u;
-				const uint32_t pair_src_count = PairDwordSourceCount(inst.op, inst.src_count);
-				for (uint32_t i = first_pair_src; i < pair_src_count; i++) {
-					CollectSequentialRegisters(registers, inst.src[i], 2);
-				}
-			}
-			if (inst.op == IR::Opcode::BitCountU64 || inst.op == IR::Opcode::FindMsbFromHighU64) {
-				CollectSequentialRegisters(registers, inst.src[0], 2);
-			}
-			if (IR::IsCompare64Opcode(inst.op)) {
-				CollectSequentialRegisters(registers, inst.src[0], 2);
-				CollectSequentialRegisters(registers, inst.src[1], 2);
-			}
-			if (IR::IsCompareOpcode(inst.op) && inst.dst.kind == IR::OperandKind::Register &&
-			    inst.dst.reg.file != IR::RegisterFile::Scc) {
-				CollectSequentialRegisters(registers, inst.dst, 2);
-			}
-			if (inst.op == IR::Opcode::IAddCarryU32 || inst.op == IR::Opcode::ISubBorrowU32) {
-				CollectSequentialRegisters(registers, inst.dst2, 2);
-			}
-			if (inst.op == IR::Opcode::UMadU64U32) {
-				CollectSequentialRegisters(registers, inst.dst, 2);
-				CollectSequentialRegisters(registers, inst.dst2, 2);
-				CollectSequentialRegisters(registers, inst.src[2], 2);
-			}
-			if (inst.op == IR::Opcode::SBufferLoadDword &&
-			    inst.memory.kind == IR::ResourceKind::ScalarBuffer) {
-				CollectRegister(registers, {IR::RegisterFile::Scalar, inst.memory.resource * 4u});
-			}
-			if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4) &&
-			    inst.memory.image_address_components > 1u) {
-				CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
-				CollectSequentialRegisters(registers, inst.src[0],
-				                           inst.memory.image_address_components);
-			}
-			if (inst.op == IR::Opcode::ImageGetLod) {
-				CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
-				CollectSequentialRegisters(registers, inst.src[0],
-				                           inst.memory.image_address_components);
-			}
-			if (inst.op == IR::Opcode::ImageLoad) {
-				CollectSequentialRegisters(registers, inst.dst, inst.memory.data_dwords);
-				CollectSequentialRegisters(registers, inst.src[0],
-				                           inst.memory.image_address_components);
-			}
-			if (inst.op == IR::Opcode::ImageStore) {
-				CollectSequentialRegisters(registers, inst.src[0], inst.memory.data_dwords);
-				CollectSequentialRegisters(registers, inst.src[1],
-				                           inst.memory.image_address_components);
-			}
-			if (inst.memory.kind == IR::ResourceKind::StorageImageUint) {
-				CollectSequentialRegisters(registers, inst.src[1],
-				                           inst.memory.image_address_components);
-			}
-		}
-	}
-	CollectMoveRelSourceRegisters(program, registers);
-}
-
 bool HasOutput(const std::vector<OutputBinding>& outputs, IR::StageOutputKind kind,
                uint32_t index) {
 	return std::any_of(outputs.begin(), outputs.end(), [kind, index](const OutputBinding& binding) {
@@ -276,9 +89,12 @@ uint32_t OutputVariableForExport(const EmitterState& state, const IR::ExportInfo
 }
 
 bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.op == IR::Opcode::ImageGetLod) {
+	if (program.values == nullptr) {
+		return false;
+	}
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			if (inst.GetOpcode() == IR::ValueOpcode::ImageQueryLod) {
 				return true;
 			}
 		}
@@ -287,9 +103,12 @@ bool ProgramNeedsComputeDerivatives(const IR::Program& program) {
 }
 
 bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.op == IR::Opcode::ImageGather4) {
+	if (program.values == nullptr) {
+		return false;
+	}
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			if (inst.GetOpcode() == IR::ValueOpcode::ImageGatherRaw) {
 				return true;
 			}
 		}
@@ -297,34 +116,43 @@ bool ProgramNeedsImageGatherExtended(const IR::Program& program) {
 	return false;
 }
 
-bool IsLdsOpcode(IR::Opcode op) {
-	switch (op) {
-		case IR::Opcode::DsReadUbyte:
-		case IR::Opcode::DsReadSbyte:
-		case IR::Opcode::DsReadUshort:
-		case IR::Opcode::DsReadSshort:
-		case IR::Opcode::DsReadB32:
-		case IR::Opcode::DsWriteByte:
-		case IR::Opcode::DsWriteShort:
-		case IR::Opcode::DsWriteB32:
-		case IR::Opcode::DsMinF32:
-		case IR::Opcode::DsMaxF32:
-		case IR::Opcode::DsSwizzleB32:
-		case IR::Opcode::DsWriteAddtidB32:
-		case IR::Opcode::DsReadAddtidB32: return true;
-		default: return false;
-	}
-}
-
 bool ProgramNeedsFunctionLds(const IR::Program& program) {
-	if (program.stage == ShaderType::Compute) {
+	if (program.stage == ShaderType::Compute || program.values == nullptr) {
 		return false;
 	}
 
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (IsLdsOpcode(inst.op)) {
-				return true;
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::SwizzleU32: return true;
+				case IR::ValueOpcode::LoadSharedU8:
+				case IR::ValueOpcode::LoadSharedU16:
+				case IR::ValueOpcode::LoadSharedU32:
+				case IR::ValueOpcode::WriteSharedU8:
+				case IR::ValueOpcode::WriteSharedU16:
+				case IR::ValueOpcode::WriteSharedU32:
+				case IR::ValueOpcode::SharedAtomicFMin32:
+				case IR::ValueOpcode::SharedAtomicFMax32:
+				case IR::ValueOpcode::SharedAtomicSwap32:
+				case IR::ValueOpcode::SharedAtomicIAdd32:
+				case IR::ValueOpcode::SharedAtomicISub32:
+				case IR::ValueOpcode::SharedAtomicSMin32:
+				case IR::ValueOpcode::SharedAtomicUMin32:
+				case IR::ValueOpcode::SharedAtomicSMax32:
+				case IR::ValueOpcode::SharedAtomicUMax32:
+				case IR::ValueOpcode::SharedAtomicAnd32:
+				case IR::ValueOpcode::SharedAtomicOr32:
+				case IR::ValueOpcode::SharedAtomicXor32:
+				case IR::ValueOpcode::DataAppend:
+				case IR::ValueOpcode::DataConsume: {
+					const auto index = inst.Flags<IR::MemoryFlags>().index;
+					if (index < program.values->memory_info.size() &&
+					    program.values->memory_info[index].kind == IR::ResourceKind::Lds) {
+						return true;
+					}
+					break;
+				}
+				default: break;
 			}
 		}
 	}
@@ -332,13 +160,18 @@ bool ProgramNeedsFunctionLds(const IR::Program& program) {
 }
 
 bool ProgramNeedsPixelValidMask(const IR::Program& program) {
-	if (program.stage != ShaderType::Pixel) {
+	if (program.stage != ShaderType::Pixel || program.values == nullptr) {
 		return false;
 	}
 
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.op == IR::Opcode::Export && inst.export_info.vm) {
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			if (inst.GetOpcode() != IR::ValueOpcode::SetAttribute) {
+				continue;
+			}
+			const auto index = inst.Flags<IR::ExportFlags>().index;
+			if (index < program.values->export_info.size() &&
+			    program.values->export_info[index].vm) {
 				return true;
 			}
 		}
@@ -346,20 +179,14 @@ bool ProgramNeedsPixelValidMask(const IR::Program& program) {
 	return false;
 }
 
-bool InstructionHasDppSource(const IR::Instruction& inst) {
-	for (uint32_t i = 0; i < inst.src_count && i < 3u; i++) {
-		if (inst.src[i].dpp) {
-			return true;
-		}
-	}
-	return false;
-}
-
 bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
+	if (program.values == nullptr) {
+		return false;
+	}
 	if (program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation) {
-		for (const auto& block: program.blocks) {
-			for (const auto& inst: block.instructions) {
-				if (inst.op == IR::Opcode::WqmB64) {
+		for (const auto* block: program.values->blocks) {
+			for (const auto& inst: *block) {
+				if (inst.GetOpcode() == IR::ValueOpcode::WqmMask) {
 					return true;
 				}
 			}
@@ -369,13 +196,22 @@ bool ProgramNeedsSubgroupBallot(const IR::Program& program) {
 }
 
 bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.op == IR::Opcode::ReadFirstLaneU32 || inst.op == IR::Opcode::ReadLaneU32 ||
-			    inst.op == IR::Opcode::Permlane16B32 || inst.op == IR::Opcode::Permlanex16B32 ||
-			    inst.op == IR::Opcode::DsSwizzleB32 || inst.op == IR::Opcode::DsConsume ||
-			    inst.op == IR::Opcode::DsAppend || InstructionHasDppSource(inst)) {
-				return true;
+	if (program.values == nullptr) {
+		return false;
+	}
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::DppMoveU32:
+				case IR::ValueOpcode::ReadFirstLane:
+				case IR::ValueOpcode::ReadLane:
+				case IR::ValueOpcode::Permlane16U32:
+				case IR::ValueOpcode::DataAppend:
+				case IR::ValueOpcode::DataConsume:
+				case IR::ValueOpcode::GdsDataAppend:
+				case IR::ValueOpcode::GdsDataConsume:
+				case IR::ValueOpcode::SwizzleU32: return true;
+				default: break;
 			}
 		}
 	}
@@ -383,25 +219,24 @@ bool ProgramNeedsSubgroupShuffle(const IR::Program& program) {
 }
 
 bool ProgramNeedsSubgroupLocalInvocationId(const IR::Program& program) {
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (inst.op == IR::Opcode::WqmB64 || inst.op == IR::Opcode::DsSwizzleB32 ||
-			    inst.op == IR::Opcode::DsConsume || inst.op == IR::Opcode::DsAppend ||
-			    inst.op == IR::Opcode::WriteLaneU32 || inst.op == IR::Opcode::Permlane16B32 ||
-			    inst.op == IR::Opcode::Permlanex16B32 || inst.op == IR::Opcode::Export ||
-			    InstructionHasDppSource(inst)) {
-				return true;
-			}
-			if (IR::IsCompareOpcode(inst.op)) {
-				return true;
-			}
-			if (inst.dst.kind == IR::OperandKind::Register &&
-			    inst.dst.reg.file == IR::RegisterFile::Vector) {
-				return true;
-			}
-			if (inst.dst2.kind == IR::OperandKind::Register &&
-			    inst.dst2.reg.file == IR::RegisterFile::Vector) {
-				return true;
+	if (program.values == nullptr) {
+		return false;
+	}
+	for (const auto* block: program.values->blocks) {
+		for (const auto& inst: *block) {
+			switch (inst.GetOpcode()) {
+				case IR::ValueOpcode::DppMoveU32:
+				case IR::ValueOpcode::DppUpdateU32:
+				case IR::ValueOpcode::WqmMask:
+				case IR::ValueOpcode::LaneId:
+				case IR::ValueOpcode::WriteLane:
+				case IR::ValueOpcode::Permlane16U32:
+				case IR::ValueOpcode::DataAppend:
+				case IR::ValueOpcode::DataConsume:
+				case IR::ValueOpcode::GdsDataAppend:
+				case IR::ValueOpcode::GdsDataConsume:
+				case IR::ValueOpcode::SwizzleU32: return true;
+				default: break;
 			}
 		}
 	}
@@ -418,9 +253,6 @@ uint32_t PointerForRegister(const EmitterState& state, IR::Register reg) {
 }
 
 uint32_t          ConstantU32(EmitterState& state, uint32_t value);
-void              EmitStoreU32(EmitterState& state, const IR::Operand& dst, uint32_t value);
-uint32_t          EmitSubgroupLocalInvocationId(EmitterState& state);
-uint32_t          EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane);
 [[noreturn]] void ExitDescriptorBindingFailure(const EmitterState&       state,
                                                IR::DescriptorBindingKind kind, uint32_t resource,
                                                const char* reason) {
@@ -584,6 +416,13 @@ uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32
 	return sampled_image;
 }
 
+uint32_t MakeSampledImage(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc,
+                          ImageViewKind view, uint32_t image_resource) {
+	auto selected     = mem;
+	selected.resource = image_resource;
+	return MakeSampledImage(state, selected, use_pc, view);
+}
+
 uint32_t StorageImageDescriptorPointer(EmitterState& state, uint32_t resource, bool uint_image,
                                        uint32_t use_pc, ImageViewKind view) {
 	(void)use_pc;
@@ -608,6 +447,48 @@ uint32_t LoadStorageImageDescriptor(EmitterState& state, uint32_t resource, bool
 	return image;
 }
 
+void EmitStorageImageWrite(EmitterState& state, uint32_t resource, bool uint_image,
+                           ImageViewKind view, uint32_t mip_lod, uint32_t coord, uint32_t texel) {
+	const auto  kind    = StorageBindingKind(uint_image, view);
+	const auto  binding = ResourceForDescriptor(state, kind, resource);
+	const auto& image   = state.program.info.images.at(resource);
+	const auto  LoadAt  = [&](uint32_t array_index) {
+		const auto pointer = DescriptorElementPointer(
+		    state, StorageImagePointerType(state, uint_image, view),
+		    StorageImageVariable(state, uint_image, view), array_index, kind, resource,
+		    "storage image descriptor array was not emitted");
+		const auto descriptor = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpLoad, StorageImageType(state, uint_image, view), descriptor, pointer});
+		return descriptor;
+	};
+	if (image.mip_mode != IR::ImageMipMode::DynamicStorage) {
+		state.builder.AddFunction({OpImageWrite, LoadAt(binding.array_index), coord, texel});
+		return;
+	}
+	if (image.mip_count == 0u) {
+		ExitDescriptorBindingFailure(state, kind, resource,
+		                             "dynamic storage image has no mip descriptors");
+	}
+
+	const auto            merge_label = state.builder.AllocateId();
+	std::vector<uint32_t> labels(image.mip_count);
+	std::vector<uint32_t> words {OpSwitch, mip_lod, merge_label};
+	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
+		labels[mip] = state.builder.AllocateId();
+		words.push_back(mip);
+		words.push_back(labels[mip]);
+	}
+	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction(words);
+	for (uint32_t mip = 0; mip < image.mip_count; mip++) {
+		state.builder.AddFunction({OpLabel, labels[mip]});
+		state.builder.AddFunction({OpImageWrite, LoadAt(binding.array_index + mip), coord, texel});
+		state.builder.AddFunction({OpBranch, merge_label});
+	}
+	state.builder.AddFunction({OpLabel, merge_label});
+}
+
 uint32_t ExecutionModelForStage(ShaderType stage) {
 	switch (stage) {
 		case ShaderType::Vertex: return ExecutionModelVertex;

src/graphics/shader/recompiler/backend/spirv/spirvEmitterControlFlow.cpp

@@ -0,0 +1,52 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+bool UserDataDwordIndex(const EmitterState& state, IR::Register reg, uint32_t& dword_index) {
+	if (reg.file != IR::RegisterFile::Scalar) {
+		return false;
+	}
+	const auto& registers = state.program.bindings.user_data_registers;
+	const auto  found     = std::lower_bound(registers.begin(), registers.end(), reg.index);
+	if (found == registers.end() || *found != reg.index) {
+		return false;
+	}
+	dword_index = static_cast<uint32_t>(found - registers.begin());
+	return true;
+}
+
+uint32_t EmitShaderDataDwordLoad(EmitterState& state, uint32_t dword_index) {
+	if (state.push_constant_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, state.ptr_push_constant_uint, pointer,
+		                           state.push_constant_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+		return value;
+	}
+	if (state.vsharp_storage_variable != 0) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
+		                           state.vsharp_storage_variable, ConstantU32(state, 0),
+		                           ConstantU32(state, dword_index)});
+		state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+		return value;
+	}
+	return ConstantU32(state, 0);
+}
+
+uint32_t InitialRegisterValue(const EmitterState& state, IR::Register reg) {
+	if (reg.file != IR::RegisterFile::Exec) {
+		return 0;
+	}
+	if (state.per_invocation_masks) {
+		return reg.index == 0 ? 1u : 0u;
+	}
+	return reg.index == 0 || state.wave_size == 64u ? 0xffffffffu : 0u;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterFlow.cpp

@@ -0,0 +1,570 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
+	uint32_t ret = ConstantU32(state, 0);
+	for (uint32_t i = 0; i < 8u; i++) {
+		const auto mask     = 0x0fu << (i * 4u);
+		const auto masked   = state.builder.AllocateId();
+		const auto non_zero = state.builder.AllocateId();
+		const auto expanded = state.builder.AllocateId();
+		const auto combined = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
+		state.builder.AddFunction(
+		    {OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
+		state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
+		                           ConstantU32(state, mask), ConstantU32(state, 0)});
+		state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
+		ret = combined;
+	}
+	return ret;
+}
+
+uint32_t BoolAsU32(ValueEmitContext& ctx, uint32_t value) {
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, result, value,
+	                               ConstantU32(ctx.state, 1), ConstantU32(ctx.state, 0)});
+	return result;
+}
+
+void StoreRegisterMirror(ValueEmitContext& ctx, IR::Register reg, uint32_t value) {
+	if (const auto pointer = PointerForRegister(ctx.state, reg); pointer != 0) {
+		if (IsInactiveWave32ExecHigh(ctx.state, reg)) {
+			value = ConstantU32(ctx.state, 0);
+		}
+		ctx.state.builder.AddFunction({OpStore, pointer, value});
+	}
+}
+
+bool IsPrologue(const ValueEmitContext& ctx) {
+	return !ctx.program.blocks.empty() && ctx.current_block == ctx.program.blocks.front();
+}
+
+uint32_t EmitBuiltinU32(ValueEmitContext& ctx, IR::StageInputKind kind, uint32_t component) {
+	auto& state = ctx.state;
+	if (kind == IR::StageInputKind::LocalInvocationIndex) {
+		return EmitLocalInvocationIndex(state);
+	}
+	const auto variable = InputVariableForKind(state, kind);
+	if (variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	if (kind == IR::StageInputKind::FrontFacing) {
+		const auto value = state.builder.AllocateId();
+		const auto bits  = state.builder.AllocateId();
+		state.builder.AddFunction({OpLoad, state.bool_type, value, variable});
+		state.builder.AddFunction(
+		    {OpSelect, state.uint_type, bits, value, ConstantU32(state, 1), ConstantU32(state, 0)});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::VertexIndex || kind == IR::StageInputKind::InstanceIndex) {
+		const auto value = state.builder.AllocateId();
+		const auto bits  = state.builder.AllocateId();
+		state.builder.AddFunction({OpLoad, state.int_type, value, variable});
+		state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+		return bits;
+	}
+	if (kind == IR::StageInputKind::FragCoord) {
+		const auto pointer = state.builder.AllocateId();
+		const auto value   = state.builder.AllocateId();
+		const auto bits    = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, state.ptr_input_float, pointer, variable,
+		                           ConstantU32(state, component)});
+		state.builder.AddFunction({OpLoad, state.float_type, value, pointer});
+		state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
+		return bits;
+	}
+	return EmitInputComponentU32(state, kind, component);
+}
+
+uint32_t EmitWqm(ValueEmitContext& ctx, uint32_t active) {
+	auto&      state  = ctx.state;
+	const auto ballot = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+	                           ConstantU32(state, ScopeSubgroup), active});
+	const auto low  = state.builder.AllocateId();
+	const auto high = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
+	const auto wqm_low  = EmitWqmLaneU32(state, low);
+	const auto wqm_high = EmitWqmLaneU32(state, high);
+	const auto lane     = EmitSubgroupLocalInvocationId(state);
+	const auto upper    = state.builder.AllocateId();
+	const auto mask     = state.builder.AllocateId();
+	const auto bit_lane = state.builder.AllocateId();
+	const auto bit      = state.builder.AllocateId();
+	const auto hit      = state.builder.AllocateId();
+	const auto result   = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 32)});
+	state.builder.AddFunction({OpSelect, state.uint_type, mask, upper, wqm_high, wqm_low});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, bit_lane, lane, ConstantU32(state, 31)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_lane});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, state.bool_type, result, hit, ConstantU32(state, 0)});
+	return result;
+}
+
+uint32_t EmitDppWriteCondition(ValueEmitContext& ctx, const IR::DppMoveFlags& flags,
+                               uint32_t exec) {
+	auto&      state      = ctx.state;
+	const auto lane       = EmitSubgroupLocalInvocationId(state);
+	const auto bank_shift = state.builder.AllocateId();
+	const auto row_shift  = state.builder.AllocateId();
+	const auto bank       = state.builder.AllocateId();
+	const auto row        = state.builder.AllocateId();
+	const auto bank_bit   = state.builder.AllocateId();
+	const auto row_bit    = state.builder.AllocateId();
+	const auto bank_hit   = state.builder.AllocateId();
+	const auto row_hit    = state.builder.AllocateId();
+	const auto bank_ok    = state.builder.AllocateId();
+	const auto row_ok     = state.builder.AllocateId();
+	const auto masks_ok   = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, bank_shift, lane, ConstantU32(state, 2)});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, row_shift, lane, ConstantU32(state, 4)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, bank, bank_shift, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, row, row_shift, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, bank_bit, ConstantU32(state, 1), bank});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, row_bit, ConstantU32(state, 1), row});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, bank_hit, ConstantU32(state, flags.bank_mask), bank_bit});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, row_hit, ConstantU32(state, flags.row_mask), row_bit});
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, bank_ok, bank_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, row_ok, row_hit, ConstantU32(state, 0)});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, masks_ok, bank_ok, row_ok});
+	uint32_t writable = masks_ok;
+	if (!flags.bound_control) {
+		const auto target  = EmitDppTargetLane(state, flags.control);
+		const auto bounded = state.builder.AllocateId();
+		state.builder.AddFunction({OpLogicalAnd, state.bool_type, bounded, writable, target.valid});
+		writable = bounded;
+	}
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, result, exec, writable});
+	return result;
+}
+
+uint32_t EmitAttribute(ValueEmitContext& ctx, uint32_t attr, uint32_t chan) {
+	auto&       state = ctx.state;
+	const auto* input = InputBindingForParameter(state, attr);
+	if (input == nullptr || input->variable_id == 0) {
+		return ConstantU32(state, 0);
+	}
+	if (state.stage == ShaderType::Vertex) {
+		return EmitVertexParameterComponentU32(state, *input, chan & 3u);
+	}
+	const auto vector    = state.builder.AllocateId();
+	const auto component = state.builder.AllocateId();
+	const auto bits      = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, state.vec4_float_type, vector, input->variable_id});
+	state.builder.AddFunction({OpCompositeExtract, state.float_type, component, vector, chan & 3u});
+	state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
+	return bits;
+}
+
+bool MrtUsesUint(const EmitterState& state, const IR::ExportInfo& exp) {
+	return state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+	       exp.index < std::size(state.input_info.pixel->target_output_mode) &&
+	       state.input_info.pixel->target_output_mode[exp.index] == 7u;
+}
+
+uint32_t ExportRawComponent(ValueEmitContext& ctx, uint32_t vector, uint32_t component) {
+	const auto value = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction(
+	    {OpCompositeExtract, ctx.state.uint_type, value, vector, component});
+	return value;
+}
+
+uint32_t ExportVector(ValueEmitContext& ctx, uint32_t data, const IR::ExportInfo& exp,
+                      bool uint_output) {
+	auto& state = ctx.state;
+	if (exp.compr && !uint_output) {
+		uint32_t f32[4] = {ConstantF32(state, 0), ConstantF32(state, 0), ConstantF32(state, 0),
+		                   ConstantF32(state, 0x3f800000u)};
+		for (uint32_t pair = 0; pair < 2u; pair++) {
+			if ((exp.en & (3u << (pair * 2u))) == 0u) {
+				continue;
+			}
+			const auto packed   = state.builder.AllocateId();
+			const auto unpacked = state.builder.AllocateId();
+			state.builder.AddFunction({OpCompositeExtract, state.uint_type, packed, data, pair});
+			state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
+			                           state.glsl_std450, GlslUnpackHalf2x16, packed});
+			for (uint32_t lane = 0; lane < 2u; lane++) {
+				const auto component = pair * 2u + lane;
+				if (((exp.en >> component) & 1u) != 0u) {
+					f32[component] = state.builder.AllocateId();
+					state.builder.AddFunction(
+					    {OpCompositeExtract, state.float_type, f32[component], unpacked, lane});
+				}
+			}
+		}
+		const auto vector = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
+		return vector;
+	}
+	uint32_t raw[4] = {
+	    ConstantU32(state, 0),
+	    ConstantU32(state, 0),
+	    ConstantU32(state, 0),
+	    ConstantU32(state, uint_output ? 1u : 0x3f800000u),
+	};
+	if (exp.compr) {
+		for (uint32_t pair = 0; pair < 2u; pair++) {
+			if ((exp.en & (3u << (pair * 2u))) == 0u) {
+				continue;
+			}
+			const auto packed = ExportRawComponent(ctx, data, pair);
+			for (uint32_t lane = 0; lane < 2u; lane++) {
+				const auto component = pair * 2u + lane;
+				if (((exp.en >> component) & 1u) == 0u) {
+					continue;
+				}
+				raw[component] = state.builder.AllocateId();
+				state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, raw[component],
+				                           packed, ConstantU32(state, lane * 16u),
+				                           ConstantU32(state, 16)});
+			}
+		}
+	} else {
+		for (uint32_t component = 0; component < 4u; component++) {
+			if (((exp.en >> component) & 1u) != 0u) {
+				raw[component] = ExportRawComponent(ctx, data, component);
+			}
+		}
+	}
+	if (uint_output) {
+		const auto vector = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpCompositeConstruct, state.vec4_uint_type, vector, raw[0], raw[1], raw[2], raw[3]});
+		return vector;
+	}
+	uint32_t f32[4] {};
+	for (uint32_t component = 0; component < 4u; component++) {
+		f32[component] = state.builder.AllocateId();
+		state.builder.AddFunction({OpBitcast, state.float_type, f32[component], raw[component]});
+	}
+	const auto vector = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpCompositeConstruct, state.vec4_float_type, vector, f32[0], f32[1], f32[2], f32[3]});
+	return vector;
+}
+
+void EmitExport(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&       state = ctx.state;
+	const auto& exp   = ctx.Export(inst);
+	const auto  exec  = ctx.Arg(inst, 1);
+	if (state.stage == ShaderType::Pixel && exp.vm && state.needs_pixel_valid_mask &&
+	    state.pixel_valid_mask_variable != 0) {
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpSelect, state.uint_type, value, exec, ConstantU32(state, 1), ConstantU32(state, 0)});
+		state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
+	}
+	if (exp.kind == IR::ExportTargetKind::Null || exp.kind == IR::ExportTargetKind::Primitive ||
+	    exp.en == 0u) {
+		return;
+	}
+	EmitIfCondition(state, exec, [&]() {
+		const auto data = ctx.Arg(inst, 0);
+		if (exp.kind == IR::ExportTargetKind::MrtZ) {
+			if ((exp.en & 1u) != 0u && state.depth_variable != 0) {
+				const auto raw = ExportRawComponent(ctx, data, 0);
+				const auto f32 = state.builder.AllocateId();
+				state.builder.AddFunction({OpBitcast, state.float_type, f32, raw});
+				state.builder.AddFunction({OpStore, state.depth_variable, f32});
+			}
+			if ((exp.en & 4u) != 0u && state.sample_mask_variable != 0) {
+				const auto raw     = ExportRawComponent(ctx, data, 2);
+				const auto value   = state.builder.AllocateId();
+				const auto pointer = state.builder.AllocateId();
+				state.builder.AddFunction({OpBitcast, state.int_type, value, raw});
+				state.builder.AddFunction({OpAccessChain, state.ptr_output_int, pointer,
+				                           state.sample_mask_variable, ConstantU32(state, 0)});
+				state.builder.AddFunction({OpStore, pointer, value});
+			}
+			return;
+		}
+		const auto variable = OutputVariableForExport(state, exp);
+		if (variable == 0) {
+			return;
+		}
+		const bool uint_output = MrtUsesUint(state, exp);
+		const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
+		auto       value       = ExportVector(ctx, data, exp, uint_output);
+		if (state.stage == ShaderType::Pixel && exp.kind == IR::ExportTargetKind::Mrt &&
+		    exp.index < state.input_info.pixel->target_export_mapping.size()) {
+			const auto mapping = state.input_info.pixel->target_export_mapping[exp.index];
+			if (!mapping.IsIdentity()) {
+				const auto mapped = state.builder.AllocateId();
+				state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value,
+				                           mapping.Map(0), mapping.Map(1), mapping.Map(2),
+				                           mapping.Map(3)});
+				value = mapped;
+			}
+		}
+		if (exp.kind == IR::ExportTargetKind::Position) {
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, state.ptr_output_vec4_float, pointer,
+			                           variable, ConstantU32(state, 0)});
+			state.builder.AddFunction({OpStore, pointer, value});
+		} else {
+			state.builder.AddFunction({OpStore, variable, value});
+		}
+	});
+}
+
+} // namespace
+
+bool EmitValueFlow(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto& state = ctx.state;
+	switch (inst.GetOpcode()) {
+		case IR::ValueOpcode::Identity:
+		case IR::ValueOpcode::ConditionRef: ctx.Define(inst, ctx.Arg(inst, 0)); return true;
+		case IR::ValueOpcode::SetScalarRegister:
+			StoreRegisterMirror(
+			    ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
+			    ctx.Arg(inst, 1));
+			return true;
+		case IR::ValueOpcode::SetThreadBitScalarRegister:
+			StoreRegisterMirror(
+			    ctx, {IR::RegisterFile::Scalar, IR::RegIndex(inst.Arg(0).ScalarRegister())},
+			    BoolAsU32(ctx, ctx.Arg(inst, 1)));
+			return true;
+		case IR::ValueOpcode::SetVectorRegister:
+			StoreRegisterMirror(
+			    ctx, {IR::RegisterFile::Vector, IR::RegIndex(inst.Arg(0).VectorRegister())},
+			    ctx.Arg(inst, 1));
+			return true;
+		case IR::ValueOpcode::SetScc: ctx.Arg(inst, 0); return true;
+		case IR::ValueOpcode::SetExec: ctx.Arg(inst, 0); return true;
+		case IR::ValueOpcode::SetExecLo:
+		case IR::ValueOpcode::SetExecHi:
+			if (!IsPrologue(ctx)) {
+				StoreRegisterMirror(ctx,
+				                    {IR::RegisterFile::Exec,
+				                     inst.GetOpcode() == IR::ValueOpcode::SetExecLo ? 0u : 1u},
+				                    ctx.Arg(inst, 0));
+			}
+			return true;
+		case IR::ValueOpcode::SetVcc: ctx.Arg(inst, 0); return true;
+		case IR::ValueOpcode::SetVccLo:
+		case IR::ValueOpcode::SetVccHi:
+			if (!IsPrologue(ctx)) {
+				StoreRegisterMirror(ctx,
+				                    {IR::RegisterFile::Vcc,
+				                     inst.GetOpcode() == IR::ValueOpcode::SetVccLo ? 0u : 1u},
+				                    ctx.Arg(inst, 0));
+			}
+			return true;
+		case IR::ValueOpcode::SetM0:
+			StoreRegisterMirror(ctx, {IR::RegisterFile::M0, 0}, ctx.Arg(inst, 0));
+			return true;
+		case IR::ValueOpcode::GetScalarRegister:
+		case IR::ValueOpcode::GetThreadBitScalarRegister:
+		case IR::ValueOpcode::GetVectorRegister:
+		case IR::ValueOpcode::GetScc:
+		case IR::ValueOpcode::GetExec:
+		case IR::ValueOpcode::GetExecLo:
+		case IR::ValueOpcode::GetExecHi:
+		case IR::ValueOpcode::GetVcc:
+		case IR::ValueOpcode::GetVccLo:
+		case IR::ValueOpcode::GetVccHi:
+		case IR::ValueOpcode::GetM0: return true;
+		case IR::ValueOpcode::Void:
+		case IR::ValueOpcode::Reference:
+		case IR::ValueOpcode::ReferenceU32:
+		case IR::ValueOpcode::Prologue:
+		case IR::ValueOpcode::Epilogue:
+		case IR::ValueOpcode::ControlNop:
+		case IR::ValueOpcode::Waitcnt:
+		case IR::ValueOpcode::Sendmsg:
+		case IR::ValueOpcode::TtraceData:
+		case IR::ValueOpcode::InstPrefetch: return true;
+		case IR::ValueOpcode::Barrier: {
+			const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
+			state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
+			                           ConstantU32(state, ScopeWorkgroup),
+			                           ConstantU32(state, semantics)});
+			return true;
+		}
+		case IR::ValueOpcode::GetUserData: {
+			const auto reg   = inst.Arg(0).ScalarRegister();
+			uint32_t   dword = 0;
+			if (!UserDataDwordIndex(state, {IR::RegisterFile::Scalar, IR::RegIndex(reg)}, dword)) {
+				ctx.Define(inst, ConstantU32(state, 0));
+			} else {
+				ctx.Define(inst, EmitShaderDataDwordLoad(state, dword));
+			}
+			return true;
+		}
+		case IR::ValueOpcode::GetBuiltin:
+			ctx.Define(inst, EmitBuiltinU32(ctx, static_cast<IR::StageInputKind>(inst.Arg(0).U32()),
+			                                inst.Arg(1).U32()));
+			return true;
+		case IR::ValueOpcode::UndefU1:
+		case IR::ValueOpcode::UndefU8:
+		case IR::ValueOpcode::UndefU16:
+		case IR::ValueOpcode::UndefU32:
+		case IR::ValueOpcode::UndefU64:
+			state.builder.AddFunction({OpUndef, ctx.TypeId(inst.GetType()), ctx.Result(inst)});
+			return true;
+		case IR::ValueOpcode::DppMoveU32: {
+			const auto flags    = inst.Flags<IR::DppMoveFlags>();
+			const auto target   = EmitDppTargetLane(state, flags.control);
+			const auto shuffled = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
+			                           target.lane});
+			if (flags.fetch_inactive) {
+				ctx.Define(inst, shuffled);
+				return true;
+			}
+			const auto ballot = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
+			const auto source_active = EmitBallotLaneActiveBool(state, ballot, target.lane);
+			const auto can_fetch     = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
+			ctx.Emit(inst, OpSelect, IR::Type::U32, {can_fetch, shuffled, ConstantU32(state, 0)});
+			return true;
+		}
+		case IR::ValueOpcode::DppUpdateU32: {
+			const auto flags = inst.Flags<IR::DppMoveFlags>();
+			const auto write = EmitDppWriteCondition(ctx, flags, ctx.Arg(inst, 2));
+			ctx.Emit(inst, OpSelect, IR::Type::U32, {write, ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		}
+		case IR::ValueOpcode::WqmMask:
+			ctx.Define(inst, EmitWqm(ctx, ctx.Arg(inst, 0)));
+			return true;
+		case IR::ValueOpcode::LaneId:
+			ctx.Define(inst, EmitSubgroupLocalInvocationId(state));
+			return true;
+		case IR::ValueOpcode::Ballot:
+			if (state.per_invocation_masks) {
+				const auto result = state.builder.AllocateId();
+				state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result,
+				                           BoolAsU32(ctx, ctx.Arg(inst, 0)), ConstantU32(state, 0),
+				                           ConstantU32(state, 0), ConstantU32(state, 0)});
+				ctx.Define(inst, result);
+			} else {
+				ctx.Emit(inst, OpGroupNonUniformBallot, IR::Type::U32x4,
+				         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0)});
+			}
+			return true;
+		case IR::ValueOpcode::ReadFirstLane: {
+			const auto ballot = state.builder.AllocateId();
+			const auto lane   = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 1)});
+			state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, lane,
+			                           ConstantU32(state, ScopeSubgroup), ballot});
+			ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), lane});
+			return true;
+		}
+		case IR::ValueOpcode::ReadLane:
+			ctx.Emit(inst, OpGroupNonUniformShuffle, IR::Type::U32,
+			         {ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0), ctx.Arg(inst, 1)});
+			return true;
+		case IR::ValueOpcode::WriteLane: {
+			const auto hit = state.builder.AllocateId();
+			state.builder.AddFunction({OpIEqual, state.bool_type, hit,
+			                           EmitSubgroupLocalInvocationId(state), ctx.Arg(inst, 2)});
+			ctx.Emit(inst, OpSelect, IR::Type::U32, {hit, ctx.Arg(inst, 1), ctx.Arg(inst, 0)});
+			return true;
+		}
+		case IR::ValueOpcode::Permlane16U32: {
+			const auto flags     = inst.Flags<IR::PermlaneFlags>();
+			const auto subid     = EmitSubgroupLocalInvocationId(state);
+			const auto row       = state.builder.AllocateId();
+			const auto row_value = state.builder.AllocateId();
+			const auto lane      = state.builder.AllocateId();
+			const auto lane8     = state.builder.AllocateId();
+			const auto shift     = state.builder.AllocateId();
+			const auto upper     = state.builder.AllocateId();
+			const auto selected  = state.builder.AllocateId();
+			const auto shifted   = state.builder.AllocateId();
+			const auto index     = state.builder.AllocateId();
+			const auto target    = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
+			if (flags.x16) {
+				state.builder.AddFunction(
+				    {OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
+			} else {
+				state.builder.AddFunction({OpCopyObject, state.uint_type, row_value, row});
+			}
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
+			state.builder.AddFunction(
+			    {OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
+			state.builder.AddFunction(
+			    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
+			state.builder.AddFunction(
+			    {OpSelect, state.uint_type, selected, upper, ctx.Arg(inst, 2), ctx.Arg(inst, 1)});
+			state.builder.AddFunction(
+			    {OpShiftRightLogical, state.uint_type, shifted, selected, shift});
+			state.builder.AddFunction(
+			    {OpBitwiseAnd, state.uint_type, index, shifted, ConstantU32(state, 15)});
+			state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index});
+			const auto shuffled = state.builder.AllocateId();
+			state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+			                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 0),
+			                           target});
+			uint32_t result = shuffled;
+			if (!flags.fetch_inactive) {
+				const auto source_exec = state.builder.AllocateId();
+				state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
+				                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 3),
+				                           target});
+				result = state.builder.AllocateId();
+				state.builder.AddFunction({OpSelect, state.uint_type, result, source_exec, shuffled,
+				                           ConstantU32(state, 0)});
+			}
+			ctx.Define(inst, result);
+			return true;
+		}
+		case IR::ValueOpcode::GetAttribute:
+			ctx.Define(inst, EmitAttribute(ctx, inst.Arg(0).U32(), inst.Arg(1).U32()));
+			return true;
+		case IR::ValueOpcode::SetAttribute: EmitExport(ctx, inst); return true;
+		case IR::ValueOpcode::GetShaderBase:
+			// Guest S_GETPC values stay shader-relative in SPIR-V, matching the runtime ABI. The
+			// runtime descriptor evaluator supplies the mapped shader base for host-side planning.
+			ctx.Define(inst, ctx.Def(IR::Value(uint64_t {0})));
+			return true;
+		case IR::ValueOpcode::GetSrtResource:
+		case IR::ValueOpcode::GetBufferResource:
+		case IR::ValueOpcode::GetAddressResource:
+		case IR::ValueOpcode::GetImageResource:
+		case IR::ValueOpcode::GetSamplerResource:
+		case IR::ValueOpcode::GetLdsResource:
+		case IR::ValueOpcode::GetGdsResource:
+		case IR::ValueOpcode::MakeImageAddress: return true;
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterHelpers.cpp

@@ -0,0 +1,247 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t EmitTrueBool(EmitterState& state) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
+	return ret;
+}
+
+DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
+	const auto quad_base = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto shift     = state.builder.AllocateId();
+	const auto selected0 = state.builder.AllocateId();
+	const auto selected  = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
+	state.builder.AddFunction(
+	    {OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
+	return {target, EmitTrueBool(state)};
+}
+
+DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
+                                        bool left) {
+	const auto row          = state.builder.AllocateId();
+	const auto lane         = state.builder.AllocateId();
+	const auto lane_shifted = state.builder.AllocateId();
+	const auto target       = state.builder.AllocateId();
+	const auto valid        = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+	if (left) {
+		state.builder.AddFunction(
+		    {OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
+		state.builder.AddFunction(
+		    {OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
+	} else {
+		state.builder.AddFunction(
+		    {OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
+	}
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
+	return {target, valid};
+}
+
+DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
+                                              uint32_t amount) {
+	const auto row      = state.builder.AllocateId();
+	const auto lane     = state.builder.AllocateId();
+	const auto in_high  = state.builder.AllocateId();
+	const auto minus    = state.builder.AllocateId();
+	const auto plus     = state.builder.AllocateId();
+	const auto selected = state.builder.AllocateId();
+	const auto target   = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
+	state.builder.AddFunction(
+	    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
+	state.builder.AddFunction(
+	    {OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
+	state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
+	return {target, EmitTrueBool(state)};
+}
+
+DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
+	const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
+	const auto lane_mask = half_row ? 7u : 15u;
+	const auto base      = state.builder.AllocateId();
+	const auto lane      = state.builder.AllocateId();
+	const auto mirrored  = state.builder.AllocateId();
+	const auto target    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
+	state.builder.AddFunction(
+	    {OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
+	return {target, EmitTrueBool(state)};
+}
+
+DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
+	const auto subid = EmitSubgroupLocalInvocationId(state);
+	if (control <= 0xffu) {
+		return EmitDppQuadPermTargetLane(state, subid, control);
+	}
+	if (control >= 0x101u && control <= 0x10fu) {
+		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
+	}
+	if (control >= 0x111u && control <= 0x11fu) {
+		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
+	}
+	if (control >= 0x121u && control <= 0x12fu) {
+		return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
+	}
+	if (control == 0x140u) {
+		return EmitDppMirrorTargetLane(state, subid, false);
+	}
+	if (control == 0x141u) {
+		return EmitDppMirrorTargetLane(state, subid, true);
+	}
+	return {subid, EmitTrueBool(state)};
+}
+
+uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
+	if (state.subgroup_local_invocation_id_variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	const auto value = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
+	return value;
+}
+
+uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
+	for (const auto& input: state.inputs) {
+		if (input.kind == kind) {
+			return input.variable_id;
+		}
+	}
+	return 0;
+}
+
+const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
+	for (const auto& input: state.inputs) {
+		if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
+			return &input;
+		}
+	}
+	return nullptr;
+}
+
+uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
+	const auto variable = InputVariableForKind(state, kind);
+	if (variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	const auto pointer = state.builder.AllocateId();
+	const auto value   = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
+	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+	return value;
+}
+
+uint32_t EmitLocalInvocationIndex(EmitterState& state) {
+	const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
+	if (variable == 0) {
+		return ConstantU32(state, 0);
+	}
+	const auto value = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
+	return value;
+}
+
+uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
+                                        uint32_t component) {
+	if ((component & 3u) != 3u) {
+		return ConstantU32(state, 0);
+	}
+	return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
+}
+
+uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
+                                         uint32_t component) {
+	const auto count = VertexParameterComponentCount(state, input);
+	const auto kind  = VertexParameterScalarKind(state, input.location);
+	if (component >= count) {
+		return VertexInputDefaultComponentU32(state, kind, component);
+	}
+
+	const auto scalar_type = VertexParameterScalarType(state, kind);
+	uint32_t   raw         = state.builder.AllocateId();
+	if (count == 1u) {
+		state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
+	} else {
+		const auto pointer_type = VertexParameterScalarPointerType(state, kind);
+		const auto pointer      = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
+		                           ConstantU32(state, component)});
+		state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
+	}
+
+	if (kind == VertexInputScalarKind::Uint) {
+		return raw;
+	}
+
+	const auto bits = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
+	return bits;
+}
+
+uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
+	const auto active_ballot = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
+	                           ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
+	return EmitBallotLaneActiveBool(state, active_ballot, lane);
+}
+uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
+	const auto low = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
+	uint32_t mask = low;
+	if (state.wave_size == 64u) {
+		const auto high     = state.builder.AllocateId();
+		const auto in_high  = state.builder.AllocateId();
+		const auto selected = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
+		state.builder.AddFunction(
+		    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
+		state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
+		mask = selected;
+	}
+
+	const auto lane_low = state.builder.AllocateId();
+	const auto bit      = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
+	state.builder.AddFunction(
+	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
+
+	const auto hit      = state.builder.AllocateId();
+	const auto active   = state.builder.AllocateId();
+	const auto in_range = state.builder.AllocateId();
+	const auto ret      = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
+	state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
+	state.builder.AddFunction(
+	    {OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
+	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
+	return ret;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterImage.cpp

@@ -0,0 +1,636 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+#include <bit>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
+	uint32_t index = 0;
+	for (uint32_t i = 0; i < component; i++) {
+		index += (dmask >> i) & 1u;
+	}
+	return index;
+}
+
+uint32_t ImageGatherComponent(uint32_t dmask) {
+	switch (dmask) {
+		case 0x2u: return 1;
+		case 0x4u: return 2;
+		case 0x8u: return 3;
+		default: return 0;
+	}
+}
+
+uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, value});
+	return result;
+}
+
+uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, lhs, rhs});
+	return result;
+}
+
+bool HasFlag(const IR::MemoryInfo& mem, uint32_t flag) {
+	return (mem.image_sample_flags & flag) != 0u;
+}
+
+bool UsesA16(const IR::MemoryInfo& mem) {
+	return HasFlag(mem, Decoder::ImageSampleFlagA16);
+}
+
+bool ComponentUsesA16(const IR::MemoryInfo& mem, uint32_t component) {
+	if (!UsesA16(mem)) return false;
+	uint32_t cursor = 0;
+	if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) {
+		if (component == cursor) return false;
+		cursor++;
+	}
+	if (HasFlag(mem, Decoder::ImageSampleFlagBias)) {
+		if (component == cursor) return true;
+		cursor++;
+	}
+	if (HasFlag(mem, Decoder::ImageSampleFlagCompare) && component == cursor) return false;
+	return true;
+}
+
+uint32_t HalfComponent(const IR::MemoryInfo& mem, uint32_t component) {
+	if (!UsesA16(mem)) return component * 2u;
+	uint32_t half = 0;
+	for (uint32_t index = 0; index < component; index++) {
+		const auto width = ComponentUsesA16(mem, index) ? 1u : 2u;
+		if (width == 2u && (half & 1u) != 0u) half++;
+		half += width;
+	}
+	if (!ComponentUsesA16(mem, component) && (half & 1u) != 0u) half++;
+	return half;
+}
+
+uint32_t AddressU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                    uint32_t component) {
+	const auto half   = HalfComponent(mem, component);
+	const auto packed = half / 2u;
+	if (packed >= address.NumArgs()) return ConstantU32(ctx.state, 0);
+	auto value = ctx.Def(address.Arg(packed));
+	if (ComponentUsesA16(mem, component)) {
+		if ((half & 1u) != 0u) {
+			value = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, value,
+			               ConstantU32(ctx.state, 16));
+		}
+		value = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, value,
+		               ConstantU32(ctx.state, 0xffffu));
+	}
+	return value;
+}
+
+uint32_t AddressF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                    uint32_t component) {
+	const auto value = AddressU32(ctx, mem, address, component);
+	return ComponentUsesA16(mem, component)
+	           ? EmitF16BitsToF32(ctx.state, value)
+	           : Unary(ctx.state, OpBitcast, ctx.state.float_type, value);
+}
+
+ImageSampleLayout Layout(const IR::MemoryInfo& mem, ImageViewKind view) {
+	ImageSampleLayout layout;
+	uint32_t          cursor = 0;
+	if (HasFlag(mem, Decoder::ImageSampleFlagOffset)) layout.offset = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagBias)) layout.bias = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagCompare)) layout.dref = cursor++;
+	if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
+		layout.grad_x = cursor;
+		cursor += ImageViewSpatialComponents(view);
+		layout.grad_y = cursor;
+		cursor += ImageViewSpatialComponents(view);
+	}
+	layout.coord = cursor;
+	cursor += ImageViewCoordinateComponents(view);
+	if (HasFlag(mem, Decoder::ImageSampleFlagLod)) layout.lod = cursor++;
+	return layout;
+}
+
+uint32_t ZeroF32(EmitterState& state) {
+	return ConstantF32(state, 0);
+}
+
+uint32_t ZeroU32x4(EmitterState& state) {
+	const auto zero  = ConstantU32(state, 0);
+	const auto value = state.builder.AllocateId();
+	state.builder.AddType(
+	    {OpConstantComposite, state.vec4_uint_type, value, zero, zero, zero, zero});
+	return value;
+}
+
+uint32_t CubeAxis(EmitterState& state, uint32_t value) {
+	return Binary(state, OpFSub, state.float_type, value, ConstantF32(state, 0x3f800000u));
+}
+
+uint32_t CubeLayer(EmitterState& state, uint32_t value) {
+	const auto guest = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertFToU, state.uint_type, guest, value});
+	const auto padding =
+	    Binary(state, OpShiftLeftLogical, state.uint_type,
+	           Binary(state, OpShiftRightLogical, state.uint_type, guest, ConstantU32(state, 3)),
+	           ConstantU32(state, 1));
+	const auto host   = Binary(state, OpISub, state.uint_type, guest, padding);
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertUToF, state.float_type, result, host});
+	return result;
+}
+
+uint32_t CoordF32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                  uint32_t first, uint32_t components) {
+	auto x = AddressF32(ctx, mem, address, first);
+	if (components == 1u) return x;
+	auto y = mem.image_address_components > first + 1u ? AddressF32(ctx, mem, address, first + 1u)
+	                                                   : ZeroF32(ctx.state);
+	if (mem.image_cube) {
+		x = CubeAxis(ctx.state, x);
+		y = CubeAxis(ctx.state, y);
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	if (components == 3u) {
+		auto z = mem.image_address_components > first + 2u
+		             ? AddressF32(ctx, mem, address, first + 2u)
+		             : ZeroF32(ctx.state);
+		if (mem.image_cube) z = CubeLayer(ctx.state, z);
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, ctx.state.vec3_float_type, result, x, y, z});
+	} else {
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, ctx.state.vec2_float_type, result, x, y});
+	}
+	return result;
+}
+
+uint32_t CoordU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                  ImageViewKind view) {
+	const auto components = ImageViewCoordinateComponents(view);
+	const auto x          = AddressU32(ctx, mem, address, 0);
+	if (components == 1u) return x;
+	const auto y      = mem.image_address_components > 1u ? AddressU32(ctx, mem, address, 1)
+	                                                      : ConstantU32(ctx.state, 0);
+	const auto result = ctx.state.builder.AllocateId();
+	if (components == 3u) {
+		const auto z = mem.image_address_components > 2u ? AddressU32(ctx, mem, address, 2)
+		                                                 : ConstantU32(ctx.state, 0);
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, ctx.state.vec3_uint_type, result, x, y, z});
+	} else {
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, ctx.state.vec2_uint_type, result, x, y});
+	}
+	return result;
+}
+
+uint32_t LodU32(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                ImageViewKind view) {
+	const auto component = ImageViewCoordinateComponents(view);
+	return mem.image_has_mip && mem.image_address_components > component
+	           ? AddressU32(ctx, mem, address, component)
+	           : ConstantU32(ctx.state, 0);
+}
+
+uint32_t FloatBits(ValueEmitContext& ctx, uint32_t value) {
+	return Unary(ctx.state, OpBitcast, ctx.state.uint_type, value);
+}
+
+uint32_t ResultVector(ValueEmitContext& ctx, uint32_t value, bool integer, bool dref) {
+	uint32_t component[4] {};
+	for (uint32_t index = 0; index < 4u; index++) {
+		if (dref) {
+			component[index] = index == 0u ? FloatBits(ctx, value) : ConstantU32(ctx.state, 0);
+			continue;
+		}
+		const auto scalar = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpCompositeExtract,
+		                               integer ? ctx.state.uint_type : ctx.state.float_type, scalar,
+		                               value, index});
+		component[index] = integer ? scalar : FloatBits(ctx, scalar);
+	}
+	const auto result = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, result,
+	                               component[0], component[1], component[2], component[3]});
+	return result;
+}
+
+uint32_t QueryDimensions(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                         const IR::Inst& address) {
+	const auto pc    = inst.Flags<IR::MemoryFlags>().pc;
+	const auto view  = SampledImageViewKind(ctx.state, mem, pc);
+	const auto image = LoadSampledImageDescriptor(ctx.state, mem, pc, view);
+	const auto size  = ctx.state.builder.AllocateId();
+	if (ImageSpirvMultisampled(view) != 0u) {
+		ctx.state.builder.AddFunction(
+		    {OpImageQuerySize, ImageViewSizeType(ctx.state, view), size, image});
+	} else {
+		ctx.state.builder.AddFunction({OpImageQuerySizeLod, ImageViewSizeType(ctx.state, view),
+		                               size, image, AddressU32(ctx, mem, address, 0)});
+	}
+	const auto components = ImageViewCoordinateComponents(view);
+	uint32_t   result[4]  = {ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0),
+	                         ConstantU32(ctx.state, 0), ConstantU32(ctx.state, 0)};
+	if (components == 1u) {
+		result[0] = size;
+	} else {
+		for (uint32_t index = 0; index < components; index++) {
+			result[index] = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpCompositeExtract, ctx.state.uint_type, result[index], size, index});
+		}
+	}
+	if (ImageSpirvMultisampled(view) == 0u) {
+		result[3] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpImageQueryLevels, ctx.state.uint_type, result[3], image});
+	}
+	const auto vector = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec4_uint_type, vector,
+	                               result[0], result[1], result[2], result[3]});
+	return vector;
+}
+
+uint32_t PackedOffset(ValueEmitContext& ctx, const IR::MemoryInfo& mem, const IR::Inst& address,
+                      const ImageSampleLayout& layout, ImageViewKind view) {
+	const auto components = ImageViewSpatialComponents(view);
+	const auto zero       = ConstantI32(ctx.state, 0);
+	if (layout.offset == NoImageComponent || mem.image_address_components <= layout.offset) {
+		if (components == 1u) return zero;
+		const auto result = ctx.state.builder.AllocateId();
+		if (components == 3u) {
+			ctx.state.builder.AddFunction(
+			    {OpCompositeConstruct, ctx.state.vec3_int_type, result, zero, zero, zero});
+		} else {
+			ctx.state.builder.AddFunction(
+			    {OpCompositeConstruct, ctx.state.vec2_int_type, result, zero, zero});
+		}
+		return result;
+	}
+	const auto packed    = Unary(ctx.state, OpBitcast, ctx.state.int_type,
+	                             AddressU32(ctx, mem, address, layout.offset));
+	uint32_t   values[3] = {zero, zero, zero};
+	for (uint32_t index = 0; index < components; index++) {
+		values[index] = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction({OpBitFieldSExtract, ctx.state.int_type, values[index],
+		                               packed, ConstantU32(ctx.state, index * 8u),
+		                               ConstantU32(ctx.state, 6)});
+	}
+	if (components == 1u) return values[0];
+	const auto result = ctx.state.builder.AllocateId();
+	if (components == 3u) {
+		ctx.state.builder.AddFunction({OpCompositeConstruct, ctx.state.vec3_int_type, result,
+		                               values[0], values[1], values[2]});
+	} else {
+		ctx.state.builder.AddFunction(
+		    {OpCompositeConstruct, ctx.state.vec2_int_type, result, values[0], values[1]});
+	}
+	return result;
+}
+
+uint32_t HorizontalOffsets(EmitterState& state, ImageViewKind view) {
+	const auto components   = ImageViewSpatialComponents(view);
+	const auto array_type   = state.builder.AllocateId();
+	const auto count        = ConstantU32(state, 4);
+	const auto element_type = components == 1u ? state.int_type : state.vec2_int_type;
+	state.builder.AddType({OpTypeArray, array_type, element_type, count});
+	uint32_t offsets[4] {};
+	for (uint32_t index = 0; index < 4u; index++) {
+		const auto x = ConstantI32(state, static_cast<int32_t>(index) - 1);
+		if (components == 1u) {
+			offsets[index] = x;
+		} else {
+			offsets[index] = state.builder.AllocateId();
+			state.builder.AddType({OpConstantComposite, state.vec2_int_type, offsets[index], x,
+			                       ConstantI32(state, 0)});
+		}
+	}
+	const auto value = state.builder.AllocateId();
+	state.builder.AddType(
+	    {OpConstantComposite, array_type, value, offsets[0], offsets[1], offsets[2], offsets[3]});
+	return value;
+}
+
+uint32_t InverseSwizzle(uint32_t swizzle, uint32_t component) {
+	for (uint32_t source = 0; source < 4u; source++) {
+		if (((swizzle >> (source * 3u)) & 7u) == 4u + component) return source;
+	}
+	return UINT32_MAX;
+}
+
+uint32_t StoreTexel(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t data, bool integer) {
+	const auto swizzle = ctx.state.program.info.images[mem.resource].storage_swizzle;
+	uint32_t   values[4] {};
+	const auto dmask = mem.dmask != 0u ? mem.dmask : 1u;
+	for (uint32_t component = 0; component < 4u; component++) {
+		const auto source = InverseSwizzle(swizzle, component);
+		uint32_t   raw    = ConstantU32(ctx.state, 0);
+		if (source < 4u && ((dmask >> source) & 1u) != 0u) {
+			const auto packed_index = DmaskComponentIndex(dmask, source);
+			raw                     = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpCompositeExtract, ctx.state.uint_type, raw, data, packed_index});
+		}
+		values[component] = integer ? raw : Unary(ctx.state, OpBitcast, ctx.state.float_type, raw);
+	}
+	const auto texel = ctx.state.builder.AllocateId();
+	ctx.state.builder.AddFunction({OpCompositeConstruct,
+	                               integer ? ctx.state.vec4_uint_type : ctx.state.vec4_float_type,
+	                               texel, values[0], values[1], values[2], values[3]});
+	return texel;
+}
+
+uint32_t ImageAtomicOpcode(IR::ValueOpcode opcode) {
+	switch (opcode) {
+		case IR::ValueOpcode::ImageAtomicIAdd32: return OpAtomicIAdd;
+		case IR::ValueOpcode::ImageAtomicUMin32: return OpAtomicUMin;
+		case IR::ValueOpcode::ImageAtomicUMax32: return OpAtomicUMax;
+		case IR::ValueOpcode::ImageAtomicAnd32: return OpAtomicAnd;
+		case IR::ValueOpcode::ImageAtomicOr32: return OpAtomicOr;
+		case IR::ValueOpcode::ImageAtomicXor32: return OpAtomicXor;
+		default: return 0;
+	}
+}
+
+} // namespace
+
+bool EmitValueImage(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto op = inst.GetOpcode();
+	if (op < IR::ValueOpcode::ImageQueryDimensions || op > IR::ValueOpcode::ImageAtomicXor32) {
+		return false;
+	}
+	auto&       state     = ctx.state;
+	const auto& mem       = ctx.Memory(inst);
+	const auto  pc        = inst.Flags<IR::MemoryFlags>().pc;
+	const auto  image_arg = inst.Arg(0);
+	ctx.ResourceIndex(image_arg, IR::ValueOpcode::GetImageResource);
+	const bool  sampled = op == IR::ValueOpcode::ImageQueryLod ||
+	                      op == IR::ValueOpcode::ImageSampleRaw ||
+	                      op == IR::ValueOpcode::ImageGatherRaw;
+	const auto* address = ctx.ImageAddress(inst.Arg(sampled ? 2 : 1));
+	if (address == nullptr) return true;
+	if (op == IR::ValueOpcode::ImageQueryDimensions) {
+		ctx.Define(inst, QueryDimensions(ctx, inst, mem, *address));
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageQueryLod) {
+		const auto view    = SampledImageViewKind(state, mem, pc);
+		const auto sampled = MakeSampledImage(state, mem, pc, view);
+		const auto lod     = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpImageQueryLod, state.vec2_float_type, lod, sampled,
+		     CoordF32(ctx, mem, *address, 0, ImageViewSpatialComponents(view))});
+		uint32_t values[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0),
+		                      ConstantU32(state, 0)};
+		for (uint32_t index = 0; index < 2u; index++) {
+			const auto component = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpCompositeExtract, state.float_type, component, lod, index});
+			values[index] = FloatBits(ctx, component);
+		}
+		const auto result = state.builder.AllocateId();
+		state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, result, values[0],
+		                           values[1], values[2], values[3]});
+		ctx.Define(inst, result);
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageRead) {
+		const auto view      = SampledImageViewKind(state, mem, pc);
+		const bool integer   = mem.kind == IR::ResourceKind::ImageUint;
+		const auto condition = ctx.Arg(inst, 2);
+		ctx.Define(
+		    inst,
+		    EmitValueOrDefaultIfCondition(
+		        state, condition, state.vec4_uint_type, ZeroU32x4(state), [&]() {
+			        const auto image = LoadSampledImageDescriptor(state, mem, pc, view);
+			        const auto color = state.builder.AllocateId();
+			        const auto coord = CoordU32(ctx, mem, *address, view);
+			        if (ImageSpirvMultisampled(view) != 0u) {
+				        state.builder.AddFunction(
+				            {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
+				             color, image, coord, ImageOperandsSampleMask,
+				             AddressU32(ctx, mem, *address, ImageViewCoordinateComponents(view))});
+			        } else {
+				        state.builder.AddFunction(
+				            {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type,
+				             color, image, coord, ImageOperandsLodMask,
+				             LodU32(ctx, mem, *address, view)});
+			        }
+			        return ResultVector(ctx, color, integer, false);
+		        }));
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageWrite) {
+		const auto uint_image = mem.kind == IR::ResourceKind::StorageImageUint;
+		const auto view       = StorageImageViewKind(state, mem, uint_image, pc);
+		EmitIfCondition(state, ctx.Arg(inst, 3), [&]() {
+			const auto mip_lod =
+			    state.program.info.images[mem.resource].mip_mode == IR::ImageMipMode::DynamicStorage
+			        ? LodU32(ctx, mem, *address, view)
+			        : 0u;
+			const auto coord = CoordU32(ctx, mem, *address, view);
+			const auto texel = StoreTexel(ctx, mem, ctx.Arg(inst, 2), uint_image);
+			EmitStorageImageWrite(state, mem.resource, uint_image, view, mip_lod, coord, texel);
+		});
+		return true;
+	}
+	if (op == IR::ValueOpcode::ImageSampleRaw || op == IR::ValueOpcode::ImageGatherRaw) {
+		const auto view    = SampledImageViewKind(state, mem, pc);
+		const auto layout  = Layout(mem, view);
+		const bool integer = mem.kind == IR::ResourceKind::ImageUint;
+		const bool dref    = HasFlag(mem, Decoder::ImageSampleFlagCompare);
+		const auto coord =
+		    CoordF32(ctx, mem, *address, layout.coord, ImageViewCoordinateComponents(view));
+		if (op == IR::ValueOpcode::ImageGatherRaw) {
+			const auto            sampled = MakeSampledImage(state, mem, pc, view);
+			const auto            sample  = state.builder.AllocateId();
+			std::vector<uint32_t> words =
+			    dref ? std::vector<uint32_t> {OpImageDrefGather,
+			                                  state.vec4_float_type,
+			                                  sample,
+			                                  sampled,
+			                                  coord,
+			                                  layout.dref != NoImageComponent
+			                                      ? AddressF32(ctx, mem, *address, layout.dref)
+			                                      : ZeroF32(state)}
+			         : std::vector<uint32_t> {
+			               OpImageGather, integer ? state.vec4_uint_type : state.vec4_float_type,
+			               sample,        sampled,
+			               coord,         ConstantU32(state, ImageGatherComponent(mem.dmask))};
+			if (HasFlag(mem, Decoder::ImageSampleFlagGatherHorizontal)) {
+				words.push_back(ImageOperandsConstOffsetsMask);
+				words.push_back(HorizontalOffsets(state, view));
+			} else if (layout.offset != NoImageComponent) {
+				words.push_back(ImageOperandsOffsetMask);
+				words.push_back(PackedOffset(ctx, mem, *address, layout, view));
+			}
+			state.builder.AddFunction(words);
+			ctx.Define(inst, ResultVector(ctx, sample, integer && !dref, false));
+			return true;
+		}
+		const bool explicit_lod = HasFlag(mem, Decoder::ImageSampleFlagDerivative) ||
+		                          HasFlag(mem, Decoder::ImageSampleFlagLod) ||
+		                          HasFlag(mem, Decoder::ImageSampleFlagLevelZero) ||
+		                          state.stage != ShaderType::Pixel;
+		const auto opcode = explicit_lod
+		                        ? (dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod)
+		                        : (dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod);
+		const auto result_type =
+		    dref ? state.float_type : (integer ? state.vec4_uint_type : state.vec4_float_type);
+		const auto dref_value =
+		    dref ? (layout.dref != NoImageComponent ? AddressF32(ctx, mem, *address, layout.dref)
+		                                            : ZeroF32(state))
+		         : 0u;
+		uint32_t              operand_mask = 0;
+		std::vector<uint32_t> operands;
+		if (HasFlag(mem, Decoder::ImageSampleFlagDerivative)) {
+			operand_mask |= ImageOperandsGradMask;
+			operands.push_back(
+			    CoordF32(ctx, mem, *address, layout.grad_x, ImageViewSpatialComponents(view)));
+			operands.push_back(
+			    CoordF32(ctx, mem, *address, layout.grad_y, ImageViewSpatialComponents(view)));
+		} else if (explicit_lod) {
+			operand_mask |= ImageOperandsLodMask;
+			operands.push_back(HasFlag(mem, Decoder::ImageSampleFlagLod) &&
+			                           layout.lod != NoImageComponent
+			                       ? AddressF32(ctx, mem, *address, layout.lod)
+			                       : ZeroF32(state));
+		} else if (layout.bias != NoImageComponent) {
+			operand_mask |= ImageOperandsBiasMask;
+			operands.push_back(AddressF32(ctx, mem, *address, layout.bias));
+		}
+		const auto EmitSample = [&](uint32_t resource) {
+			const auto            sampled = MakeSampledImage(state, mem, pc, view, resource);
+			const auto            sample  = state.builder.AllocateId();
+			std::vector<uint32_t> words {opcode, result_type, sample, sampled, coord};
+			if (dref) {
+				words.push_back(dref_value);
+			}
+			if (operand_mask != 0u) {
+				words.push_back(operand_mask);
+				words.insert(words.end(), operands.begin(), operands.end());
+			}
+			state.builder.AddFunction(words);
+			return sample;
+		};
+		const auto& image = state.program.info.images[mem.resource];
+		if (image.indirect_root != mem.resource) {
+			ctx.Define(inst, ResultVector(ctx, EmitSample(mem.resource), integer, dref));
+			return true;
+		}
+		const auto* handle = image_arg.ResolveInstruction();
+		const auto* source = image.source < ctx.program.descriptor_sources.size()
+		                         ? &ctx.program.descriptor_sources[image.source]
+		                         : nullptr;
+		if (handle == nullptr || source == nullptr || !source->indirect_image.has_value() ||
+		    source->indirect_image->key_arg >= handle->NumArgs()) {
+			ctx.Fail(inst, "has invalid indirect image key provenance");
+			return true;
+		}
+		const auto key = ctx.Def(handle->Arg(source->indirect_image->key_arg));
+		if (state.flattened_srt_variable == 0 || image.indirect_mapping_capacity == 0u ||
+		    image.indirect_resources.size() < 2u) {
+			ctx.Fail(inst, "has no indirect image runtime mapping");
+			return true;
+		}
+		const auto LoadMapping = [&](uint32_t index) {
+			const auto pointer = state.builder.AllocateId();
+			state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
+			                           state.flattened_srt_variable, ConstantU32(state, 0), index});
+			const auto value = state.builder.AllocateId();
+			state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
+			return value;
+		};
+		const auto mapping  = ConstantU32(state, image.indirect_mapping_offset);
+		auto       low      = ConstantU32(state, 0u);
+		auto       high     = LoadMapping(mapping);
+		auto       selected = ConstantU32(state, 0u);
+		for (uint32_t iteration = 0; iteration < std::bit_width(image.indirect_mapping_capacity);
+		     iteration++) {
+			const auto active = Binary(state, OpULessThan, state.bool_type, low, high);
+			const auto mid =
+			    Binary(state, OpShiftRightLogical, state.uint_type,
+			           Binary(state, OpIAdd, state.uint_type, low, high), ConstantU32(state, 1u));
+			const auto probe = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, state.uint_type, probe, active, mid, ConstantU32(state, 0u)});
+			const auto entry      = Binary(state, OpIAdd, state.uint_type, mapping,
+			                               Binary(state, OpIAdd, state.uint_type,
+			                                      Binary(state, OpShiftLeftLogical, state.uint_type,
+			                                             probe, ConstantU32(state, 1u)),
+			                                      ConstantU32(state, 1u)));
+			const auto mapped_key = LoadMapping(entry);
+			const auto candidate =
+			    LoadMapping(Binary(state, OpIAdd, state.uint_type, entry, ConstantU32(state, 1u)));
+			const auto equal         = Binary(state, OpIEqual, state.bool_type, mapped_key, key);
+			const auto match         = Binary(state, OpLogicalAnd, state.bool_type, active, equal);
+			const auto next_selected = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, state.uint_type, next_selected, match, candidate, selected});
+			selected              = next_selected;
+			const auto less       = Binary(state, OpULessThan, state.bool_type, mapped_key, key);
+			const auto take_upper = Binary(state, OpLogicalAnd, state.bool_type, active, less);
+			const auto take_lower = Binary(state, OpLogicalAnd, state.bool_type, active,
+			                               Unary(state, OpLogicalNot, state.bool_type, less));
+			const auto next_low   = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, state.uint_type, next_low, take_upper,
+			     Binary(state, OpIAdd, state.uint_type, mid, ConstantU32(state, 1u)), low});
+			low                  = next_low;
+			const auto next_high = state.builder.AllocateId();
+			state.builder.AddFunction(
+			    {OpSelect, state.uint_type, next_high, take_lower, mid, high});
+			high = next_high;
+		}
+		const auto            default_label = state.builder.AllocateId();
+		const auto            merge_label   = state.builder.AllocateId();
+		std::vector<uint32_t> labels(image.indirect_resources.size() - 1u);
+		std::vector<uint32_t> switch_words {OpSwitch, selected, default_label};
+		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
+			labels[candidate - 1u] = state.builder.AllocateId();
+			switch_words.push_back(candidate);
+			switch_words.push_back(labels[candidate - 1u]);
+		}
+		state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+		state.builder.AddFunction(switch_words);
+		std::vector<uint32_t> phi_words {OpPhi, result_type, state.builder.AllocateId()};
+		state.builder.AddFunction({OpLabel, default_label});
+		phi_words.push_back(EmitSample(image.indirect_resources[0]));
+		phi_words.push_back(default_label);
+		state.builder.AddFunction({OpBranch, merge_label});
+		for (uint32_t candidate = 1; candidate < image.indirect_resources.size(); candidate++) {
+			state.builder.AddFunction({OpLabel, labels[candidate - 1u]});
+			phi_words.push_back(EmitSample(image.indirect_resources[candidate]));
+			phi_words.push_back(labels[candidate - 1u]);
+			state.builder.AddFunction({OpBranch, merge_label});
+		}
+		state.builder.AddFunction({OpLabel, merge_label});
+		state.builder.AddFunction(phi_words);
+		ctx.Define(inst, ResultVector(ctx, phi_words[2], integer, dref));
+		return true;
+	}
+	const auto atomic_opcode = ImageAtomicOpcode(op);
+	if (atomic_opcode != 0u) {
+		const auto view = StorageImageViewKind(state, mem, true, pc);
+		ctx.Define(inst, EmitValueOrZeroIfCondition(state, ctx.Arg(inst, 3), [&]() {
+			           const auto pointer = state.builder.AllocateId();
+			           state.builder.AddFunction(
+			               {OpImageTexelPointer, state.ptr_image_uint, pointer,
+			                StorageImageDescriptorPointer(state, mem.resource, true, pc, view),
+			                CoordU32(ctx, mem, *address, view), ConstantU32(state, 0)});
+			           const auto old = state.builder.AllocateId();
+			           state.builder.AddFunction({atomic_opcode, state.uint_type, old, pointer,
+			                                      ConstantU32(state, ScopeDevice),
+			                                      ConstantU32(state, MemorySemanticsNone),
+			                                      ctx.Arg(inst, 2)});
+			           EmitDeviceAtomicMemoryBarrier(state);
+			           return old;
+		           }));
+		return true;
+	}
+	return false;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemory.cpp

@@ -0,0 +1,729 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+uint32_t Unary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, value});
+	return result;
+}
+
+uint32_t Binary(EmitterState& state, uint32_t opcode, uint32_t type, uint32_t lhs, uint32_t rhs) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, type, result, lhs, rhs});
+	return result;
+}
+
+uint32_t Select(EmitterState& state, uint32_t type, uint32_t condition, uint32_t true_value,
+                uint32_t false_value) {
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpSelect, type, result, condition, true_value, false_value});
+	return result;
+}
+
+uint32_t AndCondition(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	if (lhs == 0) return rhs;
+	if (rhs == 0) return lhs;
+	return Binary(state, OpLogicalAnd, state.bool_type, lhs, rhs);
+}
+
+uint32_t BufferByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                           uint32_t index, uint32_t offset, uint32_t soffset) {
+	auto&      state = ctx.state;
+	const auto packed =
+	    ConstantU32(state, StorageBufferPackedStride(state, mem, inst.Flags<IR::MemoryFlags>().pc));
+	if (mem.formatted && !mem.typed && inst.GetOpcode() == IR::ValueOpcode::StoreBufferU32) {
+		const auto add_tid_bit = Binary(
+		    state, OpBitwiseAnd, state.uint_type,
+		    Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 20)),
+		    ConstantU32(state, 1));
+		const auto add_tid =
+		    Binary(state, OpINotEqual, state.bool_type, add_tid_bit, ConstantU32(state, 0));
+		const auto lane = Binary(state, OpBitwiseAnd, state.uint_type,
+		                         EmitLocalInvocationIndex(state), ConstantU32(state, 63));
+		index           = Select(state, state.uint_type, add_tid,
+		                         Binary(state, OpIAdd, state.uint_type, index, lane), index);
+	}
+	const auto stride =
+	    Binary(state, OpBitwiseAnd, state.uint_type, packed, ConstantU32(state, 0x3fffu));
+	const auto swizzle =
+	    Binary(state, OpBitwiseAnd, state.uint_type,
+	           Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 14)),
+	           ConstantU32(state, 1));
+	const auto use_swizzle =
+	    Binary(state, OpLogicalAnd, state.bool_type,
+	           Binary(state, OpINotEqual, state.bool_type, stride, ConstantU32(state, 0)),
+	           Binary(state, OpINotEqual, state.bool_type, swizzle, ConstantU32(state, 0)));
+	const auto stride_enum =
+	    Binary(state, OpBitwiseAnd, state.uint_type,
+	           Binary(state, OpShiftRightLogical, state.uint_type, packed, ConstantU32(state, 16)),
+	           ConstantU32(state, 3));
+	const auto index_shift =
+	    Binary(state, OpIAdd, state.uint_type, stride_enum, ConstantU32(state, 3));
+	const auto index_stride =
+	    Binary(state, OpShiftLeftLogical, state.uint_type, ConstantU32(state, 8), stride_enum);
+	const auto index_mask =
+	    Binary(state, OpISub, state.uint_type, index_stride, ConstantU32(state, 1));
+	offset = Binary(state, OpIAdd, state.uint_type, offset, ConstantU32(state, mem.offset));
+	const auto linear    = Binary(state, OpIAdd, state.uint_type,
+	                              Binary(state, OpIMul, state.uint_type, index, stride), offset);
+	const auto index_msb = Binary(state, OpShiftRightLogical, state.uint_type, index, index_shift);
+	const auto index_lsb = Binary(state, OpBitwiseAnd, state.uint_type, index, index_mask);
+	const auto offset_msb =
+	    Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, ~3u));
+	const auto offset_lsb =
+	    Binary(state, OpBitwiseAnd, state.uint_type, offset, ConstantU32(state, 3));
+	const auto msb =
+	    Binary(state, OpIMul, state.uint_type,
+	           Binary(state, OpIAdd, state.uint_type,
+	                  Binary(state, OpIMul, state.uint_type, index_msb, stride), offset_msb),
+	           index_stride);
+	const auto lsb =
+	    Binary(state, OpIAdd, state.uint_type,
+	           Binary(state, OpShiftLeftLogical, state.uint_type, index_lsb, ConstantU32(state, 2)),
+	           offset_lsb);
+	const auto address = Select(state, state.uint_type, use_swizzle,
+	                            Binary(state, OpIAdd, state.uint_type, msb, lsb), linear);
+	return Binary(state, OpIAdd, state.uint_type, address, soffset);
+}
+
+uint32_t AddU64Low(EmitterState& state, uint32_t low, uint32_t high, uint32_t add_low,
+                   uint32_t add_high, uint32_t& out_high) {
+	const auto result = Binary(state, OpIAdd, state.uint_type, low, add_low);
+	const auto carry  = Binary(state, OpULessThan, state.bool_type, result, low);
+	out_high          = Binary(
+	    state, OpIAdd, state.uint_type, Binary(state, OpIAdd, state.uint_type, high, add_high),
+	    Select(state, state.uint_type, carry, ConstantU32(state, 1), ConstantU32(state, 0)));
+	return result;
+}
+
+uint32_t AddressByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                            uint32_t low, uint32_t high) {
+	auto& state     = ctx.state;
+	auto  immediate = static_cast<int32_t>(mem.offset);
+	if (mem.kind == IR::ResourceKind::ScalarAddress) {
+		immediate = static_cast<int32_t>(static_cast<uint32_t>(immediate) & ~3u);
+		low       = Binary(state, OpBitwiseAnd, state.uint_type, low, ConstantU32(state, ~3u));
+	}
+	const auto immediate_low  = ConstantU32(state, static_cast<uint32_t>(immediate));
+	const auto immediate_high = ConstantU32(state, immediate < 0 ? UINT32_MAX : 0u);
+	const auto base           = state.program.info.addresses[mem.resource].specialized_base;
+	if (mem.kind == IR::ResourceKind::Scratch) {
+		low              = AddU64Low(state, low, high, immediate_low, immediate_high, high);
+		const auto valid = Binary(state, OpIEqual, state.bool_type, high, ConstantU32(state, 0));
+		return Select(state, state.uint_type, valid, low, ConstantU32(state, UINT32_MAX));
+	}
+	if (mem.kind == IR::ResourceKind::Flat ||
+	    (mem.kind == IR::ResourceKind::Global &&
+	     state.program.info.addresses[mem.resource].unbased)) {
+		low                      = AddU64Low(state, low, high, immediate_low, immediate_high, high);
+		const auto base_low      = ConstantU32(state, static_cast<uint32_t>(base));
+		const auto base_high     = ConstantU32(state, static_cast<uint32_t>(base >> 32u));
+		const auto relative      = Binary(state, OpISub, state.uint_type, low, base_low);
+		const auto borrow        = Binary(state, OpULessThan, state.bool_type, low, base_low);
+		const auto relative_high = Binary(
+		    state, OpISub, state.uint_type, Binary(state, OpISub, state.uint_type, high, base_high),
+		    Select(state, state.uint_type, borrow, ConstantU32(state, 1), ConstantU32(state, 0)));
+		const auto valid =
+		    Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0));
+		return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX));
+	}
+	const auto initial       = base + static_cast<uint64_t>(static_cast<int64_t>(immediate));
+	auto       relative_high = ConstantU32(state, static_cast<uint32_t>(initial >> 32u));
+	const auto relative      = AddU64Low(state, ConstantU32(state, static_cast<uint32_t>(initial)),
+	                                     relative_high, low, ConstantU32(state, 0), relative_high);
+	const auto valid =
+	    Binary(state, OpIEqual, state.bool_type, relative_high, ConstantU32(state, 0));
+	return Select(state, state.uint_type, valid, relative, ConstantU32(state, UINT32_MAX));
+}
+
+uint32_t ByteAddress(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	if (mem.kind == IR::ResourceKind::Buffer) {
+		return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2),
+		                         ctx.Arg(inst, 3));
+	}
+	if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
+		return ctx.Arg(inst, 1);
+	}
+	return AddressByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2));
+}
+
+uint32_t DwordIndex(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	return Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, ByteAddress(ctx, inst, mem),
+	              ConstantU32(ctx.state, 2));
+}
+
+uint32_t InBounds(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
+	if (mem.kind == IR::ResourceKind::Lds) {
+		return EmitLdsElementInBounds(ctx.state, index);
+	}
+	if (mem.kind == IR::ResourceKind::Gds) {
+		return EmitGdsElementInBounds(ctx.state, index);
+	}
+	return EmitStorageBufferElementInBounds(ctx.state, mem, index, pc);
+}
+
+uint32_t Pointer(ValueEmitContext& ctx, const IR::MemoryInfo& mem, uint32_t index, uint32_t pc) {
+	return EmitMemoryElementPointer(ctx.state, mem, index, pc);
+}
+
+uint32_t LoadWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
+                  uint32_t extra_offset = 0) {
+	mem.offset += extra_offset;
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto index = DwordIndex(ctx, inst, mem);
+		return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			const auto value = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpLoad, ctx.state.uint_type, value, Pointer(ctx, mem, index, pc)});
+			return value;
+		});
+	});
+}
+
+uint32_t LoadSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
+                     bool sign_extend, uint32_t extra_offset = 0) {
+	mem.offset += extra_offset;
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto address = ByteAddress(ctx, inst, mem);
+		const auto index   = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address,
+		                            ConstantU32(ctx.state, 2));
+		return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			const auto word = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpLoad, ctx.state.uint_type, word, Pointer(ctx, mem, index, pc)});
+			const auto byte  = Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address,
+			                          ConstantU32(ctx.state, 3));
+			const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, byte,
+			                          ConstantU32(ctx.state, 3));
+			const auto value =
+			    Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type,
+			           Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, word, shift),
+			           ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu));
+			if (!sign_extend) return value;
+			const auto left = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type, value,
+			                         ConstantU32(ctx.state, 32u - bits));
+			return Binary(ctx.state, OpShiftRightArithmetic, ctx.state.uint_type, left,
+			              ConstantU32(ctx.state, 32u - bits));
+		});
+	});
+}
+
+Prospero::BufferFormat BufferFormat(const ValueEmitContext& ctx, const IR::Inst& inst,
+                                    const IR::MemoryInfo& mem) {
+	return mem.typed ? Format::DecodeTBufferFormat(mem.data_format, mem.number_format)
+	                 : StorageBufferFormat(ctx.state, mem, inst.Flags<IR::MemoryFlags>().pc);
+}
+
+IR::MemoryInfo RebaseFormattedComponent(IR::MemoryInfo mem, Prospero::BufferFormat format,
+                                        uint32_t component) {
+	if (!mem.typed) {
+		const auto component_slot = mem.component_index * 4u;
+		mem.offset                = mem.offset >= component_slot ? mem.offset - component_slot : 0u;
+		mem.offset += Format::GetFormatComponentByteOffset(format, component);
+	}
+	return mem;
+}
+
+uint32_t FormattedLoad(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	const auto format = BufferFormat(ctx, inst, mem);
+	if (!Format::IsKnownFormat(format)) return LoadWord(ctx, inst, mem);
+	const auto info      = Format::GetFormatInfo(format);
+	auto       component = mem.component_index;
+	if (!mem.typed) {
+		const auto selector =
+		    GetDstSel(ctx.state.program.info.buffers[mem.resource].descriptor_swizzle, component);
+		if (selector == 0u) return ConstantU32(ctx.state, 0);
+		if (selector == 1u) {
+			const auto integer = info.type == Format::ComponentType::Uint ||
+			                     info.type == Format::ComponentType::Sint;
+			return ConstantU32(ctx.state, integer ? 1u : 0x3f800000u);
+		}
+		if (selector < 4u) {
+			ExitDescriptorBindingFailure(ctx.state, IR::DescriptorBindingKind::Buffers,
+			                             mem.resource, "buffer descriptor has reserved dst_sel");
+		}
+		component = selector - 4u;
+	}
+	if (component >= info.component_count) return ConstantU32(ctx.state, 0);
+	uint32_t   raw  = 0;
+	const auto bits = info.component_bits[component];
+	if (info.packed_bitfield) {
+		raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component));
+		const auto type =
+		    IsSignedFormatComponent(info.type) ? ctx.state.int_type : ctx.state.uint_type;
+		const auto source =
+		    type == ctx.state.int_type ? Unary(ctx.state, OpBitcast, type, raw) : raw;
+		const auto extracted = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {IsSignedFormatComponent(info.type) ? OpBitFieldSExtract : OpBitFieldUExtract, type,
+		     extracted, source, ConstantU32(ctx.state, info.component_bit_offset[component]),
+		     ConstantU32(ctx.state, bits)});
+		raw = type == ctx.state.int_type
+		          ? Unary(ctx.state, OpBitcast, ctx.state.uint_type, extracted)
+		          : extracted;
+	} else if (bits == 32u) {
+		raw = LoadWord(ctx, inst, RebaseFormattedComponent(mem, format, component));
+	} else {
+		raw = LoadSubword(ctx, inst, RebaseFormattedComponent(mem, format, component), bits,
+		                  IsSignedFormatComponent(info.type));
+	}
+	return NormalizeFormatComponent(ctx.state, info, component, raw);
+}
+
+template <typename Fn>
+uint32_t AtomicUpdate(EmitterState& state, uint32_t pointer, IR::ResourceKind kind, Fn&& desired) {
+	const auto scope     = kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
+	const auto memory    = kind == IR::ResourceKind::Lds ? MemorySemanticsWorkgroupMemory
+	                                                     : MemorySemanticsUniformMemory;
+	const auto preheader = state.builder.AllocateId();
+	const auto header    = state.builder.AllocateId();
+	const auto cont      = state.builder.AllocateId();
+	const auto merge     = state.builder.AllocateId();
+	const auto initial   = state.builder.AllocateId();
+	const auto observed  = state.builder.AllocateId();
+	const auto exchanged = state.builder.AllocateId();
+	state.builder.AddFunction({OpBranch, preheader});
+	state.builder.AddFunction({OpLabel, preheader});
+	state.builder.AddFunction({OpAtomicLoad, state.uint_type, initial, pointer,
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone)});
+	state.builder.AddFunction({OpBranch, header});
+	state.builder.AddFunction({OpLabel, header});
+	state.builder.AddFunction(
+	    {OpPhi, state.uint_type, observed, initial, preheader, exchanged, cont});
+	const auto next = desired(observed);
+	state.builder.AddFunction({OpAtomicCompareExchange, state.uint_type, exchanged, pointer,
+	                           ConstantU32(state, scope), ConstantU32(state, MemorySemanticsNone),
+	                           ConstantU32(state, MemorySemanticsNone), next, observed});
+	const auto success = Binary(state, OpIEqual, state.bool_type, exchanged, observed);
+	state.builder.AddFunction({OpLoopMerge, merge, cont, LoopControlNone});
+	state.builder.AddFunction({OpBranchConditional, success, merge, cont});
+	state.builder.AddFunction({OpLabel, cont});
+	state.builder.AddFunction({OpBranch, header});
+	state.builder.AddFunction({OpLabel, merge});
+	state.builder.AddFunction({OpMemoryBarrier, ConstantU32(state, scope),
+	                           ConstantU32(state, MemorySemanticsAcquireRelease | memory)});
+	return observed;
+}
+
+void StoreSubword(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem, uint32_t bits,
+                  uint32_t extra_offset = 0) {
+	mem.offset += extra_offset;
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto address = ByteAddress(ctx, inst, mem);
+		const auto index   = Binary(ctx.state, OpShiftRightLogical, ctx.state.uint_type, address,
+		                            ConstantU32(ctx.state, 2));
+		EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			const auto pointer = Pointer(ctx, mem, index, pc);
+			const auto shift = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
+			                          Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, address,
+			                                 ConstantU32(ctx.state, 3)),
+			                          ConstantU32(ctx.state, 3));
+			const auto mask  = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
+			                          ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu), shift);
+			const auto value = Binary(ctx.state, OpShiftLeftLogical, ctx.state.uint_type,
+			                          Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type,
+			                                 ctx.Arg(inst, inst.NumArgs() - 2),
+			                                 ConstantU32(ctx.state, bits == 8u ? 0xffu : 0xffffu)),
+			                          shift);
+			const auto merge = [&](uint32_t old) {
+				return Binary(ctx.state, OpBitwiseOr, ctx.state.uint_type,
+				              Binary(ctx.state, OpBitwiseAnd, ctx.state.uint_type, old,
+				                     Unary(ctx.state, OpNot, ctx.state.uint_type, mask)),
+				              value);
+			};
+			if (mem.kind == IR::ResourceKind::Lds || mem.kind == IR::ResourceKind::Gds) {
+				AtomicUpdate(ctx.state, pointer, mem.kind, merge);
+			} else {
+				const auto old = ctx.state.builder.AllocateId();
+				ctx.state.builder.AddFunction({OpLoad, ctx.state.uint_type, old, pointer});
+				ctx.state.builder.AddFunction({OpStore, pointer, merge(old)});
+			}
+		});
+	});
+}
+
+void StoreWord(ValueEmitContext& ctx, const IR::Inst& inst, IR::MemoryInfo mem,
+               uint32_t extra_offset = 0) {
+	mem.offset += extra_offset;
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto index = DwordIndex(ctx, inst, mem);
+		EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			ctx.state.builder.AddFunction(
+			    {OpStore, Pointer(ctx, mem, index, pc), ctx.Arg(inst, inst.NumArgs() - 2)});
+		});
+	});
+}
+
+void FormattedStore(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem) {
+	const auto format = BufferFormat(ctx, inst, mem);
+	if (!Format::IsKnownFormat(format)) {
+		StoreWord(ctx, inst, mem);
+		return;
+	}
+	const auto info = Format::GetFormatInfo(format);
+	if (mem.component_index >= info.component_count) return;
+	const auto bits          = info.component_bits[mem.component_index];
+	const auto component_mem = RebaseFormattedComponent(mem, format, mem.component_index);
+	if (bits == 8u || bits == 16u) {
+		StoreSubword(ctx, inst, component_mem, bits);
+	} else {
+		StoreWord(ctx, inst, component_mem);
+	}
+}
+
+uint32_t AtomicOpcode(IR::ValueOpcode opcode) {
+	switch (opcode) {
+		case IR::ValueOpcode::BufferAtomicSwap32:
+		case IR::ValueOpcode::SharedAtomicSwap32:
+		case IR::ValueOpcode::GdsAtomicSwap32: return OpAtomicExchange;
+		case IR::ValueOpcode::BufferAtomicIAdd32:
+		case IR::ValueOpcode::SharedAtomicIAdd32:
+		case IR::ValueOpcode::GdsAtomicIAdd32: return OpAtomicIAdd;
+		case IR::ValueOpcode::BufferAtomicISub32:
+		case IR::ValueOpcode::SharedAtomicISub32:
+		case IR::ValueOpcode::GdsAtomicISub32: return OpAtomicISub;
+		case IR::ValueOpcode::BufferAtomicSMin32:
+		case IR::ValueOpcode::SharedAtomicSMin32:
+		case IR::ValueOpcode::GdsAtomicSMin32: return OpAtomicSMin;
+		case IR::ValueOpcode::BufferAtomicUMin32:
+		case IR::ValueOpcode::SharedAtomicUMin32:
+		case IR::ValueOpcode::GdsAtomicUMin32: return OpAtomicUMin;
+		case IR::ValueOpcode::BufferAtomicSMax32:
+		case IR::ValueOpcode::SharedAtomicSMax32:
+		case IR::ValueOpcode::GdsAtomicSMax32: return OpAtomicSMax;
+		case IR::ValueOpcode::BufferAtomicUMax32:
+		case IR::ValueOpcode::SharedAtomicUMax32:
+		case IR::ValueOpcode::GdsAtomicUMax32: return OpAtomicUMax;
+		case IR::ValueOpcode::BufferAtomicAnd32:
+		case IR::ValueOpcode::SharedAtomicAnd32:
+		case IR::ValueOpcode::GdsAtomicAnd32: return OpAtomicAnd;
+		case IR::ValueOpcode::BufferAtomicOr32:
+		case IR::ValueOpcode::SharedAtomicOr32:
+		case IR::ValueOpcode::GdsAtomicOr32: return OpAtomicOr;
+		case IR::ValueOpcode::BufferAtomicXor32:
+		case IR::ValueOpcode::SharedAtomicXor32:
+		case IR::ValueOpcode::GdsAtomicXor32: return OpAtomicXor;
+		default: return 0;
+	}
+}
+
+uint32_t EmitAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                    bool return_value) {
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	const auto result =
+	    EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		    const auto index = DwordIndex(ctx, inst, mem);
+		    return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			    const auto value = ctx.Arg(inst, inst.NumArgs() - 2);
+			    const auto old   = ctx.state.builder.AllocateId();
+			    const auto scope = mem.kind == IR::ResourceKind::Lds ? ScopeWorkgroup : ScopeDevice;
+			    ctx.state.builder.AddFunction({AtomicOpcode(inst.GetOpcode()), ctx.state.uint_type,
+			                                   old, Pointer(ctx, mem, index, pc),
+			                                   ConstantU32(ctx.state, scope),
+			                                   ConstantU32(ctx.state, MemorySemanticsNone), value});
+			    if (mem.kind == IR::ResourceKind::Lds) {
+				    const auto semantics =
+				        MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
+				    ctx.state.builder.AddFunction({OpMemoryBarrier,
+				                                   ConstantU32(ctx.state, ScopeWorkgroup),
+				                                   ConstantU32(ctx.state, semantics)});
+			    } else {
+				    EmitDeviceAtomicMemoryBarrier(ctx.state);
+			    }
+			    return old;
+		    });
+	    });
+	return return_value ? result : ConstantU32(ctx.state, 0);
+}
+
+uint32_t FloatAtomicReplacement(EmitterState& state, uint32_t old, uint32_t source,
+                                bool max_value) {
+	struct OrderedBits {
+		uint32_t nan;
+		uint32_t zero;
+		uint32_t key;
+	};
+	const auto classify = [&](uint32_t bits) {
+		const auto value    = Unary(state, OpBitcast, state.float_type, bits);
+		const auto cls      = EmitClassifyF32(state, value);
+		const auto negative = Binary(
+		    state, OpINotEqual, state.bool_type,
+		    Binary(state, OpBitwiseAnd, state.uint_type, bits, ConstantU32(state, 0x80000000u)),
+		    ConstantU32(state, 0));
+		const auto negative_key = Unary(state, OpNot, state.uint_type, bits);
+		const auto positive_key =
+		    Binary(state, OpBitwiseXor, state.uint_type, bits, ConstantU32(state, 0x80000000u));
+		return OrderedBits {cls.nan, cls.zero,
+		                    Select(state, state.uint_type, negative, negative_key, positive_key)};
+	};
+	const auto source_class = classify(source);
+	const auto old_class    = classify(old);
+	const auto unordered =
+	    Binary(state, OpLogicalOr, state.bool_type,
+	           Binary(state, OpLogicalOr, state.bool_type, source_class.nan, old_class.nan),
+	           Binary(state, OpLogicalAnd, state.bool_type, source_class.zero, old_class.zero));
+	const auto ordered = Unary(state, OpLogicalNot, state.bool_type, unordered);
+	const auto compare = Binary(state, max_value ? OpUGreaterThan : OpULessThan, state.bool_type,
+	                            source_class.key, old_class.key);
+	return Select(state, state.uint_type,
+	              Binary(state, OpLogicalAnd, state.bool_type, ordered, compare), source, old);
+}
+
+uint32_t FloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                     bool max_value) {
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	return EmitValueOrZeroIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto index = DwordIndex(ctx, inst, mem);
+		return EmitValueOrZeroIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			const auto pointer = Pointer(ctx, mem, index, pc);
+			const auto source  = ctx.Arg(inst, inst.NumArgs() - 2);
+			return AtomicUpdate(ctx.state, pointer, mem.kind, [&](uint32_t old) {
+				return FloatAtomicReplacement(ctx.state, old, source, max_value);
+			});
+		});
+	});
+}
+
+uint32_t SharedFloatAtomic(ValueEmitContext& ctx, const IR::Inst& inst, const IR::MemoryInfo& mem,
+                           bool max_value) {
+	const auto pc = inst.Flags<IR::MemoryFlags>().pc;
+	EmitIfCondition(ctx.state, ctx.Arg(inst, inst.NumArgs() - 1), [&]() {
+		const auto index = DwordIndex(ctx, inst, mem);
+		EmitIfCondition(ctx.state, InBounds(ctx, mem, index, pc), [&]() {
+			ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 2)});
+			const auto data = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction(
+			    {OpLoad, ctx.state.uint_type, data, ctx.scratch_u32_variable});
+			AtomicUpdate(ctx.state, Pointer(ctx, mem, index, pc), mem.kind, [&](uint32_t old) {
+				const auto old_f = Unary(ctx.state, OpBitcast, ctx.state.float_type, old);
+				const auto compare_f =
+				    Unary(ctx.state, OpBitcast, ctx.state.float_type, ctx.Arg(inst, 3));
+				const auto data_f  = Unary(ctx.state, OpBitcast, ctx.state.float_type, data);
+				const auto compare = Binary(
+				    ctx.state, max_value ? OpFOrdGreaterThan : OpFOrdLessThan, ctx.state.bool_type,
+				    max_value ? old_f : compare_f, max_value ? compare_f : old_f);
+				return Unary(ctx.state, OpBitcast, ctx.state.uint_type,
+				             Select(ctx.state, ctx.state.float_type, compare, data_f, old_f));
+			});
+		});
+	});
+	return 0;
+}
+
+uint32_t AppendConsume(ValueEmitContext& ctx, const IR::Inst& inst, bool gds, bool append) {
+	auto&      state = ctx.state;
+	const auto m0    = ctx.Arg(inst, 1);
+	const auto base =
+	    Binary(state, OpShiftRightLogical, state.uint_type, m0, ConstantU32(state, 16));
+	const auto size = Binary(state, OpBitwiseAnd, state.uint_type, m0, ConstantU32(state, 0xffffu));
+	const auto address =
+	    Binary(state, OpIAdd, state.uint_type, base, ConstantU32(state, ctx.Memory(inst).offset));
+	const auto index =
+	    Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2));
+	const auto exec   = ctx.Arg(inst, 2);
+	const auto ballot = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
+	                           ConstantU32(state, ScopeSubgroup), exec});
+	const auto low  = state.builder.AllocateId();
+	const auto high = state.builder.AllocateId();
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
+	state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
+	const auto count_low  = state.per_invocation_masks ? low : ctx.Arg(inst, 3);
+	const auto count_high = state.per_invocation_masks ? high : ctx.Arg(inst, 4);
+	const auto count =
+	    Binary(state, OpIAdd, state.uint_type, Unary(state, OpBitCount, state.uint_type, count_low),
+	           Unary(state, OpBitCount, state.uint_type, count_high));
+	const auto first = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first,
+	                           ConstantU32(state, ScopeSubgroup), ballot});
+	const auto is_first =
+	    Binary(state, OpIEqual, state.bool_type, EmitSubgroupLocalInvocationId(state), first);
+	const auto storage_bounds =
+	    gds ? EmitGdsElementInBounds(state, index) : EmitLdsElementInBounds(state, index);
+	const auto m0_bounds =
+	    gds ? Binary(state, OpINotEqual, state.bool_type, size, ConstantU32(state, 0))
+	        : Binary(state, OpULessThan, state.bool_type,
+	                 ConstantU32(state, ctx.Memory(inst).offset + 3u), size);
+	const auto condition = AndCondition(
+	    state, is_first, AndCondition(state, exec, AndCondition(state, storage_bounds, m0_bounds)));
+	const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() {
+		const auto value = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {append ? OpAtomicIAdd : OpAtomicISub, state.uint_type, value,
+		     gds ? EmitGdsElementPointer(state, index) : EmitLdsElementPointer(state, index),
+		     ConstantU32(state, gds ? ScopeDevice : ScopeWorkgroup),
+		     ConstantU32(state, MemorySemanticsNone), count});
+		return value;
+	});
+	const auto result = state.builder.AllocateId();
+	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, result,
+	                           ConstantU32(state, ScopeSubgroup), atomic, first});
+	return result;
+}
+
+} // namespace
+
+bool EmitValueMemory(ValueEmitContext& ctx, const IR::Inst& inst) {
+	auto&      state = ctx.state;
+	const auto op    = inst.GetOpcode();
+	if ((op == IR::ValueOpcode::LoadAddressU32 || op == IR::ValueOpcode::ReadConstBuffer) &&
+	    ctx.Memory(inst).planning_only) {
+		return true;
+	}
+	if (op == IR::ValueOpcode::ReadConst) {
+		if (state.flattened_srt_variable == 0) {
+			ctx.Fail(inst, "requires the flattened SRT descriptor");
+			return true;
+		}
+		const auto pointer = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
+		                           state.flattened_srt_variable, ConstantU32(state, 0),
+		                           ctx.Arg(inst, 1)});
+		ctx.Emit(inst, OpLoad, IR::Type::U32, {pointer});
+		return true;
+	}
+	if (op == IR::ValueOpcode::ReadConstBuffer) {
+		auto mem           = ctx.Memory(inst);
+		mem.kind           = IR::ResourceKind::ScalarBuffer;
+		const auto address = Binary(state, OpIAdd, state.uint_type, ctx.Arg(inst, 1),
+		                            ConstantU32(state, mem.offset));
+		const auto index =
+		    Binary(state, OpShiftRightLogical, state.uint_type, address, ConstantU32(state, 2));
+		const auto pc        = inst.Flags<IR::MemoryFlags>().pc;
+		const auto condition = EmitStorageBufferElementInBounds(state, mem, index, pc);
+		ctx.Define(inst, EmitValueOrZeroIfCondition(state, condition, [&]() {
+			           const auto value = state.builder.AllocateId();
+			           state.builder.AddFunction(
+			               {OpLoad, state.uint_type, value,
+			                EmitStorageBufferElementPointer(state, mem, index, pc)});
+			           return value;
+		           }));
+		return true;
+	}
+	const bool load_address = op == IR::ValueOpcode::LoadAddressU8 ||
+	                          op == IR::ValueOpcode::LoadAddressU16 ||
+	                          op == IR::ValueOpcode::LoadAddressU32;
+	const bool load_buffer  = op == IR::ValueOpcode::LoadBufferU8 ||
+	                          op == IR::ValueOpcode::LoadBufferU16 ||
+	                          op == IR::ValueOpcode::LoadBufferU32;
+	const bool load_shared =
+	    op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadSharedU16 ||
+	    op == IR::ValueOpcode::LoadSharedU32 || op == IR::ValueOpcode::LoadGdsU8 ||
+	    op == IR::ValueOpcode::LoadGdsU16 || op == IR::ValueOpcode::LoadGdsU32;
+	if (load_address || load_buffer || load_shared) {
+		auto mem = ctx.Memory(inst);
+		if (load_shared)
+			mem.kind = op == IR::ValueOpcode::LoadGdsU8 || op == IR::ValueOpcode::LoadGdsU16 ||
+			                   op == IR::ValueOpcode::LoadGdsU32
+			               ? IR::ResourceKind::Gds
+			               : IR::ResourceKind::Lds;
+		uint32_t value = 0;
+		if (op == IR::ValueOpcode::LoadBufferU32 && mem.formatted)
+			value = FormattedLoad(ctx, inst, mem);
+		else if (op == IR::ValueOpcode::LoadAddressU8 || op == IR::ValueOpcode::LoadBufferU8 ||
+		         op == IR::ValueOpcode::LoadSharedU8 || op == IR::ValueOpcode::LoadGdsU8)
+			value = LoadSubword(ctx, inst, mem, 8, false);
+		else if (op == IR::ValueOpcode::LoadAddressU16 || op == IR::ValueOpcode::LoadBufferU16 ||
+		         op == IR::ValueOpcode::LoadSharedU16 || op == IR::ValueOpcode::LoadGdsU16)
+			value = LoadSubword(ctx, inst, mem, 16, false);
+		else
+			value = LoadWord(ctx, inst, mem);
+		ctx.Define(inst, value);
+		return true;
+	}
+	const bool store_address = op == IR::ValueOpcode::StoreAddressU8 ||
+	                           op == IR::ValueOpcode::StoreAddressU16 ||
+	                           op == IR::ValueOpcode::StoreAddressU32;
+	const bool store_buffer  = op == IR::ValueOpcode::StoreBufferU8 ||
+	                           op == IR::ValueOpcode::StoreBufferU16 ||
+	                           op == IR::ValueOpcode::StoreBufferU32;
+	const bool store_shared =
+	    op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteSharedU16 ||
+	    op == IR::ValueOpcode::WriteSharedU32 || op == IR::ValueOpcode::WriteGdsU8 ||
+	    op == IR::ValueOpcode::WriteGdsU16 || op == IR::ValueOpcode::WriteGdsU32;
+	if (store_address || store_buffer || store_shared) {
+		auto mem = ctx.Memory(inst);
+		if (store_shared)
+			mem.kind = op == IR::ValueOpcode::WriteGdsU8 || op == IR::ValueOpcode::WriteGdsU16 ||
+			                   op == IR::ValueOpcode::WriteGdsU32
+			               ? IR::ResourceKind::Gds
+			               : IR::ResourceKind::Lds;
+		if (op == IR::ValueOpcode::StoreBufferU32 && mem.formatted)
+			FormattedStore(ctx, inst, mem);
+		else if (op == IR::ValueOpcode::StoreAddressU8 || op == IR::ValueOpcode::StoreBufferU8 ||
+		         op == IR::ValueOpcode::WriteSharedU8 || op == IR::ValueOpcode::WriteGdsU8)
+			StoreSubword(ctx, inst, mem, 8);
+		else if (op == IR::ValueOpcode::StoreAddressU16 || op == IR::ValueOpcode::StoreBufferU16 ||
+		         op == IR::ValueOpcode::WriteSharedU16 || op == IR::ValueOpcode::WriteGdsU16)
+			StoreSubword(ctx, inst, mem, 16);
+		else
+			StoreWord(ctx, inst, mem);
+		return true;
+	}
+	const auto atomic_opcode = AtomicOpcode(op);
+	if (atomic_opcode != 0) {
+		auto mem = ctx.Memory(inst);
+		if (op >= IR::ValueOpcode::SharedAtomicSwap32 && op <= IR::ValueOpcode::SharedAtomicXor32)
+			mem.kind = IR::ResourceKind::Lds;
+		else if (op >= IR::ValueOpcode::GdsAtomicSwap32 && op <= IR::ValueOpcode::GdsAtomicXor32)
+			mem.kind = IR::ResourceKind::Gds;
+		ctx.Define(inst, EmitAtomic(ctx, inst, mem, true));
+		return true;
+	}
+	if (op == IR::ValueOpcode::BufferAtomicFMin32 || op == IR::ValueOpcode::BufferAtomicFMax32) {
+		ctx.Define(inst, FloatAtomic(ctx, inst, ctx.Memory(inst),
+		                             op == IR::ValueOpcode::BufferAtomicFMax32));
+		return true;
+	}
+	if (op == IR::ValueOpcode::SharedAtomicFMin32 || op == IR::ValueOpcode::SharedAtomicFMax32 ||
+	    op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32) {
+		auto mem = ctx.Memory(inst);
+		mem.kind = op == IR::ValueOpcode::GdsAtomicFMin32 || op == IR::ValueOpcode::GdsAtomicFMax32
+		               ? IR::ResourceKind::Gds
+		               : IR::ResourceKind::Lds;
+		SharedFloatAtomic(ctx, inst, mem,
+		                  op == IR::ValueOpcode::SharedAtomicFMax32 ||
+		                      op == IR::ValueOpcode::GdsAtomicFMax32);
+		return true;
+	}
+	if (op == IR::ValueOpcode::DataAppend || op == IR::ValueOpcode::DataConsume ||
+	    op == IR::ValueOpcode::GdsDataAppend || op == IR::ValueOpcode::GdsDataConsume) {
+		ctx.Define(inst, AppendConsume(ctx, inst,
+		                               op == IR::ValueOpcode::GdsDataAppend ||
+		                                   op == IR::ValueOpcode::GdsDataConsume,
+		                               op == IR::ValueOpcode::DataAppend ||
+		                                   op == IR::ValueOpcode::GdsDataAppend));
+		return true;
+	}
+	if (op == IR::ValueOpcode::SwizzleU32) {
+		ctx.state.builder.AddFunction({OpStore, ctx.scratch_u32_variable, ctx.Arg(inst, 0)});
+		const auto source = ctx.state.builder.AllocateId();
+		ctx.state.builder.AddFunction(
+		    {OpLoad, ctx.state.uint_type, source, ctx.scratch_u32_variable});
+		const auto target =
+		    EmitDsSwizzleTargetLane(state, EmitSubgroupLocalInvocationId(state),
+		                            inst.Arg(1).IsImmediate() ? inst.Arg(1).U32() : 0);
+		const auto shuffled = state.builder.AllocateId();
+		state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
+		                           ConstantU32(state, ScopeSubgroup), source, target});
+		const auto source_exec = state.builder.AllocateId();
+		state.builder.AddFunction({OpGroupNonUniformShuffle, state.bool_type, source_exec,
+		                           ConstantU32(state, ScopeSubgroup), ctx.Arg(inst, 2), target});
+		const auto source_active =
+		    AndCondition(state, source_exec, EmitSubgroupLaneActiveBool(state, target));
+		ctx.Define(inst,
+		           Select(state, state.uint_type, source_active, shuffled, ConstantU32(state, 0)));
+		return true;
+	}
+	return false;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterMemoryHelpers.cpp

@@ -0,0 +1,371 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+
+uint32_t EmitAddU32(EmitterState& state, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpIAdd, state.uint_type, ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t EmitBinaryU32(EmitterState& state, uint32_t opcode, uint32_t lhs, uint32_t rhs) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, state.uint_type, ret, lhs, rhs});
+	return ret;
+}
+
+uint32_t StorageBufferPackedStride(const EmitterState& state, const IR::MemoryInfo& mem,
+                                   uint32_t use_pc) {
+	(void)use_pc;
+	if (mem.resource >= state.program.info.buffers.size()) {
+		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
+		                             "buffer specialization is missing");
+	}
+	return state.program.info.buffers[mem.resource].packed_stride;
+}
+
+Prospero::BufferFormat StorageBufferFormat(const EmitterState& state, const IR::MemoryInfo& mem,
+                                           uint32_t use_pc) {
+	(void)use_pc;
+	if (mem.resource >= state.program.info.buffers.size()) {
+		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
+		                             "buffer specialization is missing");
+	}
+	return state.program.info.buffers[mem.resource].descriptor_format;
+}
+
+void EmitStorageBufferOffsets(EmitterState& state) {
+	for (uint32_t i = 0; i < state.program.bindings.buffer_offset_count; i++) {
+		const auto word =
+		    EmitShaderDataDwordLoad(state, state.program.bindings.buffer_offset_dword + i / 4u);
+		const auto shift                = ConstantU32(state, (i % 4u) * 8u + 2u);
+		state.storage_buffer_offsets[i] = EmitBinaryU32(
+		    state, OpBitwiseAnd, EmitBinaryU32(state, OpShiftRightLogical, word, shift),
+		    ConstantU32(state, 0x3fu));
+	}
+}
+
+bool IsAddressMemoryKind(IR::ResourceKind kind) {
+	return kind == IR::ResourceKind::ScalarAddress || kind == IR::ResourceKind::Flat ||
+	       kind == IR::ResourceKind::Global || kind == IR::ResourceKind::Scratch;
+}
+
+DescriptorResourceBinding
+StorageBufferBindingForMemory(EmitterState& state, const IR::MemoryInfo& mem, uint32_t use_pc) {
+	(void)use_pc;
+	const auto binding =
+	    ResourceForDescriptor(state, IR::DescriptorBindingKind::Buffers, mem.resource);
+	if (state.storage_buffer_variable == 0) {
+		ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::Buffers, mem.resource,
+		                             "storage buffer descriptor array was not emitted");
+	}
+	return binding;
+}
+
+uint32_t EmitStorageBufferIndex(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
+                                uint32_t use_pc) {
+	if (IsAddressMemoryKind(mem.kind)) {
+		return index;
+	}
+	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
+	return EmitAddU32(state, index, state.storage_buffer_offsets[binding.array_index]);
+}
+
+uint32_t EmitStorageBufferObjectPointer(EmitterState& state, const IR::MemoryInfo& mem,
+                                        uint32_t use_pc) {
+	if (IsAddressMemoryKind(mem.kind)) {
+		if (state.address_memory_variable == 0) {
+			ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
+			                             mem.resource, "address memory binding was not emitted");
+		}
+		const auto binding =
+		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
+		const auto pointer = state.builder.AllocateId();
+		state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
+		                           state.address_memory_variable,
+		                           ConstantU32(state, binding.array_index)});
+		return pointer;
+	}
+	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer, pointer,
+	                           state.storage_buffer_variable,
+	                           ConstantU32(state, binding.array_index)});
+	return pointer;
+}
+
+uint32_t EmitStorageBufferElementInBounds(EmitterState& state, const IR::MemoryInfo& mem,
+                                          uint32_t index, uint32_t use_pc) {
+	index                = EmitStorageBufferIndex(state, mem, index, use_pc);
+	const auto object    = EmitStorageBufferObjectPointer(state, mem, use_pc);
+	const auto length    = state.builder.AllocateId();
+	const auto in_bounds = state.builder.AllocateId();
+	state.builder.AddFunction({OpArrayLength, state.uint_type, length, object, 0});
+	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
+	return in_bounds;
+}
+
+uint32_t EmitStorageBufferElementPointer(EmitterState& state, const IR::MemoryInfo& mem,
+                                         uint32_t index, uint32_t use_pc) {
+	index = EmitStorageBufferIndex(state, mem, index, use_pc);
+	if (IsAddressMemoryKind(mem.kind)) {
+		if (state.address_memory_variable == 0) {
+			ExitDescriptorBindingFailure(state, IR::DescriptorBindingKind::AddressMemory,
+			                             mem.resource, "address memory binding was not emitted");
+		}
+		const auto binding =
+		    ResourceForDescriptor(state, IR::DescriptorBindingKind::AddressMemory, mem.resource);
+		const auto pointer = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.address_memory_variable,
+		     ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
+		return pointer;
+	}
+	const auto binding = StorageBufferBindingForMemory(state, mem, use_pc);
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpAccessChain, state.ptr_storage_buffer_uint, pointer, state.storage_buffer_variable,
+	     ConstantU32(state, binding.array_index), ConstantU32(state, 0), index});
+	return pointer;
+}
+
+uint32_t EmitLdsElementPointer(EmitterState& state, uint32_t index) {
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpAccessChain, state.ptr_workgroup_uint, pointer, state.lds_variable, index});
+	return pointer;
+}
+
+uint32_t LdsDwordCount(const EmitterState& state) {
+	return state.stage == ShaderType::Compute ? state.input_info.compute->lds_size_dwords : 8192u;
+}
+
+uint32_t EmitLdsElementInBounds(EmitterState& state, uint32_t index) {
+	const auto in_bounds = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpULessThan, state.bool_type, in_bounds, index, ConstantU32(state, LdsDwordCount(state))});
+	return in_bounds;
+}
+
+uint32_t EmitMemoryElementPointer(EmitterState& state, const IR::MemoryInfo& mem, uint32_t index,
+                                  uint32_t use_pc) {
+	if (mem.kind == IR::ResourceKind::Lds) {
+		return EmitLdsElementPointer(state, index);
+	}
+	if (mem.kind == IR::ResourceKind::Gds) {
+		return EmitGdsElementPointer(state, index);
+	}
+	return EmitStorageBufferElementPointer(state, mem, index, use_pc);
+}
+
+uint32_t EmitTBufferBitcastF32ToU32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.uint_type, ret, value});
+	return ret;
+}
+
+uint32_t EmitTBufferBitcastU32ToF32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.float_type, ret, value});
+	return ret;
+}
+
+uint32_t EmitTBufferBitcastU32ToI32(EmitterState& state, uint32_t value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitcast, state.int_type, ret, value});
+	return ret;
+}
+
+uint32_t EmitTBufferCompareU32Constant(EmitterState& state, uint32_t opcode, uint32_t value,
+                                       uint32_t constant) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction({opcode, state.bool_type, ret, value, ConstantU32(state, constant)});
+	return ret;
+}
+
+uint32_t EmitTBufferSelectF32(EmitterState& state, uint32_t condition, uint32_t true_value,
+                              uint32_t false_value) {
+	const auto ret = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpSelect, state.float_type, ret, condition, true_value, false_value});
+	return ret;
+}
+
+bool IsSignedFormatComponent(Format::ComponentType type) {
+	return type == Format::ComponentType::Sint || type == Format::ComponentType::Snorm ||
+	       type == Format::ComponentType::Sscaled;
+}
+
+uint32_t EmitHalfToF32Bits(EmitterState& state, uint32_t raw) {
+	const auto unpacked = state.builder.AllocateId();
+	const auto value    = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450, GlslUnpackHalf2x16, raw});
+	state.builder.AddFunction({OpCompositeExtract, state.float_type, value, unpacked, 0});
+	return EmitTBufferBitcastF32ToU32(state, value);
+}
+
+uint32_t EmitUFloatToF32Bits(EmitterState& state, uint32_t raw, uint32_t bits) {
+	const auto mantissa_bits = bits == 11u ? 6u : 5u;
+	const auto mantissa_mask = (1u << mantissa_bits) - 1u;
+	const auto mantissa =
+	    EmitBinaryU32(state, OpBitwiseAnd, raw, ConstantU32(state, mantissa_mask));
+	const auto exponent = EmitBinaryU32(
+	    state, OpBitwiseAnd,
+	    EmitBinaryU32(state, OpShiftRightLogical, raw, ConstantU32(state, mantissa_bits)),
+	    ConstantU32(state, 0x1fu));
+
+	const auto exponent_32 = EmitAddU32(state, exponent, ConstantU32(state, 127u - 15u));
+	const auto exponent_bits =
+	    EmitBinaryU32(state, OpShiftLeftLogical, exponent_32, ConstantU32(state, 23));
+	const auto mantissa_bits_32 =
+	    EmitBinaryU32(state, OpShiftLeftLogical, mantissa, ConstantU32(state, 23u - mantissa_bits));
+	const auto normal_bits = EmitBinaryU32(state, OpBitwiseOr, exponent_bits, mantissa_bits_32);
+	const auto normal      = EmitTBufferBitcastU32ToF32(state, normal_bits);
+
+	const auto special_bits =
+	    EmitBinaryU32(state, OpBitwiseOr, ConstantU32(state, 0x7f800000u), mantissa_bits_32);
+	const auto special = EmitTBufferBitcastU32ToF32(state, special_bits);
+
+	const auto mantissa_f32 = state.builder.AllocateId();
+	const auto subnormal    = state.builder.AllocateId();
+	state.builder.AddFunction({OpConvertUToF, state.float_type, mantissa_f32, mantissa});
+	state.builder.AddFunction(
+	    {OpFMul, state.float_type, subnormal, mantissa_f32,
+	     ConstantF32Value(state, std::ldexp(1.0f, 1 - 15 - static_cast<int>(mantissa_bits)))});
+
+	const auto zero_exp    = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 0);
+	const auto special_exp = EmitTBufferCompareU32Constant(state, OpIEqual, exponent, 31);
+	const auto finite      = EmitTBufferSelectF32(state, zero_exp, subnormal, normal);
+	const auto result      = EmitTBufferSelectF32(state, special_exp, special, finite);
+	return EmitTBufferBitcastF32ToU32(state, result);
+}
+
+uint32_t NormalizeFormatComponent(EmitterState& state, const Format::BufferFormatInfo& info,
+                                  uint32_t component, uint32_t raw) {
+	const auto bits = info.component_bits[component];
+	switch (info.type) {
+		case Format::ComponentType::Uint:
+		case Format::ComponentType::Sint: return raw;
+		case Format::ComponentType::Uscaled: {
+			const auto value = state.builder.AllocateId();
+			state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
+			return EmitTBufferBitcastF32ToU32(state, value);
+		}
+		case Format::ComponentType::Sscaled: {
+			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
+			const auto value      = state.builder.AllocateId();
+			state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
+			return EmitTBufferBitcastF32ToU32(state, value);
+		}
+		case Format::ComponentType::Unorm: {
+			const auto value      = state.builder.AllocateId();
+			const auto normalized = state.builder.AllocateId();
+			const auto max_value  = static_cast<float>((1u << bits) - 1u);
+			state.builder.AddFunction({OpConvertUToF, state.float_type, value, raw});
+			state.builder.AddFunction(
+			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
+			return EmitTBufferBitcastF32ToU32(state, normalized);
+		}
+		case Format::ComponentType::Snorm: {
+			const auto signed_raw = EmitTBufferBitcastU32ToI32(state, raw);
+			const auto value      = state.builder.AllocateId();
+			const auto normalized = state.builder.AllocateId();
+			const auto clamped    = state.builder.AllocateId();
+			const auto max_value  = static_cast<float>((1u << (bits - 1u)) - 1u);
+			state.builder.AddFunction({OpConvertSToF, state.float_type, value, signed_raw});
+			state.builder.AddFunction(
+			    {OpFDiv, state.float_type, normalized, value, ConstantF32Value(state, max_value)});
+			state.builder.AddFunction({OpExtInst, state.float_type, clamped, state.glsl_std450,
+			                           GlslFMax, normalized, ConstantF32Value(state, -1.0f)});
+			return EmitTBufferBitcastF32ToU32(state, clamped);
+		}
+		case Format::ComponentType::Float:
+			if (bits == 32u) {
+				return raw;
+			}
+			if (bits == 16u) {
+				return EmitHalfToF32Bits(state, raw);
+			}
+			return EmitUFloatToF32Bits(state, raw, bits);
+		default: return raw;
+	}
+}
+
+void EmitDeviceAtomicMemoryBarrier(EmitterState& state) {
+	const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsUniformMemory;
+	state.builder.AddFunction(
+	    {OpMemoryBarrier, ConstantU32(state, ScopeDevice), ConstantU32(state, semantics)});
+}
+
+uint32_t EmitGdsElementInBounds(EmitterState& state, uint32_t index) {
+	const auto length    = state.builder.AllocateId();
+	const auto in_bounds = state.builder.AllocateId();
+	state.builder.AddFunction({OpArrayLength, state.uint_type, length, state.gds_variable, 0});
+	state.builder.AddFunction({OpULessThan, state.bool_type, in_bounds, index, length});
+	return in_bounds;
+}
+
+uint32_t EmitGdsElementPointer(EmitterState& state, uint32_t index) {
+	const auto pointer = state.builder.AllocateId();
+	state.builder.AddFunction({OpAccessChain, state.ptr_storage_buffer_uint, pointer,
+	                           state.gds_variable, ConstantU32(state, 0), index});
+	return pointer;
+}
+
+uint32_t EmitDsSwizzleTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
+	if ((control & 0xc000u) == 0xc000u) {
+		const uint32_t mask         = control & 0x1fu;
+		const uint32_t rotate       = (control >> 5u) & 0x1fu;
+		const uint32_t rotate_delta = (control & 0x400u) != 0u ? ((32u - rotate) & 0x1fu) : rotate;
+		const auto     lane         = EmitAndConstant(state, subid, 31);
+		const auto     rotated_sum  = EmitAddU32(state, lane, ConstantU32(state, rotate_delta));
+		const auto     rotated      = EmitAndConstant(state, rotated_sum, 31);
+		const auto     kept         = EmitAndConstant(state, lane, mask);
+		const auto     moved        = EmitAndConstant(state, rotated, (~mask) & 31u);
+		const auto     combined     = EmitOrU32(state, kept, moved);
+		const auto     base         = EmitAndConstant(state, subid, 0xffffffe0u);
+		return EmitOrU32(state, base, combined);
+	}
+
+	if ((control & 0x8000u) != 0) {
+		const auto lane2  = state.builder.AllocateId();
+		const auto shift  = state.builder.AllocateId();
+		const auto perm0  = state.builder.AllocateId();
+		const auto perm   = state.builder.AllocateId();
+		const auto base   = state.builder.AllocateId();
+		const auto target = state.builder.AllocateId();
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, state.uint_type, lane2, subid, ConstantU32(state, 3)});
+		state.builder.AddFunction(
+		    {OpShiftLeftLogical, state.uint_type, shift, lane2, ConstantU32(state, 1)});
+		state.builder.AddFunction(
+		    {OpShiftRightLogical, state.uint_type, perm0, ConstantU32(state, control), shift});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, state.uint_type, perm, perm0, ConstantU32(state, 3)});
+		state.builder.AddFunction(
+		    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xfffffffcu)});
+		state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, perm});
+		return target;
+	}
+
+	const auto lane   = state.builder.AllocateId();
+	const auto masked = state.builder.AllocateId();
+	const auto ored   = state.builder.AllocateId();
+	const auto xored  = state.builder.AllocateId();
+	const auto base   = state.builder.AllocateId();
+	const auto target = state.builder.AllocateId();
+	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, masked, lane, ConstantU32(state, control & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseOr, state.uint_type, ored, masked, ConstantU32(state, (control >> 5u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseXor, state.uint_type, xored, ored, ConstantU32(state, (control >> 10u) & 0x1fu)});
+	state.builder.AddFunction(
+	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, 0xffffffe0u)});
+	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, xored});
+	return target;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/backend/spirv/spirvEmitterModule.cpp

@@ -1,4 +1,6 @@
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
 
 namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
 
@@ -57,13 +59,12 @@ uint32_t ConstantF32Value(EmitterState& state, float value) {
 }
 
 VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint32_t location) {
-	if (state.stage != ShaderType::Vertex || state.vertex_input_info == nullptr ||
-	    location >= ShaderVertexInputInfo::RES_MAX ||
-	    location >= static_cast<uint32_t>(state.vertex_input_info->resources_num)) {
+	if (state.stage != ShaderType::Vertex || location >= ShaderVertexInputInfo::RES_MAX ||
+	    location >= static_cast<uint32_t>(state.input_info.vertex->resources_num)) {
 		return VertexInputScalarKind::Float;
 	}
 
-	switch (state.vertex_input_info->resources[location].Format()) {
+	switch (state.input_info.vertex->resources[location].Format()) {
 		case Prospero::BufferFormat::k8UInt:
 		case Prospero::BufferFormat::k16UInt:
 		case Prospero::BufferFormat::k8_8UInt:
@@ -90,12 +91,11 @@ VertexInputScalarKind VertexParameterScalarKind(const EmitterState& state, uint3
 
 uint32_t VertexParameterComponentCount(const EmitterState& state, const InputBinding& input) {
 	uint32_t count = input.component_count;
-	if (state.stage == ShaderType::Vertex && state.vertex_input_info != nullptr &&
-	    input.location < ShaderVertexInputInfo::RES_MAX &&
-	    input.location < static_cast<uint32_t>(state.vertex_input_info->resources_num) &&
-	    state.vertex_input_info->resources_dst[input.location].registers_num > 0) {
+	if (state.stage == ShaderType::Vertex && input.location < ShaderVertexInputInfo::RES_MAX &&
+	    input.location < static_cast<uint32_t>(state.input_info.vertex->resources_num) &&
+	    state.input_info.vertex->resources_dst[input.location].registers_num > 0) {
 		count = static_cast<uint32_t>(
-		    state.vertex_input_info->resources_dst[input.location].registers_num);
+		    state.input_info.vertex->resources_dst[input.location].registers_num);
 	}
 	return std::clamp(count, 1u, 4u);
 }
@@ -175,9 +175,9 @@ uint32_t VertexParameterInputPointerType(const EmitterState& state, VertexInputS
 }
 
 static bool MrtUsesUintOutput(const EmitterState& state, uint32_t index) {
-	return state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr &&
-	       index < std::size(state.pixel_input_info->target_output_mode) &&
-	       state.pixel_input_info->target_output_mode[index] == 7u;
+	return state.stage == ShaderType::Pixel &&
+	       index < std::size(state.input_info.pixel->target_output_mode) &&
+	       state.input_info.pixel->target_output_mode[index] == 7u;
 }
 
 void AllocateInputVariables(EmitterState& state) {
@@ -258,8 +258,8 @@ void AddInputAnnotationsAndNames(EmitterState& state) {
 			if (flat) {
 				state.builder.AddAnnotation({OpDecorate, input.variable_id, DecorationFlat});
 			}
-			if (state.stage == ShaderType::Pixel && state.pixel_input_info != nullptr &&
-			    state.pixel_input_info->ps_no_perspective && !flat) {
+			if (state.stage == ShaderType::Pixel && state.input_info.pixel->ps_no_perspective &&
+			    !flat) {
 				state.builder.AddAnnotation(
 				    {OpDecorate, input.variable_id, DecorationNoPerspective});
 			}
@@ -413,6 +413,11 @@ void EmitHeaderAndTypes(EmitterState& state) {
 	state.vec3_float_type              = state.builder.AllocateId();
 	state.vec4_float_type              = state.builder.AllocateId();
 	state.ptr_func_uint                = state.builder.AllocateId();
+	state.ptr_func_bool                = state.builder.AllocateId();
+	state.ptr_func_float               = state.builder.AllocateId();
+	state.ptr_func_uint_pair           = state.builder.AllocateId();
+	state.ptr_func_vec4_uint           = state.builder.AllocateId();
+	state.ptr_func_vec2_float          = state.builder.AllocateId();
 	state.ptr_input_float              = state.builder.AllocateId();
 	state.ptr_input_bool               = state.builder.AllocateId();
 	state.ptr_input_int                = state.builder.AllocateId();
@@ -481,6 +486,7 @@ void EmitHeaderAndTypes(EmitterState& state) {
 	state.builder.AddCapability({CapabilitySampled1D});
 	state.builder.AddCapability({CapabilityImage1D});
 	state.builder.AddCapability({CapabilityImageQuery});
+	state.builder.AddCapability({CapabilitySignedZeroInfNanPreserve});
 	if (state.needs_image_gather_extended) {
 		state.builder.AddCapability({CapabilityImageGatherExtended});
 	}
@@ -504,20 +510,22 @@ void EmitHeaderAndTypes(EmitterState& state) {
 		state.builder.AddCapability({CapabilityComputeDerivativeGroupQuadsKHR});
 		state.builder.AddExtension("SPV_KHR_compute_shader_derivatives");
 	}
+	state.builder.AddExtension("SPV_KHR_float_controls");
 	state.builder.AddExtInstImport(state.glsl_std450, "GLSL.std.450");
 	state.builder.AddMemoryModel({AddressingModelLogical, MemoryModelGLSL450});
 	state.builder.AddEntryPoint(ExecutionModelForStage(state.stage), state.main_func, "main",
 	                            state.interface_variables);
+	// GCN/RDNA arithmetic preserves 32-bit signed zero, infinity, and NaN. Declaring that
+	// contract prevents host compilers from treating synthesized IEEE values as finite.
+	state.builder.AddExecutionMode({state.main_func, ExecutionModeSignedZeroInfNanPreserve, 32u});
 	if (state.stage == ShaderType::Compute) {
-		uint32_t local_x = state.needs_compute_derivatives ? 2u : 1u;
-		uint32_t local_y = state.needs_compute_derivatives ? 2u : 1u;
-		uint32_t local_z = 1u;
-		if (state.compute_input_info != nullptr) {
-			const auto* cs = state.compute_input_info;
-			local_x        = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
-			local_y        = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
-			local_z        = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
-		}
+		uint32_t    local_x = state.needs_compute_derivatives ? 2u : 1u;
+		uint32_t    local_y = state.needs_compute_derivatives ? 2u : 1u;
+		uint32_t    local_z = 1u;
+		const auto* cs      = state.input_info.compute;
+		local_x             = cs->threads_num[0] != 0u ? cs->threads_num[0] : local_x;
+		local_y             = cs->threads_num[1] != 0u ? cs->threads_num[1] : local_y;
+		local_z             = cs->threads_num[2] != 0u ? cs->threads_num[2] : local_z;
 		state.builder.AddExecutionMode(
 		    {state.main_func, ExecutionModeLocalSize, local_x, local_y, local_z});
 	}
@@ -526,10 +534,9 @@ void EmitHeaderAndTypes(EmitterState& state) {
 		if (state.depth_variable != 0) {
 			state.builder.AddExecutionMode({state.main_func, ExecutionModeDepthReplacing});
 		}
-		if (state.pixel_input_info != nullptr && state.pixel_input_info->ps_early_z &&
-		    !state.pixel_input_info->ps_pixel_kill_enable &&
-		    !state.pixel_input_info->ps_depth_export_enable &&
-		    !state.pixel_input_info->ps_sample_mask_export_enable) {
+		if (state.input_info.pixel->ps_early_z && !state.input_info.pixel->ps_pixel_kill_enable &&
+		    !state.input_info.pixel->ps_depth_export_enable &&
+		    !state.input_info.pixel->ps_sample_mask_export_enable) {
 			state.builder.AddExecutionMode({state.main_func, ExecutionModeEarlyFragmentTests});
 		}
 	}
@@ -568,6 +575,16 @@ void EmitHeaderAndTypes(EmitterState& state) {
 	state.builder.AddType({OpTypeVector, state.vec4_float_type, state.float_type, 4});
 	state.builder.AddType(
 	    {OpTypePointer, state.ptr_func_uint, StorageClassFunction, state.uint_type});
+	state.builder.AddType(
+	    {OpTypePointer, state.ptr_func_bool, StorageClassFunction, state.bool_type});
+	state.builder.AddType(
+	    {OpTypePointer, state.ptr_func_float, StorageClassFunction, state.float_type});
+	state.builder.AddType(
+	    {OpTypePointer, state.ptr_func_uint_pair, StorageClassFunction, state.uint_pair_type});
+	state.builder.AddType(
+	    {OpTypePointer, state.ptr_func_vec4_uint, StorageClassFunction, state.vec4_uint_type});
+	state.builder.AddType(
+	    {OpTypePointer, state.ptr_func_vec2_float, StorageClassFunction, state.vec2_float_type});
 	state.builder.AddType(
 	    {OpTypePointer, state.ptr_input_float, StorageClassInput, state.float_type});
 	state.builder.AddType(
@@ -716,7 +733,8 @@ void EmitHeaderAndTypes(EmitterState& state) {
 	if (state.stage == ShaderType::Compute || state.needs_function_lds) {
 		const auto storage_class =
 		    state.needs_function_lds ? StorageClassFunction : StorageClassWorkgroup;
-		const auto lds_size = ConstantU32(state, state.needs_function_lds ? 8192u : 1024u);
+		const auto lds_dwords = std::max(LdsDwordCount(state), 1u);
+		const auto lds_size   = ConstantU32(state, lds_dwords);
 		state.builder.AddType({OpTypeArray, state.lds_array_type, state.uint_type, lds_size});
 		state.builder.AddType(
 		    {OpTypePointer, state.ptr_workgroup_array, storage_class, state.lds_array_type});

src/graphics/shader/recompiler/backend/spirv/spirvEmitterProgram.cpp

@@ -0,0 +1,567 @@
+#include "graphics/shader/recompiler/backend/spirv/spirvEmitterInternal.h"
+
+#include <algorithm>
+#include <bit>
+#include <fmt/format.h>
+#include <functional>
+#include <unordered_set>
+
+namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
+namespace {
+
+void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
+	const auto kill_label  = state.builder.AllocateId();
+	const auto merge_label = state.builder.AllocateId();
+	const auto inactive    = state.builder.AllocateId();
+	state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
+	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
+	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
+	state.builder.AddFunction({OpLabel, kill_label});
+	state.builder.AddFunction({OpKill});
+	state.builder.AddFunction({OpLabel, merge_label});
+}
+
+void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
+	if (state.pixel_valid_mask_variable == 0) {
+		return;
+	}
+
+	const auto mask_value = state.builder.AllocateId();
+	const auto active     = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
+	state.builder.AddFunction(
+	    {OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
+	EmitKillIfBoolFalse(state, active);
+}
+
+uint32_t BoolConstant(EmitterState& state, bool value) {
+	const auto id = state.builder.AllocateId();
+	state.builder.AddType({value ? 41u : 42u, state.bool_type, id});
+	return id;
+}
+
+uint32_t ConstantPair(EmitterState& state, uint64_t value) {
+	const auto id = state.builder.AllocateId();
+	state.builder.AddType({OpConstantComposite, state.uint_pair_type, id,
+	                       ConstantU32(state, static_cast<uint32_t>(value)),
+	                       ConstantU32(state, static_cast<uint32_t>(value >> 32u))});
+	return id;
+}
+
+uint32_t PhiPointerType(const EmitterState& state, IR::Type type) {
+	switch (type) {
+		case IR::Type::U1: return state.ptr_func_bool;
+		case IR::Type::U8:
+		case IR::Type::U16:
+		case IR::Type::U32:
+		case IR::Type::F16: return state.ptr_func_uint;
+		case IR::Type::F32: return state.ptr_func_float;
+		case IR::Type::U64:
+		case IR::Type::U32x2: return state.ptr_func_uint_pair;
+		case IR::Type::U32x4: return state.ptr_func_vec4_uint;
+		case IR::Type::F32x2: return state.ptr_func_vec2_float;
+		default: return 0;
+	}
+}
+
+void StorePhiEdge(ValueEmitContext& ctx, const IR::Block* from, const IR::Block* to) {
+	if (to == nullptr) {
+		return;
+	}
+	for (const auto& phi: *to) {
+		if (phi.GetOpcode() != IR::ValueOpcode::Phi) {
+			continue;
+		}
+		for (size_t index = 0; index < phi.NumArgs(); index++) {
+			if (phi.PhiBlock(index) == from) {
+				ctx.state.builder.AddFunction(
+				    {OpStore, ctx.phi_variables.at(&phi), ctx.Def(phi.Arg(index))});
+				break;
+			}
+		}
+	}
+}
+
+const IR::Block* TargetBlock(const IR::ValueProgram& program, uint32_t id) {
+	const auto found = std::ranges::find_if(
+	    program.block_info, [&](const IR::ValueBlockInfo& info) { return info.id == id; });
+	if (found == program.block_info.end()) {
+		return nullptr;
+	}
+	return program.blocks[static_cast<size_t>(found - program.block_info.begin())];
+}
+
+void EmitReturn(ValueEmitContext& ctx) {
+	EmitKillIfPixelValidMaskInactive(ctx.state);
+	ctx.state.builder.AddFunction({OpReturn});
+}
+
+void EmitStructuredTerminator(ValueEmitContext& ctx, const IR::Block* block,
+                              const IR::ValueBlockInfo& info) {
+	const auto& term       = info.terminator;
+	const auto  emit_merge = [&]() {
+		if (term.loop_header) {
+			const auto* merge = TargetBlock(ctx.program, term.merge_block);
+			const auto* cont  = TargetBlock(ctx.program, term.continue_block);
+			if (merge != nullptr && cont != nullptr) {
+				ctx.state.builder.AddFunction(
+				    {OpLoopMerge, ctx.Label(merge), ctx.Label(cont), LoopControlNone});
+			}
+		} else if (term.kind == CFG::TerminatorKind::ConditionalBranch &&
+		           term.merge_block != UINT32_MAX) {
+			if (const auto* merge = TargetBlock(ctx.program, term.merge_block); merge != nullptr) {
+				ctx.state.builder.AddFunction(
+				    {OpSelectionMerge, ctx.Label(merge), SelectionControlNone});
+			}
+		}
+	};
+
+	switch (term.kind) {
+		case CFG::TerminatorKind::Branch: {
+			const auto* target = TargetBlock(ctx.program, term.true_block);
+			if (target == nullptr) {
+				EmitReturn(ctx);
+				return;
+			}
+			StorePhiEdge(ctx, block, target);
+			emit_merge();
+			ctx.state.builder.AddFunction({OpBranch, ctx.Label(target)});
+			return;
+		}
+		case CFG::TerminatorKind::ConditionalBranch: {
+			const auto* true_block  = TargetBlock(ctx.program, term.true_block);
+			const auto* false_block = TargetBlock(ctx.program, term.false_block);
+			if (true_block == nullptr || false_block == nullptr || info.condition.IsEmpty()) {
+				EmitReturn(ctx);
+				return;
+			}
+			StorePhiEdge(ctx, block, true_block);
+			StorePhiEdge(ctx, block, false_block);
+			const auto condition = ctx.Def(info.condition);
+			emit_merge();
+			ctx.state.builder.AddFunction(
+			    {OpBranchConditional, condition, ctx.Label(true_block), ctx.Label(false_block)});
+			return;
+		}
+		default: EmitReturn(ctx); return;
+	}
+}
+
+void EmitDispatcherTarget(ValueEmitContext& ctx, const IR::Block* from, uint32_t target) {
+	const auto* block = TargetBlock(ctx.program, target);
+	if (block != nullptr) {
+		StorePhiEdge(ctx, from, block);
+	}
+}
+
+void EmitDispatcherTerminator(ValueEmitContext& ctx, const IR::Block* block,
+                              const IR::ValueBlockInfo& info, uint32_t exit_label) {
+	const auto& term = info.terminator;
+	switch (term.kind) {
+		case CFG::TerminatorKind::Branch:
+			EmitDispatcherTarget(ctx, block, term.true_block);
+			ctx.state.builder.AddFunction(
+			    {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, term.true_block)});
+			break;
+		case CFG::TerminatorKind::ConditionalBranch: {
+			EmitDispatcherTarget(ctx, block, term.true_block);
+			EmitDispatcherTarget(ctx, block, term.false_block);
+			const auto selected = ctx.state.builder.AllocateId();
+			ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, selected,
+			                               ctx.Def(info.condition),
+			                               ConstantU32(ctx.state, term.true_block),
+			                               ConstantU32(ctx.state, term.false_block)});
+			ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
+			break;
+		}
+		case CFG::TerminatorKind::IndirectBranch: {
+			for (const auto target: term.indirect_targets) {
+				EmitDispatcherTarget(ctx, block, target);
+			}
+			for (const auto target: term.indirect_selector_targets) {
+				EmitDispatcherTarget(ctx, block, target);
+			}
+			uint32_t selected = ConstantU32(ctx.state, UINT32_MAX);
+			if (!info.indirect_target.IsEmpty()) {
+				const auto  selector = ctx.Def(info.indirect_target);
+				const auto& values   = term.indirect_selector_code != UINT32_MAX
+				                           ? term.indirect_selector_values
+				                           : term.indirect_target_pcs;
+				const auto& targets  = term.indirect_selector_code != UINT32_MAX
+				                           ? term.indirect_selector_targets
+				                           : term.indirect_targets;
+				for (size_t index = 0; index < std::min(values.size(), targets.size()); index++) {
+					const auto match = ctx.state.builder.AllocateId();
+					const auto next  = ctx.state.builder.AllocateId();
+					ctx.state.builder.AddFunction({OpIEqual, ctx.state.bool_type, match, selector,
+					                               ConstantU32(ctx.state, values[index])});
+					ctx.state.builder.AddFunction({OpSelect, ctx.state.uint_type, next, match,
+					                               ConstantU32(ctx.state, targets[index]),
+					                               selected});
+					selected = next;
+				}
+			}
+			ctx.state.builder.AddFunction({OpStore, ctx.state.dispatch_pc_variable, selected});
+			break;
+		}
+		default:
+			ctx.state.builder.AddFunction(
+			    {OpStore, ctx.state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
+			break;
+	}
+	ctx.state.builder.AddFunction({OpBranch, exit_label});
+}
+
+bool EmitInstruction(ValueEmitContext& ctx, const IR::Inst& inst) {
+	const auto finish = [&]() {
+		if (const auto found = ctx.dispatcher_variables.find(&inst);
+		    found != ctx.dispatcher_variables.end()) {
+			ctx.state.builder.AddFunction(
+			    {OpStore, found->second, ctx.Def(IR::Value(const_cast<IR::Inst*>(&inst)))});
+		}
+		return true;
+	};
+	if (inst.GetOpcode() == IR::ValueOpcode::Phi) {
+		ctx.state.builder.AddFunction(
+		    {OpLoad, ctx.TypeId(inst.GetType()), ctx.Result(inst), ctx.phi_variables.at(&inst)});
+		return finish();
+	}
+	if (EmitValueFlow(ctx, inst) || EmitValueAlu(ctx, inst) || EmitValueMemory(ctx, inst) ||
+	    EmitValueImage(ctx, inst)) {
+		return finish();
+	}
+	ctx.Fail(inst, "has no direct SPIR-V emitter");
+	return false;
+}
+
+void EmitBlock(ValueEmitContext& ctx, const IR::Block* block) {
+	ctx.current_block = block;
+	ctx.state.builder.AddFunction({OpLabel, ctx.Label(block)});
+	for (const auto& inst: *block) {
+		if (!EmitInstruction(ctx, inst)) {
+			return;
+		}
+	}
+}
+
+void EmitStructuredFunction(ValueEmitContext& ctx) {
+	ctx.state.builder.AddFunction({OpBranch, ctx.Label(ctx.program.blocks.front())});
+	for (size_t index = 0; index < ctx.program.blocks.size() && !ctx.failed; index++) {
+		EmitBlock(ctx, ctx.program.blocks[index]);
+		EmitStructuredTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index]);
+	}
+}
+
+void EmitDispatcherFunction(ValueEmitContext& ctx) {
+	auto&       state = ctx.state;
+	const auto* entry = ctx.program.blocks.front();
+	state.builder.AddFunction({OpBranch, ctx.Label(entry)});
+	EmitBlock(ctx, entry);
+	EmitDispatcherTerminator(ctx, entry, ctx.program.block_info.front(),
+	                         state.dispatch_header_label);
+
+	state.builder.AddFunction({OpLabel, state.dispatch_header_label});
+	const auto pc = state.builder.AllocateId();
+	state.builder.AddFunction({OpLoad, state.uint_type, pc, state.dispatch_pc_variable});
+	const auto done = state.builder.AllocateId();
+	state.builder.AddFunction(
+	    {OpIEqual, state.bool_type, done, pc, ConstantU32(ctx.state, UINT32_MAX)});
+	state.builder.AddFunction(
+	    {OpLoopMerge, state.dispatch_merge_label, state.dispatch_continue_label, LoopControlNone});
+	state.builder.AddFunction(
+	    {OpBranchConditional, done, state.dispatch_merge_label, state.dispatch_select_label});
+
+	state.builder.AddFunction({OpLabel, state.dispatch_select_label});
+	state.builder.AddFunction(
+	    {OpSelectionMerge, state.dispatch_after_switch_label, SelectionControlNone});
+	std::vector<uint32_t> words {OpSwitch, pc, state.dispatch_default_label};
+	for (size_t index = 1; index < ctx.program.blocks.size(); index++) {
+		words.push_back(ctx.program.block_info[index].id);
+		words.push_back(ctx.Label(ctx.program.blocks[index]));
+	}
+	state.builder.AddFunction(words);
+	state.builder.AddFunction({OpLabel, state.dispatch_default_label});
+	state.builder.AddFunction(
+	    {OpStore, state.dispatch_pc_variable, ConstantU32(ctx.state, UINT32_MAX)});
+	state.builder.AddFunction({OpBranch, state.dispatch_after_switch_label});
+
+	for (size_t index = 1; index < ctx.program.blocks.size() && !ctx.failed; index++) {
+		EmitBlock(ctx, ctx.program.blocks[index]);
+		EmitDispatcherTerminator(ctx, ctx.program.blocks[index], ctx.program.block_info[index],
+		                         state.dispatch_after_switch_label);
+	}
+	state.builder.AddFunction({OpLabel, state.dispatch_after_switch_label});
+	state.builder.AddFunction({OpBranch, state.dispatch_continue_label});
+	state.builder.AddFunction({OpLabel, state.dispatch_continue_label});
+	state.builder.AddFunction({OpBranch, state.dispatch_header_label});
+	state.builder.AddFunction({OpLabel, state.dispatch_merge_label});
+	EmitReturn(ctx);
+}
+
+} // namespace
+
+uint32_t ValueEmitContext::TypeId(IR::Type type) const {
+	switch (type) {
+		case IR::Type::U1: return state.bool_type;
+		case IR::Type::U8:
+		case IR::Type::U16:
+		case IR::Type::U32:
+		case IR::Type::F16: return state.uint_type;
+		case IR::Type::U64:
+		case IR::Type::U32x2: return state.uint_pair_type;
+		case IR::Type::F32: return state.float_type;
+		case IR::Type::U32x4: return state.vec4_uint_type;
+		case IR::Type::F32x2: return state.vec2_float_type;
+		default: return 0;
+	}
+}
+
+uint32_t ValueEmitContext::Def(IR::Value value) {
+	value = value.Resolve();
+	if (value.IsImmediate()) {
+		switch (value.GetType()) {
+			case IR::Type::U1: return BoolConstant(state, value.U1());
+			case IR::Type::U8: return ConstantU32(state, value.U8());
+			case IR::Type::U16: return ConstantU32(state, value.U16());
+			case IR::Type::U32: return ConstantU32(state, value.U32());
+			case IR::Type::U64: return ConstantPair(state, value.U64());
+			case IR::Type::F16: return ConstantU32(state, value.F16Bits());
+			case IR::Type::F32:
+				return ConstantF32(state, std::bit_cast<uint32_t>(value.F32Value()));
+			default: break;
+		}
+	}
+	const auto* inst = value.ResolveInstruction();
+	if (inst == nullptr) {
+		failed = true;
+		error  = "direct SPIR-V emitter received a non-value argument";
+		return ConstantU32(state, 0);
+	}
+	if (program.dispatcher_fallback && current_block != nullptr &&
+	    inst->Parent() != current_block) {
+		if (const auto found = dispatcher_variables.find(inst);
+		    found != dispatcher_variables.end()) {
+			const auto id = state.builder.AllocateId();
+			state.builder.AddFunction({OpLoad, TypeId(inst->GetType()), id, found->second});
+			return id;
+		}
+	}
+	if (const auto found = definitions.find(inst); found != definitions.end()) {
+		return found->second;
+	}
+	const auto id = state.builder.AllocateId();
+	definitions.emplace(inst, id);
+	return id;
+}
+
+uint32_t ValueEmitContext::Arg(const IR::Inst& inst, size_t index) {
+	return Def(inst.Arg(index));
+}
+
+uint32_t ValueEmitContext::Result(const IR::Inst& inst) {
+	if (const auto found = definitions.find(&inst); found != definitions.end()) {
+		return found->second;
+	}
+	const auto id = state.builder.AllocateId();
+	definitions.emplace(&inst, id);
+	return id;
+}
+
+uint32_t ValueEmitContext::Emit(const IR::Inst& inst, uint32_t opcode, IR::Type type,
+                                std::initializer_list<uint32_t> args) {
+	std::vector<uint32_t> words {opcode, TypeId(type), Result(inst)};
+	words.insert(words.end(), args.begin(), args.end());
+	state.builder.AddFunction(words);
+	return Result(inst);
+}
+
+uint32_t ValueEmitContext::Define(const IR::Inst& inst, uint32_t value) {
+	if (const auto found = definitions.find(&inst); found != definitions.end()) {
+		if (found->second != value) {
+			state.builder.AddFunction({OpCopyObject, TypeId(inst.GetType()), found->second, value});
+		}
+		return found->second;
+	}
+	definitions.emplace(&inst, value);
+	return value;
+}
+
+uint32_t ValueEmitContext::ResourceIndex(IR::Value value, IR::ValueOpcode opcode) {
+	const auto* inst = value.ResolveInstruction();
+	if (inst == nullptr || inst->GetOpcode() != opcode) {
+		failed = true;
+		error  = "typed resource handle has the wrong producer";
+		return 0;
+	}
+	return inst->Flags<uint32_t>();
+}
+
+const IR::Inst* ValueEmitContext::ImageAddress(IR::Value value) {
+	const auto* inst = value.ResolveInstruction();
+	if (inst == nullptr || inst->GetOpcode() != IR::ValueOpcode::MakeImageAddress) {
+		failed = true;
+		error  = "typed image address was not constructed by MakeImageAddress";
+		return nullptr;
+	}
+	return inst;
+}
+
+const IR::MemoryInfo& ValueEmitContext::Memory(const IR::Inst& inst) const {
+	return program.memory_info.at(inst.Flags<IR::MemoryFlags>().index);
+}
+
+const IR::ExportInfo& ValueEmitContext::Export(const IR::Inst& inst) const {
+	return program.export_info.at(inst.Flags<IR::ExportFlags>().index);
+}
+
+uint32_t ValueEmitContext::Label(const IR::Block* block) const {
+	return labels.at(block);
+}
+
+size_t ValueEmitContext::BlockIndex(const IR::Block* block) const {
+	return static_cast<size_t>(std::ranges::find(program.blocks, block) - program.blocks.begin());
+}
+
+void ValueEmitContext::Fail(const IR::Inst& inst, const char* reason) {
+	failed = true;
+	error  = fmt::format("typed opcode {} {}", IR::ValueOpcodeName(inst.GetOpcode()), reason);
+}
+
+bool EmitValueProgram(EmitterState& state, const IR::ValueProgram& program, std::string* error) {
+	ValueEmitContext ctx(state, program);
+	for (const auto* block: program.blocks) {
+		ctx.labels.emplace(block, state.builder.AllocateId());
+		for (const auto& inst: *block) {
+			if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
+				continue;
+			}
+			const auto ptr_type = PhiPointerType(state, inst.GetType());
+			if (ptr_type == 0) {
+				ctx.failed = true;
+				ctx.error  = fmt::format("typed opcode Phi of type {} cannot be spilled across a "
+				                         "control-flow edge",
+				                         IR::TypeName(inst.GetType()));
+				break;
+			}
+			ctx.phi_variables.emplace(&inst, state.builder.AllocateId());
+		}
+	}
+	if (ctx.failed) {
+		SetError(error, ctx.error.c_str());
+		return false;
+	}
+	if (program.dispatcher_fallback) {
+		const auto mark_cross_block = [&](IR::Value value, const IR::Block* consumer) {
+			value                  = value.Resolve();
+			const auto* definition = value.TryInstruction();
+			if (definition == nullptr || definition->Parent() == consumer ||
+			    PhiPointerType(state, definition->GetType()) == 0) {
+				return;
+			}
+			if (!ctx.dispatcher_variables.contains(definition)) {
+				ctx.dispatcher_variables.emplace(definition, state.builder.AllocateId());
+			}
+		};
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				for (size_t index = 0; index < inst.NumArgs(); index++) {
+					const auto* consumer =
+					    inst.GetOpcode() == IR::ValueOpcode::Phi ? inst.PhiBlock(index) : block;
+					mark_cross_block(inst.Arg(index), consumer);
+				}
+			}
+		}
+		for (size_t index = 0; index < program.blocks.size(); index++) {
+			mark_cross_block(program.block_info[index].condition, program.blocks[index]);
+			mark_cross_block(program.block_info[index].indirect_target, program.blocks[index]);
+		}
+		state.dispatcher_fallback         = true;
+		state.dispatch_pc_variable        = state.builder.AllocateId();
+		state.dispatch_header_label       = state.builder.AllocateId();
+		state.dispatch_select_label       = state.builder.AllocateId();
+		state.dispatch_default_label      = state.builder.AllocateId();
+		state.dispatch_after_switch_label = state.builder.AllocateId();
+		state.dispatch_continue_label     = state.builder.AllocateId();
+		state.dispatch_merge_label        = state.builder.AllocateId();
+	}
+	for (const auto* block: program.blocks) {
+		if (std::ranges::any_of(*block, [](const IR::Inst& inst) {
+			    return inst.GetOpcode() == IR::ValueOpcode::SwizzleU32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMin32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::SharedAtomicFMax32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMin32 ||
+			           inst.GetOpcode() == IR::ValueOpcode::GdsAtomicFMax32;
+		    })) {
+			ctx.scratch_u32_variable = state.builder.AllocateId();
+			break;
+		}
+	}
+	state.builder.AddFunction(
+	    {OpFunction, state.void_type, state.main_func, FunctionControlNone, state.func_type});
+	state.builder.AddFunction({OpLabel, state.entry_label});
+	if (state.needs_function_lds) {
+		state.builder.AddFunction(
+		    {OpVariable, state.ptr_workgroup_array, state.lds_variable, StorageClassFunction});
+	}
+	if (state.pixel_valid_mask_variable != 0) {
+		state.builder.AddFunction({OpVariable, state.ptr_func_uint, state.pixel_valid_mask_variable,
+		                           StorageClassFunction});
+	}
+	if (program.dispatcher_fallback) {
+		state.builder.AddFunction(
+		    {OpVariable, state.ptr_func_uint, state.dispatch_pc_variable, StorageClassFunction});
+	}
+	for (const auto& binding: state.registers) {
+		state.builder.AddFunction(
+		    {OpVariable, state.ptr_func_uint, binding.pointer_id, StorageClassFunction});
+	}
+	for (const auto* block: program.blocks) {
+		for (const auto& inst: *block) {
+			if (inst.GetOpcode() != IR::ValueOpcode::Phi) {
+				continue;
+			}
+			state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
+			                           ctx.phi_variables.at(&inst), StorageClassFunction});
+		}
+	}
+	if (program.dispatcher_fallback) {
+		for (const auto* block: program.blocks) {
+			for (const auto& inst: *block) {
+				if (const auto found = ctx.dispatcher_variables.find(&inst);
+				    found != ctx.dispatcher_variables.end()) {
+					state.builder.AddFunction({OpVariable, PhiPointerType(state, inst.GetType()),
+					                           found->second, StorageClassFunction});
+				}
+			}
+		}
+	}
+	if (ctx.scratch_u32_variable != 0) {
+		state.builder.AddFunction(
+		    {OpVariable, state.ptr_func_uint, ctx.scratch_u32_variable, StorageClassFunction});
+	}
+	for (const auto& binding: state.registers) {
+		state.builder.AddFunction({OpStore, binding.pointer_id,
+		                           ConstantU32(state, InitialRegisterValue(state, binding.reg))});
+	}
+	if (state.pixel_valid_mask_variable != 0) {
+		state.builder.AddFunction(
+		    {OpStore, state.pixel_valid_mask_variable, ConstantU32(state, 1)});
+	}
+	EmitStorageBufferOffsets(state);
+	if (program.blocks.empty()) {
+		EmitReturn(ctx);
+	} else if (program.dispatcher_fallback) {
+		EmitDispatcherFunction(ctx);
+	} else {
+		EmitStructuredFunction(ctx);
+	}
+	state.builder.AddFunction({OpFunctionEnd});
+	if (ctx.failed) {
+		SetError(error, ctx.error.c_str());
+		return false;
+	}
+	return true;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/decompiler/MemoryOps.cpp

@@ -1,477 +0,0 @@
-#include "graphics/shader/recompiler/decompiler/MemoryOps.h"
-
-#include <fmt/format.h>
-#include <iterator>
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-namespace {
-
-struct MemoryOpcodeInfo {
-	uint32_t opcode      = 0;
-	Opcode   decoded     = Opcode::Unsupported;
-	uint32_t data_dwords = 1;
-	uint32_t data_bits   = 32;
-	bool     data_signed = false;
-	bool     typed       = false;
-	bool     formatted   = false;
-};
-
-constexpr MemoryOpcodeInfo SMEM_OPS[] = {
-    {0x00u, Opcode::SLoadDword, 1, 32},         {0x01u, Opcode::SLoadDwordx2, 2, 32},
-    {0x02u, Opcode::SLoadDwordx4, 4, 32},       {0x03u, Opcode::SLoadDwordx8, 8, 32},
-    {0x04u, Opcode::SLoadDwordx16, 16, 32},     {0x08u, Opcode::SBufferLoadDword, 1, 32},
-    {0x09u, Opcode::SBufferLoadDwordx2, 2, 32}, {0x0au, Opcode::SBufferLoadDwordx4, 4, 32},
-    {0x0bu, Opcode::SBufferLoadDwordx8, 8, 32}, {0x0cu, Opcode::SBufferLoadDwordx16, 16, 32},
-};
-
-constexpr MemoryOpcodeInfo MUBUF_OPS[] = {
-    {0x00u, Opcode::BufferLoadFormatX, 1, 32, false, false, true},
-    {0x01u, Opcode::BufferLoadFormatXy, 2, 32, false, false, true},
-    {0x02u, Opcode::BufferLoadFormatXyz, 3, 32, false, false, true},
-    {0x03u, Opcode::BufferLoadFormatXyzw, 4, 32, false, false, true},
-    {0x04u, Opcode::BufferStoreFormatX, 1, 32, false, false, true},
-    {0x05u, Opcode::BufferStoreFormatXy, 2, 32, false, false, true},
-    {0x06u, Opcode::BufferStoreFormatXyz, 3, 32, false, false, true},
-    {0x07u, Opcode::BufferStoreFormatXyzw, 4, 32, false, false, true},
-    {0x08u, Opcode::BufferLoadUbyte, 1, 8},
-    {0x09u, Opcode::BufferLoadSbyte, 1, 8, true},
-    {0x0au, Opcode::BufferLoadUshort, 1, 16},
-    {0x0bu, Opcode::BufferLoadSshort, 1, 16, true},
-    {0x0cu, Opcode::BufferLoadDword, 1, 32},
-    {0x0du, Opcode::BufferLoadDwordx2, 2, 32},
-    {0x0eu, Opcode::BufferLoadDwordx4, 4, 32},
-    {0x0fu, Opcode::BufferLoadDwordx3, 3, 32},
-    {0x18u, Opcode::BufferStoreByte, 1, 8},
-    {0x1au, Opcode::BufferStoreShort, 1, 16},
-    {0x1cu, Opcode::BufferStoreDword, 1, 32},
-    {0x1du, Opcode::BufferStoreDwordx2, 2, 32},
-    {0x1eu, Opcode::BufferStoreDwordx4, 4, 32},
-    {0x1fu, Opcode::BufferStoreDwordx3, 3, 32},
-    {0x30u, Opcode::BufferAtomicSwap, 1, 32},
-    {0x32u, Opcode::BufferAtomicAdd, 1, 32},
-    {0x33u, Opcode::BufferAtomicSub, 1, 32},
-    {0x35u, Opcode::BufferAtomicSMin, 1, 32},
-    {0x36u, Opcode::BufferAtomicUMin, 1, 32},
-    {0x37u, Opcode::BufferAtomicSMax, 1, 32},
-    {0x38u, Opcode::BufferAtomicUMax, 1, 32},
-    {0x39u, Opcode::BufferAtomicAnd, 1, 32},
-    {0x3au, Opcode::BufferAtomicOr, 1, 32},
-    {0x3bu, Opcode::BufferAtomicXor, 1, 32},
-    {0x3fu, Opcode::BufferAtomicFMin, 1, 32},
-    {0x40u, Opcode::BufferAtomicFMax, 1, 32},
-};
-
-constexpr MemoryOpcodeInfo MTBUF_OPS[] = {
-    {0x00u, Opcode::TBufferLoadFormatX, 1, 32, false, true, true},
-    {0x01u, Opcode::TBufferLoadFormatXy, 2, 32, false, true, true},
-    {0x02u, Opcode::TBufferLoadFormatXyz, 3, 32, false, true, true},
-    {0x03u, Opcode::TBufferLoadFormatXyzw, 4, 32, false, true, true},
-    {0x04u, Opcode::TBufferStoreFormatX, 1, 32, false, true, true},
-    {0x05u, Opcode::TBufferStoreFormatXy, 2, 32, false, true, true},
-    {0x06u, Opcode::TBufferStoreFormatXyz, 3, 32, false, true, true},
-    {0x07u, Opcode::TBufferStoreFormatXyzw, 4, 32, false, true, true},
-};
-
-constexpr MemoryOpcodeInfo FLAT_OPS[] = {
-    {0x08u, Opcode::FlatLoadUbyte, 1, 8},     {0x09u, Opcode::FlatLoadSbyte, 1, 8, true},
-    {0x0au, Opcode::FlatLoadUshort, 1, 16},   {0x0bu, Opcode::FlatLoadSshort, 1, 16, true},
-    {0x0cu, Opcode::FlatLoadDword, 1, 32},    {0x0du, Opcode::FlatLoadDwordx2, 2, 32},
-    {0x0eu, Opcode::FlatLoadDwordx4, 4, 32},  {0x0fu, Opcode::FlatLoadDwordx3, 3, 32},
-    {0x18u, Opcode::FlatStoreByte, 1, 8},     {0x1au, Opcode::FlatStoreShort, 1, 16},
-    {0x1cu, Opcode::FlatStoreDword, 1, 32},   {0x1du, Opcode::FlatStoreDwordx2, 2, 32},
-    {0x1eu, Opcode::FlatStoreDwordx4, 4, 32}, {0x1fu, Opcode::FlatStoreDwordx3, 3, 32},
-};
-
-constexpr MemoryOpcodeInfo DS_OPS[] = {
-    {0x00u, Opcode::DsAddU32, 1, 32},           {0x01u, Opcode::DsSubU32, 1, 32},
-    {0x05u, Opcode::DsMinI32, 1, 32},           {0x06u, Opcode::DsMaxI32, 1, 32},
-    {0x07u, Opcode::DsMinU32, 1, 32},           {0x08u, Opcode::DsMaxU32, 1, 32},
-    {0x09u, Opcode::DsAndB32, 1, 32},           {0x0au, Opcode::DsOrB32, 1, 32},
-    {0x0bu, Opcode::DsXorB32, 1, 32},           {0x0du, Opcode::DsWriteB32, 1, 32},
-    {0x0eu, Opcode::DsWrite2B32, 2, 32},        {0x0fu, Opcode::DsWrite2St64B32, 2, 32},
-    {0x12u, Opcode::DsMinF32, 1, 32},           {0x13u, Opcode::DsMaxF32, 1, 32},
-    {0x1eu, Opcode::DsWriteByte, 1, 8},         {0x1fu, Opcode::DsWriteShort, 1, 16},
-    {0x20u, Opcode::DsAddRtnU32, 1, 32},        {0x21u, Opcode::DsSubRtnU32, 1, 32},
-    {0x25u, Opcode::DsMinRtnI32, 1, 32},        {0x26u, Opcode::DsMaxRtnI32, 1, 32},
-    {0x27u, Opcode::DsMinRtnU32, 1, 32},        {0x28u, Opcode::DsMaxRtnU32, 1, 32},
-    {0x29u, Opcode::DsAndRtnB32, 1, 32},        {0x2au, Opcode::DsOrRtnB32, 1, 32},
-    {0x2bu, Opcode::DsXorRtnB32, 1, 32},        {0x2du, Opcode::DsWrxchgRtnB32, 1, 32},
-    {0x35u, Opcode::DsSwizzleB32, 1, 32},       {0x36u, Opcode::DsReadB32, 1, 32},
-    {0x37u, Opcode::DsRead2B32, 2, 32},         {0x38u, Opcode::DsRead2B32, 2, 32},
-    {0x39u, Opcode::DsReadSbyte, 1, 8, true},   {0x3au, Opcode::DsReadUbyte, 1, 8},
-    {0x3bu, Opcode::DsReadSshort, 1, 16, true}, {0x3cu, Opcode::DsReadUshort, 1, 16},
-    {0x3du, Opcode::DsConsume, 1, 32},          {0x3eu, Opcode::DsAppend, 1, 32},
-    {0x4du, Opcode::DsWriteB64, 2, 32},         {0x4eu, Opcode::DsWrite2B64, 4, 32},
-    {0x4fu, Opcode::DsWrite2St64B64, 4, 32},    {0x76u, Opcode::DsReadB64, 2, 32},
-    {0x77u, Opcode::DsRead2B64, 4, 32},         {0x78u, Opcode::DsRead2St64B64, 4, 32},
-    {0xb0u, Opcode::DsWriteAddtidB32, 1, 32},   {0xb1u, Opcode::DsReadAddtidB32, 1, 32},
-    {0xdeu, Opcode::DsWriteB96, 3, 32},         {0xdfu, Opcode::DsWriteB128, 4, 32},
-    {0xfeu, Opcode::DsReadB96, 3, 32},          {0xffu, Opcode::DsReadB128, 4, 32},
-};
-
-const MemoryOpcodeInfo* LookupMemoryOpcode(const MemoryOpcodeInfo* ops, uint32_t count,
-                                           uint32_t opcode) {
-	for (uint32_t i = 0; i < count; i++) {
-		if (ops[i].opcode == opcode) {
-			return &ops[i];
-		}
-	}
-	return nullptr;
-}
-
-uint32_t SignExtendU32(uint32_t value, uint32_t bits) {
-	if (bits == 0u || bits >= 32u) {
-		return value;
-	}
-	const uint32_t sign = 1u << (bits - 1u);
-	return (value ^ sign) - sign;
-}
-
-void MarkMemoryUnsupported(Instruction& inst, Family family, uint32_t opcode, const char* reason) {
-	inst.family    = family;
-	inst.opcode_id = opcode;
-	inst.opcode    = Opcode::Unsupported;
-	SetUnsupported(inst, family, opcode, reason);
-}
-
-void ApplyMemoryInfo(Instruction& inst, const MemoryOpcodeInfo* info) {
-	if (info == nullptr) {
-		inst.opcode = Opcode::Unsupported;
-		return;
-	}
-	inst.opcode      = info->decoded;
-	inst.data_dwords = info->data_dwords;
-	inst.data_bits   = info->data_bits;
-	inst.data_signed = info->data_signed;
-	inst.typed       = info->typed;
-	inst.formatted   = info->formatted;
-}
-
-bool IsDsWriteOpcode(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::DsWriteByte:
-		case Opcode::DsWriteShort:
-		case Opcode::DsWrite2B32:
-		case Opcode::DsWrite2St64B32:
-		case Opcode::DsWrite2B64:
-		case Opcode::DsWrite2St64B64:
-		case Opcode::DsWriteB32:
-		case Opcode::DsWriteB64:
-		case Opcode::DsWriteB96:
-		case Opcode::DsWriteB128: return true;
-		default: return false;
-	}
-}
-
-bool IsDsAtomicOpcode(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::DsAddU32:
-		case Opcode::DsAddRtnU32:
-		case Opcode::DsSubU32:
-		case Opcode::DsSubRtnU32:
-		case Opcode::DsMinI32:
-		case Opcode::DsMinRtnI32:
-		case Opcode::DsMaxI32:
-		case Opcode::DsMaxRtnI32:
-		case Opcode::DsMinU32:
-		case Opcode::DsMinRtnU32:
-		case Opcode::DsMaxU32:
-		case Opcode::DsMaxRtnU32:
-		case Opcode::DsAndB32:
-		case Opcode::DsAndRtnB32:
-		case Opcode::DsOrB32:
-		case Opcode::DsOrRtnB32:
-		case Opcode::DsXorB32:
-		case Opcode::DsXorRtnB32:
-		case Opcode::DsWrxchgRtnB32: return true;
-		default: return false;
-	}
-}
-
-uint32_t DsSourceCount(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::DsWrite2B32:
-		case Opcode::DsWrite2St64B32:
-		case Opcode::DsWrite2B64:
-		case Opcode::DsWrite2St64B64:
-		case Opcode::DsMinF32:
-		case Opcode::DsMaxF32: return 3u;
-		case Opcode::DsReadAddtidB32:
-		case Opcode::DsConsume:
-		case Opcode::DsAppend: return 0u;
-		default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u;
-	}
-}
-
-bool IsFlatStoreOpcode(Opcode opcode) {
-	switch (opcode) {
-		case Opcode::FlatStoreByte:
-		case Opcode::FlatStoreShort:
-		case Opcode::FlatStoreDword:
-		case Opcode::FlatStoreDwordx2:
-		case Opcode::FlatStoreDwordx3:
-		case Opcode::FlatStoreDwordx4: return true;
-		default: return false;
-	}
-}
-
-} // namespace
-
-bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated SMEM instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = (word0 >> 18u) & 0xffu;
-	const uint32_t sdst    = (word0 >> 6u) & 0x7fu;
-	const uint32_t sbase   = word0 & 0x3fu;
-	const uint32_t soffset = (word1 >> 25u) & 0x7fu;
-
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = SignExtendU32(word1 & 0x1fffffu, 21u);
-	inst.glc        = ((word0 >> 16u) & 1u) != 0;
-	inst.family     = Family::SMEM;
-	inst.opcode_id  = opcode;
-	const auto* info =
-	    LookupMemoryOpcode(SMEM_OPS, static_cast<uint32_t>(std::size(SMEM_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
-	}
-
-	DecodeScalarDestination(sdst, pc, inst.dst, nullptr);
-	// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
-	// pair index; their descriptor operand consumes four SGPRs from that base.
-	DecodeScalarSource(sbase * 2u, pc, inst.src0, nullptr);
-	DecodeScalarSource(soffset, pc, inst.src1, nullptr);
-	inst.src_count = 2;
-	return true;
-}
-
-bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated MUBUF instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = ((word0 >> 18u) & 0x7fu) | (((word0 >> 25u) & 1u) << 7u);
-	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
-	const uint32_t vaddr   = word1 & 0xffu;
-	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
-	const uint32_t soffset = (word1 >> 24u) & 0xffu;
-
-	inst.pc         = pc;
-	inst.word       = word0;
-	inst.word_count = 2;
-	inst.offset     = word0 & 0xfffu;
-	inst.idxen      = ((word0 >> 13u) & 1u) != 0;
-	inst.offen      = ((word0 >> 12u) & 1u) != 0;
-	inst.glc        = ((word0 >> 14u) & 1u) != 0;
-	inst.slc        = ((word1 >> 22u) & 1u) != 0;
-	inst.family     = Family::MUBUF;
-	inst.opcode_id  = opcode;
-	const auto* info =
-	    LookupMemoryOpcode(MUBUF_OPS, static_cast<uint32_t>(std::size(MUBUF_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented");
-	}
-
-	DecodeVectorGpr(vdata, inst.dst, nullptr);
-	DecodeVectorGpr(vaddr, inst.src0, nullptr);
-	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
-	DecodeScalarSource(soffset, pc, inst.src2, nullptr);
-	inst.src_count = 3;
-	return true;
-}
-
-bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated MTBUF instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = ((word0 >> 16u) & 0x7u) | (((word1 >> 21u) & 1u) << 3u);
-	const uint32_t dfmt    = (word0 >> 19u) & 0xfu;
-	const uint32_t nfmt    = (word0 >> 23u) & 0x7u;
-	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
-	const uint32_t vaddr   = word1 & 0xffu;
-	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
-	const uint32_t soffset = (word1 >> 24u) & 0xffu;
-
-	inst.pc            = pc;
-	inst.word          = word0;
-	inst.word_count    = 2;
-	inst.offset        = word0 & 0xfffu;
-	inst.idxen         = ((word0 >> 13u) & 1u) != 0;
-	inst.offen         = ((word0 >> 12u) & 1u) != 0;
-	inst.glc           = ((word0 >> 14u) & 1u) != 0;
-	inst.slc           = ((word1 >> 22u) & 1u) != 0;
-	inst.family        = Family::MTBUF;
-	inst.opcode_id     = opcode;
-	inst.data_format   = dfmt;
-	inst.number_format = nfmt;
-	const auto* info =
-	    LookupMemoryOpcode(MTBUF_OPS, static_cast<uint32_t>(std::size(MTBUF_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::MTBUF, opcode, "MTBUF opcode is not implemented");
-	}
-
-	DecodeVectorGpr(vdata, inst.dst, nullptr);
-	DecodeVectorGpr(vaddr, inst.src0, nullptr);
-	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
-	DecodeScalarSource(soffset, pc, inst.src2, nullptr);
-	inst.src_count = 3;
-	return true;
-}
-
-bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated FLAT instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0  = code[word_index];
-	const uint32_t word1  = code[word_index + 1u];
-	const uint32_t offset = word0 & 0xfffu;
-	const uint32_t dlc    = (word0 >> 12u) & 1u;
-	const uint32_t lds    = (word0 >> 13u) & 1u;
-	const uint32_t seg    = (word0 >> 14u) & 0x3u;
-	const uint32_t opcode = (word0 >> 18u) & 0x7fu;
-	const uint32_t vdst   = (word1 >> 24u) & 0xffu;
-	const uint32_t saddr  = (word1 >> 16u) & 0x7fu;
-	const uint32_t data   = (word1 >> 8u) & 0xffu;
-	const uint32_t addr   = word1 & 0xffu;
-
-	inst.pc             = pc;
-	inst.word           = word0;
-	inst.word_count     = 2;
-	inst.offset         = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
-	inst.glc            = ((word0 >> 16u) & 1u) != 0;
-	inst.slc            = ((word0 >> 17u) & 1u) != 0;
-	inst.family         = Family::FLAT;
-	inst.opcode_id      = opcode;
-	inst.memory_segment = seg;
-	const auto* info =
-	    LookupMemoryOpcode(FLAT_OPS, static_cast<uint32_t>(std::size(FLAT_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-
-	if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) {
-		SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
-		return true;
-	}
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::FLAT, opcode, "FLAT opcode is not implemented");
-		return true;
-	}
-
-	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst, nullptr);
-	DecodeVectorGpr(addr, inst.src0, nullptr);
-	inst.src_count = 1;
-	if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
-		DecodeVectorGpr(addr + 1u, inst.src1, nullptr);
-		inst.src_count = 2;
-	} else {
-		DecodeScalarSource(saddr, pc, inst.src1, nullptr);
-		inst.src_count = 2;
-	}
-	return true;
-}
-
-bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-              std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated DS instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
-	const uint32_t word0   = code[word_index];
-	const uint32_t word1   = code[word_index + 1u];
-	const uint32_t opcode  = (word0 >> 18u) & 0xffu;
-	const uint32_t offset0 = word0 & 0xffu;
-	const uint32_t offset1 = (word0 >> 8u) & 0xffu;
-	const uint32_t vdst    = (word1 >> 24u) & 0xffu;
-	const uint32_t data1   = (word1 >> 16u) & 0xffu;
-	const uint32_t data0   = (word1 >> 8u) & 0xffu;
-	const uint32_t addr    = word1 & 0xffu;
-
-	inst.pc          = pc;
-	inst.word        = word0;
-	inst.word_count  = 2;
-	inst.offset      = offset0 | (offset1 << 8u);
-	inst.gds         = ((word0 >> 17u) & 1u) != 0u;
-	inst.family      = Family::DS;
-	inst.opcode_id   = opcode;
-	const auto* info = LookupMemoryOpcode(DS_OPS, static_cast<uint32_t>(std::size(DS_OPS)), opcode);
-	ApplyMemoryInfo(inst, info);
-	SetRawWords(inst, code, word_index, 2);
-	if (inst.opcode == Opcode::Unsupported) {
-		MarkMemoryUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
-	}
-	if (inst.opcode == Opcode::DsSwizzleB32 && inst.offset >= 0xe000u) {
-		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
-	}
-	if (inst.gds &&
-	    (inst.opcode == Opcode::DsSwizzleB32 || inst.opcode == Opcode::DsWriteAddtidB32 ||
-	     inst.opcode == Opcode::DsReadAddtidB32)) {
-		SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
-	}
-	if (inst.opcode == Opcode::DsWriteAddtidB32 && data1 != 0u) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "DS write addtid data1 operand is not implemented");
-	}
-	if (inst.opcode == Opcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) {
-		SetUnsupported(inst, Family::DS, opcode,
-		               "DS read addtid data operands are not implemented");
-	}
-	if (inst.opcode == Opcode::DsWrite2B32 || opcode == 0x37u) {
-		inst.offset           = offset0 * 4u;
-		inst.secondary_offset = offset1 * 4u;
-	} else if (inst.opcode == Opcode::DsWrite2St64B32 || opcode == 0x38u) {
-		inst.offset           = offset0 * 256u;
-		inst.secondary_offset = offset1 * 256u;
-	} else if (inst.opcode == Opcode::DsWrite2B64 || inst.opcode == Opcode::DsRead2B64) {
-		inst.offset           = offset0 * 8u;
-		inst.secondary_offset = offset1 * 8u;
-	} else if (inst.opcode == Opcode::DsWrite2St64B64 || inst.opcode == Opcode::DsRead2St64B64) {
-		inst.offset           = offset0 * 512u;
-		inst.secondary_offset = offset1 * 512u;
-	}
-
-	DecodeVectorGpr(vdst, inst.dst, nullptr);
-	DecodeVectorGpr(addr, inst.src0, nullptr);
-	DecodeVectorGpr(data0, inst.src1, nullptr);
-	DecodeVectorGpr(data1, inst.src2, nullptr);
-	inst.src_count = DsSourceCount(inst.opcode);
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/decompiler/MemoryOps.h

@@ -1,21 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
-
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-
-bool DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error);
-bool DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error);
-bool DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                 Instruction& inst, std::string* error);
-bool DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error);
-bool DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-              std::string* error);
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ */

src/graphics/shader/recompiler/decompiler/OpcodeTable.h

@@ -1,45 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
-
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
-
-#include <cstddef>
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder::Detail {
-
-struct OpcodeMap {
-	uint32_t encoding = 0;
-	Opcode   decoded  = Opcode::Unknown;
-};
-
-template <typename Entry, size_t N>
-constexpr const Entry* FindOpcode(const Entry (&table)[N], uint32_t encoding) {
-	for (const auto& entry: table) {
-		if (entry.encoding == encoding) {
-			return &entry;
-		}
-	}
-	return nullptr;
-}
-
-template <typename Entry, size_t N>
-constexpr Opcode LookupOpcode(const Entry (&table)[N], uint32_t encoding) {
-	const auto* entry = FindOpcode(table, encoding);
-	return entry != nullptr ? entry->decoded : Opcode::Unsupported;
-}
-
-template <typename Entry, size_t N>
-constexpr bool HasUniqueEncodings(const Entry (&table)[N]) {
-	for (size_t i = 0; i < N; i++) {
-		for (size_t j = i + 1; j < N; j++) {
-			if (table[i].encoding == table[j].encoding) {
-				return false;
-			}
-		}
-	}
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder::Detail
-
-#endif // EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_

src/graphics/shader/recompiler/decompiler/ScalarAluOps.cpp

@@ -1,277 +0,0 @@
-#include "graphics/shader/recompiler/decompiler/ScalarAluOps.h"
-
-#include "graphics/shader/recompiler/decompiler/OpcodeTable.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-namespace {
-
-using Detail::OpcodeMap;
-
-constexpr OpcodeMap SOP2_OPS[] = {
-    {0x00u, Opcode::SAddU32},       {0x01u, Opcode::SSubU32},       {0x02u, Opcode::SAddI32},
-    {0x03u, Opcode::SSubI32},       {0x04u, Opcode::SAddcU32},      {0x05u, Opcode::SSubbU32},
-    {0x06u, Opcode::SMinI32},       {0x07u, Opcode::SMinU32},       {0x08u, Opcode::SMaxI32},
-    {0x09u, Opcode::SMaxU32},       {0x0au, Opcode::SCselectB32},   {0x0bu, Opcode::SCselectB64},
-    {0x0eu, Opcode::SAndB32},       {0x0fu, Opcode::SAndB64},       {0x10u, Opcode::SOrB32},
-    {0x11u, Opcode::SOrB64},        {0x12u, Opcode::SXorB32},       {0x13u, Opcode::SXorB64},
-    {0x14u, Opcode::SAndn2B32},     {0x15u, Opcode::SAndn2B64},     {0x16u, Opcode::SOrn2B32},
-    {0x17u, Opcode::SOrn2B64},      {0x18u, Opcode::SNandB32},      {0x19u, Opcode::SNandB64},
-    {0x1au, Opcode::SNorB32},       {0x1bu, Opcode::SNorB64},       {0x1cu, Opcode::SXnorB32},
-    {0x1du, Opcode::SXnorB64},      {0x1eu, Opcode::SLshlB32},      {0x1fu, Opcode::SLshlB64},
-    {0x20u, Opcode::SLshrB32},      {0x21u, Opcode::SLshrB64},      {0x22u, Opcode::SAshrI32},
-    {0x24u, Opcode::SBfmB32},       {0x25u, Opcode::SBfmB64},       {0x26u, Opcode::SMulI32},
-    {0x27u, Opcode::SBfeU32},       {0x29u, Opcode::SBfeU64},       {0x2eu, Opcode::SLshl1AddU32},
-    {0x2fu, Opcode::SLshl2AddU32},  {0x30u, Opcode::SLshl3AddU32},  {0x31u, Opcode::SLshl4AddU32},
-    {0x32u, Opcode::SPackLlB32B16}, {0x33u, Opcode::SPackLhB32B16}, {0x34u, Opcode::SPackHhB32B16},
-    {0x35u, Opcode::SMulHiU32},
-};
-
-constexpr OpcodeMap SOP1_OPS[] = {
-    {0x03u, Opcode::SMovB32},
-    {0x04u, Opcode::SMovB64},
-    {0x07u, Opcode::SNotB32},
-    {0x08u, Opcode::SNotB64},
-    {0x0au, Opcode::SWqmB64},
-    {0x0bu, Opcode::SBrevB32},
-    {0x0fu, Opcode::SBcnt1I32B32},
-    {0x10u, Opcode::SBcnt1I32B64},
-    {0x13u, Opcode::SFf1I32B32},
-    {0x16u, Opcode::SFlbitI32B64},
-    {0x1bu, Opcode::SBitset0B32},
-    {0x1du, Opcode::SBitset1B32},
-    {0x1fu, Opcode::SGetpcB64},
-    {0x20u, Opcode::SSetpcB64},
-    {0x24u, Opcode::SAndSaveexecB64},
-    {0x28u, Opcode::SOrn2SaveexecB64},
-    {0x34u, Opcode::SAbsI32},
-    {0x37u, Opcode::SAndn1SaveexecB64},
-    {0x3bu, Opcode::SBitreplicateB64B32},
-    {0x3cu, Opcode::SAndSaveexecB32},
-    {0x44u, Opcode::SAndn1SaveexecB32},
-};
-
-constexpr OpcodeMap SOPC_OPS[] = {
-    {0x00u, Opcode::SCmpEqI32},   {0x01u, Opcode::SCmpLgI32},   {0x02u, Opcode::SCmpGtI32},
-    {0x03u, Opcode::SCmpGeI32},   {0x04u, Opcode::SCmpLtI32},   {0x05u, Opcode::SCmpLeI32},
-    {0x06u, Opcode::SCmpEqU32},   {0x07u, Opcode::SCmpLgU32},   {0x08u, Opcode::SCmpGtU32},
-    {0x09u, Opcode::SCmpGeU32},   {0x0au, Opcode::SCmpLtU32},   {0x0bu, Opcode::SCmpLeU32},
-    {0x0cu, Opcode::SBitcmp0B32}, {0x0du, Opcode::SBitcmp1B32}, {0x12u, Opcode::SCmpEqU64},
-    {0x13u, Opcode::SCmpLgU64},
-};
-
-constexpr OpcodeMap SOPK_OPS[] = {
-    {0x00u, Opcode::SMovkI32},   {0x03u, Opcode::SCmpEqI32}, {0x04u, Opcode::SCmpLgI32},
-    {0x05u, Opcode::SCmpGtI32},  {0x06u, Opcode::SCmpGeI32}, {0x07u, Opcode::SCmpLtI32},
-    {0x08u, Opcode::SCmpLeI32},  {0x09u, Opcode::SCmpEqU32}, {0x0au, Opcode::SCmpLgU32},
-    {0x0bu, Opcode::SCmpGtU32},  {0x0cu, Opcode::SCmpGeU32}, {0x0du, Opcode::SCmpLtU32},
-    {0x0eu, Opcode::SCmpLeU32},  {0x0fu, Opcode::SAddI32},   {0x10u, Opcode::SMulkI32},
-    {0x13u, Opcode::SSetregB32}, {0x17u, Opcode::SWaitcnt},  {0x18u, Opcode::SWaitcnt},
-    {0x19u, Opcode::SWaitcnt},   {0x1au, Opcode::SWaitcnt},
-};
-
-constexpr OpcodeMap SOPP_OPS[] = {
-    {0x00u, Opcode::SNop},          {0x01u, Opcode::SEndpgm},       {0x02u, Opcode::SBranch},
-    {0x04u, Opcode::SCbranchScc0},  {0x05u, Opcode::SCbranchScc1},  {0x06u, Opcode::SCbranchVccz},
-    {0x07u, Opcode::SCbranchVccnz}, {0x08u, Opcode::SCbranchExecz}, {0x09u, Opcode::SCbranchExecnz},
-    {0x0au, Opcode::SBarrier},      {0x0cu, Opcode::SWaitcnt},      {0x0eu, Opcode::SSleep},
-    {0x10u, Opcode::SSendmsg},      {0x16u, Opcode::STtraceData},   {0x20u, Opcode::SInstPrefetch},
-};
-static_assert(Detail::HasUniqueEncodings(SOP1_OPS));
-static_assert(Detail::HasUniqueEncodings(SOP2_OPS));
-static_assert(Detail::HasUniqueEncodings(SOPK_OPS));
-static_assert(Detail::HasUniqueEncodings(SOPC_OPS));
-static_assert(Detail::HasUniqueEncodings(SOPP_OPS));
-
-bool DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                         Instruction& inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) {
-	if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
-	    !DecodeScalarSource(ssrc1, pc, inst.src1, error)) {
-		return false;
-	}
-	inst.src_count = 2;
-	return ReadLiteralOperands(code, word_index, inst, error);
-}
-
-} // namespace
-
-bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 8u) & 0xffu;
-	const uint32_t ssrc0  = word & 0xffu;
-	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-
-	inst.pc        = pc;
-	inst.word      = word;
-	inst.family    = Family::SOP1;
-	inst.opcode_id = opcode;
-	inst.opcode    = Detail::LookupOpcode(SOP1_OPS, opcode);
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented");
-		return true;
-	}
-
-	switch (inst.opcode) {
-		case Opcode::SGetpcB64:
-			inst.src_count = 0;
-			return DecodeScalarDestination(sdst, pc, inst.dst, error);
-		case Opcode::SSetpcB64:
-			inst.src_count = 1;
-			inst.dst.kind  = OperandKind::Null;
-			if (!DecodeScalarSource(ssrc0, pc, inst.src0, error)) {
-				return false;
-			}
-			return ReadLiteralOperands(code, word_index, inst, error);
-		default: break;
-	}
-
-	if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
-	    !DecodeScalarDestination(sdst, pc, inst.dst, error)) {
-		return false;
-	}
-	inst.src_count = 1;
-	return ReadLiteralOperands(code, word_index, inst, error);
-}
-
-bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 23u) & 0x7fu;
-	const uint32_t ssrc1  = (word >> 8u) & 0xffu;
-	const uint32_t ssrc0  = word & 0xffu;
-	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-
-	inst.pc        = pc;
-	inst.word      = word;
-	inst.family    = Family::SOP2;
-	inst.opcode_id = opcode;
-	inst.opcode    = Detail::LookupOpcode(SOP2_OPS, opcode);
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented");
-		return true;
-	}
-
-	if (!DecodeScalarDestination(sdst, pc, inst.dst, error)) {
-		return false;
-	}
-	return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
-}
-
-bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 23u) & 0x1fu;
-	const uint32_t sdst   = (word >> 16u) & 0x7fu;
-	const auto     imm    = static_cast<int16_t>(word & 0xffffu);
-
-	(void)error;
-
-	inst.pc              = pc;
-	inst.word            = word;
-	inst.family          = Family::SOPK;
-	inst.opcode_id       = opcode;
-	inst.opcode          = Detail::LookupOpcode(SOPK_OPS, opcode);
-	inst.src0.kind       = OperandKind::IntegerInlineConstant;
-	inst.src0.signed_val = imm;
-	inst.src0.value      = static_cast<uint32_t>(imm);
-	inst.src_count       = 1;
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented");
-		return true;
-	}
-
-	switch (inst.opcode) {
-		case Opcode::SMovkI32: return DecodeScalarDestination(sdst, pc, inst.dst, error);
-		case Opcode::SWaitcnt: {
-			const uint32_t waitcnt = word & 0xffffu;
-			inst.dst.kind          = OperandKind::Null;
-			inst.src0.signed_val   = static_cast<int32_t>(waitcnt);
-			inst.src0.value        = waitcnt;
-			inst.src_count         = 1;
-			return true;
-		}
-		case Opcode::SSetregB32:
-			inst.dst.kind        = OperandKind::Null;
-			inst.src1.kind       = OperandKind::LiteralConstant;
-			inst.src1.value      = word & 0xffffu;
-			inst.src1.signed_val = static_cast<int32_t>(imm);
-			inst.src_count       = 2;
-			return DecodeScalarSource(sdst, pc, inst.src0, error);
-		default: break;
-	}
-
-	inst.src1 = inst.src0;
-	if (!DecodeScalarSource(sdst, pc, inst.src0, error)) {
-		return false;
-	}
-	if (inst.opcode == Opcode::SAddI32 || inst.opcode == Opcode::SMulkI32) {
-		inst.src_count = 2;
-		return DecodeScalarDestination(sdst, pc, inst.dst, error);
-	}
-
-	inst.dst.kind  = OperandKind::Scc;
-	inst.src_count = 2;
-	return true;
-}
-
-bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t ssrc1  = (word >> 8u) & 0xffu;
-	const uint32_t ssrc0  = word & 0xffu;
-	const uint32_t opcode = (word >> 16u) & 0x7fu;
-
-	inst.pc        = pc;
-	inst.word      = word;
-	inst.family    = Family::SOPC;
-	inst.opcode_id = opcode;
-	inst.opcode    = Detail::LookupOpcode(SOPC_OPS, opcode);
-	inst.dst.kind  = OperandKind::Scc;
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented");
-		return true;
-	}
-
-	return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
-}
-
-bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	const uint32_t word   = code[word_index];
-	const uint32_t opcode = (word >> 16u) & 0x7fu;
-	const uint32_t simm   = word & 0xffffu;
-
-	(void)error;
-
-	inst.pc              = pc;
-	inst.word            = word;
-	inst.family          = Family::SOPP;
-	inst.opcode_id       = opcode;
-	inst.opcode          = Detail::LookupOpcode(SOPP_OPS, opcode);
-	inst.src0.kind       = OperandKind::LiteralConstant;
-	inst.src0.value      = simm;
-	inst.src0.signed_val = static_cast<int16_t>(simm);
-	inst.src_count = (inst.opcode == Opcode::SNop || inst.opcode == Opcode::SWaitcnt ||
-	                  inst.opcode == Opcode::SSleep || inst.opcode == Opcode::SSendmsg ||
-	                  inst.opcode == Opcode::STtraceData || inst.opcode == Opcode::SInstPrefetch)
-	                     ? 1
-	                     : 0;
-	inst.branch_offset = static_cast<int32_t>(static_cast<int16_t>(simm)) * 4;
-	inst.branch_target = pc + 4u + static_cast<uint32_t>(inst.branch_offset);
-	SetRawWords(inst, code, word_index, 1);
-
-	if (inst.opcode == Opcode::Unsupported) {
-		SetUnsupported(inst, Family::SOPP, opcode, "SOPP control-flow opcode is not implemented");
-	}
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/decompiler/ShaderDecoder.h

@@ -1,686 +0,0 @@
-#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_
-#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_
-
-#include "common/common.h"
-#include "common/stringUtils.h"
-
-#include <span>
-#include <vector>
-
-namespace Libs::Graphics::ShaderRecompiler::Decoder {
-
-enum class Family {
-	Unknown,
-	SOP1,
-	SOP2,
-	SOPK,
-	SOPC,
-	SOPP,
-	VOP1,
-	VOP2,
-	VOP3,
-	VOP3P,
-	VOPC,
-	VINTRP,
-	SMEM,
-	MUBUF,
-	MTBUF,
-	FLAT,
-	DS,
-	MIMG,
-	EXP
-};
-
-enum class Opcode {
-	Unknown,
-	Unsupported,
-
-	SMovB32,
-	SMovB64,
-	SMovkI32,
-	SAbsI32,
-	SBrevB32,
-	SBcnt1I32B32,
-	SBcnt1I32B64,
-	SFf1I32B32,
-	SFlbitI32B64,
-	SBitreplicateB64B32,
-	SGetpcB64,
-	SSetpcB64,
-	SAndSaveexecB32,
-	SAndn1SaveexecB32,
-	SAndSaveexecB64,
-	SOrn2SaveexecB64,
-	SAndn1SaveexecB64,
-	SNotB32,
-	SNotB64,
-	SWqmB64,
-	SAddU32,
-	SAddcU32,
-	SSubbU32,
-	SSubU32,
-	SAddI32,
-	SSubI32,
-	SBitcmp0B32,
-	SBitcmp1B32,
-	SBitset0B32,
-	SBitset1B32,
-	SMinI32,
-	SMaxI32,
-	SMinU32,
-	SMaxU32,
-	SAndB32,
-	SAndB64,
-	SAndn2B32,
-	SAndn2B64,
-	SOrB32,
-	SOrB64,
-	SOrn2B32,
-	SOrn2B64,
-	SXorB32,
-	SXorB64,
-	SNandB32,
-	SNandB64,
-	SNorB32,
-	SNorB64,
-	SXnorB32,
-	SXnorB64,
-	SLshlB32,
-	SLshlB64,
-	SLshl1AddU32,
-	SLshl2AddU32,
-	SLshl3AddU32,
-	SLshl4AddU32,
-	SLshrB32,
-	SLshrB64,
-	SAshrI32,
-	SMulI32,
-	SMulHiU32,
-	SMulkI32,
-	SBfeU32,
-	SBfeU64,
-	SBfmB32,
-	SBfmB64,
-	SCselectB32,
-	SCselectB64,
-	SPackLlB32B16,
-	SPackLhB32B16,
-	SPackHhB32B16,
-	SCmpEqI32,
-	SCmpLgI32,
-	SCmpGtI32,
-	SCmpGeI32,
-	SCmpLtI32,
-	SCmpLeI32,
-	SCmpEqU32,
-	SCmpLgU32,
-	SCmpGtU32,
-	SCmpGeU32,
-	SCmpLtU32,
-	SCmpLeU32,
-	SCmpEqU64,
-	SCmpLgU64,
-
-	VNop,
-	VMovB32,
-	VMovreldB32,
-	VMovrelsB32,
-	VReadfirstlaneB32,
-	VReadlaneB32,
-	VWritelaneB32,
-	VPermlane16B32,
-	VPermlanex16B32,
-	VCubeidF32,
-	VCubescF32,
-	VCubetcF32,
-	VCubemaF32,
-	VCndmaskB32,
-	VDot2cF32F16,
-	VCvtF32I32,
-	VCvtF32U32,
-	VCvtU32F32,
-	VCvtI32F32,
-	VCvtF16F32,
-	VCvtF32F16,
-	VCvtF16U16,
-	VCvtU16F16,
-	VCvtF16I16,
-	VCvtI16F16,
-	VCvtRpiI32F32,
-	VCvtFlrI32F32,
-	VCvtOffF32I4,
-	VCvtF32Ubyte0,
-	VCvtF32Ubyte1,
-	VCvtF32Ubyte2,
-	VCvtF32Ubyte3,
-	VRcpF32,
-	VFractF32,
-	VTruncF32,
-	VCeilF32,
-	VRndneF32,
-	VFloorF32,
-	VExpF32,
-	VLogF32,
-	VRsqF32,
-	VSqrtF32,
-	VRcpF16,
-	VSqrtF16,
-	VRsqF16,
-	VLogF16,
-	VExpF16,
-	VFloorF16,
-	VCeilF16,
-	VTruncF16,
-	VRndneF16,
-	VSinF32,
-	VCosF32,
-	VNotB32,
-	VBfrevB32,
-	VFfbhU32,
-	VFfblB32,
-	VAddF32,
-	VSubF32,
-	VSubrevF32,
-	VMulF32,
-	VMacF32,
-	VMadmkF32,
-	VMadakF32,
-	VMinF32,
-	VMaxF32,
-	VAddF16,
-	VSubF16,
-	VSubrevF16,
-	VMulF16,
-	VFmacF16,
-	VFmamkF16,
-	VFmaakF16,
-	VMinI32,
-	VMaxI32,
-	VMulI32I24,
-	VMulU32U24,
-	VMulLoU32,
-	VMulHiU32,
-	VMulLoI32,
-	VMulHiI32,
-	VCvtPkrtzF16F32,
-	VCvtPkU8F32,
-	VMadF32,
-	VMadI32I24,
-	VMadU32U24,
-	VMadU64U32,
-	VFmaF32,
-	VFmaF16,
-	VPackB32F16,
-	VBfeU32,
-	VBfeI32,
-	VBfiB32,
-	VAlignbitB32,
-	VMin3F32,
-	VMin3I32,
-	VMin3U32,
-	VMin3F16,
-	VMax3F32,
-	VMax3I32,
-	VMax3U32,
-	VMax3F16,
-	VMed3F32,
-	VMed3I32,
-	VMed3U32,
-	VMed3F16,
-	VSadU32,
-	VAdd3U32,
-	VLshlAddU32,
-	VAddLshlU32,
-	VXadU32,
-	VLshlOrB32,
-	VAndOrB32,
-	VOr3B32,
-	VXor3B32,
-	VAddI32,
-	VSubI32,
-	VSubrevI32,
-	VBfmB32,
-	VLdexpF32,
-	VCvtPknormI16F32,
-	VCvtPknormU16F32,
-	VCvtPkU16U32,
-	VPkMadI16,
-	VPkMulLoU16,
-	VPkAddI16,
-	VPkSubI16,
-	VPkLshlrevB16,
-	VPkLshrrevB16,
-	VPkAshrrevI16,
-	VPkMaxI16,
-	VPkMinI16,
-	VPkMadU16,
-	VPkAddF16,
-	VPkMulF16,
-	VPkMinF16,
-	VPkMaxF16,
-	VPkFmaF16,
-	VPkFmacF16,
-	VPkAddU16,
-	VPkSubU16,
-	VPkMaxU16,
-	VPkMinU16,
-	VMadMixloF16,
-	VMadMixhiF16,
-	VAddNcU32,
-	VAddcU32,
-	VSubNcU32,
-	VSubrevNcU32,
-	VAddNcU16,
-	VSubNcU16,
-	VMaxU16,
-	VMaxI16,
-	VMinU16,
-	VMinI16,
-	VAddNcI16,
-	VSubNcI16,
-	VAndB32,
-	VOrB32,
-	VXorB32,
-	VXnorB32,
-	VLshlB32,
-	VLshlrevB32,
-	VLshrB32,
-	VLshrrevB32,
-	VAshrI32,
-	VAshrrevI32,
-	VLshlrevB16,
-	VLshrrevB16,
-	VAshrrevI16,
-	VBcntU32B32,
-	VMbcntLoU32B32,
-	VMbcntHiU32B32,
-	VMinU32,
-	VMaxU32,
-	VMaxF16,
-	VMinF16,
-	VCmpFF32,
-	VCmpLtF32,
-	VCmpEqF32,
-	VCmpLeF32,
-	VCmpGtF32,
-	VCmpLgF32,
-	VCmpGeF32,
-	VCmpOF32,
-	VCmpUF32,
-	VCmpNgeF32,
-	VCmpNlgF32,
-	VCmpNgtF32,
-	VCmpNleF32,
-	VCmpNeqF32,
-	VCmpNltF32,
-	VCmpTruF32,
-	VCmpxLtF32,
-	VCmpxEqF32,
-	VCmpxLeF32,
-	VCmpxGtF32,
-	VCmpxLgF32,
-	VCmpxGeF32,
-	VCmpxNgeF32,
-	VCmpxNlgF32,
-	VCmpxNgtF32,
-	VCmpxNleF32,
-	VCmpxNeqF32,
-	VCmpxNltF32,
-	VCmpFI32,
-	VCmpLtI32,
-	VCmpEqI32,
-	VCmpLeI32,
-	VCmpGtI32,
-	VCmpNeI32,
-	VCmpGeI32,
-	VCmpTI32,
-	VCmpClassF32,
-	VCmpLtI16,
-	VCmpEqI16,
-	VCmpLeI16,
-	VCmpGtI16,
-	VCmpNeI16,
-	VCmpGeI16,
-	VCmpLtF16,
-	VCmpEqF16,
-	VCmpLeF16,
-	VCmpGtF16,
-	VCmpLgF16,
-	VCmpGeF16,
-	VCmpNeqF16,
-	VCmpxLtF16,
-	VCmpxEqF16,
-	VCmpxLeF16,
-	VCmpxGtF16,
-	VCmpxGeF16,
-	VCmpxNeqF16,
-	VCmpxNltF16,
-	VCmpxLtI32,
-	VCmpxEqI32,
-	VCmpxLeI32,
-	VCmpxGtI32,
-	VCmpxNeI32,
-	VCmpxGeI32,
-	VCmpLtU16,
-	VCmpEqU16,
-	VCmpLeU16,
-	VCmpGtU16,
-	VCmpNeU16,
-	VCmpGeU16,
-	VCmpFU32,
-	VCmpLtU32,
-	VCmpEqU32,
-	VCmpLeU32,
-	VCmpGtU32,
-	VCmpNeU32,
-	VCmpGeU32,
-	VCmpTU32,
-	VCmpEqI64,
-	VCmpNeU64,
-	VCmpxLtU32,
-	VCmpxEqU32,
-	VCmpxLeU32,
-	VCmpxGtU32,
-	VCmpxNeU32,
-	VCmpxGeU32,
-
-	SLoadDword,
-	SLoadDwordx2,
-	SLoadDwordx4,
-	SLoadDwordx8,
-	SLoadDwordx16,
-	SBufferLoadDword,
-	SBufferLoadDwordx2,
-	SBufferLoadDwordx4,
-	SBufferLoadDwordx8,
-	SBufferLoadDwordx16,
-	BufferLoadFormatX,
-	BufferLoadFormatXy,
-	BufferLoadFormatXyz,
-	BufferLoadFormatXyzw,
-	BufferStoreFormatX,
-	BufferStoreFormatXy,
-	BufferStoreFormatXyz,
-	BufferStoreFormatXyzw,
-	BufferLoadUbyte,
-	BufferLoadSbyte,
-	BufferLoadUshort,
-	BufferLoadSshort,
-	BufferLoadDword,
-	BufferLoadDwordx2,
-	BufferLoadDwordx3,
-	BufferLoadDwordx4,
-	BufferStoreByte,
-	BufferStoreShort,
-	BufferStoreDword,
-	BufferStoreDwordx2,
-	BufferStoreDwordx3,
-	BufferStoreDwordx4,
-	TBufferLoadFormatX,
-	TBufferLoadFormatXy,
-	TBufferLoadFormatXyz,
-	TBufferLoadFormatXyzw,
-	TBufferStoreFormatX,
-	TBufferStoreFormatXy,
-	TBufferStoreFormatXyz,
-	TBufferStoreFormatXyzw,
-	BufferAtomicSwap,
-	BufferAtomicAdd,
-	BufferAtomicSub,
-	BufferAtomicSMin,
-	BufferAtomicUMin,
-	BufferAtomicSMax,
-	BufferAtomicUMax,
-	BufferAtomicAnd,
-	BufferAtomicOr,
-	BufferAtomicXor,
-	BufferAtomicFMin,
-	BufferAtomicFMax,
-	FlatLoadUbyte,
-	FlatLoadSbyte,
-	FlatLoadUshort,
-	FlatLoadSshort,
-	FlatLoadDword,
-	FlatLoadDwordx2,
-	FlatLoadDwordx3,
-	FlatLoadDwordx4,
-	FlatStoreByte,
-	FlatStoreShort,
-	FlatStoreDword,
-	FlatStoreDwordx2,
-	FlatStoreDwordx3,
-	FlatStoreDwordx4,
-	DsAddU32,
-	DsAddRtnU32,
-	DsSubU32,
-	DsSubRtnU32,
-	DsMinI32,
-	DsMinRtnI32,
-	DsMaxI32,
-	DsMaxRtnI32,
-	DsMinU32,
-	DsMinRtnU32,
-	DsMaxU32,
-	DsMaxRtnU32,
-	DsAndB32,
-	DsAndRtnB32,
-	DsOrB32,
-	DsOrRtnB32,
-	DsXorB32,
-	DsXorRtnB32,
-	DsWrxchgRtnB32,
-	DsMinF32,
-	DsMaxF32,
-	DsSwizzleB32,
-	DsConsume,
-	DsAppend,
-	DsReadSbyte,
-	DsReadUbyte,
-	DsReadSshort,
-	DsReadUshort,
-	DsRead2B32,
-	DsReadB32,
-	DsReadB64,
-	DsRead2B64,
-	DsRead2St64B64,
-	DsReadB96,
-	DsReadB128,
-	DsWriteByte,
-	DsWriteShort,
-	DsWrite2B32,
-	DsWrite2St64B32,
-	DsWrite2B64,
-	DsWrite2St64B64,
-	DsWriteB32,
-	DsWriteB64,
-	DsWriteB96,
-	DsWriteB128,
-	DsWriteAddtidB32,
-	DsReadAddtidB32,
-	ImageGetResinfo,
-	ImageGetLod,
-	ImageLoad,
-	ImageLoadMip,
-	ImageStore,
-	ImageStoreMip,
-	ImageAtomicAdd,
-	ImageAtomicUMin,
-	ImageAtomicUMax,
-	ImageAtomicAnd,
-	ImageAtomicOr,
-	ImageAtomicXor,
-	ImageSample,
-	ImageGather4Lz,
-	ImageGather4C,
-	ImageGather4CLz,
-	ImageGather4LzO,
-	ImageGather4CO,
-	ImageGather4CLzO,
-	ImageGather4H,
-	VInterpP1F32,
-	VInterpP2F32,
-	VInterpMovF32,
-
-	SNop,
-	SWaitcnt,
-	SBarrier,
-	SBranch,
-	SCbranchScc0,
-	SCbranchScc1,
-	SCbranchVccz,
-	SCbranchVccnz,
-	SCbranchExecz,
-	SCbranchExecnz,
-	SSendmsg,
-	SSetregB32,
-	SSleep,
-	STtraceData,
-	SInstPrefetch,
-	SEndpgm,
-	Exp
-};
-
-enum class OperandKind {
-	Unknown,
-	LiteralConstant,
-	IntegerInlineConstant,
-	FloatInlineConstant,
-	Sgpr,
-	VccLo,
-	VccHi,
-	VccZ,
-	ExecLo,
-	ExecHi,
-	ExecZ,
-	Scc,
-	M0,
-	PopsExitingWaveId,
-	Null,
-	Vgpr,
-};
-
-enum ImageSampleFlag : uint32_t {
-	ImageSampleFlagLod              = 1u << 0u,
-	ImageSampleFlagBias             = 1u << 1u,
-	ImageSampleFlagDerivative       = 1u << 2u,
-	ImageSampleFlagCompare          = 1u << 3u,
-	ImageSampleFlagOffset           = 1u << 4u,
-	ImageSampleFlagLevelZero        = 1u << 5u,
-	ImageSampleFlagLodClamp         = 1u << 6u,
-	ImageSampleFlagA16              = 1u << 7u,
-	ImageSampleFlagCd               = 1u << 8u,
-	ImageSampleFlagGatherHorizontal = 1u << 9u,
-};
-
-enum class ImageDimension : uint32_t {
-	Unknown,
-	Dim1D,
-	Dim1DArray,
-	Dim2D,
-	Dim3D,
-	Dim2DArray,
-	Dim2DMsaa,
-	Dim2DMsaaArray,
-};
-
-constexpr uint32_t MaxInstructionRawWords       = 5u;
-constexpr uint32_t MaxImageNsaAddressComponents = 12u;
-
-struct Operand {
-	OperandKind kind               = OperandKind::Unknown;
-	uint32_t    value              = 0;
-	int32_t     signed_val         = 0;
-	float       float_val          = 0.0f;
-	uint32_t    reg                = 0;
-	uint32_t    sdwa_sel           = 6;
-	uint32_t    sdwa_dst_unused    = 0;
-	uint32_t    omod               = 0;
-	uint32_t    dpp_ctrl           = 0;
-	uint32_t    dpp_row_mask       = 0xf;
-	uint32_t    dpp_bank_mask      = 0xf;
-	bool        sdwa_sext          = false;
-	bool        dpp_fetch_inactive = false;
-	bool        dpp_bound_ctrl     = false;
-	bool        op_sel             = false;
-	bool        op_sel_hi          = false;
-	bool        negate             = false;
-	bool        negate_hi          = false;
-	bool        absolute           = false;
-	bool        clamp              = false;
-	bool        dpp                = false;
-};
-
-struct Instruction {
-	uint32_t       pc                          = 0;
-	uint32_t       word                        = 0;
-	uint32_t       word_count                  = 1;
-	uint32_t       raw[MaxInstructionRawWords] = {};
-	uint32_t       raw_count                   = 1;
-	Family         family                      = Family::Unknown;
-	uint32_t       opcode_id                   = 0;
-	Opcode         opcode                      = Opcode::Unknown;
-	Operand        dst;
-	Operand        dst2;
-	Operand        src0;
-	Operand        src1;
-	Operand        src2;
-	Operand        src3;
-	uint32_t       src_count                                    = 0;
-	uint32_t       offset                                       = 0;
-	uint32_t       secondary_offset                             = 0;
-	uint32_t       dmask                                        = 0;
-	uint32_t       data_dwords                                  = 1;
-	uint32_t       data_bits                                    = 32;
-	uint32_t       data_format                                  = 0;
-	uint32_t       number_format                                = 0;
-	uint32_t       image_sample_flags                           = 0;
-	ImageDimension image_dimension                              = ImageDimension::Unknown;
-	uint32_t       image_address_components                     = 0;
-	uint32_t       image_nsa_dwords                             = 0;
-	uint32_t       image_nsa_addr[MaxImageNsaAddressComponents] = {};
-	uint32_t       memory_segment                               = 0;
-	bool           data_signed                                  = false;
-	bool           typed                                        = false;
-	bool           formatted                                    = false;
-	bool           gds                                          = false;
-	bool           glc                                          = false;
-	bool           slc                                          = false;
-	bool           idxen                                        = false;
-	bool           offen                                        = false;
-	int32_t        branch_offset                                = 0;
-	uint32_t       branch_target                                = 0;
-	struct {
-		uint32_t target = 0;
-		uint32_t en     = 0;
-		bool     done   = false;
-		bool     compr  = false;
-		bool     vm     = false;
-	} exp;
-	std::string unsupported_reason;
-};
-
-struct Program {
-	std::span<const uint32_t> code;
-	std::vector<Instruction>  instructions;
-};
-
-bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error);
-
-bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error);
-bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error);
-bool DecodeVectorGpr(uint32_t reg, Operand& operand, std::string* error);
-bool ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                         std::string* error);
-void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index,
-                 uint32_t word_count);
-void SetUnsupported(Instruction& inst, Family family, uint32_t opcode_id, const char* reason);
-std::string FamilyToString(Family family);
-std::string OpcodeToString(Opcode opcode);
-std::string OperandToString(const Operand& operand);
-const char* ImageDimensionToString(ImageDimension dimension);
-std::string InstructionToString(const Instruction& inst);
-std::string ProgramToString(const Program& program);
-
-} // namespace Libs::Graphics::ShaderRecompiler::Decoder
-
-#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_ */

src/graphics/shader/recompiler/emitter/SpirvEmitter.cpp

@@ -1,525 +0,0 @@
-#include "graphics/shader/recompiler/emitter/SpirvEmitter.h"
-
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
-#include "graphics/shader/recompiler/ir/SrtWalker.h"
-
-#include <algorithm>
-#include <array>
-#include <cinttypes>
-#include <cstdio>
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv {
-
-namespace {
-
-bool Fail(std::string* error, const std::string& message) {
-	if (error != nullptr) {
-		*error = message;
-	}
-	return false;
-}
-
-bool ImageBinding(const IR::ImageResource& image, IR::DescriptorBindingKind& kind) {
-	using Kind = IR::DescriptorBindingKind;
-	using Dim  = Decoder::ImageDimension;
-	if (image.kind == IR::ResourceKind::Image || image.kind == IR::ResourceKind::ImageUint) {
-		const bool integer = image.kind == IR::ResourceKind::ImageUint;
-		switch (image.dimension) {
-			case Dim::Dim1D: kind = integer ? Kind::SampledUint1D : Kind::Sampled1D; return true;
-			case Dim::Dim1DArray:
-				kind = integer ? Kind::SampledUint1DArray : Kind::Sampled1DArray;
-				return true;
-			case Dim::Dim2D: kind = integer ? Kind::SampledUint2D : Kind::Sampled2D; return true;
-			case Dim::Dim2DMsaa:
-				kind = integer ? Kind::SampledUint2DMsaa : Kind::Sampled2DMsaa;
-				return true;
-			case Dim::Dim3D: kind = integer ? Kind::SampledUint3D : Kind::Sampled3D; return true;
-			case Dim::Dim2DArray:
-				kind = integer ? Kind::SampledUint2DArray : Kind::Sampled2DArray;
-				return true;
-			case Dim::Dim2DMsaaArray:
-				kind = integer ? Kind::SampledUint2DMsaaArray : Kind::Sampled2DMsaaArray;
-				return true;
-			case Dim::Unknown: return false;
-		}
-	}
-	const bool uint_image = image.kind == IR::ResourceKind::StorageImageUint;
-	if (image.kind != IR::ResourceKind::StorageImage && !uint_image) {
-		return false;
-	}
-	switch (image.dimension) {
-		case Dim::Dim1D: kind = uint_image ? Kind::StorageUint1D : Kind::Storage1D; return true;
-		case Dim::Dim1DArray:
-			kind = uint_image ? Kind::StorageUint1DArray : Kind::Storage1DArray;
-			return true;
-		case Dim::Dim2D: kind = uint_image ? Kind::StorageUint2D : Kind::Storage2D; return true;
-		case Dim::Dim3D: kind = uint_image ? Kind::StorageUint3D : Kind::Storage3D; return true;
-		case Dim::Dim2DArray:
-			kind = uint_image ? Kind::StorageUint2DArray : Kind::Storage2DArray;
-			return true;
-		case Dim::Dim2DMsaa:
-		case Dim::Dim2DMsaaArray: return false;
-		case Dim::Unknown: return false;
-	}
-	return false;
-}
-
-bool NeedsSampler(IR::Opcode op) {
-	return op == IR::Opcode::ImageSample || op == IR::Opcode::ImageGather4 ||
-	       op == IR::Opcode::ImageGetLod;
-}
-
-bool IsBufferLoad(IR::Opcode op) {
-	switch (op) {
-		case IR::Opcode::BufferLoadUbyte:
-		case IR::Opcode::BufferLoadSbyte:
-		case IR::Opcode::BufferLoadUshort:
-		case IR::Opcode::BufferLoadSshort:
-		case IR::Opcode::BufferLoadDword: return true;
-		default: return false;
-	}
-}
-
-bool IsBufferStore(IR::Opcode op) {
-	return op == IR::Opcode::BufferStoreByte || op == IR::Opcode::BufferStoreShort ||
-	       op == IR::Opcode::BufferStoreDword;
-}
-
-bool IsFlatLoad(IR::Opcode op) {
-	return op == IR::Opcode::FlatLoadUbyte || op == IR::Opcode::FlatLoadSbyte ||
-	       op == IR::Opcode::FlatLoadUshort || op == IR::Opcode::FlatLoadSshort ||
-	       op == IR::Opcode::FlatLoadDword;
-}
-
-bool IsFlatStore(IR::Opcode op) {
-	return op == IR::Opcode::FlatStoreByte || op == IR::Opcode::FlatStoreShort ||
-	       op == IR::Opcode::FlatStoreDword;
-}
-
-bool IsDsKind(IR::ResourceKind kind) {
-	return kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds;
-}
-
-bool IsDsRead(IR::Opcode op) {
-	switch (op) {
-		case IR::Opcode::DsReadUbyte:
-		case IR::Opcode::DsReadSbyte:
-		case IR::Opcode::DsReadUshort:
-		case IR::Opcode::DsReadSshort:
-		case IR::Opcode::DsReadB32: return true;
-		default: return false;
-	}
-}
-
-bool IsDsWrite(IR::Opcode op) {
-	switch (op) {
-		case IR::Opcode::DsWriteByte:
-		case IR::Opcode::DsWriteShort:
-		case IR::Opcode::DsWriteB32: return true;
-		default: return false;
-	}
-}
-
-bool IsAtomic(IR::Opcode op) {
-	switch (op) {
-		case IR::Opcode::AtomicSwapU32:
-		case IR::Opcode::AtomicAddU32:
-		case IR::Opcode::AtomicSubU32:
-		case IR::Opcode::AtomicSMinI32:
-		case IR::Opcode::AtomicUMinU32:
-		case IR::Opcode::AtomicSMaxI32:
-		case IR::Opcode::AtomicUMaxU32:
-		case IR::Opcode::AtomicAndU32:
-		case IR::Opcode::AtomicOrU32:
-		case IR::Opcode::AtomicXorU32:
-		case IR::Opcode::AtomicFMinF32:
-		case IR::Opcode::AtomicFMaxF32: return true;
-		default: return false;
-	}
-}
-
-uint32_t BufferAddressOperandCount(const IR::Instruction& inst) {
-	return 1u + (inst.memory.idxen ? 1u : 0u) + (inst.memory.offen ? 1u : 0u);
-}
-
-bool ValidateInstructionContract(const IR::Instruction& inst, std::string* error) {
-	if (static_cast<size_t>(inst.op) >= static_cast<size_t>(IR::Opcode::Count)) {
-		return Fail(error, "IR instruction opcode is out of range");
-	}
-	if (inst.src_count > std::size(inst.src)) {
-		return Fail(error, "instruction source count exceeds fixed IR storage");
-	}
-	const auto kind      = inst.memory.kind;
-	const auto flat_kind = kind == IR::ResourceKind::Flat || kind == IR::ResourceKind::Global ||
-	                       kind == IR::ResourceKind::Scratch;
-	const auto buffer_address_count = BufferAddressOperandCount(inst);
-	if ((IsBufferLoad(inst.op) &&
-	     (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count)) ||
-	    (IsBufferStore(inst.op) &&
-	     (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) ||
-	    (inst.op == IR::Opcode::SLoadDword &&
-	     (kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
-	    (inst.op == IR::Opcode::SBufferLoadDword &&
-	     (kind != IR::ResourceKind::ScalarBuffer || inst.src_count != 1)) ||
-	    (IsFlatLoad(inst.op) && (!flat_kind || inst.src_count != 2)) ||
-	    (IsFlatStore(inst.op) && (!flat_kind || inst.src_count != 3))) {
-		return Fail(error, "memory instruction opcode/kind/operand contract is invalid");
-	}
-	const auto ds_kind     = IsDsKind(kind);
-	const auto ds_resource = !ds_kind || inst.memory.resource == 0;
-	if ((IsDsRead(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 1 ||
-	                           inst.dst.kind != IR::OperandKind::Register)) ||
-	    (IsDsWrite(inst.op) && (!ds_kind || !ds_resource || inst.src_count != 2 ||
-	                            inst.dst.kind != IR::OperandKind::Null)) ||
-	    ((inst.op == IR::Opcode::DsAppend || inst.op == IR::Opcode::DsConsume) &&
-	     (!ds_kind || !ds_resource || inst.src_count != 1 ||
-	      inst.dst.kind != IR::OperandKind::Register)) ||
-	    ((inst.op == IR::Opcode::DsMinF32 || inst.op == IR::Opcode::DsMaxF32) &&
-	     (!ds_kind || !ds_resource || inst.src_count != 3 ||
-	      inst.dst.kind != IR::OperandKind::Null)) ||
-	    (inst.op == IR::Opcode::DsWriteAddtidB32 &&
-	     (kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 2 ||
-	      inst.dst.kind != IR::OperandKind::Null)) ||
-	    (inst.op == IR::Opcode::DsReadAddtidB32 &&
-	     (kind != IR::ResourceKind::Lds || !ds_resource || inst.src_count != 1 ||
-	      inst.dst.kind != IR::OperandKind::Register)) ||
-	    (inst.op == IR::Opcode::DsSwizzleB32 &&
-	     (kind != IR::ResourceKind::None || inst.src_count != 2 ||
-	      inst.dst.kind != IR::OperandKind::Register))) {
-		return Fail(error, "DS instruction has an invalid resource/operand contract");
-	}
-	if ((inst.op == IR::Opcode::ImageSample || inst.op == IR::Opcode::ImageGather4 ||
-	     inst.op == IR::Opcode::ImageGetLod || inst.op == IR::Opcode::ImageLoad ||
-	     inst.op == IR::Opcode::ImageGetResinfo) &&
-	    ((kind != IR::ResourceKind::Image && kind != IR::ResourceKind::ImageUint) ||
-	     inst.src_count != 1)) {
-		return Fail(error, "sampled image instruction has an invalid resource class");
-	}
-	if (inst.op == IR::Opcode::ImageGather4 &&
-	    (inst.memory.image_dimension == Decoder::ImageDimension::Dim1D ||
-	     inst.memory.image_dimension == Decoder::ImageDimension::Dim1DArray)) {
-		return Fail(error, "1D image gather is not supported by SPIR-V");
-	}
-	if (inst.op == IR::Opcode::ImageStore &&
-	    ((kind != IR::ResourceKind::StorageImage && kind != IR::ResourceKind::StorageImageUint) ||
-	     inst.src_count != 2)) {
-		return Fail(error, "storage image instruction has an invalid resource class");
-	}
-	const auto float_atomic =
-	    inst.op == IR::Opcode::AtomicFMinF32 || inst.op == IR::Opcode::AtomicFMaxF32;
-	if (float_atomic &&
-	    (kind != IR::ResourceKind::Buffer || inst.src_count != buffer_address_count + 1u)) {
-		return Fail(error, "floating-point atomic instruction must target a buffer");
-	}
-	if (IsAtomic(inst.op) &&
-	    !((kind == IR::ResourceKind::Buffer && inst.src_count == buffer_address_count + 1u) ||
-	      ((kind == IR::ResourceKind::Lds || kind == IR::ResourceKind::Gds) &&
-	       inst.memory.resource == 0 && inst.src_count == 2 &&
-	       (inst.dst.kind == IR::OperandKind::Null ||
-	        inst.dst.kind == IR::OperandKind::Register)) ||
-	      (kind == IR::ResourceKind::StorageImageUint && inst.src_count == 2))) {
-		return Fail(error, "atomic instruction has an invalid resource/operand contract");
-	}
-	return true;
-}
-
-bool ValidateNativeProgram(const IR::Program& program, std::string* error) {
-	using Kind                                             = IR::DescriptorBindingKind;
-	constexpr auto                               KindCount = static_cast<size_t>(Kind::Count);
-	std::array<std::vector<uint32_t>, KindCount> expected;
-	std::array<bool, KindCount>                  present {};
-	const auto                                   Dense = [](size_t size) {
-		std::vector<uint32_t> values(size);
-		for (uint32_t i = 0; i < values.size(); i++) {
-			values[i] = i;
-		}
-		return values;
-	};
-	auto Expect = [&](Kind kind, std::vector<uint32_t> resources = {}) {
-		const auto index = static_cast<size_t>(kind);
-		present[index]   = true;
-		expected[index]  = std::move(resources);
-	};
-	if (!program.info.buffers.empty()) {
-		Expect(Kind::Buffers, Dense(program.info.buffers.size()));
-	}
-	for (uint32_t i = 0; i < program.info.images.size(); i++) {
-		Kind kind;
-		if (!ImageBinding(program.info.images[i], kind)) {
-			return Fail(error, "native shader plan has an invalid image class");
-		}
-		present[static_cast<size_t>(kind)] = true;
-		expected[static_cast<size_t>(kind)].push_back(i);
-	}
-	if (!program.info.samplers.empty()) {
-		Expect(Kind::Samplers, Dense(program.info.samplers.size()));
-	}
-	const auto uses_gds =
-	    std::any_of(program.blocks.begin(), program.blocks.end(), [](const auto& block) {
-		    return std::any_of(
-		        block.instructions.begin(), block.instructions.end(),
-		        [](const auto& inst) { return inst.memory.kind == IR::ResourceKind::Gds; });
-	    });
-	if (uses_gds) {
-		Expect(Kind::Gds);
-	}
-	if (!program.info.addresses.empty()) {
-		Expect(Kind::AddressMemory, Dense(program.info.addresses.size()));
-	}
-	if (!program.srt.reads.empty()) {
-		Expect(Kind::FlattenedSrt);
-	}
-	if (program.bindings.ShaderDataDwords() != 0 && program.bindings.push_constant_size == 0) {
-		Expect(Kind::UserData);
-	}
-
-	std::array<bool, KindCount> seen {};
-	for (uint32_t i = 0; i < program.bindings.descriptors.size(); i++) {
-		const auto& binding = program.bindings.descriptors[i];
-		const auto  kind    = static_cast<size_t>(binding.kind);
-		if (kind >= KindCount || seen[kind] || !present[kind] ||
-		    binding.resources != expected[kind]) {
-			return Fail(error, "native descriptor groups do not match shader topology");
-		}
-		for (uint32_t j = 0; j < i; j++) {
-			if (program.bindings.descriptors[j].binding == binding.binding) {
-				return Fail(error, "native descriptor binding numbers are not unique");
-			}
-		}
-		seen[kind] = true;
-	}
-	for (size_t i = 0; i < KindCount; i++) {
-		if (present[i] != seen[i]) {
-			return Fail(error, "native shader plan is missing a required descriptor group");
-		}
-	}
-	const auto has_user_storage = present[static_cast<size_t>(Kind::UserData)];
-	if (program.bindings.buffer_offset_dword != program.bindings.user_data_registers.size() ||
-	    program.bindings.buffer_offset_count != program.info.buffers.size() ||
-	    (!has_user_storage &&
-	     program.bindings.push_constant_size != program.bindings.ShaderDataDwords() * 4u) ||
-	    (has_user_storage && program.bindings.push_constant_size != 0) ||
-	    !std::is_sorted(program.bindings.user_data_registers.begin(),
-	                    program.bindings.user_data_registers.end()) ||
-	    std::adjacent_find(program.bindings.user_data_registers.begin(),
-	                       program.bindings.user_data_registers.end()) !=
-	        program.bindings.user_data_registers.end()) {
-		return Fail(error, "native user-data layout is inconsistent");
-	}
-
-	for (const auto& block: program.blocks) {
-		for (const auto& inst: block.instructions) {
-			if (!ValidateInstructionContract(inst, error)) {
-				return false;
-			}
-			const auto memory = inst.memory.kind;
-			if ((memory == IR::ResourceKind::Buffer || (memory == IR::ResourceKind::ScalarBuffer &&
-			                                            inst.op == IR::Opcode::SBufferLoadDword)) &&
-			    inst.memory.resource >= program.info.buffers.size()) {
-				return Fail(error, "buffer instruction has an invalid dense resource");
-			}
-			if ((memory == IR::ResourceKind::Image || memory == IR::ResourceKind::ImageUint ||
-			     memory == IR::ResourceKind::StorageImage ||
-			     memory == IR::ResourceKind::StorageImageUint) &&
-			    (inst.memory.resource >= program.info.images.size() ||
-			     program.info.images[inst.memory.resource].kind != memory ||
-			     program.info.images[inst.memory.resource].dimension !=
-			         inst.memory.image_dimension)) {
-				return Fail(error, "image instruction has an invalid dense resource");
-			}
-			const bool address =
-			    inst.op == IR::Opcode::SLoadDword || memory == IR::ResourceKind::Flat ||
-			    memory == IR::ResourceKind::Global || memory == IR::ResourceKind::Scratch;
-			if (address && (inst.memory.resource >= program.info.addresses.size() ||
-			                program.info.addresses[inst.memory.resource].kind != memory)) {
-				return Fail(error, "address instruction has an invalid dense resource");
-			}
-			if (NeedsSampler(inst.op) && inst.memory.sampler >= program.info.samplers.size()) {
-				return Fail(error, "sample instruction has an invalid dense sampler");
-			}
-			if (inst.op == IR::Opcode::LoadSrtDword &&
-			    (inst.src_count != 1 || inst.src[0].kind != IR::OperandKind::ImmediateU32 ||
-			     inst.src[0].imm >= program.srt.reads.size())) {
-				return Fail(error, "flattened SRT instruction has an invalid dense slot");
-			}
-		}
-	}
-	return true;
-}
-
-} // namespace
-
-static constexpr size_t InitialEmitterVectorReserve = 4096;
-
-static void ReportReserveExceeded(const IR::Program& program, const char* vector_name,
-                                  size_t size) {
-	if (size <= InitialEmitterVectorReserve) {
-		return;
-	}
-	std::printf("SPIR-V emitter reserve exceeded: hash=0x%016" PRIx64
-	            " stage=%u vector=%s size=%zu reserve=%zu\n",
-	            program.shader_hash, static_cast<unsigned>(program.stage), vector_name, size,
-	            InitialEmitterVectorReserve);
-}
-
-static bool IsExecDestination(const IR::Operand& operand) {
-	return operand.kind == IR::OperandKind::Register && operand.reg.file == IR::RegisterFile::Exec;
-}
-
-static bool IsUniformZeroExecWrite(const IR::Program& program, const IR::Instruction& inst) {
-	if (inst.op != IR::Opcode::BitFieldMaskU64 || !IsExecDestination(inst.dst) ||
-	    inst.src_count < 1u) {
-		return false;
-	}
-	uint32_t count = 0;
-	return IR::FoldScalarConstant(program.provenance, inst.scalar_sources[0], count) &&
-	       (count & 63u) == 0u;
-}
-
-static bool ProducesLaneMask(const IR::Instruction& inst) {
-	switch (inst.op) {
-		case IR::Opcode::IAddCarryU32:
-		case IR::Opcode::ISubBorrowU32:
-		case IR::Opcode::UMadU64U32: return inst.dst2.kind != IR::OperandKind::Null;
-		default: return false;
-	}
-}
-
-bool ProgramRequiresExactSubgroupSize(const IR::Program& program) {
-	for (const auto& block: program.blocks) {
-		if (block.terminator.kind == CFG::TerminatorKind::ConditionalBranch) {
-			switch (block.terminator.condition) {
-				case CFG::BranchCondition::VccZero:
-				case CFG::BranchCondition::VccNonZero:
-				case CFG::BranchCondition::ExecZero:
-				case CFG::BranchCondition::ExecNonZero: return true;
-				default: break;
-			}
-		}
-		for (const auto& inst: block.instructions) {
-			if (Emitter::InstructionHasDppSource(inst)) {
-				return true;
-			}
-			if (IR::IsCompareOpcode(inst.op) && !Emitter::IsSccOperand(inst.dst)) {
-				return true;
-			}
-			if ((IsExecDestination(inst.dst) || IsExecDestination(inst.dst2)) &&
-			    !IsUniformZeroExecWrite(program, inst)) {
-				return true;
-			}
-			if (ProducesLaneMask(inst)) {
-				return true;
-			}
-			if (inst.op == IR::Opcode::SelectMaskU32 || inst.op == IR::Opcode::SelectMaskF32Bits) {
-				return true;
-			}
-			if (inst.op == IR::Opcode::SaveexecB32 || inst.op == IR::Opcode::SaveexecB64) {
-				return true;
-			}
-			switch (inst.op) {
-				case IR::Opcode::WqmB64:
-					if (inst.dst.kind == IR::OperandKind::Register &&
-					    inst.dst.reg.file != IR::RegisterFile::Scalar) {
-						return true;
-					}
-					break;
-				case IR::Opcode::MaskedBitCountLowU32:
-				case IR::Opcode::MaskedBitCountHighU32:
-				case IR::Opcode::ReadFirstLaneU32:
-				case IR::Opcode::ReadLaneU32:
-				case IR::Opcode::WriteLaneU32:
-				case IR::Opcode::Permlane16B32:
-				case IR::Opcode::Permlanex16B32:
-				case IR::Opcode::DsSwizzleB32:
-				case IR::Opcode::DsConsume:
-				case IR::Opcode::DsAppend: return true;
-				default: break;
-			}
-		}
-	}
-	return false;
-}
-
-bool EmitProgram(const IR::Program& program, const IR::ResourceSnapshot& resources,
-                 const ShaderVertexInputInfo*  vertex_input_info,
-                 const ShaderPixelInputInfo*   pixel_input_info,
-                 const ShaderComputeInputInfo* compute_input_info, std::vector<uint32_t>& spirv,
-                 std::string* error) {
-	using namespace Emitter;
-
-	if (program.stage != ShaderType::Compute && program.stage != ShaderType::Vertex &&
-	    program.stage != ShaderType::Pixel) {
-		SetError(error, "binary SPIR-V emitter supports compute, vertex, and pixel shaders");
-		return false;
-	}
-	if (!program.srt_patching_complete || !program.resource_tracking_complete ||
-	    !program.shader_info_complete || !program.binding_layout_complete) {
-		SetError(error, "SPIR-V emitter requires a fully planned native shader program");
-		return false;
-	}
-	if (!IR::ValidateResourceSnapshot(program, resources, error)) {
-		return false;
-	}
-	if (!IR::ValidateResourceSpecialization(program, resources, error)) {
-		return false;
-	}
-	if (!ValidateNativeProgram(program, error)) {
-		return false;
-	}
-
-	EmitterState state(program, resources);
-	state.vertex_input_info    = vertex_input_info;
-	state.pixel_input_info     = pixel_input_info;
-	state.compute_input_info   = compute_input_info;
-	state.stage                = program.stage;
-	state.wave_size            = program.wave_size;
-	state.per_invocation_masks = program.lane_mask_mode == ShaderLaneMaskMode::PerInvocation;
-	state.exact_subgroup_operations =
-	    !state.per_invocation_masks && ProgramRequiresExactSubgroupSize(program);
-	state.registers.reserve(InitialEmitterVectorReserve);
-	state.inputs.reserve(InitialEmitterVectorReserve);
-	state.outputs.reserve(InitialEmitterVectorReserve);
-	state.interface_variables.reserve(InitialEmitterVectorReserve);
-	state.reachable_blocks.reserve(InitialEmitterVectorReserve);
-	CollectRegisters(program, state.registers);
-	CopyProgramInputsAndOutputs(state, program);
-	state.needs_subgroup_ballot              = ProgramNeedsSubgroupBallot(program);
-	state.needs_subgroup_shuffle             = ProgramNeedsSubgroupShuffle(program);
-	state.needs_subgroup_local_invocation_id = ProgramNeedsSubgroupLocalInvocationId(program);
-	state.needs_compute_derivatives          = ProgramNeedsComputeDerivatives(program);
-	state.needs_image_gather_extended        = ProgramNeedsImageGatherExtended(program);
-	state.needs_function_lds                 = ProgramNeedsFunctionLds(program);
-	state.needs_pixel_valid_mask             = ProgramNeedsPixelValidMask(program);
-	ComputeReachableBlocks(state, program);
-	AllocateInputVariables(state);
-	AllocateOutputVariables(state);
-	AllocateDescriptorVariables(state);
-	EmitHeaderAndTypes(state);
-	AllocateRegisterVariables(state);
-	AllocateBlockLabels(state, program);
-	AllocateDispatcherState(state, program);
-	EmitFunction(state, program);
-	if (state.unsupported_ir_instruction) {
-		char pc[9] {};
-		std::snprintf(pc, sizeof(pc), "%08x", state.unsupported_ir_pc);
-		return Fail(error, "SPIR-V emitter has no handler for IR opcode " +
-		                       std::string(IR::OpcodeName(state.unsupported_ir_opcode)) +
-		                       " at pc 0x" + pc);
-	}
-
-	ReportReserveExceeded(program, "registers", state.registers.size());
-	ReportReserveExceeded(program, "inputs", state.inputs.size());
-	ReportReserveExceeded(program, "outputs", state.outputs.size());
-	ReportReserveExceeded(program, "interface_variables", state.interface_variables.size());
-	ReportReserveExceeded(program, "reachable_blocks", state.reachable_blocks.size());
-
-	auto binary = state.builder.Build();
-	if (binary.empty()) {
-		SetError(error, "SPIR-V builder returned an empty module");
-		return false;
-	}
-	spirv = std::move(binary);
-	return true;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv

src/graphics/shader/recompiler/emitter/spirvEmitterExport.cpp

@@ -1,227 +0,0 @@
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-
-uint32_t EmitExportComponentF32(EmitterState& state, const IR::Instruction& inst,
-                                uint32_t component) {
-	const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
-	if (!enabled || component >= inst.src_count || component >= 4u) {
-		return ConstantF32(state, component == 3u ? 0x3f800000u : 0u);
-	}
-	return EmitFloatLoad(state, inst.src[component]);
-}
-
-uint32_t EmitExportVec4F32(EmitterState& state, const IR::Instruction& inst) {
-	if (inst.export_info.compr) {
-		uint32_t components[4] = {
-		    ConstantF32(state, 0u),
-		    ConstantF32(state, 0u),
-		    ConstantF32(state, 0u),
-		    ConstantF32(state, 0x3f800000u),
-		};
-		for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
-		     pair_index++) {
-			const auto raw      = EmitValueLoad(state, inst.src[pair_index]);
-			const auto unpacked = state.builder.AllocateId();
-			state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked,
-			                           state.glsl_std450, GlslUnpackHalf2x16, raw});
-			for (uint32_t lane = 0; lane < 2u; lane++) {
-				const auto component = pair_index * 2u + lane;
-				if (((inst.export_info.en >> component) & 1u) == 0) {
-					continue;
-				}
-				components[component] = state.builder.AllocateId();
-				state.builder.AddFunction(
-				    {OpCompositeExtract, state.float_type, components[component], unpacked, lane});
-			}
-		}
-		const auto vec = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, components[0],
-		                           components[1], components[2], components[3]});
-		return vec;
-	}
-
-	const auto x   = EmitExportComponentF32(state, inst, 0);
-	const auto y   = EmitExportComponentF32(state, inst, 1);
-	const auto z   = EmitExportComponentF32(state, inst, 2);
-	const auto w   = EmitExportComponentF32(state, inst, 3);
-	const auto vec = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, vec, x, y, z, w});
-	return vec;
-}
-
-uint32_t EmitExportComponentU32(EmitterState& state, const IR::Instruction& inst,
-                                uint32_t component) {
-	const bool enabled = ((inst.export_info.en >> component) & 1u) != 0;
-	if (!enabled || component >= inst.src_count || component >= 4u) {
-		return ConstantU32(state, component == 3u ? 1u : 0u);
-	}
-	return EmitValueLoad(state, inst.src[component]);
-}
-
-uint32_t EmitExportVec4U32(EmitterState& state, const IR::Instruction& inst) {
-	uint32_t components[4] = {
-	    ConstantU32(state, 0u),
-	    ConstantU32(state, 0u),
-	    ConstantU32(state, 0u),
-	    ConstantU32(state, 1u),
-	};
-
-	if (inst.export_info.compr) {
-		for (uint32_t pair_index = 0; pair_index < 2u && pair_index < inst.src_count;
-		     pair_index++) {
-			const auto raw = EmitValueLoad(state, inst.src[pair_index]);
-			for (uint32_t lane = 0; lane < 2u; lane++) {
-				const auto component = pair_index * 2u + lane;
-				if (((inst.export_info.en >> component) & 1u) == 0) {
-					continue;
-				}
-				components[component] = state.builder.AllocateId();
-				state.builder.AddFunction(
-				    {OpBitFieldUExtract, state.uint_type, components[component], raw,
-				     ConstantU32(state, lane * 16u), ConstantU32(state, 16u)});
-			}
-		}
-	} else {
-		for (uint32_t component = 0; component < 4u; component++) {
-			components[component] = EmitExportComponentU32(state, inst, component);
-		}
-	}
-
-	const auto vec = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, vec, components[0],
-	                           components[1], components[2], components[3]});
-	return vec;
-}
-
-static bool MrtUsesUintOutput(const EmitterState& state, const IR::Instruction& inst) {
-	return inst.export_info.kind == IR::ExportTargetKind::Mrt &&
-	       state.pixel_input_info != nullptr &&
-	       inst.export_info.index < std::size(state.pixel_input_info->target_output_mode) &&
-	       state.pixel_input_info->target_output_mode[inst.export_info.index] == 7u;
-}
-
-uint32_t ApplyMrtExportMapping(EmitterState& state, const IR::Instruction& inst, uint32_t value,
-                               uint32_t vector_type) {
-	if (inst.export_info.kind != IR::ExportTargetKind::Mrt || state.pixel_input_info == nullptr ||
-	    inst.export_info.index >= state.pixel_input_info->target_export_mapping.size()) {
-		return value;
-	}
-
-	const auto mapping = state.pixel_input_info->target_export_mapping[inst.export_info.index];
-	if (mapping.IsIdentity()) {
-		return value;
-	}
-
-	const auto mapped = state.builder.AllocateId();
-	state.builder.AddFunction({OpVectorShuffle, vector_type, mapped, value, value, mapping.Map(0),
-	                           mapping.Map(1), mapping.Map(2), mapping.Map(3)});
-	return mapped;
-}
-
-bool ExportWritesData(const IR::Instruction& inst) {
-	switch (inst.export_info.kind) {
-		case IR::ExportTargetKind::Null:
-		case IR::ExportTargetKind::Primitive: return false;
-		case IR::ExportTargetKind::MrtZ: return (inst.export_info.en & 0x5u) != 0;
-		default: return inst.export_info.en != 0;
-	}
-}
-
-void EmitMrtZExport(EmitterState& state, const IR::Instruction& inst) {
-	if ((inst.export_info.en & 0x1u) != 0 && state.depth_variable != 0) {
-		state.builder.AddFunction(
-		    {OpStore, state.depth_variable, EmitExportComponentF32(state, inst, 0)});
-	}
-
-	if ((inst.export_info.en & 0x4u) != 0 && state.sample_mask_variable != 0) {
-		const auto raw =
-		    inst.src_count > 2u ? EmitValueLoad(state, inst.src[2]) : ConstantU32(state, 0);
-		const auto mask = state.builder.AllocateId();
-		const auto ptr  = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, state.int_type, mask, raw});
-		state.builder.AddFunction({OpAccessChain, state.ptr_output_int, ptr,
-		                           state.sample_mask_variable, ConstantU32(state, 0)});
-		state.builder.AddFunction({OpStore, ptr, mask});
-	}
-}
-
-void EmitExport(EmitterState& state, const IR::Instruction& inst) {
-	if (inst.export_info.kind == IR::ExportTargetKind::Null ||
-	    inst.export_info.kind == IR::ExportTargetKind::Primitive) {
-		return;
-	}
-
-	if (!ExportWritesData(inst)) {
-		return;
-	}
-
-	if (inst.export_info.kind == IR::ExportTargetKind::MrtZ) {
-		EmitMrtZExport(state, inst);
-		return;
-	}
-
-	const auto variable = OutputVariableForExport(state, inst.export_info);
-	if (variable == 0) {
-		return;
-	}
-
-	const auto uint_output = MrtUsesUintOutput(state, inst);
-	const auto vector_type = uint_output ? state.vec4_uint_type : state.vec4_float_type;
-	const auto value       = ApplyMrtExportMapping(
-	    state, inst, uint_output ? EmitExportVec4U32(state, inst) : EmitExportVec4F32(state, inst),
-	    vector_type);
-	if (inst.export_info.kind == IR::ExportTargetKind::Position) {
-		const auto pointer = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpAccessChain, state.ptr_output_vec4_float, pointer, variable, ConstantU32(state, 0)});
-		state.builder.AddFunction({OpStore, pointer, value});
-		return;
-	}
-
-	state.builder.AddFunction({OpStore, variable, value});
-}
-
-bool ExportUsesPixelValidMask(const EmitterState& state, const IR::Instruction& inst) {
-	return state.stage == ShaderType::Pixel && inst.export_info.vm && state.needs_pixel_valid_mask;
-}
-
-void EmitKillIfBoolFalse(EmitterState& state, uint32_t active) {
-	const auto kill_label  = state.builder.AllocateId();
-	const auto merge_label = state.builder.AllocateId();
-	const auto inactive    = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
-	state.builder.AddFunction({OpSelectionMerge, merge_label, SelectionControlNone});
-	state.builder.AddFunction({OpBranchConditional, inactive, kill_label, merge_label});
-	state.builder.AddFunction({OpLabel, kill_label});
-	state.builder.AddFunction({OpKill});
-	state.builder.AddFunction({OpLabel, merge_label});
-}
-
-void EmitUpdatePixelValidMask(EmitterState& state) {
-	if (state.pixel_valid_mask_variable == 0) {
-		return;
-	}
-
-	const auto active = EmitExecActiveBool(state);
-	const auto value  = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, value, active, ConstantU32(state, 1), ConstantU32(state, 0)});
-	state.builder.AddFunction({OpStore, state.pixel_valid_mask_variable, value});
-}
-
-void EmitKillIfPixelValidMaskInactive(EmitterState& state) {
-	if (state.pixel_valid_mask_variable == 0) {
-		return;
-	}
-
-	const auto mask_value = state.builder.AllocateId();
-	const auto active     = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpLoad, state.uint_type, mask_value, state.pixel_valid_mask_variable});
-	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, active, mask_value, ConstantU32(state, 0)});
-	EmitKillIfBoolFalse(state, active);
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/emitter/spirvEmitterFlowOps.cpp

@@ -1,379 +0,0 @@
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-
-void EmitMoveU32(EmitterState& state, const IR::Instruction& inst) {
-	const auto value = EmitValueLoad(state, inst.src[0]);
-	EmitStoreU32(state, inst.dst, value);
-}
-
-void EmitMoveF32Bits(EmitterState& state, const IR::Instruction& inst) {
-	const auto value = EmitFloatLoad(state, inst.src[0]);
-	const auto bits  = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, value});
-	EmitStoreU32(state, inst.dst, bits);
-}
-
-uint32_t MoveRelRegisterLimit(const EmitterState& state, IR::Register base) {
-	uint32_t limit = base.index + 1u;
-	for (const auto& binding: state.registers) {
-		if (binding.reg.file == IR::RegisterFile::Vector) {
-			limit = std::max(limit, binding.reg.index + 1u);
-		}
-	}
-	return std::min(limit, 256u);
-}
-
-void EmitMoveRelSourceU32(EmitterState& state, const IR::Instruction& inst) {
-	if (inst.src[0].kind != IR::OperandKind::Register ||
-	    inst.src[0].reg.file != IR::RegisterFile::Vector) {
-		EmitMoveU32(state, inst);
-		return;
-	}
-
-	const auto base_reg = inst.src[0].reg;
-	const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
-	auto       selected = EmitRegisterLoad(state, base_reg);
-	const auto limit    = MoveRelRegisterLimit(state, base_reg);
-	for (uint32_t reg = base_reg.index + 1u; reg < limit; reg++) {
-		const auto candidate = EmitRegisterLoad(state, {IR::RegisterFile::Vector, reg});
-		const auto match     = state.builder.AllocateId();
-		const auto next      = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
-		state.builder.AddFunction({OpSelect, state.uint_type, next, match, candidate, selected});
-		selected = next;
-	}
-	EmitStoreU32(state, inst.dst, selected);
-}
-
-void EmitMoveRelDestU32(EmitterState& state, const IR::Instruction& inst) {
-	if (inst.dst.kind != IR::OperandKind::Register ||
-	    inst.dst.reg.file != IR::RegisterFile::Vector) {
-		EmitMoveU32(state, inst);
-		return;
-	}
-
-	const auto base_reg = inst.dst.reg;
-	const auto value    = EmitValueLoad(state, inst.src[0]);
-	const auto m0 = inst.src_count > 1u ? EmitValueLoad(state, inst.src[1]) : ConstantU32(state, 0);
-	const auto exec_active = EmitExecActiveBool(state);
-	const auto limit       = MoveRelRegisterLimit(state, base_reg);
-	for (uint32_t reg = base_reg.index; reg < limit; reg++) {
-		const auto pointer = PointerForRegister(state, {IR::RegisterFile::Vector, reg});
-		if (pointer == 0) {
-			continue;
-		}
-		const auto old_value = state.builder.AllocateId();
-		const auto match     = state.builder.AllocateId();
-		const auto write     = state.builder.AllocateId();
-		const auto merged    = state.builder.AllocateId();
-		state.builder.AddFunction({OpLoad, state.uint_type, old_value, pointer});
-		state.builder.AddFunction(
-		    {OpIEqual, state.bool_type, match, m0, ConstantU32(state, reg - base_reg.index)});
-		state.builder.AddFunction({OpLogicalAnd, state.bool_type, write, exec_active, match});
-		state.builder.AddFunction({OpSelect, state.uint_type, merged, write, value, old_value});
-		state.builder.AddFunction({OpStore, pointer, merged});
-	}
-}
-
-void EmitMoveU64(EmitterState& state, const IR::Instruction& inst) {
-	if (state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
-	    IsMaskRegisterFile(inst.dst.reg.file)) {
-		EmitPerInvocationMask(state, inst.dst, EmitLaneMaskOperandActiveBool(state, inst.src[0]));
-		return;
-	}
-	const auto low  = EmitSequentialValueLoad(state, inst.src[0], 0);
-	const auto high = EmitSequentialValueLoad(state, inst.src[0], 1);
-	EmitStoreU32(state, inst.dst, low);
-	EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), high);
-}
-
-uint32_t EmitWqmLaneU32(EmitterState& state, uint32_t src) {
-	uint32_t ret = ConstantU32(state, 0);
-	for (uint32_t i = 0; i < 8u; i++) {
-		const auto mask     = 0x0fu << (i * 4u);
-		const auto masked   = state.builder.AllocateId();
-		const auto non_zero = state.builder.AllocateId();
-		const auto expanded = state.builder.AllocateId();
-		const auto combined = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, masked, src, ConstantU32(state, mask)});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, non_zero, masked, ConstantU32(state, 0)});
-		state.builder.AddFunction({OpSelect, state.uint_type, expanded, non_zero,
-		                           ConstantU32(state, mask), ConstantU32(state, 0)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, ret, expanded});
-		ret = combined;
-	}
-	return ret;
-}
-
-void EmitWqmB64(EmitterState& state, const IR::Instruction& inst) {
-	if (!state.per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
-	    inst.dst.reg.file == IR::RegisterFile::Scalar) {
-		const auto low      = EmitSequentialValueLoad(state, inst.src[0], 0);
-		const auto high     = EmitSequentialValueLoad(state, inst.src[0], 1);
-		const auto ret_low  = EmitWqmLaneU32(state, low);
-		const auto ret_high = EmitWqmLaneU32(state, high);
-		EmitStoreU32(state, inst.dst, ret_low);
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ret_high);
-		return;
-	}
-	const auto ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
-	                           ConstantU32(state, ScopeSubgroup),
-	                           EmitLaneMaskOperandActiveBool(state, inst.src[0])});
-	const auto low  = state.builder.AllocateId();
-	const auto high = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, ballot, 0});
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, ballot, 1});
-	const auto wqm_low   = EmitWqmLaneU32(state, low);
-	const auto wqm_high  = EmitWqmLaneU32(state, high);
-	const auto lane      = EmitSubgroupLocalInvocationId(state);
-	const auto high_half = state.builder.AllocateId();
-	const auto mask      = state.builder.AllocateId();
-	const auto bit_index = state.builder.AllocateId();
-	const auto bit       = state.builder.AllocateId();
-	const auto hit       = state.builder.AllocateId();
-	const auto active    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, high_half, lane, ConstantU32(state, 32)});
-	state.builder.AddFunction({OpSelect, state.uint_type, mask, high_half, wqm_high, wqm_low});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, bit_index, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), bit_index});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
-	if (state.per_invocation_masks) {
-		EmitPerInvocationMask(state, inst.dst, active);
-	} else {
-		const auto result = state.builder.AllocateId();
-		state.builder.AddFunction({OpSelect, state.uint_type, result, active, ConstantU32(state, 1),
-		                           ConstantU32(state, 0)});
-		EmitStoreU32(state, inst.dst, result);
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), ConstantU32(state, 0));
-	}
-}
-
-void EmitSaveexecPerInvocation(EmitterState& state, const IR::Instruction& inst) {
-	const auto old_exec = EmitExecActiveBool(state);
-	const auto src      = EmitLaneMaskOperandActiveBool(state, inst.src[0]);
-	EmitPerInvocationMask(state, inst.dst, old_exec);
-
-	uint32_t lhs    = old_exec;
-	uint32_t rhs    = src;
-	bool     use_or = false;
-	switch (inst.saveexec_mode) {
-		case IR::SaveexecMode::And: break;
-		case IR::SaveexecMode::Andn1: rhs = EmitLogicalNotBool(state, src); break;
-		case IR::SaveexecMode::Orn2:
-			lhs    = EmitLogicalNotBool(state, old_exec);
-			use_or = true;
-			break;
-	}
-	const auto result =
-	    use_or ? EmitLogicalOrBool(state, lhs, rhs) : EmitLogicalAndBool(state, lhs, rhs);
-	EmitPerInvocationMask(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), result);
-	EmitStoreSccBool(state, result);
-}
-
-void EmitSaveexecB32(EmitterState& state, const IR::Instruction& inst) {
-	if (state.per_invocation_masks) {
-		EmitSaveexecPerInvocation(state, inst);
-		return;
-	}
-	const auto old_low = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
-	const auto src_low = EmitValueLoad(state, inst.src[0]);
-
-	EmitStoreU32(state, inst.dst, old_low);
-
-	uint32_t rhs_low = src_low;
-	if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
-		rhs_low = state.builder.AllocateId();
-		state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
-	}
-
-	const auto new_low = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, new_low, rhs_low, old_low});
-	EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
-
-	if (state.wave_size == 32u) {
-		EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), ConstantU32(state, 0));
-	}
-
-	const auto cond = state.builder.AllocateId();
-	const auto scc  = state.builder.AllocateId();
-	state.builder.AddFunction({OpINotEqual, state.bool_type, cond, new_low, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
-	EmitStoreU32(state, SccOperand(), scc);
-}
-
-void EmitSaveexecB64(EmitterState& state, const IR::Instruction& inst) {
-	if (state.per_invocation_masks) {
-		EmitSaveexecPerInvocation(state, inst);
-		return;
-	}
-	const auto old_low  = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
-	const auto old_high = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
-	const auto src_low  = EmitSequentialValueLoad(state, inst.src[0], 0);
-	const auto src_high = EmitSequentialValueLoad(state, inst.src[0], 1);
-
-	EmitStoreU32(state, inst.dst, old_low);
-	EmitStoreU32(state, OffsetRegisterOperand(inst.dst, 1), old_high);
-
-	uint32_t lhs_low  = old_low;
-	uint32_t lhs_high = old_high;
-	uint32_t rhs_low  = src_low;
-	uint32_t rhs_high = src_high;
-	uint32_t opcode   = OpBitwiseAnd;
-	if (inst.saveexec_mode == IR::SaveexecMode::Andn1) {
-		rhs_low  = state.builder.AllocateId();
-		rhs_high = state.builder.AllocateId();
-		state.builder.AddFunction({OpNot, state.uint_type, rhs_low, src_low});
-		state.builder.AddFunction({OpNot, state.uint_type, rhs_high, src_high});
-	}
-	if (inst.saveexec_mode == IR::SaveexecMode::Orn2) {
-		lhs_low  = state.builder.AllocateId();
-		lhs_high = state.builder.AllocateId();
-		state.builder.AddFunction({OpNot, state.uint_type, lhs_low, old_low});
-		state.builder.AddFunction({OpNot, state.uint_type, lhs_high, old_high});
-		opcode = OpBitwiseOr;
-	}
-
-	const auto new_low  = state.builder.AllocateId();
-	const auto new_high = state.builder.AllocateId();
-	state.builder.AddFunction({opcode, state.uint_type, new_low, lhs_low, rhs_low});
-	state.builder.AddFunction({opcode, state.uint_type, new_high, lhs_high, rhs_high});
-	EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 0), new_low);
-	EmitStoreU32(state, MakeRegisterOperand(IR::RegisterFile::Exec, 1), new_high);
-
-	const auto active_new_high = state.wave_size == 32u ? ConstantU32(state, 0) : new_high;
-	const auto mask            = state.builder.AllocateId();
-	const auto cond            = state.builder.AllocateId();
-	const auto scc             = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, mask, new_low, active_new_high});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, cond, mask, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, scc, cond, ConstantU32(state, 1), ConstantU32(state, 0)});
-	EmitStoreU32(state, SccOperand(), scc);
-}
-
-void EmitReadFirstLaneU32(EmitterState& state, const IR::Instruction& inst) {
-	const auto src         = EmitValueLoad(state, inst.src[0]);
-	const auto active      = EmitExecActiveBool(state);
-	const auto ballot      = state.builder.AllocateId();
-	const auto first_lane  = state.builder.AllocateId();
-	const auto first_value = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
-	                           ConstantU32(state, ScopeSubgroup), active});
-	state.builder.AddFunction({OpGroupNonUniformBallotFindLSB, state.uint_type, first_lane,
-	                           ConstantU32(state, ScopeSubgroup), ballot});
-	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, first_value,
-	                           ConstantU32(state, ScopeSubgroup), src, first_lane});
-	EmitStoreU32(state, inst.dst, first_value);
-}
-
-uint32_t EmitLaneIndex(EmitterState& state, const IR::Operand& operand) {
-	const auto lane = state.builder.AllocateId();
-	const auto mask = state.wave_size == 32u ? 31u : 63u;
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, EmitValueLoad(state, operand),
-	                           ConstantU32(state, mask)});
-	return lane;
-}
-
-void EmitReadLaneU32(EmitterState& state, const IR::Instruction& inst) {
-	const auto src   = EmitValueLoad(state, inst.src[0]);
-	const auto lane  = EmitLaneIndex(state, inst.src[1]);
-	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, value,
-	                           ConstantU32(state, ScopeSubgroup), src, lane});
-	EmitStoreU32(state, inst.dst, value);
-}
-
-void EmitWriteLaneU32(EmitterState& state, const IR::Instruction& inst) {
-	const auto value = EmitValueLoad(state, inst.src[0]);
-	const auto lane  = EmitLaneIndex(state, inst.src[1]);
-	const auto subid = EmitSubgroupLocalInvocationId(state);
-	const auto hit   = state.builder.AllocateId();
-	const auto pointer =
-	    inst.dst.kind == IR::OperandKind::Register ? PointerForRegister(state, inst.dst.reg) : 0;
-	const auto old    = pointer != 0 ? state.builder.AllocateId() : ConstantU32(state, 0);
-	const auto merged = state.builder.AllocateId();
-	if (pointer != 0) {
-		state.builder.AddFunction({OpLoad, state.uint_type, old, pointer});
-	}
-	state.builder.AddFunction({OpIEqual, state.bool_type, hit, subid, lane});
-	state.builder.AddFunction({OpSelect, state.uint_type, merged, hit, value, old});
-	if (pointer != 0) {
-		// RDNA2 V_WRITELANE_B32 ignores EXEC, unlike ordinary VGPR destination writes.
-		state.builder.AddFunction({OpStore, pointer, merged});
-	}
-}
-
-void EmitPermlaneB32(EmitterState& state, const IR::Instruction& inst, bool x16) {
-	const auto subid     = EmitSubgroupLocalInvocationId(state);
-	const auto row       = state.builder.AllocateId();
-	const auto lane      = state.builder.AllocateId();
-	const auto lane8     = state.builder.AllocateId();
-	const auto shift     = state.builder.AllocateId();
-	const auto upper     = state.builder.AllocateId();
-	const auto selected  = state.builder.AllocateId();
-	const auto index0    = state.builder.AllocateId();
-	const auto index1    = state.builder.AllocateId();
-	const auto target    = state.builder.AllocateId();
-	const auto shuffled  = state.builder.AllocateId();
-	const auto value     = EmitValueLoad(state, inst.src[0]);
-	const auto src1      = EmitValueLoad(state, inst.src[1]);
-	const auto src2      = EmitValueLoad(state, inst.src[2]);
-	uint32_t   row_value = row;
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	if (x16) {
-		row_value = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseXor, state.uint_type, row_value, row, ConstantU32(state, 16)});
-	}
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane8, lane, ConstantU32(state, 7)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, shift, lane8, ConstantU32(state, 2)});
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, upper, lane, ConstantU32(state, 8)});
-	state.builder.AddFunction({OpSelect, state.uint_type, selected, upper, src2, src1});
-	state.builder.AddFunction({OpShiftRightLogical, state.uint_type, index0, selected, shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, index1, index0, ConstantU32(state, 15)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row_value, index1});
-	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
-	                           ConstantU32(state, ScopeSubgroup), value, target});
-	uint32_t ret = shuffled;
-	if (!inst.dst.op_sel) {
-		const auto source_active = EmitLaneIndexActiveBool(state, target);
-		ret                      = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, ret, source_active, shuffled, ConstantU32(state, 0)});
-	}
-	EmitStoreU32(state, inst.dst, ret);
-}
-
-void EmitControlNop(EmitterState& state, const IR::Instruction& inst) {
-	(void)state;
-	(void)inst;
-}
-
-void EmitWaitcnt(EmitterState& state, const IR::Instruction& inst) {
-	(void)state;
-	(void)inst;
-}
-
-void EmitBarrier(EmitterState& state, const IR::Instruction& inst) {
-	(void)inst;
-	const auto semantics = MemorySemanticsAcquireRelease | MemorySemanticsWorkgroupMemory;
-	state.builder.AddFunction({OpControlBarrier, ConstantU32(state, ScopeWorkgroup),
-	                           ConstantU32(state, ScopeWorkgroup), ConstantU32(state, semantics)});
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/emitter/spirvEmitterImageHelpers.cpp

@@ -1,333 +0,0 @@
-#include "common/assert.h"
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-namespace {
-
-uint32_t EmitCubeAxisF32(EmitterState& state, uint32_t value) {
-	const auto normalized = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpFSub, state.float_type, normalized, value, ConstantF32(state, 0x3f800000u)});
-	return normalized;
-}
-
-uint32_t EmitCubeLayerF32(EmitterState& state, uint32_t face_id) {
-	// Sampled RDNA2 cubemaps encode face_id as slice * 8 + face. The native
-	// 2D-array view stores six contiguous faces per slice, so remove the two
-	// reserved face IDs from every preceding slice.
-	const auto guest_layer = state.builder.AllocateId();
-	const auto slice       = state.builder.AllocateId();
-	const auto padding     = state.builder.AllocateId();
-	const auto host_layer  = state.builder.AllocateId();
-	const auto result      = state.builder.AllocateId();
-	state.builder.AddFunction({OpConvertFToU, state.uint_type, guest_layer, face_id});
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, slice, guest_layer, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, padding, slice, ConstantU32(state, 1)});
-	state.builder.AddFunction({OpISub, state.uint_type, host_layer, guest_layer, padding});
-	state.builder.AddFunction({OpConvertUToF, state.float_type, result, host_layer});
-	return result;
-}
-
-uint32_t EmitImageCoordF32Impl(EmitterState& state, const IR::Instruction& inst,
-                               const IR::Operand& address, uint32_t first_component,
-                               uint32_t components) {
-	auto x = EmitImageAddressFloatLoad(state, inst, address, first_component);
-	if (components == 1u) {
-		return x;
-	}
-	auto y = inst.memory.image_address_components > first_component + 1u
-	             ? EmitImageAddressFloatLoad(state, inst, address, first_component + 1u)
-	             : EmitZeroF32(state);
-	if (inst.memory.image_cube) {
-		// RDNA2 sampled cubemap S/T coordinates are biased by +1 relative to
-		// normalized 2D-array coordinates.
-		x = EmitCubeAxisF32(state, x);
-		y = EmitCubeAxisF32(state, y);
-	}
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		auto z = inst.memory.image_address_components > first_component + 2u
-		             ? EmitImageAddressFloatLoad(state, inst, address, first_component + 2u)
-		             : EmitZeroF32(state);
-		if (inst.memory.image_cube) {
-			z = EmitCubeLayerF32(state, z);
-		}
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, coord, x, y});
-	}
-	return coord;
-}
-
-} // namespace
-
-bool HasImageSampleFlag(const IR::Instruction& inst, uint32_t flag) {
-	return (inst.memory.image_sample_flags & flag) != 0;
-}
-
-ImageSampleLayout MakeImageSampleLayout(const IR::Instruction& inst, ImageViewKind view) {
-	ImageSampleLayout layout;
-	uint32_t          cursor = 0;
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
-		layout.offset = cursor++;
-	}
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
-		layout.bias = cursor++;
-	}
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare)) {
-		layout.dref = cursor++;
-	}
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
-		const auto components = ImageViewSpatialComponents(view);
-		layout.grad_x         = cursor;
-		cursor += components;
-		layout.grad_y = cursor;
-		cursor += components;
-	}
-	layout.coord = cursor;
-	cursor += ImageViewCoordinateComponents(view);
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod)) {
-		layout.lod = cursor++;
-	}
-	return layout;
-}
-
-uint32_t EmitImageCoordF32(EmitterState& state, const IR::Instruction& inst,
-                           const ImageSampleLayout& layout, ImageViewKind view) {
-	return EmitImageCoordF32Impl(state, inst, inst.src[0], layout.coord,
-	                             ImageViewCoordinateComponents(view));
-}
-
-uint32_t EmitImageLodF32(EmitterState& state, const IR::Instruction& inst,
-                         const ImageSampleLayout& layout) {
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero) ||
-	    layout.lod == NoImageComponent || inst.memory.image_address_components <= layout.lod) {
-		return EmitZeroF32(state);
-	}
-	return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.lod);
-}
-
-uint32_t EmitImageDrefF32(EmitterState& state, const IR::Instruction& inst,
-                          const ImageSampleLayout& layout) {
-	if (layout.dref == NoImageComponent || inst.memory.image_address_components <= layout.dref) {
-		return EmitZeroF32(state);
-	}
-	return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.dref);
-}
-
-uint32_t EmitImageBiasF32(EmitterState& state, const IR::Instruction& inst,
-                          const ImageSampleLayout& layout) {
-	if (layout.bias == NoImageComponent || inst.memory.image_address_components <= layout.bias) {
-		return EmitZeroF32(state);
-	}
-	return EmitImageAddressFloatLoad(state, inst, inst.src[0], layout.bias);
-}
-
-uint32_t EmitImageGradientF32(EmitterState& state, const IR::Instruction& inst,
-                              uint32_t first_component, ImageViewKind view) {
-	const auto x = inst.memory.image_address_components > first_component
-	                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component)
-	                   : EmitZeroF32(state);
-	const auto components = ImageViewSpatialComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y = inst.memory.image_address_components > first_component + 1u
-	                   ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 1u)
-	                   : EmitZeroF32(state);
-	const auto grad = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z =
-		    inst.memory.image_address_components > first_component + 2u
-		        ? EmitImageAddressFloatLoad(state, inst, inst.src[0], first_component + 2u)
-		        : EmitZeroF32(state);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_float_type, grad, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_float_type, grad, x, y});
-	}
-	return grad;
-}
-
-uint32_t EmitImagePackedOffsetI32(EmitterState& state, const IR::Instruction& inst,
-                                  const ImageSampleLayout& layout, ImageViewKind view) {
-	const auto components = ImageViewSpatialComponents(view);
-	if (layout.offset == NoImageComponent ||
-	    inst.memory.image_address_components <= layout.offset) {
-		const auto zero = ConstantI32(state, 0);
-		if (components == 1u) {
-			return zero;
-		}
-		const auto ret = state.builder.AllocateId();
-		if (components == 3u) {
-			state.builder.AddFunction(
-			    {OpCompositeConstruct, state.vec3_int_type, ret, zero, zero, zero});
-		} else {
-			state.builder.AddFunction({OpCompositeConstruct, state.vec2_int_type, ret, zero, zero});
-		}
-		return ret;
-	}
-
-	const auto packed_bits = EmitImageAddressValueLoad(state, inst, inst.src[0], layout.offset);
-	const auto packed_i32  = state.builder.AllocateId();
-	const auto offset_x    = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.int_type, packed_i32, packed_bits});
-	state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_x, packed_i32,
-	                           ConstantI32(state, 0), ConstantI32(state, 6)});
-	if (components == 1u) {
-		return offset_x;
-	}
-	const auto offset_y = state.builder.AllocateId();
-	const auto offset   = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_y, packed_i32,
-	                           ConstantI32(state, 8), ConstantI32(state, 6)});
-	if (components == 3u) {
-		const auto offset_z = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, offset_z, packed_i32,
-		                           ConstantI32(state, 16), ConstantI32(state, 6)});
-		state.builder.AddFunction(
-		    {OpCompositeConstruct, state.vec3_int_type, offset, offset_x, offset_y, offset_z});
-	} else {
-		state.builder.AddFunction(
-		    {OpCompositeConstruct, state.vec2_int_type, offset, offset_x, offset_y});
-	}
-	return offset;
-}
-
-uint32_t EmitImageCoordU32(EmitterState& state, const IR::Instruction& inst, ImageViewKind view) {
-	const auto x          = EmitImageAddressValueLoad(state, inst, inst.src[1], 0);
-	const auto components = ImageViewCoordinateComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y     = inst.memory.image_address_components > 1u
-	                       ? EmitImageAddressValueLoad(state, inst, inst.src[1], 1)
-	                       : ConstantU32(state, 0);
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > 2u
-		                   ? EmitImageAddressValueLoad(state, inst, inst.src[1], 2)
-		                   : ConstantU32(state, 0);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
-	}
-	return coord;
-}
-
-uint32_t EmitImageLoadCoordU32(EmitterState& state, const IR::Instruction& inst,
-                               ImageViewKind view) {
-	const auto x          = EmitImageAddressValueLoad(state, inst, inst.src[0], 0);
-	const auto components = ImageViewCoordinateComponents(view);
-	if (components == 1u) {
-		return x;
-	}
-	const auto y     = inst.memory.image_address_components > 1u
-	                       ? EmitImageAddressValueLoad(state, inst, inst.src[0], 1)
-	                       : ConstantU32(state, 0);
-	const auto coord = state.builder.AllocateId();
-	if (components == 3u) {
-		const auto z = inst.memory.image_address_components > 2u
-		                   ? EmitImageAddressValueLoad(state, inst, inst.src[0], 2)
-		                   : ConstantU32(state, 0);
-		state.builder.AddFunction({OpCompositeConstruct, state.vec3_uint_type, coord, x, y, z});
-	} else {
-		state.builder.AddFunction({OpCompositeConstruct, state.vec2_uint_type, coord, x, y});
-	}
-	return coord;
-}
-
-uint32_t EmitImageMipLodU32(EmitterState& state, const IR::Instruction& inst,
-                            const IR::Operand& address, ImageViewKind view) {
-	if (!inst.memory.image_has_mip || inst.memory.image_address_components == 0u) {
-		return ConstantU32(state, 0);
-	}
-	const auto lod_component = ImageViewCoordinateComponents(view);
-	if (inst.memory.image_address_components <= lod_component) {
-		return ConstantU32(state, 0);
-	}
-	return EmitImageAddressValueLoad(state, inst, address, lod_component);
-}
-
-uint32_t EmitImageQueryCoordF32(EmitterState& state, const IR::Instruction& inst,
-                                ImageViewKind view) {
-	// OpImageQueryLod takes only the spatial coordinates, even for arrayed images.
-	return EmitImageCoordF32Impl(state, inst, inst.src[0], 0, ImageViewSpatialComponents(view));
-}
-
-uint32_t DmaskComponentIndex(uint32_t dmask, uint32_t component) {
-	uint32_t index = 0;
-	for (uint32_t i = 0; i < component; i++) {
-		index += (dmask >> i) & 1u;
-	}
-	return index;
-}
-
-uint32_t EmitImageStoreComponentF32(EmitterState& state, const IR::Instruction& inst,
-                                    uint32_t component) {
-	const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
-	if (((dmask >> component) & 1u) == 0) {
-		return EmitZeroF32(state);
-	}
-	return EmitSequentialFloatLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
-}
-
-uint32_t InverseStorageSwizzleComponent(uint32_t swizzle, uint32_t component) {
-	const auto target = 4u + component;
-	for (uint32_t source = 0; source < 4u; source++) {
-		if (((swizzle >> (source * 3u)) & 0x7u) == target) {
-			return source;
-		}
-	}
-	return UINT32_MAX;
-}
-
-uint32_t StorageImageSwizzle(const EmitterState& state, const IR::Instruction& inst) {
-	if (inst.memory.resource >= state.program.info.images.size()) {
-		EXIT("storage image instruction has no specialized swizzle\n");
-	}
-	return state.program.info.images[inst.memory.resource].storage_swizzle;
-}
-
-uint32_t EmitImageStoreTexelF32(EmitterState& state, const IR::Instruction& inst) {
-	const auto swizzle         = StorageImageSwizzle(state, inst);
-	const auto component_value = [&](uint32_t component) {
-		const auto source = InverseStorageSwizzleComponent(swizzle, component);
-		return source < 4u ? EmitImageStoreComponentF32(state, inst, source) : EmitZeroF32(state);
-	};
-	const auto x     = component_value(0);
-	const auto y     = component_value(1);
-	const auto z     = component_value(2);
-	const auto w     = component_value(3);
-	const auto texel = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, state.vec4_float_type, texel, x, y, z, w});
-	return texel;
-}
-
-uint32_t EmitImageStoreComponentU32(EmitterState& state, const IR::Instruction& inst,
-                                    uint32_t component) {
-	const auto dmask = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
-	if (((dmask >> component) & 1u) == 0) {
-		return ConstantU32(state, 0);
-	}
-	return EmitSequentialValueLoad(state, inst.src[0], DmaskComponentIndex(dmask, component));
-}
-
-uint32_t EmitImageStoreTexelU32(EmitterState& state, const IR::Instruction& inst) {
-	const auto swizzle         = StorageImageSwizzle(state, inst);
-	const auto component_value = [&](uint32_t component) {
-		const auto source = InverseStorageSwizzleComponent(swizzle, component);
-		return source < 4u ? EmitImageStoreComponentU32(state, inst, source)
-		                   : ConstantU32(state, 0);
-	};
-	const auto x     = component_value(0);
-	const auto y     = component_value(1);
-	const auto z     = component_value(2);
-	const auto w     = component_value(3);
-	const auto texel = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeConstruct, state.vec4_uint_type, texel, x, y, z, w});
-	return texel;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/emitter/spirvEmitterImageOps.cpp

@@ -1,342 +0,0 @@
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-
-namespace {
-
-uint32_t ConstantVec2I32(EmitterState& state, int32_t x, int32_t y) {
-	const auto value = state.builder.AllocateId();
-	state.builder.AddType({OpConstantComposite, state.vec2_int_type, value, ConstantI32(state, x),
-	                       ConstantI32(state, y)});
-	return value;
-}
-
-uint32_t ConstantImageGatherHorizontalOffsets(EmitterState& state, ImageViewKind view) {
-	if (ImageViewSpatialComponents(view) == 1u) {
-		const auto type = state.builder.AllocateId();
-		state.builder.AddType({OpTypeArray, type, state.int_type, ConstantU32(state, 4)});
-		const auto value = state.builder.AllocateId();
-		state.builder.AddType({OpConstantComposite, type, value, ConstantI32(state, -1),
-		                       ConstantI32(state, 0), ConstantI32(state, 1),
-		                       ConstantI32(state, 2)});
-		return value;
-	}
-	const auto offset0 = ConstantVec2I32(state, -1, 0);
-	const auto offset1 = ConstantVec2I32(state, 0, 0);
-	const auto offset2 = ConstantVec2I32(state, 1, 0);
-	const auto offset3 = ConstantVec2I32(state, 2, 0);
-	const auto type    = state.builder.AllocateId();
-	state.builder.AddType({OpTypeArray, type, state.vec2_int_type, ConstantU32(state, 4)});
-	const auto value = state.builder.AllocateId();
-	state.builder.AddType({OpConstantComposite, type, value, offset0, offset1, offset2, offset3});
-	return value;
-}
-
-uint32_t LoadStorageImageDescriptorAtIndex(EmitterState& state, uint32_t resource,
-                                           uint32_t array_index, bool uint_image,
-                                           ImageViewKind view) {
-	const auto  kind        = StorageBindingKind(uint_image, view);
-	const auto& descriptors = state.storage_images[StorageImageIndex(uint_image, view)];
-	const auto  pointer =
-	    DescriptorElementPointer(state, descriptors.pointer_type, descriptors.variable, array_index,
-	                             kind, resource, "storage image descriptor array was not emitted");
-	const auto image = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, descriptors.image_type, image, pointer});
-	return image;
-}
-
-} // namespace
-
-uint32_t EmitImageGetResinfoComponent(EmitterState& state, uint32_t image, uint32_t size,
-                                      uint32_t component_index, ImageViewKind view) {
-	const auto components = ImageViewCoordinateComponents(view);
-	switch (component_index) {
-		case 0: {
-			if (components == 1u) {
-				return size;
-			}
-			const auto width = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, state.uint_type, width, size, 0});
-			return width;
-		}
-		case 1: {
-			if (components < 2u) {
-				return ConstantU32(state, 0);
-			}
-			const auto height = state.builder.AllocateId();
-			state.builder.AddFunction({OpCompositeExtract, state.uint_type, height, size, 1});
-			return height;
-		}
-		case 2:
-			if (components == 3u) {
-				const auto depth = state.builder.AllocateId();
-				state.builder.AddFunction({OpCompositeExtract, state.uint_type, depth, size, 2});
-				return depth;
-			}
-			return ConstantU32(state, 0);
-		case 3: {
-			const auto levels = state.builder.AllocateId();
-			state.builder.AddFunction({OpImageQueryLevels, state.uint_type, levels, image});
-			return levels;
-		}
-		default: return ConstantU32(state, 0);
-	}
-}
-
-void EmitImageGetResinfo(EmitterState& state, const IR::Instruction& inst) {
-	const auto view      = SampledImageViewKind(state, inst.memory, inst.pc);
-	const auto image     = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
-	const auto mip_level = EmitValueLoad(state, inst.src[0]);
-
-	const auto size = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpImageQuerySizeLod, ImageViewSizeType(state, view), size, image, mip_level});
-
-	const auto dmask     = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
-	uint32_t   dst_index = 0;
-	for (uint32_t component_index = 0; component_index < 4u; component_index++) {
-		if (((dmask >> component_index) & 1u) == 0) {
-			continue;
-		}
-		const auto component =
-		    EmitImageGetResinfoComponent(state, image, size, component_index, view);
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), component);
-	}
-}
-
-void EmitImageGetLod(EmitterState& state, const IR::Instruction& inst) {
-	const auto view          = SampledImageViewKind(state, inst.memory, inst.pc);
-	const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
-
-	const auto lod = state.builder.AllocateId();
-	state.builder.AddFunction({OpImageQueryLod, state.vec2_float_type, lod, sampled_image,
-	                           EmitImageQueryCoordF32(state, inst, view)});
-
-	const auto dmask     = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
-	uint32_t   dst_index = 0;
-	for (uint32_t component_index = 0; component_index < 2u; component_index++) {
-		if (((dmask >> component_index) & 1u) == 0) {
-			continue;
-		}
-		const auto component = state.builder.AllocateId();
-		const auto bits      = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpCompositeExtract, state.float_type, component, lod, component_index});
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
-	}
-}
-
-void EmitImageLoad(EmitterState& state, const IR::Instruction& inst) {
-	const auto view    = SampledImageViewKind(state, inst.memory, inst.pc);
-	const auto image   = LoadSampledImageDescriptor(state, inst.memory, inst.pc, view);
-	const bool integer = inst.memory.kind == IR::ResourceKind::ImageUint;
-
-	const auto color = state.builder.AllocateId();
-	const auto coord = EmitImageLoadCoordU32(state, inst, view);
-	if (ImageSpirvMultisampled(view) != 0) {
-		const auto sample = EmitImageAddressValueLoad(state, inst, inst.src[0],
-		                                              ImageViewCoordinateComponents(view));
-		state.builder.AddFunction({OpImageFetch,
-		                           integer ? state.vec4_uint_type : state.vec4_float_type, color,
-		                           image, coord, ImageOperandsSampleMask, sample});
-	} else {
-		state.builder.AddFunction(
-		    {OpImageFetch, integer ? state.vec4_uint_type : state.vec4_float_type, color, image,
-		     coord, ImageOperandsLodMask, EmitImageMipLodU32(state, inst, inst.src[0], view)});
-	}
-
-	const auto dmask     = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
-	uint32_t   dst_index = 0;
-	for (uint32_t component_index = 0; component_index < 4u; component_index++) {
-		if (((dmask >> component_index) & 1u) == 0) {
-			continue;
-		}
-		const auto component = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
-		                           component, color, component_index});
-		const auto bits = integer ? component : state.builder.AllocateId();
-		if (!integer) {
-			state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
-		}
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
-	}
-}
-
-void EmitImageStore(EmitterState& state, const IR::Instruction& inst) {
-	const auto uint_image = inst.memory.kind == IR::ResourceKind::StorageImageUint;
-	const auto view       = StorageImageViewKind(state, inst.memory, uint_image, inst.pc);
-	const auto binding =
-	    ResourceForDescriptor(state, StorageBindingKind(uint_image, view), inst.memory.resource);
-	const auto image = LoadStorageImageDescriptorAtIndex(state, inst.memory.resource,
-	                                                     binding.array_index, uint_image, view);
-
-	state.builder.AddFunction(
-	    {OpImageWrite, image, EmitImageCoordU32(state, inst, view),
-	     uint_image ? EmitImageStoreTexelU32(state, inst) : EmitImageStoreTexelF32(state, inst)});
-}
-
-void EmitImageSampleResult(EmitterState& state, const IR::Instruction& inst, uint32_t sample,
-                           bool dref, bool integer) {
-	if (dref) {
-		const auto bits = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, sample});
-		for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
-			EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
-		}
-		return;
-	}
-
-	const auto dmask     = inst.memory.dmask != 0 ? inst.memory.dmask : 1u;
-	uint32_t   dst_index = 0;
-	for (uint32_t component_index = 0; component_index < 4u; component_index++) {
-		if (((dmask >> component_index) & 1u) == 0) {
-			continue;
-		}
-		const auto component = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
-		                           component, sample, component_index});
-		const auto bits = integer ? component : state.builder.AllocateId();
-		if (!integer) {
-			state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
-		}
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, dst_index++), bits);
-	}
-}
-
-bool ImageSampleNeedsExplicitLod(const EmitterState& state, const IR::Instruction& inst) {
-	// RDNA2 plain IMAGE_SAMPLE is derivative-based in pixel shaders. Do not translate it
-	// to Lod(0): only _L/_LZ/_D variants supply explicit LOD/gradient information.
-	// SPIR-V implicit-lod sampling is fragment-only, so non-pixel stages still need
-	// an explicit operand even for the unsuffixed sample opcode.
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative) ||
-	    HasImageSampleFlag(inst, Decoder::ImageSampleFlagLod) ||
-	    HasImageSampleFlag(inst, Decoder::ImageSampleFlagLevelZero)) {
-		return true;
-	}
-	return state.stage != ShaderType::Pixel;
-}
-
-void AddImageSampleOperands(EmitterState& state, const IR::Instruction& inst,
-                            const ImageSampleLayout& layout, bool explicit_lod,
-                            std::vector<uint32_t>& words) {
-	uint32_t              mask = 0;
-	std::vector<uint32_t> operands;
-
-	// Keep the SPIR-V operand class aligned with the RDNA2 opcode suffix. Grad/Lod
-	// operands belong to explicit-lod instructions; bias belongs to implicit-lod
-	// sampling because it modifies the hardware-computed LOD.
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagDerivative)) {
-		mask |= ImageOperandsGradMask;
-		const auto view = SampledImageViewKind(state, inst.memory, inst.pc);
-		operands.push_back(EmitImageGradientF32(state, inst, layout.grad_x, view));
-		operands.push_back(EmitImageGradientF32(state, inst, layout.grad_y, view));
-	} else if (explicit_lod) {
-		mask |= ImageOperandsLodMask;
-		operands.push_back(EmitImageLodF32(state, inst, layout));
-	} else if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagBias)) {
-		mask |= ImageOperandsBiasMask;
-		operands.push_back(EmitImageBiasF32(state, inst, layout));
-	}
-	if (mask != 0) {
-		words.push_back(mask);
-		words.insert(words.end(), operands.begin(), operands.end());
-	}
-}
-
-uint32_t ImageSampleOpcode(const EmitterState& state, const IR::Instruction& inst) {
-	const auto flags        = inst.memory.image_sample_flags;
-	const bool dref         = (flags & Decoder::ImageSampleFlagCompare) != 0;
-	const bool explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
-	if (explicit_lod) {
-		return dref ? OpImageSampleDrefExplicitLod : OpImageSampleExplicitLod;
-	}
-	return dref ? OpImageSampleDrefImplicitLod : OpImageSampleImplicitLod;
-}
-
-void EmitImageSample(EmitterState& state, const IR::Instruction& inst) {
-	const auto view          = SampledImageViewKind(state, inst.memory, inst.pc);
-	const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
-
-	const auto layout      = MakeImageSampleLayout(inst, view);
-	const auto base_coord  = EmitImageCoordF32(state, inst, layout, view);
-	const auto sample      = state.builder.AllocateId();
-	const auto dref        = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
-	const bool integer     = inst.memory.kind == IR::ResourceKind::ImageUint;
-	uint32_t   result_type = state.vec4_float_type;
-	if (dref) {
-		result_type = state.float_type;
-	} else if (integer) {
-		result_type = state.vec4_uint_type;
-	}
-	const auto            explicit_lod = ImageSampleNeedsExplicitLod(state, inst);
-	const auto            opcode       = ImageSampleOpcode(state, inst);
-	std::vector<uint32_t> words        = {opcode, result_type, sample, sampled_image, base_coord};
-	if (dref) {
-		words.push_back(EmitImageDrefF32(state, inst, layout));
-	}
-	AddImageSampleOperands(state, inst, layout, explicit_lod, words);
-	state.builder.AddFunction(words);
-	EmitImageSampleResult(state, inst, sample, dref, integer);
-}
-
-uint32_t ImageGatherComponent(uint32_t dmask) {
-	switch (dmask) {
-		case 0x2u: return 1;
-		case 0x4u: return 2;
-		case 0x8u: return 3;
-		default: return 0;
-	}
-}
-
-void AddImageGatherOperands(EmitterState& state, const IR::Instruction& inst,
-                            const ImageSampleLayout& layout, ImageViewKind view,
-                            std::vector<uint32_t>& words) {
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagGatherHorizontal)) {
-		words.push_back(ImageOperandsConstOffsetsMask);
-		words.push_back(ConstantImageGatherHorizontalOffsets(state, view));
-		return;
-	}
-	if (HasImageSampleFlag(inst, Decoder::ImageSampleFlagOffset)) {
-		words.push_back(ImageOperandsOffsetMask);
-		words.push_back(EmitImagePackedOffsetI32(state, inst, layout, view));
-	}
-}
-
-void EmitImageGather4(EmitterState& state, const IR::Instruction& inst) {
-	const auto view          = SampledImageViewKind(state, inst.memory, inst.pc);
-	const auto sampled_image = MakeSampledImage(state, inst.memory, inst.pc, view);
-
-	const auto            layout      = MakeImageSampleLayout(inst, view);
-	const auto            coord       = EmitImageCoordF32(state, inst, layout, view);
-	const auto            texels      = state.builder.AllocateId();
-	const auto            dref        = HasImageSampleFlag(inst, Decoder::ImageSampleFlagCompare);
-	const bool            integer     = inst.memory.kind == IR::ResourceKind::ImageUint;
-	const auto            result_type = integer ? state.vec4_uint_type : state.vec4_float_type;
-	std::vector<uint32_t> words =
-	    dref ? std::vector<uint32_t> {OpImageDrefGather,
-	                                  state.vec4_float_type,
-	                                  texels,
-	                                  sampled_image,
-	                                  coord,
-	                                  EmitImageDrefF32(state, inst, layout)}
-	         : std::vector<uint32_t> {
-	               OpImageGather, result_type,
-	               texels,        sampled_image,
-	               coord,         ConstantU32(state, ImageGatherComponent(inst.memory.dmask))};
-	AddImageGatherOperands(state, inst, layout, view, words);
-	state.builder.AddFunction(words);
-
-	for (uint32_t i = 0; i < inst.memory.data_dwords; i++) {
-		const auto component = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, integer ? state.uint_type : state.float_type,
-		                           component, texels, i});
-		const auto bits = integer ? component : state.builder.AllocateId();
-		if (!integer) {
-			state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
-		}
-		EmitStoreU32(state, OffsetRegisterOperand(inst.dst, i), bits);
-	}
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/emitter/spirvEmitterValues.cpp

@@ -1,838 +0,0 @@
-#include "graphics/shader/recompiler/emitter/spirvEmitterInternal.h"
-
-namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter {
-
-bool SdwaSelectorOffsetWidth(uint32_t sel, uint32_t& offset, uint32_t& width) {
-	if (sel <= 3u) {
-		offset = sel * 8u;
-		width  = 8u;
-		return true;
-	}
-	if (sel == 4u || sel == 5u) {
-		offset = sel == 5u ? 16u : 0u;
-		width  = 16u;
-		return true;
-	}
-	if (sel == 6u) {
-		offset = 0;
-		width  = 32u;
-		return true;
-	}
-	return false;
-}
-
-uint32_t EmitSdwaExtractU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
-	if (operand.sdwa_sel == 6u && !operand.sdwa_sext) {
-		return value;
-	}
-
-	uint32_t offset = 0;
-	uint32_t width  = 0;
-	if (!SdwaSelectorOffsetWidth(operand.sdwa_sel, offset, width) || width == 32u) {
-		return value;
-	}
-
-	if (operand.sdwa_sext) {
-		const auto signed_value = state.builder.AllocateId();
-		const auto extracted    = state.builder.AllocateId();
-		const auto bits         = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitcast, state.int_type, signed_value, value});
-		state.builder.AddFunction({OpBitFieldSExtract, state.int_type, extracted, signed_value,
-		                           ConstantU32(state, offset), ConstantU32(state, width)});
-		state.builder.AddFunction({OpBitcast, state.uint_type, bits, extracted});
-		return bits;
-	}
-
-	const auto extracted = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitFieldUExtract, state.uint_type, extracted, value,
-	                           ConstantU32(state, offset), ConstantU32(state, width)});
-	return extracted;
-}
-
-uint32_t EmitTrueBool(EmitterState& state) {
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, ret, ConstantU32(state, 1), ConstantU32(state, 0)});
-	return ret;
-}
-
-DppTargetLane EmitDppQuadPermTargetLane(EmitterState& state, uint32_t subid, uint32_t control) {
-	const auto quad_base = state.builder.AllocateId();
-	const auto lane      = state.builder.AllocateId();
-	const auto shift     = state.builder.AllocateId();
-	const auto selected0 = state.builder.AllocateId();
-	const auto selected  = state.builder.AllocateId();
-	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, quad_base, subid, ConstantU32(state, 0xfffffffcu)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 3)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, shift, lane, ConstantU32(state, 1)});
-	state.builder.AddFunction(
-	    {OpShiftRightLogical, state.uint_type, selected0, ConstantU32(state, control), shift});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, selected, selected0, ConstantU32(state, 3)});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, quad_base, selected});
-	return {target, EmitTrueBool(state)};
-}
-
-DppTargetLane EmitDppRowShiftTargetLane(EmitterState& state, uint32_t subid, uint32_t amount,
-                                        bool left) {
-	const auto row          = state.builder.AllocateId();
-	const auto lane         = state.builder.AllocateId();
-	const auto lane_shifted = state.builder.AllocateId();
-	const auto target       = state.builder.AllocateId();
-	const auto valid        = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
-	if (left) {
-		state.builder.AddFunction(
-		    {OpIAdd, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
-		state.builder.AddFunction(
-		    {OpULessThan, state.bool_type, valid, lane, ConstantU32(state, 16u - amount)});
-	} else {
-		state.builder.AddFunction(
-		    {OpISub, state.uint_type, lane_shifted, lane, ConstantU32(state, amount)});
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, valid, lane, ConstantU32(state, amount)});
-	}
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, lane_shifted});
-	return {target, valid};
-}
-
-DppTargetLane EmitDppRowRotateRightTargetLane(EmitterState& state, uint32_t subid,
-                                              uint32_t amount) {
-	const auto row      = state.builder.AllocateId();
-	const auto lane     = state.builder.AllocateId();
-	const auto in_high  = state.builder.AllocateId();
-	const auto minus    = state.builder.AllocateId();
-	const auto plus     = state.builder.AllocateId();
-	const auto selected = state.builder.AllocateId();
-	const auto target   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, row, subid, ConstantU32(state, 0xfffffff0u)});
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 15)});
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction({OpISub, state.uint_type, minus, lane, ConstantU32(state, amount)});
-	state.builder.AddFunction(
-	    {OpIAdd, state.uint_type, plus, lane, ConstantU32(state, 16u - amount)});
-	state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, minus, plus});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, row, selected});
-	return {target, EmitTrueBool(state)};
-}
-
-DppTargetLane EmitDppMirrorTargetLane(EmitterState& state, uint32_t subid, bool half_row) {
-	const auto base_mask = half_row ? 0xfffffff8u : 0xfffffff0u;
-	const auto lane_mask = half_row ? 7u : 15u;
-	const auto base      = state.builder.AllocateId();
-	const auto lane      = state.builder.AllocateId();
-	const auto mirrored  = state.builder.AllocateId();
-	const auto target    = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, base, subid, ConstantU32(state, base_mask)});
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, lane_mask)});
-	state.builder.AddFunction(
-	    {OpISub, state.uint_type, mirrored, ConstantU32(state, lane_mask), lane});
-	state.builder.AddFunction({OpBitwiseOr, state.uint_type, target, base, mirrored});
-	return {target, EmitTrueBool(state)};
-}
-
-DppTargetLane EmitDppTargetLane(EmitterState& state, uint32_t control) {
-	const auto subid = EmitSubgroupLocalInvocationId(state);
-	if (control <= 0xffu) {
-		return EmitDppQuadPermTargetLane(state, subid, control);
-	}
-	if (control >= 0x101u && control <= 0x10fu) {
-		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, true);
-	}
-	if (control >= 0x111u && control <= 0x11fu) {
-		return EmitDppRowShiftTargetLane(state, subid, control & 0xfu, false);
-	}
-	if (control >= 0x121u && control <= 0x12fu) {
-		return EmitDppRowRotateRightTargetLane(state, subid, control & 0xfu);
-	}
-	if (control == 0x140u) {
-		return EmitDppMirrorTargetLane(state, subid, false);
-	}
-	if (control == 0x141u) {
-		return EmitDppMirrorTargetLane(state, subid, true);
-	}
-	return {subid, EmitTrueBool(state)};
-}
-
-uint32_t EmitDppValueU32(EmitterState& state, const IR::Operand& operand, uint32_t value) {
-	if (!operand.dpp) {
-		return value;
-	}
-
-	const auto target   = EmitDppTargetLane(state, operand.dpp_ctrl);
-	const auto shuffled = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformShuffle, state.uint_type, shuffled,
-	                           ConstantU32(state, ScopeSubgroup), value, target.lane});
-	if (operand.dpp_fetch_inactive) {
-		return shuffled;
-	}
-
-	const auto source_active = EmitLaneIndexActiveBool(state, target.lane);
-	const auto can_fetch     = state.builder.AllocateId();
-	const auto ret           = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpLogicalAnd, state.bool_type, can_fetch, target.valid, source_active});
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, ret, can_fetch, shuffled, ConstantU32(state, 0)});
-	return ret;
-}
-
-uint32_t EmitValueLoad(EmitterState& state, const IR::Operand& operand) {
-	uint32_t value = 0;
-	switch (operand.kind) {
-		case IR::OperandKind::ImmediateU32:
-		case IR::OperandKind::PcRelativeU32: value = ConstantU32(state, operand.imm); break;
-		case IR::OperandKind::Register: {
-			const auto pointer = PointerForRegister(state, operand.reg);
-			if (pointer == 0) {
-				value = ConstantU32(state, 0);
-				break;
-			}
-			value = state.builder.AllocateId();
-			state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-			break;
-		}
-		case IR::OperandKind::Null: value = ConstantU32(state, 0); break;
-		default: value = ConstantU32(state, 0); break;
-	}
-	value = EmitDppValueU32(state, operand, value);
-	return EmitSdwaExtractU32(state, operand, value);
-}
-
-uint32_t EmitRegisterLoad(EmitterState& state, IR::Register reg) {
-	if (IsInactiveWave32ExecHigh(state, reg)) {
-		return ConstantU32(state, 0);
-	}
-	const auto pointer = PointerForRegister(state, reg);
-	if (pointer == 0) {
-		return ConstantU32(state, 0);
-	}
-	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-	return value;
-}
-
-uint32_t WaveMaskHighLoad(EmitterState& state, IR::RegisterFile file) {
-	if (state.wave_size == 32u && IsMaskRegisterFile(file)) {
-		return ConstantU32(state, 0);
-	}
-	return EmitRegisterLoad(state, {file, 1});
-}
-
-uint32_t EmitLaneMaskPartU32(EmitterState& state, uint32_t part) {
-	const auto subid = EmitSubgroupLocalInvocationId(state);
-	const auto lane  = state.builder.AllocateId();
-	const auto bit   = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
-	if (part == 0) {
-		const auto in_low = state.builder.AllocateId();
-		const auto ret    = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpULessThan, state.bool_type, in_low, subid, ConstantU32(state, 32)});
-		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, ret, in_low, bit, ConstantU32(state, 0)});
-		return ret;
-	}
-	if (state.wave_size == 32u) {
-		return ConstantU32(state, 0);
-	}
-	const auto in_high = state.builder.AllocateId();
-	const auto ret     = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, ret, in_high, bit, ConstantU32(state, 0)});
-	return ret;
-}
-
-uint32_t EmitThreadMaskBelowPartU32(EmitterState& state, uint32_t part) {
-	const auto subid = EmitSubgroupLocalInvocationId(state);
-	const auto lane  = state.builder.AllocateId();
-	const auto bit   = state.builder.AllocateId();
-	const auto below = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, lane, subid, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane});
-	state.builder.AddFunction({OpISub, state.uint_type, below, bit, ConstantU32(state, 1)});
-
-	if (part == 0) {
-		const auto in_high = state.builder.AllocateId();
-		const auto ret     = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
-		state.builder.AddFunction(
-		    {OpSelect, state.uint_type, ret, in_high, ConstantU32(state, 0xffffffffu), below});
-		return ret;
-	}
-	if (state.wave_size == 32u) {
-		return ConstantU32(state, 0);
-	}
-	const auto in_high = state.builder.AllocateId();
-	const auto ret     = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpUGreaterThanEqual, state.bool_type, in_high, subid, ConstantU32(state, 32)});
-	state.builder.AddFunction(
-	    {OpSelect, state.uint_type, ret, in_high, below, ConstantU32(state, 0)});
-	return ret;
-}
-
-uint32_t EmitMaskActiveBool(EmitterState& state, IR::RegisterFile file) {
-	const auto low       = EmitRegisterLoad(state, {file, 0});
-	const auto condition = state.builder.AllocateId();
-	if (state.per_invocation_masks) {
-		const auto combined = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseOr, state.uint_type, combined, low, WaveMaskHighLoad(state, file)});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, condition, combined, ConstantU32(state, 0)});
-		return condition;
-	}
-	if (state.exact_subgroup_operations) {
-		const auto high     = WaveMaskHighLoad(state, file);
-		const auto low_hit  = state.builder.AllocateId();
-		const auto high_hit = state.builder.AllocateId();
-		const auto active   = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, active, low_hit, high_hit});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, condition, active, ConstantU32(state, 0)});
-		return condition;
-	}
-	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, condition, low, ConstantU32(state, 0)});
-	return condition;
-}
-
-uint32_t EmitMaskZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
-	const auto active = EmitMaskActiveBool(state, file);
-	if (!zero) {
-		return active;
-	}
-	const auto cond = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, cond, active});
-	return cond;
-}
-
-uint32_t EmitMaskSummaryZeroBool(EmitterState& state, IR::RegisterFile file, bool zero) {
-	if (state.per_invocation_masks) {
-		return EmitMaskZeroBool(state, file, zero);
-	}
-	if (state.exact_subgroup_operations) {
-		const auto low      = EmitRegisterLoad(state, {file, 0});
-		const auto high     = WaveMaskHighLoad(state, file);
-		const auto combined = state.builder.AllocateId();
-		const auto active   = state.builder.AllocateId();
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
-		if (!zero) {
-			return active;
-		}
-		const auto inactive = state.builder.AllocateId();
-		state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
-		return inactive;
-	}
-	const auto active = EmitMaskActiveBool(state, file);
-	if (!zero) {
-		return active;
-	}
-	const auto inactive = state.builder.AllocateId();
-	state.builder.AddFunction({OpLogicalNot, state.bool_type, inactive, active});
-	return inactive;
-}
-
-uint32_t EmitExecActiveBool(EmitterState& state) {
-	return EmitMaskZeroBool(state, IR::RegisterFile::Exec, false);
-}
-
-uint32_t EmitConditionBool(EmitterState& state, const IR::Operand& operand) {
-	if (operand.kind == IR::OperandKind::Register) {
-		if (operand.reg.file == IR::RegisterFile::Exec ||
-		    operand.reg.file == IR::RegisterFile::Vcc) {
-			return EmitMaskActiveBool(state, operand.reg.file);
-		}
-	}
-	const auto cond_value = EmitValueLoad(state, operand);
-	const auto cond       = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpINotEqual, state.bool_type, cond, cond_value, ConstantU32(state, 0)});
-	return cond;
-}
-
-uint32_t EmitSubgroupLocalInvocationId(EmitterState& state) {
-	if (state.subgroup_local_invocation_id_variable == 0) {
-		return ConstantU32(state, 0);
-	}
-	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpLoad, state.uint_type, value, state.subgroup_local_invocation_id_variable});
-	return value;
-}
-
-uint32_t InputVariableForKind(const EmitterState& state, IR::StageInputKind kind) {
-	for (const auto& input: state.inputs) {
-		if (input.kind == kind) {
-			return input.variable_id;
-		}
-	}
-	return 0;
-}
-
-uint32_t InputVariableForParameter(const EmitterState& state, uint32_t location) {
-	for (const auto& input: state.inputs) {
-		if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
-			return input.variable_id;
-		}
-	}
-	return 0;
-}
-
-const InputBinding* InputBindingForParameter(const EmitterState& state, uint32_t location) {
-	for (const auto& input: state.inputs) {
-		if (input.kind == IR::StageInputKind::Parameter && input.location == location) {
-			return &input;
-		}
-	}
-	return nullptr;
-}
-
-uint32_t EmitInputComponentU32(EmitterState& state, IR::StageInputKind kind, uint32_t component) {
-	const auto variable = InputVariableForKind(state, kind);
-	if (variable == 0) {
-		return ConstantU32(state, 0);
-	}
-	const auto pointer = state.builder.AllocateId();
-	const auto value   = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpAccessChain, state.ptr_input_uint, pointer, variable, ConstantU32(state, component)});
-	state.builder.AddFunction({OpLoad, state.uint_type, value, pointer});
-	return value;
-}
-
-uint32_t EmitLocalInvocationIndex(EmitterState& state) {
-	const auto variable = InputVariableForKind(state, IR::StageInputKind::LocalInvocationIndex);
-	if (variable == 0) {
-		return ConstantU32(state, 0);
-	}
-	const auto value = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.uint_type, value, variable});
-	return value;
-}
-
-uint32_t VertexInputDefaultComponentU32(EmitterState& state, VertexInputScalarKind kind,
-                                        uint32_t component) {
-	if ((component & 3u) != 3u) {
-		return ConstantU32(state, 0);
-	}
-	return ConstantU32(state, kind == VertexInputScalarKind::Float ? 0x3f800000u : 1u);
-}
-
-uint32_t EmitVertexParameterComponentU32(EmitterState& state, const InputBinding& input,
-                                         uint32_t component) {
-	const auto count = VertexParameterComponentCount(state, input);
-	const auto kind  = VertexParameterScalarKind(state, input.location);
-	if (component >= count) {
-		return VertexInputDefaultComponentU32(state, kind, component);
-	}
-
-	const auto scalar_type = VertexParameterScalarType(state, kind);
-	uint32_t   raw         = state.builder.AllocateId();
-	if (count == 1u) {
-		state.builder.AddFunction({OpLoad, scalar_type, raw, input.variable_id});
-	} else {
-		const auto pointer_type = VertexParameterScalarPointerType(state, kind);
-		const auto pointer      = state.builder.AllocateId();
-		state.builder.AddFunction({OpAccessChain, pointer_type, pointer, input.variable_id,
-		                           ConstantU32(state, component)});
-		state.builder.AddFunction({OpLoad, scalar_type, raw, pointer});
-	}
-
-	if (kind == VertexInputScalarKind::Uint) {
-		return raw;
-	}
-
-	const auto bits = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, raw});
-	return bits;
-}
-
-void EmitLoadInputF32(EmitterState& state, const IR::Instruction& inst) {
-	const auto* input = InputBindingForParameter(state, inst.input_info.attr);
-	if (input == nullptr || input->variable_id == 0) {
-		EmitStoreU32(state, inst.dst, ConstantU32(state, 0));
-		return;
-	}
-
-	if (state.stage == ShaderType::Vertex) {
-		EmitStoreU32(state, inst.dst,
-		             EmitVertexParameterComponentU32(state, *input, inst.input_info.chan & 3u));
-		return;
-	}
-
-	const auto input_value = state.builder.AllocateId();
-	const auto component   = state.builder.AllocateId();
-	const auto bits        = state.builder.AllocateId();
-	state.builder.AddFunction({OpLoad, state.vec4_float_type, input_value, input->variable_id});
-	state.builder.AddFunction(
-	    {OpCompositeExtract, state.float_type, component, input_value, inst.input_info.chan & 3u});
-	state.builder.AddFunction({OpBitcast, state.uint_type, bits, component});
-	EmitStoreU32(state, inst.dst, bits);
-}
-
-uint32_t EmitSccBool(EmitterState& state, bool non_zero) {
-	const auto value = EmitRegisterLoad(state, SccRegister());
-	const auto cond  = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {non_zero ? OpINotEqual : OpIEqual, state.bool_type, cond, value, ConstantU32(state, 0)});
-	return cond;
-}
-
-uint32_t EmitFloatLoad(EmitterState& state, const IR::Operand& operand) {
-	const auto uint_value  = EmitValueLoad(state, operand);
-	auto       float_value = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, float_value, uint_value});
-	if (operand.absolute) {
-		const auto abs_value = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, float_value});
-		float_value = abs_value;
-	}
-	if (operand.negate) {
-		const auto neg_value = state.builder.AllocateId();
-		state.builder.AddFunction({OpFNegate, state.float_type, neg_value, float_value});
-		float_value = neg_value;
-	}
-	return float_value;
-}
-
-uint32_t ApplyResultModifiersF32(EmitterState& state, uint32_t value, const IR::Operand& dst) {
-	uint32_t result = value;
-	if (dst.omod != 0) {
-		uint32_t bits = 0x3f000000u;
-		switch (dst.omod) {
-			case 1u: bits = 0x40000000u; break;
-			case 2u: bits = 0x40800000u; break;
-			default: break;
-		}
-		const auto scaled = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpFMul, state.float_type, scaled, result, ConstantF32(state, bits)});
-		result = scaled;
-	}
-	if (!dst.clamp) {
-		return result;
-	}
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpExtInst, state.float_type, ret, state.glsl_std450, GlslFClamp,
-	                           result, ConstantF32(state, 0), ConstantF32(state, 0x3f800000u)});
-	return ret;
-}
-
-uint32_t EmitMixF32Load(EmitterState& state, const IR::Operand& operand) {
-	auto value_operand      = operand;
-	value_operand.negate_hi = false;
-
-	if (operand.op_sel_hi) {
-		auto raw               = operand;
-		raw.sdwa_sel           = 6;
-		raw.sdwa_sext          = false;
-		raw.op_sel             = false;
-		raw.op_sel_hi          = false;
-		raw.negate             = false;
-		raw.negate_hi          = false;
-		raw.absolute           = false;
-		raw.dpp                = false;
-		raw.dpp_fetch_inactive = false;
-		raw.dpp_bound_ctrl     = false;
-
-		const auto bits     = EmitValueLoad(state, raw);
-		const auto unpacked = state.builder.AllocateId();
-		auto       f32      = state.builder.AllocateId();
-		state.builder.AddFunction({OpExtInst, state.vec2_float_type, unpacked, state.glsl_std450,
-		                           GlslUnpackHalf2x16, bits});
-		state.builder.AddFunction(
-		    {OpCompositeExtract, state.float_type, f32, unpacked, operand.op_sel ? 1u : 0u});
-		if (operand.absolute) {
-			const auto abs_value = state.builder.AllocateId();
-			state.builder.AddFunction(
-			    {OpExtInst, state.float_type, abs_value, state.glsl_std450, GlslFAbs, f32});
-			f32 = abs_value;
-		}
-		if (value_operand.negate) {
-			const auto neg_value = state.builder.AllocateId();
-			state.builder.AddFunction({OpFNegate, state.float_type, neg_value, f32});
-			f32 = neg_value;
-		}
-		return f32;
-	}
-
-	value_operand.op_sel    = false;
-	value_operand.op_sel_hi = false;
-	return EmitFloatLoad(state, value_operand);
-}
-
-IR::Operand OffsetRegisterOperand(const IR::Operand& operand, uint32_t offset) {
-	if (offset == 0 || operand.kind != IR::OperandKind::Register) {
-		return operand;
-	}
-	auto ret = operand;
-	ret.reg.index += offset;
-	ret.sdwa_sel           = 6;
-	ret.omod               = 0;
-	ret.sdwa_sext          = false;
-	ret.op_sel             = false;
-	ret.op_sel_hi          = false;
-	ret.negate             = false;
-	ret.negate_hi          = false;
-	ret.absolute           = false;
-	ret.clamp              = false;
-	ret.dpp_ctrl           = 0;
-	ret.dpp_row_mask       = 0xf;
-	ret.dpp_bank_mask      = 0xf;
-	ret.dpp_fetch_inactive = false;
-	ret.dpp_bound_ctrl     = false;
-	ret.dpp                = false;
-	return ret;
-}
-
-uint32_t EmitLaneMaskOperandActiveBool(EmitterState& state, const IR::Operand& operand) {
-	const auto low    = EmitValueLoad(state, operand);
-	const auto active = state.builder.AllocateId();
-	if (state.per_invocation_masks || state.exact_subgroup_operations) {
-		uint32_t high = ConstantU32(state, 0);
-		if (operand.kind == IR::OperandKind::Register &&
-		    (operand.reg.file == IR::RegisterFile::Scalar ||
-		     operand.reg.file == IR::RegisterFile::Vcc ||
-		     operand.reg.file == IR::RegisterFile::Exec)) {
-			high = IsMaskRegisterFile(operand.reg.file)
-			           ? WaveMaskHighLoad(state, operand.reg.file)
-			           : EmitValueLoad(state, OffsetRegisterOperand(operand, 1));
-		}
-		if (state.per_invocation_masks) {
-			const auto combined = state.builder.AllocateId();
-			state.builder.AddFunction({OpBitwiseOr, state.uint_type, combined, low, high});
-			state.builder.AddFunction(
-			    {OpINotEqual, state.bool_type, active, combined, ConstantU32(state, 0)});
-			return active;
-		}
-		const auto low_hit  = state.builder.AllocateId();
-		const auto high_hit = state.builder.AllocateId();
-		const auto hit      = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, low_hit, low, EmitLaneMaskPartU32(state, 0)});
-		state.builder.AddFunction(
-		    {OpBitwiseAnd, state.uint_type, high_hit, high, EmitLaneMaskPartU32(state, 1)});
-		state.builder.AddFunction({OpBitwiseOr, state.uint_type, hit, low_hit, high_hit});
-		state.builder.AddFunction(
-		    {OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
-		return active;
-	}
-	state.builder.AddFunction({OpINotEqual, state.bool_type, active, low, ConstantU32(state, 0)});
-	return active;
-}
-
-uint32_t EmitLaneIndexActiveBool(EmitterState& state, uint32_t lane) {
-	if (state.per_invocation_masks) {
-		const auto ballot = state.builder.AllocateId();
-		state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, ballot,
-		                           ConstantU32(state, ScopeSubgroup), EmitExecActiveBool(state)});
-		return EmitBallotLaneActiveBool(state, ballot, lane);
-	}
-	const auto lane_low = state.builder.AllocateId();
-	const auto bit      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
-
-	uint32_t mask = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 0});
-	if (state.wave_size == 64u) {
-		const auto high     = EmitRegisterLoad(state, {IR::RegisterFile::Exec, 1});
-		const auto in_high  = state.builder.AllocateId();
-		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, mask});
-		mask = selected;
-	}
-
-	const auto hit = state.builder.AllocateId();
-	const auto ret = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, ret, hit, ConstantU32(state, 0)});
-	return ret;
-}
-
-uint32_t EmitSubgroupLaneActiveBool(EmitterState& state, uint32_t lane) {
-	const auto active_ballot = state.builder.AllocateId();
-	state.builder.AddFunction({OpGroupNonUniformBallot, state.vec4_uint_type, active_ballot,
-	                           ConstantU32(state, ScopeSubgroup), EmitTrueBool(state)});
-	return EmitBallotLaneActiveBool(state, active_ballot, lane);
-}
-uint32_t EmitBallotLaneActiveBool(EmitterState& state, uint32_t active_ballot, uint32_t lane) {
-	const auto low = state.builder.AllocateId();
-	state.builder.AddFunction({OpCompositeExtract, state.uint_type, low, active_ballot, 0});
-	uint32_t mask = low;
-	if (state.wave_size == 64u) {
-		const auto high     = state.builder.AllocateId();
-		const auto in_high  = state.builder.AllocateId();
-		const auto selected = state.builder.AllocateId();
-		state.builder.AddFunction({OpCompositeExtract, state.uint_type, high, active_ballot, 1});
-		state.builder.AddFunction(
-		    {OpUGreaterThanEqual, state.bool_type, in_high, lane, ConstantU32(state, 32)});
-		state.builder.AddFunction({OpSelect, state.uint_type, selected, in_high, high, low});
-		mask = selected;
-	}
-
-	const auto lane_low = state.builder.AllocateId();
-	const auto bit      = state.builder.AllocateId();
-	state.builder.AddFunction(
-	    {OpBitwiseAnd, state.uint_type, lane_low, lane, ConstantU32(state, 31)});
-	state.builder.AddFunction(
-	    {OpShiftLeftLogical, state.uint_type, bit, ConstantU32(state, 1), lane_low});
-
-	const auto hit      = state.builder.AllocateId();
-	const auto active   = state.builder.AllocateId();
-	const auto in_range = state.builder.AllocateId();
-	const auto ret      = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitwiseAnd, state.uint_type, hit, mask, bit});
-	state.builder.AddFunction({OpINotEqual, state.bool_type, active, hit, ConstantU32(state, 0)});
-	state.builder.AddFunction(
-	    {OpULessThan, state.bool_type, in_range, lane, ConstantU32(state, state.wave_size)});
-	state.builder.AddFunction({OpLogicalAnd, state.bool_type, ret, active, in_range});
-	return ret;
-}
-
-uint32_t EmitSequentialValueLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
-	if (offset == 0 || operand.kind == IR::OperandKind::Register) {
-		return EmitValueLoad(state, OffsetRegisterOperand(operand, offset));
-	}
-	if (operand.kind == IR::OperandKind::ImmediateU32 && offset == 1) {
-		return ConstantU32(state, operand.sext_64 ? 0xffffffffu : 0u);
-	}
-	return ConstantU32(state, 0);
-}
-
-uint32_t EmitSequentialFloatLoad(EmitterState& state, const IR::Operand& operand, uint32_t offset) {
-	const auto value = EmitSequentialValueLoad(state, operand, offset);
-	const auto f32   = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
-	return f32;
-}
-
-bool ImageAddressUsesA16(const IR::Instruction& inst) {
-	return (inst.memory.image_sample_flags & Decoder::ImageSampleFlagA16) != 0u;
-}
-
-bool ImageAddressComponentUsesA16(const IR::Instruction& inst, uint32_t component) {
-	if (!ImageAddressUsesA16(inst)) {
-		return false;
-	}
-
-	uint32_t cursor = 0;
-	if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagOffset) != 0u) {
-		if (component == cursor) {
-			return false;
-		}
-		cursor++;
-	}
-	if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagBias) != 0u) {
-		if (component == cursor) {
-			return true;
-		}
-		cursor++;
-	}
-	if ((inst.memory.image_sample_flags & Decoder::ImageSampleFlagCompare) != 0u &&
-	    component == cursor) {
-		return false;
-	}
-	return true;
-}
-
-uint32_t ImageAddressHalfComponent(const IR::Instruction& inst, uint32_t component) {
-	if (!ImageAddressUsesA16(inst)) {
-		return component * 2u;
-	}
-
-	uint32_t half_component = 0;
-	for (uint32_t i = 0; i < component; i++) {
-		const auto half_width = ImageAddressComponentUsesA16(inst, i) ? 1u : 2u;
-		if (half_width == 2u && (half_component & 1u) != 0u) {
-			half_component++;
-		}
-		half_component += half_width;
-	}
-	if (!ImageAddressComponentUsesA16(inst, component) && (half_component & 1u) != 0u) {
-		half_component++;
-	}
-	return half_component;
-}
-
-uint32_t ImageAddressPackedComponent(const IR::Instruction& inst, uint32_t component) {
-	return ImageAddressHalfComponent(inst, component) / 2u;
-}
-
-IR::Operand ImageAddressOperand(const IR::Instruction& inst, const IR::Operand& base,
-                                uint32_t component) {
-	const auto packed_component = ImageAddressPackedComponent(inst, component);
-	if (packed_component > 0 && inst.memory.image_nsa_dwords != 0) {
-		const auto nsa_index = packed_component - 1u;
-		const auto nsa_count = std::min<uint32_t>(inst.memory.image_nsa_dwords * 4u,
-		                                          Decoder::MaxImageNsaAddressComponents);
-		if (nsa_index < nsa_count) {
-			return MakeRegisterOperand(IR::RegisterFile::Vector,
-			                           inst.memory.image_nsa_addr[nsa_index]);
-		}
-	}
-	return OffsetRegisterOperand(base, packed_component);
-}
-
-uint32_t EmitImageAddressValueLoad(EmitterState& state, const IR::Instruction& inst,
-                                   const IR::Operand& base, uint32_t component) {
-	const auto operand          = ImageAddressOperand(inst, base, component);
-	const auto packed_component = ImageAddressPackedComponent(inst, component);
-	if (packed_component == 0 || operand.kind == IR::OperandKind::Register) {
-		auto value = EmitValueLoad(state, operand);
-		if (ImageAddressComponentUsesA16(inst, component)) {
-			if ((ImageAddressHalfComponent(inst, component) & 1u) != 0u) {
-				value = EmitShiftRightConstant(state, value, 16u);
-			}
-			value = EmitAndConstant(state, value, 0xffffu);
-		}
-		return value;
-	}
-	return ConstantU32(state, 0);
-}
-
-uint32_t EmitImageAddressFloatLoad(EmitterState& state, const IR::Instruction& inst,
-                                   const IR::Operand& base, uint32_t component) {
-	const auto value = EmitImageAddressValueLoad(state, inst, base, component);
-	if (ImageAddressComponentUsesA16(inst, component)) {
-		return EmitF16BitsToF32(state, value);
-	}
-	const auto f32 = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, f32, value});
-	return f32;
-}
-
-uint32_t EmitZeroF32(EmitterState& state) {
-	const auto f32 = state.builder.AllocateId();
-	state.builder.AddFunction({OpBitcast, state.float_type, f32, ConstantU32(state, 0)});
-	return f32;
-}
-
-} // namespace Libs::Graphics::ShaderRecompiler::Spirv::Emitter

src/graphics/shader/recompiler/frontend/cfg/ShaderCFG.h

@@ -3,7 +3,7 @@
 
 #include "common/common.h"
 #include "common/stringUtils.h"
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 #include <vector>
 
@@ -17,6 +17,7 @@ enum class BranchCondition {
 	VccNonZero,
 	ExecZero,
 	ExecNonZero,
+	GotoVariable,
 	Unknown
 };
 
@@ -46,7 +47,9 @@ struct Terminator {
 	std::vector<uint32_t> indirect_targets;
 	std::vector<uint32_t> indirect_selector_values;
 	std::vector<uint32_t> indirect_selector_targets;
-	bool                  loop_header = false;
+	uint32_t              goto_variable = UINT32_MAX;
+	int32_t               goto_value    = -1;
+	bool                  loop_header   = false;
 };
 
 struct BasicBlock {
@@ -107,10 +110,9 @@ struct Graph {
 
 bool        BuildGraph(const Decoder::Program& program, Graph& graph, std::string* error);
 bool        Structurize(Graph& graph, std::string* error);
+bool        IsolateLoopHeader(Graph& graph, uint32_t header_id, std::string* error);
 std::string BranchConditionToString(BranchCondition condition);
 std::string FailureKindToString(FailureKind kind);
-std::string FormatBlockDiagnostic(const Graph& graph, uint32_t block_id, const char* stage,
-                                  const std::string& reason);
 std::string GraphToString(const Graph& graph);
 
 } // namespace Libs::Graphics::ShaderRecompiler::CFG

src/graphics/shader/recompiler/frontend/decode/ExportOps.cpp

@@ -1,18 +1,9 @@
-#include "graphics/shader/recompiler/decompiler/ExportOps.h"
-
-#include <fmt/format.h>
+#include "graphics/shader/recompiler/frontend/decode/ExportOps.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
-bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-               std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated EXP instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
+void DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+               Instruction& inst) {
 	const uint32_t word0  = code[word_index];
 	const uint32_t word1  = code[word_index + 1u];
 	const uint32_t target = (word0 >> 4u) & 0x3fu;
@@ -23,7 +14,7 @@ bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	inst.word_count = 2;
 	inst.family     = Family::EXP;
 	inst.opcode_id  = target;
-	inst.opcode     = Opcode::Exp;
+	inst.opcode     = Opcode::EXP;
 	inst.exp.target = target;
 	inst.exp.en     = en;
 	inst.exp.compr  = ((word0 >> 10u) & 1u) != 0;
@@ -31,12 +22,10 @@ bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	inst.exp.vm     = ((word0 >> 12u) & 1u) != 0;
 	SetRawWords(inst, code, word_index, 2);
 
-	if (!DecodeVectorGpr(word1 & 0xffu, inst.src0, error) ||
-	    !DecodeVectorGpr((word1 >> 8u) & 0xffu, inst.src1, error) ||
-	    !DecodeVectorGpr((word1 >> 16u) & 0xffu, inst.src2, error) ||
-	    !DecodeVectorGpr((word1 >> 24u) & 0xffu, inst.src3, error)) {
-		return false;
-	}
+	DecodeVectorGpr(word1 & 0xffu, inst.src0);
+	DecodeVectorGpr((word1 >> 8u) & 0xffu, inst.src1);
+	DecodeVectorGpr((word1 >> 16u) & 0xffu, inst.src2);
+	DecodeVectorGpr((word1 >> 24u) & 0xffu, inst.src3);
 
 	if (target == 0x14u && inst.exp.done && en == 0x1u) {
 		inst.src_count = 1u;
@@ -46,8 +35,6 @@ bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
 	if (en == 0u) {
 		inst.src_count = 0;
 	}
-
-	return true;
 }
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/frontend/decode/ExportOps.h

@@ -1,12 +1,11 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_EXPORTOPS_H_
 
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
-bool DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-               std::string* error);
+void DecodeExp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder
 

src/graphics/shader/recompiler/frontend/decode/ImageOps.cpp

@@ -1,31 +1,31 @@
-#include "graphics/shader/recompiler/decompiler/ImageOps.h"
+#include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
+
+#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h"
 
 #include <algorithm>
-#include <fmt/format.h>
-#include <iterator>
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 namespace {
 
 struct MimgSampleInfo {
-	uint32_t    opcode             = 0;
+	uint32_t    encoding           = 0;
 	const char* name               = nullptr;
 	uint32_t    flags              = 0;
 	uint32_t    address_components = 3;
 };
 
 struct MimgGatherInfo {
-	uint32_t    opcode             = 0;
+	uint32_t    encoding           = 0;
 	const char* name               = nullptr;
-	Opcode      decoded            = Opcode::Unsupported;
+	Opcode      decoded            = Opcode::UNSUPPORTED;
 	uint32_t    flags              = 0;
 	uint32_t    address_components = 3;
 };
 
 struct MimgAtomicInfo {
-	uint32_t    opcode  = 0;
-	const char* name    = nullptr;
-	Opcode      decoded = Opcode::Unsupported;
+	uint32_t    encoding = 0;
+	const char* name     = nullptr;
+	Opcode      decoded  = Opcode::UNSUPPORTED;
 };
 
 constexpr ImageDimension DecodeImageDimension(uint32_t dim) {
@@ -88,7 +88,7 @@ constexpr MimgSampleInfo SampleInfo(uint32_t opcode, const char* name, uint32_t
 	return {opcode, name, flags, ImageSampleAddressComponents(flags, ImageDimension::Dim2D)};
 }
 
-constexpr MimgSampleInfo MIMG_SAMPLE_OPS[] = {
+constexpr MimgSampleInfo MIMG_SAMPLE_OPCODE_LIST[] = {
     SampleInfo(0x20u, "image_sample", 0),
     SampleInfo(0x21u, "image_sample_cl", ImageSampleFlagLodClamp),
     SampleInfo(0x22u, "image_sample_d", ImageSampleFlagDerivative),
@@ -177,60 +177,49 @@ constexpr MimgSampleInfo MIMG_SAMPLE_OPS[] = {
                    ImageSampleFlagOffset),
 };
 
-constexpr MimgGatherInfo MIMG_GATHER_OPS[] = {
-    {0x47u, "image_gather4_lz", Opcode::ImageGather4Lz, ImageSampleFlagLevelZero, 2u},
-    {0x48u, "image_gather4_c", Opcode::ImageGather4C, ImageSampleFlagCompare, 3u},
-    {0x4fu, "image_gather4_c_lz", Opcode::ImageGather4CLz,
+constexpr MimgGatherInfo MIMG_GATHER_OPCODE_LIST[] = {
+    {0x47u, "image_gather4_lz", Opcode::IMAGE_GATHER4_LZ, ImageSampleFlagLevelZero, 2u},
+    {0x48u, "image_gather4_c", Opcode::IMAGE_GATHER4_C, ImageSampleFlagCompare, 3u},
+    {0x4fu, "image_gather4_c_lz", Opcode::IMAGE_GATHER4_C_LZ,
      ImageSampleFlagCompare | ImageSampleFlagLevelZero, 3u},
-    {0x57u, "image_gather4_lz_o", Opcode::ImageGather4LzO,
+    {0x57u, "image_gather4_lz_o", Opcode::IMAGE_GATHER4_LZ_O,
      ImageSampleFlagLevelZero | ImageSampleFlagOffset, 3u},
-    {0x58u, "image_gather4_c_o", Opcode::ImageGather4CO,
+    {0x58u, "image_gather4_c_o", Opcode::IMAGE_GATHER4_C_O,
      ImageSampleFlagCompare | ImageSampleFlagOffset, 4u},
-    {0x5fu, "image_gather4_c_lz_o", Opcode::ImageGather4CLzO,
+    {0x5fu, "image_gather4_c_lz_o", Opcode::IMAGE_GATHER4_C_LZ_O,
      ImageSampleFlagCompare | ImageSampleFlagLevelZero | ImageSampleFlagOffset, 4u},
-    {0x61u, "image_gather4h", Opcode::ImageGather4H, ImageSampleFlagGatherHorizontal, 2u},
+    {0x61u, "image_gather4h", Opcode::IMAGE_GATHER4H, ImageSampleFlagGatherHorizontal, 2u},
 };
 
-constexpr MimgAtomicInfo MIMG_ATOMIC_OPS[] = {
-    {0x11u, "image_atomic_add", Opcode::ImageAtomicAdd},
-    {0x15u, "image_atomic_umin", Opcode::ImageAtomicUMin},
-    {0x17u, "image_atomic_umax", Opcode::ImageAtomicUMax},
-    {0x18u, "image_atomic_and", Opcode::ImageAtomicAnd},
-    {0x19u, "image_atomic_or", Opcode::ImageAtomicOr},
-    {0x1au, "image_atomic_xor", Opcode::ImageAtomicXor},
+constexpr MimgAtomicInfo MIMG_ATOMIC_OPCODE_LIST[] = {
+    {0x11u, "image_atomic_add", Opcode::IMAGE_ATOMIC_ADD},
+    {0x15u, "image_atomic_umin", Opcode::IMAGE_ATOMIC_UMIN},
+    {0x17u, "image_atomic_umax", Opcode::IMAGE_ATOMIC_UMAX},
+    {0x18u, "image_atomic_and", Opcode::IMAGE_ATOMIC_AND},
+    {0x19u, "image_atomic_or", Opcode::IMAGE_ATOMIC_OR},
+    {0x1au, "image_atomic_xor", Opcode::IMAGE_ATOMIC_XOR},
 };
 
+constexpr auto MIMG_SAMPLE_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_SAMPLE_OPCODE_LIST);
+constexpr auto MIMG_GATHER_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_GATHER_OPCODE_LIST);
+constexpr auto MIMG_ATOMIC_OPS = Detail::MakeOpcodeTable<0x100>(MIMG_ATOMIC_OPCODE_LIST);
+
 const MimgSampleInfo* LookupSample(uint32_t opcode) {
-	for (const auto& info: MIMG_SAMPLE_OPS) {
-		if (info.opcode == opcode) {
-			return &info;
-		}
-	}
-	return nullptr;
+	return Detail::FindOpcode(MIMG_SAMPLE_OPS, opcode);
 }
 
 const MimgGatherInfo* LookupGather(uint32_t opcode) {
-	for (const auto& info: MIMG_GATHER_OPS) {
-		if (info.opcode == opcode) {
-			return &info;
-		}
-	}
-	return nullptr;
+	return Detail::FindOpcode(MIMG_GATHER_OPS, opcode);
 }
 
 const MimgAtomicInfo* LookupAtomic(uint32_t opcode) {
-	for (const auto& info: MIMG_ATOMIC_OPS) {
-		if (info.opcode == opcode) {
-			return &info;
-		}
-	}
-	return nullptr;
+	return Detail::FindOpcode(MIMG_ATOMIC_OPS, opcode);
 }
 
 Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const MimgGatherInfo* gather,
                         const MimgAtomicInfo* atomic) {
 	if (sample != nullptr) {
-		return Opcode::ImageSample;
+		return Opcode::IMAGE_SAMPLE;
 	}
 	if (gather != nullptr) {
 		return gather->decoded;
@@ -240,13 +229,13 @@ Opcode DecodeMimgOpcode(uint32_t opcode, const MimgSampleInfo* sample, const Mim
 	}
 
 	switch (opcode) {
-		case 0x00u: return Opcode::ImageLoad;
-		case 0x01u: return Opcode::ImageLoadMip;
-		case 0x08u: return Opcode::ImageStore;
-		case 0x09u: return Opcode::ImageStoreMip;
-		case 0x0eu: return Opcode::ImageGetResinfo;
-		case 0x60u: return Opcode::ImageGetLod;
-		default: return Opcode::Unsupported;
+		case 0x00u: return Opcode::IMAGE_LOAD;
+		case 0x01u: return Opcode::IMAGE_LOAD_MIP;
+		case 0x08u: return Opcode::IMAGE_STORE;
+		case 0x09u: return Opcode::IMAGE_STORE_MIP;
+		case 0x0eu: return Opcode::IMAGE_GET_RESINFO;
+		case 0x60u: return Opcode::IMAGE_GET_LOD;
+		default: return Opcode::UNSUPPORTED;
 	}
 }
 
@@ -297,27 +286,14 @@ bool IsSingleDmaskBit(uint32_t dmask) {
 
 } // namespace
 
-bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error) {
-	if (word_index + 1u >= code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated MIMG instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
-
+void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst) {
 	const uint32_t word0      = code[word_index];
 	const uint32_t word1      = code[word_index + 1u];
 	const uint32_t opcode     = ((word0 >> 18u) & 0x7fu) | ((word0 & 1u) << 7u);
 	const uint32_t nsa_dwords = (word0 >> 1u) & 0x3u;
 	const auto     dimension  = DecodeImageDimension((word0 >> 3u) & 0x7u);
 	const uint32_t word_count = 2u + nsa_dwords;
-	if (word_index + word_count > code.size()) {
-		if (error != nullptr) {
-			*error = fmt::format("truncated MIMG NSA instruction at pc 0x{:08x}", pc);
-		}
-		return false;
-	}
 
 	const uint32_t vdata  = (word1 >> 8u) & 0xffu;
 	const uint32_t vaddr  = word1 & 0xffu;
@@ -344,14 +320,14 @@ bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 	}
 	inst.image_dimension  = dimension;
 	inst.image_nsa_dwords = nsa_dwords;
-	for (uint32_t i = 0; i < nsa_dwords * 4u && i < MaxImageNsaAddressComponents; i++) {
+	for (uint32_t i = 0; i < nsa_dwords * 4u; i++) {
 		inst.image_nsa_addr[i] = (code[word_index + 2u + i / 4u] >> ((i % 4u) * 8u)) & 0xffu;
 	}
 	inst.image_address_components =
 	    DecodeMimgAddressComponents(opcode, dimension, sample, gather, atomic);
 	SetRawWords(inst, code, word_index, word_count);
 
-	if (inst.opcode == Opcode::Unsupported) {
+	if (inst.opcode == Opcode::UNSUPPORTED) {
 		SetUnsupported(inst, Family::MIMG, opcode, "MIMG opcode is not implemented");
 	}
 	if (gather != nullptr && !IsSingleDmaskBit(inst.dmask)) {
@@ -359,12 +335,11 @@ bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
 		               "MIMG image gather requires exactly one dmask bit");
 	}
 
-	DecodeVectorGpr(vdata, inst.dst, nullptr);
-	DecodeVectorGpr(vaddr, inst.src0, nullptr);
+	DecodeVectorGpr(vdata, inst.dst);
+	DecodeVectorGpr(vaddr, inst.src0);
 	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
 	DecodeScalarSource(ssamp * 4u, pc, inst.src2, nullptr);
 	inst.src_count = 3;
-	return true;
 }
 
 const char* MimgSampleOpcodeName(uint32_t opcode) {

src/graphics/shader/recompiler/frontend/decode/ImageOps.h

@@ -1,12 +1,12 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_IMAGEOPS_H_
 
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
-bool DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error);
+void DecodeMimg(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst);
 
 const char* MimgSampleOpcodeName(uint32_t opcode);
 

src/graphics/shader/recompiler/frontend/decode/MemoryOps.cpp

@@ -0,0 +1,428 @@
+#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h"
+
+#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Decoder {
+namespace {
+
+struct MemoryOpcodeInfo {
+	uint32_t encoding    = 0;
+	Opcode   decoded     = Opcode::UNSUPPORTED;
+	uint32_t data_dwords = 1;
+	uint32_t data_bits   = 32;
+	bool     data_signed = false;
+	bool     typed       = false;
+	bool     formatted   = false;
+};
+
+constexpr MemoryOpcodeInfo SMEM_OPCODE_LIST[] = {
+    {0x00u, Opcode::S_LOAD_DWORD, 1, 32},          {0x01u, Opcode::S_LOAD_DWORDX2, 2, 32},
+    {0x02u, Opcode::S_LOAD_DWORDX4, 4, 32},        {0x03u, Opcode::S_LOAD_DWORDX8, 8, 32},
+    {0x04u, Opcode::S_LOAD_DWORDX16, 16, 32},      {0x08u, Opcode::S_BUFFER_LOAD_DWORD, 1, 32},
+    {0x09u, Opcode::S_BUFFER_LOAD_DWORDX2, 2, 32}, {0x0au, Opcode::S_BUFFER_LOAD_DWORDX4, 4, 32},
+    {0x0bu, Opcode::S_BUFFER_LOAD_DWORDX8, 8, 32}, {0x0cu, Opcode::S_BUFFER_LOAD_DWORDX16, 16, 32},
+};
+
+constexpr MemoryOpcodeInfo MUBUF_OPCODE_LIST[] = {
+    {0x00u, Opcode::BUFFER_LOAD_FORMAT_X, 1, 32, false, false, true},
+    {0x01u, Opcode::BUFFER_LOAD_FORMAT_XY, 2, 32, false, false, true},
+    {0x02u, Opcode::BUFFER_LOAD_FORMAT_XYZ, 3, 32, false, false, true},
+    {0x03u, Opcode::BUFFER_LOAD_FORMAT_XYZW, 4, 32, false, false, true},
+    {0x04u, Opcode::BUFFER_STORE_FORMAT_X, 1, 32, false, false, true},
+    {0x05u, Opcode::BUFFER_STORE_FORMAT_XY, 2, 32, false, false, true},
+    {0x06u, Opcode::BUFFER_STORE_FORMAT_XYZ, 3, 32, false, false, true},
+    {0x07u, Opcode::BUFFER_STORE_FORMAT_XYZW, 4, 32, false, false, true},
+    {0x08u, Opcode::BUFFER_LOAD_UBYTE, 1, 8},
+    {0x09u, Opcode::BUFFER_LOAD_SBYTE, 1, 8, true},
+    {0x0au, Opcode::BUFFER_LOAD_USHORT, 1, 16},
+    {0x0bu, Opcode::BUFFER_LOAD_SSHORT, 1, 16, true},
+    {0x0cu, Opcode::BUFFER_LOAD_DWORD, 1, 32},
+    {0x0du, Opcode::BUFFER_LOAD_DWORDX2, 2, 32},
+    {0x0eu, Opcode::BUFFER_LOAD_DWORDX4, 4, 32},
+    {0x0fu, Opcode::BUFFER_LOAD_DWORDX3, 3, 32},
+    {0x18u, Opcode::BUFFER_STORE_BYTE, 1, 8},
+    {0x1au, Opcode::BUFFER_STORE_SHORT, 1, 16},
+    {0x1cu, Opcode::BUFFER_STORE_DWORD, 1, 32},
+    {0x1du, Opcode::BUFFER_STORE_DWORDX2, 2, 32},
+    {0x1eu, Opcode::BUFFER_STORE_DWORDX4, 4, 32},
+    {0x1fu, Opcode::BUFFER_STORE_DWORDX3, 3, 32},
+    {0x30u, Opcode::BUFFER_ATOMIC_SWAP, 1, 32},
+    {0x32u, Opcode::BUFFER_ATOMIC_ADD, 1, 32},
+    {0x33u, Opcode::BUFFER_ATOMIC_SUB, 1, 32},
+    {0x35u, Opcode::BUFFER_ATOMIC_SMIN, 1, 32},
+    {0x36u, Opcode::BUFFER_ATOMIC_UMIN, 1, 32},
+    {0x37u, Opcode::BUFFER_ATOMIC_SMAX, 1, 32},
+    {0x38u, Opcode::BUFFER_ATOMIC_UMAX, 1, 32},
+    {0x39u, Opcode::BUFFER_ATOMIC_AND, 1, 32},
+    {0x3au, Opcode::BUFFER_ATOMIC_OR, 1, 32},
+    {0x3bu, Opcode::BUFFER_ATOMIC_XOR, 1, 32},
+    {0x3fu, Opcode::BUFFER_ATOMIC_FMIN, 1, 32},
+    {0x40u, Opcode::BUFFER_ATOMIC_FMAX, 1, 32},
+};
+
+constexpr MemoryOpcodeInfo MTBUF_OPCODE_LIST[] = {
+    {0x00u, Opcode::TBUFFER_LOAD_FORMAT_X, 1, 32, false, true, true},
+    {0x01u, Opcode::TBUFFER_LOAD_FORMAT_XY, 2, 32, false, true, true},
+    {0x02u, Opcode::TBUFFER_LOAD_FORMAT_XYZ, 3, 32, false, true, true},
+    {0x03u, Opcode::TBUFFER_LOAD_FORMAT_XYZW, 4, 32, false, true, true},
+    {0x04u, Opcode::TBUFFER_STORE_FORMAT_X, 1, 32, false, true, true},
+    {0x05u, Opcode::TBUFFER_STORE_FORMAT_XY, 2, 32, false, true, true},
+    {0x06u, Opcode::TBUFFER_STORE_FORMAT_XYZ, 3, 32, false, true, true},
+    {0x07u, Opcode::TBUFFER_STORE_FORMAT_XYZW, 4, 32, false, true, true},
+};
+
+constexpr MemoryOpcodeInfo FLAT_OPCODE_LIST[] = {
+    {0x08u, Opcode::FLAT_LOAD_UBYTE, 1, 8},     {0x09u, Opcode::FLAT_LOAD_SBYTE, 1, 8, true},
+    {0x0au, Opcode::FLAT_LOAD_USHORT, 1, 16},   {0x0bu, Opcode::FLAT_LOAD_SSHORT, 1, 16, true},
+    {0x0cu, Opcode::FLAT_LOAD_DWORD, 1, 32},    {0x0du, Opcode::FLAT_LOAD_DWORDX2, 2, 32},
+    {0x0eu, Opcode::FLAT_LOAD_DWORDX4, 4, 32},  {0x0fu, Opcode::FLAT_LOAD_DWORDX3, 3, 32},
+    {0x18u, Opcode::FLAT_STORE_BYTE, 1, 8},     {0x1au, Opcode::FLAT_STORE_SHORT, 1, 16},
+    {0x1cu, Opcode::FLAT_STORE_DWORD, 1, 32},   {0x1du, Opcode::FLAT_STORE_DWORDX2, 2, 32},
+    {0x1eu, Opcode::FLAT_STORE_DWORDX4, 4, 32}, {0x1fu, Opcode::FLAT_STORE_DWORDX3, 3, 32},
+};
+
+constexpr MemoryOpcodeInfo DS_OPCODE_LIST[] = {
+    {0x00u, Opcode::DS_ADD_U32, 1, 32},          {0x01u, Opcode::DS_SUB_U32, 1, 32},
+    {0x05u, Opcode::DS_MIN_I32, 1, 32},          {0x06u, Opcode::DS_MAX_I32, 1, 32},
+    {0x07u, Opcode::DS_MIN_U32, 1, 32},          {0x08u, Opcode::DS_MAX_U32, 1, 32},
+    {0x09u, Opcode::DS_AND_B32, 1, 32},          {0x0au, Opcode::DS_OR_B32, 1, 32},
+    {0x0bu, Opcode::DS_XOR_B32, 1, 32},          {0x0du, Opcode::DS_WRITE_B32, 1, 32},
+    {0x0eu, Opcode::DS_WRITE2_B32, 2, 32},       {0x0fu, Opcode::DS_WRITE2ST64_B32, 2, 32},
+    {0x12u, Opcode::DS_MIN_F32, 1, 32},          {0x13u, Opcode::DS_MAX_F32, 1, 32},
+    {0x1eu, Opcode::DS_WRITE_B8, 1, 8},          {0x1fu, Opcode::DS_WRITE_B16, 1, 16},
+    {0x20u, Opcode::DS_ADD_RTN_U32, 1, 32},      {0x21u, Opcode::DS_SUB_RTN_U32, 1, 32},
+    {0x25u, Opcode::DS_MIN_RTN_I32, 1, 32},      {0x26u, Opcode::DS_MAX_RTN_I32, 1, 32},
+    {0x27u, Opcode::DS_MIN_RTN_U32, 1, 32},      {0x28u, Opcode::DS_MAX_RTN_U32, 1, 32},
+    {0x29u, Opcode::DS_AND_RTN_B32, 1, 32},      {0x2au, Opcode::DS_OR_RTN_B32, 1, 32},
+    {0x2bu, Opcode::DS_XOR_RTN_B32, 1, 32},      {0x2du, Opcode::DS_WRXCHG_RTN_B32, 1, 32},
+    {0x35u, Opcode::DS_SWIZZLE_B32, 1, 32},      {0x36u, Opcode::DS_READ_B32, 1, 32},
+    {0x37u, Opcode::DS_READ2_B32, 2, 32},        {0x38u, Opcode::DS_READ2_B32, 2, 32},
+    {0x39u, Opcode::DS_READ_I8, 1, 8, true},     {0x3au, Opcode::DS_READ_U8, 1, 8},
+    {0x3bu, Opcode::DS_READ_I16, 1, 16, true},   {0x3cu, Opcode::DS_READ_U16, 1, 16},
+    {0x3du, Opcode::DS_CONSUME, 1, 32},          {0x3eu, Opcode::DS_APPEND, 1, 32},
+    {0x4du, Opcode::DS_WRITE_B64, 2, 32},        {0x4eu, Opcode::DS_WRITE2_B64, 4, 32},
+    {0x4fu, Opcode::DS_WRITE2ST64_B64, 4, 32},   {0x76u, Opcode::DS_READ_B64, 2, 32},
+    {0x77u, Opcode::DS_READ2_B64, 4, 32},        {0x78u, Opcode::DS_READ2ST64_B64, 4, 32},
+    {0xb0u, Opcode::DS_WRITE_ADDTID_B32, 1, 32}, {0xb1u, Opcode::DS_READ_ADDTID_B32, 1, 32},
+    {0xdeu, Opcode::DS_WRITE_B96, 3, 32},        {0xdfu, Opcode::DS_WRITE_B128, 4, 32},
+    {0xfeu, Opcode::DS_READ_B96, 3, 32},         {0xffu, Opcode::DS_READ_B128, 4, 32},
+};
+
+constexpr auto SMEM_OPS  = Detail::MakeOpcodeTable<0x100>(SMEM_OPCODE_LIST);
+constexpr auto MUBUF_OPS = Detail::MakeOpcodeTable<0x100>(MUBUF_OPCODE_LIST);
+constexpr auto MTBUF_OPS = Detail::MakeOpcodeTable<0x10>(MTBUF_OPCODE_LIST);
+constexpr auto FLAT_OPS  = Detail::MakeOpcodeTable<0x80>(FLAT_OPCODE_LIST);
+constexpr auto DS_OPS    = Detail::MakeOpcodeTable<0x100>(DS_OPCODE_LIST);
+
+uint32_t SignExtendU32(uint32_t value, uint32_t bits) {
+	if (bits == 0u || bits >= 32u) {
+		return value;
+	}
+	const uint32_t sign = 1u << (bits - 1u);
+	return (value ^ sign) - sign;
+}
+
+void MarkMemoryUnsupported(Instruction& inst, Family family, uint32_t opcode, const char* reason) {
+	inst.family    = family;
+	inst.opcode_id = opcode;
+	inst.opcode    = Opcode::UNSUPPORTED;
+	SetUnsupported(inst, family, opcode, reason);
+}
+
+void ApplyMemoryInfo(Instruction& inst, const MemoryOpcodeInfo* info) {
+	if (info == nullptr) {
+		inst.opcode = Opcode::UNSUPPORTED;
+		return;
+	}
+	inst.opcode      = info->decoded;
+	inst.data_dwords = info->data_dwords;
+	inst.data_bits   = info->data_bits;
+	inst.data_signed = info->data_signed;
+	inst.typed       = info->typed;
+	inst.formatted   = info->formatted;
+}
+
+bool IsDsWriteOpcode(Opcode opcode) {
+	switch (opcode) {
+		case Opcode::DS_WRITE_B8:
+		case Opcode::DS_WRITE_B16:
+		case Opcode::DS_WRITE2_B32:
+		case Opcode::DS_WRITE2ST64_B32:
+		case Opcode::DS_WRITE2_B64:
+		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_WRITE_B32:
+		case Opcode::DS_WRITE_B64:
+		case Opcode::DS_WRITE_B96:
+		case Opcode::DS_WRITE_B128: return true;
+		default: return false;
+	}
+}
+
+bool IsDsAtomicOpcode(Opcode opcode) {
+	switch (opcode) {
+		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_RTN_U32:
+		case Opcode::DS_SUB_U32:
+		case Opcode::DS_SUB_RTN_U32:
+		case Opcode::DS_MIN_I32:
+		case Opcode::DS_MIN_RTN_I32:
+		case Opcode::DS_MAX_I32:
+		case Opcode::DS_MAX_RTN_I32:
+		case Opcode::DS_MIN_U32:
+		case Opcode::DS_MIN_RTN_U32:
+		case Opcode::DS_MAX_U32:
+		case Opcode::DS_MAX_RTN_U32:
+		case Opcode::DS_AND_B32:
+		case Opcode::DS_AND_RTN_B32:
+		case Opcode::DS_OR_B32:
+		case Opcode::DS_OR_RTN_B32:
+		case Opcode::DS_XOR_B32:
+		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_WRXCHG_RTN_B32: return true;
+		default: return false;
+	}
+}
+
+uint32_t DsSourceCount(Opcode opcode) {
+	switch (opcode) {
+		case Opcode::DS_WRITE2_B32:
+		case Opcode::DS_WRITE2ST64_B32:
+		case Opcode::DS_WRITE2_B64:
+		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_MIN_F32:
+		case Opcode::DS_MAX_F32: return 3u;
+		case Opcode::DS_READ_ADDTID_B32:
+		case Opcode::DS_CONSUME:
+		case Opcode::DS_APPEND: return 0u;
+		default: return IsDsWriteOpcode(opcode) || IsDsAtomicOpcode(opcode) ? 2u : 1u;
+	}
+}
+
+bool IsFlatStoreOpcode(Opcode opcode) {
+	switch (opcode) {
+		case Opcode::FLAT_STORE_BYTE:
+		case Opcode::FLAT_STORE_SHORT:
+		case Opcode::FLAT_STORE_DWORD:
+		case Opcode::FLAT_STORE_DWORDX2:
+		case Opcode::FLAT_STORE_DWORDX3:
+		case Opcode::FLAT_STORE_DWORDX4: return true;
+		default: return false;
+	}
+}
+
+} // namespace
+
+void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst) {
+	const uint32_t word0   = code[word_index];
+	const uint32_t word1   = code[word_index + 1u];
+	const uint32_t opcode  = (word0 >> 18u) & 0xffu;
+	const uint32_t sdst    = (word0 >> 6u) & 0x7fu;
+	const uint32_t sbase   = word0 & 0x3fu;
+	const uint32_t soffset = (word1 >> 25u) & 0x7fu;
+
+	inst.pc          = pc;
+	inst.word        = word0;
+	inst.word_count  = 2;
+	inst.offset      = SignExtendU32(word1 & 0x1fffffu, 21u);
+	inst.glc         = ((word0 >> 16u) & 1u) != 0;
+	inst.family      = Family::SMEM;
+	inst.opcode_id   = opcode;
+	const auto* info = Detail::FindOpcode(SMEM_OPS, opcode);
+	ApplyMemoryInfo(inst, info);
+	SetRawWords(inst, code, word_index, 2);
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		MarkMemoryUnsupported(inst, Family::SMEM, opcode, "SMEM opcode is not implemented");
+	}
+
+	DecodeScalarDestination(sdst, pc, inst.dst, nullptr);
+	// SMEM encodes SBASE in SGPR pairs. Scalar-buffer loads still use the same
+	// pair index; their descriptor operand consumes four SGPRs from that base.
+	DecodeScalarSource(sbase * 2u, pc, inst.src0, nullptr);
+	DecodeScalarSource(soffset, pc, inst.src1, nullptr);
+	inst.src_count = 2;
+}
+
+void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                 Instruction& inst) {
+	const uint32_t word0   = code[word_index];
+	const uint32_t word1   = code[word_index + 1u];
+	const uint32_t opcode  = ((word0 >> 18u) & 0x7fu) | (((word0 >> 25u) & 1u) << 7u);
+	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
+	const uint32_t vaddr   = word1 & 0xffu;
+	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
+	const uint32_t soffset = (word1 >> 24u) & 0xffu;
+
+	inst.pc          = pc;
+	inst.word        = word0;
+	inst.word_count  = 2;
+	inst.offset      = word0 & 0xfffu;
+	inst.idxen       = ((word0 >> 13u) & 1u) != 0;
+	inst.offen       = ((word0 >> 12u) & 1u) != 0;
+	inst.glc         = ((word0 >> 14u) & 1u) != 0;
+	inst.slc         = ((word1 >> 22u) & 1u) != 0;
+	inst.family      = Family::MUBUF;
+	inst.opcode_id   = opcode;
+	const auto* info = Detail::FindOpcode(MUBUF_OPS, opcode);
+	ApplyMemoryInfo(inst, info);
+	SetRawWords(inst, code, word_index, 2);
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		MarkMemoryUnsupported(inst, Family::MUBUF, opcode, "MUBUF opcode is not implemented");
+	}
+
+	DecodeVectorGpr(vdata, inst.dst);
+	DecodeVectorGpr(vaddr, inst.src0);
+	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
+	DecodeScalarSource(soffset, pc, inst.src2, nullptr);
+	inst.src_count = 3;
+}
+
+void DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                 Instruction& inst) {
+	const uint32_t word0   = code[word_index];
+	const uint32_t word1   = code[word_index + 1u];
+	const uint32_t opcode  = ((word0 >> 16u) & 0x7u) | (((word1 >> 21u) & 1u) << 3u);
+	const uint32_t dfmt    = (word0 >> 19u) & 0xfu;
+	const uint32_t nfmt    = (word0 >> 23u) & 0x7u;
+	const uint32_t vdata   = (word1 >> 8u) & 0xffu;
+	const uint32_t vaddr   = word1 & 0xffu;
+	const uint32_t srsrc   = (word1 >> 16u) & 0x1fu;
+	const uint32_t soffset = (word1 >> 24u) & 0xffu;
+
+	inst.pc            = pc;
+	inst.word          = word0;
+	inst.word_count    = 2;
+	inst.offset        = word0 & 0xfffu;
+	inst.idxen         = ((word0 >> 13u) & 1u) != 0;
+	inst.offen         = ((word0 >> 12u) & 1u) != 0;
+	inst.glc           = ((word0 >> 14u) & 1u) != 0;
+	inst.slc           = ((word1 >> 22u) & 1u) != 0;
+	inst.family        = Family::MTBUF;
+	inst.opcode_id     = opcode;
+	inst.data_format   = dfmt;
+	inst.number_format = nfmt;
+	const auto* info   = Detail::FindOpcode(MTBUF_OPS, opcode);
+	ApplyMemoryInfo(inst, info);
+	SetRawWords(inst, code, word_index, 2);
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		MarkMemoryUnsupported(inst, Family::MTBUF, opcode, "MTBUF opcode is not implemented");
+	}
+
+	DecodeVectorGpr(vdata, inst.dst);
+	DecodeVectorGpr(vaddr, inst.src0);
+	DecodeScalarSource(srsrc * 4u, pc, inst.src1, nullptr);
+	DecodeScalarSource(soffset, pc, inst.src2, nullptr);
+	inst.src_count = 3;
+}
+
+void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst) {
+	const uint32_t word0  = code[word_index];
+	const uint32_t word1  = code[word_index + 1u];
+	const uint32_t offset = word0 & 0xfffu;
+	const uint32_t dlc    = (word0 >> 12u) & 1u;
+	const uint32_t lds    = (word0 >> 13u) & 1u;
+	const uint32_t seg    = (word0 >> 14u) & 0x3u;
+	const uint32_t opcode = (word0 >> 18u) & 0x7fu;
+	const uint32_t vdst   = (word1 >> 24u) & 0xffu;
+	const uint32_t saddr  = (word1 >> 16u) & 0x7fu;
+	const uint32_t data   = (word1 >> 8u) & 0xffu;
+	const uint32_t addr   = word1 & 0xffu;
+
+	inst.pc             = pc;
+	inst.word           = word0;
+	inst.word_count     = 2;
+	inst.offset         = seg == 0u ? (offset & 0x7ffu) : SignExtendU32(offset, 12u);
+	inst.glc            = ((word0 >> 16u) & 1u) != 0;
+	inst.slc            = ((word0 >> 17u) & 1u) != 0;
+	inst.family         = Family::FLAT;
+	inst.opcode_id      = opcode;
+	inst.memory_segment = seg;
+	const auto* info    = Detail::FindOpcode(FLAT_OPS, opcode);
+	ApplyMemoryInfo(inst, info);
+	SetRawWords(inst, code, word_index, 2);
+
+	if (dlc != 0 || lds != 0 || inst.glc || inst.slc || seg == 3u) {
+		SetUnsupported(inst, Family::FLAT, opcode, "FLAT modifiers or segment are not implemented");
+		return;
+	}
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		MarkMemoryUnsupported(inst, Family::FLAT, opcode, "FLAT opcode is not implemented");
+		return;
+	}
+
+	DecodeVectorGpr(IsFlatStoreOpcode(inst.opcode) ? data : vdst, inst.dst);
+	DecodeVectorGpr(addr, inst.src0);
+	inst.src_count = 1;
+	if (seg == 0u || saddr == 0x7du || saddr == 0x7fu) {
+		DecodeVectorGpr(addr + 1u, inst.src1);
+		inst.src_count = 2;
+	} else {
+		DecodeScalarSource(saddr, pc, inst.src1, nullptr);
+		inst.src_count = 2;
+	}
+}
+
+void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst) {
+	const uint32_t word0   = code[word_index];
+	const uint32_t word1   = code[word_index + 1u];
+	const uint32_t opcode  = (word0 >> 18u) & 0xffu;
+	const uint32_t offset0 = word0 & 0xffu;
+	const uint32_t offset1 = (word0 >> 8u) & 0xffu;
+	const uint32_t vdst    = (word1 >> 24u) & 0xffu;
+	const uint32_t data1   = (word1 >> 16u) & 0xffu;
+	const uint32_t data0   = (word1 >> 8u) & 0xffu;
+	const uint32_t addr    = word1 & 0xffu;
+
+	inst.pc          = pc;
+	inst.word        = word0;
+	inst.word_count  = 2;
+	inst.offset      = offset0 | (offset1 << 8u);
+	inst.gds         = ((word0 >> 17u) & 1u) != 0u;
+	inst.family      = Family::DS;
+	inst.opcode_id   = opcode;
+	const auto* info = Detail::FindOpcode(DS_OPS, opcode);
+	ApplyMemoryInfo(inst, info);
+	SetRawWords(inst, code, word_index, 2);
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		MarkMemoryUnsupported(inst, Family::DS, opcode, "DS opcode is not implemented");
+	}
+	if (inst.opcode == Opcode::DS_SWIZZLE_B32 && inst.offset >= 0xe000u) {
+		SetUnsupported(inst, Family::DS, opcode, "DS swizzle FFT mode is not implemented");
+	}
+	if (inst.gds &&
+	    (inst.opcode == Opcode::DS_SWIZZLE_B32 || inst.opcode == Opcode::DS_WRITE_ADDTID_B32 ||
+	     inst.opcode == Opcode::DS_READ_ADDTID_B32)) {
+		SetUnsupported(inst, Family::DS, opcode, "DS swizzle/addtid is available only for LDS");
+	}
+	if (inst.opcode == Opcode::DS_WRITE_ADDTID_B32 && data1 != 0u) {
+		SetUnsupported(inst, Family::DS, opcode,
+		               "DS write addtid data1 operand is not implemented");
+	}
+	if (inst.opcode == Opcode::DS_READ_ADDTID_B32 && (data0 != 0u || data1 != 0u)) {
+		SetUnsupported(inst, Family::DS, opcode,
+		               "DS read addtid data operands are not implemented");
+	}
+	if (inst.opcode == Opcode::DS_WRITE2_B32 || opcode == 0x37u) {
+		inst.offset           = offset0 * 4u;
+		inst.secondary_offset = offset1 * 4u;
+	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B32 || opcode == 0x38u) {
+		inst.offset           = offset0 * 256u;
+		inst.secondary_offset = offset1 * 256u;
+	} else if (inst.opcode == Opcode::DS_WRITE2_B64 || inst.opcode == Opcode::DS_READ2_B64) {
+		inst.offset           = offset0 * 8u;
+		inst.secondary_offset = offset1 * 8u;
+	} else if (inst.opcode == Opcode::DS_WRITE2ST64_B64 ||
+	           inst.opcode == Opcode::DS_READ2ST64_B64) {
+		inst.offset           = offset0 * 512u;
+		inst.secondary_offset = offset1 * 512u;
+	}
+
+	DecodeVectorGpr(vdst, inst.dst);
+	DecodeVectorGpr(addr, inst.src0);
+	DecodeVectorGpr(data0, inst.src1);
+	DecodeVectorGpr(data1, inst.src2);
+	inst.src_count = DsSourceCount(inst.opcode);
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/frontend/decode/MemoryOps.h

@@ -0,0 +1,20 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_
+
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Decoder {
+
+void DecodeSmem(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst);
+void DecodeMubuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                 Instruction& inst);
+void DecodeMtbuf(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                 Instruction& inst);
+void DecodeFlat(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst);
+void DecodeDs(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst);
+
+} // namespace Libs::Graphics::ShaderRecompiler::Decoder
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_MEMORYOPS_H_ */

src/graphics/shader/recompiler/frontend/decode/OpcodeTable.h

@@ -0,0 +1,57 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_
+
+#include "common/assert.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+
+namespace Libs::Graphics::ShaderRecompiler::Decoder::Detail {
+
+struct OpcodeMap {
+	uint32_t encoding = 0;
+	Opcode   decoded  = Opcode::UNKNOWN;
+};
+
+template <typename Entry, size_t EncodingCount, size_t EntryCount>
+struct OpcodeTable {
+	std::array<Entry, EntryCount>       entries = {};
+	std::array<uint16_t, EncodingCount> indices = {};
+};
+
+template <size_t EncodingCount, typename Entry, size_t EntryCount>
+consteval auto MakeOpcodeTable(const Entry (&entries)[EntryCount]) {
+	static_assert(EntryCount < UINT16_MAX);
+	OpcodeTable<Entry, EncodingCount, EntryCount> table;
+	for (size_t i = 0; i < EntryCount; i++) {
+		const auto encoding = entries[i].encoding;
+		if (encoding >= EncodingCount || table.indices[encoding] != 0) {
+			EXIT("invalid opcode table\n");
+		}
+		table.entries[i]        = entries[i];
+		table.indices[encoding] = static_cast<uint16_t>(i + 1);
+	}
+	return table;
+}
+
+template <typename Entry, size_t EncodingCount, size_t EntryCount>
+constexpr const Entry* FindOpcode(const OpcodeTable<Entry, EncodingCount, EntryCount>& table,
+                                  uint32_t                                             encoding) {
+	if (table.indices[encoding] == 0) {
+		return nullptr;
+	}
+	return &table.entries[table.indices[encoding] - 1];
+}
+
+template <typename Entry, size_t EncodingCount, size_t EntryCount>
+constexpr Opcode LookupOpcode(const OpcodeTable<Entry, EncodingCount, EntryCount>& table,
+                              uint32_t                                             encoding) {
+	const auto* entry = FindOpcode(table, encoding);
+	return entry != nullptr ? entry->decoded : Opcode::UNSUPPORTED;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Decoder::Detail
+
+#endif // EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_DECOMPILER_OPCODETABLE_H_

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.cpp

@@ -0,0 +1,300 @@
+#include "graphics/shader/recompiler/frontend/decode/ScalarAluOps.h"
+
+#include "graphics/shader/recompiler/frontend/decode/OpcodeTable.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Decoder {
+namespace {
+
+using Detail::OpcodeMap;
+
+constexpr OpcodeMap SOP2_OPCODE_LIST[] = {
+    {0x00u, Opcode::S_ADD_U32},         {0x01u, Opcode::S_SUB_U32},
+    {0x02u, Opcode::S_ADD_I32},         {0x03u, Opcode::S_SUB_I32},
+    {0x04u, Opcode::S_ADDC_U32},        {0x05u, Opcode::S_SUBB_U32},
+    {0x06u, Opcode::S_MIN_I32},         {0x07u, Opcode::S_MIN_U32},
+    {0x08u, Opcode::S_MAX_I32},         {0x09u, Opcode::S_MAX_U32},
+    {0x0au, Opcode::S_CSELECT_B32},     {0x0bu, Opcode::S_CSELECT_B64},
+    {0x0eu, Opcode::S_AND_B32},         {0x0fu, Opcode::S_AND_B64},
+    {0x10u, Opcode::S_OR_B32},          {0x11u, Opcode::S_OR_B64},
+    {0x12u, Opcode::S_XOR_B32},         {0x13u, Opcode::S_XOR_B64},
+    {0x14u, Opcode::S_ANDN2_B32},       {0x15u, Opcode::S_ANDN2_B64},
+    {0x16u, Opcode::S_ORN2_B32},        {0x17u, Opcode::S_ORN2_B64},
+    {0x18u, Opcode::S_NAND_B32},        {0x19u, Opcode::S_NAND_B64},
+    {0x1au, Opcode::S_NOR_B32},         {0x1bu, Opcode::S_NOR_B64},
+    {0x1cu, Opcode::S_XNOR_B32},        {0x1du, Opcode::S_XNOR_B64},
+    {0x1eu, Opcode::S_LSHL_B32},        {0x1fu, Opcode::S_LSHL_B64},
+    {0x20u, Opcode::S_LSHR_B32},        {0x21u, Opcode::S_LSHR_B64},
+    {0x22u, Opcode::S_ASHR_I32},        {0x24u, Opcode::S_BFM_B32},
+    {0x25u, Opcode::S_BFM_B64},         {0x26u, Opcode::S_MUL_I32},
+    {0x27u, Opcode::S_BFE_U32},         {0x28u, Opcode::S_BFE_I32},
+    {0x29u, Opcode::S_BFE_U64},         {0x2eu, Opcode::S_LSHL1_ADD_U32},
+    {0x2fu, Opcode::S_LSHL2_ADD_U32},   {0x30u, Opcode::S_LSHL3_ADD_U32},
+    {0x31u, Opcode::S_LSHL4_ADD_U32},   {0x32u, Opcode::S_PACK_LL_B32_B16},
+    {0x33u, Opcode::S_PACK_LH_B32_B16}, {0x34u, Opcode::S_PACK_HH_B32_B16},
+    {0x35u, Opcode::S_MUL_HI_U32},
+};
+
+constexpr OpcodeMap SOP1_OPCODE_LIST[] = {
+    {0x03u, Opcode::S_MOV_B32},
+    {0x04u, Opcode::S_MOV_B64},
+    {0x07u, Opcode::S_NOT_B32},
+    {0x08u, Opcode::S_NOT_B64},
+    {0x0au, Opcode::S_WQM_B64},
+    {0x0bu, Opcode::S_BREV_B32},
+    {0x0fu, Opcode::S_BCNT1_I32_B32},
+    {0x10u, Opcode::S_BCNT1_I32_B64},
+    {0x13u, Opcode::S_FF1_I32_B32},
+    {0x14u, Opcode::S_FF1_I32_B64},
+    {0x16u, Opcode::S_FLBIT_I32_B64},
+    {0x1bu, Opcode::S_BITSET0_B32},
+    {0x1du, Opcode::S_BITSET1_B32},
+    {0x1fu, Opcode::S_GETPC_B64},
+    {0x20u, Opcode::S_SETPC_B64},
+    {0x24u, Opcode::S_AND_SAVEEXEC_B64},
+    {0x28u, Opcode::S_ORN2_SAVEEXEC_B64},
+    {0x34u, Opcode::S_ABS_I32},
+    {0x37u, Opcode::S_ANDN1_SAVEEXEC_B64},
+    {0x3bu, Opcode::S_BITREPLICATE_B64_B32},
+    {0x3cu, Opcode::S_AND_SAVEEXEC_B32},
+    {0x44u, Opcode::S_ANDN1_SAVEEXEC_B32},
+};
+
+constexpr OpcodeMap SOPC_OPCODE_LIST[] = {
+    {0x00u, Opcode::S_CMP_EQ_I32},  {0x01u, Opcode::S_CMP_LG_I32},  {0x02u, Opcode::S_CMP_GT_I32},
+    {0x03u, Opcode::S_CMP_GE_I32},  {0x04u, Opcode::S_CMP_LT_I32},  {0x05u, Opcode::S_CMP_LE_I32},
+    {0x06u, Opcode::S_CMP_EQ_U32},  {0x07u, Opcode::S_CMP_LG_U32},  {0x08u, Opcode::S_CMP_GT_U32},
+    {0x09u, Opcode::S_CMP_GE_U32},  {0x0au, Opcode::S_CMP_LT_U32},  {0x0bu, Opcode::S_CMP_LE_U32},
+    {0x0cu, Opcode::S_BITCMP0_B32}, {0x0du, Opcode::S_BITCMP1_B32}, {0x12u, Opcode::S_CMP_EQ_U64},
+    {0x13u, Opcode::S_CMP_LG_U64},
+};
+
+constexpr OpcodeMap SOPK_OPCODE_LIST[] = {
+    {0x00u, Opcode::S_MOVK_I32},   {0x03u, Opcode::S_CMP_EQ_I32}, {0x04u, Opcode::S_CMP_LG_I32},
+    {0x05u, Opcode::S_CMP_GT_I32}, {0x06u, Opcode::S_CMP_GE_I32}, {0x07u, Opcode::S_CMP_LT_I32},
+    {0x08u, Opcode::S_CMP_LE_I32}, {0x09u, Opcode::S_CMP_EQ_U32}, {0x0au, Opcode::S_CMP_LG_U32},
+    {0x0bu, Opcode::S_CMP_GT_U32}, {0x0cu, Opcode::S_CMP_GE_U32}, {0x0du, Opcode::S_CMP_LT_U32},
+    {0x0eu, Opcode::S_CMP_LE_U32}, {0x0fu, Opcode::S_ADD_I32},    {0x10u, Opcode::S_MULK_I32},
+    {0x13u, Opcode::S_SETREG_B32}, {0x17u, Opcode::S_WAITCNT},    {0x18u, Opcode::S_WAITCNT},
+    {0x19u, Opcode::S_WAITCNT},    {0x1au, Opcode::S_WAITCNT},
+};
+
+constexpr OpcodeMap SOPP_OPCODE_LIST[] = {
+    {0x00u, Opcode::S_NOP},
+    {0x01u, Opcode::S_ENDPGM},
+    {0x02u, Opcode::S_BRANCH},
+    {0x04u, Opcode::S_CBRANCH_SCC0},
+    {0x05u, Opcode::S_CBRANCH_SCC1},
+    {0x06u, Opcode::S_CBRANCH_VCCZ},
+    {0x07u, Opcode::S_CBRANCH_VCCNZ},
+    {0x08u, Opcode::S_CBRANCH_EXECZ},
+    {0x09u, Opcode::S_CBRANCH_EXECNZ},
+    {0x0au, Opcode::S_BARRIER},
+    {0x0cu, Opcode::S_WAITCNT},
+    {0x0eu, Opcode::S_SLEEP},
+    {0x10u, Opcode::S_SENDMSG},
+    {0x12u, Opcode::S_TRAP},
+    {0x16u, Opcode::S_TTRACEDATA},
+    {0x20u, Opcode::S_INST_PREFETCH},
+    {0x23u, Opcode::S_WAITCNT_DEPCTR},
+};
+
+constexpr auto SOP1_OPS = Detail::MakeOpcodeTable<0x100>(SOP1_OPCODE_LIST);
+constexpr auto SOP2_OPS = Detail::MakeOpcodeTable<0x80>(SOP2_OPCODE_LIST);
+constexpr auto SOPK_OPS = Detail::MakeOpcodeTable<0x20>(SOPK_OPCODE_LIST);
+constexpr auto SOPC_OPS = Detail::MakeOpcodeTable<0x80>(SOPC_OPCODE_LIST);
+constexpr auto SOPP_OPS = Detail::MakeOpcodeTable<0x80>(SOPP_OPCODE_LIST);
+
+bool DecodeBinarySources(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                         Instruction& inst, uint32_t ssrc0, uint32_t ssrc1, std::string* error) {
+	if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
+	    !DecodeScalarSource(ssrc1, pc, inst.src1, error)) {
+		return false;
+	}
+	inst.src_count = 2;
+	ReadLiteralOperands(code, word_index, inst);
+	return true;
+}
+
+} // namespace
+
+bool DecodeSop1(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
+                std::string* error) {
+	const uint32_t word   = code[word_index];
+	const uint32_t opcode = (word >> 8u) & 0xffu;
+	const uint32_t ssrc0  = word & 0xffu;
+	const uint32_t sdst   = (word >> 16u) & 0x7fu;
+
+	inst.pc        = pc;
+	inst.word      = word;
+	inst.family    = Family::SOP1;
+	inst.opcode_id = opcode;
+	inst.opcode    = Detail::LookupOpcode(SOP1_OPS, opcode);
+	SetRawWords(inst, code, word_index, 1);
+
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		SetUnsupported(inst, Family::SOP1, opcode, "SOP1 opcode is not implemented");
+		return true;
+	}
+
+	switch (inst.opcode) {
+		case Opcode::S_GETPC_B64:
+			inst.src_count = 0;
+			return DecodeScalarDestination(sdst, pc, inst.dst, error);
+		case Opcode::S_SETPC_B64:
+			inst.src_count = 1;
+			inst.dst.kind  = OperandKind::Null;
+			if (!DecodeScalarSource(ssrc0, pc, inst.src0, error)) {
+				return false;
+			}
+			ReadLiteralOperands(code, word_index, inst);
+			return true;
+		default: break;
+	}
+
+	if (!DecodeScalarSource(ssrc0, pc, inst.src0, error) ||
+	    !DecodeScalarDestination(sdst, pc, inst.dst, error)) {
+		return false;
+	}
+	inst.src_count = 1;
+	ReadLiteralOperands(code, word_index, inst);
+	return true;
+}
+
+bool DecodeSop2(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
+                std::string* error) {
+	const uint32_t word   = code[word_index];
+	const uint32_t opcode = (word >> 23u) & 0x7fu;
+	const uint32_t ssrc1  = (word >> 8u) & 0xffu;
+	const uint32_t ssrc0  = word & 0xffu;
+	const uint32_t sdst   = (word >> 16u) & 0x7fu;
+
+	inst.pc        = pc;
+	inst.word      = word;
+	inst.family    = Family::SOP2;
+	inst.opcode_id = opcode;
+	inst.opcode    = Detail::LookupOpcode(SOP2_OPS, opcode);
+	SetRawWords(inst, code, word_index, 1);
+
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		SetUnsupported(inst, Family::SOP2, opcode, "SOP2 opcode is not implemented");
+		return true;
+	}
+
+	if (!DecodeScalarDestination(sdst, pc, inst.dst, error)) {
+		return false;
+	}
+	return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
+}
+
+bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
+                std::string* error) {
+	const uint32_t word   = code[word_index];
+	const uint32_t opcode = (word >> 23u) & 0x1fu;
+	const uint32_t sdst   = (word >> 16u) & 0x7fu;
+	const auto     imm    = static_cast<int16_t>(word & 0xffffu);
+
+	inst.pc              = pc;
+	inst.word            = word;
+	inst.family          = Family::SOPK;
+	inst.opcode_id       = opcode;
+	inst.opcode          = Detail::LookupOpcode(SOPK_OPS, opcode);
+	inst.src0.kind       = OperandKind::IntegerInlineConstant;
+	inst.src0.signed_val = imm;
+	inst.src0.value      = static_cast<uint32_t>(imm);
+	inst.src_count       = 1;
+	SetRawWords(inst, code, word_index, 1);
+
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		SetUnsupported(inst, Family::SOPK, opcode, "SOPK opcode is not implemented");
+		return true;
+	}
+
+	switch (inst.opcode) {
+		case Opcode::S_MOVK_I32: return DecodeScalarDestination(sdst, pc, inst.dst, error);
+		case Opcode::S_WAITCNT: {
+			const uint32_t waitcnt = word & 0xffffu;
+			inst.dst.kind          = OperandKind::Null;
+			inst.src0.signed_val   = static_cast<int32_t>(waitcnt);
+			inst.src0.value        = waitcnt;
+			inst.src_count         = 1;
+			return true;
+		}
+		case Opcode::S_SETREG_B32:
+			inst.dst.kind        = OperandKind::Null;
+			inst.src1.kind       = OperandKind::LiteralConstant;
+			inst.src1.value      = word & 0xffffu;
+			inst.src1.signed_val = static_cast<int32_t>(imm);
+			inst.src_count       = 2;
+			return DecodeScalarSource(sdst, pc, inst.src0, error);
+		default: break;
+	}
+
+	inst.src1 = inst.src0;
+	if (!DecodeScalarSource(sdst, pc, inst.src0, error)) {
+		return false;
+	}
+	if (inst.opcode == Opcode::S_ADD_I32 || inst.opcode == Opcode::S_MULK_I32) {
+		inst.src_count = 2;
+		return DecodeScalarDestination(sdst, pc, inst.dst, error);
+	}
+
+	inst.dst.kind  = OperandKind::Scc;
+	inst.src_count = 2;
+	return true;
+}
+
+bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
+                std::string* error) {
+	const uint32_t word   = code[word_index];
+	const uint32_t ssrc1  = (word >> 8u) & 0xffu;
+	const uint32_t ssrc0  = word & 0xffu;
+	const uint32_t opcode = (word >> 16u) & 0x7fu;
+
+	inst.pc        = pc;
+	inst.word      = word;
+	inst.family    = Family::SOPC;
+	inst.opcode_id = opcode;
+	inst.opcode    = Detail::LookupOpcode(SOPC_OPS, opcode);
+	inst.dst.kind  = OperandKind::Scc;
+	SetRawWords(inst, code, word_index, 1);
+
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		SetUnsupported(inst, Family::SOPC, opcode, "SOPC opcode is not implemented");
+		return true;
+	}
+
+	return DecodeBinarySources(pc, code, word_index, inst, ssrc0, ssrc1, error);
+}
+
+void DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst) {
+	const uint32_t word   = code[word_index];
+	const uint32_t opcode = (word >> 16u) & 0x7fu;
+	const uint32_t simm   = word & 0xffffu;
+
+	inst.pc              = pc;
+	inst.word            = word;
+	inst.family          = Family::SOPP;
+	inst.opcode_id       = opcode;
+	inst.opcode          = Detail::LookupOpcode(SOPP_OPS, opcode);
+	inst.src0.kind       = OperandKind::LiteralConstant;
+	inst.src0.value      = inst.opcode == Opcode::S_TRAP ? simm & 0xffu : simm;
+	inst.src0.signed_val = inst.opcode == Opcode::S_TRAP
+	                           ? static_cast<int32_t>(inst.src0.value)
+	                           : static_cast<int32_t>(static_cast<int16_t>(simm));
+	inst.src_count = (inst.opcode == Opcode::S_NOP || inst.opcode == Opcode::S_WAITCNT ||
+	                  inst.opcode == Opcode::S_WAITCNT_DEPCTR || inst.opcode == Opcode::S_SLEEP ||
+	                  inst.opcode == Opcode::S_SENDMSG || inst.opcode == Opcode::S_TRAP ||
+	                  inst.opcode == Opcode::S_TTRACEDATA || inst.opcode == Opcode::S_INST_PREFETCH)
+	                     ? 1
+	                     : 0;
+	inst.branch_offset = static_cast<int32_t>(static_cast<int16_t>(simm)) * 4;
+	inst.branch_target = pc + 4u + static_cast<uint32_t>(inst.branch_offset);
+	SetRawWords(inst, code, word_index, 1);
+
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		SetUnsupported(inst, Family::SOPP, opcode, "SOPP control-flow opcode is not implemented");
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/frontend/decode/ScalarAluOps.h

@@ -1,7 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SCALARALUOPS_H_
 
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
@@ -13,8 +13,8 @@ bool DecodeSopk(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
                 std::string* error);
 bool DecodeSopc(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
                 std::string* error);
-bool DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
-                std::string* error);
+void DecodeSopp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                Instruction& inst);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder
 

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.cpp

@@ -0,0 +1,692 @@
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
+
+#include "graphics/shader/recompiler/frontend/decode/ExportOps.h"
+#include "graphics/shader/recompiler/frontend/decode/ImageOps.h"
+#include "graphics/shader/recompiler/frontend/decode/MemoryOps.h"
+#include "graphics/shader/recompiler/frontend/decode/ScalarAluOps.h"
+#include "graphics/shader/recompiler/frontend/decode/VectorAluOps.h"
+
+#include <algorithm>
+#include <bit>
+#include <fmt/format.h>
+
+namespace Libs::Graphics::ShaderRecompiler::Decoder {
+namespace {
+
+uint32_t FloatBits(float value) {
+	return std::bit_cast<uint32_t>(value);
+}
+
+void SetError(std::string* error, const char* message) {
+	if (error != nullptr) {
+		*error = message;
+	}
+}
+
+bool HasLiteral(const Instruction& inst) {
+	return inst.src0.kind == OperandKind::LiteralConstant ||
+	       inst.src1.kind == OperandKind::LiteralConstant ||
+	       inst.src2.kind == OperandKind::LiteralConstant ||
+	       inst.src3.kind == OperandKind::LiteralConstant;
+}
+
+bool IsControlFlowBranch(Opcode opcode) {
+	switch (opcode) {
+		case Opcode::S_BRANCH:
+		case Opcode::S_CBRANCH_SCC0:
+		case Opcode::S_CBRANCH_SCC1:
+		case Opcode::S_CBRANCH_VCCZ:
+		case Opcode::S_CBRANCH_VCCNZ:
+		case Opcode::S_CBRANCH_EXECZ:
+		case Opcode::S_CBRANCH_EXECNZ: return true;
+		default: return false;
+	}
+}
+
+void ApplyLiteral(Operand& operand, uint32_t literal) {
+	if (operand.kind == OperandKind::LiteralConstant) {
+		operand.value      = literal;
+		operand.signed_val = static_cast<int32_t>(literal);
+	}
+}
+
+std::string RawWordsToString(const Instruction& inst) {
+	std::string text;
+	for (uint32_t i = 0; i < inst.raw_count; i++) {
+		if (i != 0) {
+			text += " ";
+		}
+		text += fmt::format("0x{:08x}", inst.raw[i]);
+	}
+	return text;
+}
+
+std::string FormatBinary(const Instruction& inst) {
+	std::string text = fmt::format("0x{:08x}: {} {}", inst.pc, magic_enum::enum_name(inst.opcode),
+	                               OperandToString(inst.dst).c_str());
+	if (inst.dst2.kind != OperandKind::Unknown) {
+		text += ", ";
+		text += OperandToString(inst.dst2);
+	}
+	const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3};
+	for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
+		text += ", ";
+		text += OperandToString(*sources[i]);
+	}
+	return text;
+}
+
+std::string FormatSources(const Instruction& inst) {
+	std::string    text = fmt::format("0x{:08x}: {}", inst.pc, magic_enum::enum_name(inst.opcode));
+	const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3};
+	for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
+		text += i == 0 ? " " : ", ";
+		text += OperandToString(*sources[i]);
+	}
+	return text;
+}
+
+std::string FormatMemory(const Instruction& inst) {
+	std::string text = fmt::format("0x{:08x}: {} {}", inst.pc, magic_enum::enum_name(inst.opcode),
+	                               OperandToString(inst.dst).c_str());
+	const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3};
+	for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
+		text += ", ";
+		text += OperandToString(*sources[i]);
+	}
+	text += fmt::format(" ; offset={} offset2={} dwords={} bits={} dfmt={} nfmt={} signed={} "
+	                    "typed={} formatted={} segment={} glc={} slc={} idxen={} offen={}",
+	                    inst.offset, inst.secondary_offset, inst.data_dwords, inst.data_bits,
+	                    inst.data_format, inst.number_format, inst.data_signed ? 1u : 0u,
+	                    inst.typed ? 1u : 0u, inst.formatted ? 1u : 0u, inst.memory_segment,
+	                    inst.glc ? 1u : 0u, inst.slc ? 1u : 0u, inst.idxen ? 1u : 0u,
+	                    inst.offen ? 1u : 0u);
+	return text;
+}
+
+std::string WithUnsupportedReason(const Instruction& inst, const std::string& text) {
+	if (inst.unsupported_reason.empty()) {
+		return text;
+	}
+	return text + fmt::format(" ; family={} opcode=0x{:02x} raw=[{}] reason={}",
+	                          magic_enum::enum_name(inst.family), inst.opcode_id,
+	                          RawWordsToString(inst).c_str(), inst.unsupported_reason);
+}
+
+void AppendFlag(std::string* text, bool* first, uint32_t flags, uint32_t flag, const char* name) {
+	if ((flags & flag) == 0) {
+		return;
+	}
+	if (!*first) {
+		*text += "|";
+	}
+	*text += name;
+	*first = false;
+}
+
+std::string ImageSampleFlagsToString(uint32_t flags) {
+	if (flags == 0) {
+		return "none";
+	}
+	std::string text;
+	bool        first = true;
+	AppendFlag(&text, &first, flags, ImageSampleFlagLod, "lod");
+	AppendFlag(&text, &first, flags, ImageSampleFlagBias, "bias");
+	AppendFlag(&text, &first, flags, ImageSampleFlagDerivative, "derivative");
+	AppendFlag(&text, &first, flags, ImageSampleFlagCompare, "compare");
+	AppendFlag(&text, &first, flags, ImageSampleFlagOffset, "offset");
+	AppendFlag(&text, &first, flags, ImageSampleFlagLevelZero, "level_zero");
+	AppendFlag(&text, &first, flags, ImageSampleFlagLodClamp, "lod_clamp");
+	AppendFlag(&text, &first, flags, ImageSampleFlagA16, "a16");
+	AppendFlag(&text, &first, flags, ImageSampleFlagCd, "cd");
+	AppendFlag(&text, &first, flags, ImageSampleFlagGatherHorizontal, "gather_horizontal");
+	return text;
+}
+
+std::string FormatMimg(const Instruction& inst) {
+	const char* sample_name =
+	    inst.opcode == Opcode::IMAGE_SAMPLE ? MimgSampleOpcodeName(inst.opcode_id) : nullptr;
+	const std::string_view name =
+	    sample_name != nullptr ? sample_name : magic_enum::enum_name(inst.opcode);
+	std::string text =
+	    fmt::format("0x{:08x}: {} {}, {}, {}, {} ; dmask=0x{:x} image_dim={}", inst.pc, name,
+	                OperandToString(inst.dst).c_str(), OperandToString(inst.src0).c_str(),
+	                OperandToString(inst.src1).c_str(), OperandToString(inst.src2).c_str(),
+	                inst.dmask, ImageDimensionToString(inst.image_dimension));
+	switch (inst.opcode) {
+		case Opcode::IMAGE_SAMPLE:
+		case Opcode::IMAGE_GATHER4_LZ:
+		case Opcode::IMAGE_GATHER4_C:
+		case Opcode::IMAGE_GATHER4_C_LZ:
+		case Opcode::IMAGE_GATHER4_LZ_O:
+		case Opcode::IMAGE_GATHER4_C_O:
+		case Opcode::IMAGE_GATHER4_C_LZ_O:
+		case Opcode::IMAGE_GATHER4H:
+			text += fmt::format(" sample_flags={} addr_components={}",
+			                    ImageSampleFlagsToString(inst.image_sample_flags).c_str(),
+			                    inst.image_address_components);
+			break;
+		default: break;
+	}
+	if (inst.image_nsa_dwords != 0) {
+		text += fmt::format(" nsa_dwords={} nsa_addr=", inst.image_nsa_dwords);
+		const auto count =
+		    std::min<uint32_t>(inst.image_nsa_dwords * 4u, MaxImageNsaAddressComponents);
+		for (uint32_t i = 0; i < count; i++) {
+			if (i != 0) {
+				text += ",";
+			}
+			text += fmt::format("{}", inst.image_nsa_addr[i]);
+		}
+	}
+	return text;
+}
+
+std::string FormatExp(const Instruction& inst) {
+	std::string text = fmt::format("0x{:08x}: exp target=0x{:02x} en=0x{:x} done={} compr={} vm={}",
+	                               inst.pc, inst.exp.target, inst.exp.en, inst.exp.done ? 1u : 0u,
+	                               inst.exp.compr ? 1u : 0u, inst.exp.vm ? 1u : 0u);
+	const Operand* sources[] = {&inst.src0, &inst.src1, &inst.src2, &inst.src3};
+	for (uint32_t i = 0; i < inst.src_count && i < 4u; i++) {
+		text += i == 0 ? " " : ", ";
+		text += OperandToString(*sources[i]);
+	}
+	return text;
+}
+
+} // namespace
+
+const char* ImageDimensionToString(ImageDimension dimension) {
+	switch (dimension) {
+		case ImageDimension::Dim1D: return "1d";
+		case ImageDimension::Dim1DArray: return "1d_array";
+		case ImageDimension::Dim2D: return "2d";
+		case ImageDimension::Dim3D: return "3d";
+		case ImageDimension::Dim2DArray: return "2d_array";
+		case ImageDimension::Dim2DMsaa: return "2d_msaa";
+		case ImageDimension::Dim2DMsaaArray: return "2d_msaa_array";
+		default: return "unknown";
+	}
+}
+
+bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error) {
+	operand = {};
+
+	if (code <= 105u) {
+		operand.kind = OperandKind::Sgpr;
+		operand.reg  = code;
+		return true;
+	}
+	if (code >= 128u && code <= 192u) {
+		operand.kind       = OperandKind::IntegerInlineConstant;
+		operand.signed_val = static_cast<int32_t>(code - 128u);
+		operand.value      = static_cast<uint32_t>(operand.signed_val);
+		return true;
+	}
+	if (code >= 193u && code <= 208u) {
+		operand.kind       = OperandKind::IntegerInlineConstant;
+		operand.signed_val = 192 - static_cast<int32_t>(code);
+		operand.value      = static_cast<uint32_t>(operand.signed_val);
+		return true;
+	}
+	if (code >= 240u && code <= 247u) {
+		constexpr float values[] = {0.5f, -0.5f, 1.0f, -1.0f, 2.0f, -2.0f, 4.0f, -4.0f};
+		operand.kind             = OperandKind::FloatInlineConstant;
+		operand.float_val        = values[code - 240u];
+		operand.value            = FloatBits(operand.float_val);
+		return true;
+	}
+	if (code >= 256u && code <= 511u) {
+		DecodeVectorGpr(code - 256u, operand);
+		return true;
+	}
+
+	switch (code) {
+		case 106u: operand.kind = OperandKind::VccLo; return true;
+		case 107u: operand.kind = OperandKind::VccHi; return true;
+		case 124u: operand.kind = OperandKind::M0; return true;
+		case 125u: operand.kind = OperandKind::Null; return true;
+		case 126u: operand.kind = OperandKind::ExecLo; return true;
+		case 127u: operand.kind = OperandKind::ExecHi; return true;
+		case 239u: operand.kind = OperandKind::PopsExitingWaveId; return true;
+		case 248u:
+			operand.kind      = OperandKind::FloatInlineConstant;
+			operand.float_val = 0.15915494309189535f;
+			operand.value     = FloatBits(operand.float_val);
+			return true;
+		case 251u: operand.kind = OperandKind::VccZ; return true;
+		case 252u: operand.kind = OperandKind::ExecZ; return true;
+		case 253u: operand.kind = OperandKind::Scc; return true;
+		case 255u: operand.kind = OperandKind::LiteralConstant; return true;
+		default:
+			if (error != nullptr) {
+				*error = fmt::format("unsupported scalar source operand 0x{:08x} at pc 0x{:08x}",
+				                     code, pc);
+			}
+			return false;
+	}
+}
+
+bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error) {
+	operand = {};
+
+	if (code <= 105u) {
+		operand.kind = OperandKind::Sgpr;
+		operand.reg  = code;
+		return true;
+	}
+
+	switch (code) {
+		case 106u: operand.kind = OperandKind::VccLo; return true;
+		case 107u: operand.kind = OperandKind::VccHi; return true;
+		case 124u: operand.kind = OperandKind::M0; return true;
+		case 125u: operand.kind = OperandKind::Null; return true;
+		case 126u: operand.kind = OperandKind::ExecLo; return true;
+		case 127u: operand.kind = OperandKind::ExecHi; return true;
+		default:
+			if (error != nullptr) {
+				*error = fmt::format(
+				    "unsupported scalar destination operand 0x{:08x} at pc 0x{:08x}", code, pc);
+			}
+			return false;
+	}
+}
+
+void DecodeVectorGpr(uint32_t reg, Operand& operand) {
+	operand      = {};
+	operand.kind = OperandKind::Vgpr;
+	operand.reg  = reg;
+}
+
+void ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst) {
+	if (!HasLiteral(inst)) {
+		return;
+	}
+
+	const auto literal = code[word_index + inst.word_count];
+	ApplyLiteral(inst.src0, literal);
+	ApplyLiteral(inst.src1, literal);
+	ApplyLiteral(inst.src2, literal);
+	ApplyLiteral(inst.src3, literal);
+	inst.word_count++;
+	SetRawWords(inst, code, word_index, inst.word_count);
+}
+
+void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index,
+                 uint32_t word_count) {
+	inst.word_count = word_count;
+	inst.raw_count  = word_count;
+	for (uint32_t i = 0; i < inst.raw_count; i++) {
+		inst.raw[i] = code[word_index + i];
+	}
+}
+
+void SetUnsupported(Instruction& inst, Family family, uint32_t opcode_id, const char* reason) {
+	inst.opcode             = Opcode::UNSUPPORTED;
+	inst.family             = family;
+	inst.opcode_id          = opcode_id;
+	inst.unsupported_reason = reason;
+}
+
+Family GetInstructionFamily(uint32_t word) {
+	if ((word & 0x80000000u) == 0u) {
+		switch ((word >> 25u) & 0x3fu) {
+			case 0x3eu: return Family::VOPC;
+			case 0x3fu: return Family::VOP1;
+			default: return Family::VOP2;
+		}
+	}
+	if ((word & 0xc0000000u) == 0x80000000u) {
+		const auto opcode = (word >> 23u) & 0x7fu;
+		switch (opcode) {
+			case 0x7du: return Family::SOP1;
+			case 0x7eu: return Family::SOPC;
+			case 0x7fu: return Family::SOPP;
+			default: return opcode >= 0x60u ? Family::SOPK : Family::SOP2;
+		}
+	}
+
+	switch (word >> 26u) {
+		case 0x32u: return Family::VINTRP;
+		case 0x33u: return Family::VOP3P;
+		case 0x35u: return Family::VOP3;
+		case 0x36u: return Family::DS;
+		case 0x37u: return Family::FLAT;
+		case 0x38u: return Family::MUBUF;
+		case 0x3au: return Family::MTBUF;
+		case 0x3cu: return Family::MIMG;
+		case 0x3du: return Family::SMEM;
+		case 0x3eu: return Family::EXP;
+		default: return Family::Unknown;
+	}
+}
+
+bool DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
+                       std::string* error) {
+	const uint32_t pc = word_index * sizeof(uint32_t);
+	switch (GetInstructionFamily(code[word_index])) {
+		case Family::SOP1: return DecodeSop1(pc, code, word_index, inst, error);
+		case Family::SOP2: return DecodeSop2(pc, code, word_index, inst, error);
+		case Family::SOPK: return DecodeSopk(pc, code, word_index, inst, error);
+		case Family::SOPC: return DecodeSopc(pc, code, word_index, inst, error);
+		case Family::SOPP: DecodeSopp(pc, code, word_index, inst); return true;
+		case Family::VOP1: return DecodeVop1(pc, code, word_index, inst, error);
+		case Family::VOP2: return DecodeVop2(pc, code, word_index, inst, error);
+		case Family::VOP3: return DecodeVop3(pc, code, word_index, inst, error);
+		case Family::VOP3P: return DecodeVop3p(pc, code, word_index, inst, error);
+		case Family::VOPC: return DecodeVopc(pc, code, word_index, inst, error);
+		case Family::VINTRP: DecodeVintrp(pc, code, word_index, inst); return true;
+		case Family::SMEM: DecodeSmem(pc, code, word_index, inst); return true;
+		case Family::MUBUF: DecodeMubuf(pc, code, word_index, inst); return true;
+		case Family::MTBUF: DecodeMtbuf(pc, code, word_index, inst); return true;
+		case Family::FLAT: DecodeFlat(pc, code, word_index, inst); return true;
+		case Family::DS: DecodeDs(pc, code, word_index, inst); return true;
+		case Family::MIMG: DecodeMimg(pc, code, word_index, inst); return true;
+		case Family::EXP: DecodeExp(pc, code, word_index, inst); return true;
+		default:
+			if (error != nullptr) {
+				*error =
+				    fmt::format("unknown RDNA2 instruction family at pc 0x{:08x}, raw=0x{:08x}", pc,
+				                code[word_index]);
+			}
+			return false;
+	}
+}
+
+bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error) {
+	program.instructions.clear();
+	program.instructions.reserve(code.size());
+	program.code = code;
+
+	std::vector<bool> branch_targets;
+	for (uint32_t word_index = 0; word_index < code.size();) {
+		program.instructions.emplace_back();
+		if (!DecodeInstruction(code, word_index, program.instructions.back(), error)) {
+			program.instructions.pop_back();
+			return false;
+		}
+
+		const auto& inst = program.instructions.back();
+		word_index += inst.word_count;
+
+		if (IsControlFlowBranch(inst.opcode)) {
+			const auto target_index = inst.branch_target / sizeof(uint32_t);
+			if (branch_targets.empty()) {
+				branch_targets.resize(code.size());
+			}
+			branch_targets[target_index] = true;
+		}
+		if (inst.opcode == Opcode::S_ENDPGM &&
+		    (word_index >= code.size() || branch_targets.empty() || !branch_targets[word_index])) {
+			return true;
+		}
+	}
+
+	SetError(error, "shader decode reached the code boundary before S_ENDPGM");
+	return false;
+}
+
+std::string OperandToString(const Operand& operand) {
+	std::string text;
+	switch (operand.kind) {
+		case OperandKind::LiteralConstant: text = fmt::format("0x{:08x}", operand.value); break;
+		case OperandKind::IntegerInlineConstant:
+			text = fmt::format("{}", operand.signed_val);
+			break;
+		case OperandKind::FloatInlineConstant: text = fmt::format("{:f}", operand.float_val); break;
+		case OperandKind::Sgpr: text = fmt::format("s{}", operand.reg); break;
+		case OperandKind::Vgpr: text = fmt::format("v{}", operand.reg); break;
+		case OperandKind::VccLo: text = "vcc_lo"; break;
+		case OperandKind::VccHi: text = "vcc_hi"; break;
+		case OperandKind::VccZ: text = "vccz"; break;
+		case OperandKind::ExecLo: text = "exec_lo"; break;
+		case OperandKind::ExecHi: text = "exec_hi"; break;
+		case OperandKind::ExecZ: text = "execz"; break;
+		case OperandKind::Scc: text = "scc"; break;
+		case OperandKind::M0: text = "m0"; break;
+		case OperandKind::PopsExitingWaveId: text = "pops_exiting_wave_id"; break;
+		case OperandKind::Null: text = "null"; break;
+		default: text = "unknown"; break;
+	}
+	if (operand.sdwa_sel != 6 || operand.sdwa_sext) {
+		text += fmt::format(".sdwa(sel={},sext={})", operand.sdwa_sel, operand.sdwa_sext ? 1u : 0u);
+	}
+	if (operand.absolute) {
+		text += ".abs";
+	}
+	if (operand.negate) {
+		text += ".neg";
+	}
+	if (operand.op_sel || operand.op_sel_hi || operand.negate_hi) {
+		text += fmt::format(".opsel(lo={},hi={},neghi={})", operand.op_sel ? 1u : 0u,
+		                    operand.op_sel_hi ? 1u : 0u, operand.negate_hi ? 1u : 0u);
+	}
+	if (operand.omod != 0) {
+		text += fmt::format(".omod({})", operand.omod);
+	}
+	if (operand.clamp) {
+		text += ".clamp";
+	}
+	if (operand.dpp) {
+		text += fmt::format(".dpp(ctrl=0x{:x},fi={},bc={})", operand.dpp_ctrl,
+		                    operand.dpp_fetch_inactive ? 1u : 0u, operand.dpp_bound_ctrl ? 1u : 0u);
+	}
+	return text;
+}
+
+std::string InstructionToString(const Instruction& inst) {
+	if (inst.opcode == Opcode::UNSUPPORTED) {
+		return fmt::format("0x{:08x}: unsupported family={} opcode=0x{:02x} raw=[{}] reason={}",
+		                   inst.pc, magic_enum::enum_name(inst.family), inst.opcode_id,
+		                   RawWordsToString(inst).c_str(), inst.unsupported_reason);
+	}
+	if (inst.family == Family::SOPC) {
+		return WithUnsupportedReason(inst, FormatSources(inst));
+	}
+
+	switch (inst.opcode) {
+		case Opcode::S_MOV_B32:
+		case Opcode::S_MOV_B64:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               OperandToString(inst.dst).c_str(),
+			                                               OperandToString(inst.src0).c_str()));
+		case Opcode::S_ABS_I32:
+		case Opcode::S_BREV_B32:
+		case Opcode::S_BCNT1_I32_B32:
+		case Opcode::S_FF1_I32_B32:
+		case Opcode::S_FF1_I32_B64:
+		case Opcode::S_NOT_B64:
+		case Opcode::S_WQM_B64:
+		case Opcode::S_AND_SAVEEXEC_B32:
+		case Opcode::S_ANDN1_SAVEEXEC_B32:
+		case Opcode::S_AND_SAVEEXEC_B64:
+		case Opcode::S_ORN2_SAVEEXEC_B64:
+		case Opcode::S_ANDN1_SAVEEXEC_B64:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}, {}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               OperandToString(inst.dst).c_str(),
+			                                               OperandToString(inst.src0).c_str()));
+		case Opcode::S_GETPC_B64:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_getpc_b64 {}", inst.pc,
+			                                               OperandToString(inst.dst).c_str()));
+		case Opcode::S_SETPC_B64:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setpc_b64 {}", inst.pc,
+			                                               OperandToString(inst.src0).c_str()));
+		case Opcode::S_SETREG_B32:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_setreg_b32 {}, {}", inst.pc,
+			                                               OperandToString(inst.src0).c_str(),
+			                                               OperandToString(inst.src1).c_str()));
+		case Opcode::S_NOP:
+		case Opcode::S_WAITCNT:
+		case Opcode::S_WAITCNT_DEPCTR:
+		case Opcode::S_SLEEP:
+		case Opcode::S_TRAP:
+		case Opcode::S_SENDMSG:
+		case Opcode::S_TTRACEDATA:
+		case Opcode::S_INST_PREFETCH:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} {}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               OperandToString(inst.src0).c_str()));
+		case Opcode::S_BARRIER:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_barrier", inst.pc));
+		case Opcode::V_NOP:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: v_nop", inst.pc));
+		case Opcode::S_ENDPGM:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: s_endpgm", inst.pc));
+		case Opcode::S_BRANCH:
+		case Opcode::S_CBRANCH_SCC0:
+		case Opcode::S_CBRANCH_SCC1:
+		case Opcode::S_CBRANCH_VCCZ:
+		case Opcode::S_CBRANCH_VCCNZ:
+		case Opcode::S_CBRANCH_EXECZ:
+		case Opcode::S_CBRANCH_EXECNZ:
+			return WithUnsupportedReason(inst, fmt::format("0x{:08x}: {} 0x{:08x}", inst.pc,
+			                                               magic_enum::enum_name(inst.opcode),
+			                                               inst.branch_target));
+		case Opcode::EXP: return WithUnsupportedReason(inst, FormatExp(inst));
+		case Opcode::IMAGE_SAMPLE:
+		case Opcode::IMAGE_STORE:
+		case Opcode::IMAGE_STORE_MIP:
+		case Opcode::IMAGE_ATOMIC_ADD:
+		case Opcode::IMAGE_ATOMIC_UMIN:
+		case Opcode::IMAGE_ATOMIC_UMAX:
+		case Opcode::IMAGE_ATOMIC_AND:
+		case Opcode::IMAGE_ATOMIC_OR:
+		case Opcode::IMAGE_ATOMIC_XOR:
+		case Opcode::IMAGE_LOAD:
+		case Opcode::IMAGE_LOAD_MIP:
+		case Opcode::IMAGE_GET_RESINFO:
+		case Opcode::IMAGE_GET_LOD:
+		case Opcode::IMAGE_GATHER4_LZ:
+		case Opcode::IMAGE_GATHER4_C:
+		case Opcode::IMAGE_GATHER4_C_LZ:
+		case Opcode::IMAGE_GATHER4_LZ_O:
+		case Opcode::IMAGE_GATHER4_C_O:
+		case Opcode::IMAGE_GATHER4_C_LZ_O:
+		case Opcode::IMAGE_GATHER4H: return WithUnsupportedReason(inst, FormatMimg(inst));
+		case Opcode::S_LOAD_DWORD:
+		case Opcode::S_LOAD_DWORDX2:
+		case Opcode::S_LOAD_DWORDX4:
+		case Opcode::S_LOAD_DWORDX8:
+		case Opcode::S_LOAD_DWORDX16:
+		case Opcode::S_BUFFER_LOAD_DWORD:
+		case Opcode::S_BUFFER_LOAD_DWORDX2:
+		case Opcode::S_BUFFER_LOAD_DWORDX4:
+		case Opcode::S_BUFFER_LOAD_DWORDX8:
+		case Opcode::S_BUFFER_LOAD_DWORDX16:
+		case Opcode::BUFFER_LOAD_FORMAT_X:
+		case Opcode::BUFFER_LOAD_FORMAT_XY:
+		case Opcode::BUFFER_LOAD_FORMAT_XYZ:
+		case Opcode::BUFFER_LOAD_FORMAT_XYZW:
+		case Opcode::BUFFER_STORE_FORMAT_X:
+		case Opcode::BUFFER_STORE_FORMAT_XY:
+		case Opcode::BUFFER_STORE_FORMAT_XYZ:
+		case Opcode::BUFFER_STORE_FORMAT_XYZW:
+		case Opcode::BUFFER_LOAD_UBYTE:
+		case Opcode::BUFFER_LOAD_USHORT:
+		case Opcode::BUFFER_LOAD_DWORD:
+		case Opcode::BUFFER_LOAD_DWORDX2:
+		case Opcode::BUFFER_LOAD_DWORDX3:
+		case Opcode::BUFFER_LOAD_DWORDX4:
+		case Opcode::BUFFER_STORE_BYTE:
+		case Opcode::BUFFER_STORE_SHORT:
+		case Opcode::BUFFER_STORE_DWORD:
+		case Opcode::BUFFER_STORE_DWORDX2:
+		case Opcode::BUFFER_STORE_DWORDX3:
+		case Opcode::BUFFER_STORE_DWORDX4:
+		case Opcode::TBUFFER_LOAD_FORMAT_X:
+		case Opcode::TBUFFER_LOAD_FORMAT_XY:
+		case Opcode::TBUFFER_LOAD_FORMAT_XYZ:
+		case Opcode::TBUFFER_LOAD_FORMAT_XYZW:
+		case Opcode::TBUFFER_STORE_FORMAT_X:
+		case Opcode::TBUFFER_STORE_FORMAT_XY:
+		case Opcode::TBUFFER_STORE_FORMAT_XYZ:
+		case Opcode::TBUFFER_STORE_FORMAT_XYZW:
+		case Opcode::BUFFER_ATOMIC_SWAP:
+		case Opcode::BUFFER_ATOMIC_ADD:
+		case Opcode::BUFFER_ATOMIC_SUB:
+		case Opcode::BUFFER_ATOMIC_SMIN:
+		case Opcode::BUFFER_ATOMIC_UMIN:
+		case Opcode::BUFFER_ATOMIC_SMAX:
+		case Opcode::BUFFER_ATOMIC_UMAX:
+		case Opcode::BUFFER_ATOMIC_AND:
+		case Opcode::BUFFER_ATOMIC_OR:
+		case Opcode::BUFFER_ATOMIC_XOR:
+		case Opcode::BUFFER_ATOMIC_FMIN:
+		case Opcode::BUFFER_ATOMIC_FMAX:
+		case Opcode::BUFFER_LOAD_SBYTE:
+		case Opcode::BUFFER_LOAD_SSHORT:
+		case Opcode::FLAT_LOAD_UBYTE:
+		case Opcode::FLAT_LOAD_SBYTE:
+		case Opcode::FLAT_LOAD_USHORT:
+		case Opcode::FLAT_LOAD_SSHORT:
+		case Opcode::FLAT_LOAD_DWORD:
+		case Opcode::FLAT_LOAD_DWORDX2:
+		case Opcode::FLAT_LOAD_DWORDX3:
+		case Opcode::FLAT_LOAD_DWORDX4:
+		case Opcode::FLAT_STORE_BYTE:
+		case Opcode::FLAT_STORE_SHORT:
+		case Opcode::FLAT_STORE_DWORD:
+		case Opcode::FLAT_STORE_DWORDX2:
+		case Opcode::FLAT_STORE_DWORDX3:
+		case Opcode::FLAT_STORE_DWORDX4:
+		case Opcode::DS_ADD_U32:
+		case Opcode::DS_ADD_RTN_U32:
+		case Opcode::DS_SUB_U32:
+		case Opcode::DS_SUB_RTN_U32:
+		case Opcode::DS_MIN_I32:
+		case Opcode::DS_MIN_RTN_I32:
+		case Opcode::DS_MAX_I32:
+		case Opcode::DS_MAX_RTN_I32:
+		case Opcode::DS_MIN_U32:
+		case Opcode::DS_MIN_RTN_U32:
+		case Opcode::DS_MAX_U32:
+		case Opcode::DS_MAX_RTN_U32:
+		case Opcode::DS_AND_B32:
+		case Opcode::DS_AND_RTN_B32:
+		case Opcode::DS_OR_B32:
+		case Opcode::DS_OR_RTN_B32:
+		case Opcode::DS_XOR_B32:
+		case Opcode::DS_XOR_RTN_B32:
+		case Opcode::DS_WRXCHG_RTN_B32:
+		case Opcode::DS_MIN_F32:
+		case Opcode::DS_MAX_F32:
+		case Opcode::DS_SWIZZLE_B32:
+		case Opcode::DS_READ_I8:
+		case Opcode::DS_READ_U8:
+		case Opcode::DS_READ_I16:
+		case Opcode::DS_READ_U16:
+		case Opcode::DS_READ2_B32:
+		case Opcode::DS_READ_B32:
+		case Opcode::DS_READ_B64:
+		case Opcode::DS_READ2_B64:
+		case Opcode::DS_READ2ST64_B64:
+		case Opcode::DS_READ_B96:
+		case Opcode::DS_READ_B128:
+		case Opcode::DS_WRITE_B8:
+		case Opcode::DS_WRITE_B16:
+		case Opcode::DS_WRITE2_B32:
+		case Opcode::DS_WRITE2ST64_B32:
+		case Opcode::DS_WRITE2_B64:
+		case Opcode::DS_WRITE2ST64_B64:
+		case Opcode::DS_WRITE_B32:
+		case Opcode::DS_WRITE_B64:
+		case Opcode::DS_WRITE_B96:
+		case Opcode::DS_WRITE_B128:
+		case Opcode::DS_WRITE_ADDTID_B32:
+		case Opcode::DS_READ_ADDTID_B32: return WithUnsupportedReason(inst, FormatMemory(inst));
+		default: return WithUnsupportedReason(inst, FormatBinary(inst));
+	}
+}
+
+std::string ProgramToString(const Program& program) {
+	std::string text;
+	for (const auto& inst: program.instructions) {
+		text += InstructionToString(inst);
+		text += "\n";
+	}
+	return text;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Decoder

src/graphics/shader/recompiler/frontend/decode/ShaderDecoder.h

@@ -0,0 +1,712 @@
+#ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_
+#define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_
+
+#include "common/common.h"
+#include "common/magicEnum.h"
+
+#include <span>
+#include <string_view>
+#include <vector>
+
+namespace Libs::Graphics::ShaderRecompiler::Decoder {
+
+enum class Family {
+	Unknown,
+	SOP1,
+	SOP2,
+	SOPK,
+	SOPC,
+	SOPP,
+	VOP1,
+	VOP2,
+	VOP3,
+	VOP3P,
+	VOPC,
+	VINTRP,
+	SMEM,
+	MUBUF,
+	MTBUF,
+	FLAT,
+	DS,
+	MIMG,
+	EXP
+};
+
+enum class Opcode {
+	UNKNOWN,
+	UNSUPPORTED,
+
+	S_MOV_B32,
+	S_MOV_B64,
+	S_MOVK_I32,
+	S_ABS_I32,
+	S_BREV_B32,
+	S_BCNT1_I32_B32,
+	S_BCNT1_I32_B64,
+	S_FF1_I32_B32,
+	S_FF1_I32_B64,
+	S_FLBIT_I32_B64,
+	S_BITREPLICATE_B64_B32,
+	S_GETPC_B64,
+	S_SETPC_B64,
+	S_AND_SAVEEXEC_B32,
+	S_ANDN1_SAVEEXEC_B32,
+	S_AND_SAVEEXEC_B64,
+	S_ORN2_SAVEEXEC_B64,
+	S_ANDN1_SAVEEXEC_B64,
+	S_NOT_B32,
+	S_NOT_B64,
+	S_WQM_B64,
+	S_ADD_U32,
+	S_ADDC_U32,
+	S_SUBB_U32,
+	S_SUB_U32,
+	S_ADD_I32,
+	S_SUB_I32,
+	S_BITCMP0_B32,
+	S_BITCMP1_B32,
+	S_BITSET0_B32,
+	S_BITSET1_B32,
+	S_MIN_I32,
+	S_MAX_I32,
+	S_MIN_U32,
+	S_MAX_U32,
+	S_AND_B32,
+	S_AND_B64,
+	S_ANDN2_B32,
+	S_ANDN2_B64,
+	S_OR_B32,
+	S_OR_B64,
+	S_ORN2_B32,
+	S_ORN2_B64,
+	S_XOR_B32,
+	S_XOR_B64,
+	S_NAND_B32,
+	S_NAND_B64,
+	S_NOR_B32,
+	S_NOR_B64,
+	S_XNOR_B32,
+	S_XNOR_B64,
+	S_LSHL_B32,
+	S_LSHL_B64,
+	S_LSHL1_ADD_U32,
+	S_LSHL2_ADD_U32,
+	S_LSHL3_ADD_U32,
+	S_LSHL4_ADD_U32,
+	S_LSHR_B32,
+	S_LSHR_B64,
+	S_ASHR_I32,
+	S_MUL_I32,
+	S_MUL_HI_U32,
+	S_MULK_I32,
+	S_BFE_U32,
+	S_BFE_I32,
+	S_BFE_U64,
+	S_BFM_B32,
+	S_BFM_B64,
+	S_CSELECT_B32,
+	S_CSELECT_B64,
+	S_PACK_LL_B32_B16,
+	S_PACK_LH_B32_B16,
+	S_PACK_HH_B32_B16,
+	S_CMP_EQ_I32,
+	S_CMP_LG_I32,
+	S_CMP_GT_I32,
+	S_CMP_GE_I32,
+	S_CMP_LT_I32,
+	S_CMP_LE_I32,
+	S_CMP_EQ_U32,
+	S_CMP_LG_U32,
+	S_CMP_GT_U32,
+	S_CMP_GE_U32,
+	S_CMP_LT_U32,
+	S_CMP_LE_U32,
+	S_CMP_EQ_U64,
+	S_CMP_LG_U64,
+
+	V_NOP,
+	V_MOV_B32,
+	V_MOVRELD_B32,
+	V_MOVRELS_B32,
+	V_READFIRSTLANE_B32,
+	V_READLANE_B32,
+	V_WRITELANE_B32,
+	V_PERMLANE16_B32,
+	V_PERMLANEX16_B32,
+	V_CUBEID_F32,
+	V_CUBESC_F32,
+	V_CUBETC_F32,
+	V_CUBEMA_F32,
+	V_CNDMASK_B32,
+	V_DOT2C_F32_F16,
+	V_CVT_F32_I32,
+	V_CVT_F32_U32,
+	V_CVT_U32_F32,
+	V_CVT_I32_F32,
+	V_CVT_F16_F32,
+	V_CVT_F32_F16,
+	V_CVT_F16_U16,
+	V_CVT_U16_F16,
+	V_CVT_F16_I16,
+	V_CVT_I16_F16,
+	V_CVT_RPI_I32_F32,
+	V_CVT_FLR_I32_F32,
+	V_FREXP_EXP_I32_F32,
+	V_FREXP_MANT_F32,
+	V_CVT_OFF_F32_I4,
+	V_CVT_F32_UBYTE0,
+	V_CVT_F32_UBYTE1,
+	V_CVT_F32_UBYTE2,
+	V_CVT_F32_UBYTE3,
+	V_RCP_F32,
+	V_RCP_IFLAG_F32,
+	V_FRACT_F32,
+	V_TRUNC_F32,
+	V_CEIL_F32,
+	V_RNDNE_F32,
+	V_FLOOR_F32,
+	V_EXP_F32,
+	V_LOG_F32,
+	V_RSQ_F32,
+	V_SQRT_F32,
+	V_RCP_F16,
+	V_SQRT_F16,
+	V_RSQ_F16,
+	V_LOG_F16,
+	V_EXP_F16,
+	V_FLOOR_F16,
+	V_CEIL_F16,
+	V_TRUNC_F16,
+	V_RNDNE_F16,
+	V_SIN_F32,
+	V_COS_F32,
+	V_NOT_B32,
+	V_BFREV_B32,
+	V_FFBH_U32,
+	V_FFBL_B32,
+	V_ADD_F32,
+	V_SUB_F32,
+	V_SUBREV_F32,
+	V_MUL_F32,
+	V_MAC_F32,
+	V_MADMK_F32,
+	V_MADAK_F32,
+	V_MIN_F32,
+	V_MAX_F32,
+	V_ADD_F16,
+	V_SUB_F16,
+	V_SUBREV_F16,
+	V_MUL_F16,
+	V_FMAC_F16,
+	V_FMAMK_F16,
+	V_FMAAK_F16,
+	V_MIN_I32,
+	V_MAX_I32,
+	V_MUL_I32_I24,
+	V_MUL_U32_U24,
+	V_MUL_LO_U32,
+	V_MUL_HI_U32,
+	V_MUL_LO_I32,
+	V_MUL_HI_I32,
+	V_CVT_PKRTZ_F16_F32,
+	V_CVT_PK_U8_F32,
+	V_MAD_F32,
+	V_MAD_I32_I24,
+	V_MAD_U32_U24,
+	V_MAD_U64_U32,
+	V_FMA_F32,
+	V_FMA_F16,
+	V_PACK_B32_F16,
+	V_BFE_U32,
+	V_BFE_I32,
+	V_BFI_B32,
+	V_ALIGNBIT_B32,
+	V_ALIGNBYTE_B32,
+	V_MIN3_F32,
+	V_MIN3_I32,
+	V_MIN3_U32,
+	V_MIN3_F16,
+	V_MAX3_F32,
+	V_MAX3_I32,
+	V_MAX3_U32,
+	V_MAX3_F16,
+	V_MED3_F32,
+	V_MED3_I32,
+	V_MED3_U32,
+	V_MED3_F16,
+	V_MED3_I16,
+	V_SAD_U32,
+	V_ADD3_U32,
+	V_LSHL_ADD_U32,
+	V_ADD_LSHL_U32,
+	V_XAD_U32,
+	V_LSHL_OR_B32,
+	V_AND_OR_B32,
+	V_OR3_B32,
+	V_XOR3_B32,
+	V_ADD_I32,
+	V_SUB_I32,
+	V_SUBREV_I32,
+	V_BFM_B32,
+	V_LDEXP_F32,
+	V_CVT_PKNORM_I16_F32,
+	V_CVT_PKNORM_U16_F32,
+	V_CVT_PK_U16_U32,
+	V_CVT_PK_I16_I32,
+	V_PK_MAD_I16,
+	V_PK_MUL_LO_U16,
+	V_PK_ADD_I16,
+	V_PK_SUB_I16,
+	V_PK_LSHLREV_B16,
+	V_PK_LSHRREV_B16,
+	V_PK_ASHRREV_I16,
+	V_PK_MAX_I16,
+	V_PK_MIN_I16,
+	V_PK_MAD_U16,
+	V_PK_ADD_F16,
+	V_PK_MUL_F16,
+	V_PK_MIN_F16,
+	V_PK_MAX_F16,
+	V_PK_FMA_F16,
+	V_PK_FMAC_F16,
+	V_PK_ADD_U16,
+	V_PK_SUB_U16,
+	V_PK_MAX_U16,
+	V_PK_MIN_U16,
+	V_MAD_MIXLO_F16,
+	V_MAD_MIXHI_F16,
+	V_ADD_NC_U32,
+	V_ADDC_U32,
+	V_SUBREV_CO_CI_U32,
+	V_SUB_NC_U32,
+	V_SUBREV_NC_U32,
+	V_ADD_NC_U16,
+	V_SUB_NC_U16,
+	V_MAX_U16,
+	V_MAX_I16,
+	V_MIN_U16,
+	V_MIN_I16,
+	V_ADD_NC_I16,
+	V_SUB_NC_I16,
+	V_AND_B32,
+	V_OR_B32,
+	V_XOR_B32,
+	V_XNOR_B32,
+	V_LSHL_B32,
+	V_LSHLREV_B32,
+	V_LSHR_B32,
+	V_LSHRREV_B32,
+	V_ASHR_I32,
+	V_ASHRREV_I32,
+	V_LSHLREV_B16,
+	V_LSHRREV_B16,
+	V_ASHRREV_I16,
+	V_BCNT_U32_B32,
+	V_MBCNT_LO_U32_B32,
+	V_MBCNT_HI_U32_B32,
+	V_MIN_U32,
+	V_MAX_U32,
+	V_MAX_F16,
+	V_MIN_F16,
+	V_CMP_F_F32,
+	V_CMP_LT_F32,
+	V_CMP_EQ_F32,
+	V_CMP_LE_F32,
+	V_CMP_GT_F32,
+	V_CMP_LG_F32,
+	V_CMP_GE_F32,
+	V_CMP_O_F32,
+	V_CMP_U_F32,
+	V_CMP_NGE_F32,
+	V_CMP_NLG_F32,
+	V_CMP_NGT_F32,
+	V_CMP_NLE_F32,
+	V_CMP_NEQ_F32,
+	V_CMP_NLT_F32,
+	V_CMP_TRU_F32,
+	V_CMPX_LT_F32,
+	V_CMPX_EQ_F32,
+	V_CMPX_LE_F32,
+	V_CMPX_GT_F32,
+	V_CMPX_LG_F32,
+	V_CMPX_GE_F32,
+	V_CMPX_NGE_F32,
+	V_CMPX_NLG_F32,
+	V_CMPX_NGT_F32,
+	V_CMPX_NLE_F32,
+	V_CMPX_NEQ_F32,
+	V_CMPX_NLT_F32,
+	V_CMP_F_I32,
+	V_CMP_LT_I32,
+	V_CMP_EQ_I32,
+	V_CMP_LE_I32,
+	V_CMP_GT_I32,
+	V_CMP_NE_I32,
+	V_CMP_GE_I32,
+	V_CMP_T_I32,
+	V_CMP_CLASS_F32,
+	V_CMP_LT_I16,
+	V_CMP_EQ_I16,
+	V_CMP_LE_I16,
+	V_CMP_GT_I16,
+	V_CMP_NE_I16,
+	V_CMP_GE_I16,
+	V_CMP_LT_F16,
+	V_CMP_EQ_F16,
+	V_CMP_LE_F16,
+	V_CMP_GT_F16,
+	V_CMP_LG_F16,
+	V_CMP_GE_F16,
+	V_CMP_NEQ_F16,
+	V_CMPX_LT_F16,
+	V_CMPX_EQ_F16,
+	V_CMPX_LE_F16,
+	V_CMPX_GT_F16,
+	V_CMPX_GE_F16,
+	V_CMPX_NEQ_F16,
+	V_CMPX_NLT_F16,
+	V_CMPX_LT_I32,
+	V_CMPX_EQ_I32,
+	V_CMPX_LE_I32,
+	V_CMPX_GT_I32,
+	V_CMPX_NE_I32,
+	V_CMPX_GE_I32,
+	V_CMP_LT_U16,
+	V_CMP_EQ_U16,
+	V_CMP_LE_U16,
+	V_CMP_GT_U16,
+	V_CMP_NE_U16,
+	V_CMP_GE_U16,
+	V_CMP_F_U32,
+	V_CMP_LT_U32,
+	V_CMP_EQ_U32,
+	V_CMP_LE_U32,
+	V_CMP_GT_U32,
+	V_CMP_NE_U32,
+	V_CMP_GE_U32,
+	V_CMP_T_U32,
+	V_CMP_EQ_I64,
+	V_CMP_GT_U64,
+	V_CMP_NE_U64,
+	V_CMPX_LT_U32,
+	V_CMPX_EQ_U32,
+	V_CMPX_LE_U32,
+	V_CMPX_GT_U32,
+	V_CMPX_NE_U32,
+	V_CMPX_GE_U32,
+
+	S_LOAD_DWORD,
+	S_LOAD_DWORDX2,
+	S_LOAD_DWORDX4,
+	S_LOAD_DWORDX8,
+	S_LOAD_DWORDX16,
+	S_BUFFER_LOAD_DWORD,
+	S_BUFFER_LOAD_DWORDX2,
+	S_BUFFER_LOAD_DWORDX4,
+	S_BUFFER_LOAD_DWORDX8,
+	S_BUFFER_LOAD_DWORDX16,
+	BUFFER_LOAD_FORMAT_X,
+	BUFFER_LOAD_FORMAT_XY,
+	BUFFER_LOAD_FORMAT_XYZ,
+	BUFFER_LOAD_FORMAT_XYZW,
+	BUFFER_STORE_FORMAT_X,
+	BUFFER_STORE_FORMAT_XY,
+	BUFFER_STORE_FORMAT_XYZ,
+	BUFFER_STORE_FORMAT_XYZW,
+	BUFFER_LOAD_UBYTE,
+	BUFFER_LOAD_SBYTE,
+	BUFFER_LOAD_USHORT,
+	BUFFER_LOAD_SSHORT,
+	BUFFER_LOAD_DWORD,
+	BUFFER_LOAD_DWORDX2,
+	BUFFER_LOAD_DWORDX3,
+	BUFFER_LOAD_DWORDX4,
+	BUFFER_STORE_BYTE,
+	BUFFER_STORE_SHORT,
+	BUFFER_STORE_DWORD,
+	BUFFER_STORE_DWORDX2,
+	BUFFER_STORE_DWORDX3,
+	BUFFER_STORE_DWORDX4,
+	TBUFFER_LOAD_FORMAT_X,
+	TBUFFER_LOAD_FORMAT_XY,
+	TBUFFER_LOAD_FORMAT_XYZ,
+	TBUFFER_LOAD_FORMAT_XYZW,
+	TBUFFER_STORE_FORMAT_X,
+	TBUFFER_STORE_FORMAT_XY,
+	TBUFFER_STORE_FORMAT_XYZ,
+	TBUFFER_STORE_FORMAT_XYZW,
+	BUFFER_ATOMIC_SWAP,
+	BUFFER_ATOMIC_ADD,
+	BUFFER_ATOMIC_SUB,
+	BUFFER_ATOMIC_SMIN,
+	BUFFER_ATOMIC_UMIN,
+	BUFFER_ATOMIC_SMAX,
+	BUFFER_ATOMIC_UMAX,
+	BUFFER_ATOMIC_AND,
+	BUFFER_ATOMIC_OR,
+	BUFFER_ATOMIC_XOR,
+	BUFFER_ATOMIC_FMIN,
+	BUFFER_ATOMIC_FMAX,
+	FLAT_LOAD_UBYTE,
+	FLAT_LOAD_SBYTE,
+	FLAT_LOAD_USHORT,
+	FLAT_LOAD_SSHORT,
+	FLAT_LOAD_DWORD,
+	FLAT_LOAD_DWORDX2,
+	FLAT_LOAD_DWORDX3,
+	FLAT_LOAD_DWORDX4,
+	FLAT_STORE_BYTE,
+	FLAT_STORE_SHORT,
+	FLAT_STORE_DWORD,
+	FLAT_STORE_DWORDX2,
+	FLAT_STORE_DWORDX3,
+	FLAT_STORE_DWORDX4,
+	DS_ADD_U32,
+	DS_ADD_RTN_U32,
+	DS_SUB_U32,
+	DS_SUB_RTN_U32,
+	DS_MIN_I32,
+	DS_MIN_RTN_I32,
+	DS_MAX_I32,
+	DS_MAX_RTN_I32,
+	DS_MIN_U32,
+	DS_MIN_RTN_U32,
+	DS_MAX_U32,
+	DS_MAX_RTN_U32,
+	DS_AND_B32,
+	DS_AND_RTN_B32,
+	DS_OR_B32,
+	DS_OR_RTN_B32,
+	DS_XOR_B32,
+	DS_XOR_RTN_B32,
+	DS_WRXCHG_RTN_B32,
+	DS_MIN_F32,
+	DS_MAX_F32,
+	DS_SWIZZLE_B32,
+	DS_CONSUME,
+	DS_APPEND,
+	DS_READ_I8,
+	DS_READ_U8,
+	DS_READ_I16,
+	DS_READ_U16,
+	DS_READ2_B32,
+	DS_READ_B32,
+	DS_READ_B64,
+	DS_READ2_B64,
+	DS_READ2ST64_B64,
+	DS_READ_B96,
+	DS_READ_B128,
+	DS_WRITE_B8,
+	DS_WRITE_B16,
+	DS_WRITE2_B32,
+	DS_WRITE2ST64_B32,
+	DS_WRITE2_B64,
+	DS_WRITE2ST64_B64,
+	DS_WRITE_B32,
+	DS_WRITE_B64,
+	DS_WRITE_B96,
+	DS_WRITE_B128,
+	DS_WRITE_ADDTID_B32,
+	DS_READ_ADDTID_B32,
+	IMAGE_GET_RESINFO,
+	IMAGE_GET_LOD,
+	IMAGE_LOAD,
+	IMAGE_LOAD_MIP,
+	IMAGE_STORE,
+	IMAGE_STORE_MIP,
+	IMAGE_ATOMIC_ADD,
+	IMAGE_ATOMIC_UMIN,
+	IMAGE_ATOMIC_UMAX,
+	IMAGE_ATOMIC_AND,
+	IMAGE_ATOMIC_OR,
+	IMAGE_ATOMIC_XOR,
+	IMAGE_SAMPLE,
+	IMAGE_GATHER4_LZ,
+	IMAGE_GATHER4_C,
+	IMAGE_GATHER4_C_LZ,
+	IMAGE_GATHER4_LZ_O,
+	IMAGE_GATHER4_C_O,
+	IMAGE_GATHER4_C_LZ_O,
+	IMAGE_GATHER4H,
+	V_INTERP_P1_F32,
+	V_INTERP_P2_F32,
+	V_INTERP_MOV_F32,
+
+	S_NOP,
+	S_WAITCNT,
+	S_WAITCNT_DEPCTR,
+	S_BARRIER,
+	S_BRANCH,
+	S_CBRANCH_SCC0,
+	S_CBRANCH_SCC1,
+	S_CBRANCH_VCCZ,
+	S_CBRANCH_VCCNZ,
+	S_CBRANCH_EXECZ,
+	S_CBRANCH_EXECNZ,
+	S_SENDMSG,
+	S_SETREG_B32,
+	S_SLEEP,
+	S_TRAP,
+	S_TTRACEDATA,
+	S_INST_PREFETCH,
+	S_ENDPGM,
+	EXP,
+	COUNT
+};
+
+enum class OperandKind {
+	Unknown,
+	LiteralConstant,
+	IntegerInlineConstant,
+	FloatInlineConstant,
+	Sgpr,
+	VccLo,
+	VccHi,
+	VccZ,
+	ExecLo,
+	ExecHi,
+	ExecZ,
+	Scc,
+	M0,
+	PopsExitingWaveId,
+	Null,
+	Vgpr,
+};
+
+enum ImageSampleFlag : uint32_t {
+	ImageSampleFlagLod              = 1u << 0u,
+	ImageSampleFlagBias             = 1u << 1u,
+	ImageSampleFlagDerivative       = 1u << 2u,
+	ImageSampleFlagCompare          = 1u << 3u,
+	ImageSampleFlagOffset           = 1u << 4u,
+	ImageSampleFlagLevelZero        = 1u << 5u,
+	ImageSampleFlagLodClamp         = 1u << 6u,
+	ImageSampleFlagA16              = 1u << 7u,
+	ImageSampleFlagCd               = 1u << 8u,
+	ImageSampleFlagGatherHorizontal = 1u << 9u,
+};
+
+enum class ImageDimension : uint32_t {
+	Unknown,
+	Dim1D,
+	Dim1DArray,
+	Dim2D,
+	Dim3D,
+	Dim2DArray,
+	Dim2DMsaa,
+	Dim2DMsaaArray,
+};
+
+constexpr uint32_t MaxInstructionRawWords       = 5u;
+constexpr uint32_t MaxImageNsaAddressComponents = 12u;
+
+struct Operand {
+	OperandKind kind       = OperandKind::Unknown;
+	uint32_t    value      = 0;
+	int32_t     signed_val = 0;
+	float       float_val  = 0.0f;
+	uint32_t    reg        = 0;
+	uint32_t    sdwa_sel   = 6;
+	// Native 16-bit destinations use the same selector fields internally but preserve the
+	// untouched half. Real SDWA encodings overwrite this with their decoded DST_U value.
+	uint32_t sdwa_dst_unused    = 2;
+	uint32_t omod               = 0;
+	uint32_t dpp_ctrl           = 0;
+	uint32_t dpp_row_mask       = 0xf;
+	uint32_t dpp_bank_mask      = 0xf;
+	bool     sdwa_sext          = false;
+	bool     dpp_fetch_inactive = false;
+	bool     dpp_bound_ctrl     = false;
+	bool     op_sel             = false;
+	bool     op_sel_hi          = false;
+	bool     negate             = false;
+	bool     negate_hi          = false;
+	bool     absolute           = false;
+	bool     clamp              = false;
+	bool     dpp                = false;
+};
+
+struct Instruction {
+	uint32_t       pc                          = 0;
+	uint32_t       word                        = 0;
+	uint32_t       word_count                  = 1;
+	uint32_t       raw[MaxInstructionRawWords] = {};
+	uint32_t       raw_count                   = 1;
+	Family         family                      = Family::Unknown;
+	uint32_t       opcode_id                   = 0;
+	Opcode         opcode                      = Opcode::UNKNOWN;
+	Operand        dst;
+	Operand        dst2;
+	Operand        src0;
+	Operand        src1;
+	Operand        src2;
+	Operand        src3;
+	uint32_t       src_count                                    = 0;
+	uint32_t       offset                                       = 0;
+	uint32_t       secondary_offset                             = 0;
+	uint32_t       dmask                                        = 0;
+	uint32_t       data_dwords                                  = 1;
+	uint32_t       data_bits                                    = 32;
+	uint32_t       data_format                                  = 0;
+	uint32_t       number_format                                = 0;
+	uint32_t       image_sample_flags                           = 0;
+	ImageDimension image_dimension                              = ImageDimension::Unknown;
+	uint32_t       image_address_components                     = 0;
+	uint32_t       image_nsa_dwords                             = 0;
+	uint32_t       image_nsa_addr[MaxImageNsaAddressComponents] = {};
+	uint32_t       memory_segment                               = 0;
+	bool           data_signed                                  = false;
+	bool           typed                                        = false;
+	bool           formatted                                    = false;
+	bool           gds                                          = false;
+	bool           glc                                          = false;
+	bool           slc                                          = false;
+	bool           idxen                                        = false;
+	bool           offen                                        = false;
+	int32_t        branch_offset                                = 0;
+	uint32_t       branch_target                                = 0;
+	struct {
+		uint32_t target = 0;
+		uint32_t en     = 0;
+		bool     done   = false;
+		bool     compr  = false;
+		bool     vm     = false;
+	} exp;
+	std::string_view unsupported_reason;
+};
+
+struct Program {
+	std::span<const uint32_t> code;
+	std::vector<Instruction>  instructions;
+};
+
+// Code spans are trusted to contain complete instructions, valid branch targets, and 32-bit PCs.
+Family GetInstructionFamily(uint32_t word);
+// The output object must be freshly initialized.
+bool DecodeInstruction(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst,
+                       std::string* error);
+bool DecodeProgram(std::span<const uint32_t> code, Program& program, std::string* error);
+
+bool DecodeScalarSource(uint32_t code, uint32_t pc, Operand& operand, std::string* error);
+bool DecodeScalarDestination(uint32_t code, uint32_t pc, Operand& operand, std::string* error);
+void DecodeVectorGpr(uint32_t reg, Operand& operand);
+void ReadLiteralOperands(std::span<const uint32_t> code, uint32_t word_index, Instruction& inst);
+void SetRawWords(Instruction& inst, std::span<const uint32_t> code, uint32_t word_index,
+                 uint32_t word_count);
+void SetUnsupported(Instruction& inst, Family family, uint32_t opcode_id, const char* reason);
+std::string OperandToString(const Operand& operand);
+const char* ImageDimensionToString(ImageDimension dimension);
+std::string InstructionToString(const Instruction& inst);
+std::string ProgramToString(const Program& program);
+
+} // namespace Libs::Graphics::ShaderRecompiler::Decoder
+
+template <>
+struct magic_enum::customize::enum_range<Libs::Graphics::ShaderRecompiler::Decoder::Opcode> {
+	static constexpr int min =
+	    static_cast<int>(Libs::Graphics::ShaderRecompiler::Decoder::Opcode::UNKNOWN);
+	static constexpr int max =
+	    static_cast<int>(Libs::Graphics::ShaderRecompiler::Decoder::Opcode::COUNT) - 1;
+};
+
+#endif /* EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_SHADERDECODER_H_ */

src/graphics/shader/recompiler/frontend/decode/VectorAluOps.h

@@ -1,7 +1,7 @@
 #ifndef EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
 #define EMULATOR_INCLUDE_EMULATOR_GRAPHICS_SHADER_RECOMPILER_VECTORALUOPS_H_
 
-#include "graphics/shader/recompiler/decompiler/ShaderDecoder.h"
+#include "graphics/shader/recompiler/frontend/decode/ShaderDecoder.h"
 
 namespace Libs::Graphics::ShaderRecompiler::Decoder {
 
@@ -15,8 +15,8 @@ bool DecodeVop3(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index
                 std::string* error);
 bool DecodeVop3p(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
                  Instruction& inst, std::string* error);
-bool DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
-                  Instruction& inst, std::string* error);
+void DecodeVintrp(uint32_t pc, std::span<const uint32_t> code, uint32_t word_index,
+                  Instruction& inst);
 
 } // namespace Libs::Graphics::ShaderRecompiler::Decoder
 

src/graphics/shader/recompiler/frontend/translate/Attribute.cpp

@@ -0,0 +1,35 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+#include <algorithm>
+#include <array>
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+
+IR::ExportFlags Translator::AddExportInfo(const IR::Instruction& inst) {
+	const auto index = static_cast<uint32_t>(value_program.export_info.size());
+	value_program.export_info.push_back(inst.export_info);
+	return {.index = index, .pc = inst.pc};
+}
+
+bool Translator::TranslateAttributeOperation(const IR::Instruction& inst) {
+	if (inst.op == IR::Opcode::LoadInputF32) {
+		WriteOperand(inst.dst,
+		             ir.Emit(IR::ValueOpcode::GetAttribute,
+		                     {IR::Value(inst.input_info.attr), IR::Value(inst.input_info.chan)}));
+		return true;
+	}
+	if (inst.op == IR::Opcode::Export) {
+		std::array<IR::Value, 4> components {IR::Value(0u), IR::Value(0u), IR::Value(0u),
+		                                     IR::Value(0u)};
+		for (uint32_t index = 0; index < std::min(inst.src_count, 4u); index++) {
+			components[index] = ReadRawU32(PlainOperand(inst.src[index]));
+		}
+		const auto data = ir.Emit(IR::ValueOpcode::CompositeConstructU32x4,
+		                          {components[0], components[1], components[2], components[3]});
+		ir.Emit(IR::ValueOpcode::SetAttribute, {data, ir.GetExec()}, AddExportInfo(inst));
+		return true;
+	}
+	return false;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

src/graphics/shader/recompiler/frontend/translate/Compare.cpp

@@ -0,0 +1,199 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+
+bool Translator::TranslateIntegerCompare(const IR::Instruction& inst) {
+	IR::ValueOpcode opcode {};
+	IR::Type        type = IR::Type::U32;
+	switch (inst.op) {
+		case IR::Opcode::CompareFalse:
+			WriteCompareResult(inst.dst, IR::U1(IR::Value(false)));
+			return true;
+		case IR::Opcode::CompareTrue:
+			WriteCompareResult(inst.dst, IR::U1(IR::Value(true)));
+			return true;
+		case IR::Opcode::CompareEqU32:
+		case IR::Opcode::CompareMaskEqU32:
+		case IR::Opcode::CompareEqI32:
+		case IR::Opcode::CompareMaskEqI32: opcode = IR::ValueOpcode::IEqual32; break;
+		case IR::Opcode::CompareNeU32:
+		case IR::Opcode::CompareMaskNeU32:
+		case IR::Opcode::CompareNeI32:
+		case IR::Opcode::CompareMaskNeI32: opcode = IR::ValueOpcode::INotEqual32; break;
+		case IR::Opcode::CompareGtU32:
+		case IR::Opcode::CompareMaskGtU32: opcode = IR::ValueOpcode::UGreaterThan32; break;
+		case IR::Opcode::CompareGeU32:
+		case IR::Opcode::CompareMaskGeU32: opcode = IR::ValueOpcode::UGreaterThanEqual32; break;
+		case IR::Opcode::CompareLtU32:
+		case IR::Opcode::CompareMaskLtU32: opcode = IR::ValueOpcode::ULessThan32; break;
+		case IR::Opcode::CompareLeU32:
+		case IR::Opcode::CompareMaskLeU32: opcode = IR::ValueOpcode::ULessThanEqual32; break;
+		case IR::Opcode::CompareGtI32:
+		case IR::Opcode::CompareMaskGtI32: opcode = IR::ValueOpcode::SGreaterThan32; break;
+		case IR::Opcode::CompareGeI32:
+		case IR::Opcode::CompareMaskGeI32: opcode = IR::ValueOpcode::SGreaterThanEqual32; break;
+		case IR::Opcode::CompareLtI32:
+		case IR::Opcode::CompareMaskLtI32: opcode = IR::ValueOpcode::SLessThan32; break;
+		case IR::Opcode::CompareLeI32:
+		case IR::Opcode::CompareMaskLeI32: opcode = IR::ValueOpcode::SLessThanEqual32; break;
+		case IR::Opcode::CompareEqU64:
+			opcode = IR::ValueOpcode::IEqual64;
+			type   = IR::Type::U64;
+			break;
+		case IR::Opcode::CompareGtU64:
+			opcode = IR::ValueOpcode::UGreaterThan64;
+			type   = IR::Type::U64;
+			break;
+		case IR::Opcode::CompareNeU64:
+			opcode = IR::ValueOpcode::INotEqual64;
+			type   = IR::Type::U64;
+			break;
+		default: return false;
+	}
+	const auto lhs = ReadOperand(inst.src[0], type);
+	const auto rhs = ReadOperand(inst.src[1], type);
+	WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs})));
+	return true;
+}
+
+bool Translator::TranslateInteger16Compare(const IR::Instruction& inst) {
+	IR::ValueOpcode opcode {};
+	bool            signed_value = false;
+	switch (inst.op) {
+		case IR::Opcode::CompareEqU16:
+		case IR::Opcode::CompareEqI16: opcode = IR::ValueOpcode::IEqual32; break;
+		case IR::Opcode::CompareNeU16:
+		case IR::Opcode::CompareNeI16: opcode = IR::ValueOpcode::INotEqual32; break;
+		case IR::Opcode::CompareGtU16: opcode = IR::ValueOpcode::UGreaterThan32; break;
+		case IR::Opcode::CompareGeU16: opcode = IR::ValueOpcode::UGreaterThanEqual32; break;
+		case IR::Opcode::CompareLtU16: opcode = IR::ValueOpcode::ULessThan32; break;
+		case IR::Opcode::CompareLeU16: opcode = IR::ValueOpcode::ULessThanEqual32; break;
+		case IR::Opcode::CompareGtI16:
+			opcode       = IR::ValueOpcode::SGreaterThan32;
+			signed_value = true;
+			break;
+		case IR::Opcode::CompareGeI16:
+			opcode       = IR::ValueOpcode::SGreaterThanEqual32;
+			signed_value = true;
+			break;
+		case IR::Opcode::CompareLtI16:
+			opcode       = IR::ValueOpcode::SLessThan32;
+			signed_value = true;
+			break;
+		case IR::Opcode::CompareLeI16:
+			opcode       = IR::ValueOpcode::SLessThanEqual32;
+			signed_value = true;
+			break;
+		default: return false;
+	}
+	if (inst.op == IR::Opcode::CompareEqI16 || inst.op == IR::Opcode::CompareNeI16) {
+		signed_value = true;
+	}
+	WriteCompareResult(inst.dst,
+	                   IR::U1(ir.Emit(opcode, {ReadU16AsU32(inst.src[0], signed_value),
+	                                           ReadU16AsU32(inst.src[1], signed_value)})));
+	return true;
+}
+
+bool Translator::TranslateFloatCompare(const IR::Instruction& inst) {
+	IR::ValueOpcode opcode {};
+	bool            half = false;
+	switch (inst.op) {
+		case IR::Opcode::CompareEqF32:
+		case IR::Opcode::CompareMaskEqF32: opcode = IR::ValueOpcode::FPOrdEqual32; break;
+		case IR::Opcode::CompareNeF32:
+		case IR::Opcode::CompareMaskNeF32: opcode = IR::ValueOpcode::FPOrdNotEqual32; break;
+		case IR::Opcode::CompareGtF32:
+		case IR::Opcode::CompareMaskGtF32: opcode = IR::ValueOpcode::FPOrdGreaterThan32; break;
+		case IR::Opcode::CompareGeF32:
+		case IR::Opcode::CompareMaskGeF32: opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32; break;
+		case IR::Opcode::CompareLtF32:
+		case IR::Opcode::CompareMaskLtF32: opcode = IR::ValueOpcode::FPOrdLessThan32; break;
+		case IR::Opcode::CompareLeF32:
+		case IR::Opcode::CompareMaskLeF32: opcode = IR::ValueOpcode::FPOrdLessThanEqual32; break;
+		case IR::Opcode::CompareUnordEqF32:
+		case IR::Opcode::CompareMaskUnordEqF32: opcode = IR::ValueOpcode::FPUnordEqual32; break;
+		case IR::Opcode::CompareUnordNeF32:
+		case IR::Opcode::CompareMaskUnordNeF32: opcode = IR::ValueOpcode::FPUnordNotEqual32; break;
+		case IR::Opcode::CompareUnordGtF32:
+		case IR::Opcode::CompareMaskUnordGtF32:
+			opcode = IR::ValueOpcode::FPUnordGreaterThan32;
+			break;
+		case IR::Opcode::CompareUnordGeF32:
+		case IR::Opcode::CompareMaskUnordGeF32:
+			opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32;
+			break;
+		case IR::Opcode::CompareUnordLtF32:
+		case IR::Opcode::CompareMaskUnordLtF32: opcode = IR::ValueOpcode::FPUnordLessThan32; break;
+		case IR::Opcode::CompareUnordLeF32:
+		case IR::Opcode::CompareMaskUnordLeF32:
+			opcode = IR::ValueOpcode::FPUnordLessThanEqual32;
+			break;
+		case IR::Opcode::CompareEqF16:
+		case IR::Opcode::CompareMaskEqF16:
+			opcode = IR::ValueOpcode::FPOrdEqual32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareNeF16:
+		case IR::Opcode::CompareMaskNeF16:
+			opcode = IR::ValueOpcode::FPOrdNotEqual32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareGtF16:
+		case IR::Opcode::CompareMaskGtF16:
+			opcode = IR::ValueOpcode::FPOrdGreaterThan32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareGeF16:
+		case IR::Opcode::CompareMaskGeF16:
+			opcode = IR::ValueOpcode::FPOrdGreaterThanEqual32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareLtF16:
+		case IR::Opcode::CompareMaskLtF16:
+			opcode = IR::ValueOpcode::FPOrdLessThan32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareLeF16:
+		case IR::Opcode::CompareMaskLeF16:
+			opcode = IR::ValueOpcode::FPOrdLessThanEqual32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareUnordNeF16:
+		case IR::Opcode::CompareMaskUnordNeF16:
+			opcode = IR::ValueOpcode::FPUnordNotEqual32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareMaskUnordGeF16:
+			opcode = IR::ValueOpcode::FPUnordGreaterThanEqual32;
+			half   = true;
+			break;
+		case IR::Opcode::CompareOrderedF32:
+		case IR::Opcode::CompareUnorderedF32: {
+			const auto lhs       = IR::F32(ReadOperand(inst.src[0], IR::Type::F32));
+			const auto rhs       = IR::F32(ReadOperand(inst.src[1], IR::Type::F32));
+			const auto unordered = ir.LogicalOr(IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {lhs})),
+			                                    IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {rhs})));
+			const auto result =
+			    inst.op == IR::Opcode::CompareUnorderedF32 ? unordered : ir.LogicalNot(unordered);
+			WriteCompareResult(inst.dst, result);
+			return true;
+		}
+		case IR::Opcode::CompareClassF32: {
+			const auto value = ReadOperand(inst.src[0], IR::Type::F32);
+			const auto mask  = ReadOperand(inst.src[1], IR::Type::U32);
+			WriteCompareResult(inst.dst,
+			                   IR::U1(ir.Emit(IR::ValueOpcode::FPCmpClass32, {value, mask})));
+			return true;
+		}
+		default: return false;
+	}
+	const auto lhs =
+	    half ? IR::Value(ReadF16AsF32(inst.src[0])) : ReadOperand(inst.src[0], IR::Type::F32);
+	const auto rhs =
+	    half ? IR::Value(ReadF16AsF32(inst.src[1])) : ReadOperand(inst.src[1], IR::Type::F32);
+	WriteCompareResult(inst.dst, IR::U1(ir.Emit(opcode, {lhs, rhs})));
+	return true;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

src/graphics/shader/recompiler/frontend/translate/Control.cpp

@@ -0,0 +1,266 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+
+bool Translator::TranslateStateOperation(const IR::Instruction& inst) {
+	const auto zero = IR::U32(IR::Value(0u));
+	switch (inst.op) {
+		case IR::Opcode::SaveexecB32:
+		case IR::Opcode::SaveexecB64: {
+			const auto old_low  = ir.GetExecLo();
+			const auto old_high = ir.GetExecHi();
+			const auto src_low  = ReadRawU32(inst.src[0]);
+			const auto src_high = inst.op == IR::Opcode::SaveexecB64
+			                          ? ReadRawU32(OffsetOperand(inst.src[0], 1))
+			                          : IR::U32(IR::Value(0u));
+			const auto apply    = [&](IR::U32 old_value, IR::U32 src_value) {
+				switch (inst.saveexec_mode) {
+					case IR::SaveexecMode::And: return ir.BitwiseAnd(old_value, src_value);
+					case IR::SaveexecMode::Andn1:
+						return ir.BitwiseAnd(old_value, ir.BitwiseNot(src_value));
+					case IR::SaveexecMode::Orn2:
+						return ir.BitwiseOr(ir.BitwiseNot(old_value), src_value);
+				}
+				return old_value;
+			};
+			const auto result_low = apply(old_low, src_low);
+			const auto result_high =
+			    inst.op == IR::Opcode::SaveexecB64 ? apply(old_high, src_high) : old_high;
+			if (inst.op == IR::Opcode::SaveexecB64) {
+				WriteOperand(inst.dst, ir.PackUint2x32(old_low, old_high));
+			} else {
+				WriteOperand(inst.dst, old_low);
+			}
+			ir.SetExecLo(result_low);
+			ir.SetExecHi(result_high);
+			ir.SetExec(ThreadBit(result_low, result_high));
+			const auto nonzero = inst.op == IR::Opcode::SaveexecB64
+			                         ? ir.BitwiseOr(result_low, result_high)
+			                         : result_low;
+			ir.SetScc(ir.INotEqual(nonzero, zero));
+			return true;
+		}
+		case IR::Opcode::ScalarAddCarryU32:
+		case IR::Opcode::IAddCarryU32: {
+			const auto lhs      = ReadU32(inst.src[0]);
+			const auto rhs      = ReadU32(inst.src[1]);
+			const auto carry_in = ConditionBit(inst.src[2]);
+			const auto add0     = ir.Emit(IR::ValueOpcode::IAddCarry32, {lhs, rhs});
+			const auto partial  = ir.CompositeExtract(add0, 0);
+			const auto carry0   = ir.CompositeExtract(add0, 1);
+			const auto add1     = ir.Emit(IR::ValueOpcode::IAddCarry32, {partial, carry_in});
+			const auto result   = ir.CompositeExtract(add1, 0);
+			const auto carry1   = ir.CompositeExtract(add1, 1);
+			const auto carry = ir.INotEqual(ir.BitwiseOr(carry0, carry1), IR::U32(IR::Value(0u)));
+			WriteOperand(inst.dst, result);
+			if (inst.op == IR::Opcode::ScalarAddCarryU32) {
+				ir.SetScc(carry);
+			} else {
+				WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), carry));
+			}
+			return true;
+		}
+		case IR::Opcode::ScalarSubBorrowU32:
+		case IR::Opcode::ISubBorrowU32: {
+			const auto lhs    = ReadU32(inst.src[0]);
+			const auto rhs    = ReadU32(inst.src[1]);
+			const auto result = ir.ISub(lhs, rhs);
+			const auto borrow = ir.UGreaterThan(rhs, lhs);
+			WriteOperand(inst.dst, result);
+			if (inst.op == IR::Opcode::ScalarSubBorrowU32) {
+				ir.SetScc(borrow);
+			} else {
+				WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
+			}
+			return true;
+		}
+		case IR::Opcode::ScalarSubBorrowCarryU32:
+		case IR::Opcode::ISubBorrowCarryU32: {
+			const auto lhs       = ReadU32(inst.src[0]);
+			const auto rhs       = ReadU32(inst.src[1]);
+			const auto borrow_in = ConditionBit(inst.src[2]);
+			const auto partial   = ir.ISub(lhs, rhs);
+			const auto result    = ir.ISub(partial, borrow_in);
+			const auto borrow0   = ir.UGreaterThan(rhs, lhs);
+			const auto borrow1   = ir.UGreaterThan(borrow_in, partial);
+			WriteOperand(inst.dst, result);
+			const auto borrow = ir.LogicalOr(borrow0, borrow1);
+			if (inst.op == IR::Opcode::ScalarSubBorrowCarryU32) {
+				ir.SetScc(borrow);
+			} else {
+				WriteMask(inst.dst2, ir.LogicalAnd(ir.GetExec(), borrow));
+			}
+			return true;
+		}
+		case IR::Opcode::ScalarSignedAddOverflowI32:
+		case IR::Opcode::ScalarSignedSubOverflowI32: {
+			const auto lhs      = ReadU32(inst.src[0]);
+			const auto rhs      = ReadU32(inst.src[1]);
+			const bool subtract = inst.op == IR::Opcode::ScalarSignedSubOverflowI32;
+			const auto result   = subtract ? ir.ISub(lhs, rhs) : ir.IAdd(lhs, rhs);
+			const auto shift    = IR::U32(IR::Value(31u));
+			const auto lhs_sign = ir.ShiftRightLogical(lhs, shift);
+			const auto rhs_sign = ir.ShiftRightLogical(rhs, shift);
+			const auto out_sign = ir.ShiftRightLogical(result, shift);
+			const auto inputs =
+			    subtract ? ir.INotEqual(lhs_sign, rhs_sign) : ir.IEqual(lhs_sign, rhs_sign);
+			const auto changed = ir.INotEqual(lhs_sign, out_sign);
+			WriteOperand(inst.dst, result);
+			ir.SetScc(ir.LogicalAnd(inputs, changed));
+			return true;
+		}
+		case IR::Opcode::ScalarShiftLeftAddCarryU32: {
+			const auto lhs           = ReadU32(inst.src[0]);
+			const auto shift         = ReadU32(inst.src[1]);
+			const auto rhs           = ReadU32(inst.src[2]);
+			const auto shifted       = ir.ShiftLeftLogical(lhs, shift);
+			const auto result        = ir.IAdd(shifted, rhs);
+			const auto add_carry     = ir.ULessThan(result, shifted);
+			const auto inverse_shift = ir.ISub(IR::U32(IR::Value(32u)), shift);
+			const auto shifted_out   = ir.ShiftRightLogical(lhs, inverse_shift);
+			const auto shift_carry   = ir.INotEqual(shifted_out, zero);
+			WriteOperand(inst.dst, result);
+			ir.SetScc(ir.LogicalOr(add_carry, shift_carry));
+			return true;
+		}
+		default: return false;
+	}
+}
+
+bool Translator::TranslateControlOperation(const IR::Instruction& inst) {
+	switch (inst.op) {
+		case IR::Opcode::ControlNop: ir.Emit(IR::ValueOpcode::ControlNop); return true;
+		case IR::Opcode::Waitcnt: ir.Emit(IR::ValueOpcode::Waitcnt); return true;
+		case IR::Opcode::Barrier: ir.Emit(IR::ValueOpcode::Barrier); return true;
+		case IR::Opcode::Sendmsg: ir.Emit(IR::ValueOpcode::Sendmsg); return true;
+		case IR::Opcode::TtraceData: ir.Emit(IR::ValueOpcode::TtraceData); return true;
+		case IR::Opcode::InstPrefetch: ir.Emit(IR::ValueOpcode::InstPrefetch); return true;
+		default: return false;
+	}
+}
+
+bool Translator::TranslateMove(const IR::Instruction& inst) {
+	switch (inst.op) {
+		case IR::Opcode::MoveU32:
+			if (inst.src[0].kind == IR::OperandKind::Register &&
+			    (inst.src[0].reg.file == IR::RegisterFile::Exec ||
+			     inst.src[0].reg.file == IR::RegisterFile::Vcc) &&
+			    inst.dst.kind == IR::OperandKind::Register &&
+			    (inst.dst.reg.file == IR::RegisterFile::Exec ||
+			     inst.dst.reg.file == IR::RegisterFile::Vcc)) {
+				WriteMask(inst.dst, ReadMask(inst.src[0]));
+			} else {
+				WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U32));
+			}
+			return true;
+		case IR::Opcode::MoveF32Bits:
+			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::F32));
+			return true;
+		case IR::Opcode::MoveU64: {
+			const bool scalar_pair = inst.dst.kind == IR::OperandKind::Register &&
+			                         inst.src[0].kind == IR::OperandKind::Register &&
+			                         inst.dst.reg.file == IR::RegisterFile::Scalar &&
+			                         inst.src[0].reg.file == IR::RegisterFile::Scalar;
+			WriteOperand(inst.dst, ReadOperand(inst.src[0], IR::Type::U64));
+			if (scalar_pair) {
+				ir.SetThreadBitScalarReg(
+				    static_cast<IR::ScalarReg>(inst.dst.reg.index),
+				    ir.GetThreadBitScalarReg(static_cast<IR::ScalarReg>(inst.src[0].reg.index)));
+			}
+			return true;
+		}
+		case IR::Opcode::WqmB64: {
+			if (!current_per_invocation_masks && inst.dst.kind == IR::OperandKind::Register &&
+			    inst.dst.reg.file == IR::RegisterFile::Scalar) {
+				const auto result =
+				    ExpandWholeQuadMask(IR::U64(ReadOperand(inst.src[0], IR::Type::U64)));
+				WriteOperand(inst.dst, result);
+				ir.SetScc(IR::U1(
+				    ir.Emit(IR::ValueOpcode::INotEqual64, {result, IR::Value(uint64_t {0})})));
+				return true;
+			}
+			const auto result = IR::U1(ir.Emit(IR::ValueOpcode::WqmMask, {ReadMask(inst.src[0])}));
+			WriteMask64(inst.dst, result);
+			ir.SetScc(result);
+			return true;
+		}
+		default: return false;
+	}
+}
+
+bool Translator::TranslateLaneOperation(const IR::Instruction& inst) {
+	const auto lane_mask = IR::U32(IR::Value(current_wave_size == 32u ? 31u : 63u));
+	switch (inst.op) {
+		case IR::Opcode::MoveRelSourceU32: {
+			EXIT_IF(inst.src[0].kind != IR::OperandKind::Register ||
+			        inst.src[0].reg.file != IR::RegisterFile::Vector);
+			const auto base     = inst.src[0].reg.index;
+			const auto m0       = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu)));
+			auto       selected = ir.GetVectorReg(static_cast<IR::VectorReg>(base));
+			for (uint32_t index = base + 1u; index < current_vector_limit; index++) {
+				const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
+				selected =
+				    ir.Select(match, ir.GetVectorReg(static_cast<IR::VectorReg>(index)), selected);
+			}
+			WriteOperand(inst.dst, selected);
+			return true;
+		}
+		case IR::Opcode::MoveRelDestU32: {
+			EXIT_IF(inst.dst.kind != IR::OperandKind::Register ||
+			        inst.dst.reg.file != IR::RegisterFile::Vector);
+			const auto base  = inst.dst.reg.index;
+			const auto value = ReadU32(inst.src[0]);
+			const auto m0    = ir.BitwiseAnd(ReadU32(inst.src[1]), IR::U32(IR::Value(0xffu)));
+			for (uint32_t index = base; index < current_vector_limit; index++) {
+				const auto reg   = static_cast<IR::VectorReg>(index);
+				const auto match = ir.IEqual(m0, IR::U32(IR::Value(index - base)));
+				const auto write = ir.LogicalAnd(ir.GetExec(), match);
+				ir.SetVectorReg(reg, ir.Select(write, value, ir.GetVectorReg(reg)));
+			}
+			return true;
+		}
+		case IR::Opcode::ReadFirstLaneU32: {
+			const auto result =
+			    ir.Emit(IR::ValueOpcode::ReadFirstLane, {ReadU32(inst.src[0]), ir.GetExec()});
+			WriteOperand(inst.dst, result);
+			return true;
+		}
+		case IR::Opcode::ReadLaneU32: {
+			const auto lane = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask);
+			WriteOperand(inst.dst,
+			             ir.Emit(IR::ValueOpcode::ReadLane, {ReadU32(inst.src[0]), lane}));
+			return true;
+		}
+		case IR::Opcode::WriteLaneU32: {
+			EXIT_IF(inst.dst.kind != IR::OperandKind::Register ||
+			        inst.dst.reg.file != IR::RegisterFile::Vector);
+			const auto reg    = static_cast<IR::VectorReg>(inst.dst.reg.index);
+			const auto lane   = ir.BitwiseAnd(ReadU32(inst.src[1]), lane_mask);
+			const auto result = IR::U32(ir.Emit(
+			    IR::ValueOpcode::WriteLane, {ir.GetVectorReg(reg), ReadU32(inst.src[0]), lane}));
+			// V_WRITELANE_B32 ignores EXEC; bypass the ordinary VGPR destination path.
+			ir.SetVectorReg(reg, result);
+			return true;
+		}
+		case IR::Opcode::Permlane16B32:
+		case IR::Opcode::Permlanex16B32: {
+			const IR::PermlaneFlags flags {
+			    .x16            = inst.op == IR::Opcode::Permlanex16B32,
+			    .fetch_inactive = inst.dst.op_sel,
+			    .bound_control  = inst.dst.op_sel_hi,
+			};
+			const auto result = ir.Emit(
+			    IR::ValueOpcode::Permlane16U32,
+			    {ReadU32(inst.src[0]), ReadU32(inst.src[1]), ReadU32(inst.src[2]), ir.GetExec()},
+			    flags);
+			auto dst      = inst.dst;
+			dst.op_sel    = false;
+			dst.op_sel_hi = false;
+			WriteOperand(dst, result);
+			return true;
+		}
+		default: return false;
+	}
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

src/graphics/shader/recompiler/frontend/translate/Convert.cpp

@@ -0,0 +1,175 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+
+IR::F32 Translator::SelectF32(IR::U1 condition, IR::F32 true_value, IR::F32 false_value) {
+	return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
+}
+
+IR::U32 Translator::ConvertF32ToU32Saturated(IR::F32 value, float upper_bound, float safe_upper,
+                                             uint32_t upper_result) {
+	const auto zero = IR::F32(IR::Value::F32(0.0f));
+	const auto nan  = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value}));
+	const auto low  = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, zero}));
+	const auto high = IR::U1(
+	    ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)}));
+	const auto truncated = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value}));
+	const auto safe_low  = SelectF32(ir.LogicalOr(nan, low), zero, truncated);
+	const auto safe      = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low);
+	const auto converted = IR::U32(ir.Emit(IR::ValueOpcode::ConvertU32F32, {safe}));
+	return ir.Select(high, IR::U32(IR::Value(upper_result)), converted);
+}
+
+IR::U32 Translator::ConvertF32ToI32Saturated(IR::F32 value, float lower_bound, float upper_bound,
+                                             float safe_upper, uint32_t lower_result,
+                                             uint32_t upper_result) {
+	const auto nan = IR::U1(ir.Emit(IR::ValueOpcode::FPIsNan32, {value}));
+	const auto low = IR::U1(
+	    ir.Emit(IR::ValueOpcode::FPOrdLessThanEqual32, {value, IR::Value::F32(lower_bound)}));
+	const auto high = IR::U1(
+	    ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {value, IR::Value::F32(upper_bound)}));
+	const auto truncated    = IR::F32(ir.Emit(IR::ValueOpcode::FPTrunc32, {value}));
+	const auto safe_low     = SelectF32(low, IR::F32(IR::Value::F32(lower_bound)), truncated);
+	const auto safe_high    = SelectF32(high, IR::F32(IR::Value::F32(safe_upper)), safe_low);
+	const auto safe         = SelectF32(nan, IR::F32(IR::Value::F32(0.0f)), safe_high);
+	const auto converted    = IR::U32(ir.Emit(IR::ValueOpcode::ConvertS32F32, {safe}));
+	const auto clamped_high = ir.Select(high, IR::U32(IR::Value(upper_result)), converted);
+	const auto clamped      = ir.Select(low, IR::U32(IR::Value(lower_result)), clamped_high);
+	return ir.Select(nan, IR::U32(IR::Value(0u)), clamped);
+}
+
+bool Translator::TranslateConversion(const IR::Instruction& inst) {
+	const auto f32 = [&](uint32_t index) {
+		return IR::F32(ReadOperand(inst.src[index], IR::Type::F32));
+	};
+	const auto u32         = [&](uint32_t index) { return ReadU32(inst.src[index]); };
+	const auto convert_u32 = [&](IR::F32 value) {
+		return ConvertF32ToU32Saturated(value, 4294967296.0f, 4294967040.0f, 0xffffffffu);
+	};
+	const auto convert_i32 = [&](IR::F32 value) {
+		return ConvertF32ToI32Saturated(value, -2147483648.0f, 2147483648.0f, 2147483520.0f,
+		                                0x80000000u, 0x7fffffffu);
+	};
+
+	switch (inst.op) {
+		case IR::Opcode::ConvertByteU32ToF32: {
+			const auto index =
+			    inst.src[1].kind == IR::OperandKind::ImmediateU32 ? inst.src[1].imm & 3u : 0u;
+			const auto byte =
+			    ir.BitwiseAnd(ir.ShiftRightLogical(u32(0), IR::U32(IR::Value(index * 8u))),
+			                  IR::U32(IR::Value(0xffu)));
+			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {byte}));
+			return true;
+		}
+		case IR::Opcode::ConvertU32ToF32:
+			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32U32, {u32(0)}));
+			return true;
+		case IR::Opcode::ConvertI32ToF32:
+			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::ConvertF32S32, {u32(0)}));
+			return true;
+		case IR::Opcode::ConvertF32ToU32: WriteOperand(inst.dst, convert_u32(f32(0))); return true;
+		case IR::Opcode::ConvertF32ToI32: WriteOperand(inst.dst, convert_i32(f32(0))); return true;
+		case IR::Opcode::ConvertF32ToF16: WriteF16(inst.dst, f32(0)); return true;
+		case IR::Opcode::ConvertF16ToF32:
+			WriteOperand(inst.dst, ReadF16AsF32(inst.src[0]));
+			return true;
+		case IR::Opcode::ConvertU16ToF16: {
+			const auto value = IR::F32(
+			    ir.Emit(IR::ValueOpcode::ConvertF32U32, {ReadU16AsU32(inst.src[0], false)}));
+			WriteF16(inst.dst, value);
+			return true;
+		}
+		case IR::Opcode::ConvertI16ToF16: {
+			const auto value =
+			    IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {ReadU16AsU32(inst.src[0], true)}));
+			WriteF16(inst.dst, value);
+			return true;
+		}
+		case IR::Opcode::ConvertF16ToU16: {
+			const auto converted =
+			    ConvertF32ToU32Saturated(ReadF16AsF32(inst.src[0]), 65536.0f, 65535.0f, 0xffffu);
+			WriteU16(inst.dst, converted);
+			return true;
+		}
+		case IR::Opcode::ConvertF16ToI16: {
+			const auto converted = ConvertF32ToI32Saturated(
+			    ReadF16AsF32(inst.src[0]), -32768.0f, 32768.0f, 32767.0f, 0xffff8000u, 0x7fffu);
+			WriteU16(inst.dst, ir.BitwiseAnd(converted, IR::U32(IR::Value(0xffffu))));
+			return true;
+		}
+		case IR::Opcode::ConvertRoundPlusInfF32ToI32: {
+			const auto biased =
+			    IR::F32(ir.Emit(IR::ValueOpcode::FPAdd32, {f32(0), IR::Value::F32(0.5f)}));
+			const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {biased}));
+			WriteOperand(inst.dst, convert_i32(rounded));
+			return true;
+		}
+		case IR::Opcode::ConvertFloorF32ToI32: {
+			const auto rounded = IR::F32(ir.Emit(IR::ValueOpcode::FPFloor32, {f32(0)}));
+			WriteOperand(inst.dst, convert_i32(rounded));
+			return true;
+		}
+		case IR::Opcode::FrexpExpI32F32: {
+			const auto bits     = ir.BitCastU32(f32(0));
+			const auto exponent = IR::U32(
+			    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
+			const auto mantissa  = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu)));
+			const auto normal    = ir.ISub(exponent, IR::U32(IR::Value(126u)));
+			const auto msb       = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa}));
+			const auto subnormal = ir.ISub(msb, IR::U32(IR::Value(148u)));
+			const auto denormal  = ir.Select(ir.INotEqual(mantissa, IR::U32(IR::Value(0u))),
+			                                 subnormal, IR::U32(IR::Value(0u)));
+			const auto finite    = ir.Select(
+			    ir.INotEqual(exponent, IR::U32(IR::Value(0xffu))),
+			    ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal, denormal),
+			    IR::U32(IR::Value(0u)));
+			WriteOperand(inst.dst, finite);
+			return true;
+		}
+		case IR::Opcode::ConvertI4ToOffsetF32: {
+			const auto nibble = IR::U32(
+			    ir.Emit(IR::ValueOpcode::BitFieldSExtract, {u32(0), IR::Value(0u), IR::Value(4u)}));
+			const auto value = IR::F32(ir.Emit(IR::ValueOpcode::ConvertF32S32, {nibble}));
+			WriteOperand(inst.dst,
+			             ir.Emit(IR::ValueOpcode::FPMul32, {value, IR::Value::F32(1.0f / 16.0f)}));
+			return true;
+		}
+		case IR::Opcode::PackF32ToF16Rtz:
+			WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::PackFloat2x16Rtz, {f32(0), f32(1)}));
+			return true;
+		case IR::Opcode::PackSnorm2x16F32:
+		case IR::Opcode::PackUnorm2x16F32: {
+			const auto pair   = ir.Emit(IR::ValueOpcode::CompositeConstructF32x2, {f32(0), f32(1)});
+			const auto opcode = inst.op == IR::Opcode::PackSnorm2x16F32
+			                        ? IR::ValueOpcode::PackSnorm2x16
+			                        : IR::ValueOpcode::PackUnorm2x16;
+			WriteOperand(inst.dst, ir.Emit(opcode, {pair}));
+			return true;
+		}
+		case IR::Opcode::PackU8F32: {
+			const auto byte  = ConvertF32ToU32Saturated(f32(0), 255.0f, 255.0f, 255u);
+			const auto index = ir.BitwiseAnd(u32(1), IR::U32(IR::Value(3u)));
+			const auto shift = ir.ShiftLeftLogical(index, IR::U32(IR::Value(3u)));
+			const auto mask  = ir.ShiftLeftLogical(IR::U32(IR::Value(0xffu)), shift);
+			const auto base  = ir.BitwiseAnd(u32(2), ir.BitwiseNot(mask));
+			WriteOperand(inst.dst, ir.BitwiseOr(base, ir.ShiftLeftLogical(byte, shift)));
+			return true;
+		}
+		case IR::Opcode::PackB32F16: {
+			const auto low = ReadF16LaneBits(inst.src[0], false);
+			const auto high =
+			    ir.ShiftLeftLogical(ReadF16LaneBits(inst.src[1], false), IR::U32(IR::Value(16u)));
+			WriteOperand(inst.dst, ir.BitwiseOr(low, high));
+			return true;
+		}
+		default: return false;
+	}
+}
+
+IR::U32 Translator::PackU16Lanes(IR::U32 low, IR::U32 high) {
+	const auto mask = IR::U32(IR::Value(0xffffu));
+	return ir.BitwiseOr(ir.BitwiseAnd(low, mask),
+	                    ir.ShiftLeftLogical(ir.BitwiseAnd(high, mask), IR::U32(IR::Value(16u))));
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

src/graphics/shader/recompiler/frontend/translate/Dispatch.cpp

@@ -0,0 +1,42 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+
+bool Translator::TranslateU64MaskOperation(const IR::Instruction& inst) {
+	if (current_per_invocation_masks) {
+		return TranslatePerInvocationU64Mask(inst);
+	}
+	return inst.op == IR::Opcode::BitwiseAndU64 || inst.op == IR::Opcode::BitwiseOrU64
+	           ? TranslateSimpleInteger(inst)
+	           : TranslateComposedInteger(inst);
+}
+
+bool Translator::TranslateInstruction(const IR::Instruction&          inst,
+                                      const BufferAddressValues*      address_snapshot,
+                                      const ScalarMemorySourceValues* scalar_source_snapshot) {
+	switch (IR::GetOpcodeInfo(inst.op).lowering_class) {
+		case IR::LoweringClass::Control: return TranslateControlOperation(inst);
+		case IR::LoweringClass::Move: return TranslateMove(inst);
+		case IR::LoweringClass::Lane: return TranslateLaneOperation(inst);
+		case IR::LoweringClass::State: return TranslateStateOperation(inst);
+		case IR::LoweringClass::Memory:
+			return TranslateMemoryOperation(inst, address_snapshot, scalar_source_snapshot);
+		case IR::LoweringClass::Attribute: return TranslateAttributeOperation(inst);
+		case IR::LoweringClass::IntegerCompare: return TranslateIntegerCompare(inst);
+		case IR::LoweringClass::Integer16Compare: return TranslateInteger16Compare(inst);
+		case IR::LoweringClass::FloatCompare: return TranslateFloatCompare(inst);
+		case IR::LoweringClass::Conversion: return TranslateConversion(inst);
+		case IR::LoweringClass::Integer16: return TranslateInteger16Operation(inst);
+		case IR::LoweringClass::PackedInteger16: return TranslatePackedInteger16(inst);
+		case IR::LoweringClass::PackedFloat16: return TranslatePackedFloat16(inst);
+		case IR::LoweringClass::Float16: return TranslateFloat16Operation(inst);
+		case IR::LoweringClass::Float: return TranslateFloatOperation(inst);
+		case IR::LoweringClass::U64Mask: return TranslateU64MaskOperation(inst);
+		case IR::LoweringClass::SimpleInteger: return TranslateSimpleInteger(inst);
+		case IR::LoweringClass::ComposedInteger: return TranslateComposedInteger(inst);
+		case IR::LoweringClass::ExtendedInteger: return TranslateExtendedInteger(inst);
+	}
+	return false;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail

src/graphics/shader/recompiler/frontend/translate/Float.cpp

@@ -0,0 +1,232 @@
+#include "graphics/shader/recompiler/frontend/translate/Translator.h"
+
+#include <array>
+#include <utility>
+
+namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail {
+
+bool Translator::TranslatePackedFloat16(const IR::Instruction& inst) {
+	IR::ValueOpcode opcode;
+	switch (inst.op) {
+		case IR::Opcode::PackedAddF16: opcode = IR::ValueOpcode::FPAdd32; break;
+		case IR::Opcode::PackedMulF16: opcode = IR::ValueOpcode::FPMul32; break;
+		case IR::Opcode::PackedMinF16: opcode = IR::ValueOpcode::FPMin32; break;
+		case IR::Opcode::PackedMaxF16: opcode = IR::ValueOpcode::FPMax32; break;
+		case IR::Opcode::PackedFmaF16: opcode = IR::ValueOpcode::FPFma32; break;
+		default: return false;
+	}
+	const auto translate_lane = [&](bool high) {
+		const auto lhs = ReadF16LaneAsF32(inst.src[0], high);
+		const auto rhs = ReadF16LaneAsF32(inst.src[1], high);
+		IR::F32    result;
+		if (inst.src_count == 3u) {
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs, ReadF16LaneAsF32(inst.src[2], high)}));
+		} else {
+			result = IR::F32(ir.Emit(opcode, {lhs, rhs}));
+		}
+		return ApplyF32ResultModifiers(inst.dst, result);
+	};
+	auto raw    = inst.dst;
+	raw.omod    = 0u;
+	raw.clamp   = false;
+	auto result = PackHalf2x16(translate_lane(false), translate_lane(true));
+	if (inst.op == IR::Opcode::PackedMinF16 || inst.op == IR::Opcode::PackedMaxF16) {
+		const auto quiet_snan = [&](const IR::Operand& operand, bool high) {
+			const auto bits     = ReadU16LaneAsU32(operand, high, false);
+			const auto exponent = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x7c00u)));
+			const auto payload  = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x01ffu)));
+			const auto snan     = ir.LogicalAnd(ir.IEqual(exponent, IR::U32(IR::Value(0x7c00u))),
+			                                    ir.INotEqual(payload, IR::U32(IR::Value(0u))));
+			return std::pair {snan, ir.BitwiseOr(bits, IR::U32(IR::Value(0x0200u)))};
+		};
+		const auto override_lane = [&](bool high) {
+			const auto [lhs_snan, lhs_quiet] = quiet_snan(inst.src[0], high);
+			const auto [rhs_snan, rhs_quiet] = quiet_snan(inst.src[1], high);
+			const auto normal = high ? ir.ShiftRightLogical(result, IR::U32(IR::Value(16u)))
+			                         : ir.BitwiseAnd(result, IR::U32(IR::Value(0xffffu)));
+			return ir.Select(lhs_snan, lhs_quiet, ir.Select(rhs_snan, rhs_quiet, normal));
+		};
+		result = PackU16Lanes(override_lane(false), override_lane(true));
+	}
+	WriteOperand(raw, result);
+	return true;
+}
+
+bool Translator::TranslateFloat16Operation(const IR::Instruction& inst) {
+	IR::ValueOpcode opcode;
+	switch (inst.op) {
+		case IR::Opcode::AddF16: opcode = IR::ValueOpcode::FPAdd32; break;
+		case IR::Opcode::SubF16: opcode = IR::ValueOpcode::FPSub32; break;
+		case IR::Opcode::MulF16: opcode = IR::ValueOpcode::FPMul32; break;
+		case IR::Opcode::MinF16: opcode = IR::ValueOpcode::FPMin32; break;
+		case IR::Opcode::MaxF16: opcode = IR::ValueOpcode::FPMax32; break;
+		case IR::Opcode::FmaF16:
+		case IR::Opcode::MadMixF16: opcode = IR::ValueOpcode::FPFma32; break;
+		case IR::Opcode::RcpF16: opcode = IR::ValueOpcode::FPRecip32; break;
+		case IR::Opcode::SqrtF16: opcode = IR::ValueOpcode::FPSqrt; break;
+		case IR::Opcode::InverseSqrtF16: opcode = IR::ValueOpcode::FPRecipSqrt32; break;
+		case IR::Opcode::Log2F16: opcode = IR::ValueOpcode::FPLog2; break;
+		case IR::Opcode::Exp2F16: opcode = IR::ValueOpcode::FPExp2; break;
+		case IR::Opcode::FloorF16: opcode = IR::ValueOpcode::FPFloor32; break;
+		case IR::Opcode::CeilF16: opcode = IR::ValueOpcode::FPCeil32; break;
+		case IR::Opcode::TruncF16: opcode = IR::ValueOpcode::FPTrunc32; break;
+		case IR::Opcode::RoundEvenF16: opcode = IR::ValueOpcode::FPRoundEven32; break;
+		case IR::Opcode::Min3F16: opcode = IR::ValueOpcode::FPMinTri32; break;
+		case IR::Opcode::Max3F16: opcode = IR::ValueOpcode::FPMaxTri32; break;
+		case IR::Opcode::Med3F16: opcode = IR::ValueOpcode::FPMedTri32; break;
+		default: return false;
+	}
+	std::array<IR::Value, 3> args;
+	for (uint32_t index = 0; index < inst.src_count; index++) {
+		args[index] = inst.op == IR::Opcode::MadMixF16 ? IR::Value(ReadMixF32(inst.src[index]))
+		                                               : IR::Value(ReadF16AsF32(inst.src[index]));
+	}
+	IR::F32 result;
+	switch (inst.src_count) {
+		case 1: result = IR::F32(ir.Emit(opcode, {args[0]})); break;
+		case 2: result = IR::F32(ir.Emit(opcode, {args[0], args[1]})); break;
+		case 3: result = IR::F32(ir.Emit(opcode, {args[0], args[1], args[2]})); break;
+		default: EXIT("invalid half-float source count: %u", inst.src_count);
+	}
+	WriteF16(inst.dst, result);
+	return true;
+}
+
+bool Translator::TranslateFloatOperation(const IR::Instruction& inst) {
+	if (inst.op == IR::Opcode::FrexpMantF32) {
+		const auto bits     = ReadU32(inst.src[0]);
+		const auto exponent = IR::U32(
+		    ir.Emit(IR::ValueOpcode::BitFieldUExtract, {bits, IR::Value(23u), IR::Value(8u)}));
+		const auto mantissa = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x007fffffu)));
+		const auto sign     = ir.BitwiseAnd(bits, IR::U32(IR::Value(0x80000000u)));
+		const auto base     = ir.BitwiseOr(sign, IR::U32(IR::Value(0x3f000000u)));
+		const auto normal   = ir.BitwiseOr(base, mantissa);
+		const auto msb      = IR::U32(ir.Emit(IR::ValueOpcode::FindUMsb32, {mantissa}));
+		const auto shift    = ir.ISub(IR::U32(IR::Value(23u)), msb);
+		const auto fraction =
+		    ir.BitwiseAnd(ir.ShiftLeftLogical(mantissa, shift), IR::U32(IR::Value(0x007fffffu)));
+		const auto subnormal = ir.BitwiseOr(base, fraction);
+		const auto zero      = ir.IEqual(mantissa, IR::U32(IR::Value(0u)));
+		const auto finite    = ir.Select(ir.INotEqual(exponent, IR::U32(IR::Value(0u))), normal,
+		                                 ir.Select(zero, bits, subnormal));
+		const auto result = ir.Select(ir.IEqual(exponent, IR::U32(IR::Value(0xffu))), bits, finite);
+		WriteOperand(inst.dst, ir.BitCastF32(result));
+		return true;
+	}
+	IR::ValueOpcode opcode {};
+	switch (inst.op) {
+		case IR::Opcode::RcpF32: opcode = IR::ValueOpcode::FPRecip32; break;
+		case IR::Opcode::RcpIflagF32: opcode = IR::ValueOpcode::FPRecipIFlag32; break;
+		case IR::Opcode::FractF32: opcode = IR::ValueOpcode::FPFract32; break;
+		case IR::Opcode::TruncF32: opcode = IR::ValueOpcode::FPTrunc32; break;
+		case IR::Opcode::CeilF32: opcode = IR::ValueOpcode::FPCeil32; break;
+		case IR::Opcode::RoundEvenF32: opcode = IR::ValueOpcode::FPRoundEven32; break;
+		case IR::Opcode::FloorF32: opcode = IR::ValueOpcode::FPFloor32; break;
+		case IR::Opcode::Exp2F32: opcode = IR::ValueOpcode::FPExp2; break;
+		case IR::Opcode::Log2F32: opcode = IR::ValueOpcode::FPLog2; break;
+		case IR::Opcode::InverseSqrtF32: opcode = IR::ValueOpcode::FPRecipSqrt32; break;
+		case IR::Opcode::SqrtF32: opcode = IR::ValueOpcode::FPSqrt; break;
+		case IR::Opcode::SinF32: opcode = IR::ValueOpcode::FPSin; break;
+		case IR::Opcode::CosF32: opcode = IR::ValueOpcode::FPCos; break;
+		case IR::Opcode::FAddF32: opcode = IR::ValueOpcode::FPAdd32; break;
+		case IR::Opcode::FSubF32: opcode = IR::ValueOpcode::FPSub32; break;
+		case IR::Opcode::FMulF32: opcode = IR::ValueOpcode::FPMul32; break;
+		case IR::Opcode::FMinF32: opcode = IR::ValueOpcode::FPMin32; break;
+		case IR::Opcode::FMaxF32: opcode = IR::ValueOpcode::FPMax32; break;
+		case IR::Opcode::FMadF32: opcode = IR::ValueOpcode::FPFma32; break;
+		case IR::Opcode::FMin3F32: opcode = IR::ValueOpcode::FPMinTri32; break;
+		case IR::Opcode::FMax3F32: opcode = IR::ValueOpcode::FPMaxTri32; break;
+		case IR::Opcode::FMed3F32: opcode = IR::ValueOpcode::FPMedTri32; break;
+		case IR::Opcode::LdexpF32: opcode = IR::ValueOpcode::FPLdexp; break;
+		default: break;
+	}
+	if (opcode != IR::ValueOpcode {}) {
+		std::array<IR::Value, 3> args;
+		for (uint32_t index = 0; index < inst.src_count; index++) {
+			const auto arg_type = IR::ArgTypeOf(opcode, index);
+			args[index] =
+			    arg_type == IR::Type::F32
+			        ? (inst.op == IR::Opcode::FMadF32 ? IR::Value(ReadMixF32(inst.src[index]))
+			                                          : ReadOperand(inst.src[index], IR::Type::F32))
+			        : ReadOperand(inst.src[index], arg_type);
+		}
+		IR::Value result;
+		switch (inst.src_count) {
+			case 1: result = ir.Emit(opcode, {args[0]}); break;
+			case 2: result = ir.Emit(opcode, {args[0], args[1]}); break;
+			case 3: result = ir.Emit(opcode, {args[0], args[1], args[2]}); break;
+			default: EXIT("invalid floating-point source count: %u", inst.src_count);
+		}
+		WriteOperand(inst.dst, result);
+		return true;
+	}
+
+	const auto select_f32 = [&](IR::U1 condition, IR::F32 true_value, IR::F32 false_value) {
+		return IR::F32(ir.Emit(IR::ValueOpcode::SelectF32, {condition, true_value, false_value}));
+	};
+	if (inst.op == IR::Opcode::Dot2AccF32F16) {
+		auto a          = inst.src[0];
+		a.op_sel        = false;
+		a.op_sel_hi     = true;
+		auto b          = inst.src[1];
+		b.op_sel        = false;
+		b.op_sel_hi     = true;
+		const auto a_lo = ReadF16LaneAsF32(a, false);
+		const auto a_hi = ReadF16LaneAsF32(a, true);
+		const auto b_lo = ReadF16LaneAsF32(b, false);
+		const auto b_hi = ReadF16LaneAsF32(b, true);
+		const auto acc  = ReadMixF32(inst.src[2]);
+		const auto lo   = IR::F32(ir.Emit(IR::ValueOpcode::FPFma32, {a_lo, b_lo, acc}));
+		WriteOperand(inst.dst, ir.Emit(IR::ValueOpcode::FPFma32, {a_hi, b_hi, lo}));
+		return true;
+	}
+	if (inst.op != IR::Opcode::CubeIdF32 && inst.op != IR::Opcode::CubeScF32 &&
+	    inst.op != IR::Opcode::CubeTcF32 && inst.op != IR::Opcode::CubeMaF32) {
+		return false;
+	}
+
+	const auto x  = ReadMixF32(inst.src[0]);
+	const auto y  = ReadMixF32(inst.src[1]);
+	const auto z  = ReadMixF32(inst.src[2]);
+	const auto nx = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {x}));
+	const auto ny = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {y}));
+	const auto nz = IR::F32(ir.Emit(IR::ValueOpcode::FPNeg32, {z}));
+	const auto ax = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {x}));
+	const auto ay = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {y}));
+	const auto az = IR::F32(ir.Emit(IR::ValueOpcode::FPAbs32, {z}));
+	const auto z_face =
+	    ir.LogicalAnd(IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {az, ax})),
+	                  IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {az, ay})));
+	const auto y_face = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdGreaterThanEqual32, {ay, ax}));
+	const auto x_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {x, IR::Value::F32(0.0f)}));
+	const auto y_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {y, IR::Value::F32(0.0f)}));
+	const auto z_neg = IR::U1(ir.Emit(IR::ValueOpcode::FPOrdLessThan32, {z, IR::Value::F32(0.0f)}));
+	const auto select_face = [&](IR::F32 x_value, IR::F32 y_value, IR::F32 z_value) {
+		return select_f32(z_face, z_value, select_f32(y_face, y_value, x_value));
+	};
+	IR::F32 result;
+	switch (inst.op) {
+		case IR::Opcode::CubeIdF32:
+			result = select_face(
+			    select_f32(x_neg, IR::F32(IR::Value::F32(1.0f)), IR::F32(IR::Value::F32(0.0f))),
+			    select_f32(y_neg, IR::F32(IR::Value::F32(3.0f)), IR::F32(IR::Value::F32(2.0f))),
+			    select_f32(z_neg, IR::F32(IR::Value::F32(5.0f)), IR::F32(IR::Value::F32(4.0f))));
+			break;
+		case IR::Opcode::CubeScF32:
+			result = select_face(select_f32(x_neg, z, nz), x, select_f32(z_neg, nx, x));
+			break;
+		case IR::Opcode::CubeTcF32: result = select_face(ny, select_f32(y_neg, nz, z), ny); break;
+		case IR::Opcode::CubeMaF32: {
+			const auto two = IR::F32(IR::Value::F32(2.0f));
+			result         = select_face(IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {x, two})),
+			                             IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {y, two})),
+			                             IR::F32(ir.Emit(IR::ValueOpcode::FPMul32, {z, two})));
+			break;
+		}
+		default: return false;
+	}
+	WriteOperand(inst.dst, result);
+	return true;
+}
+
+} // namespace Libs::Graphics::ShaderRecompiler::Frontend::Detail