Files
NativeMetal/NativeAdrenoMetalPlugin.mm
2026-11-16 10:03:58 +08:00

846 lines
53 KiB
Plaintext

// SPDX-License-Identifier: MIT
// Adreno Metal plugin, reached through the original discovery (IOAcceleratorES -> MetalPluginName -> NSBundle -> principal class ->
// initWithAcceleratorPort:). Default (stage 1): it runs the shared userspace GPU test (NativeAdrenoUserTest.h) from inside a real system
// process, under that process's sandbox, and then returns nil. No device is registered, so the original software rendering keeps running unchanged.
// Published (the image carries the file NativeAdrenoMetal.publish beside the bundle): backboardd, the one process that owns the GPU, returns the native
// Metal frontend's device (the metal2vulkan runtime on Turnip, on the Adreno 740), which the original discovery registers; the CARenderServer then
// renders the system UI through it, and the display driver's frames are written to the scanout by the GPU (GpuScanoutClient.h). Another process
// gets its own native device too (its own driver connection, GPU address space and shader cache), as every process does with Apple's
// own GPU plugin.
#import <Foundation/Foundation.h>
#include <string>
#include <sys/stat.h>
#include <dirent.h>
#include <sys/mount.h>
#include <os/log.h>
#include "NativeRenderServer.h"
#include "SharedShaderCache.h"
#include <pthread.h>
#include <thread>
#include <dlfcn.h>
#if __has_feature(ptrauth_calls)
#include <ptrauth.h>
#endif
#include <unistd.h>
#include <pwd.h>
#include <fts.h>
#include <mach/mach.h>
#include <sys/sysctl.h>
#include <libkern/OSCacheControl.h>
#include <objc/message.h>
#include "NativeAdrenoUserTest.h"
#include "NativeTurnipSwapTest.h"
#include "NativeGpuTrace.h"
#include "NativeMetalBridgePhase.h"
#include "CacheExport.h"
#if !TARGET_OS_OSX
#include "PrimeShaders.h"
#endif
#include "ExceptionDiag.h"
#if TARGET_OS_OSX
#include "IOGPUMetalDeviceShim.h"
// macOS: +[IOGPUMetalDevice registerAcceleratorService:] takes the bundle's MetalPluginClassName only if it subclasses IOGPUMetalDevice.
@interface IOGPUMetalDevice : NSObject
@end
#else
@interface _MTLDevice : NSObject
@end
#endif
@interface NativeAdrenoMetalDeviceBase : NSObject // only a lock object for the published mode's one-time setup
@end
@implementation NativeAdrenoMetalDeviceBase
@end
#if TARGET_OS_OSX
@interface NativeAdrenoMetalDevice : IOGPUMetalDevice
#else
@interface NativeAdrenoMetalDevice : _MTLDevice
#endif
- (id)initWithAcceleratorPort:(unsigned)port;
@end
// ---- published mode ----
namespace publish {
// The image's marker: a file beside the bundle in the union view of /System/Library/Extensions (the bundle's own seal stays untouched).
// A file beside the bundle: its path in path, true when it is readable.
static bool requestedPath(const char *name, char *path, size_t size) {
Dl_info info = {};
if (!dladdr(reinterpret_cast<const void *>(&requestedPath), &info) || !info.dli_fname) return false;
snprintf(path, size, "%s", info.dli_fname);
// The bundle's executable: <dir>/NativeAdrenoMetal.bundle/NativeAdrenoMetal (iOS) or
// <dir>/NativeAdrenoMetal.bundle/Contents/MacOS/NativeAdrenoMetal (macOS).
char *bundle = strstr(path, "/NativeAdrenoMetal.bundle/");
if (!bundle) return false;
snprintf(bundle + 1, size - size_t(bundle + 1 - path), "%s", name);
return access(path, R_OK) == 0;
}
static bool requested() {
char path[1024];
return requestedPath("NativeAdrenoMetal.publish", path, sizeof path);
}
static Gpu *makeGpu(unsigned port) {
void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL);
if (!iokit) return nullptr;
Gpu *gpu = new Gpu;
gpu->call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod");
gpu->map = (MapFn)dlsym(iokit, "IOConnectMapMemory64");
gpu->unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64");
gpu->openFn = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen");
gpu->closeFn = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose");
gpu->port = port;
return gpu;
}
static std::string gCacheDirectory; // the runtime's cache directory (the one that opened the device), exported by CacheExport.h
static bool gFillsSystemCache; // the render server opened its device in the system shader cache's directory (SharedShaderCache.h)
// Progress markers 0x68xx (GPU6): 6800 entered, 6801 Turnip found, 6802 device made, 6803 returned to the discovery, 68f1 no Turnip, 68f2 no device.
static unsigned gDiagBits;
// Reads of the render server's own state for the framebuffer-format question (diagnostic only, for the exact QuartzCore of the target image; a read
// that fails or a build that differs reports zeros). Each value is one marker, 0x7Kvv: K picks the item, vv the value.
static bool readTask(uintptr_t address, void *out, size_t size) {
vm_size_t got = 0;
return vm_read_overwrite(mach_task_self(), address, size, (vm_address_t) out, &got) == KERN_SUCCESS && got == size;
}
[[maybe_unused]] static bool writeTask(uintptr_t address, const void *data, size_t size) { // iPadOS only (24A446 QuartzCore)
return vm_write(mach_task_self(), address, (vm_offset_t) data, (mach_msg_type_number_t) size) == KERN_SUCCESS;
}
// EXPERIMENT (second report onward): this display path takes plain 8-bit BGRA only, so tell the render server's display that it has neither the deep
// (10-bit) framebuffer nor universal buffer compression. Whether the state it reads those from is the whole story is what the markers show.
// EXPERIMENT (boot-arg fb-bgra=1): the render server's framebuffer format with every other virtualized answer left as it is.
// framebuffer_pixel_format (24A446 0x184461838) takes RGhA on `tbnz w8, #0` at 0x184461928 when CADeviceIsVirtualized (byte 0x1e701bdd9) is 1;
// 40 other places in QuartzCore read the same byte (XR formats, backdrop compute, ASTC ...). Holding the byte at 0 turned all of them, and the
// tablet drew light blue blocks. Here the byte stays 1 and only this branch is made a NOP, so the framebuffer is chosen as on a real device and
// nothing else changes. Marks 0x6E40 unreadable, 0x6E41 other instruction, 0x6E42 patched, 0x6E50|stage failures.
// The boot argument token that starts with prefix (a whole name, or name=value), or nullptr: a name is matched only at the start of the
// command line or after a space, so "log=1" does not find "cache-log=1".
static const char *bootArgToken(const char *args, const char *prefix) {
const size_t length = strlen(prefix);
for (const char *at = args; (at = strstr(at, prefix)); at += length)
if (at == args || at[-1] == ' ') return at;
return nullptr;
}
static bool bootArgSet(const char *key) {
char args[1100] = {};
size_t size = sizeof args - 1;
if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) != 0) return false;
const char *at = bootArgToken(args, key);
const size_t length = strlen(key);
return at && (at[length] == 0 || at[length] == ' ');
}
// `site` is the slid address of that branch in the running QuartzCore; any other word there is left alone (0x6E41).
[[maybe_unused]] static unsigned patchFramebufferBranch(uintptr_t site) {
uint32_t word = 0;
if (!readTask(site, &word, sizeof word)) return 0x6E40;
if (word == 0xd503201f) return 0x6E42;
if (word != 0x37000608) return 0x6E41;
const vm_address_t page = site & ~(uintptr_t)(vm_page_size - 1);
if (vm_protect(mach_task_self(), page, vm_page_size, FALSE, VM_PROT_READ | VM_PROT_WRITE | VM_PROT_COPY) != KERN_SUCCESS) return 0x6E51;
const uint32_t nop = 0xd503201f;
const bool wrote = vm_write(mach_task_self(), site, (vm_offset_t)&nop, sizeof nop) == KERN_SUCCESS;
const bool restored = vm_protect(mach_task_self(), page, vm_page_size, FALSE, VM_PROT_READ | VM_PROT_EXECUTE) == KERN_SUCCESS;
sys_icache_invalidate((void *)site, sizeof nop);
if (!wrote) return 0x6E52;
if (!restored) return 0x6E53;
return 0x6E42;
}
// Every process other than the render server: the same runtime on its own connection, with none of the render server's threads (no
// display hand-off, telemetry, benchmark or cache export). Its shader cache lives in its own cache or temp directory.
// The reason a client's device failed, as a text line the driver keeps per process (the render server's report row is rewritten every ten seconds):
// "<step> <detail>", through the marker stream (a connection is not needed). The Gpu object is made only on this path.
// Deletes every "*.import" file under `root`: the copying import of an earlier image wrote each file beside its place and renamed it only
// once the whole pack was read; a boot that ended first left them all there.
static void removeHalfImported(const char *root) {
char *paths[] = {const_cast<char *>(root), nullptr};
FTS *fts = fts_open(paths, FTS_PHYSICAL | FTS_NOCHDIR | FTS_XDEV, nullptr);
if (!fts) return;
while (FTSENT *e = fts_read(fts))
if (e->fts_info == FTS_F && e->fts_namelen > 7 && !strcmp(e->fts_name + e->fts_namelen - 7, ".import")) unlink(e->fts_path);
fts_close(fts);
}
// diag=1 in the render server: the bridge's telemetry table (native_bridge_metal_telemetry) to the kernel's board log every six
// seconds and to the system log every thirty. The kernel keeps 48 rows of at most 95 characters (its [metal table] section): the
// marker 0xC000 | row starts a row, 0x8000 | a << 7 | b appends two characters. A row that reads as it did when last sent is not
// sent again, and a row the table no longer has is cleared. The system log has every row, past the 48th too.
static void *streamTelemetry(void *arg) {
Gpu *g = static_cast<Gpu *>(arg);
pthread_setname_np("native-metal-telemetry");
pthread_set_qos_class_self_np(QOS_CLASS_UTILITY, 0); // a row is up to 46 marker calls: behind the render server's own work
sleep(5);
static char table[8192], sent[48][92];
for (unsigned pass = 0;; ++pass) {
native_bridge_metal_telemetry(table, sizeof table);
unsigned row = 0;
for (char *line = table; *line; ++row) {
char *end = strchr(line, '\n');
if (end) *end = 0;
if (pass % 5 == 0) os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal table: %{public}s", line);
if (row < 48 && strncmp(sent[row], line, sizeof sent[row] - 1)) {
strlcpy(sent[row], line, sizeof sent[row]);
g->mark(0xC000u | row);
const size_t length = strlen(sent[row]);
for (size_t i = 0; i < length; i += 2) {
const unsigned a = (unsigned char)sent[row][i] & 0x7f, b = i + 1 < length ? (unsigned char)sent[row][i + 1] & 0x7f : 0;
g->mark(0x8000u | (a << 7) | b);
}
}
if (!end) {++row; break;}
line = end + 1;
}
for (; row < 48; ++row)
if (sent[row][0]) {sent[row][0] = 0; g->mark(0xC000u | row);}
sleep(6);
}
return nullptr;
}
static unsigned gClientPort;
static void clientFailure(unsigned port, const char *step, const char *detail) {
if (Gpu *gpu = makeGpu(port)) {
// The row holds 63 characters: the common prefix goes, and a long reason keeps its start (what was refused) and its end (why).
if (!detail) detail = "";
if (!strncmp(detail, "native-metal: ", 14)) detail += 14;
char shown[48];
const size_t length = strlen(detail);
if (length <= 46) strlcpy(shown, detail, sizeof shown);
else snprintf(shown, sizeof shown, "%.18s~%s", detail, detail + length - 27);
char text[64];
snprintf(text, sizeof text, "%.14s %s", step, shown);
streamText(*gpu, text);
// A refusal's text is what says why a client ends (iconservicesagent's pipeline refusal showed 18 characters of it): the rest of it
// follows in rows of its own, up to six, each 46 characters.
if (!strcmp(step, "problem") && length > 46)
for (unsigned part = 1, at = 46; part <= 6 && at < length; ++part, at += 46) {
// The log keeps the latest text row only: each part follows the one before it by 4 s so a sample catches every one.
std::string row = "problem" + std::to_string(part) + " " + std::string(detail + at).substr(0, 46);
auto *target = gpu;
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, int64_t(part) * 4 * NSEC_PER_SEC),
dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{streamText(*target, row.c_str());});
}
}
}
static id clientDevice(unsigned port) {
char turnip[1024];
if (!turnip::libraryPath(turnip, sizeof turnip) || access(turnip, R_OK) != 0) {clientFailure(port, "no-turnip", ""); return nil;}
uint64_t registryID = 0;
if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) {
if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port);
if (auto entryID = (int (*)(unsigned, uint64_t *))dlsym(iokit, "IORegistryEntryGetRegistryEntryID")) entryID(port, &registryID);
}
// IOSurfaces this GPU reads or writes must be plain rows, as in the render server.
CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
#if TARGET_OS_OSX
if (geteuid() == 0 && getenv("NATIVE_METAL_PRIMER")) {
const std::string root = systemcache::writerDirectory();
if (root.empty()) return nil;
native_bridge_metal_system_cache(nullptr, nullptr, 0, true);
return native_bridge_metal_open(turnip, root.c_str(), registryID, nullptr);
}
#endif
char candidates[3][512] = {};
if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0;
if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0;
strlcpy(candidates[2], sharedShaderCacheDirectory().c_str(), sizeof candidates[2]);
id device = nil;
const char *lastError = "";
unsigned usable = 0;
for (unsigned i : {2u, 0u, 1u}) {
if (device) break;
if (!candidates[i][0]) continue;
// In the user cache directory the cache sits inside Metal's own shader cache directory (com.apple.metal): a sandboxed process may
// write there whenever its profile lets Metal cache shaders at all (macOS iconservicesagent: its profile allows METAL_CACHE_PATH
// and denied every write to <cache>/m2v, so each of its launches compiled every pipeline again and it was killed at its memory
// limit within seconds).
if (i == 0) {strlcat(candidates[i], "com.apple.metal", sizeof candidates[i]); mkdir(candidates[i], 0700); strlcat(candidates[i], "/", sizeof candidates[i]);}
if (i != 2) strlcat(candidates[i], "m2v", sizeof candidates[i]);
mkdir(candidates[i], 0700);
if (access(candidates[i], W_OK) != 0) continue;
++usable;
@try {
device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr);
} @catch (NSException *exception) {
device = nil;
lastError = "exception";
}
if (!device && !lastError[0]) lastError = nmb_last_error();
}
if (!device && !usable) {
setenv("METAL2VULKAN_SHADER_CACHE_OPTIONAL", "1", 1);
const char *path = candidates[0][0] ? candidates[0] : candidates[1][0] ? candidates[1] : "/var/mobile/tmp/m2v";
@try {
device = native_bridge_metal_open(turnip, path, registryID, nullptr);
} @catch (NSException *exception) {
device = nil;
lastError = "exception";
}
if (!device && !lastError[0]) lastError = nmb_last_error();
}
if (!device) clientFailure(port, usable ? "no-device" : "no-cachedir", lastError);
// The client's first problem, once, on the same text line, sent from the thread that hit it: WidgetRenderer crashed on a nil render
// encoder with no reason anywhere readable (its own log lines need the plugin's format strings, which the device does not keep), and a
// once-a-second reader never ran before the crash (i568).
if (device) {
gClientPort = port;
native_bridge_metal_problem_hook = [](const char *problem) {clientFailure(gClientPort, "problem", problem);};
char problem[200] = {};
if (native_bridge_metal_first_problem(problem, sizeof problem) && problem[0]) clientFailure(port, "problem", problem);
}
return device;
}
#if TARGET_OS_OSX
// The compositor's QuartzCore asserts metallib_url != nil (CA::OGL::new_metal_context: [[NSBundle bundleForClass:[CALayer class]]
// URLForResource:@"default" withExtension:@"metallib"]) on this layout (macOS run32), though both the root and the SystemOS volume hold
// QuartzCore.framework/Versions/A/Resources/default.metallib. Measured here, in the render server, with the same calls, before QuartzCore
// makes its Metal context: what the bundle resolves to, and what readdir and NSFileManager list at the bundle's root and in Versions.
static void quartzCoreBundleReport() {
@autoreleasepool {
Class layer = NSClassFromString(@"CALayer");
NSBundle *bundle = layer ? [NSBundle bundleForClass:layer] : nil;
NSURL *url = [bundle URLForResource:@"default" withExtension:@"metallib"];
os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal qc: bundle %{public}s resources %{public}s exec %{public}s url %{public}s",
bundle.bundlePath.UTF8String ?: "-", bundle.resourcePath.UTF8String ?: "-", bundle.executablePath.UTF8String ?: "-",
url.path.UTF8String ?: "nil");
NSString *root = bundle.bundlePath ?: @"/System/Library/Frameworks/QuartzCore.framework";
for (NSString *sub in @[@"", @"/Versions", @"/Versions/A", @"/Versions/A/Resources", @"/Resources"]) {
NSString *path = [root stringByAppendingString:sub];
std::string listing;
if (DIR *d = opendir(path.fileSystemRepresentation)) {
while (struct dirent *e = readdir(d)) {listing += e->d_name; listing += ' ';}
closedir(d);
} else listing = "(opendir " + std::to_string(errno) + ")";
NSArray *items = [NSFileManager.defaultManager contentsOfDirectoryAtPath:path error:nil];
struct stat st = {};
const int sr = lstat(path.fileSystemRepresentation, &st);
// opendir reads a whole union directory (both layers) only when fstatfs reports MNT_UNION (0x20) for it.
struct statfs sf = {};
const int fr = statfs(path.fileSystemRepresentation, &sf);
os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal qc: %{public}s statfs %d flags 0x%x %{public}s on %{public}s",
path.UTF8String, fr, sf.f_flags, sf.f_fstypename, sf.f_mntonname);
os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal qc: %{public}s lstat %d mode %o dev %d readdir [%{public}s] fm [%{public}s]",
path.UTF8String, sr, st.st_mode, st.st_dev, listing.c_str(),
[[items componentsJoinedByString:@" "] UTF8String] ?: "nil");
}
}
}
#endif
static id deviceForPort(unsigned port) {
static id device;
static bool tried;
@synchronized([NativeAdrenoMetalDeviceBase class]) {
if (tried) return device;
tried = true;
exceptiondiag::start(); // the throw site of an uncaught exception, for the kernel's THROW row (ExceptionDiag.h)
// /private/var/ngpu/tu_debug, when present, holds TU_DEBUG for every Metal process (the boot argument tu-debug= comes after it).
if (FILE *flags = fopen("/private/var/ngpu/tu_debug", "r")) {
char line[128] = {};
if (fgets(line, sizeof line, flags)) {line[strcspn(line, "\n")] = 0; setenv("TU_DEBUG", line, 1);}
fclose(flags);
}
// tu-debug=<flags>: TU_DEBUG for every Metal process (an experiment: sysmem renders without GMEM load and store, gmem always with them).
{
char args[1100] = {};
size_t size = sizeof args - 1;
if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0)
if (const char *at = bootArgToken(args, "tu-debug=")) {
char value[64] = {};
strlcpy(value, at + strlen("tu-debug="), sizeof value);
value[strcspn(value, " ")] = 0;
setenv("TU_DEBUG", value, 1);
// Turnip's log (what perf and the other flags print) goes to the system log and to a file of this process beside
// the frame trace: a daemon's stderr goes nowhere, and the system log keeps warnings and errors only.
setenv("MESA_LOG", "file,syslog", 0);
char log[96];
mkdir("/private/var/tmp/NativeAdrenoMetal.trace", 0755);
snprintf(log, sizeof log, "/private/var/tmp/NativeAdrenoMetal.trace/mesa-%s-%d.log", getprogname(), (int)getpid());
setenv("MESA_LOG_FILE", log, 0);
}
}
// Turnip chooses each pass's rendering: tile memory (GMEM) or system memory, by its own rules (a pass with a memoryless
// attachment or a feedback loop in tile memory, a pass of few draws in system memory, the others by the bandwidth each would
// take). A pass in system memory has no tile load and store, which is what a screen-size pass that
// draws on a small part of its target otherwise pays for whole. The colour order of GMEM loads and stores is the image's own
// swap on the memory side (patches/mesa/turnip-gmem-memory-side-swap.patch). The RM row of the memory log counts both kinds;
// tu-debug= (for instance gmem,nocb: every pass in tile memory, no concurrent binning) overrides this.
// A7xx bins on BV while BR renders; Turnip retains its per-pass LRZ, query and hardware-binning checks.
setenv("tu_allow_concurrent_binning", bootArgSet("tu-cb=0") ? "false" : "true", 1);
// A740 has the sampler's built-in border constants; custom colors retain Turnip's table route.
setenv("tu_enable_fast_border_color_for_undefined_formats", bootArgSet("tu-fbc=0") ? "false" : "true", 1);
// diag=1 turns on every diagnostic of the GPU stack in this process: the frontend's frame trace, Turnip's pass stamps and
// submission ring, the runtime's stage ring, and in the render server the status and telemetry lines. Without it none of them runs.
if (bootArgSet("diag=1") || bootArgSet("gpu-trace=1")) setenv("SHENG_DIAG", "1", 1);
else unsetenv("SHENG_DIAG");
if (getenv("SHENG_DIAG")) {
char boot[64] = {};
size_t size = sizeof boot - 1;
if (!sysctlbyname("kern.bootsessionuuid", boot, &size, nullptr, 0)) setenv("NATIVE_METAL_BOOT_SESSION", boot, 1);
setenv("METAL2VULKAN_ENCODER_TIMES", "8ms", 1);
setenv("METAL2VULKAN_QUEUE_TIMES", "1", 1);
}
if (bootArgSet("gpu-trace=1")) setenv("SHENG_GPU_TRACE", "1", 1);
else unsetenv("SHENG_GPU_TRACE");
// The bridge's work placement (metal-flow=<bits>, read below). Any value logs the FL counts
// (native_bridge_metal_open), so two runs compare without diag=1's own cost.
// gpu-purge=0: the bridge discards nothing of what callers declare Volatile (NativeMetalPurge.rs), for a comparison on the
// same image. Read by the bridge as the device opens.
if (bootArgSet("gpu-purge=0")) setenv("NATIVE_METAL_PURGE", "0", 1);
else unsetenv("NATIVE_METAL_PURGE");
// gpu-ycbcr-planes=0: every two-plane YCbCr surface texture is made with its converted image, as before bridge patch 0035,
// for a comparison on the same image. Read by the bridge at the first such texture.
if (bootArgSet("gpu-ycbcr-planes=0")) setenv("NATIVE_METAL_YCBCR_PLANES", "0", 1);
else unsetenv("NATIVE_METAL_YCBCR_PLANES");
// Turnip compiles inside vkCreate*Pipelines on the calling thread: the runtime makes the call there instead of on a thread it
// starts for it (its relay exists for a driver whose compiler service takes the asking thread's class).
setenv("METAL2VULKAN_COMPILE_ON_ASKING_THREAD", "1", 1);
// metal-tile=1: the device also answers the family that has tile shaders (bridge: tile_shaders_offered).
if (bootArgSet("metal-tile=1")) setenv("NATIVE_METAL_TILE", "1", 1); else unsetenv("NATIVE_METAL_TILE");
{
// metal-flow=<bits>: 1 the lowering thread, 2 commit submission, 4 the warm pass; all three without the argument.
char args[1100] = {}, flow[16] = "";
size_t size = sizeof args - 1;
if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0)
if (const char *at = bootArgToken(args, "metal-flow=")) {
strlcpy(flow, at + strlen("metal-flow="), sizeof flow);
flow[strcspn(flow, " ")] = 0;
}
if (flow[0]) setenv("NATIVE_METAL_FLOW", flow, 1);
else unsetenv("NATIVE_METAL_FLOW");
}
if (bootArgSet("wallpaper-freeze=1")) setenv("NATIVE_METAL_FREEZE_WALLPAPER", "1", 1);
// log-gpu=1: the GPU stack's log lines (NSLog, the pipeline journal); off without it.
if (bootArgSet("log-gpu=1")) setenv("SHENG_LOG_GPU", "1", 1);
else unsetenv("SHENG_LOG_GPU");
#if TARGET_OS_OSX
// ci-mps=0 selects Core Image's own blur kernels. Core Image reads this at its first blur;
// an explicit environment value takes precedence.
if (bootArgSet("ci-mps=0")) setenv("CI_ENABLE_MPS", "0", 0);
if (!strcmp(getprogname(), "iconservicesagent")) {
char args[1100] = {};
size_t size = sizeof args - 1;
if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0)
if (const char *at = bootArgToken(args, "icon-cb=")) {
unsigned first = 0, count = 0, pass = 0, skipFirst = 0, skipLast = 0;
const int fields = sscanf(at + strlen("icon-cb="), "%u:%u:%u:%u:%u", &first, &count, &pass, &skipFirst, &skipLast);
if ((fields == 2 || fields == 5) && first && count && count <= 32) {
char range[32];snprintf(range, sizeof range, "%u:%u", first, count);
setenv("NATIVE_METAL_DUMP_CB", range, 0);
setenv("NATIVE_METAL_DUMP_LOG", "1", 0);
setenv("NATIVE_METAL_DUMP_FILE", "1", 0);
setenv("NATIVE_METAL_DUMP_PASS", "1", 0);
setenv("NATIVE_METAL_DUMP_INPUTS", "1", 0);
setenv("NATIVE_METAL_DUMP_DRAWS", "1:64", 0);
setenv("NATIVE_METAL_DUMP_WORDS", "32", 0);
setenv("NATIVE_METAL_DUMP_FUNCTIONS", "alpha_effect,accumulator_color,stroke_lines,plane_color,primitive_image", 0);
if (fields == 5 && pass && skipFirst && skipLast >= skipFirst) {
char skip[48];snprintf(skip, sizeof skip, "%u:%u:%u", pass, skipFirst, skipLast);
setenv("NATIVE_METAL_SKIP_DRAWS", skip, 0);
}
}
}
}
#endif
#if TARGET_OS_OSX
// macOS: Turnip opens the accelerator by class (AGXAccelerator here); the display takes plain rows (no Apple framebuffer or IOSurface
// compression). The runtime keeps its default block size: the virtual-machine board shrank it for a 512 MiB GPU pool, which this
// GPU's window does not have. The render server (WindowServer) takes the same path as backboardd on iPadOS below (display hand-off
// client, shader cache directory, pipeline pack, priming), without the iPadOS QuartzCore addresses; every device gets IOGPU's
// answers (IOGPUMetalDeviceShim.h).
setenv("TU_DARWIN_SERVICE", "AGXAccelerator", 1);
setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1);
CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
if (!nativeIsRenderServer()) {
// The system shader cache the render server fills (SharedShaderCache.h), read for what this process's own cache lacks.
// gpu-prime-system=0: no process reads it and the render server keeps its cache and its priming to itself, as before, for
// a comparison on the same image.
if (!bootArgSet("gpu-prime-system=0"))
native_bridge_metal_system_cache(systemcache::kReaderPath, systemcache::kOwners, 2, false);
device = clientDevice(port);
if (device) iogpu::install(device, port);
return device;
}
#else
if (!nativeIsRenderServer()) return device = clientDevice(port);
#endif
// The discovery releases the accelerator port when it is done; the monitor and the scanout client keep using it.
if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) {
if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port);
}
Gpu *gpu = makeGpu(port);
if (!gpu) return nil;
gpu->mark(0x6800);
char turnip[1024];
if (!turnip::libraryPath(turnip, sizeof turnip) || access(turnip, R_OK) != 0) {gpu->mark(0x68f1); return nil;}
gpu->mark(0x6801);
// The runtime refuses to open without a writable shader-cache directory, and which directories backboardd's sandbox lets it write is not
// known from here: try the user cache and temp directories, then /tmp and /var/tmp, and use the first that opens a device.
uint64_t registryID = 0;
if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) {
if (auto entryID = (int (*)(unsigned, uint64_t *))dlsym(iokit, "IORegistryEntryGetRegistryEntryID")) entryID(port, &registryID);
}
// What the render server sees for its framebuffer choice (marker 0x6Axx): bit0 the deep-framebuffer preference is set, bit1 it is false,
// bit2 MobileGestalt's answer behind the render server's deep flag is true, bit3 it exists, bit4 the compression capability dictionary exists,
// bit5 the process is not root.
{
unsigned bits = 0;
if (CFTypeRef value = CFPreferencesCopyAppValue(CFSTR("CAEnableDeepFramebuffer"), kCFPreferencesCurrentApplication)) {
bits |= 1;
if (CFGetTypeID(value) == CFBooleanGetTypeID() && !CFBooleanGetValue((CFBooleanRef) value)) bits |= 2;
CFRelease(value);
}
if (auto copyAnswer = (CFTypeRef (*)(CFStringRef)) dlsym(RTLD_DEFAULT, "MGCopyAnswer")) {
if (CFTypeRef deep = copyAnswer(CFSTR("Aixt/MEN2O2B7f+8m4TxUA"))) {
bits |= 8;
if (CFGetTypeID(deep) == CFBooleanGetTypeID() && CFBooleanGetValue((CFBooleanRef) deep)) bits |= 4;
CFRelease(deep);
}
if (CFTypeRef caps = copyAnswer(CFSTR("AoKnINTLPoKML3ctoP0AZg"))) {bits |= 16; CFRelease(caps);}
}
if (getuid() != 0) bits |= 32;
gDiagBits = bits;
gpu->mark(0x6A00 | bits);
}
// The render server compresses its framebuffer (an Apple/AGX layout this GPU does not write) unless told not to; the display driver takes plain rows.
setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1);
#if defined(__arm64e__)
// iPadOS 27 picks a 64-bit half-float framebuffer ('RGhA') whenever the display's EDR headroom is above 1 (this panel reports XDR
// headroom), which the display path can only show converted, slowly and greyed. As on iPadOS 17, where the render server drew plain
// 8-bit BGRA, the headroom is held at 1. IOMFBDisplay::framebuffer_pixel_format (24A446 0x184461838, asked on every update) takes
// RGhA when the headroom (Display::edr_headroom, CA_DISPLAY_EDR_HEADROOM first when nonzero) is above CA_FP16_MIN_HEADROOM and
// the panel's own above 8, or when CA_FORCE_FP16_FRAMEBUFFER, CA_ALLOW_EDR_MIRRORING or the display's own flag says so. Both
// options are QuartzCore's float options: init_debug reads them from the environment into debug_values (index 21 at
// __DATA_DIRTY 0x1e7021014, index 81 at 0x1e7021104), so they are set in the environment for a read after this and written for
// one before it. i84edf75a1 wrote only the stored values (marks 6E01, 6E02) and the framebuffer stayed RGhA: an option read after
// the plugin loaded put the threshold back to 0. CADefaultsGetFloat takes the environment first (getenv, atof), then the defaults:
// a threshold of 1e30 is above any headroom.
// id02fe1633 read back 0x6E19 in one process: CA_ALLOW_EDR_MIRRORING set and the threshold not raised (a process whose options
// were read after the write). The mirroring option is held off the same two ways (bool option 156 at _ca_debug_options
// 0x1e702116c + 156; init_debug stores value != 0).
setenv("CA_DISPLAY_EDR_HEADROOM", "1", 1);
setenv("CA_FP16_MIN_HEADROOM", "1e30", 1);
setenv("CA_ALLOW_EDR_MIRRORING", "0", 1);
#if !TARGET_OS_OSX
// The stored option values and the framebuffer branch are addresses of iPadOS 27's QuartzCore (24A446).
if (void *start = dlsym(RTLD_DEFAULT, "CARenderServerStart")) {
const uintptr_t slide = (uintptr_t)ptrauth_strip(start, ptrauth_key_function_pointer) - 0x1846dce88;
const float one = 1.0f, never = 1e30f;
gpu->mark(writeTask(0x1e7021014 + slide, &one, sizeof one) ? 0x6E01 : 0x6EF1);
gpu->mark(writeTask(0x1e7021104 + slide, &never, sizeof never) ? 0x6E02 : 0x6EF2);
const uint8_t off = 0;
gpu->mark(writeTask(0x1e702116c + 156 + slide, &off, sizeof off) ? 0x6E03 : 0x6EF3);
if (!strcmp(getprogname(), "backboardd") && bootArgSet("fb-bgra=1")) gpu->mark(patchFramebufferBranch(0x184461928 + slide));
// What framebuffer_pixel_format reads in the render server, once the desktop has started (other processes draw no
// framebuffer): 0x6E20 | 1 headroom override 1, 2 threshold above 1, 4 CA_FORCE_FP16_FRAMEBUFFER (_ca_debug_options
// 0x1e702116c + 128), 8 CA_ALLOW_EDR_MIRRORING (+ 156), 0x10 the device-is-virtualized byte (0x1e701bdd9).
// i35abaf83 read 0x6E23 and the framebuffer stayed RGhA (48388K swap surfaces, 8 bytes a pixel). The function has two more
// ways to RGhA: the display's own flag (+0x6840 of the display) and, when no compression format was picked, the virtualized
// byte, which sub_184541f10 stores once from a call with the constant CFString at 0x1e9021680; the string's text is noted
// ("ca virtualized key ...") so the answer behind it can be found.
static uintptr_t gSlide;
gSlide = slide;
pthread_t reader;
if (!strcmp(getprogname(), "backboardd") && pthread_create(&reader, nullptr, [](void *arg) -> void * {
sleep(30);
const float headroom = *reinterpret_cast<volatile const float *>(0x1e7021014 + gSlide);
const float threshold = *reinterpret_cast<volatile const float *>(0x1e7021104 + gSlide);
const uint8_t force = *reinterpret_cast<volatile const uint8_t *>(0x1e702116c + 128 + gSlide);
const uint8_t mirroring = *reinterpret_cast<volatile const uint8_t *>(0x1e702116c + 156 + gSlide);
const uint8_t virtualized = *reinterpret_cast<volatile const uint8_t *>(0x1e701bdd9 + gSlide);
static_cast<Gpu *>(arg)->mark(0x6E20 | (headroom == 1.0f ? 1 : 0) | (threshold > 1.0f ? 2 : 0) | (force ? 4 : 0) | (mirroring ? 8 : 0) |
(virtualized ? 0x10 : 0));
// A constant CFString: isa, flags, bytes, length. Only an 8-bit constant (flags 0x7c8) with a short length is read.
const uint64_t *cf = reinterpret_cast<const uint64_t *>(0x1e9021680 + gSlide);
char note[96];
if ((uint32_t)cf[1] == 0x7c8 && cf[3] > 0 && cf[3] < 64) {
const char *text = reinterpret_cast<const char *>(ptrauth_strip(reinterpret_cast<void *>(cf[2]), ptrauth_key_process_independent_data));
snprintf(note, sizeof note, "ca virtualized key %.*s = %u", (int)cf[3], text, virtualized);
} else
snprintf(note, sizeof note, "ca virtualized key ? flags %llx = %u", (unsigned long long)cf[1], virtualized);
native_bridge_metal_note(note);
return nullptr;
}, gpu) == 0)
pthread_detach(reader);
}
#else
// macOS: the framebuffer format is changed as a fixup of signed text (the board's payload text_fixups), applied when the page
// passes code signing validation. A write into the mapped page from here made the page invalid, and code signing killed
// WindowServer at its next validation of that page.
#endif // !TARGET_OS_OSX
#endif
// The runtime's creation log reports each VkPipeline making's time to the device's sink, which the telemetry's V row sums
// (diag=1).
unsetenv("METAL2VULKAN_LOG_PIPELINES");
CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
char candidates[7][512] = {};
if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0;
if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0;
strlcpy(candidates[2], "/tmp/", sizeof candidates[2]);
strlcpy(candidates[3], "/private/var/tmp/", sizeof candidates[3]);
// The user temp directory of the mobile user is under /var/mobile/tmp, which launchd's boot task dirs_cleaner empties at every boot,
// so with a persistent Data store the pipelines were still compiled again each boot (V row hit0). Try the user's Library/Caches first
// (marker 0x6814); the earlier candidates keep their markers.
if (const passwd *user = getpwuid(getuid()); user && user->pw_dir && user->pw_dir[0]) {
snprintf(candidates[4], sizeof candidates[4] - 16, "%s/Library/Caches/com.apple." NATIVE_RENDER_SERVER, user->pw_dir);
mkdir(candidates[4], 0700);
strlcat(candidates[4], "/", sizeof candidates[4]);
}
strlcpy(candidates[5], sharedShaderCacheDirectory().c_str(), sizeof candidates[5]);
#if TARGET_OS_OSX
// First the system shader cache's directory, where the kernel made it for this process (marker 0x6816): what the render
// server compiles and primes there, every process reads (SharedShaderCache.h). Not with gpu-prime-system=0.
if (!bootArgSet("gpu-prime-system=0")) strlcpy(candidates[6], systemcache::writerDirectory().c_str(), sizeof candidates[6]);
#endif
static const unsigned order[] = {6, 5, 4, 0, 1, 2, 3};
for (unsigned i : order) {
if (device) break;
if (!candidates[i][0]) continue;
if (i == 0) {strlcat(candidates[i], "com.apple.metal", sizeof candidates[i]); mkdir(candidates[i], 0700); strlcat(candidates[i], "/", sizeof candidates[i]);} // as for the clients below
if (i < 5) strlcat(candidates[i], "m2v", sizeof candidates[i]);
if (i != 6) mkdir(candidates[i], 0700);
if (access(candidates[i], W_OK) != 0) continue;
// Files an earlier image's copying import left half written here (int 590aaee2), which hold memory of the Data disk.
{
static bool once;
if (!once && i < 5) {once = true; removeHalfImported(candidates[i]);}
}
// The image's pipeline pack (an earlier boot's export), into this directory before the runtime reads it. Marker 0x6Cnn: files written
// (saturated at 0xfe), 0x6Cff the pack was refused.
if (char pack[1024]; requestedPath("NativeAdrenoMetal.m2vpack", pack, sizeof pack)) {
const int imported = cacheexport::importPack(pack, candidates[i]);
if (imported) gpu->mark(0x6C00 | (imported < 0 ? 0xff : imported > 0xfe ? 0xfe : imported));
}
// What the device writes in the system cache's directory is readable by everyone; in any other, by this user alone.
native_bridge_metal_system_cache(nullptr, nullptr, 0, i == 6);
@try {
device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr);
} @catch (NSException *exception) {
device = nil;
}
if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i]; gFillsSystemCache = i == 6;} // which directory served
}
if (!device) {
gpu->mark(0x68f2);
char text[64];
snprintf(text, sizeof text, "PUB no device: %.40s", nmb_last_error());
streamText(*gpu, text);
return nil;
}
gpu->mark(0x6802);
// The display hand-off client on its own thread: it only serves, not the discovery.
pthread_t scanout;
pthread_attr_t attributes;
pthread_attr_init(&attributes);
pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED);
pthread_attr_setstacksize(&attributes, 1u << 20);
// The telemetry table with diag=1 (streamTelemetry); without it the thread does not exist.
if (getenv("SHENG_DIAG")) {
pthread_t telemetry;
pthread_create(&telemetry, &attributes, streamTelemetry, gpu);
}
pthread_create(&scanout, &attributes, [](void *arg) -> void * {
Gpu *g = static_cast<Gpu *>(arg);
sleep(3);
@try {
bridgephase::serveScanout(*g, device);
} @catch (NSException *exception) {
g->mark(0x68f3);
}
return nullptr;
}, gpu);
// The cache directory's export to the UFS log only when NativeAdrenoMetal.cacheexport sits beside the bundle: Data persists across
// boots now, so the caches no longer need the trip through the host, and once priming has filled the directory, every rescan and
// export competed with the render server (int-245: the scanning thread at 45% CPU, the sending one at 48%, frames twice as slow).
char exportPath[1024];
if (requestedPath("NativeAdrenoMetal.cacheexport", exportPath, sizeof exportPath)) {
pthread_t exporter;
pthread_create(&exporter, &attributes, [](void *arg) -> void * {
cacheexport::run(*static_cast<Gpu *>(arg), gCacheDirectory);
return nullptr;
}, gpu);
}
#if !TARGET_OS_OSX
// The iPadOS primer uses an 8 MiB stack for large translated functions.
pthread_attr_setstacksize(&attributes, 8u << 20);
pthread_t primer;
if (!bootArgSet("gpu-prime=0")) pthread_create(&primer, &attributes, [](void *) -> void * {
pthread_setname_np("native-metal-prime"); // a process sample and the telemetry's TH rows name its CPU time
char list[1024];
prime::run((id<MTLDevice>)device, gCacheDirectory, 90,
requestedPath("NativeAdrenoMetal.primelist", list, sizeof list) ? std::string(list) : std::string(),
gFillsSystemCache);
return nullptr;
}, nullptr);
#endif
pthread_attr_destroy(&attributes);
// The GMEM colour order test (NativeTurnipSwapTest.h), once, 20 s after the device is published: each result line goes to the
// kernel's GPU trace as a problem record (event 65).
if (bootArgSet("swaptest=1") && nativeIsRenderServer()) {
pthread_t tester;
pthread_create(&tester, nullptr, [](void *) -> void * {
sleep(20);
char path[1024];
if (!turnip::libraryPath(path, sizeof path)) return nullptr;
void *lib = dlopen(path, RTLD_NOW | RTLD_LOCAL);
auto send = lib ? reinterpret_cast<bool (*)(uint32_t, const void *, uint32_t)>(dlsym(lib, "tu_darwin_frame_diagnostic"))
: nullptr;
turnip::swaptest::run(path, [&](const char *text) {
if (!send) return;
NativeGpuTraceRecord record = {};
record.event = GpuTraceProblem;
strlcpy(reinterpret_cast<char *>(&record.value[1]), text, 11 * sizeof(uint64_t));
send(NativeGpuTraceMethod, &record, sizeof record);
});
return nullptr;
}, nullptr);
pthread_detach(tester);
}
gpu->mark(0x6803);
#if TARGET_OS_OSX
iogpu::install(device, port);
quartzCoreBundleReport();
#endif
return device;
}
}
} // namespace publish
@implementation NativeAdrenoMetalDevice
- (id)initWithAcceleratorPort:(unsigned)port {
#if TARGET_OS_OSX
// The registered device is the frontend's object, not this one: IOGPUMetalDevice's own initializer would open an IOGPU user client.
// This object is kept, because -[IOGPUMetalDevice dealloc] tears down state its initializer never made.
static NSMutableArray *kept;
static dispatch_once_t once;
dispatch_once(&once, ^{ kept = [NSMutableArray array]; });
@synchronized(kept) {[kept addObject:self];}
if (!port) return nil;
#else
self = [super init];
if (!self || !port) return nil;
#endif
if (publish::requested()) return publish::deviceForPort(port);
static bool attempted;
if (attempted) return nil;
attempted = true;
// The test runs on its own thread: a kernel call that never returns (int-042: BoFree) then stalls that thread only, not SpringBoard's
// main thread (the UI froze with the test running inline).
// The discovery code releases the accelerator port as soon as this returns nil, which is before the test thread has started: the first
// marker (and every later open) then went to a dead name and never reached the kernel (K0 in int-043..052, with the plugin file opened by
// SpringBoard). Keep one extra user reference on the port, and let the thread look the service up itself as well.
if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) {
if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port);
}
struct Job {unsigned port; char name[32];};
Job *job = new Job{port, {}};
strlcpy(job->name, getprogname(), sizeof job->name);
pthread_t thread;
pthread_attr_t attributes;
pthread_attr_init(&attributes);
pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED);
if (pthread_create(&thread, &attributes, [](void *arg) -> void * {
Job *j = static_cast<Job *>(arg);
unsigned port = j->port;
if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) {
auto matching = (void *(*)(const char *))dlsym(iokit, "IOServiceMatching");
auto getService = (unsigned (*)(unsigned, void *))dlsym(iokit, "IOServiceGetMatchingService");
if (matching && getService) {
#if TARGET_OS_OSX
const unsigned own = getService(0, matching("AGXAccelerator"));
#else
const unsigned own = getService(0, matching("IOAcceleratorES"));
#endif
if (own) port = own;
}
}
gAfterTurnip = bridgephase::run;
adrenoUserTest(port, j->name);
delete j;
return nullptr;
}, job) != 0) delete job;
pthread_attr_destroy(&attributes);
return nil;
}
@end
#if TARGET_OS_OSX
#include <objc/runtime.h>
#include <mach/mach_time.h>
#include <atomic>
// WindowServer's frame pacing, as macOS-DroidVM's plugin (24af265, kc253-kc292), each behind its boot argument.
//
// ca-triple-buffer=1 sets QuartzCore's debug option CA_FORCE_TRIPLE_BUFFERED_LATENCY (read by IOMFBDisplay::can_update_):
// CoreAnimation renders a frame ahead instead of starting each frame only after the last one's buffer came back, which it otherwise
// does whenever a frame took longer than a refresh (DroidVM: a heavy window drag from 56-60 to 61-67 frames a second).
__attribute__((constructor)) static void tripleBufferFromBootArgs() {
if (strcmp(getprogname(), "WindowServer") || !publish::bootArgSet("ca-triple-buffer=1")) return;
auto setOption = reinterpret_cast<void (*)(unsigned, bool)>(dlsym(RTLD_DEFAULT, "CASetDebugOption"));
auto optionName = reinterpret_cast<const char *(*)(unsigned)>(dlsym(RTLD_DEFAULT, "CAGetDebugOptionEnvVariableName"));
// The option's index differs between QuartzCore builds, so it is found by its name.
unsigned index = ~0u;
for (unsigned i = 0; optionName && i < 1024; ++i) {
const char *name = optionName(i);
if (!name) break;
if (!strcmp(name, "CA_FORCE_TRIPLE_BUFFERED_LATENCY")) { index = i; break; }
}
const bool set = setOption && index != ~0u;
if (set) setOption(index, true);
os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal: CA_FORCE_TRIPLE_BUFFERED_LATENCY %{public}s (option %u)", set ? "set" : "not found", index);
}
// ca-free-wakeup=1: SkyLight starts each WindowServer update at -[CAWindowServerDisplay nextWakeupTime], which QuartzCore puts on the
// refresh grid (the last VBL plus a frame), so an update that took 13 ms is followed by one at 16.7 ms. This answers the earlier of that
// and a refresh after the last update's start (-beginExternalUpdate:usingSoftwareRenderer:): the display shows a swap when it comes,
// not at a VBL. SkyLight adds its updater delay (0.5 ms) and updates at once only when the last update is more than 1.25 frames before
// the answer, so before the due time the answer is the due time less that delay, after it the least time 1.25 frames past the last
// start. Without an update in the last second the grid's answer stands. The refresh is vsync='s (120 by default).
// DroidVM: a heavy window drag from 60 to 96 frames a second.
static std::atomic<uint64_t> gUpdateBegan{0}, gLastAsk{0};
static double hostSeconds(uint64_t t) {
static mach_timebase_info_data_t base;
if (!base.denom) mach_timebase_info(&base);
return (double)t * base.numer / base.denom / 1e9;
}
static uint64_t hostTicks(double seconds) {
static mach_timebase_info_data_t base;
if (!base.denom) mach_timebase_info(&base);
return (uint64_t)(seconds * 1e9 * base.denom / base.numer);
}
__attribute__((constructor)) static void freeWakeupFromBootArgs() {
if (strcmp(getprogname(), "WindowServer") || !publish::bootArgSet("ca-free-wakeup=1")) return;
char args[1100] = {};
size_t size = sizeof args - 1;
unsigned hz = 120;
if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0)
if (const char *at = publish::bootArgToken(args, "vsync=")) {
const int value = atoi(at + 6);
if (value >= 30 && value <= 240) hz = (unsigned)value;
}
const double refresh = 1.0 / hz;
Class display = objc_getClass("CAWindowServerDisplay");
static SEL wakeSel = sel_registerName("nextWakeupTime"), beginSel = sel_registerName("beginExternalUpdate:usingSoftwareRenderer:");
Method wake = display ? class_getInstanceMethod(display, wakeSel) : nullptr;
Method begin = display ? class_getInstanceMethod(display, beginSel) : nullptr;
if (!wake || !begin || method_getTypeEncoding(wake)[0] != 'd' || method_getNumberOfArguments(begin) != 4) {
os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal: free wakeup: CAWindowServerDisplay lacks %{public}s",
!wake ? "nextWakeupTime" : "beginExternalUpdate:usingSoftwareRenderer:");
return;
}
static auto wakeGrid = reinterpret_cast<double (*)(id, SEL)>(method_getImplementation(wake));
static auto beginOriginal = reinterpret_cast<uint64_t (*)(id, SEL, void *, uint64_t)>(method_getImplementation(begin));
method_setImplementation(begin, imp_implementationWithBlock(^uint64_t(id self, void *update, uint64_t software) {
// The update's start is SkyLight's last ask before it, where its update callback began.
const uint64_t now = mach_absolute_time(), asked = gLastAsk.load(std::memory_order_relaxed);
gUpdateBegan.store(asked && now - asked < hostTicks(0.003) ? asked : now, std::memory_order_relaxed);
return beginOriginal(self, beginSel, update, software);
}));
method_setImplementation(wake, imp_implementationWithBlock(^double(id self) {
const double grid = wakeGrid(self, wakeSel);
gLastAsk.store(mach_absolute_time(), std::memory_order_relaxed);
const uint64_t began = gUpdateBegan.load(std::memory_order_relaxed);
if (grid <= 0 || !began) return grid;
const double now = hostSeconds(mach_absolute_time()), start = hostSeconds(began);
if (now - start > 1.0) return grid;
const double due = start + refresh - 0.0005, ahead = start + 1.25 * refresh + 0.001;
if (now < due) return due < grid ? due : grid;
return now > ahead ? now : ahead;
}));
os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal: free wakeup on (%.2f ms)", refresh * 1e3);
}
#endif