// SPDX-License-Identifier: MIT // Adreno Metal plugin, reached through the original discovery (IOAcceleratorES -> MetalPluginName -> NSBundle -> principal class -> // initWithAcceleratorPort:). Default (stage 1): it runs the shared userspace GPU test (NativeAdrenoUserTest.h) from inside a real system // process, under that process's sandbox, and then returns nil. No device is registered, so the original software rendering keeps running unchanged. // Published (the image carries the file NativeAdrenoMetal.publish beside the bundle): backboardd, the one process that owns the GPU, returns the native // Metal frontend's device (the metal2vulkan runtime on Turnip, on the Adreno 740), which the original discovery registers; the CARenderServer then // renders the system UI through it, and the display driver's frames are written to the scanout by the GPU (GpuScanoutClient.h). Another process // gets its own native device too (its own driver connection, GPU address space and shader cache), as every process does with Apple's // own GPU plugin. #import #include #include #include #include #include #include "NativeRenderServer.h" #include "SharedShaderCache.h" #include #include #include #if __has_feature(ptrauth_calls) #include #endif #include #include #include #include #include #include #include #include "NativeAdrenoUserTest.h" #include "NativeTurnipSwapTest.h" #include "NativeGpuTrace.h" #include "NativeMetalBridgePhase.h" #include "CacheExport.h" #if !TARGET_OS_OSX #include "PrimeShaders.h" #endif #include "ExceptionDiag.h" #if TARGET_OS_OSX #include "IOGPUMetalDeviceShim.h" // macOS: +[IOGPUMetalDevice registerAcceleratorService:] takes the bundle's MetalPluginClassName only if it subclasses IOGPUMetalDevice. @interface IOGPUMetalDevice : NSObject @end #else @interface _MTLDevice : NSObject @end #endif @interface NativeAdrenoMetalDeviceBase : NSObject // only a lock object for the published mode's one-time setup @end @implementation NativeAdrenoMetalDeviceBase @end #if TARGET_OS_OSX @interface NativeAdrenoMetalDevice : IOGPUMetalDevice #else @interface NativeAdrenoMetalDevice : _MTLDevice #endif - (id)initWithAcceleratorPort:(unsigned)port; @end // ---- published mode ---- namespace publish { // The image's marker: a file beside the bundle in the union view of /System/Library/Extensions (the bundle's own seal stays untouched). // A file beside the bundle: its path in path, true when it is readable. static bool requestedPath(const char *name, char *path, size_t size) { Dl_info info = {}; if (!dladdr(reinterpret_cast(&requestedPath), &info) || !info.dli_fname) return false; snprintf(path, size, "%s", info.dli_fname); // The bundle's executable: /NativeAdrenoMetal.bundle/NativeAdrenoMetal (iOS) or // /NativeAdrenoMetal.bundle/Contents/MacOS/NativeAdrenoMetal (macOS). char *bundle = strstr(path, "/NativeAdrenoMetal.bundle/"); if (!bundle) return false; snprintf(bundle + 1, size - size_t(bundle + 1 - path), "%s", name); return access(path, R_OK) == 0; } static bool requested() { char path[1024]; return requestedPath("NativeAdrenoMetal.publish", path, sizeof path); } static Gpu *makeGpu(unsigned port) { void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); if (!iokit) return nullptr; Gpu *gpu = new Gpu; gpu->call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod"); gpu->map = (MapFn)dlsym(iokit, "IOConnectMapMemory64"); gpu->unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64"); gpu->openFn = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen"); gpu->closeFn = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose"); gpu->port = port; return gpu; } static std::string gCacheDirectory; // the runtime's cache directory (the one that opened the device), exported by CacheExport.h static bool gFillsSystemCache; // the render server opened its device in the system shader cache's directory (SharedShaderCache.h) // Progress markers 0x68xx (GPU6): 6800 entered, 6801 Turnip found, 6802 device made, 6803 returned to the discovery, 68f1 no Turnip, 68f2 no device. static unsigned gDiagBits; // Reads of the render server's own state for the framebuffer-format question (diagnostic only, for the exact QuartzCore of the target image; a read // that fails or a build that differs reports zeros). Each value is one marker, 0x7Kvv: K picks the item, vv the value. static bool readTask(uintptr_t address, void *out, size_t size) { vm_size_t got = 0; return vm_read_overwrite(mach_task_self(), address, size, (vm_address_t) out, &got) == KERN_SUCCESS && got == size; } [[maybe_unused]] static bool writeTask(uintptr_t address, const void *data, size_t size) { // iPadOS only (24A446 QuartzCore) return vm_write(mach_task_self(), address, (vm_offset_t) data, (mach_msg_type_number_t) size) == KERN_SUCCESS; } // EXPERIMENT (second report onward): this display path takes plain 8-bit BGRA only, so tell the render server's display that it has neither the deep // (10-bit) framebuffer nor universal buffer compression. Whether the state it reads those from is the whole story is what the markers show. // EXPERIMENT (boot-arg fb-bgra=1): the render server's framebuffer format with every other virtualized answer left as it is. // framebuffer_pixel_format (24A446 0x184461838) takes RGhA on `tbnz w8, #0` at 0x184461928 when CADeviceIsVirtualized (byte 0x1e701bdd9) is 1; // 40 other places in QuartzCore read the same byte (XR formats, backdrop compute, ASTC ...). Holding the byte at 0 turned all of them, and the // tablet drew light blue blocks. Here the byte stays 1 and only this branch is made a NOP, so the framebuffer is chosen as on a real device and // nothing else changes. Marks 0x6E40 unreadable, 0x6E41 other instruction, 0x6E42 patched, 0x6E50|stage failures. // The boot argument token that starts with prefix (a whole name, or name=value), or nullptr: a name is matched only at the start of the // command line or after a space, so "log=1" does not find "cache-log=1". static const char *bootArgToken(const char *args, const char *prefix) { const size_t length = strlen(prefix); for (const char *at = args; (at = strstr(at, prefix)); at += length) if (at == args || at[-1] == ' ') return at; return nullptr; } static bool bootArgSet(const char *key) { char args[1100] = {}; size_t size = sizeof args - 1; if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) != 0) return false; const char *at = bootArgToken(args, key); const size_t length = strlen(key); return at && (at[length] == 0 || at[length] == ' '); } // `site` is the slid address of that branch in the running QuartzCore; any other word there is left alone (0x6E41). [[maybe_unused]] static unsigned patchFramebufferBranch(uintptr_t site) { uint32_t word = 0; if (!readTask(site, &word, sizeof word)) return 0x6E40; if (word == 0xd503201f) return 0x6E42; if (word != 0x37000608) return 0x6E41; const vm_address_t page = site & ~(uintptr_t)(vm_page_size - 1); if (vm_protect(mach_task_self(), page, vm_page_size, FALSE, VM_PROT_READ | VM_PROT_WRITE | VM_PROT_COPY) != KERN_SUCCESS) return 0x6E51; const uint32_t nop = 0xd503201f; const bool wrote = vm_write(mach_task_self(), site, (vm_offset_t)&nop, sizeof nop) == KERN_SUCCESS; const bool restored = vm_protect(mach_task_self(), page, vm_page_size, FALSE, VM_PROT_READ | VM_PROT_EXECUTE) == KERN_SUCCESS; sys_icache_invalidate((void *)site, sizeof nop); if (!wrote) return 0x6E52; if (!restored) return 0x6E53; return 0x6E42; } // Every process other than the render server: the same runtime on its own connection, with none of the render server's threads (no // display hand-off, telemetry, benchmark or cache export). Its shader cache lives in its own cache or temp directory. // The reason a client's device failed, as a text line the driver keeps per process (the render server's report row is rewritten every ten seconds): // " ", through the marker stream (a connection is not needed). The Gpu object is made only on this path. // Deletes every "*.import" file under `root`: the copying import of an earlier image wrote each file beside its place and renamed it only // once the whole pack was read; a boot that ended first left them all there. static void removeHalfImported(const char *root) { char *paths[] = {const_cast(root), nullptr}; FTS *fts = fts_open(paths, FTS_PHYSICAL | FTS_NOCHDIR | FTS_XDEV, nullptr); if (!fts) return; while (FTSENT *e = fts_read(fts)) if (e->fts_info == FTS_F && e->fts_namelen > 7 && !strcmp(e->fts_name + e->fts_namelen - 7, ".import")) unlink(e->fts_path); fts_close(fts); } // diag=1 in the render server: the bridge's telemetry table (native_bridge_metal_telemetry) to the kernel's board log every six // seconds and to the system log every thirty. The kernel keeps 48 rows of at most 95 characters (its [metal table] section): the // marker 0xC000 | row starts a row, 0x8000 | a << 7 | b appends two characters. A row that reads as it did when last sent is not // sent again, and a row the table no longer has is cleared. The system log has every row, past the 48th too. static void *streamTelemetry(void *arg) { Gpu *g = static_cast(arg); pthread_setname_np("native-metal-telemetry"); pthread_set_qos_class_self_np(QOS_CLASS_UTILITY, 0); // a row is up to 46 marker calls: behind the render server's own work sleep(5); static char table[8192], sent[48][92]; for (unsigned pass = 0;; ++pass) { native_bridge_metal_telemetry(table, sizeof table); unsigned row = 0; for (char *line = table; *line; ++row) { char *end = strchr(line, '\n'); if (end) *end = 0; if (pass % 5 == 0) os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal table: %{public}s", line); if (row < 48 && strncmp(sent[row], line, sizeof sent[row] - 1)) { strlcpy(sent[row], line, sizeof sent[row]); g->mark(0xC000u | row); const size_t length = strlen(sent[row]); for (size_t i = 0; i < length; i += 2) { const unsigned a = (unsigned char)sent[row][i] & 0x7f, b = i + 1 < length ? (unsigned char)sent[row][i + 1] & 0x7f : 0; g->mark(0x8000u | (a << 7) | b); } } if (!end) {++row; break;} line = end + 1; } for (; row < 48; ++row) if (sent[row][0]) {sent[row][0] = 0; g->mark(0xC000u | row);} sleep(6); } return nullptr; } static unsigned gClientPort; static void clientFailure(unsigned port, const char *step, const char *detail) { if (Gpu *gpu = makeGpu(port)) { // The row holds 63 characters: the common prefix goes, and a long reason keeps its start (what was refused) and its end (why). if (!detail) detail = ""; if (!strncmp(detail, "native-metal: ", 14)) detail += 14; char shown[48]; const size_t length = strlen(detail); if (length <= 46) strlcpy(shown, detail, sizeof shown); else snprintf(shown, sizeof shown, "%.18s~%s", detail, detail + length - 27); char text[64]; snprintf(text, sizeof text, "%.14s %s", step, shown); streamText(*gpu, text); // A refusal's text is what says why a client ends (iconservicesagent's pipeline refusal showed 18 characters of it): the rest of it // follows in rows of its own, up to six, each 46 characters. if (!strcmp(step, "problem") && length > 46) for (unsigned part = 1, at = 46; part <= 6 && at < length; ++part, at += 46) { // The log keeps the latest text row only: each part follows the one before it by 4 s so a sample catches every one. std::string row = "problem" + std::to_string(part) + " " + std::string(detail + at).substr(0, 46); auto *target = gpu; dispatch_after(dispatch_time(DISPATCH_TIME_NOW, int64_t(part) * 4 * NSEC_PER_SEC), dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{streamText(*target, row.c_str());}); } } } static id clientDevice(unsigned port) { char turnip[1024]; if (!turnip::libraryPath(turnip, sizeof turnip) || access(turnip, R_OK) != 0) {clientFailure(port, "no-turnip", ""); return nil;} uint64_t registryID = 0; if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); if (auto entryID = (int (*)(unsigned, uint64_t *))dlsym(iokit, "IORegistryEntryGetRegistryEntryID")) entryID(port, ®istryID); } // IOSurfaces this GPU reads or writes must be plain rows, as in the render server. CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); #if TARGET_OS_OSX if (geteuid() == 0 && getenv("NATIVE_METAL_PRIMER")) { const std::string root = systemcache::writerDirectory(); if (root.empty()) return nil; native_bridge_metal_system_cache(nullptr, nullptr, 0, true); return native_bridge_metal_open(turnip, root.c_str(), registryID, nullptr); } #endif char candidates[3][512] = {}; if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0; if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0; strlcpy(candidates[2], sharedShaderCacheDirectory().c_str(), sizeof candidates[2]); id device = nil; const char *lastError = ""; unsigned usable = 0; for (unsigned i : {2u, 0u, 1u}) { if (device) break; if (!candidates[i][0]) continue; // In the user cache directory the cache sits inside Metal's own shader cache directory (com.apple.metal): a sandboxed process may // write there whenever its profile lets Metal cache shaders at all (macOS iconservicesagent: its profile allows METAL_CACHE_PATH // and denied every write to /m2v, so each of its launches compiled every pipeline again and it was killed at its memory // limit within seconds). if (i == 0) {strlcat(candidates[i], "com.apple.metal", sizeof candidates[i]); mkdir(candidates[i], 0700); strlcat(candidates[i], "/", sizeof candidates[i]);} if (i != 2) strlcat(candidates[i], "m2v", sizeof candidates[i]); mkdir(candidates[i], 0700); if (access(candidates[i], W_OK) != 0) continue; ++usable; @try { device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr); } @catch (NSException *exception) { device = nil; lastError = "exception"; } if (!device && !lastError[0]) lastError = nmb_last_error(); } if (!device && !usable) { setenv("METAL2VULKAN_SHADER_CACHE_OPTIONAL", "1", 1); const char *path = candidates[0][0] ? candidates[0] : candidates[1][0] ? candidates[1] : "/var/mobile/tmp/m2v"; @try { device = native_bridge_metal_open(turnip, path, registryID, nullptr); } @catch (NSException *exception) { device = nil; lastError = "exception"; } if (!device && !lastError[0]) lastError = nmb_last_error(); } if (!device) clientFailure(port, usable ? "no-device" : "no-cachedir", lastError); // The client's first problem, once, on the same text line, sent from the thread that hit it: WidgetRenderer crashed on a nil render // encoder with no reason anywhere readable (its own log lines need the plugin's format strings, which the device does not keep), and a // once-a-second reader never ran before the crash (i568). if (device) { gClientPort = port; native_bridge_metal_problem_hook = [](const char *problem) {clientFailure(gClientPort, "problem", problem);}; char problem[200] = {}; if (native_bridge_metal_first_problem(problem, sizeof problem) && problem[0]) clientFailure(port, "problem", problem); } return device; } #if TARGET_OS_OSX // The compositor's QuartzCore asserts metallib_url != nil (CA::OGL::new_metal_context: [[NSBundle bundleForClass:[CALayer class]] // URLForResource:@"default" withExtension:@"metallib"]) on this layout (macOS run32), though both the root and the SystemOS volume hold // QuartzCore.framework/Versions/A/Resources/default.metallib. Measured here, in the render server, with the same calls, before QuartzCore // makes its Metal context: what the bundle resolves to, and what readdir and NSFileManager list at the bundle's root and in Versions. static void quartzCoreBundleReport() { @autoreleasepool { Class layer = NSClassFromString(@"CALayer"); NSBundle *bundle = layer ? [NSBundle bundleForClass:layer] : nil; NSURL *url = [bundle URLForResource:@"default" withExtension:@"metallib"]; os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal qc: bundle %{public}s resources %{public}s exec %{public}s url %{public}s", bundle.bundlePath.UTF8String ?: "-", bundle.resourcePath.UTF8String ?: "-", bundle.executablePath.UTF8String ?: "-", url.path.UTF8String ?: "nil"); NSString *root = bundle.bundlePath ?: @"/System/Library/Frameworks/QuartzCore.framework"; for (NSString *sub in @[@"", @"/Versions", @"/Versions/A", @"/Versions/A/Resources", @"/Resources"]) { NSString *path = [root stringByAppendingString:sub]; std::string listing; if (DIR *d = opendir(path.fileSystemRepresentation)) { while (struct dirent *e = readdir(d)) {listing += e->d_name; listing += ' ';} closedir(d); } else listing = "(opendir " + std::to_string(errno) + ")"; NSArray *items = [NSFileManager.defaultManager contentsOfDirectoryAtPath:path error:nil]; struct stat st = {}; const int sr = lstat(path.fileSystemRepresentation, &st); // opendir reads a whole union directory (both layers) only when fstatfs reports MNT_UNION (0x20) for it. struct statfs sf = {}; const int fr = statfs(path.fileSystemRepresentation, &sf); os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal qc: %{public}s statfs %d flags 0x%x %{public}s on %{public}s", path.UTF8String, fr, sf.f_flags, sf.f_fstypename, sf.f_mntonname); os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal qc: %{public}s lstat %d mode %o dev %d readdir [%{public}s] fm [%{public}s]", path.UTF8String, sr, st.st_mode, st.st_dev, listing.c_str(), [[items componentsJoinedByString:@" "] UTF8String] ?: "nil"); } } } #endif static id deviceForPort(unsigned port) { static id device; static bool tried; @synchronized([NativeAdrenoMetalDeviceBase class]) { if (tried) return device; tried = true; exceptiondiag::start(); // the throw site of an uncaught exception, for the kernel's THROW row (ExceptionDiag.h) // /private/var/ngpu/tu_debug, when present, holds TU_DEBUG for every Metal process (the boot argument tu-debug= comes after it). if (FILE *flags = fopen("/private/var/ngpu/tu_debug", "r")) { char line[128] = {}; if (fgets(line, sizeof line, flags)) {line[strcspn(line, "\n")] = 0; setenv("TU_DEBUG", line, 1);} fclose(flags); } // tu-debug=: TU_DEBUG for every Metal process (an experiment: sysmem renders without GMEM load and store, gmem always with them). { char args[1100] = {}; size_t size = sizeof args - 1; if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0) if (const char *at = bootArgToken(args, "tu-debug=")) { char value[64] = {}; strlcpy(value, at + strlen("tu-debug="), sizeof value); value[strcspn(value, " ")] = 0; setenv("TU_DEBUG", value, 1); // Turnip's log (what perf and the other flags print) goes to the system log and to a file of this process beside // the frame trace: a daemon's stderr goes nowhere, and the system log keeps warnings and errors only. setenv("MESA_LOG", "file,syslog", 0); char log[96]; mkdir("/private/var/tmp/NativeAdrenoMetal.trace", 0755); snprintf(log, sizeof log, "/private/var/tmp/NativeAdrenoMetal.trace/mesa-%s-%d.log", getprogname(), (int)getpid()); setenv("MESA_LOG_FILE", log, 0); } } // Turnip chooses each pass's rendering: tile memory (GMEM) or system memory, by its own rules (a pass with a memoryless // attachment or a feedback loop in tile memory, a pass of few draws in system memory, the others by the bandwidth each would // take). A pass in system memory has no tile load and store, which is what a screen-size pass that // draws on a small part of its target otherwise pays for whole. The colour order of GMEM loads and stores is the image's own // swap on the memory side (patches/mesa/turnip-gmem-memory-side-swap.patch). The RM row of the memory log counts both kinds; // tu-debug= (for instance gmem,nocb: every pass in tile memory, no concurrent binning) overrides this. // A7xx bins on BV while BR renders; Turnip retains its per-pass LRZ, query and hardware-binning checks. setenv("tu_allow_concurrent_binning", bootArgSet("tu-cb=0") ? "false" : "true", 1); // A740 has the sampler's built-in border constants; custom colors retain Turnip's table route. setenv("tu_enable_fast_border_color_for_undefined_formats", bootArgSet("tu-fbc=0") ? "false" : "true", 1); // diag=1 turns on every diagnostic of the GPU stack in this process: the frontend's frame trace, Turnip's pass stamps and // submission ring, the runtime's stage ring, and in the render server the status and telemetry lines. Without it none of them runs. if (bootArgSet("diag=1") || bootArgSet("gpu-trace=1")) setenv("SHENG_DIAG", "1", 1); else unsetenv("SHENG_DIAG"); if (getenv("SHENG_DIAG")) { char boot[64] = {}; size_t size = sizeof boot - 1; if (!sysctlbyname("kern.bootsessionuuid", boot, &size, nullptr, 0)) setenv("NATIVE_METAL_BOOT_SESSION", boot, 1); setenv("METAL2VULKAN_ENCODER_TIMES", "8ms", 1); setenv("METAL2VULKAN_QUEUE_TIMES", "1", 1); } if (bootArgSet("gpu-trace=1")) setenv("SHENG_GPU_TRACE", "1", 1); else unsetenv("SHENG_GPU_TRACE"); // The bridge's work placement (metal-flow=, read below). Any value logs the FL counts // (native_bridge_metal_open), so two runs compare without diag=1's own cost. // gpu-purge=0: the bridge discards nothing of what callers declare Volatile (NativeMetalPurge.rs), for a comparison on the // same image. Read by the bridge as the device opens. if (bootArgSet("gpu-purge=0")) setenv("NATIVE_METAL_PURGE", "0", 1); else unsetenv("NATIVE_METAL_PURGE"); // gpu-ycbcr-planes=0: every two-plane YCbCr surface texture is made with its converted image, as before bridge patch 0035, // for a comparison on the same image. Read by the bridge at the first such texture. if (bootArgSet("gpu-ycbcr-planes=0")) setenv("NATIVE_METAL_YCBCR_PLANES", "0", 1); else unsetenv("NATIVE_METAL_YCBCR_PLANES"); // Turnip compiles inside vkCreate*Pipelines on the calling thread: the runtime makes the call there instead of on a thread it // starts for it (its relay exists for a driver whose compiler service takes the asking thread's class). setenv("METAL2VULKAN_COMPILE_ON_ASKING_THREAD", "1", 1); // metal-tile=1: the device also answers the family that has tile shaders (bridge: tile_shaders_offered). if (bootArgSet("metal-tile=1")) setenv("NATIVE_METAL_TILE", "1", 1); else unsetenv("NATIVE_METAL_TILE"); { // metal-flow=: 1 the lowering thread, 2 commit submission, 4 the warm pass; all three without the argument. char args[1100] = {}, flow[16] = ""; size_t size = sizeof args - 1; if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0) if (const char *at = bootArgToken(args, "metal-flow=")) { strlcpy(flow, at + strlen("metal-flow="), sizeof flow); flow[strcspn(flow, " ")] = 0; } if (flow[0]) setenv("NATIVE_METAL_FLOW", flow, 1); else unsetenv("NATIVE_METAL_FLOW"); } if (bootArgSet("wallpaper-freeze=1")) setenv("NATIVE_METAL_FREEZE_WALLPAPER", "1", 1); // log-gpu=1: the GPU stack's log lines (NSLog, the pipeline journal); off without it. if (bootArgSet("log-gpu=1")) setenv("SHENG_LOG_GPU", "1", 1); else unsetenv("SHENG_LOG_GPU"); #if TARGET_OS_OSX // ci-mps=0 selects Core Image's own blur kernels. Core Image reads this at its first blur; // an explicit environment value takes precedence. if (bootArgSet("ci-mps=0")) setenv("CI_ENABLE_MPS", "0", 0); if (!strcmp(getprogname(), "iconservicesagent")) { char args[1100] = {}; size_t size = sizeof args - 1; if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0) if (const char *at = bootArgToken(args, "icon-cb=")) { unsigned first = 0, count = 0, pass = 0, skipFirst = 0, skipLast = 0; const int fields = sscanf(at + strlen("icon-cb="), "%u:%u:%u:%u:%u", &first, &count, &pass, &skipFirst, &skipLast); if ((fields == 2 || fields == 5) && first && count && count <= 32) { char range[32];snprintf(range, sizeof range, "%u:%u", first, count); setenv("NATIVE_METAL_DUMP_CB", range, 0); setenv("NATIVE_METAL_DUMP_LOG", "1", 0); setenv("NATIVE_METAL_DUMP_FILE", "1", 0); setenv("NATIVE_METAL_DUMP_PASS", "1", 0); setenv("NATIVE_METAL_DUMP_INPUTS", "1", 0); setenv("NATIVE_METAL_DUMP_DRAWS", "1:64", 0); setenv("NATIVE_METAL_DUMP_WORDS", "32", 0); setenv("NATIVE_METAL_DUMP_FUNCTIONS", "filter_distance,alpha_effect,accumulator,stroke_lines,plane_color,primitive_image", 0); setenv("NATIVE_METAL_WARNINGS", "1", 0); setenv("METAL2VULKAN_LOG_PASSES", "RG16Float:*", 0); setenv("METAL2VULKAN_LOG_SHADOWS", "1", 0); if (fields == 5 && pass && skipFirst && skipLast >= skipFirst) { char skip[48];snprintf(skip, sizeof skip, "%u:%u:%u", pass, skipFirst, skipLast); setenv("NATIVE_METAL_SKIP_DRAWS", skip, 0); } } } } #endif #if TARGET_OS_OSX // macOS: Turnip opens the accelerator by class (AGXAccelerator here); the display takes plain rows (no Apple framebuffer or IOSurface // compression). The runtime keeps its default block size: the virtual-machine board shrank it for a 512 MiB GPU pool, which this // GPU's window does not have. The render server (WindowServer) takes the same path as backboardd on iPadOS below (display hand-off // client, shader cache directory, pipeline pack, priming), without the iPadOS QuartzCore addresses; every device gets IOGPU's // answers (IOGPUMetalDeviceShim.h). setenv("TU_DARWIN_SERVICE", "AGXAccelerator", 1); setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1); CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); if (!nativeIsRenderServer()) { // The system shader cache the render server fills (SharedShaderCache.h), read for what this process's own cache lacks. // gpu-prime-system=0: no process reads it and the render server keeps its cache and its priming to itself, as before, for // a comparison on the same image. if (!bootArgSet("gpu-prime-system=0")) native_bridge_metal_system_cache(systemcache::kReaderPath, systemcache::kOwners, 2, false); device = clientDevice(port); if (device) iogpu::install(device, port); return device; } #else if (!nativeIsRenderServer()) return device = clientDevice(port); #endif // The discovery releases the accelerator port when it is done; the monitor and the scanout client keep using it. if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); } Gpu *gpu = makeGpu(port); if (!gpu) return nil; gpu->mark(0x6800); char turnip[1024]; if (!turnip::libraryPath(turnip, sizeof turnip) || access(turnip, R_OK) != 0) {gpu->mark(0x68f1); return nil;} gpu->mark(0x6801); // The runtime refuses to open without a writable shader-cache directory, and which directories backboardd's sandbox lets it write is not // known from here: try the user cache and temp directories, then /tmp and /var/tmp, and use the first that opens a device. uint64_t registryID = 0; if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { if (auto entryID = (int (*)(unsigned, uint64_t *))dlsym(iokit, "IORegistryEntryGetRegistryEntryID")) entryID(port, ®istryID); } // What the render server sees for its framebuffer choice (marker 0x6Axx): bit0 the deep-framebuffer preference is set, bit1 it is false, // bit2 MobileGestalt's answer behind the render server's deep flag is true, bit3 it exists, bit4 the compression capability dictionary exists, // bit5 the process is not root. { unsigned bits = 0; if (CFTypeRef value = CFPreferencesCopyAppValue(CFSTR("CAEnableDeepFramebuffer"), kCFPreferencesCurrentApplication)) { bits |= 1; if (CFGetTypeID(value) == CFBooleanGetTypeID() && !CFBooleanGetValue((CFBooleanRef) value)) bits |= 2; CFRelease(value); } if (auto copyAnswer = (CFTypeRef (*)(CFStringRef)) dlsym(RTLD_DEFAULT, "MGCopyAnswer")) { if (CFTypeRef deep = copyAnswer(CFSTR("Aixt/MEN2O2B7f+8m4TxUA"))) { bits |= 8; if (CFGetTypeID(deep) == CFBooleanGetTypeID() && CFBooleanGetValue((CFBooleanRef) deep)) bits |= 4; CFRelease(deep); } if (CFTypeRef caps = copyAnswer(CFSTR("AoKnINTLPoKML3ctoP0AZg"))) {bits |= 16; CFRelease(caps);} } if (getuid() != 0) bits |= 32; gDiagBits = bits; gpu->mark(0x6A00 | bits); } // The render server compresses its framebuffer (an Apple/AGX layout this GPU does not write) unless told not to; the display driver takes plain rows. setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1); #if defined(__arm64e__) // iPadOS 27 picks a 64-bit half-float framebuffer ('RGhA') whenever the display's EDR headroom is above 1 (this panel reports XDR // headroom), which the display path can only show converted, slowly and greyed. As on iPadOS 17, where the render server drew plain // 8-bit BGRA, the headroom is held at 1. IOMFBDisplay::framebuffer_pixel_format (24A446 0x184461838, asked on every update) takes // RGhA when the headroom (Display::edr_headroom, CA_DISPLAY_EDR_HEADROOM first when nonzero) is above CA_FP16_MIN_HEADROOM and // the panel's own above 8, or when CA_FORCE_FP16_FRAMEBUFFER, CA_ALLOW_EDR_MIRRORING or the display's own flag says so. Both // options are QuartzCore's float options: init_debug reads them from the environment into debug_values (index 21 at // __DATA_DIRTY 0x1e7021014, index 81 at 0x1e7021104), so they are set in the environment for a read after this and written for // one before it. i84edf75a1 wrote only the stored values (marks 6E01, 6E02) and the framebuffer stayed RGhA: an option read after // the plugin loaded put the threshold back to 0. CADefaultsGetFloat takes the environment first (getenv, atof), then the defaults: // a threshold of 1e30 is above any headroom. // id02fe1633 read back 0x6E19 in one process: CA_ALLOW_EDR_MIRRORING set and the threshold not raised (a process whose options // were read after the write). The mirroring option is held off the same two ways (bool option 156 at _ca_debug_options // 0x1e702116c + 156; init_debug stores value != 0). setenv("CA_DISPLAY_EDR_HEADROOM", "1", 1); setenv("CA_FP16_MIN_HEADROOM", "1e30", 1); setenv("CA_ALLOW_EDR_MIRRORING", "0", 1); #if !TARGET_OS_OSX // The stored option values and the framebuffer branch are addresses of iPadOS 27's QuartzCore (24A446). if (void *start = dlsym(RTLD_DEFAULT, "CARenderServerStart")) { const uintptr_t slide = (uintptr_t)ptrauth_strip(start, ptrauth_key_function_pointer) - 0x1846dce88; const float one = 1.0f, never = 1e30f; gpu->mark(writeTask(0x1e7021014 + slide, &one, sizeof one) ? 0x6E01 : 0x6EF1); gpu->mark(writeTask(0x1e7021104 + slide, &never, sizeof never) ? 0x6E02 : 0x6EF2); const uint8_t off = 0; gpu->mark(writeTask(0x1e702116c + 156 + slide, &off, sizeof off) ? 0x6E03 : 0x6EF3); if (!strcmp(getprogname(), "backboardd") && bootArgSet("fb-bgra=1")) gpu->mark(patchFramebufferBranch(0x184461928 + slide)); // What framebuffer_pixel_format reads in the render server, once the desktop has started (other processes draw no // framebuffer): 0x6E20 | 1 headroom override 1, 2 threshold above 1, 4 CA_FORCE_FP16_FRAMEBUFFER (_ca_debug_options // 0x1e702116c + 128), 8 CA_ALLOW_EDR_MIRRORING (+ 156), 0x10 the device-is-virtualized byte (0x1e701bdd9). // i35abaf83 read 0x6E23 and the framebuffer stayed RGhA (48388K swap surfaces, 8 bytes a pixel). The function has two more // ways to RGhA: the display's own flag (+0x6840 of the display) and, when no compression format was picked, the virtualized // byte, which sub_184541f10 stores once from a call with the constant CFString at 0x1e9021680; the string's text is noted // ("ca virtualized key ...") so the answer behind it can be found. static uintptr_t gSlide; gSlide = slide; pthread_t reader; if (!strcmp(getprogname(), "backboardd") && pthread_create(&reader, nullptr, [](void *arg) -> void * { sleep(30); const float headroom = *reinterpret_cast(0x1e7021014 + gSlide); const float threshold = *reinterpret_cast(0x1e7021104 + gSlide); const uint8_t force = *reinterpret_cast(0x1e702116c + 128 + gSlide); const uint8_t mirroring = *reinterpret_cast(0x1e702116c + 156 + gSlide); const uint8_t virtualized = *reinterpret_cast(0x1e701bdd9 + gSlide); static_cast(arg)->mark(0x6E20 | (headroom == 1.0f ? 1 : 0) | (threshold > 1.0f ? 2 : 0) | (force ? 4 : 0) | (mirroring ? 8 : 0) | (virtualized ? 0x10 : 0)); // A constant CFString: isa, flags, bytes, length. Only an 8-bit constant (flags 0x7c8) with a short length is read. const uint64_t *cf = reinterpret_cast(0x1e9021680 + gSlide); char note[96]; if ((uint32_t)cf[1] == 0x7c8 && cf[3] > 0 && cf[3] < 64) { const char *text = reinterpret_cast(ptrauth_strip(reinterpret_cast(cf[2]), ptrauth_key_process_independent_data)); snprintf(note, sizeof note, "ca virtualized key %.*s = %u", (int)cf[3], text, virtualized); } else snprintf(note, sizeof note, "ca virtualized key ? flags %llx = %u", (unsigned long long)cf[1], virtualized); native_bridge_metal_note(note); return nullptr; }, gpu) == 0) pthread_detach(reader); } #else // macOS: the framebuffer format is changed as a fixup of signed text (the board's payload text_fixups), applied when the page // passes code signing validation. A write into the mapped page from here made the page invalid, and code signing killed // WindowServer at its next validation of that page. #endif // !TARGET_OS_OSX #endif // The runtime's creation log reports each VkPipeline making's time to the device's sink, which the telemetry's V row sums // (diag=1). unsetenv("METAL2VULKAN_LOG_PIPELINES"); CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); char candidates[7][512] = {}; if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0; if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0; strlcpy(candidates[2], "/tmp/", sizeof candidates[2]); strlcpy(candidates[3], "/private/var/tmp/", sizeof candidates[3]); // The user temp directory of the mobile user is under /var/mobile/tmp, which launchd's boot task dirs_cleaner empties at every boot, // so with a persistent Data store the pipelines were still compiled again each boot (V row hit0). Try the user's Library/Caches first // (marker 0x6814); the earlier candidates keep their markers. if (const passwd *user = getpwuid(getuid()); user && user->pw_dir && user->pw_dir[0]) { snprintf(candidates[4], sizeof candidates[4] - 16, "%s/Library/Caches/com.apple." NATIVE_RENDER_SERVER, user->pw_dir); mkdir(candidates[4], 0700); strlcat(candidates[4], "/", sizeof candidates[4]); } strlcpy(candidates[5], sharedShaderCacheDirectory().c_str(), sizeof candidates[5]); #if TARGET_OS_OSX // First the system shader cache's directory, where the kernel made it for this process (marker 0x6816): what the render // server compiles and primes there, every process reads (SharedShaderCache.h). Not with gpu-prime-system=0. if (!bootArgSet("gpu-prime-system=0")) strlcpy(candidates[6], systemcache::writerDirectory().c_str(), sizeof candidates[6]); #endif static const unsigned order[] = {6, 5, 4, 0, 1, 2, 3}; for (unsigned i : order) { if (device) break; if (!candidates[i][0]) continue; if (i == 0) {strlcat(candidates[i], "com.apple.metal", sizeof candidates[i]); mkdir(candidates[i], 0700); strlcat(candidates[i], "/", sizeof candidates[i]);} // as for the clients below if (i < 5) strlcat(candidates[i], "m2v", sizeof candidates[i]); if (i != 6) mkdir(candidates[i], 0700); if (access(candidates[i], W_OK) != 0) continue; // Files an earlier image's copying import left half written here (int 590aaee2), which hold memory of the Data disk. { static bool once; if (!once && i < 5) {once = true; removeHalfImported(candidates[i]);} } // The image's pipeline pack (an earlier boot's export), into this directory before the runtime reads it. Marker 0x6Cnn: files written // (saturated at 0xfe), 0x6Cff the pack was refused. if (char pack[1024]; requestedPath("NativeAdrenoMetal.m2vpack", pack, sizeof pack)) { const int imported = cacheexport::importPack(pack, candidates[i]); if (imported) gpu->mark(0x6C00 | (imported < 0 ? 0xff : imported > 0xfe ? 0xfe : imported)); } // What the device writes in the system cache's directory is readable by everyone; in any other, by this user alone. native_bridge_metal_system_cache(nullptr, nullptr, 0, i == 6); @try { device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr); } @catch (NSException *exception) { device = nil; } if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i]; gFillsSystemCache = i == 6;} // which directory served } if (!device) { gpu->mark(0x68f2); char text[64]; snprintf(text, sizeof text, "PUB no device: %.40s", nmb_last_error()); streamText(*gpu, text); return nil; } gpu->mark(0x6802); // The display hand-off client on its own thread: it only serves, not the discovery. pthread_t scanout; pthread_attr_t attributes; pthread_attr_init(&attributes); pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); pthread_attr_setstacksize(&attributes, 1u << 20); // The telemetry table with diag=1 (streamTelemetry); without it the thread does not exist. if (getenv("SHENG_DIAG")) { pthread_t telemetry; pthread_create(&telemetry, &attributes, streamTelemetry, gpu); } pthread_create(&scanout, &attributes, [](void *arg) -> void * { Gpu *g = static_cast(arg); sleep(3); @try { bridgephase::serveScanout(*g, device); } @catch (NSException *exception) { g->mark(0x68f3); } return nullptr; }, gpu); // The cache directory's export to the UFS log only when NativeAdrenoMetal.cacheexport sits beside the bundle: Data persists across // boots now, so the caches no longer need the trip through the host, and once priming has filled the directory, every rescan and // export competed with the render server (int-245: the scanning thread at 45% CPU, the sending one at 48%, frames twice as slow). char exportPath[1024]; if (requestedPath("NativeAdrenoMetal.cacheexport", exportPath, sizeof exportPath)) { pthread_t exporter; pthread_create(&exporter, &attributes, [](void *arg) -> void * { cacheexport::run(*static_cast(arg), gCacheDirectory); return nullptr; }, gpu); } #if !TARGET_OS_OSX // The iPadOS primer uses an 8 MiB stack for large translated functions. pthread_attr_setstacksize(&attributes, 8u << 20); pthread_t primer; if (!bootArgSet("gpu-prime=0")) pthread_create(&primer, &attributes, [](void *) -> void * { pthread_setname_np("native-metal-prime"); // a process sample and the telemetry's TH rows name its CPU time char list[1024]; prime::run((id)device, gCacheDirectory, 90, requestedPath("NativeAdrenoMetal.primelist", list, sizeof list) ? std::string(list) : std::string(), gFillsSystemCache); return nullptr; }, nullptr); #endif pthread_attr_destroy(&attributes); // The GMEM colour order test (NativeTurnipSwapTest.h), once, 20 s after the device is published: each result line goes to the // kernel's GPU trace as a problem record (event 65). if (bootArgSet("swaptest=1") && nativeIsRenderServer()) { pthread_t tester; pthread_create(&tester, nullptr, [](void *) -> void * { sleep(20); char path[1024]; if (!turnip::libraryPath(path, sizeof path)) return nullptr; void *lib = dlopen(path, RTLD_NOW | RTLD_LOCAL); auto send = lib ? reinterpret_cast(dlsym(lib, "tu_darwin_frame_diagnostic")) : nullptr; turnip::swaptest::run(path, [&](const char *text) { if (!send) return; NativeGpuTraceRecord record = {}; record.event = GpuTraceProblem; strlcpy(reinterpret_cast(&record.value[1]), text, 11 * sizeof(uint64_t)); send(NativeGpuTraceMethod, &record, sizeof record); }); return nullptr; }, nullptr); pthread_detach(tester); } gpu->mark(0x6803); #if TARGET_OS_OSX iogpu::install(device, port); quartzCoreBundleReport(); #endif return device; } } } // namespace publish @implementation NativeAdrenoMetalDevice - (id)initWithAcceleratorPort:(unsigned)port { #if TARGET_OS_OSX // The registered device is the frontend's object, not this one: IOGPUMetalDevice's own initializer would open an IOGPU user client. // This object is kept, because -[IOGPUMetalDevice dealloc] tears down state its initializer never made. static NSMutableArray *kept; static dispatch_once_t once; dispatch_once(&once, ^{ kept = [NSMutableArray array]; }); @synchronized(kept) {[kept addObject:self];} if (!port) return nil; #else self = [super init]; if (!self || !port) return nil; #endif if (publish::requested()) return publish::deviceForPort(port); static bool attempted; if (attempted) return nil; attempted = true; // The test runs on its own thread: a kernel call that never returns (int-042: BoFree) then stalls that thread only, not SpringBoard's // main thread (the UI froze with the test running inline). // The discovery code releases the accelerator port as soon as this returns nil, which is before the test thread has started: the first // marker (and every later open) then went to a dead name and never reached the kernel (K0 in int-043..052, with the plugin file opened by // SpringBoard). Keep one extra user reference on the port, and let the thread look the service up itself as well. if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); } struct Job {unsigned port; char name[32];}; Job *job = new Job{port, {}}; strlcpy(job->name, getprogname(), sizeof job->name); pthread_t thread; pthread_attr_t attributes; pthread_attr_init(&attributes); pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); if (pthread_create(&thread, &attributes, [](void *arg) -> void * { Job *j = static_cast(arg); unsigned port = j->port; if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { auto matching = (void *(*)(const char *))dlsym(iokit, "IOServiceMatching"); auto getService = (unsigned (*)(unsigned, void *))dlsym(iokit, "IOServiceGetMatchingService"); if (matching && getService) { #if TARGET_OS_OSX const unsigned own = getService(0, matching("AGXAccelerator")); #else const unsigned own = getService(0, matching("IOAcceleratorES")); #endif if (own) port = own; } } gAfterTurnip = bridgephase::run; adrenoUserTest(port, j->name); delete j; return nullptr; }, job) != 0) delete job; pthread_attr_destroy(&attributes); return nil; } @end #if TARGET_OS_OSX #include #include #include // WindowServer's frame pacing, as macOS-DroidVM's plugin (24af265, kc253-kc292), each behind its boot argument. // // ca-triple-buffer=1 sets QuartzCore's debug option CA_FORCE_TRIPLE_BUFFERED_LATENCY (read by IOMFBDisplay::can_update_): // CoreAnimation renders a frame ahead instead of starting each frame only after the last one's buffer came back, which it otherwise // does whenever a frame took longer than a refresh (DroidVM: a heavy window drag from 56-60 to 61-67 frames a second). __attribute__((constructor)) static void tripleBufferFromBootArgs() { if (strcmp(getprogname(), "WindowServer") || !publish::bootArgSet("ca-triple-buffer=1")) return; auto setOption = reinterpret_cast(dlsym(RTLD_DEFAULT, "CASetDebugOption")); auto optionName = reinterpret_cast(dlsym(RTLD_DEFAULT, "CAGetDebugOptionEnvVariableName")); // The option's index differs between QuartzCore builds, so it is found by its name. unsigned index = ~0u; for (unsigned i = 0; optionName && i < 1024; ++i) { const char *name = optionName(i); if (!name) break; if (!strcmp(name, "CA_FORCE_TRIPLE_BUFFERED_LATENCY")) { index = i; break; } } const bool set = setOption && index != ~0u; if (set) setOption(index, true); os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal: CA_FORCE_TRIPLE_BUFFERED_LATENCY %{public}s (option %u)", set ? "set" : "not found", index); } // ca-free-wakeup=1: SkyLight starts each WindowServer update at -[CAWindowServerDisplay nextWakeupTime], which QuartzCore puts on the // refresh grid (the last VBL plus a frame), so an update that took 13 ms is followed by one at 16.7 ms. This answers the earlier of that // and a refresh after the last update's start (-beginExternalUpdate:usingSoftwareRenderer:): the display shows a swap when it comes, // not at a VBL. SkyLight adds its updater delay (0.5 ms) and updates at once only when the last update is more than 1.25 frames before // the answer, so before the due time the answer is the due time less that delay, after it the least time 1.25 frames past the last // start. Without an update in the last second the grid's answer stands. The refresh is vsync='s (120 by default). // DroidVM: a heavy window drag from 60 to 96 frames a second. static std::atomic gUpdateBegan{0}, gLastAsk{0}; static double hostSeconds(uint64_t t) { static mach_timebase_info_data_t base; if (!base.denom) mach_timebase_info(&base); return (double)t * base.numer / base.denom / 1e9; } static uint64_t hostTicks(double seconds) { static mach_timebase_info_data_t base; if (!base.denom) mach_timebase_info(&base); return (uint64_t)(seconds * 1e9 * base.denom / base.numer); } __attribute__((constructor)) static void freeWakeupFromBootArgs() { if (strcmp(getprogname(), "WindowServer") || !publish::bootArgSet("ca-free-wakeup=1")) return; char args[1100] = {}; size_t size = sizeof args - 1; unsigned hz = 120; if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0) if (const char *at = publish::bootArgToken(args, "vsync=")) { const int value = atoi(at + 6); if (value >= 30 && value <= 240) hz = (unsigned)value; } const double refresh = 1.0 / hz; Class display = objc_getClass("CAWindowServerDisplay"); static SEL wakeSel = sel_registerName("nextWakeupTime"), beginSel = sel_registerName("beginExternalUpdate:usingSoftwareRenderer:"); Method wake = display ? class_getInstanceMethod(display, wakeSel) : nullptr; Method begin = display ? class_getInstanceMethod(display, beginSel) : nullptr; if (!wake || !begin || method_getTypeEncoding(wake)[0] != 'd' || method_getNumberOfArguments(begin) != 4) { os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal: free wakeup: CAWindowServerDisplay lacks %{public}s", !wake ? "nextWakeupTime" : "beginExternalUpdate:usingSoftwareRenderer:"); return; } static auto wakeGrid = reinterpret_cast(method_getImplementation(wake)); static auto beginOriginal = reinterpret_cast(method_getImplementation(begin)); method_setImplementation(begin, imp_implementationWithBlock(^uint64_t(id self, void *update, uint64_t software) { // The update's start is SkyLight's last ask before it, where its update callback began. const uint64_t now = mach_absolute_time(), asked = gLastAsk.load(std::memory_order_relaxed); gUpdateBegan.store(asked && now - asked < hostTicks(0.003) ? asked : now, std::memory_order_relaxed); return beginOriginal(self, beginSel, update, software); })); method_setImplementation(wake, imp_implementationWithBlock(^double(id self) { const double grid = wakeGrid(self, wakeSel); gLastAsk.store(mach_absolute_time(), std::memory_order_relaxed); const uint64_t began = gUpdateBegan.load(std::memory_order_relaxed); if (grid <= 0 || !began) return grid; const double now = hostSeconds(mach_absolute_time()), start = hostSeconds(began); if (now - start > 1.0) return grid; const double due = start + refresh - 0.0005, ahead = start + 1.25 * refresh + 0.001; if (now < due) return due < grid ? due : grid; return now > ahead ? now : ahead; })); os_log(OS_LOG_DEFAULT, "NativeAdrenoMetal: free wakeup on (%.2f ms)", refresh * 1e3); } #endif