diff --git a/NativeAdrenoMetalPlugin.mm b/NativeAdrenoMetalPlugin.mm index e96c685..869c137 100644 --- a/NativeAdrenoMetalPlugin.mm +++ b/NativeAdrenoMetalPlugin.mm @@ -89,6 +89,7 @@ static Gpu *makeGpu(unsigned port) { return gpu; } static std::string gCacheDirectory; // the runtime's cache directory (the one that opened the device), exported by CacheExport.h +static bool gFillsSystemCache; // the render server opened its device in the system shader cache's directory (SharedShaderCache.h) // Progress markers 0x68xx (GPU6): 6800 entered, 6801 Turnip found, 6802 device made, 6803 returned to the discovery, 68f1 no Turnip, 68f2 no device. static unsigned gDiagBits; // Reads of the render server's own state for the framebuffer-format question (diagnostic only, for the exact QuartzCore of the target image; a read @@ -375,6 +376,11 @@ static id deviceForPort(unsigned port) { setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1); CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); if (!nativeIsRenderServer()) { + // The system shader cache the render server fills (SharedShaderCache.h), read for what this process's own cache lacks. + // gpu-prime-system=0: no process reads it and the render server keeps its cache and its priming to itself, as before, for + // a comparison on the same image. + if (!bootArgSet("gpu-prime-system=0")) + native_bridge_metal_system_cache(systemcache::kReaderPath, systemcache::kOwners, 2, false); device = clientDevice(port); if (device) iogpu::install(device, port); return device; @@ -497,7 +503,7 @@ static id deviceForPort(unsigned port) { unsetenv("METAL2VULKAN_LOG_PIPELINES"); CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); - char candidates[6][512] = {}; + char candidates[7][512] = {}; if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0; if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0; strlcpy(candidates[2], "/tmp/", sizeof candidates[2]); @@ -511,18 +517,23 @@ static id deviceForPort(unsigned port) { strlcat(candidates[4], "/", sizeof candidates[4]); } strlcpy(candidates[5], sharedShaderCacheDirectory().c_str(), sizeof candidates[5]); - static const unsigned order[] = {5, 4, 0, 1, 2, 3}; +#if TARGET_OS_OSX + // First the system shader cache's directory, where the kernel made it for this process (marker 0x6816): what the render + // server compiles and primes there, every process reads (SharedShaderCache.h). Not with gpu-prime-system=0. + if (!bootArgSet("gpu-prime-system=0")) strlcpy(candidates[6], systemcache::writerDirectory().c_str(), sizeof candidates[6]); +#endif + static const unsigned order[] = {6, 5, 4, 0, 1, 2, 3}; for (unsigned i : order) { if (device) break; if (!candidates[i][0]) continue; if (i == 0) {strlcat(candidates[i], "com.apple.metal", sizeof candidates[i]); mkdir(candidates[i], 0700); strlcat(candidates[i], "/", sizeof candidates[i]);} // as for the clients below - if (i != 5) strlcat(candidates[i], "m2v", sizeof candidates[i]); - mkdir(candidates[i], 0700); + if (i < 5) strlcat(candidates[i], "m2v", sizeof candidates[i]); + if (i != 6) mkdir(candidates[i], 0700); if (access(candidates[i], W_OK) != 0) continue; // Files an earlier image's copying import left half written here (int 590aaee2), which hold memory of the Data disk. { static bool once; - if (!once && i != 5) {once = true; removeHalfImported(candidates[i]);} + if (!once && i < 5) {once = true; removeHalfImported(candidates[i]);} } // The image's pipeline pack (an earlier boot's export), into this directory before the runtime reads it. Marker 0x6Cnn: files written // (saturated at 0xfe), 0x6Cff the pack was refused. @@ -530,12 +541,14 @@ static id deviceForPort(unsigned port) { const int imported = cacheexport::importPack(pack, candidates[i]); if (imported) gpu->mark(0x6C00 | (imported < 0 ? 0xff : imported > 0xfe ? 0xfe : imported)); } + // What the device writes in the system cache's directory is readable by everyone; in any other, by this user alone. + native_bridge_metal_system_cache(nullptr, nullptr, 0, i == 6); @try { device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr); } @catch (NSException *exception) { device = nil; } - if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i];} // which directory served + if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i]; gFillsSystemCache = i == 6;} // which directory served } if (!device) { gpu->mark(0x68f2); @@ -586,7 +599,8 @@ static id deviceForPort(unsigned port) { pthread_setname_np("native-metal-prime"); // a process sample and the telemetry's TH rows name its CPU time char list[1024]; prime::run((id)device, gCacheDirectory, 90, - requestedPath("NativeAdrenoMetal.primelist", list, sizeof list) ? std::string(list) : std::string()); + requestedPath("NativeAdrenoMetal.primelist", list, sizeof list) ? std::string(list) : std::string(), + gFillsSystemCache); return nullptr; }, nullptr); pthread_attr_destroy(&attributes); diff --git a/NativeBridgeMetalFrontend.h b/NativeBridgeMetalFrontend.h index 9a91ea7..380b11c 100644 --- a/NativeBridgeMetalFrontend.h +++ b/NativeBridgeMetalFrontend.h @@ -9,6 +9,11 @@ extern "C" { // Opens the process's one device: loader exports vkGetInstanceProcAddr (Turnip), cache is a writable shader-cache directory, registryID is the // accelerator service's registry entry. missing (may be null) is told the class and selector of every unimplemented call. id native_bridge_metal_open(const char *loader, const char *cache, uint64_t registryID, void (*missing)(const char *, const char *)); +// The system shader cache (SharedShaderCache.h), named before native_bridge_metal_open. A reader passes the layer's directory and the +// user ids whose files it trusts there: its device then asks that directory, read only and by the layer's owner and mode rules, for +// what its own cache lacks. The process that fills the layer passes writer true and opens its device in the layer's directory: what +// it writes is readable by everyone. Returns whether the layer was taken; the reason of a refusal is in the `cache:` log line. +bool native_bridge_metal_system_cache(const char *reader, const uint32_t *owners, unsigned count, bool writer); // Bare AIR bitcode as a library (the bridge's fixtures; Metal itself accepts only metallibs). id native_bridge_metal_library_with_air(const void *bytes, size_t length); // One line for the plugin's status report: command buffers committed/failed locally, pipelines made, refusals, the last refusal's text. diff --git a/NativeBridgeMetalFrontend.mm b/NativeBridgeMetalFrontend.mm index 59e6875..33118d5 100644 --- a/NativeBridgeMetalFrontend.mm +++ b/NativeBridgeMetalFrontend.mm @@ -619,6 +619,9 @@ __attribute__((weak)) volatile uint32_t native_prime_libraries_total; __attribute__((weak)) volatile uint32_t native_prime_made; __attribute__((weak)) volatile uint32_t native_prime_rate; __attribute__((weak)) volatile uint32_t native_prime_functions_resumed; +__attribute__((weak)) volatile uint32_t native_prime_libraries_primed; // libraries this process's passes compiled +__attribute__((weak)) volatile uint32_t native_prime_libraries_skipped; // other GPUs' driver bundles (whole system) or not loaded here +__attribute__((weak)) volatile uint32_t native_prime_scope; // 0 no pass yet, 1 loaded bundles only, 2 the whole system } static void refuseSurface(uint32_t step,IOSurfaceRef surface,MTLTextureDescriptor *d,NSUInteger plane,uint64_t offset,uint64_t row) { gSurfaceRefusals++; @@ -660,15 +663,32 @@ static void onLargeStack(void (^body)(void)) { // its Metal cache directory but not Library/Caches: with the replies only there it kept none, and each of its launches compiled // every request again. static std::string gCacheRoot; // the pipeline cache directory the device opened with (native_bridge_metal_open) +// The system shader cache as this process uses it (native_bridge_metal_system_cache): 0 not named, 1 read as a layer, 2 filled by +// this process, 3 refused; its directory or the refusal's reason; and the compiler replies the layer answered. +static int gSystemCacheState; +static std::string gSystemCacheText; +static std::atomic gCompileSystem; struct NBShaderCacheHeader {uint32_t magic,warningBytes;uint64_t metallibBytes;unsigned char digest[CC_SHA256_DIGEST_LENGTH];}; static const uint32_t kNBShaderCacheMagic=0x3143534e; // "NSC1" -static NSString *shaderCachePath(const unsigned char uuid[16],uint32_t type,const std::vector &request) { +static NSString *shaderCacheName(const unsigned char uuid[16],uint32_t type,const std::vector &request) { + CC_SHA256_CTX context;CC_SHA256_Init(&context); + CC_SHA256_Update(&context,uuid,16);CC_SHA256_Update(&context,&type,sizeof type); + CC_SHA256_Update(&context,request.data(),(CC_LONG)request.size()); + unsigned char key[CC_SHA256_DIGEST_LENGTH];CC_SHA256_Final(key,&context); + char name[2*CC_SHA256_DIGEST_LENGTH+8]; + for(int i=0;i *roots=[NSMutableArray array]; + // The process that fills the system cache keeps its replies there, where every process reads them. + if(gSystemCacheState==2 && !gCacheRoot.empty())[roots addObject:@(gCacheRoot.c_str())]; if(!shared.empty())[roots addObject:@(shared.c_str())]; if(!gCacheRoot.empty())[roots addObject:@(gCacheRoot.c_str())]; if(NSString *caches=NSSearchPathForDirectoriesInDomains(NSCachesDirectory,NSUserDomainMask,YES).firstObject)[roots addObject:caches]; @@ -679,17 +699,9 @@ static NSString *shaderCachePath(const unsigned char uuid[16],uint32_t type,cons } }); if(!directory)return nil; - CC_SHA256_CTX context;CC_SHA256_Init(&context); - CC_SHA256_Update(&context,uuid,16);CC_SHA256_Update(&context,&type,sizeof type); - CC_SHA256_Update(&context,request.data(),(CC_LONG)request.size()); - unsigned char key[CC_SHA256_DIGEST_LENGTH];CC_SHA256_Final(key,&context); - char name[2*CC_SHA256_DIGEST_LENGTH+8]; - for(int i=0;i &request, } }); if(!service){*failure=@"the system's Metal compiler (MTLCompiler.framework) is not available";return nil;} - NSString *cached=shaderCachePath(compilerUuid,type,request); + NSString *cachedName=shaderCacheName(compilerUuid,type,request); + NSString *cached=shaderCachePath(cachedName); NSData *kept=nil;NSString *keptWarnings=nil; - if(shaderCacheRead(cached,&kept,&keptWarnings)) { + if(shaderCacheRead(cached,cachedName,&kept,&keptWarnings)) { gCompileCached++; *warnings=keptWarnings;*failure=nil; return kept; @@ -4831,20 +4858,24 @@ static void logClientWork() { dispatch_source_set_timer(timer,dispatch_time(DISPATCH_TIME_NOW,30*NSEC_PER_SEC),30*NSEC_PER_SEC,5*NSEC_PER_SEC); dispatch_source_set_event_handler(timer,^{ static uint64_t last[3]; - uint64_t vp[8]={},work[10]={},mc[10]={}; + uint64_t vp[8]={},work[10]={},mc[10]={},sy[4]={}; if(!nmb_device_pipeline_counts(gDevice,vp))return; + (void)nmb_system_cache_counts(sy); const uint64_t now[3]={gTimePipeline.count.load(),gTimeCompile.count.load()+gCompileCached.load(),vp[0]}; if(!memcmp(now,last,sizeof now))return; memcpy(last,now,sizeof now); (void)nmb_device_work_stats(gDevice,work);(void)nmb_device_memory_counts(gDevice,mc); os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal work: pipeline %llu/%llums/%llums library %llu/%llums compile %llu/%llums cached %llu | vk %llu " - "create %llu/%llu step %llu drv %llu+%llums | cache hit %llu miss %llu wr %llu | mem %llu/%llu/%lluM | problems %llu", + "create %llu/%llu step %llu drv %llu+%llums | cache hit %llu miss %llu wr %llu | system %llu/%lluK rej %llu refused %llu replies %llu | " + "mem %llu/%llu/%lluM | problems %llu", (unsigned long long)gTimePipeline.count.load(),(unsigned long long)(gTimePipeline.total.load()/1000), (unsigned long long)(gTimePipeline.slowest.load()/1000),(unsigned long long)gTimeLibrary.count.load(), (unsigned long long)(gTimeLibrary.total.load()/1000),(unsigned long long)gTimeCompile.count.load(), (unsigned long long)(gTimeCompile.total.load()/1000),(unsigned long long)gCompileCached.load(),(unsigned long long)vp[0], (unsigned long long)vp[1],(unsigned long long)vp[2],(unsigned long long)vp[3],(unsigned long long)(vp[4]/1000000), (unsigned long long)(vp[5]/1000000),(unsigned long long)work[7],(unsigned long long)work[8],(unsigned long long)work[9], + (unsigned long long)sy[0],(unsigned long long)(sy[3]>>10),(unsigned long long)sy[1],(unsigned long long)sy[2], + (unsigned long long)gCompileSystem.load(), (unsigned long long)(mc[0]>>20),(unsigned long long)(mc[2]>>20),(unsigned long long)(mc[3]>>20),(unsigned long long)gProblems.load()); }); dispatch_resume(timer); @@ -4888,6 +4919,15 @@ static void logClientMemory() { dispatch_resume(timer); } +extern "C" bool native_bridge_metal_system_cache(const char *reader,const uint32_t *owners,unsigned count,bool writer) { + if(gDeviceObject)return false; + nmb_cache_readable_by_all(writer); + gSystemCacheState=writer?2:0;gSystemCacheText=writer?"filled here":""; + if(writer || !reader){(void)nmb_system_cache_open(nullptr,nullptr,0);return writer;} + if(nmb_system_cache_open(reader,owners,count)){gSystemCacheState=1;gSystemCacheText=reader;return true;} + gSystemCacheState=3;gSystemCacheText=nmb_last_error(); + return false; +} extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint64_t registryID,void (*missing)(const char *,const char *)) { if(gDeviceObject)return gDeviceObject; gMissing=missing;gInFlight=[NSMutableSet set]; @@ -4905,7 +4945,12 @@ extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint logClientWork(); logClientMemory(); // Where this process's pipeline cache is and whose it is: a cache serves the processes of its user that may write its directory. - if(diagOn() || logGpuOn())os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal cache: %{public}s uid %d",cache?cache:"none",(int)getuid()); + // Then the system layer: read from its directory, filled by this process, refused with the level and why, or not named. + if(diagOn() || logGpuOn()) { + static const char *const kStates[]={"none","read","filled","refused"}; + os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal cache: %{public}s uid %d system %{public}s %{public}s",cache?cache:"none",(int)getuid(), + kStates[gSystemCacheState],gSystemCacheText.c_str()); + } if(cache && logGpuOn()){gJournalPath=std::string(cache)+"/pipelines-"+std::to_string(getpid())+".log";gJournalBegan=nowUs();} char name[128]={};nmb_device_name(gDevice,name,sizeof name);gName=[NSString stringWithUTF8String:name];gRegistryID=registryID; // The target families supportsFamily: answers true, with the rows of Apple's tables the device does not meet in them (nmb_device_family_waivers). @@ -5283,6 +5328,20 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) { } } row("PR compiling=%u libs=%u/%u made=%u rate=%u resumed=%u",native_prime_compiling,native_prime_libraries_done,native_prime_libraries_total,native_prime_made,native_prime_rate,native_prime_functions_resumed); + // Priming's reach and the system shader cache. PRb: the pass's scope (1 the bundles loaded here, 2 the whole system), the libraries + // it compiled itself, those it leaves out (other GPUs' driver bundles, or not loaded here), the system cache's state here (0 not + // named, 1 read, 2 filled by this process, 3 refused) and what this process published: the entries and KiB its cache directory + // took while it is the system layer. SY, in a process that reads the layer: the entries it answered, their KiB, the contents + // refused, the levels refused for owner or mode, and the compiler replies it answered. + { + uint64_t sc[24]={},sy[4]={}; + if(gDevice)(void)nmb_device_shader_cache_detail(gDevice,sc); + (void)nmb_system_cache_counts(sy); + row("PRb scope=%u primed=%u left=%u | system %d pub %llu/%lluK",native_prime_scope,native_prime_libraries_primed, + native_prime_libraries_skipped,gSystemCacheState,(ull)(gSystemCacheState==2?sc[11]+sc[12]:0),(ull)(gSystemCacheState==2?sc[13]>>10:0)); + if(gSystemCacheState==1 || gSystemCacheState==3) + row("SY read %llu/%lluK rej %llu refused %llu replies %llu",(ull)sy[0],(ull)(sy[3]>>10),(ull)sy[1],(ull)sy[2],(ull)gCompileSystem.load()); + } if(gDevice) { // The warm pass (records warmed, failed, unread, KiB of driver data) and where work ran: scheduler passes and batches at commit, render // log waits, encoder ends on the lowering thread. diff --git a/PrimeShaders.h b/PrimeShaders.h index 9460f59..6db8352 100644 --- a/PrimeShaders.h +++ b/PrimeShaders.h @@ -39,7 +39,10 @@ extern "C" unsigned native_bridge_metal_recent_commits(uint64_t windowUs); // settle, 3 it opens a library and its functions), libraries done of the list, pipelines made, and the most command buffers the last wait // saw in 250 ms. extern "C" {volatile uint32_t native_prime_compiling, native_prime_libraries_done, native_prime_libraries_total, native_prime_made, native_prime_rate, - native_prime_functions_resumed;} + native_prime_functions_resumed; +// For the PRb row: the libraries this process's passes compiled, those a pass leaves out, and the pass's scope (1 the bundles loaded +// in this process, 2 the whole system). +volatile uint32_t native_prime_libraries_primed, native_prime_libraries_skipped, native_prime_scope;} extern "C" void native_bridge_metal_quiet_problems(bool); namespace prime { @@ -187,6 +190,17 @@ static bool libraryOfLoadedBundle(const std::string &path) { } return false; } +// macOS: whether a library is in another GPU's driver bundle (/System/Library/Extensions/.bundle: AGXMetal* and the like, 167 of +// the 400 libraries of a macOS 27 image). IOGPU loads the bundle of the accelerator found, which here is this plugin, so nothing +// asks for those libraries on this GPU. +static bool libraryOfDriverBundle(const std::string &path) { + static const char kCryptex[] = "/System/Cryptexes/OS", kExtensions[] = "/System/Library/Extensions/"; + const char *plain = path.c_str(); + if (strncmp(plain, kCryptex, sizeof kCryptex - 1) == 0 && plain[sizeof kCryptex - 1] == '/') plain += sizeof kCryptex - 1; + if (strncmp(plain, kExtensions, sizeof kExtensions - 1) != 0) return false; + const char *slash = strchr(plain + sizeof kExtensions - 1, '/'); + return slash && slash - plain > 7 && strncmp(slash - 7, ".bundle", 7) == 0; +} #endif extern "C" const char *nmb_runtime_build(void); @@ -349,7 +363,10 @@ static std::string library(id device, const std::string &path, const } // The thread: waits for the desktop to settle, then works through the libraries not finished yet. -static void run(id device, std::string root, unsigned delaySeconds, const std::string &listPath) { +// `wholeSystem` (macOS): this process fills the system shader cache (SharedShaderCache.h), which every process reads, so every +// library of the list is primed but those of other GPUs' driver bundles; without it only the libraries of bundles loaded here are, +// since the cache then serves this process alone. +static void run(id device, std::string root, unsigned delaySeconds, const std::string &listPath, bool wholeSystem) { pthread_set_qos_class_self_np(QOS_CLASS_BACKGROUND, 0); native_bridge_metal_quiet_problems(true); while (root.size() > 1 && root.back() == '/') root.pop_back(); @@ -370,6 +387,11 @@ static void run(id device, std::string root, unsigned delaySeconds, c const std::string build = buildId(); appendLine(logPath, note + " build=" + build); native_prime_libraries_total = (uint32_t)paths.size(); +#if TARGET_OS_OSX + native_prime_scope = wholeSystem ? 2 : 1; +#else + native_prime_scope = 2; // iPadOS primes every library of the list, whatever its bundle +#endif for (const std::string &path : paths) { ++libs; native_prime_libraries_done = finished; @@ -379,10 +401,15 @@ static void run(id device, std::string root, unsigned delaySeconds, c const std::string id = path + "\t" + std::to_string((long long)st.st_size) + "\t" + std::to_string((long long)st.st_mtimespec.tv_sec) + "\t" + build; if (done.count(id)) {++finished; continue;} #if TARGET_OS_OSX - // Counted as done for the telemetry's progress, and not listed as finished (libraryOfLoadedBundle). - if (!libraryOfLoadedBundle(path)) {++foreign; ++finished; continue;} + // Counted as done for the telemetry's progress, and not listed as finished (libraryOfDriverBundle, libraryOfLoadedBundle). + if (wholeSystem ? libraryOfDriverBundle(path) : !libraryOfLoadedBundle(path)) { + ++foreign; ++finished; + native_prime_libraries_skipped = foreign; + continue; + } #endif appendLine(logPath, library(device, path, current, bad, functionsPath, functionsDone, build)); + native_prime_libraries_primed = native_prime_libraries_primed + 1; appendLine(donePath, id); done.insert(id); ++finished; @@ -390,7 +417,8 @@ static void run(id device, std::string root, unsigned delaySeconds, c native_prime_libraries_done = finished; native_prime_compiling = 0; char line[120]; - snprintf(line, sizeof line, "pass libraries=%u finished=%u not-loaded=%u ms=%llu", libs, finished, foreign, (unsigned long long)(nowMs() - began)); + snprintf(line, sizeof line, "pass libraries=%u finished=%u %s=%u ms=%llu", libs, finished, wholeSystem ? "driver-bundles" : "not-loaded", + foreign, (unsigned long long)(nowMs() - began)); appendLine(logPath, line); } } // namespace prime diff --git a/SharedShaderCache.h b/SharedShaderCache.h index f02320e..1544515 100644 --- a/SharedShaderCache.h +++ b/SharedShaderCache.h @@ -33,6 +33,33 @@ inline std::string sharedShaderCacheDirectory() { return path; } +// The system shader cache (macOS): one directory the render server fills and every process reads. +// The render server (WindowServer, the user _windowserver) is not the console user, so the per-user directory above serves it alone, +// and its sandbox lets it write only its own cache and temporary directories, /private/tmp and /private/var/tmp, while the one +// directory every sandbox that may use graphics can read is /Library/GPUBundles (system.sb's system-graphics, the WebKit GPU +// process's profile). No path is both, so the kernel makes the directory at boot where the render server may write it +// (kWriterPath: owner _windowserver, mode 0755) and shows it read-only where everyone may read it (kReaderPath, a read-only bind +// mount beside the plugin). The render server opens its device in the first (its cache directory, made readable by everyone) and +// primes the system's Metal libraries into it; every other process asks the second for what its own cache lacks. +// A reader trusts the directory only by who can write it: it, every directory on the way to an entry and the entry must be owned by +// the superuser or the render server's user and not writable by group or others, each opened without following a link (the +// runtime's system layer, bridge patch 0036). An entry is then checked as one of the process's own cache is (name, build, key, +// profile, checksum). gpu-prime-system=0 turns all of it off. +namespace systemcache { +static const char kWriterPath[] = "/private/var/tmp/NativeAdrenoMetal.cache"; +static const char kReaderPath[] = "/Library/GPUBundles/NativeAdrenoMetal.cache"; +static const uint32_t kOwners[] = {0, 88}; // the superuser and _windowserver +// The directory this process fills, when the kernel made it for this process's user (or this process is the superuser): never +// made here, since /private/var/tmp takes anyone's directories. +inline std::string writerDirectory() { + struct stat st = {}; + if (lstat(kWriterPath, &st) || !S_ISDIR(st.st_mode) || (st.st_mode & 0022)) return {}; + if (st.st_uid != kOwners[0] && st.st_uid != kOwners[1]) return {}; + if (st.st_uid != getuid() && getuid() != 0) return {}; + return access(kWriterPath, W_OK) == 0 ? std::string(kWriterPath) : std::string(); +} +} // namespace systemcache + // Call before the first Vulkan device is opened, including the optional smoke path. inline void configureShaderDiskCache(const std::string &root) { if (root.empty()) return; diff --git a/bridge/NativeMetalObjects.h b/bridge/NativeMetalObjects.h index d864a9b..f60198f 100644 --- a/bridge/NativeMetalObjects.h +++ b/bridge/NativeMetalObjects.h @@ -45,6 +45,21 @@ bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]); * rejected, read failures, writes, writes queued, writes on a calling thread, bytes written, queued hits, write failures, evictions, driver cache loads, * driver pipeline hits, driver pipeline misses, driver saves, then three spare zeros. */ bool nmb_device_shader_cache_detail(const NmbDevice *, uint64_t out[24]); +/* The read-only system cache layer of the device opened after this call: the directory at root, read only where it, every directory + * on the way to an entry and the entry are owned by one of the count user ids and not writable by group or others, never through a + * link. A device asks its own cache directory first and the layer for what that lacks. False with nmb_last_error where the directory + * fails the test (no layer then); a null root names none. */ +bool nmb_system_cache_open(const char *root, const uint32_t *owners, size_t count); +/* The cache directory of the device opened after this call makes its directories and entries readable by everyone (0755, 0644): + * the process that fills the system layer through its own directory. */ +void nmb_cache_readable_by_all(bool on); +/* Entries read from the system layer and accepted, entries whose contents were refused, levels refused for kind, owner or mode, + * bytes of the accepted entries. False without a layer. */ +bool nmb_system_cache_counts(uint64_t out[4]); +/* A file under the system layer by the layer's rules, at most limit bytes: a malloc block the caller frees, or null. The caller + * checks the contents and reports its verdict. */ +uint8_t *nmb_system_cache_read(const char *relative, uint64_t limit, size_t *length); +void nmb_system_cache_verdict(bool accepted, uint64_t bytes); /* The translator source the runtime was built from, which keys the shader cache (build-native-metal-bridge.py). */ const char *nmb_runtime_build(void); // Linear images made/refused/refused again, shadowed textures, shadow copies, texels copied back, linear passes in place/through copies. diff --git a/bridge/NativeMetalObjects.rs b/bridge/NativeMetalObjects.rs index 105374c..0e984e3 100644 --- a/bridge/NativeMetalObjects.rs +++ b/bridge/NativeMetalObjects.rs @@ -572,6 +572,102 @@ pub extern "C" fn nmb_stage_ring_read(from: u64, out: *mut metal2vulkan_runtime: count } +/// Name the read-only system cache layer for the device opened after this call (bridge patch 0036): the directory at `root`, read +/// only where it, every directory on the way to an entry and the entry are owned by one of the `count` user ids at `owners` and +/// not writable by group or others, and never through a link. False with the reason in `nmb_last_error` where the directory fails +/// that test; no layer is named then. A null `root` names none. +/// +/// # Safety +/// `root` is null or a C string; `owners` points at `count` user ids. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_system_cache_open(root: *const c_char, owners: *const u32, count: usize) -> bool { + guard(false, || { + metal2vulkan_runtime::set_system_layer(None); + if root.is_null() { + return Ok(true); + } + if owners.is_null() || count == 0 { + return Err("system cache: no owner".into()); + } + // SAFETY: the caller's contract. + let (root, owners) = unsafe { (CStr::from_ptr(root), std::slice::from_raw_parts(owners, count)) }; + let layer = metal2vulkan_runtime::SystemLayer::open(&PathBuf::from(root.to_string_lossy().into_owned()), owners) + .map_err(|why| format!("system cache: {why}"))?; + metal2vulkan_runtime::set_system_layer(Some(std::sync::Arc::new(layer))); + Ok(true) + }) +} + +/// Whether the cache directory of the device opened after this call makes its directories and entries readable by everyone +/// (0755, 0644) instead of by their owner alone: the directory of the process that fills the system layer. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_cache_readable_by_all(on: bool) { + metal2vulkan_runtime::set_cache_readable_by_all(on); +} + +/// The system layer's counts into the caller's four words: entries read from it and accepted, entries whose contents were refused, +/// levels refused for their kind, owner or mode, and the accepted entries' bytes. False without a layer. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_system_cache_counts(out: *mut u64) -> bool { + guard(false, || { + if out.is_null() { + return Err("null counts".into()); + } + let Some(layer) = metal2vulkan_runtime::system_layer() else { + return Ok(false); + }; + let c = layer.counts(); + // SAFETY: the header's four words. + unsafe { std::slice::from_raw_parts_mut(out, 4) }.copy_from_slice(&[c.accepted, c.rejected, c.refused, c.accepted_bytes]); + Ok(true) + }) +} + +/// The bytes of the file at `relative` under the system layer, read by the layer's rules, at most `limit`: a block of `malloc` +/// the caller frees, its length in `length`; null where there is no layer or the file is absent, refused or longer. The caller +/// checks the contents and reports the verdict (`nmb_system_cache_verdict`). +/// +/// # Safety +/// `relative` is a C string and `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_system_cache_read(relative: *const c_char, limit: u64, length: *mut usize) -> *mut u8 { + guard(std::ptr::null_mut(), || { + if relative.is_null() || length.is_null() { + return Err("system cache read: null argument".into()); + } + let Some(layer) = metal2vulkan_runtime::system_layer() else { + return Ok(std::ptr::null_mut()); + }; + // SAFETY: the caller's contract. + let relative = unsafe { CStr::from_ptr(relative) }.to_string_lossy().into_owned(); + let Some(bytes) = layer.read(std::path::Path::new(&relative), limit) else { + return Ok(std::ptr::null_mut()); + }; + // SAFETY: a block of the bytes' length (at least one byte), filled from them; `length` is writable. + unsafe { + let block = libc::malloc(bytes.len().max(1)).cast::(); + if block.is_null() { + return Err("system cache read: no memory".into()); + } + std::ptr::copy_nonoverlapping(bytes.as_ptr(), block, bytes.len()); + *length = bytes.len(); + Ok(block) + } + }) +} + +/// Count what the caller made of `bytes` bytes `nmb_system_cache_read` gave: accepted, or refused by its own check. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_system_cache_verdict(accepted: bool, bytes: u64) { + if let Some(layer) = metal2vulkan_runtime::system_layer() { + if accepted { + layer.accepted(bytes); + } else { + layer.rejected(); + } + } +} + /// Free the runtime's spare dedicated blocks and kept empty pooled blocks (bridge patch 0021), as Metal gives its resource cache's /// memory back when the system reports memory pressure: how many blocks went. #[unsafe(no_mangle)] diff --git a/bridge/patches/0036-system-shader-cache-layer.patch b/bridge/patches/0036-system-shader-cache-layer.patch new file mode 100644 index 0000000..bb5e09e --- /dev/null +++ b/bridge/patches/0036-system-shader-cache-layer.patch @@ -0,0 +1,516 @@ +diff --git a/crates/metal2vulkan-runtime/src/lib.rs b/crates/metal2vulkan-runtime/src/lib.rs +index 03287cc..048d3d4 100644 +--- a/crates/metal2vulkan-runtime/src/lib.rs ++++ b/crates/metal2vulkan-runtime/src/lib.rs +@@ -360,6 +360,9 @@ pub use resource_ids::{IdPlacement, ResourceId, ResourceIdKind}; + pub use sampler::{sampler_border_color, SamplerState, SamplerStateOptions}; + pub use creation::{allocation_sites, AllocationSite}; + pub use memory::{MemoryKinds, MEMORY_KIND_CLASSES}; ++pub use shader_cache::system::{ ++ set_cache_readable_by_all, set_system_layer, system_layer, SystemLayer, SystemLayerCounts, ++}; + pub use shader_cache::{ + ShaderCacheOptions, ShaderCacheStatistics, DEFAULT_CAPACITY, DEFAULT_MEMORY_CAPACITY, + }; +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +index 508af31..ab148b2 100644 +--- a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs ++++ b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +@@ -71,6 +71,7 @@ pub(crate) use creation::VariantStages; + pub(crate) mod driver; + mod held; + pub(super) mod store; ++pub(crate) mod system; + pub(super) mod worker; + + use super::error::{invalid, Error, Problem, ShaderCacheError}; +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/store.rs b/crates/metal2vulkan-runtime/src/shader_cache/store.rs +index 95f4d86..539a35f 100644 +--- a/crates/metal2vulkan-runtime/src/shader_cache/store.rs ++++ b/crates/metal2vulkan-runtime/src/shader_cache/store.rs +@@ -76,6 +76,9 @@ pub(crate) struct Directory { + root: PathBuf, + build: Hash256, + capacity: u64, ++ /// Its directories and entries are made readable by everyone (`system.rs`): the directory ++ /// of the process that fills the system layer. ++ public: bool, + } + + impl Directory { +@@ -92,17 +95,20 @@ impl Directory { + root, + build, + capacity, ++ public: super::system::cache_readable_by_all(), + }; + let failed = |path: &Path| { + let path = path.to_path_buf(); + move |error| Failed { path, error } + }; + let made = (|| { +- create_private_dir_all(&directory.root).map_err(failed(&directory.root))?; ++ directory.create_dir_all(&directory.root).map_err(failed(&directory.root))?; ++ let layout_dir = directory.layout_dir(); ++ directory.create_dir_all(&layout_dir).map_err(failed(&layout_dir))?; + let build_dir = directory.build_dir(); +- create_private_dir_all(&build_dir).map_err(failed(&build_dir))?; ++ directory.create_dir_all(&build_dir).map_err(failed(&build_dir))?; + let probe = build_dir.join(format!(".probe.{}.tmp", unique())); +- let written = new_private_file(&probe).and_then(|mut file| file.write_all(b"probe")); ++ let written = directory.new_file(&probe).and_then(|mut file| file.write_all(b"probe")); + let removed = fs::remove_file(&probe); + written.and(removed).map_err(failed(&directory.root)) + })(); +@@ -133,6 +139,41 @@ impl Directory { + self.layout_dir().join(hex(&self.build)) + } + ++ /// Create `dir` and its missing parents: each only its owner may enter, or, in a directory ++ /// made for every reader (`system.rs`), everyone may read and enter, whatever the process's ++ /// file mode mask says. ++ fn create_dir_all(&self, dir: &Path) -> io::Result<()> { ++ if !self.public { ++ return create_private_dir_all(dir); ++ } ++ let mut builder = fs::DirBuilder::new(); ++ builder.recursive(true); ++ #[cfg(unix)] ++ { ++ use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; ++ builder.mode(0o755); ++ builder.create(dir)?; ++ if fs::metadata(dir)?.permissions().mode() & 0o777 != 0o755 { ++ fs::set_permissions(dir, fs::Permissions::from_mode(0o755))?; ++ } ++ Ok(()) ++ } ++ #[cfg(not(unix))] ++ builder.create(dir) ++ } ++ ++ /// Create the new file `path`, which only its owner may read, or everyone in a directory ++ /// made for every reader; fails if it exists. ++ fn new_file(&self, path: &Path) -> io::Result { ++ let file = new_private_file(path)?; ++ #[cfg(unix)] ++ if self.public { ++ use std::os::unix::fs::PermissionsExt; ++ file.set_permissions(fs::Permissions::from_mode(0o644))?; ++ } ++ Ok(file) ++ } ++ + /// The path of the entry named `name`. + pub(crate) fn entry_path(&self, name: &str) -> PathBuf { + self.build_dir().join(&name[..2]).join(name) +@@ -204,6 +245,9 @@ enum Found { + Bytes(Vec, fs::File), + /// The entry's bytes, from a write the cache thread has not made yet: no file to refresh. + Queued(Vec), ++ /// The entry's bytes, from the system layer (`system.rs`): not this directory's to refresh ++ /// or remove. ++ System(Vec), + } + + /// The counts behind [`ShaderCacheStatistics`](super::ShaderCacheStatistics). +@@ -347,6 +391,8 @@ pub(crate) struct Store { + /// The device's cache thread, which writes the entries this store is given; `None` in a + /// store with no thread, which writes them on the calling thread. + asked: Option>, ++ /// The system layer this store asks for what its directory lacks (`system.rs`). ++ system: Option>, + } + + /// What a device's store says once each, through its instance's message sink, else on stderr: +@@ -383,9 +429,17 @@ impl Store { + write_failed: AtomicBool::new(false), + }, + asked, ++ system: super::system::system_layer(), + } + } + ++ /// The system layer's bytes of the file this directory would hold at `path`. ++ fn system_read(&self, path: &Path) -> Option> { ++ let system = self.system.as_ref()?; ++ let relative = path.strip_prefix(self.directory.root()).ok()?; ++ system.read(relative, self.entry_limit()) ++ } ++ + /// Count an entry not written (`ShaderCacheStatistics::write_failures`); the first is + /// announced as [`Degradation::ShaderCacheEntryNotWritten`], naming the directory and its + /// cap. Never an error: the work the entry would have saved is the directory's regardless. +@@ -474,6 +528,10 @@ impl Store { + count(&self.counters.holds_questions); + timed(&self.counters.read_nanos, || { + fs::metadata(path).is_ok_and(|metadata| metadata.len() <= self.entry_limit()) ++ || self.system.as_ref().is_some_and(|system| { ++ path.strip_prefix(self.directory.root()) ++ .is_ok_and(|relative| system.holds(relative, self.entry_limit())) ++ }) + }) + } + +@@ -513,6 +571,10 @@ impl Store { + } + Found::Bytes(bytes, file) => (bytes, Some(file)), + Found::Queued(bytes) => (bytes, None), ++ Found::System(bytes) => { ++ let length = bytes.len() as u64; ++ return self.system_verdict(decode(bytes), length); ++ } + }; + match decode(bytes) { + Some(value) => { +@@ -526,6 +588,23 @@ impl Store { + } + } + ++ /// What a decoder made of `length` bytes the system layer gave: counted there, and a ++ /// refusal in this store's rejections too. Nothing is touched or removed. ++ fn system_verdict(&self, decoded: Option, length: u64) -> Lookup { ++ let system = self.system.as_ref().expect("bytes of the system layer"); ++ match decoded { ++ Some(value) => { ++ system.accepted(length); ++ Lookup::Hit(value) ++ } ++ None => { ++ system.rejected(); ++ count(&self.counters.rejected); ++ Lookup::Rejected ++ } ++ } ++ } ++ + /// [`Store::load`] of the file at `path`. + fn load_at(&self, path: &Path, decode: impl FnOnce(&[u8]) -> Result) -> Lookup { + let (bytes, file) = match self.read(path) { +@@ -536,6 +615,9 @@ impl Store { + } + Found::Bytes(bytes, file) => (bytes, Some(file)), + Found::Queued(bytes) => (bytes, None), ++ Found::System(bytes) => { ++ return self.system_verdict(decode(&bytes).ok(), bytes.len() as u64); ++ } + }; + match decode(&bytes) { + Ok(value) => { +@@ -594,7 +676,19 @@ impl Store { + return Found::Queued(bytes); + } + count(&self.counters.entry_reads); +- timed(&self.counters.read_nanos, || { ++ timed(&self.counters.read_nanos, || match self.read_own(path) { ++ // What this directory lacks or cannot give, the system layer may hold. ++ found @ (Found::Absent | Found::Failed) => match self.system_read(path) { ++ Some(bytes) => Found::System(bytes), ++ None => found, ++ }, ++ found => found, ++ }) ++ } ++ ++ /// [`Store::read`] of this directory's own file at `path`. ++ fn read_own(&self, path: &Path) -> Found { ++ { + let file = match fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Found::Absent, +@@ -617,7 +711,7 @@ impl Store { + } + Err(_) => Found::Failed, + } +- }) ++ } + } + + /// Give `bytes` to the cache thread to write as the entry named `name` (`worker.rs`): the +@@ -678,10 +772,10 @@ impl Store { + return Err(io::Error::other("the entry is larger than the cache takes")); + } + let parent = path.parent().expect("an entry has a directory"); +- create_private_dir_all(parent)?; ++ self.directory.create_dir_all(parent)?; + let temporary = parent.join(format!(".{name}.{}.tmp", unique())); + let written = (|| { +- let mut file = new_private_file(&temporary)?; ++ let mut file = self.directory.new_file(&temporary)?; + file.write_all(bytes)?; + drop(file); + fs::rename(&temporary, path)?; +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/system.rs b/crates/metal2vulkan-runtime/src/shader_cache/system.rs +new file mode 100644 +index 0000000..c6e5cd1 +--- /dev/null ++++ b/crates/metal2vulkan-runtime/src/shader_cache/system.rs +@@ -0,0 +1,268 @@ ++//! The system layer: a second cache directory, read only, that one trusted writer fills for every ++//! process of the machine (the consumer's render server primes the OS's Metal libraries into it). ++//! ++//! - **Laid out as a device's own directory** (`store.rs`): the same `v1///` ++//! entries and content records, so what the writer's device wrote there under its own rules is ++//! what a reader's device looks up. A lookup asks the device's own directory first and this one ++//! only for what that lacks: a record the process added to (a making's driver data) is its own ++//! from then on. ++//! - **Trusted by who may write it, at every level opened.** The directory, each directory ++//! under it on the way to an entry, and the entry are opened without following a link ++//! (`O_NOFOLLOW`, relative to the level above) and judged by `fstat`: a directory or a regular ++//! file, owned by one of the owners the consumer named, that neither its group nor others may ++//! write. The directories above the layer are judged by `lstat` when it is opened: no link, ++//! owned by the superuser or one of the owners, and written by others only where the sticky bit ++//! keeps them from replacing what is not theirs. An entry's envelope carries a checksum, not a ++//! signature, and the driver's pipeline data is parsed by the driver as it is: a process must ++//! therefore never read an entry that anyone but those owners could have put there, and a ++//! level that fails the test is refused and counted, never read. ++//! - **Never written, touched or removed from here.** A hit refreshes no access time and ++//! contents a decoder refuses stay (counted): the directory is its writer's. ++//! - **Process-wide.** [`set_system_layer`] names the layer for the devices made after it; ++//! [`set_cache_readable_by_all`] makes the directories and entries a device writes readable by ++//! everyone (`0755`, `0644`) instead of by their owner alone, for the process that fills the ++//! layer through its own directory. ++ ++use std::ffi::CString; ++use std::io::Read; ++use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; ++use std::os::unix::ffi::OsStrExt; ++use std::path::{Component, Path, PathBuf}; ++use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; ++use std::sync::{Arc, RwLock}; ++ ++/// What a system layer did for this process ([`SystemLayer::counts`]). ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++pub struct SystemLayerCounts { ++ /// Entries and records read from the layer and accepted by their decoder: work not done ++ /// in this process. ++ pub accepted: u64, ++ /// Entries read from the layer whose contents the decoder refused (left in place). ++ pub rejected: u64, ++ /// Levels (a directory or an entry) refused for their kind, owner or mode: never read. ++ pub refused: u64, ++ /// Bytes of the accepted entries. ++ pub accepted_bytes: u64, ++} ++ ++/// An opened system layer. ++#[derive(Debug)] ++pub struct SystemLayer { ++ root: OwnedFd, ++ path: PathBuf, ++ owners: Vec, ++ accepted: AtomicU64, ++ rejected: AtomicU64, ++ refused: AtomicU64, ++ accepted_bytes: AtomicU64, ++} ++ ++static LAYER: RwLock>> = RwLock::new(None); ++static READABLE_BY_ALL: AtomicBool = AtomicBool::new(false); ++ ++/// Name the system layer the devices made from now on consult, or none. ++pub fn set_system_layer(layer: Option>) { ++ *LAYER.write().unwrap_or_else(|e| e.into_inner()) = layer; ++} ++ ++/// The system layer named last. ++pub fn system_layer() -> Option> { ++ LAYER.read().unwrap_or_else(|e| e.into_inner()).clone() ++} ++ ++/// Whether the cache directories opened from now on make their directories and entries readable ++/// by everyone. ++pub fn set_cache_readable_by_all(on: bool) { ++ READABLE_BY_ALL.store(on, Ordering::Relaxed); ++} ++ ++pub(super) fn cache_readable_by_all() -> bool { ++ READABLE_BY_ALL.load(Ordering::Relaxed) ++} ++ ++fn c_path(bytes: &[u8]) -> Option { ++ CString::new(bytes).ok() ++} ++ ++fn fstat(fd: &OwnedFd) -> Option { ++ // SAFETY: a zeroed stat is a valid out value and the descriptor is open. ++ unsafe { ++ let mut st: libc::stat = std::mem::zeroed(); ++ (libc::fstat(fd.as_raw_fd(), &mut st) == 0).then_some(st) ++ } ++} ++ ++/// Open `name` under `parent` (or by its path where `parent` is `None`), never through a link. ++fn open_level(parent: Option<&OwnedFd>, name: &[u8], directory: bool) -> Result { ++ let name = c_path(name).ok_or(libc::EINVAL)?; ++ let kind = if directory { libc::O_DIRECTORY } else { libc::O_NONBLOCK }; ++ let flags = libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | kind; ++ // SAFETY: the name is a C string and the parent descriptor is open. ++ let fd = unsafe { ++ match parent { ++ Some(parent) => libc::openat(parent.as_raw_fd(), name.as_ptr(), flags), ++ None => libc::open(name.as_ptr(), flags), ++ } ++ }; ++ if fd < 0 { ++ return Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)); ++ } ++ // SAFETY: a descriptor this call opened and nothing else owns. ++ Ok(unsafe { OwnedFd::from_raw_fd(fd) }) ++} ++ ++impl SystemLayer { ++ /// Open the layer at `root`, whose every level must be owned by one of `owners`. The error ++ /// says which level failed and how. ++ pub fn open(root: &Path, owners: &[u32]) -> Result { ++ if !root.is_absolute() || owners.is_empty() { ++ return Err("the layer needs an absolute path and an owner".into()); ++ } ++ // The directories above the layer: no link among them, each the superuser's or an ++ // owner's, written by others only under the sticky bit. A level the sandbox lets this ++ // process see nothing of (EPERM, which no file mode gives) is one the OS made. ++ let mut above = PathBuf::new(); ++ let mut parts = root.components().peekable(); ++ while let Some(part) = parts.next() { ++ match part { ++ Component::RootDir | Component::Normal(_) => above.push(part), ++ _ => return Err(format!("{}: not a plain path", root.display())), ++ } ++ if parts.peek().is_none() { ++ break; ++ } ++ let path = c_path(above.as_os_str().as_bytes()).ok_or("a path with a zero byte")?; ++ // SAFETY: a zeroed stat is a valid out value and the path is a C string. ++ let (result, st) = unsafe { ++ let mut st: libc::stat = std::mem::zeroed(); ++ (libc::lstat(path.as_ptr(), &mut st), st) ++ }; ++ if result != 0 { ++ let error = std::io::Error::last_os_error(); ++ if error.raw_os_error() == Some(libc::EPERM) { ++ continue; ++ } ++ return Err(format!("{}: {error}", above.display())); ++ } ++ let directory = st.st_mode & libc::S_IFMT == libc::S_IFDIR; ++ let owned = st.st_uid == 0 || owners.contains(&st.st_uid); ++ let closed = st.st_mode & 0o022 == 0 || st.st_mode & libc::S_ISVTX != 0; ++ if !directory || !owned || !closed { ++ return Err(format!( ++ "{}: above the layer, uid {} mode {:o}", ++ above.display(), ++ st.st_uid, ++ st.st_mode ++ )); ++ } ++ } ++ let fd = open_level(None, root.as_os_str().as_bytes(), true) ++ .map_err(|errno| format!("{}: errno {errno}", root.display()))?; ++ let layer = SystemLayer { ++ root: fd, ++ path: root.to_path_buf(), ++ owners: owners.to_vec(), ++ accepted: AtomicU64::new(0), ++ rejected: AtomicU64::new(0), ++ refused: AtomicU64::new(0), ++ accepted_bytes: AtomicU64::new(0), ++ }; ++ match fstat(&layer.root) { ++ Some(st) if layer.trusted(&st, true) => Ok(layer), ++ Some(st) => Err(format!( ++ "{}: uid {} mode {:o}", ++ root.display(), ++ st.st_uid, ++ st.st_mode ++ )), ++ None => Err(format!("{}: no fstat", root.display())), ++ } ++ } ++ ++ /// The layer's directory. ++ pub fn path(&self) -> &Path { ++ &self.path ++ } ++ ++ /// Whether a level of `st` is what the layer reads: a directory or a regular file, an ++ /// owner's, that neither its group nor others may write. ++ fn trusted(&self, st: &libc::stat, directory: bool) -> bool { ++ let kind = if directory { libc::S_IFDIR } else { libc::S_IFREG }; ++ st.st_mode & libc::S_IFMT == kind && self.owners.contains(&st.st_uid) && st.st_mode & 0o022 == 0 ++ } ++ ++ /// The open file of `relative` under the layer and its length, every level on the way judged. ++ /// `None` where a level is absent or refused (refusals are counted). ++ fn entry(&self, relative: &Path) -> Option<(std::fs::File, u64)> { ++ let mut parts = relative.components().peekable(); ++ let mut level: Option = None; ++ while let Some(part) = parts.next() { ++ let Component::Normal(name) = part else { ++ return None; ++ }; ++ let last = parts.peek().is_none(); ++ let parent = level.as_ref().unwrap_or(&self.root); ++ let fd = match open_level(Some(parent), name.as_bytes(), !last) { ++ Ok(fd) => fd, ++ Err(errno) => { ++ // A link in an entry's place (ELOOP), or a file where a directory belongs. ++ if errno == libc::ELOOP || errno == libc::ENOTDIR { ++ self.refused.fetch_add(1, Ordering::Relaxed); ++ } ++ return None; ++ } ++ }; ++ let st = fstat(&fd)?; ++ if !self.trusted(&st, !last) { ++ self.refused.fetch_add(1, Ordering::Relaxed); ++ return None; ++ } ++ if last { ++ return Some((std::fs::File::from(fd), st.st_size as u64)); ++ } ++ level = Some(fd); ++ } ++ None ++ } ++ ++ /// Whether the layer holds `relative`, of at most `limit` bytes. ++ pub fn holds(&self, relative: &Path, limit: u64) -> bool { ++ self.entry(relative).is_some_and(|(_, length)| length <= limit) ++ } ++ ++ /// The bytes of `relative` under the layer, at most `limit`; `None` where it is absent, ++ /// refused, longer or unreadable. The caller's decoder judges the contents and reports the ++ /// verdict ([`accepted`](Self::accepted), [`rejected`](Self::rejected)). ++ pub fn read(&self, relative: &Path, limit: u64) -> Option> { ++ let (file, length) = self.entry(relative)?; ++ if length > limit { ++ return None; ++ } ++ let mut bytes = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); ++ match file.take(limit + 1).read_to_end(&mut bytes) { ++ Ok(_) if bytes.len() as u64 <= limit => Some(bytes), ++ _ => None, ++ } ++ } ++ ++ /// Count an entry of `bytes` bytes read from the layer that its decoder accepted. ++ pub fn accepted(&self, bytes: u64) { ++ self.accepted.fetch_add(1, Ordering::Relaxed); ++ self.accepted_bytes.fetch_add(bytes, Ordering::Relaxed); ++ } ++ ++ /// Count an entry read from the layer that its decoder refused. ++ pub fn rejected(&self) { ++ self.rejected.fetch_add(1, Ordering::Relaxed); ++ } ++ ++ /// The counts so far. ++ pub fn counts(&self) -> SystemLayerCounts { ++ SystemLayerCounts { ++ accepted: self.accepted.load(Ordering::Relaxed), ++ rejected: self.rejected.load(Ordering::Relaxed), ++ refused: self.refused.load(Ordering::Relaxed), ++ accepted_bytes: self.accepted_bytes.load(Ordering::Relaxed), ++ } ++ } ++} diff --git a/bridge/system_cache_host.mm b/bridge/system_cache_host.mm new file mode 100644 index 0000000..0fcd35c --- /dev/null +++ b/bridge/system_cache_host.mm @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +// The system shader cache (SharedShaderCache.h, bridge patch 0036) between processes on the host. +// system_cache_host fill +// opens the device in that directory as the process that fills the layer (entries readable by everyone), makes a render and a +// compute pipeline and a library from source, draws, and leaves the entries there. +// system_cache_host read +// names the layer with that one trusted owner, opens the device in its own (empty) directory, makes the same pipelines and +// library, checks the drawn pixels, and prints what its own cache missed and what the layer answered: +// `RESULT layer=<0|1> accepted= rejected= refused= translated= replies= pixels=ok`. +// The cases to run it for: a second process hitting what the first published (translated=0, accepted above 0), a layer of +// another owner or writable by its group (layer=0), a directory or an entry others may write and a link in a directory's place +// (refused above 0, those entries translated here), and entries with changed bytes (rejected above 0, left in place). +#include "FrontendChecks.h" +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" +#include + +int main(int argc, char **argv) { + if (argc < 4) return 2; + const bool fill = !strcmp(argv[3], "fill"); + if (!fill && argc != 6) return 2; + @autoreleasepool { + bool layer = false; + if (fill) native_bridge_metal_system_cache(nullptr, nullptr, 0, true); + else { + const uint32_t owner = (uint32_t)strtoul(argv[5], nullptr, 10); + layer = native_bridge_metal_system_cache(argv[4], &owner, 1, false); + if (!layer) std::printf("layer refused: %s\n", nmb_last_error()); + } + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("no device: %s\n", nmb_last_error()); return 2;} + size_t length = 0; + const void *air = nmb_fixture_air(&length); + id library = native_bridge_metal_library_with_air(air, length); + frontendchecks::Report report = {}; + frontendchecks::gReport = &report; + id pipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + NSError *error = nil; + id compute = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"scale_and_offset"] error:&error]; + if (!library || !pipeline || !compute) {std::puts("FAIL a pipeline was refused"); return 1;} + // A library from source: its compiler reply is kept beside the pipeline cache, and read from the layer by a reader. + id source = [device newLibraryWithSource:@"#include \nusing namespace metal;\n" + "kernel void system_cache_probe(device float *data [[buffer(0)]], uint id [[thread_position_in_grid]]) {data[id] = 7.0f * float(id);}\n" + options:nil error:&error]; + if (!source) std::printf("source library not made on this host: %s\n", error.localizedDescription.UTF8String); + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id texture = [device newTextureWithDescriptor:td]; + id render = [[device newCommandQueue] commandBuffer]; + frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6); + [render commit]; + if (!frontendchecks::completed(render, "render")) return 1; + uint8_t pixels[16 * 16 * 4] = {}; + [texture getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + if (frontendchecks::wrongTexels(pixels, 64, 16)) {std::puts("FAIL wrong pixels"); return 1;} + // The cache thread writes what the creations queued. + usleep(1500000); + NmbShaderCacheStats stats = {}; + native_bridge_metal_shader_cache_stats(&stats); + uint64_t sy[4] = {}; + (void)nmb_system_cache_counts(sy); + char text[8192] = {}; + native_bridge_metal_telemetry(text, sizeof text); + unsigned long long translated = 0, found = 0, replies = 0; + for (char *line = strtok(text, "\n"); line; line = strtok(nullptr, "\n")) { + if (!strncmp(line, "SC ", 3) || !strncmp(line, "PRb ", 4) || !strncmp(line, "SY ", 3)) std::printf("%s\n", line); + sscanf(line, "SC fn %llu/%llu", &found, &translated); + if (const char *at = strstr(line, "replies ")) if (!strncmp(line, "SY ", 3)) sscanf(at, "replies %llu", &replies); + } + std::printf("RESULT layer=%d accepted=%llu rejected=%llu refused=%llu translated=%llu found=%llu misses=%llu replies=%llu pixels=ok\n", + layer, (unsigned long long)sy[0], (unsigned long long)sy[1], (unsigned long long)sy[2], translated, found, + (unsigned long long)stats.misses, replies); + } +}