feat(cache): system shader cache the render server fills and every process reads

On macOS the render server (WindowServer, user _windowserver) and the apps
are different users, so what it primed served it alone, and a sandboxed
process that cannot write the user's shared directory compiled every
pipeline at first use.

The render server now opens its device in /private/var/tmp/NativeAdrenoMetal.cache
when the kernel made that directory for it, writes its entries readable by
everyone, and primes every library of the list except other GPUs' driver
bundles. Every other process names /Library/GPUBundles/NativeAdrenoMetal.cache
(the same directory, shown read-only by the kernel) as a read-only layer
its device asks for what its own cache lacks: translated functions,
pipelines, the driver's pipeline data and compiler replies.

Bridge patch 0036 adds the layer to the runtime. A level (the directory,
each directory on the way to an entry, the entry) is opened without
following a link and read only when owned by root or _windowserver and not
writable by group or others; entries are then checked as the process's own
are. Nothing is written, touched or removed there.

gpu-prime-system=0 restores the per-user behaviour. Telemetry: PRb and SY
rows, system counts in the work line, the layer's state in the cache line.
iPadOS is unchanged.

Host: bridge/system_cache_host.mm over MoltenVK (second process hits the
published entries; wrong owner, writable levels, links and corrupt entries
are refused or ignored). Syntax-checked for macOS 27, iPadOS 27 and
iPadOS 17. Has not run on hardware; needs the kernel step that makes and
binds the directory.
This commit is contained in:
rponeawa
2026-11-11 00:00:00 +08:00
Unverified
parent 9780273e4f
commit 5e939ccd19
9 changed files with 862 additions and 28 deletions
+21 -7
View File
@@ -89,6 +89,7 @@ static Gpu *makeGpu(unsigned port) {
return gpu;
}
static std::string gCacheDirectory; // the runtime's cache directory (the one that opened the device), exported by CacheExport.h
static bool gFillsSystemCache; // the render server opened its device in the system shader cache's directory (SharedShaderCache.h)
// Progress markers 0x68xx (GPU6): 6800 entered, 6801 Turnip found, 6802 device made, 6803 returned to the discovery, 68f1 no Turnip, 68f2 no device.
static unsigned gDiagBits;
// Reads of the render server's own state for the framebuffer-format question (diagnostic only, for the exact QuartzCore of the target image; a read
@@ -375,6 +376,11 @@ static id deviceForPort(unsigned port) {
setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1);
CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
if (!nativeIsRenderServer()) {
// The system shader cache the render server fills (SharedShaderCache.h), read for what this process's own cache lacks.
// gpu-prime-system=0: no process reads it and the render server keeps its cache and its priming to itself, as before, for
// a comparison on the same image.
if (!bootArgSet("gpu-prime-system=0"))
native_bridge_metal_system_cache(systemcache::kReaderPath, systemcache::kOwners, 2, false);
device = clientDevice(port);
if (device) iogpu::install(device, port);
return device;
@@ -497,7 +503,7 @@ static id deviceForPort(unsigned port) {
unsetenv("METAL2VULKAN_LOG_PIPELINES");
CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
char candidates[6][512] = {};
char candidates[7][512] = {};
if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0;
if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0;
strlcpy(candidates[2], "/tmp/", sizeof candidates[2]);
@@ -511,18 +517,23 @@ static id deviceForPort(unsigned port) {
strlcat(candidates[4], "/", sizeof candidates[4]);
}
strlcpy(candidates[5], sharedShaderCacheDirectory().c_str(), sizeof candidates[5]);
static const unsigned order[] = {5, 4, 0, 1, 2, 3};
#if TARGET_OS_OSX
// First the system shader cache's directory, where the kernel made it for this process (marker 0x6816): what the render
// server compiles and primes there, every process reads (SharedShaderCache.h). Not with gpu-prime-system=0.
if (!bootArgSet("gpu-prime-system=0")) strlcpy(candidates[6], systemcache::writerDirectory().c_str(), sizeof candidates[6]);
#endif
static const unsigned order[] = {6, 5, 4, 0, 1, 2, 3};
for (unsigned i : order) {
if (device) break;
if (!candidates[i][0]) continue;
if (i == 0) {strlcat(candidates[i], "com.apple.metal", sizeof candidates[i]); mkdir(candidates[i], 0700); strlcat(candidates[i], "/", sizeof candidates[i]);} // as for the clients below
if (i != 5) strlcat(candidates[i], "m2v", sizeof candidates[i]);
mkdir(candidates[i], 0700);
if (i < 5) strlcat(candidates[i], "m2v", sizeof candidates[i]);
if (i != 6) mkdir(candidates[i], 0700);
if (access(candidates[i], W_OK) != 0) continue;
// Files an earlier image's copying import left half written here (int 590aaee2), which hold memory of the Data disk.
{
static bool once;
if (!once && i != 5) {once = true; removeHalfImported(candidates[i]);}
if (!once && i < 5) {once = true; removeHalfImported(candidates[i]);}
}
// The image's pipeline pack (an earlier boot's export), into this directory before the runtime reads it. Marker 0x6Cnn: files written
// (saturated at 0xfe), 0x6Cff the pack was refused.
@@ -530,12 +541,14 @@ static id deviceForPort(unsigned port) {
const int imported = cacheexport::importPack(pack, candidates[i]);
if (imported) gpu->mark(0x6C00 | (imported < 0 ? 0xff : imported > 0xfe ? 0xfe : imported));
}
// What the device writes in the system cache's directory is readable by everyone; in any other, by this user alone.
native_bridge_metal_system_cache(nullptr, nullptr, 0, i == 6);
@try {
device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr);
} @catch (NSException *exception) {
device = nil;
}
if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i];} // which directory served
if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i]; gFillsSystemCache = i == 6;} // which directory served
}
if (!device) {
gpu->mark(0x68f2);
@@ -586,7 +599,8 @@ static id deviceForPort(unsigned port) {
pthread_setname_np("native-metal-prime"); // a process sample and the telemetry's TH rows name its CPU time
char list[1024];
prime::run((id<MTLDevice>)device, gCacheDirectory, 90,
requestedPath("NativeAdrenoMetal.primelist", list, sizeof list) ? std::string(list) : std::string());
requestedPath("NativeAdrenoMetal.primelist", list, sizeof list) ? std::string(list) : std::string(),
gFillsSystemCache);
return nullptr;
}, nullptr);
pthread_attr_destroy(&attributes);
+5
View File
@@ -9,6 +9,11 @@ extern "C" {
// Opens the process's one device: loader exports vkGetInstanceProcAddr (Turnip), cache is a writable shader-cache directory, registryID is the
// accelerator service's registry entry. missing (may be null) is told the class and selector of every unimplemented call.
id native_bridge_metal_open(const char *loader, const char *cache, uint64_t registryID, void (*missing)(const char *, const char *));
// The system shader cache (SharedShaderCache.h), named before native_bridge_metal_open. A reader passes the layer's directory and the
// user ids whose files it trusts there: its device then asks that directory, read only and by the layer's owner and mode rules, for
// what its own cache lacks. The process that fills the layer passes writer true and opens its device in the layer's directory: what
// it writes is readable by everyone. Returns whether the layer was taken; the reason of a refusal is in the `cache:` log line.
bool native_bridge_metal_system_cache(const char *reader, const uint32_t *owners, unsigned count, bool writer);
// Bare AIR bitcode as a library (the bridge's fixtures; Metal itself accepts only metallibs).
id native_bridge_metal_library_with_air(const void *bytes, size_t length);
// One line for the plugin's status report: command buffers committed/failed locally, pipelines made, refusals, the last refusal's text.
+75 -16
View File
@@ -619,6 +619,9 @@ __attribute__((weak)) volatile uint32_t native_prime_libraries_total;
__attribute__((weak)) volatile uint32_t native_prime_made;
__attribute__((weak)) volatile uint32_t native_prime_rate;
__attribute__((weak)) volatile uint32_t native_prime_functions_resumed;
__attribute__((weak)) volatile uint32_t native_prime_libraries_primed; // libraries this process's passes compiled
__attribute__((weak)) volatile uint32_t native_prime_libraries_skipped; // other GPUs' driver bundles (whole system) or not loaded here
__attribute__((weak)) volatile uint32_t native_prime_scope; // 0 no pass yet, 1 loaded bundles only, 2 the whole system
}
static void refuseSurface(uint32_t step,IOSurfaceRef surface,MTLTextureDescriptor *d,NSUInteger plane,uint64_t offset,uint64_t row) {
gSurfaceRefusals++;
@@ -660,15 +663,32 @@ static void onLargeStack(void (^body)(void)) {
// its Metal cache directory but not Library/Caches: with the replies only there it kept none, and each of its launches compiled
// every request again.
static std::string gCacheRoot; // the pipeline cache directory the device opened with (native_bridge_metal_open)
// The system shader cache as this process uses it (native_bridge_metal_system_cache): 0 not named, 1 read as a layer, 2 filled by
// this process, 3 refused; its directory or the refusal's reason; and the compiler replies the layer answered.
static int gSystemCacheState;
static std::string gSystemCacheText;
static std::atomic<uint64_t> gCompileSystem;
struct NBShaderCacheHeader {uint32_t magic,warningBytes;uint64_t metallibBytes;unsigned char digest[CC_SHA256_DIGEST_LENGTH];};
static const uint32_t kNBShaderCacheMagic=0x3143534e; // "NSC1"
static NSString *shaderCachePath(const unsigned char uuid[16],uint32_t type,const std::vector<uint8_t> &request) {
static NSString *shaderCacheName(const unsigned char uuid[16],uint32_t type,const std::vector<uint8_t> &request) {
CC_SHA256_CTX context;CC_SHA256_Init(&context);
CC_SHA256_Update(&context,uuid,16);CC_SHA256_Update(&context,&type,sizeof type);
CC_SHA256_Update(&context,request.data(),(CC_LONG)request.size());
unsigned char key[CC_SHA256_DIGEST_LENGTH];CC_SHA256_Final(key,&context);
char name[2*CC_SHA256_DIGEST_LENGTH+8];
for(int i=0;i<CC_SHA256_DIGEST_LENGTH;i++)snprintf(name+2*i,3,"%02x",key[i]);
strlcat(name,".reply",sizeof name);
return @(name);
}
static NSString *shaderCachePath(NSString *name) {
static NSString *directory;
static dispatch_once_t once;
dispatch_once(&once,^{
if(getenv("NATIVE_METAL_NO_SHADER_CACHE"))return;
const std::string shared=sharedShaderCacheDirectory();
NSMutableArray<NSString *> *roots=[NSMutableArray array];
// The process that fills the system cache keeps its replies there, where every process reads them.
if(gSystemCacheState==2 && !gCacheRoot.empty())[roots addObject:@(gCacheRoot.c_str())];
if(!shared.empty())[roots addObject:@(shared.c_str())];
if(!gCacheRoot.empty())[roots addObject:@(gCacheRoot.c_str())];
if(NSString *caches=NSSearchPathForDirectoriesInDomains(NSCachesDirectory,NSUserDomainMask,YES).firstObject)[roots addObject:caches];
@@ -679,17 +699,9 @@ static NSString *shaderCachePath(const unsigned char uuid[16],uint32_t type,cons
}
});
if(!directory)return nil;
CC_SHA256_CTX context;CC_SHA256_Init(&context);
CC_SHA256_Update(&context,uuid,16);CC_SHA256_Update(&context,&type,sizeof type);
CC_SHA256_Update(&context,request.data(),(CC_LONG)request.size());
unsigned char key[CC_SHA256_DIGEST_LENGTH];CC_SHA256_Final(key,&context);
char name[2*CC_SHA256_DIGEST_LENGTH+8];
for(int i=0;i<CC_SHA256_DIGEST_LENGTH;i++)snprintf(name+2*i,3,"%02x",key[i]);
strlcat(name,".reply",sizeof name);
return [directory stringByAppendingPathComponent:@(name)];
return [directory stringByAppendingPathComponent:name];
}
static bool shaderCacheRead(NSString *path,NSData **metallib,NSString **warnings) {
NSData *file=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:nil]:nil;
static bool shaderCacheDecode(NSData *file,NSData **metallib,NSString **warnings) {
NBShaderCacheHeader header;
if(file.length<sizeof header)return false;
memcpy(&header,file.bytes,sizeof header);
@@ -702,6 +714,20 @@ static bool shaderCacheRead(NSString *path,NSData **metallib,NSString **warnings
*metallib=[NSData dataWithBytes:payload+header.warningBytes length:(NSUInteger)header.metallibBytes];
return true;
}
// The reply at `path` in this process's directory, else the one of that name the system layer holds (the render server's, read by
// the layer's owner and mode rules), checked the same way: a reply the layer holds is one this process does not compile.
static bool shaderCacheRead(NSString *path,NSString *name,NSData **metallib,NSString **warnings) {
NSData *file=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:nil]:nil;
if(shaderCacheDecode(file,metallib,warnings))return true;
if(gSystemCacheState!=1 || getenv("NATIVE_METAL_NO_SHADER_CACHE"))return false;
size_t length=0;
uint8_t *bytes=nmb_system_cache_read([@"native-metal-shaders/" stringByAppendingString:name].UTF8String,256u<<20,&length);
if(!bytes)return false;
const bool good=shaderCacheDecode([NSData dataWithBytesNoCopy:bytes length:length freeWhenDone:YES],metallib,warnings);
nmb_system_cache_verdict(good,length);
if(good)gCompileSystem++;
return good;
}
static void shaderCacheWrite(NSString *path,NSData *metallib,NSString *warnings) {
if(!path || !metallib.length)return;
NSData *warned=[warnings dataUsingEncoding:NSUTF8StringEncoding];
@@ -783,9 +809,10 @@ static NSData *codeGenRequest(uint32_t type,const std::vector<uint8_t> &request,
}
});
if(!service){*failure=@"the system's Metal compiler (MTLCompiler.framework) is not available";return nil;}
NSString *cached=shaderCachePath(compilerUuid,type,request);
NSString *cachedName=shaderCacheName(compilerUuid,type,request);
NSString *cached=shaderCachePath(cachedName);
NSData *kept=nil;NSString *keptWarnings=nil;
if(shaderCacheRead(cached,&kept,&keptWarnings)) {
if(shaderCacheRead(cached,cachedName,&kept,&keptWarnings)) {
gCompileCached++;
*warnings=keptWarnings;*failure=nil;
return kept;
@@ -4831,20 +4858,24 @@ static void logClientWork() {
dispatch_source_set_timer(timer,dispatch_time(DISPATCH_TIME_NOW,30*NSEC_PER_SEC),30*NSEC_PER_SEC,5*NSEC_PER_SEC);
dispatch_source_set_event_handler(timer,^{
static uint64_t last[3];
uint64_t vp[8]={},work[10]={},mc[10]={};
uint64_t vp[8]={},work[10]={},mc[10]={},sy[4]={};
if(!nmb_device_pipeline_counts(gDevice,vp))return;
(void)nmb_system_cache_counts(sy);
const uint64_t now[3]={gTimePipeline.count.load(),gTimeCompile.count.load()+gCompileCached.load(),vp[0]};
if(!memcmp(now,last,sizeof now))return;
memcpy(last,now,sizeof now);
(void)nmb_device_work_stats(gDevice,work);(void)nmb_device_memory_counts(gDevice,mc);
os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal work: pipeline %llu/%llums/%llums library %llu/%llums compile %llu/%llums cached %llu | vk %llu "
"create %llu/%llu step %llu drv %llu+%llums | cache hit %llu miss %llu wr %llu | mem %llu/%llu/%lluM | problems %llu",
"create %llu/%llu step %llu drv %llu+%llums | cache hit %llu miss %llu wr %llu | system %llu/%lluK rej %llu refused %llu replies %llu | "
"mem %llu/%llu/%lluM | problems %llu",
(unsigned long long)gTimePipeline.count.load(),(unsigned long long)(gTimePipeline.total.load()/1000),
(unsigned long long)(gTimePipeline.slowest.load()/1000),(unsigned long long)gTimeLibrary.count.load(),
(unsigned long long)(gTimeLibrary.total.load()/1000),(unsigned long long)gTimeCompile.count.load(),
(unsigned long long)(gTimeCompile.total.load()/1000),(unsigned long long)gCompileCached.load(),(unsigned long long)vp[0],
(unsigned long long)vp[1],(unsigned long long)vp[2],(unsigned long long)vp[3],(unsigned long long)(vp[4]/1000000),
(unsigned long long)(vp[5]/1000000),(unsigned long long)work[7],(unsigned long long)work[8],(unsigned long long)work[9],
(unsigned long long)sy[0],(unsigned long long)(sy[3]>>10),(unsigned long long)sy[1],(unsigned long long)sy[2],
(unsigned long long)gCompileSystem.load(),
(unsigned long long)(mc[0]>>20),(unsigned long long)(mc[2]>>20),(unsigned long long)(mc[3]>>20),(unsigned long long)gProblems.load());
});
dispatch_resume(timer);
@@ -4888,6 +4919,15 @@ static void logClientMemory() {
dispatch_resume(timer);
}
extern "C" bool native_bridge_metal_system_cache(const char *reader,const uint32_t *owners,unsigned count,bool writer) {
if(gDeviceObject)return false;
nmb_cache_readable_by_all(writer);
gSystemCacheState=writer?2:0;gSystemCacheText=writer?"filled here":"";
if(writer || !reader){(void)nmb_system_cache_open(nullptr,nullptr,0);return writer;}
if(nmb_system_cache_open(reader,owners,count)){gSystemCacheState=1;gSystemCacheText=reader;return true;}
gSystemCacheState=3;gSystemCacheText=nmb_last_error();
return false;
}
extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint64_t registryID,void (*missing)(const char *,const char *)) {
if(gDeviceObject)return gDeviceObject;
gMissing=missing;gInFlight=[NSMutableSet set];
@@ -4905,7 +4945,12 @@ extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint
logClientWork();
logClientMemory();
// Where this process's pipeline cache is and whose it is: a cache serves the processes of its user that may write its directory.
if(diagOn() || logGpuOn())os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal cache: %{public}s uid %d",cache?cache:"none",(int)getuid());
// Then the system layer: read from its directory, filled by this process, refused with the level and why, or not named.
if(diagOn() || logGpuOn()) {
static const char *const kStates[]={"none","read","filled","refused"};
os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal cache: %{public}s uid %d system %{public}s %{public}s",cache?cache:"none",(int)getuid(),
kStates[gSystemCacheState],gSystemCacheText.c_str());
}
if(cache && logGpuOn()){gJournalPath=std::string(cache)+"/pipelines-"+std::to_string(getpid())+".log";gJournalBegan=nowUs();}
char name[128]={};nmb_device_name(gDevice,name,sizeof name);gName=[NSString stringWithUTF8String:name];gRegistryID=registryID;
// The target families supportsFamily: answers true, with the rows of Apple's tables the device does not meet in them (nmb_device_family_waivers).
@@ -5283,6 +5328,20 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) {
}
}
row("PR compiling=%u libs=%u/%u made=%u rate=%u resumed=%u",native_prime_compiling,native_prime_libraries_done,native_prime_libraries_total,native_prime_made,native_prime_rate,native_prime_functions_resumed);
// Priming's reach and the system shader cache. PRb: the pass's scope (1 the bundles loaded here, 2 the whole system), the libraries
// it compiled itself, those it leaves out (other GPUs' driver bundles, or not loaded here), the system cache's state here (0 not
// named, 1 read, 2 filled by this process, 3 refused) and what this process published: the entries and KiB its cache directory
// took while it is the system layer. SY, in a process that reads the layer: the entries it answered, their KiB, the contents
// refused, the levels refused for owner or mode, and the compiler replies it answered.
{
uint64_t sc[24]={},sy[4]={};
if(gDevice)(void)nmb_device_shader_cache_detail(gDevice,sc);
(void)nmb_system_cache_counts(sy);
row("PRb scope=%u primed=%u left=%u | system %d pub %llu/%lluK",native_prime_scope,native_prime_libraries_primed,
native_prime_libraries_skipped,gSystemCacheState,(ull)(gSystemCacheState==2?sc[11]+sc[12]:0),(ull)(gSystemCacheState==2?sc[13]>>10:0));
if(gSystemCacheState==1 || gSystemCacheState==3)
row("SY read %llu/%lluK rej %llu refused %llu replies %llu",(ull)sy[0],(ull)(sy[3]>>10),(ull)sy[1],(ull)sy[2],(ull)gCompileSystem.load());
}
if(gDevice) {
// The warm pass (records warmed, failed, unread, KiB of driver data) and where work ran: scheduler passes and batches at commit, render
// log waits, encoder ends on the lowering thread.
+33 -5
View File
@@ -39,7 +39,10 @@ extern "C" unsigned native_bridge_metal_recent_commits(uint64_t windowUs);
// settle, 3 it opens a library and its functions), libraries done of the list, pipelines made, and the most command buffers the last wait
// saw in 250 ms.
extern "C" {volatile uint32_t native_prime_compiling, native_prime_libraries_done, native_prime_libraries_total, native_prime_made, native_prime_rate,
native_prime_functions_resumed;}
native_prime_functions_resumed;
// For the PRb row: the libraries this process's passes compiled, those a pass leaves out, and the pass's scope (1 the bundles loaded
// in this process, 2 the whole system).
volatile uint32_t native_prime_libraries_primed, native_prime_libraries_skipped, native_prime_scope;}
extern "C" void native_bridge_metal_quiet_problems(bool);
namespace prime {
@@ -187,6 +190,17 @@ static bool libraryOfLoadedBundle(const std::string &path) {
}
return false;
}
// macOS: whether a library is in another GPU's driver bundle (/System/Library/Extensions/<name>.bundle: AGXMetal* and the like, 167 of
// the 400 libraries of a macOS 27 image). IOGPU loads the bundle of the accelerator found, which here is this plugin, so nothing
// asks for those libraries on this GPU.
static bool libraryOfDriverBundle(const std::string &path) {
static const char kCryptex[] = "/System/Cryptexes/OS", kExtensions[] = "/System/Library/Extensions/";
const char *plain = path.c_str();
if (strncmp(plain, kCryptex, sizeof kCryptex - 1) == 0 && plain[sizeof kCryptex - 1] == '/') plain += sizeof kCryptex - 1;
if (strncmp(plain, kExtensions, sizeof kExtensions - 1) != 0) return false;
const char *slash = strchr(plain + sizeof kExtensions - 1, '/');
return slash && slash - plain > 7 && strncmp(slash - 7, ".bundle", 7) == 0;
}
#endif
extern "C" const char *nmb_runtime_build(void);
@@ -349,7 +363,10 @@ static std::string library(id<MTLDevice> device, const std::string &path, const
}
// The thread: waits for the desktop to settle, then works through the libraries not finished yet.
static void run(id<MTLDevice> device, std::string root, unsigned delaySeconds, const std::string &listPath) {
// `wholeSystem` (macOS): this process fills the system shader cache (SharedShaderCache.h), which every process reads, so every
// library of the list is primed but those of other GPUs' driver bundles; without it only the libraries of bundles loaded here are,
// since the cache then serves this process alone.
static void run(id<MTLDevice> device, std::string root, unsigned delaySeconds, const std::string &listPath, bool wholeSystem) {
pthread_set_qos_class_self_np(QOS_CLASS_BACKGROUND, 0);
native_bridge_metal_quiet_problems(true);
while (root.size() > 1 && root.back() == '/') root.pop_back();
@@ -370,6 +387,11 @@ static void run(id<MTLDevice> device, std::string root, unsigned delaySeconds, c
const std::string build = buildId();
appendLine(logPath, note + " build=" + build);
native_prime_libraries_total = (uint32_t)paths.size();
#if TARGET_OS_OSX
native_prime_scope = wholeSystem ? 2 : 1;
#else
native_prime_scope = 2; // iPadOS primes every library of the list, whatever its bundle
#endif
for (const std::string &path : paths) {
++libs;
native_prime_libraries_done = finished;
@@ -379,10 +401,15 @@ static void run(id<MTLDevice> device, std::string root, unsigned delaySeconds, c
const std::string id = path + "\t" + std::to_string((long long)st.st_size) + "\t" + std::to_string((long long)st.st_mtimespec.tv_sec) + "\t" + build;
if (done.count(id)) {++finished; continue;}
#if TARGET_OS_OSX
// Counted as done for the telemetry's progress, and not listed as finished (libraryOfLoadedBundle).
if (!libraryOfLoadedBundle(path)) {++foreign; ++finished; continue;}
// Counted as done for the telemetry's progress, and not listed as finished (libraryOfDriverBundle, libraryOfLoadedBundle).
if (wholeSystem ? libraryOfDriverBundle(path) : !libraryOfLoadedBundle(path)) {
++foreign; ++finished;
native_prime_libraries_skipped = foreign;
continue;
}
#endif
appendLine(logPath, library(device, path, current, bad, functionsPath, functionsDone, build));
native_prime_libraries_primed = native_prime_libraries_primed + 1;
appendLine(donePath, id);
done.insert(id);
++finished;
@@ -390,7 +417,8 @@ static void run(id<MTLDevice> device, std::string root, unsigned delaySeconds, c
native_prime_libraries_done = finished;
native_prime_compiling = 0;
char line[120];
snprintf(line, sizeof line, "pass libraries=%u finished=%u not-loaded=%u ms=%llu", libs, finished, foreign, (unsigned long long)(nowMs() - began));
snprintf(line, sizeof line, "pass libraries=%u finished=%u %s=%u ms=%llu", libs, finished, wholeSystem ? "driver-bundles" : "not-loaded",
foreign, (unsigned long long)(nowMs() - began));
appendLine(logPath, line);
}
} // namespace prime
+27
View File
@@ -33,6 +33,33 @@ inline std::string sharedShaderCacheDirectory() {
return path;
}
// The system shader cache (macOS): one directory the render server fills and every process reads.
// The render server (WindowServer, the user _windowserver) is not the console user, so the per-user directory above serves it alone,
// and its sandbox lets it write only its own cache and temporary directories, /private/tmp and /private/var/tmp, while the one
// directory every sandbox that may use graphics can read is /Library/GPUBundles (system.sb's system-graphics, the WebKit GPU
// process's profile). No path is both, so the kernel makes the directory at boot where the render server may write it
// (kWriterPath: owner _windowserver, mode 0755) and shows it read-only where everyone may read it (kReaderPath, a read-only bind
// mount beside the plugin). The render server opens its device in the first (its cache directory, made readable by everyone) and
// primes the system's Metal libraries into it; every other process asks the second for what its own cache lacks.
// A reader trusts the directory only by who can write it: it, every directory on the way to an entry and the entry must be owned by
// the superuser or the render server's user and not writable by group or others, each opened without following a link (the
// runtime's system layer, bridge patch 0036). An entry is then checked as one of the process's own cache is (name, build, key,
// profile, checksum). gpu-prime-system=0 turns all of it off.
namespace systemcache {
static const char kWriterPath[] = "/private/var/tmp/NativeAdrenoMetal.cache";
static const char kReaderPath[] = "/Library/GPUBundles/NativeAdrenoMetal.cache";
static const uint32_t kOwners[] = {0, 88}; // the superuser and _windowserver
// The directory this process fills, when the kernel made it for this process's user (or this process is the superuser): never
// made here, since /private/var/tmp takes anyone's directories.
inline std::string writerDirectory() {
struct stat st = {};
if (lstat(kWriterPath, &st) || !S_ISDIR(st.st_mode) || (st.st_mode & 0022)) return {};
if (st.st_uid != kOwners[0] && st.st_uid != kOwners[1]) return {};
if (st.st_uid != getuid() && getuid() != 0) return {};
return access(kWriterPath, W_OK) == 0 ? std::string(kWriterPath) : std::string();
}
} // namespace systemcache
// Call before the first Vulkan device is opened, including the optional smoke path.
inline void configureShaderDiskCache(const std::string &root) {
if (root.empty()) return;
+15
View File
@@ -45,6 +45,21 @@ bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]);
* rejected, read failures, writes, writes queued, writes on a calling thread, bytes written, queued hits, write failures, evictions, driver cache loads,
* driver pipeline hits, driver pipeline misses, driver saves, then three spare zeros. */
bool nmb_device_shader_cache_detail(const NmbDevice *, uint64_t out[24]);
/* The read-only system cache layer of the device opened after this call: the directory at root, read only where it, every directory
* on the way to an entry and the entry are owned by one of the count user ids and not writable by group or others, never through a
* link. A device asks its own cache directory first and the layer for what that lacks. False with nmb_last_error where the directory
* fails the test (no layer then); a null root names none. */
bool nmb_system_cache_open(const char *root, const uint32_t *owners, size_t count);
/* The cache directory of the device opened after this call makes its directories and entries readable by everyone (0755, 0644):
* the process that fills the system layer through its own directory. */
void nmb_cache_readable_by_all(bool on);
/* Entries read from the system layer and accepted, entries whose contents were refused, levels refused for kind, owner or mode,
* bytes of the accepted entries. False without a layer. */
bool nmb_system_cache_counts(uint64_t out[4]);
/* A file under the system layer by the layer's rules, at most limit bytes: a malloc block the caller frees, or null. The caller
* checks the contents and reports its verdict. */
uint8_t *nmb_system_cache_read(const char *relative, uint64_t limit, size_t *length);
void nmb_system_cache_verdict(bool accepted, uint64_t bytes);
/* The translator source the runtime was built from, which keys the shader cache (build-native-metal-bridge.py). */
const char *nmb_runtime_build(void);
// Linear images made/refused/refused again, shadowed textures, shadow copies, texels copied back, linear passes in place/through copies.
+96
View File
@@ -572,6 +572,102 @@ pub extern "C" fn nmb_stage_ring_read(from: u64, out: *mut metal2vulkan_runtime:
count
}
/// Name the read-only system cache layer for the device opened after this call (bridge patch 0036): the directory at `root`, read
/// only where it, every directory on the way to an entry and the entry are owned by one of the `count` user ids at `owners` and
/// not writable by group or others, and never through a link. False with the reason in `nmb_last_error` where the directory fails
/// that test; no layer is named then. A null `root` names none.
///
/// # Safety
/// `root` is null or a C string; `owners` points at `count` user ids.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_system_cache_open(root: *const c_char, owners: *const u32, count: usize) -> bool {
guard(false, || {
metal2vulkan_runtime::set_system_layer(None);
if root.is_null() {
return Ok(true);
}
if owners.is_null() || count == 0 {
return Err("system cache: no owner".into());
}
// SAFETY: the caller's contract.
let (root, owners) = unsafe { (CStr::from_ptr(root), std::slice::from_raw_parts(owners, count)) };
let layer = metal2vulkan_runtime::SystemLayer::open(&PathBuf::from(root.to_string_lossy().into_owned()), owners)
.map_err(|why| format!("system cache: {why}"))?;
metal2vulkan_runtime::set_system_layer(Some(std::sync::Arc::new(layer)));
Ok(true)
})
}
/// Whether the cache directory of the device opened after this call makes its directories and entries readable by everyone
/// (0755, 0644) instead of by their owner alone: the directory of the process that fills the system layer.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_cache_readable_by_all(on: bool) {
metal2vulkan_runtime::set_cache_readable_by_all(on);
}
/// The system layer's counts into the caller's four words: entries read from it and accepted, entries whose contents were refused,
/// levels refused for their kind, owner or mode, and the accepted entries' bytes. False without a layer.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_system_cache_counts(out: *mut u64) -> bool {
guard(false, || {
if out.is_null() {
return Err("null counts".into());
}
let Some(layer) = metal2vulkan_runtime::system_layer() else {
return Ok(false);
};
let c = layer.counts();
// SAFETY: the header's four words.
unsafe { std::slice::from_raw_parts_mut(out, 4) }.copy_from_slice(&[c.accepted, c.rejected, c.refused, c.accepted_bytes]);
Ok(true)
})
}
/// The bytes of the file at `relative` under the system layer, read by the layer's rules, at most `limit`: a block of `malloc`
/// the caller frees, its length in `length`; null where there is no layer or the file is absent, refused or longer. The caller
/// checks the contents and reports the verdict (`nmb_system_cache_verdict`).
///
/// # Safety
/// `relative` is a C string and `length` is writable.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_system_cache_read(relative: *const c_char, limit: u64, length: *mut usize) -> *mut u8 {
guard(std::ptr::null_mut(), || {
if relative.is_null() || length.is_null() {
return Err("system cache read: null argument".into());
}
let Some(layer) = metal2vulkan_runtime::system_layer() else {
return Ok(std::ptr::null_mut());
};
// SAFETY: the caller's contract.
let relative = unsafe { CStr::from_ptr(relative) }.to_string_lossy().into_owned();
let Some(bytes) = layer.read(std::path::Path::new(&relative), limit) else {
return Ok(std::ptr::null_mut());
};
// SAFETY: a block of the bytes' length (at least one byte), filled from them; `length` is writable.
unsafe {
let block = libc::malloc(bytes.len().max(1)).cast::<u8>();
if block.is_null() {
return Err("system cache read: no memory".into());
}
std::ptr::copy_nonoverlapping(bytes.as_ptr(), block, bytes.len());
*length = bytes.len();
Ok(block)
}
})
}
/// Count what the caller made of `bytes` bytes `nmb_system_cache_read` gave: accepted, or refused by its own check.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_system_cache_verdict(accepted: bool, bytes: u64) {
if let Some(layer) = metal2vulkan_runtime::system_layer() {
if accepted {
layer.accepted(bytes);
} else {
layer.rejected();
}
}
}
/// Free the runtime's spare dedicated blocks and kept empty pooled blocks (bridge patch 0021), as Metal gives its resource cache's
/// memory back when the system reports memory pressure: how many blocks went.
#[unsafe(no_mangle)]
@@ -0,0 +1,516 @@
diff --git a/crates/metal2vulkan-runtime/src/lib.rs b/crates/metal2vulkan-runtime/src/lib.rs
index 03287cc..048d3d4 100644
--- a/crates/metal2vulkan-runtime/src/lib.rs
+++ b/crates/metal2vulkan-runtime/src/lib.rs
@@ -360,6 +360,9 @@ pub use resource_ids::{IdPlacement, ResourceId, ResourceIdKind};
pub use sampler::{sampler_border_color, SamplerState, SamplerStateOptions};
pub use creation::{allocation_sites, AllocationSite};
pub use memory::{MemoryKinds, MEMORY_KIND_CLASSES};
+pub use shader_cache::system::{
+ set_cache_readable_by_all, set_system_layer, system_layer, SystemLayer, SystemLayerCounts,
+};
pub use shader_cache::{
ShaderCacheOptions, ShaderCacheStatistics, DEFAULT_CAPACITY, DEFAULT_MEMORY_CAPACITY,
};
diff --git a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs
index 508af31..ab148b2 100644
--- a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs
+++ b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs
@@ -71,6 +71,7 @@ pub(crate) use creation::VariantStages;
pub(crate) mod driver;
mod held;
pub(super) mod store;
+pub(crate) mod system;
pub(super) mod worker;
use super::error::{invalid, Error, Problem, ShaderCacheError};
diff --git a/crates/metal2vulkan-runtime/src/shader_cache/store.rs b/crates/metal2vulkan-runtime/src/shader_cache/store.rs
index 95f4d86..539a35f 100644
--- a/crates/metal2vulkan-runtime/src/shader_cache/store.rs
+++ b/crates/metal2vulkan-runtime/src/shader_cache/store.rs
@@ -76,6 +76,9 @@ pub(crate) struct Directory {
root: PathBuf,
build: Hash256,
capacity: u64,
+ /// Its directories and entries are made readable by everyone (`system.rs`): the directory
+ /// of the process that fills the system layer.
+ public: bool,
}
impl Directory {
@@ -92,17 +95,20 @@ impl Directory {
root,
build,
capacity,
+ public: super::system::cache_readable_by_all(),
};
let failed = |path: &Path| {
let path = path.to_path_buf();
move |error| Failed { path, error }
};
let made = (|| {
- create_private_dir_all(&directory.root).map_err(failed(&directory.root))?;
+ directory.create_dir_all(&directory.root).map_err(failed(&directory.root))?;
+ let layout_dir = directory.layout_dir();
+ directory.create_dir_all(&layout_dir).map_err(failed(&layout_dir))?;
let build_dir = directory.build_dir();
- create_private_dir_all(&build_dir).map_err(failed(&build_dir))?;
+ directory.create_dir_all(&build_dir).map_err(failed(&build_dir))?;
let probe = build_dir.join(format!(".probe.{}.tmp", unique()));
- let written = new_private_file(&probe).and_then(|mut file| file.write_all(b"probe"));
+ let written = directory.new_file(&probe).and_then(|mut file| file.write_all(b"probe"));
let removed = fs::remove_file(&probe);
written.and(removed).map_err(failed(&directory.root))
})();
@@ -133,6 +139,41 @@ impl Directory {
self.layout_dir().join(hex(&self.build))
}
+ /// Create `dir` and its missing parents: each only its owner may enter, or, in a directory
+ /// made for every reader (`system.rs`), everyone may read and enter, whatever the process's
+ /// file mode mask says.
+ fn create_dir_all(&self, dir: &Path) -> io::Result<()> {
+ if !self.public {
+ return create_private_dir_all(dir);
+ }
+ let mut builder = fs::DirBuilder::new();
+ builder.recursive(true);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
+ builder.mode(0o755);
+ builder.create(dir)?;
+ if fs::metadata(dir)?.permissions().mode() & 0o777 != 0o755 {
+ fs::set_permissions(dir, fs::Permissions::from_mode(0o755))?;
+ }
+ Ok(())
+ }
+ #[cfg(not(unix))]
+ builder.create(dir)
+ }
+
+ /// Create the new file `path`, which only its owner may read, or everyone in a directory
+ /// made for every reader; fails if it exists.
+ fn new_file(&self, path: &Path) -> io::Result<fs::File> {
+ let file = new_private_file(path)?;
+ #[cfg(unix)]
+ if self.public {
+ use std::os::unix::fs::PermissionsExt;
+ file.set_permissions(fs::Permissions::from_mode(0o644))?;
+ }
+ Ok(file)
+ }
+
/// The path of the entry named `name`.
pub(crate) fn entry_path(&self, name: &str) -> PathBuf {
self.build_dir().join(&name[..2]).join(name)
@@ -204,6 +245,9 @@ enum Found {
Bytes(Vec<u8>, fs::File),
/// The entry's bytes, from a write the cache thread has not made yet: no file to refresh.
Queued(Vec<u8>),
+ /// The entry's bytes, from the system layer (`system.rs`): not this directory's to refresh
+ /// or remove.
+ System(Vec<u8>),
}
/// The counts behind [`ShaderCacheStatistics`](super::ShaderCacheStatistics).
@@ -347,6 +391,8 @@ pub(crate) struct Store {
/// The device's cache thread, which writes the entries this store is given; `None` in a
/// store with no thread, which writes them on the calling thread.
asked: Option<Arc<Asked>>,
+ /// The system layer this store asks for what its directory lacks (`system.rs`).
+ system: Option<Arc<super::system::SystemLayer>>,
}
/// What a device's store says once each, through its instance's message sink, else on stderr:
@@ -383,9 +429,17 @@ impl Store {
write_failed: AtomicBool::new(false),
},
asked,
+ system: super::system::system_layer(),
}
}
+ /// The system layer's bytes of the file this directory would hold at `path`.
+ fn system_read(&self, path: &Path) -> Option<Vec<u8>> {
+ let system = self.system.as_ref()?;
+ let relative = path.strip_prefix(self.directory.root()).ok()?;
+ system.read(relative, self.entry_limit())
+ }
+
/// Count an entry not written (`ShaderCacheStatistics::write_failures`); the first is
/// announced as [`Degradation::ShaderCacheEntryNotWritten`], naming the directory and its
/// cap. Never an error: the work the entry would have saved is the directory's regardless.
@@ -474,6 +528,10 @@ impl Store {
count(&self.counters.holds_questions);
timed(&self.counters.read_nanos, || {
fs::metadata(path).is_ok_and(|metadata| metadata.len() <= self.entry_limit())
+ || self.system.as_ref().is_some_and(|system| {
+ path.strip_prefix(self.directory.root())
+ .is_ok_and(|relative| system.holds(relative, self.entry_limit()))
+ })
})
}
@@ -513,6 +571,10 @@ impl Store {
}
Found::Bytes(bytes, file) => (bytes, Some(file)),
Found::Queued(bytes) => (bytes, None),
+ Found::System(bytes) => {
+ let length = bytes.len() as u64;
+ return self.system_verdict(decode(bytes), length);
+ }
};
match decode(bytes) {
Some(value) => {
@@ -526,6 +588,23 @@ impl Store {
}
}
+ /// What a decoder made of `length` bytes the system layer gave: counted there, and a
+ /// refusal in this store's rejections too. Nothing is touched or removed.
+ fn system_verdict<T>(&self, decoded: Option<T>, length: u64) -> Lookup<T> {
+ let system = self.system.as_ref().expect("bytes of the system layer");
+ match decoded {
+ Some(value) => {
+ system.accepted(length);
+ Lookup::Hit(value)
+ }
+ None => {
+ system.rejected();
+ count(&self.counters.rejected);
+ Lookup::Rejected
+ }
+ }
+ }
+
/// [`Store::load`] of the file at `path`.
fn load_at<T, E>(&self, path: &Path, decode: impl FnOnce(&[u8]) -> Result<T, E>) -> Lookup<T> {
let (bytes, file) = match self.read(path) {
@@ -536,6 +615,9 @@ impl Store {
}
Found::Bytes(bytes, file) => (bytes, Some(file)),
Found::Queued(bytes) => (bytes, None),
+ Found::System(bytes) => {
+ return self.system_verdict(decode(&bytes).ok(), bytes.len() as u64);
+ }
};
match decode(&bytes) {
Ok(value) => {
@@ -594,7 +676,19 @@ impl Store {
return Found::Queued(bytes);
}
count(&self.counters.entry_reads);
- timed(&self.counters.read_nanos, || {
+ timed(&self.counters.read_nanos, || match self.read_own(path) {
+ // What this directory lacks or cannot give, the system layer may hold.
+ found @ (Found::Absent | Found::Failed) => match self.system_read(path) {
+ Some(bytes) => Found::System(bytes),
+ None => found,
+ },
+ found => found,
+ })
+ }
+
+ /// [`Store::read`] of this directory's own file at `path`.
+ fn read_own(&self, path: &Path) -> Found {
+ {
let file = match fs::File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Found::Absent,
@@ -617,7 +711,7 @@ impl Store {
}
Err(_) => Found::Failed,
}
- })
+ }
}
/// Give `bytes` to the cache thread to write as the entry named `name` (`worker.rs`): the
@@ -678,10 +772,10 @@ impl Store {
return Err(io::Error::other("the entry is larger than the cache takes"));
}
let parent = path.parent().expect("an entry has a directory");
- create_private_dir_all(parent)?;
+ self.directory.create_dir_all(parent)?;
let temporary = parent.join(format!(".{name}.{}.tmp", unique()));
let written = (|| {
- let mut file = new_private_file(&temporary)?;
+ let mut file = self.directory.new_file(&temporary)?;
file.write_all(bytes)?;
drop(file);
fs::rename(&temporary, path)?;
diff --git a/crates/metal2vulkan-runtime/src/shader_cache/system.rs b/crates/metal2vulkan-runtime/src/shader_cache/system.rs
new file mode 100644
index 0000000..c6e5cd1
--- /dev/null
+++ b/crates/metal2vulkan-runtime/src/shader_cache/system.rs
@@ -0,0 +1,268 @@
+//! The system layer: a second cache directory, read only, that one trusted writer fills for every
+//! process of the machine (the consumer's render server primes the OS's Metal libraries into it).
+//!
+//! - **Laid out as a device's own directory** (`store.rs`): the same `v1/<build>/<xx>/<name>`
+//! entries and content records, so what the writer's device wrote there under its own rules is
+//! what a reader's device looks up. A lookup asks the device's own directory first and this one
+//! only for what that lacks: a record the process added to (a making's driver data) is its own
+//! from then on.
+//! - **Trusted by who may write it, at every level opened.** The directory, each directory
+//! under it on the way to an entry, and the entry are opened without following a link
+//! (`O_NOFOLLOW`, relative to the level above) and judged by `fstat`: a directory or a regular
+//! file, owned by one of the owners the consumer named, that neither its group nor others may
+//! write. The directories above the layer are judged by `lstat` when it is opened: no link,
+//! owned by the superuser or one of the owners, and written by others only where the sticky bit
+//! keeps them from replacing what is not theirs. An entry's envelope carries a checksum, not a
+//! signature, and the driver's pipeline data is parsed by the driver as it is: a process must
+//! therefore never read an entry that anyone but those owners could have put there, and a
+//! level that fails the test is refused and counted, never read.
+//! - **Never written, touched or removed from here.** A hit refreshes no access time and
+//! contents a decoder refuses stay (counted): the directory is its writer's.
+//! - **Process-wide.** [`set_system_layer`] names the layer for the devices made after it;
+//! [`set_cache_readable_by_all`] makes the directories and entries a device writes readable by
+//! everyone (`0755`, `0644`) instead of by their owner alone, for the process that fills the
+//! layer through its own directory.
+
+use std::ffi::CString;
+use std::io::Read;
+use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
+use std::os::unix::ffi::OsStrExt;
+use std::path::{Component, Path, PathBuf};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, RwLock};
+
+/// What a system layer did for this process ([`SystemLayer::counts`]).
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct SystemLayerCounts {
+ /// Entries and records read from the layer and accepted by their decoder: work not done
+ /// in this process.
+ pub accepted: u64,
+ /// Entries read from the layer whose contents the decoder refused (left in place).
+ pub rejected: u64,
+ /// Levels (a directory or an entry) refused for their kind, owner or mode: never read.
+ pub refused: u64,
+ /// Bytes of the accepted entries.
+ pub accepted_bytes: u64,
+}
+
+/// An opened system layer.
+#[derive(Debug)]
+pub struct SystemLayer {
+ root: OwnedFd,
+ path: PathBuf,
+ owners: Vec<u32>,
+ accepted: AtomicU64,
+ rejected: AtomicU64,
+ refused: AtomicU64,
+ accepted_bytes: AtomicU64,
+}
+
+static LAYER: RwLock<Option<Arc<SystemLayer>>> = RwLock::new(None);
+static READABLE_BY_ALL: AtomicBool = AtomicBool::new(false);
+
+/// Name the system layer the devices made from now on consult, or none.
+pub fn set_system_layer(layer: Option<Arc<SystemLayer>>) {
+ *LAYER.write().unwrap_or_else(|e| e.into_inner()) = layer;
+}
+
+/// The system layer named last.
+pub fn system_layer() -> Option<Arc<SystemLayer>> {
+ LAYER.read().unwrap_or_else(|e| e.into_inner()).clone()
+}
+
+/// Whether the cache directories opened from now on make their directories and entries readable
+/// by everyone.
+pub fn set_cache_readable_by_all(on: bool) {
+ READABLE_BY_ALL.store(on, Ordering::Relaxed);
+}
+
+pub(super) fn cache_readable_by_all() -> bool {
+ READABLE_BY_ALL.load(Ordering::Relaxed)
+}
+
+fn c_path(bytes: &[u8]) -> Option<CString> {
+ CString::new(bytes).ok()
+}
+
+fn fstat(fd: &OwnedFd) -> Option<libc::stat> {
+ // SAFETY: a zeroed stat is a valid out value and the descriptor is open.
+ unsafe {
+ let mut st: libc::stat = std::mem::zeroed();
+ (libc::fstat(fd.as_raw_fd(), &mut st) == 0).then_some(st)
+ }
+}
+
+/// Open `name` under `parent` (or by its path where `parent` is `None`), never through a link.
+fn open_level(parent: Option<&OwnedFd>, name: &[u8], directory: bool) -> Result<OwnedFd, i32> {
+ let name = c_path(name).ok_or(libc::EINVAL)?;
+ let kind = if directory { libc::O_DIRECTORY } else { libc::O_NONBLOCK };
+ let flags = libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | kind;
+ // SAFETY: the name is a C string and the parent descriptor is open.
+ let fd = unsafe {
+ match parent {
+ Some(parent) => libc::openat(parent.as_raw_fd(), name.as_ptr(), flags),
+ None => libc::open(name.as_ptr(), flags),
+ }
+ };
+ if fd < 0 {
+ return Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO));
+ }
+ // SAFETY: a descriptor this call opened and nothing else owns.
+ Ok(unsafe { OwnedFd::from_raw_fd(fd) })
+}
+
+impl SystemLayer {
+ /// Open the layer at `root`, whose every level must be owned by one of `owners`. The error
+ /// says which level failed and how.
+ pub fn open(root: &Path, owners: &[u32]) -> Result<SystemLayer, String> {
+ if !root.is_absolute() || owners.is_empty() {
+ return Err("the layer needs an absolute path and an owner".into());
+ }
+ // The directories above the layer: no link among them, each the superuser's or an
+ // owner's, written by others only under the sticky bit. A level the sandbox lets this
+ // process see nothing of (EPERM, which no file mode gives) is one the OS made.
+ let mut above = PathBuf::new();
+ let mut parts = root.components().peekable();
+ while let Some(part) = parts.next() {
+ match part {
+ Component::RootDir | Component::Normal(_) => above.push(part),
+ _ => return Err(format!("{}: not a plain path", root.display())),
+ }
+ if parts.peek().is_none() {
+ break;
+ }
+ let path = c_path(above.as_os_str().as_bytes()).ok_or("a path with a zero byte")?;
+ // SAFETY: a zeroed stat is a valid out value and the path is a C string.
+ let (result, st) = unsafe {
+ let mut st: libc::stat = std::mem::zeroed();
+ (libc::lstat(path.as_ptr(), &mut st), st)
+ };
+ if result != 0 {
+ let error = std::io::Error::last_os_error();
+ if error.raw_os_error() == Some(libc::EPERM) {
+ continue;
+ }
+ return Err(format!("{}: {error}", above.display()));
+ }
+ let directory = st.st_mode & libc::S_IFMT == libc::S_IFDIR;
+ let owned = st.st_uid == 0 || owners.contains(&st.st_uid);
+ let closed = st.st_mode & 0o022 == 0 || st.st_mode & libc::S_ISVTX != 0;
+ if !directory || !owned || !closed {
+ return Err(format!(
+ "{}: above the layer, uid {} mode {:o}",
+ above.display(),
+ st.st_uid,
+ st.st_mode
+ ));
+ }
+ }
+ let fd = open_level(None, root.as_os_str().as_bytes(), true)
+ .map_err(|errno| format!("{}: errno {errno}", root.display()))?;
+ let layer = SystemLayer {
+ root: fd,
+ path: root.to_path_buf(),
+ owners: owners.to_vec(),
+ accepted: AtomicU64::new(0),
+ rejected: AtomicU64::new(0),
+ refused: AtomicU64::new(0),
+ accepted_bytes: AtomicU64::new(0),
+ };
+ match fstat(&layer.root) {
+ Some(st) if layer.trusted(&st, true) => Ok(layer),
+ Some(st) => Err(format!(
+ "{}: uid {} mode {:o}",
+ root.display(),
+ st.st_uid,
+ st.st_mode
+ )),
+ None => Err(format!("{}: no fstat", root.display())),
+ }
+ }
+
+ /// The layer's directory.
+ pub fn path(&self) -> &Path {
+ &self.path
+ }
+
+ /// Whether a level of `st` is what the layer reads: a directory or a regular file, an
+ /// owner's, that neither its group nor others may write.
+ fn trusted(&self, st: &libc::stat, directory: bool) -> bool {
+ let kind = if directory { libc::S_IFDIR } else { libc::S_IFREG };
+ st.st_mode & libc::S_IFMT == kind && self.owners.contains(&st.st_uid) && st.st_mode & 0o022 == 0
+ }
+
+ /// The open file of `relative` under the layer and its length, every level on the way judged.
+ /// `None` where a level is absent or refused (refusals are counted).
+ fn entry(&self, relative: &Path) -> Option<(std::fs::File, u64)> {
+ let mut parts = relative.components().peekable();
+ let mut level: Option<OwnedFd> = None;
+ while let Some(part) = parts.next() {
+ let Component::Normal(name) = part else {
+ return None;
+ };
+ let last = parts.peek().is_none();
+ let parent = level.as_ref().unwrap_or(&self.root);
+ let fd = match open_level(Some(parent), name.as_bytes(), !last) {
+ Ok(fd) => fd,
+ Err(errno) => {
+ // A link in an entry's place (ELOOP), or a file where a directory belongs.
+ if errno == libc::ELOOP || errno == libc::ENOTDIR {
+ self.refused.fetch_add(1, Ordering::Relaxed);
+ }
+ return None;
+ }
+ };
+ let st = fstat(&fd)?;
+ if !self.trusted(&st, !last) {
+ self.refused.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
+ if last {
+ return Some((std::fs::File::from(fd), st.st_size as u64));
+ }
+ level = Some(fd);
+ }
+ None
+ }
+
+ /// Whether the layer holds `relative`, of at most `limit` bytes.
+ pub fn holds(&self, relative: &Path, limit: u64) -> bool {
+ self.entry(relative).is_some_and(|(_, length)| length <= limit)
+ }
+
+ /// The bytes of `relative` under the layer, at most `limit`; `None` where it is absent,
+ /// refused, longer or unreadable. The caller's decoder judges the contents and reports the
+ /// verdict ([`accepted`](Self::accepted), [`rejected`](Self::rejected)).
+ pub fn read(&self, relative: &Path, limit: u64) -> Option<Vec<u8>> {
+ let (file, length) = self.entry(relative)?;
+ if length > limit {
+ return None;
+ }
+ let mut bytes = Vec::with_capacity(usize::try_from(length).unwrap_or(0));
+ match file.take(limit + 1).read_to_end(&mut bytes) {
+ Ok(_) if bytes.len() as u64 <= limit => Some(bytes),
+ _ => None,
+ }
+ }
+
+ /// Count an entry of `bytes` bytes read from the layer that its decoder accepted.
+ pub fn accepted(&self, bytes: u64) {
+ self.accepted.fetch_add(1, Ordering::Relaxed);
+ self.accepted_bytes.fetch_add(bytes, Ordering::Relaxed);
+ }
+
+ /// Count an entry read from the layer that its decoder refused.
+ pub fn rejected(&self) {
+ self.rejected.fetch_add(1, Ordering::Relaxed);
+ }
+
+ /// The counts so far.
+ pub fn counts(&self) -> SystemLayerCounts {
+ SystemLayerCounts {
+ accepted: self.accepted.load(Ordering::Relaxed),
+ rejected: self.rejected.load(Ordering::Relaxed),
+ refused: self.refused.load(Ordering::Relaxed),
+ accepted_bytes: self.accepted_bytes.load(Ordering::Relaxed),
+ }
+ }
+}
+74
View File
@@ -0,0 +1,74 @@
// SPDX-License-Identifier: MIT
// The system shader cache (SharedShaderCache.h, bridge patch 0036) between processes on the host.
// system_cache_host <loader> <own cache directory> fill
// opens the device in that directory as the process that fills the layer (entries readable by everyone), makes a render and a
// compute pipeline and a library from source, draws, and leaves the entries there.
// system_cache_host <loader> <own cache directory> read <layer directory> <owner uid>
// names the layer with that one trusted owner, opens the device in its own (empty) directory, makes the same pipelines and
// library, checks the drawn pixels, and prints what its own cache missed and what the layer answered:
// `RESULT layer=<0|1> accepted=<n> rejected=<n> refused=<n> translated=<functions translated here> replies=<n> pixels=ok`.
// The cases to run it for: a second process hitting what the first published (translated=0, accepted above 0), a layer of
// another owner or writable by its group (layer=0), a directory or an entry others may write and a link in a directory's place
// (refused above 0, those entries translated here), and entries with changed bytes (rejected above 0, left in place).
#include "FrontendChecks.h"
#include "NativeMetalObjects.h"
#include "../NativeBridgeMetalFrontend.h"
#include <string>
int main(int argc, char **argv) {
if (argc < 4) return 2;
const bool fill = !strcmp(argv[3], "fill");
if (!fill && argc != 6) return 2;
@autoreleasepool {
bool layer = false;
if (fill) native_bridge_metal_system_cache(nullptr, nullptr, 0, true);
else {
const uint32_t owner = (uint32_t)strtoul(argv[5], nullptr, 10);
layer = native_bridge_metal_system_cache(argv[4], &owner, 1, false);
if (!layer) std::printf("layer refused: %s\n", nmb_last_error());
}
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) {std::printf("no device: %s\n", nmb_last_error()); return 2;}
size_t length = 0;
const void *air = nmb_fixture_air(&length);
id<MTLLibrary> library = native_bridge_metal_library_with_air(air, length);
frontendchecks::Report report = {};
frontendchecks::gReport = &report;
id<MTLRenderPipelineState> pipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm);
NSError *error = nil;
id<MTLComputePipelineState> compute = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"scale_and_offset"] error:&error];
if (!library || !pipeline || !compute) {std::puts("FAIL a pipeline was refused"); return 1;}
// A library from source: its compiler reply is kept beside the pipeline cache, and read from the layer by a reader.
id<MTLLibrary> source = [device newLibraryWithSource:@"#include <metal_stdlib>\nusing namespace metal;\n"
"kernel void system_cache_probe(device float *data [[buffer(0)]], uint id [[thread_position_in_grid]]) {data[id] = 7.0f * float(id);}\n"
options:nil error:&error];
if (!source) std::printf("source library not made on this host: %s\n", error.localizedDescription.UTF8String);
auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO];
td.usage = MTLTextureUsageRenderTarget;
id<MTLTexture> texture = [device newTextureWithDescriptor:td];
id<MTLCommandBuffer> render = [[device newCommandQueue] commandBuffer];
frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6);
[render commit];
if (!frontendchecks::completed(render, "render")) return 1;
uint8_t pixels[16 * 16 * 4] = {};
[texture getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0];
if (frontendchecks::wrongTexels(pixels, 64, 16)) {std::puts("FAIL wrong pixels"); return 1;}
// The cache thread writes what the creations queued.
usleep(1500000);
NmbShaderCacheStats stats = {};
native_bridge_metal_shader_cache_stats(&stats);
uint64_t sy[4] = {};
(void)nmb_system_cache_counts(sy);
char text[8192] = {};
native_bridge_metal_telemetry(text, sizeof text);
unsigned long long translated = 0, found = 0, replies = 0;
for (char *line = strtok(text, "\n"); line; line = strtok(nullptr, "\n")) {
if (!strncmp(line, "SC ", 3) || !strncmp(line, "PRb ", 4) || !strncmp(line, "SY ", 3)) std::printf("%s\n", line);
sscanf(line, "SC fn %llu/%llu", &found, &translated);
if (const char *at = strstr(line, "replies ")) if (!strncmp(line, "SY ", 3)) sscanf(at, "replies %llu", &replies);
}
std::printf("RESULT layer=%d accepted=%llu rejected=%llu refused=%llu translated=%llu found=%llu misses=%llu replies=%llu pixels=ok\n",
layer, (unsigned long long)sy[0], (unsigned long long)sy[1], (unsigned long long)sy[2], translated, found,
(unsigned long long)stats.misses, replies);
}
}