From 9f336eb4a4d4b0436dc67d325b0356af823ecd2e Mon Sep 17 00:00:00 2001 From: rponeawa Date: Mon, 16 Nov 2026 10:03:58 +0800 Subject: [PATCH] perf(metal): integrate AdrenoMetal runtime changes --- NativeAdrenoMetalPlugin.mm | 62 +- NativeBridgeMetalFrontend.mm | 289 +- PrimeShaders.h | 257 +- ShaderCache/ShaderCacheApp.mm | 154 + ShaderCache/ShaderCacheState.h | 40 + ShaderCache/ShaderCacheWorker.mm | 71 + bridge/NativeMetalBridge.rs | 3 +- bridge/NativeMetalObjects.h | 11 +- bridge/NativeMetalObjects.rs | 135 +- bridge/RustPacBoundary.c | 5 + bridge/interface_regressions_host.mm | 2 +- ...ptimizations-on-development-555b13d5.patch | 4492 +++++++++++++++++ tools/audit-runtime-fast-paths.py | 186 + tools/build-shader-cache-app.py | 50 + tools/decode-command-buffer-log.py | 76 + tools/gpu-acceptance/build.py | 278 + tools/gpu-acceptance/lib.rs | 825 +++ tools/gpu-acceptance/main.c | 59 + tools/replay-renderbox-alpha.mm | 38 + 19 files changed, 6905 insertions(+), 128 deletions(-) create mode 100644 ShaderCache/ShaderCacheApp.mm create mode 100644 ShaderCache/ShaderCacheState.h create mode 100644 ShaderCache/ShaderCacheWorker.mm create mode 100644 bridge/patches/0046-main-optimizations-on-development-555b13d5.patch create mode 100644 tools/audit-runtime-fast-paths.py create mode 100644 tools/build-shader-cache-app.py create mode 100644 tools/decode-command-buffer-log.py create mode 100644 tools/gpu-acceptance/build.py create mode 100644 tools/gpu-acceptance/lib.rs create mode 100644 tools/gpu-acceptance/main.c create mode 100644 tools/replay-renderbox-alpha.mm diff --git a/NativeAdrenoMetalPlugin.mm b/NativeAdrenoMetalPlugin.mm index ffa797d..d6f0d4a 100644 --- a/NativeAdrenoMetalPlugin.mm +++ b/NativeAdrenoMetalPlugin.mm @@ -33,7 +33,9 @@ #include "NativeGpuTrace.h" #include "NativeMetalBridgePhase.h" #include "CacheExport.h" +#if !TARGET_OS_OSX #include "PrimeShaders.h" +#endif #include "ExceptionDiag.h" #if TARGET_OS_OSX @@ -224,6 +226,14 @@ static id clientDevice(unsigned port) { } // IOSurfaces this GPU reads or writes must be plain rows, as in the render server. CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); +#if TARGET_OS_OSX + if (geteuid() == 0 && getenv("NATIVE_METAL_PRIMER")) { + const std::string root = systemcache::writerDirectory(); + if (root.empty()) return nil; + native_bridge_metal_system_cache(nullptr, nullptr, 0, true); + return native_bridge_metal_open(turnip, root.c_str(), registryID, nullptr); + } +#endif char candidates[3][512] = {}; if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0; if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0; @@ -348,14 +358,21 @@ static id deviceForPort(unsigned port) { // draws on a small part of its target otherwise pays for whole. The colour order of GMEM loads and stores is the image's own // swap on the memory side (patches/mesa/turnip-gmem-memory-side-swap.patch). The RM row of the memory log counts both kinds; // tu-debug= (for instance gmem,nocb: every pass in tile memory, no concurrent binning) overrides this. - // Concurrent binning is a Turnip option that is off unless asked for (tu_allow_concurrent_binning, read from the environment - // by Mesa's option parser): with it the BV thread bins a pass while BR still renders the one before. tu-cb=1 turns it on. - // It stays at Mesa's default, off: what it gains here was never measured. - if (bootArgSet("tu-cb=1")) setenv("tu_allow_concurrent_binning", "true", 0); + // A7xx bins on BV while BR renders; Turnip retains its per-pass LRZ, query and hardware-binning checks. + setenv("tu_allow_concurrent_binning", bootArgSet("tu-cb=0") ? "false" : "true", 1); + // A740 has the sampler's built-in border constants; custom colors retain Turnip's table route. + setenv("tu_enable_fast_border_color_for_undefined_formats", bootArgSet("tu-fbc=0") ? "false" : "true", 1); // diag=1 turns on every diagnostic of the GPU stack in this process: the frontend's frame trace, Turnip's pass stamps and // submission ring, the runtime's stage ring, and in the render server the status and telemetry lines. Without it none of them runs. if (bootArgSet("diag=1") || bootArgSet("gpu-trace=1")) setenv("SHENG_DIAG", "1", 1); else unsetenv("SHENG_DIAG"); + if (getenv("SHENG_DIAG")) { + char boot[64] = {}; + size_t size = sizeof boot - 1; + if (!sysctlbyname("kern.bootsessionuuid", boot, &size, nullptr, 0)) setenv("NATIVE_METAL_BOOT_SESSION", boot, 1); + setenv("METAL2VULKAN_ENCODER_TIMES", "8ms", 1); + setenv("METAL2VULKAN_QUEUE_TIMES", "1", 1); + } if (bootArgSet("gpu-trace=1")) setenv("SHENG_GPU_TRACE", "1", 1); else unsetenv("SHENG_GPU_TRACE"); // The bridge's work placement (metal-flow=, read below). Any value logs the FL counts @@ -389,6 +406,35 @@ static id deviceForPort(unsigned port) { // log-gpu=1: the GPU stack's log lines (NSLog, the pipeline journal); off without it. if (bootArgSet("log-gpu=1")) setenv("SHENG_LOG_GPU", "1", 1); else unsetenv("SHENG_LOG_GPU"); +#if TARGET_OS_OSX + // ci-mps=0 selects Core Image's own blur kernels. Core Image reads this at its first blur; + // an explicit environment value takes precedence. + if (bootArgSet("ci-mps=0")) setenv("CI_ENABLE_MPS", "0", 0); + if (!strcmp(getprogname(), "iconservicesagent")) { + char args[1100] = {}; + size_t size = sizeof args - 1; + if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0) + if (const char *at = bootArgToken(args, "icon-cb=")) { + unsigned first = 0, count = 0, pass = 0, skipFirst = 0, skipLast = 0; + const int fields = sscanf(at + strlen("icon-cb="), "%u:%u:%u:%u:%u", &first, &count, &pass, &skipFirst, &skipLast); + if ((fields == 2 || fields == 5) && first && count && count <= 32) { + char range[32];snprintf(range, sizeof range, "%u:%u", first, count); + setenv("NATIVE_METAL_DUMP_CB", range, 0); + setenv("NATIVE_METAL_DUMP_LOG", "1", 0); + setenv("NATIVE_METAL_DUMP_FILE", "1", 0); + setenv("NATIVE_METAL_DUMP_PASS", "1", 0); + setenv("NATIVE_METAL_DUMP_INPUTS", "1", 0); + setenv("NATIVE_METAL_DUMP_DRAWS", "1:64", 0); + setenv("NATIVE_METAL_DUMP_WORDS", "32", 0); + setenv("NATIVE_METAL_DUMP_FUNCTIONS", "alpha_effect,accumulator_color,stroke_lines,plane_color,primitive_image", 0); + if (fields == 5 && pass && skipFirst && skipLast >= skipFirst) { + char skip[48];snprintf(skip, sizeof skip, "%u:%u:%u", pass, skipFirst, skipLast); + setenv("NATIVE_METAL_SKIP_DRAWS", skip, 0); + } + } + } + } +#endif #if TARGET_OS_OSX // macOS: Turnip opens the accelerator by class (AGXAccelerator here); the display takes plain rows (no Apple framebuffer or IOSurface // compression). The runtime keeps its default block size: the virtual-machine board shrank it for a 512 MiB GPU pool, which this @@ -608,12 +654,11 @@ static id deviceForPort(unsigned port) { return nullptr; }, gpu); } - // Every Metal library of the OS through the runtime, 90 s after the render server opened the device (PrimeShaders.h); the boots that froze - // or crashed before three minutes never primed anything. Progress is kept in the Data cache directory, so later boots resume. - // The translator recurses deeply on large functions: give it the stack of a main thread. +#if !TARGET_OS_OSX + // The iPadOS primer uses an 8 MiB stack for large translated functions. pthread_attr_setstacksize(&attributes, 8u << 20); pthread_t primer; - pthread_create(&primer, &attributes, [](void *) -> void * { + if (!bootArgSet("gpu-prime=0")) pthread_create(&primer, &attributes, [](void *) -> void * { pthread_setname_np("native-metal-prime"); // a process sample and the telemetry's TH rows name its CPU time char list[1024]; prime::run((id)device, gCacheDirectory, 90, @@ -621,6 +666,7 @@ static id deviceForPort(unsigned port) { gFillsSystemCache); return nullptr; }, nullptr); +#endif pthread_attr_destroy(&attributes); // The GMEM colour order test (NativeTurnipSwapTest.h), once, 20 s after the device is published: each result line goes to the // kernel's GPU trace as a problem record (event 65). diff --git a/NativeBridgeMetalFrontend.mm b/NativeBridgeMetalFrontend.mm index 65b8aa5..31614c9 100644 --- a/NativeBridgeMetalFrontend.mm +++ b/NativeBridgeMetalFrontend.mm @@ -5,7 +5,6 @@ // Every result comes from the runtime and the GPU: a refused call returns nil/NO with the runtime's reason, an encoder call the runtime // refuses fails its command buffer (status Error, handlers run) instead of being dropped. A selector this frontend does not implement is // reported through the missing-selector hook before the usual exception, so a hardware run names the gap. -// Shared events work within this process (no cross-process handle yet). Not yet implemented here: heaps, argument buffers, indirect commands, drawables/presentation. #import #import #import @@ -19,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -580,7 +580,7 @@ static bool isRenderServer() {return nativeIsRenderServer();} static void endRenderServerAfterGpuLoss() { static std::atomic ending{false}; if(!isRenderServer() || ending.exchange(true))return; - NBLOG(@"native-metal: the render server's GPU context is lost; ending backboardd so launchd starts it again"); + NBLOG(@"native-metal: the render server's GPU context is lost; ending %s so launchd starts it again",NATIVE_RENDER_SERVER); dispatch_after(dispatch_time(DISPATCH_TIME_NOW,(int64_t)(500*NSEC_PER_MSEC)),dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE,0),^{ kill(getpid(),SIGTERM); }); @@ -624,6 +624,7 @@ static void logTexture(const char *kind,MTLTextureDescriptor *d,IOSurfaceRef sur // planes, requested plane, plane offset, plane row bytes and allocation, and the descriptor's size and pixel format. struct SurfaceRefusal {uint32_t step,format,planes,plane,pixelFormat;uint64_t offset,row,allocation,width,height,surfaceWidth,surfaceHeight,rotation;bool framebufferOnly;}; static SurfaceRefusal gSurfaceLastRefusal; +#if !TARGET_OS_OSX // Shader priming's progress (PrimeShaders.h, in the plugin), for the PR row. Weak here: the plugin's own definition wins, and a host // binary that links the frontend without the plugin (every check-*-host.py test) has zeroes instead of an undefined symbol. extern "C" { @@ -635,8 +636,14 @@ __attribute__((weak)) volatile uint32_t native_prime_rate; __attribute__((weak)) volatile uint32_t native_prime_functions_resumed; __attribute__((weak)) volatile uint32_t native_prime_libraries_primed; // libraries this process's passes compiled __attribute__((weak)) volatile uint32_t native_prime_libraries_skipped; // other GPUs' driver bundles (whole system) or not loaded here +__attribute__((weak)) volatile uint32_t native_prime_pass_complete; +__attribute__((weak)) volatile uint32_t native_prime_libraries_failed; +__attribute__((weak)) volatile uint32_t native_prime_functions_refused; +__attribute__((weak)) volatile uint32_t native_prime_functions_skipped; +__attribute__((weak)) volatile uint32_t native_prime_inventory_failed; __attribute__((weak)) volatile uint32_t native_prime_scope; // 0 no pass yet, 1 loaded bundles only, 2 the whole system } +#endif static void refuseSurface(uint32_t step,IOSurfaceRef surface,MTLTextureDescriptor *d,NSUInteger plane,uint64_t offset,uint64_t row) { gSurfaceRefusals++; SurfaceRefusal r={step,surface?IOSurfaceGetPixelFormat(surface):0,surface?(uint32_t)IOSurfaceGetPlaneCount(surface):0,(uint32_t)plane, @@ -872,7 +879,7 @@ static NSData *compileMetalSource(NSString *source,NSString *arguments,NSString memcpy(request.data()+16,text.bytes,text.length);memcpy(request.data()+16+sourceBytes,options,optionBytes); return codeGenRequest(13,request,warnings,failure); } -// The front end's arguments for a set of compile options, as Metal passes them: the language version, fast math (Metal's default), +// The front end's arguments for a set of compile options, as Metal passes them: the language version, math mode and FP32 functions, // invariance, size optimization and the preprocessor macros; module caches in this process's temporary directory, which a sandboxed // client may write. static NSString *compileArgument(NSString *text) { @@ -885,8 +892,26 @@ static NSString *compileArguments(MTLCompileOptions *options) { const NSUInteger version=options && options.languageVersion?(NSUInteger)options.languageVersion:(NSUInteger)MTLLanguageVersion3_1; const unsigned major=(unsigned)(version>>16),minor=(unsigned)(version&0xffff); [arguments addObject:[NSString stringWithFormat:major>=3?@"-std=metal%u.%u":@"-std=ios-metal%u.%u",major,minor]]; - const BOOL fast=options?((BOOL (*)(id,SEL))objc_msgSend)(options,sel_registerName("fastMathEnabled")):YES; - [arguments addObject:fast?@"-ffast-math":@"-fno-fast-math"]; + const SEL modeSelector=sel_registerName("mathMode"),functionsSelector=sel_registerName("mathFloatingPointFunctions"); + NSString *modeName=nil,*functionsName=nil; + if([options respondsToSelector:modeSelector] && [options respondsToSelector:functionsSelector]) { + switch(((NSInteger (*)(id,SEL))objc_msgSend)(options,modeSelector)) { + case MTLMathModeSafe:modeName=@"safe";break; + case MTLMathModeRelaxed:modeName=@"relaxed";break; + case MTLMathModeFast:modeName=@"fast";break; + } + switch(((NSInteger (*)(id,SEL))objc_msgSend)(options,functionsSelector)) { + case MTLMathFloatingPointFunctionsFast:functionsName=@"fast";break; + case MTLMathFloatingPointFunctionsPrecise:functionsName=@"precise";break; + } + } + if(modeName && functionsName) { + [arguments addObject:[@"-fmetal-math-mode=" stringByAppendingString:modeName]]; + [arguments addObject:[@"-fmetal-math-fp32-functions=" stringByAppendingString:functionsName]]; + } else { + const BOOL fast=options?((BOOL (*)(id,SEL))objc_msgSend)(options,sel_registerName("fastMathEnabled")):YES; + [arguments addObject:fast?@"-ffast-math":@"-fno-fast-math"]; + } if(options.preserveInvariance)[arguments addObject:@"-fpreserve-invariance"]; if(options.optimizationLevel==MTLLibraryOptimizationLevelSize)[arguments addObject:@"-Os"]; [options.preprocessorMacros enumerateKeysAndObjectsUsingBlock:^(NSString *name,NSObject *value,BOOL *) { @@ -1584,7 +1609,46 @@ static void noteUnalignedTexture(MTLTextureDescriptor *d,NSUInteger offset,NSUIn @end @implementation NBFunctionConstant @end +@interface NBFunctionAttribute : NSObject +@property(nonatomic,copy) NSString *name; +@property(nonatomic) NSUInteger attributeIndex; +@property(nonatomic) MTLDataType attributeType; +@property(nonatomic,getter=isActive) BOOL active; +@property(nonatomic,getter=isPatchData) BOOL patchData; +@property(nonatomic,getter=isPatchControlPointData) BOOL patchControlPointData; +@end +@implementation NBFunctionAttribute +@end +static void libraryAirBytes(NBLibrary *library,const void *bytes,size_t length) { + const uint8_t *file=(const uint8_t *)bytes; + library.bytes=file;library.length=length; + if(length<8 || memcmp(file,"\xca\xfe\xba\xbe",4))return; + auto word=[&](size_t at) {return ((uint32_t)file[at]<<24)|((uint32_t)file[at+1]<<16)|((uint32_t)file[at+2]<<8)|file[at+3];}; + const uint32_t count=word(4); + if(count>(length-8)/20)return; + for(uint32_t i=0;ilength || length-at<20)return; + if(word(at)!=0x01000017)continue; + const size_t offset=word(at+8),size=word(at+12); + if(offset>length || size>length-offset)return; + library.bytes=file+offset;library.length=size;return; + } +} @implementation NBFunction +- (NSArray *)stageInputAttributes { + if(!_handle)return nil; + int32_t count=nmb_function_attribute_count(_handle);if(count<=0)return nil; + NSMutableArray *attributes=[NSMutableArray arrayWithCapacity:count]; + for(int32_t i=0;isizeof name)return nil; + NBFunctionAttribute *a=[NBFunctionAttribute new];a.name=@(name);a.attributeIndex=facts[0];a.attributeType=(MTLDataType)facts[1]; + a.active=facts[2];a.patchData=facts[3];a.patchControlPointData=facts[4];[attributes addObject:a]; + } + return attributes; +} +- (NSArray *)vertexAttributes {return self.functionType==MTLFunctionTypeVertex?[self stageInputAttributes]:nil;} - (NSDictionary *)functionConstantsDictionary { NSMutableDictionary *result=[NSMutableDictionary dictionary]; if(!_handle)return result; @@ -1759,6 +1823,14 @@ static NSError *functionLookupError(NBLibrary *library, NSString *name, NSString } return names; } +- (NSArray *)nb_primeFunctionNames { + NSMutableArray *names=[NSMutableArray array]; + if(!forEachLibraryEntry(self,[&](const std::string &entry,int type) { + if((type==0 || type==2) && !entry.empty())[names addObject:@(entry.c_str())]; + return true; + }))return self.functionNames; + return names; +} // _MTLLibrary's externFunctionNames: the entries of external linkage (TYPE 5), an empty array for a library without any. iPadOS 27's Core // Image collects the precompiled kernels' names so (DAGPrecompiledKernels: [CIKernelLibrary externFunctionNames] of ci_stdlib and // ci_filters) and links a kernel into its DAG only when its name is there or in a stitchable library; without the answer (nil, an unknown @@ -1789,6 +1861,7 @@ static NSError *functionLookupError(NBLibrary *library, NSString *name, NSString NmbFunction *handle=name?nmb_function_new_zero_constants(_handle,name.UTF8String):nullptr; if(!handle)return nil; NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.sourceName=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + function.library=self; return (id)function; } @end @@ -2067,8 +2140,10 @@ static void dumpSurfacesOnce(NSSet *textures,NSSet *owners) { // NATIVE_METAL_DUMP_CB: the textures this command buffer wrote, met while it was encoded (dumpTargets), the render pass's attachments // waiting for their encoder (dumpPending), how many passes and kernels it encoded so far, and whether it is the plugin's own readback. @property(nonatomic,strong) NSMutableArray *dumpTargets,*dumpPending; +@property(nonatomic,strong) NSMutableArray *dumpPassReads; +@property(nonatomic) uint64_t dumpAppIndex; @property(nonatomic) uint32_t dumpPasses,dumpKernels; -@property(nonatomic) BOOL dumpReadback; +@property(nonatomic) BOOL dumpReadback,dumpCapture; @property(nonatomic,strong) NBTexture *wallpaperOutput; @property(nonatomic,strong) NSMutableArray *dumpLog; // its passes and first draws, as encoded (NBEncoder dumpDraw:) - (void)refuse:(NSString *)what; @@ -2125,9 +2200,18 @@ static std::pair dumpRange() { static bool dumpMaybe(NBCommandBuffer *commandBuffer) { const auto range=dumpRange(); if(!range.first || commandBuffer.dumpReadback)return false; + if(commandBuffer.dumpCapture)return true; const uint64_t next=gAppCommits.load()+1; - return next+16>=range.first && next<(uint64_t)range.first+range.second; + const bool capture=next+16>=range.first && next<(uint64_t)range.first+range.second; + // Keep a selected buffer's complete encoding when other buffers commit before it. + if(capture)commandBuffer.dumpCapture=YES; + return capture; } +static bool dumpAtPassEnd() { + static const bool enabled=getenv("NATIVE_METAL_DUMP_PASS")!=nullptr; + return enabled; +} +static void dumpPassSnapshot(NBCommandBuffer *commands,NSArray *targets); // A line of the command buffer's NATIVE_METAL_DUMP_CB log (its passes, first draws and their state), printed before its readback. // NATIVE_METAL_DUMP_ONLY=1: only the NATIVE_METAL_DUMP_IMAGE targets are read back and written, with no log, so a long range of // command buffers (a small texture over hundreds of frames) costs little. @@ -2143,6 +2227,8 @@ static NSString *dumpWords(const uint8_t *p,size_t n) { static const size_t words=[]{const char *v=getenv("NATIVE_METAL_DUMP_WORDS");return v && atoi(v)>0?size_t(atoi(v)):size_t(16);}(); NSMutableString *s=[NSMutableString string]; for(size_t i=0;i(n/4,words);i++){float f;memcpy(&f,p+4*i,4);[s appendFormat:@" %.4g",f];} + [s appendString:@" hex"]; + for(size_t i=0;i(n/4,words);i++){uint32_t v;memcpy(&v,p+4*i,4);[s appendFormat:@" %08x",v];} size_t zero=0,nan=0;const size_t all=std::min(n/4,256); for(size_t i=0;i)texture index:(NSUInteger)index stage:(uint32_t)stage { if(texture && ![(id)texture isKindOfClass:[NBTexture class]]){[_commandBuffer refuse:@"foreign texture"];return;} if(gDrawStats)[self noteState:kDSTexture key:stage<<8|(uint32_t)index object:(__bridge const void *)texture value:0]; - if(texture && stage==0 && (texture.usage&MTLTextureUsageShaderWrite) && dumpMaybe(_commandBuffer)) { + static const bool captureInputs=getenv("NATIVE_METAL_DUMP_INPUTS")!=nullptr; + const bool captureInput=stage==2 && captureInputs; + if(texture && ((stage==0 && (texture.usage&MTLTextureUsageShaderWrite)) || captureInput) && dumpMaybe(_commandBuffer)) { if(!_dumpEntries)_dumpEntries=[NSMutableArray array]; if(!_commandBuffer.dumpTargets)_commandBuffer.dumpTargets=[NSMutableArray array]; const NSUInteger before=_commandBuffer.dumpTargets.count; dumpNote(_commandBuffer,_commandBuffer.dumpTargets,texture,0,0, - [NSString stringWithFormat:@"kernel %u texture %lu",_commandBuffer.dumpKernels,(unsigned long)index]); + captureInput?[NSString stringWithFormat:@"pass %u input texture %lu",_commandBuffer.dumpPasses,(unsigned long)index]: + [NSString stringWithFormat:@"kernel %u texture %lu",_commandBuffer.dumpKernels,(unsigned long)index]); if(_commandBuffer.dumpTargets.count>before)[_dumpEntries addObject:_commandBuffer.dumpTargets.lastObject]; } if(texture){[_commandBuffer own:texture];} @@ -2632,6 +2721,7 @@ static uint64_t encodeCommandOf(id object) { _targetW=0; } [super endEncoding]; + if(dumpAtPassEnd() && dumpMaybe(self.commandBuffer))dumpPassSnapshot(self.commandBuffer,self.dumpEntries); } - (void)setRenderPipelineState:(id)state { if(![(id)state isKindOfClass:[NBRenderPipeline class]]){[self.commandBuffer refuse:@"foreign render pipeline"];return;} @@ -3024,19 +3114,80 @@ static unsigned dumpTexel(uint32_t format,const uint8_t *p,float c[4],unsigned * default: return 0; } } +static void dumpWrite(const char *format,...) { + va_list args;va_start(args,format);char *text=nullptr; + const int length=vasprintf(&text,format,args);va_end(args); + if(length<0 || !text)return; + fwrite(text,1,(size_t)length,stderr); + static const bool file=getenv("NATIVE_METAL_DUMP_FILE")!=nullptr; + if(file && !gCacheRoot.empty()) { + static std::mutex lock; + static int fd=-1; + static size_t written=0; + static bool stopped=false; + std::lock_guard hold(lock); + if(!stopped) { + if(fd<0)fd=open((gCacheRoot+"/command-buffer.log").c_str(),O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOFOLLOW,0600); + constexpr size_t limit=2*1024*1024; + const char capped[]="native-metal dump capture limit reached\n"; + const bool fits=(size_t)length<=limit-sizeof capped-written; + const char *data=fits?text:capped; + size_t left=fits?(size_t)length:sizeof capped-1; + while(fd>=0 && left) { + const ssize_t n=write(fd,data,left); + if(n<0 && errno==EINTR)continue; + if(n<=0){close(fd);fd=-1;stopped=true;break;} + written+=(size_t)n;data+=n;left-=(size_t)n; + } + if(!fits && fd>=0){close(fd);fd=-1;stopped=true;} + } + } + static const bool unified=[]{const char *value=getenv("NATIVE_METAL_DUMP_LOG");return value && !strcmp(value,"1");}(); + if(unified) { + static std::atomic sequence; + const uint64_t entry=sequence.fetch_add(1)+1; + const size_t parts=((size_t)length+511)/512; + for(size_t part=0;part(512,(size_t)length-at)); + os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal CB dump %llu %zu/%zu: %{public}s",(unsigned long long)entry,part+1,parts,chunk.c_str()); + } + } + free(text); +} +static void dumpShaderFunction(NBFunction *function) { + static NSArray *wanted=[]{ + const char *value=getenv("NATIVE_METAL_DUMP_FUNCTIONS"); + return value?[@(value) componentsSeparatedByString:@","]:@[]; + }(); + NSString *name=function.sourceName?:function.name; + bool selected=false; + for(NSString *prefix in wanted)if(prefix.length && ([prefix isEqualToString:@"all"] || [name containsString:prefix])){selected=true;break;} + if(!selected)return; + static std::mutex lock;static std::unordered_set seen;static uint64_t sequence; + const std::string key=std::to_string((uintptr_t)function.library.bytes)+":"+(name.UTF8String?:""); + uint64_t entry; + {std::lock_guard guard(lock);if(!seen.insert(key).second)return;entry=++sequence;} + NSData *data=functionBitcode(function); + if(!data.length || data.length>512*1024){dumpWrite("native-metal dump function %llu %s: unavailable (%lu bytes)\n", + (unsigned long long)entry,name.UTF8String,(unsigned long)data.length);return;} + NSString *encoded=[data base64EncodedStringWithOptions:0]; + dumpWrite("native-metal dump function %llu %s: %s\n",(unsigned long long)entry,name.UTF8String,encoded.UTF8String); +} static void dumpPrint(uint64_t appIndex,NSDictionary *read) { NSDictionary *t=read[@"target"];NBTexture *texture=t[@"texture"];const NmbTextureDescriptor d=texture.descriptor; NSString *encoder=t[@"encoder"]; char head[400]; - snprintf(head,sizeof head,"native-metal dump cb %llu: %s%s%s%s texture %s %llux%llu f%llu L%lu S%lu",(unsigned long long)appIndex, + snprintf(head,sizeof head,"native-metal dump cb %llu: %s%s%s%s texture %s@%p %llux%llu f%llu L%lu S%lu",(unsigned long long)appIndex, [t[@"where"] UTF8String],encoder?" (":"",encoder?encoder.UTF8String:"",encoder?")":"",texture.label.length?texture.label.UTF8String:"-", + (__bridge void *)texture, (unsigned long long)d.width,(unsigned long long)d.height,(unsigned long long)d.format, (unsigned long)[t[@"level"] unsignedIntegerValue],(unsigned long)[t[@"slice"] unsignedIntegerValue]); - if(read[@"skip"]){fprintf(stderr,"%s: %s\n",head,[read[@"skip"] UTF8String]);return;} + if(read[@"skip"]){dumpWrite("%s: %s\n",head,[read[@"skip"] UTF8String]);return;} const unsigned w=[read[@"w"] unsignedIntValue],rows=[read[@"rows"] unsignedIntValue],bpp=[read[@"bpp"] unsignedIntValue]; const uint32_t format=[read[@"format"] unsignedIntValue]; const uint8_t *bytes=(const uint8_t *)[(id)read[@"buffer"] contents]; - if(!bytes){fprintf(stderr,"%s: no readback contents\n",head);return;} + if(!bytes){dumpWrite("%s: no readback contents\n",head);return;} double sum[4]={},lo[4]={INFINITY,INFINITY,INFINITY,INFINITY},hi[4]={-INFINITY,-INFINITY,-INFINITY,-INFINITY}; uint64_t finite[4]={},nans=0,infs=0,black=0,texels=0;unsigned channels=1; double cell[8][32]={};uint32_t cellCount[8][32]={},cellNan[8][32]={},cellLit[8][32]={}; @@ -3062,21 +3213,21 @@ static void dumpPrint(uint64_t appIndex,NSDictionary *read) { } snprintf(part,sizeof part,"; nan %llu inf %llu black %.1f%% of %llu",(unsigned long long)nans,(unsigned long long)infs, texels?100.0*black/texels:0.0,(unsigned long long)texels);line+=part; - fprintf(stderr,"%s\n",line.c_str()); + dumpWrite("%s\n",line.c_str()); for(unsigned gy=0;gy<8;gy++) { char grid[33]={}; for(unsigned gx=0;gx<32;gx++) { const uint32_t n=cellCount[gy][gx];const double mean=n?cell[gy][gx]/n:0; grid[gx]=!n?' ':cellNan[gy][gx]?'!':!cellLit[gy][gx]?'.':mean>1?'+':mean<0?'-':(char)('0'+std::min(9,(int)(mean*10))); } - fprintf(stderr,"native-metal dump cb %llu: |%s|\n",(unsigned long long)appIndex,grid); + dumpWrite("native-metal dump cb %llu: |%s|\n",(unsigned long long)appIndex,grid); } // NATIVE_METAL_DUMP_IMAGE: the whole texture, its rows top down as the format stores them, in base64 lines of 6 KiB. if([read[@"image"] boolValue]) { NSString *text=[[NSData dataWithBytesNoCopy:(void *)bytes length:(NSUInteger)rows*w*bpp freeWhenDone:NO] base64EncodedStringWithOptions:0]; const NSUInteger part=8192,parts=(text.length+part-1)/part; for(NSUInteger i=0;i)source.queue commandBuffer]; - if(!readback)return; - readback.dumpReadback=YES; - id blit=[(id)readback blitCommandEncoder]; +static NSArray *dumpEncodeReads(NBCommandBuffer *commands,NSArray *targets) { + id blit=[(id)commands blitCommandEncoder]; NSMutableArray *reads=[NSMutableArray array]; for(NSDictionary *t in targets) { NBTexture *texture=t[@"texture"];const NmbTextureDescriptor d=texture.descriptor; @@ -3111,7 +3250,8 @@ static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) { float c[4];unsigned channels=0;const uint8_t zero[16]={}; const unsigned bpp=texture.pvrtc?0:dumpTexel((uint32_t)d.format,zero,c,&channels); NSString *skip=!bpp?@"format not read back":d.samples>1?@"multisampled":d.type==MTLTextureType3D?@"3D": - texture.storageMode==MTLStorageModeMemoryless?@"memoryless":nil; + texture.storageMode==MTLStorageModeMemoryless?@"memoryless": + dumpAtPassEnd() && t[@"store"] && [t[@"store"] unsignedIntegerValue]==MTLStoreActionDontCare?@"store action DontCare":nil; if(skip){[reads addObject:@{@"target":t,@"skip":skip}];continue;} const NSUInteger w=std::max(1,(NSUInteger)d.width>>level),h=std::max(1,(NSUInteger)d.height>>level); const bool image=dumpImageWanted(t[@"where"]); @@ -3125,8 +3265,33 @@ static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) { [reads addObject:@{@"target":t,@"buffer":buffer,@"w":@(w),@"rows":@(rows),@"bpp":@(bpp),@"format":@((uint32_t)d.format),@"image":@(image)}]; } [blit endEncoding]; + return reads; +} +// Snapshot a stored attachment immediately after its encoder, before heap reuse or a later pass can overwrite it. +static void dumpPassSnapshot(NBCommandBuffer *commands,NSArray *targets) { + if(dumpOnly())targets=[targets filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(NSDictionary *t,NSDictionary *bindings){(void)bindings; + return dumpImageWanted(t[@"where"]); + }]]; + if(!targets.count)return; + if(!commands.dumpPassReads)commands.dumpPassReads=[NSMutableArray array]; + [commands.dumpPassReads addObjectsFromArray:dumpEncodeReads(commands,targets)]; +} +// The readback of `source`'s targets, by a command buffer of the plugin's own committed after it on its queue. +static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) { + NSArray *targets=[source.dumpTargets copy]; + if(dumpOnly())targets=[targets filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(NSDictionary *t,NSDictionary *bindings){(void)bindings; + return dumpImageWanted(t[@"where"]); + }]]; + else dumpWrite("native-metal dump cb %llu: %lu targets, %u passes, %u kernels%s\n",(unsigned long long)appIndex, + (unsigned long)targets.count,source.dumpPasses,source.dumpKernels,dumpAtPassEnd()?"; pass snapshots":""); + for(NSString *line in source.dumpLog)dumpWrite("native-metal dump cb %llu: %s\n",(unsigned long long)appIndex,line.UTF8String); + if(dumpAtPassEnd() || !targets.count)return; + NBCommandBuffer *readback=(NBCommandBuffer *)[(id)source.queue commandBuffer]; + if(!readback)return; + readback.dumpReadback=YES; + NSArray *reads=dumpEncodeReads(readback,targets); [(id)readback addCompletedHandler:^(id done) { - if(done.error){fprintf(stderr,"native-metal dump cb %llu: readback failed: %s\n",(unsigned long long)appIndex, + if(done.error){dumpWrite("native-metal dump cb %llu: readback failed: %s\n",(unsigned long long)appIndex, done.error.localizedDescription.UTF8String);return;} for(NSDictionary *read in reads)dumpPrint(appIndex,read); }]; @@ -3180,6 +3345,8 @@ static char kUserDictionaryKey; - (void)setProtectionOptions:(uint64_t)options {if(options)[self refuse:[NSString stringWithFormat:@"protection options 0x%llx",options]];} - (void)refuse:(NSString *)what { if(!self.encodeFailure){self.encodeFailure=what;NBLOG(@"native-metal: command buffer refused: %@",what);} + // Recovery creates a new runtime device; launchd restarts a render server whose retained objects belong to the previous device. + if([what rangeOfString:@"MadeBeforeRecovery"].location!=NSNotFound)endRenderServerAfterGpuLoss(); } - (MTLCommandBufferStatus)status { if(self.failedLocally)return MTLCommandBufferStatusError; @@ -3255,9 +3422,11 @@ static char kUserDictionaryKey; _dumpPending=[NSMutableArray array]; for(unsigned i=0;i<8;i++) { MTLRenderPassColorAttachmentDescriptor *c=d.colorAttachments[i]; + const NSUInteger before=_dumpPending.count; if(c.resolveTexture)dumpNote(self,_dumpPending,c.resolveTexture,c.resolveLevel,c.resolveSlice, [NSString stringWithFormat:@"pass %u resolve %u",_dumpPasses,i]); else dumpNote(self,_dumpPending,c.texture,c.level,c.slice,[NSString stringWithFormat:@"pass %u color %u",_dumpPasses,i]); + if(_dumpPending.count>before)_dumpPending.lastObject[@"store"]=@(c.storeAction); } dumpNote(self,_dumpPending,d.depthAttachment.texture,d.depthAttachment.level,d.depthAttachment.slice, [NSString stringWithFormat:@"pass %u depth",_dumpPasses]); @@ -3418,6 +3587,11 @@ static char kUserDictionaryKey; frontStage(traceNumber,self.commitIndex,8); const uint64_t flushed=flushSurfaces(_surfaces); frontStage(traceNumber,self.commitIndex,9,flushed,(uint32_t)_surfaces.count); + if(_dumpAppIndex) { + if(self.status==MTLCommandBufferStatusCompleted)for(NSDictionary *read in _dumpPassReads)dumpPrint(_dumpAppIndex,read); + else dumpWrite("native-metal dump cb %llu: pass snapshots failed: %s\n",(unsigned long long)_dumpAppIndex,self.error.localizedDescription.UTF8String?:"command refused"); + _dumpPassReads=nil; + } if(freezeWallpaperOn() && !self.failedLocally && self.status==MTLCommandBufferStatusCompleted && self.wallpaperOutput && encodedWork.draws) { std::lock_guard guard(gWallpaperLock); @@ -3519,13 +3693,15 @@ static char kUserDictionaryKey; // Completion owns this reference; the runtime orders the non-owning scheduled callback before it. void *context=(__bridge_retained void *)self; const uint64_t appIndex=self.dumpReadback?0:gAppCommits.fetch_add(1)+1; + const auto range=dumpRange(); + const bool capture=appIndex && range.first && appIndex>=range.first && appIndex<(uint64_t)range.first+range.second; + if(capture && dumpAtPassEnd()){self.dumpAppIndex=appIndex;dumpReadback(self,appIndex);} frontStage(traceNumber,self.commitIndex,6); if(nmb_command_buffer_on_completed(_handle,completed,context) && nmb_command_buffer_on_scheduled(_handle,scheduled,context) && nmb_command_buffer_commit(_handle)) { frontStage(traceNumber,self.commitIndex,7); - const auto range=dumpRange(); self.submitted=YES; - if(appIndex && range.first && appIndex>=range.first && appIndex<(uint64_t)range.first+range.second)dumpReadback(self,appIndex); + if(capture && !dumpAtPassEnd())dumpReadback(self,appIndex); [self fireEarly];return; } (void)CFBridgingRelease(context); // refused registration/commit cannot call the registered handlers @@ -3859,6 +4035,14 @@ static bool renderDescriptor(MTLRenderPipelineDescriptor *d,NBRenderDescriptor * // NATIVE_METAL_DUMP_CB: a render pipeline's functions, color formats (b blends, m its write mask), depth format and vertex layout // (attribute=format@offset/buffer, layout=stride/step/rate), for its draws' lines. static NSString *renderDumpInfo(MTLRenderPipelineDescriptor *d) { + if(dumpRange().first) { + NBFunction *vertex=[(id)d.vertexFunction isKindOfClass:[NBFunction class]]?(NBFunction *)d.vertexFunction:nil; + NBFunction *fragment=[(id)d.fragmentFunction isKindOfClass:[NBFunction class]]?(NBFunction *)d.fragmentFunction:nil; + dumpWrite("native-metal dump pipeline %s | v %s source %s %s | f %s source %s %s\n",d.label.UTF8String?:"-", + vertex.name.UTF8String?:"-",vertex.sourceName.UTF8String?:"-",vertex?vertex.constantsText.c_str():"", + fragment.name.UTF8String?:"-",fragment.sourceName.UTF8String?:"-",fragment?fragment.constantsText.c_str():""); + if(vertex)dumpShaderFunction(vertex);if(fragment)dumpShaderFunction(fragment); + } NSMutableString *s=[NSMutableString stringWithFormat:@"%@ v=%@ f=%@",d.label?:@"-",d.vertexFunction.name?:@"-",d.fragmentFunction.name?:@"-"]; for(unsigned i=0;i<8;i++) { MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];if(a.pixelFormat==MTLPixelFormatInvalid)continue; @@ -4220,10 +4404,11 @@ static char dvmMemoryReleaseKey; NSString *why=lastError(); uint64_t answer[2]={0,0}; (void)nmb_device_heap_texture_size_and_align(gDevice,&descriptor,answer); - GLOG(@"native-metal: heap texture refused: %@ | fmt %lu %lux%lu L%lu type %lu needs %#llx@%#llx heap %#lx used %#lx room %#lx",why, + GLOG(@"native-metal: heap texture refused: %@ | fmt %lu %lux%lu L%lu type %lu needs %#llx@%#llx heap %#lx used %#lx room %#lx usage %#lx options %#lx optimized %d descriptor %p",why, (unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount, (unsigned long)_heapType,(unsigned long long)answer[0],(unsigned long long)answer[1],(unsigned long)nmb_heap_size(_handle), - (unsigned long)nmb_heap_used_size(_handle),(unsigned long)nmb_heap_max_available_size(_handle,answer[1]?answer[1]:1)); + (unsigned long)nmb_heap_used_size(_handle),(unsigned long)nmb_heap_max_available_size(_handle,answer[1]?answer[1]:1), + (unsigned long)d.usage,(unsigned long)d.resourceOptions,(int)d.allowGPUOptimizedContents,d); if(offset!=UINT64_MAX)return nil; handle=nmb_texture_new(gDevice,&descriptor); if(!handle){NBLOG(@"native-metal: texture (fmt %lu %lux%lu, %lu levels) refused: %@",(unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount,lastError());return nil;} @@ -4432,6 +4617,13 @@ static void fillBaseLimits(id device); NmbTextureDescriptor descriptor;uint64_t answer[2]={0,0}; if(textureDescriptor(d,&descriptor)&&!nmb_device_heap_texture_size_and_align(gDevice,&descriptor,answer)) NBLOG(@"native-metal: heap texture size (fmt %lu %lux%lu, %lu levels) refused: %@",(unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount,lastError()); + if(iconMetalTrace()) { + static std::atomic queries{0}; + if(queries.fetch_add(1)<64)GLOG(@"native-metal: heap query fmt %lu %lux%lu L%lu usage %#lx options %#lx optimized %d -> %#llx@%#llx descriptor %p", + (unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount, + (unsigned long)d.usage,(unsigned long)d.resourceOptions,(int)d.allowGPUOptimizedContents, + (unsigned long long)answer[0],(unsigned long long)answer[1],d); + } return (MTLSizeAndAlign){answer[0],answer[1]}; } - (MTLSizeAndAlign)heapBufferSizeAndAlignWithLength:(NSUInteger)length options:(MTLResourceOptions)options { @@ -4539,7 +4731,7 @@ static void fillBaseLimits(id device); const uint64_t libraryBegan=nowUs(); NmbLibrary *handle=nmb_library_new(gDevice,bytes,size,false);gTimeLibrary.add(libraryBegan); if(!handle)return failWith>(error,@"library",nil); - NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=mapped;library.bytes=(const uint8_t *)bytes;library.length=size; + NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=mapped;libraryAirBytes(library,bytes,size); if(error)*error=nil;return (id)library; } // A library file that cannot be read (macOS run36: QuartzCore's default.metallib failed with NSCocoaErrorDomain 260 on a fresh Data @@ -4564,7 +4756,7 @@ static void reportUnreadableLibrary(NSString *path) { const uint64_t libraryBegan=nowUs(); NmbLibrary *handle=nmb_library_new(gDevice,bytes.bytes,bytes.length,false);gTimeLibrary.add(libraryBegan); if(!handle)return failWith>(error,@"library",nil); - NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=bytes;library.bytes=(const uint8_t *)bytes.bytes;library.length=bytes.length; + NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=bytes;libraryAirBytes(library,bytes.bytes,bytes.length); if(error)*error=nil;return (id)library; } - (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { @@ -5495,18 +5687,23 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) { row("V vkpipe n=%llu total=%llums max=%llums waited=%llums failed=%llu",(ull)work[0],ms(work[1]),ms(work[2]),ms(work[3]),(ull)work[4]); row("V cache hit%llu miss%llu wr%llu | msg i%llu w%llu",(ull)work[7],(ull)work[8],(ull)work[9],(ull)work[5],(ull)work[6]); } - // The shader cache's full counts (nmb_device_shader_cache_detail): functions found/translated, compute and render pipelines found/compiled, entries + // The shader cache's full counts (nmb_device_shader_cache_detail): functions found/translated, driver data found/unchanged, render pipelines found/compiled, entries // rejected, unreadable, handed to the directory (queued, written on the asking thread), bytes written, failed writes, evictions, the driver's own // pipeline cache: loads, saves, hits, misses. uint64_t sc[24]={}; if(nmb_device_shader_cache_detail(gDevice,sc)) { - row("SC fn %llu/%llu cp %llu/%llu rp %llu/%llu rej %llu rd-fail %llu",(ull)sc[2],(ull)sc[3],(ull)sc[4],(ull)sc[5],(ull)sc[6],(ull)sc[7], + row("SC fn %llu/%llu data %llu/%llu rp %llu/%llu rej %llu rd-fail %llu",(ull)sc[2],(ull)sc[3],(ull)sc[4],(ull)sc[5],(ull)sc[6],(ull)sc[7], (ull)sc[8],(ull)sc[9]); row("SCb wr %llu q %llu thr %llu %lluK wfail %llu evict %llu qhit %llu | drv load %llu save %llu hit %llu miss %llu",(ull)sc[10],(ull)sc[11], (ull)sc[12],(ull)(sc[13]>>10),(ull)sc[15],(ull)sc[16],(ull)sc[14],(ull)sc[17],(ull)sc[20],(ull)sc[18],(ull)sc[19]); + row("SCc drv truncated %llu mismatched %llu",(ull)sc[21],(ull)sc[22]); } } +#if !TARGET_OS_OSX row("PR compiling=%u libs=%u/%u made=%u rate=%u resumed=%u",native_prime_compiling,native_prime_libraries_done,native_prime_libraries_total,native_prime_made,native_prime_rate,native_prime_functions_resumed); + row("PRc state=%u failed=%u refused=%u bad=%u missing=%u",native_prime_pass_complete,native_prime_libraries_failed, + native_prime_functions_refused,native_prime_functions_skipped,native_prime_inventory_failed); +#endif // Priming's reach and the system shader cache. PRb: the pass's scope (1 the bundles loaded here, 2 the whole system), the libraries // it compiled itself, those it leaves out (other GPUs' driver bundles, or not loaded here), the system cache's state here (0 not // named, 1 read, 2 filled by this process, 3 refused) and what this process published: the entries and KiB its cache directory @@ -5516,12 +5713,24 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) { uint64_t sc[24]={},sy[4]={}; if(gDevice)(void)nmb_device_shader_cache_detail(gDevice,sc); (void)nmb_system_cache_counts(sy); +#if !TARGET_OS_OSX row("PRb scope=%u primed=%u left=%u | system %d pub %llu/%lluK",native_prime_scope,native_prime_libraries_primed, native_prime_libraries_skipped,gSystemCacheState,(ull)(gSystemCacheState==2?sc[11]+sc[12]:0),(ull)(gSystemCacheState==2?sc[13]>>10:0)); +#else + row("SY system %d pub %llu/%lluK",gSystemCacheState,(ull)(gSystemCacheState==2?sc[11]+sc[12]:0),(ull)(gSystemCacheState==2?sc[13]>>10:0)); +#endif if(gSystemCacheState==1 || gSystemCacheState==3) row("SY read %llu/%lluK rej %llu refused %llu replies %llu",(ull)sy[0],(ull)(sy[3]>>10),(ull)sy[1],(ull)sy[2],(ull)gCompileSystem.load()); } if(gDevice) { + uint64_t fast[15]={}; + if(nmb_device_fast_path_counts(gDevice,fast)) { + row("FP group %llu/%llu pools %llu/%llu linear %llu/%llu evict %llu",(ull)fast[0],(ull)fast[1],(ull)fast[2],(ull)fast[3], + (ull)fast[4],(ull)fast[5],(ull)fast[6]); + if(fast[9]==UINT64_MAX)row("CM calls %llu total %lluus",(ull)fast[7],(ull)fast[8]); + else row("CM calls %llu us %llu/%llu/%llu/%llu/%llu/%llu/%llu",(ull)fast[7],(ull)fast[8],(ull)fast[9],(ull)fast[10],(ull)fast[11], + (ull)fast[12],(ull)fast[13],(ull)fast[14]); + } // The warm pass (records warmed, failed, unread, KiB of driver data) and where work ran: scheduler passes and batches at commit, render // log waits, encoder ends on the lowering thread. uint64_t flow[8]={}; diff --git a/PrimeShaders.h b/PrimeShaders.h index 6db8352..7a9253e 100644 --- a/PrimeShaders.h +++ b/PrimeShaders.h @@ -1,15 +1,5 @@ // SPDX-License-Identifier: MIT -// Shader priming in the render server: once the desktop has settled, every Metal library of the OS volume goes through the runtime on a -// background thread, so its functions are translated and compiled into the persistent cache directory before anything asks for them. -// Each kernel function becomes a compute pipeline; each vertex function becomes a render pipeline with the library's fragment function of -// the same stem (`_vert_` -> `_frag_`, or a fragment-less pipeline) and one BGRA8 attachment. Pipelines a caller makes with other constants -// or attachments still compile then, but their functions' modules are decoded and the translator has seen every function once. -// State in the cache directory: prime-done lists finished libraries (path, size, mtime), prime-bad the functions that ended the process -// (prime-current names the function being compiled; a start that finds it moves it to prime-bad), prime.log one line per library, and -// prime-fn the functions finished inside the libraries not finished yet (path, name and the build that made them). -// A library listed as finished is never read again, and a function listed in prime-fn is not compiled again, so a restart resumes where -// the previous run stopped even inside a library with thousands of functions (the one that held priming at 162 of 280 libraries for -// every boot: each boot ended before it did and began it again). +// Inventory compilation shared by Shader Cache on macOS and the iPadOS render server. #pragma once #import #import @@ -24,15 +14,18 @@ #include #include #include +#include #include #include #include #include +#include extern "C" unsigned native_bridge_metal_recent_commits(uint64_t windowUs); @interface NSObject (NBPrimeZeroConstants) - (id)nb_newFunctionWithZeroConstants:(NSString *)name; +- (NSArray *)nb_primeFunctionNames; @end // Priming's progress for the telemetry table (the PR row): what the thread does (1 a pipeline compiles, 2 it waits for the desktop to @@ -42,10 +35,18 @@ extern "C" {volatile uint32_t native_prime_compiling, native_prime_libraries_don native_prime_functions_resumed; // For the PRb row: the libraries this process's passes compiled, those a pass leaves out, and the pass's scope (1 the bundles loaded // in this process, 2 the whole system). -volatile uint32_t native_prime_libraries_primed, native_prime_libraries_skipped, native_prime_scope;} +volatile uint32_t native_prime_libraries_primed, native_prime_libraries_skipped, native_prime_scope; +volatile uint32_t native_prime_pass_complete, native_prime_libraries_failed, native_prime_functions_refused, + native_prime_functions_skipped, native_prime_inventory_failed;} extern "C" void native_bridge_metal_quiet_problems(bool); namespace prime { +static std::function progress; +static std::function cancelled; +static bool stopping() {return cancelled && cancelled();} +static void report(const std::string &library = {}, const std::string &function = {}) { + if (progress) progress(library, function); +} #if TARGET_OS_OSX // macOS: the system apps are in /System/Applications; the OS cryptex is reached through /System/Cryptexes/OS as on iPadOS. static const char *const kRoots[] = {"/System/Library", "/System/Applications", "/Applications", "/System/Cryptexes/OS/System/Library"}; @@ -91,6 +92,14 @@ static void writeCurrent(const std::string &path, const std::string &line) { } static uint64_t nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1000000;} +static std::string bootSession() { + char uuid[64] = {}; + size_t size = sizeof uuid; + if (sysctlbyname("kern.bootsessionuuid", uuid, &size, nullptr, 0) != 0) return "unknown"; + uuid[sizeof uuid - 1] = 0; + return uuid; +} + // The libraries: first the paths of the list file beside the bundle (NativeAdrenoMetal.primelist, every .metallib of the OS image, one // absolute path per line), because the render server's sandbox refuses to list most directories (a scan alone found 8 of 166). Only // without a list does the scan run: it walks every directory it can reach and held a core for 8 s with frames stalling (i557u, 120 s @@ -99,22 +108,25 @@ static uint64_t nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 100000 static std::vector libraries(const std::string &listPath, std::string ¬e) { std::vector found; std::set> seen; - unsigned lines = 0, failed = 0; + unsigned lines = 0, failed = 0, invalid = 0; int lastErrno = 0; std::set list = listPath.empty() ? std::set() : readLines(listPath); const char *source = listPath.empty() ? "(none)" : listPath.c_str(); if (list.empty()) {list = embeddedLines(); source = "embedded";} + const bool inventory = !list.empty(); for (const std::string &path : list) { ++lines; struct stat st = {}; - if (path.empty() || path[0] != '/') continue; + if (path.empty() || path[0] != '/') {++invalid; continue;} if (stat(path.c_str(), &st) != 0) {++failed; lastErrno = errno; continue;} - if (S_ISREG(st.st_mode) && seen.insert({st.st_dev, st.st_ino}).second) found.push_back(path); + if (!S_ISREG(st.st_mode)) {++invalid; continue;} + if (seen.insert({st.st_dev, st.st_ino}).second) found.push_back(path); } char line[200]; - snprintf(line, sizeof line, "list %s lines=%u kept=%zu stat-failed=%u errno=%d", source, lines, - found.size(), failed, lastErrno); + snprintf(line, sizeof line, "list %s lines=%u kept=%zu stat-failed=%u errno=%d invalid=%u inventory=%u", source, lines, + found.size(), failed, lastErrno, invalid, inventory ? 1u : 0u); note = line; + native_prime_inventory_failed = failed + invalid + (inventory ? 0u : 1u); for (const char *root : kRoots) { if (!found.empty()) break; char *paths[] = {const_cast(root), nullptr}; @@ -139,12 +151,13 @@ static std::vector libraries(const std::string &listPath, std::stri // a quarter of a run's two-second windows, and a desktop in use ran at 18 to 20 submissions a second, under the bar above, so priming // compiled through it (in four runs' process samples the priming thread was compiling or writing its progress in 88 to 94 % of the // samples, in two of them beside the encoding thread's draws). There any command buffer in the window is the render server at work. -#if TARGET_OS_OSX -static constexpr unsigned kQuietCommits = 0; -#else +#if !TARGET_OS_OSX static constexpr unsigned kQuietCommits = 8; #endif static uint64_t waitForQuiet() { +#if TARGET_OS_OSX + return 0; +#else const uint64_t began = nowMs(); uint64_t quietSince = 0; bool busy = false; @@ -161,6 +174,7 @@ static uint64_t waitForQuiet() { else if (now - quietSince >= 2000) return now - began; usleep(100000); } +#endif } #if TARGET_OS_OSX @@ -201,6 +215,26 @@ static bool libraryOfDriverBundle(const std::string &path) { const char *slash = strchr(plain + sizeof kExtensions - 1, '/'); return slash && slash - plain > 7 && strncmp(slash - 7, ".bundle", 7) == 0; } +// A universal archive whose slices all contain Apple GPU machine code. +static bool appleGpuArchive(const std::string &path) { + FILE *file = fopen(path.c_str(), "rb"); + if (!file) return false; + struct stat st = {}; + uint8_t header[8]; + auto word = [](const uint8_t *p) {return (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | p[3];}; + bool archive = fstat(fileno(file), &st) == 0 && fread(header, 1, sizeof header, file) == sizeof header && + !memcmp(header, "\xca\xfe\xba\xbe", 4); + const uint32_t count = archive ? word(header + 4) : 0; + archive = archive && count && st.st_size >= 8 && count <= (uint64_t)(st.st_size - 8) / 20; + for (uint32_t i = 0; archive && i < count; ++i) { + uint8_t slice[20]; + if (fread(slice, 1, sizeof slice, file) != sizeof slice) {archive = false; break;} + const uint64_t offset = word(slice + 8), size = word(slice + 12); + archive = word(slice) == 0x01000013 && size && offset <= (uint64_t)st.st_size && size <= (uint64_t)st.st_size - offset; + } + fclose(file); + return archive; +} #endif extern "C" const char *nmb_runtime_build(void); @@ -253,6 +287,9 @@ static bool linkedKernel(id device, id lib, id callee = [lib newFunctionWithName:symbol]; if (!callee) return false; + if (callee.functionConstantsDictionary.count && [lib respondsToSelector:@selector(nb_newFunctionWithZeroConstants:)]) + callee = [(NSObject *)lib nb_newFunctionWithZeroConstants:symbol]; + if (!callee) return false; [linked addObject:callee]; MTLComputePipelineDescriptor *d = [MTLComputePipelineDescriptor new]; d.computeFunction = kernel; @@ -265,10 +302,57 @@ static bool linkedKernel(id device, id lib, id defaultFunction(id lib, NSString *name, unsigned &zeroed) { + id function = [lib newFunctionWithName:name]; + if (function.functionConstantsDictionary.count && [lib respondsToSelector:@selector(nb_newFunctionWithZeroConstants:)]) { + function = [(NSObject *)lib nb_newFunctionWithZeroConstants:name]; + if (function) ++zeroed; + } + return function; +} + +// Canonical packed inputs follow the function's reflected scalar type and lane count. +static MTLVertexDescriptor *vertexInputs(id function) { + MTLVertexDescriptor *descriptor = [MTLVertexDescriptor vertexDescriptor]; + for (MTLVertexAttribute *attribute in function.vertexAttributes) { + const NSUInteger index = attribute.attributeIndex, type = attribute.attributeType; + if (index >= 31 || !attribute.isActive) continue; + MTLVertexFormat format = MTLVertexFormatInvalid; + NSUInteger bytes = 0; + struct Family {NSUInteger type, bytes; MTLVertexFormat scalar, vector2;}; + static const Family families[] = { + {MTLDataTypeFloat,4,MTLVertexFormatFloat,MTLVertexFormatFloat2}, + {MTLDataTypeHalf,2,MTLVertexFormatHalf,MTLVertexFormatHalf2}, + {MTLDataTypeInt,4,MTLVertexFormatInt,MTLVertexFormatInt2}, + {MTLDataTypeUInt,4,MTLVertexFormatUInt,MTLVertexFormatUInt2}, + {MTLDataTypeShort,2,MTLVertexFormatShort,MTLVertexFormatShort2}, + {MTLDataTypeUShort,2,MTLVertexFormatUShort,MTLVertexFormatUShort2}, + {MTLDataTypeChar,1,MTLVertexFormatChar,MTLVertexFormatChar2}, + {MTLDataTypeUChar,1,MTLVertexFormatUChar,MTLVertexFormatUChar2}, + }; + for (const Family &family : families) if (type >= family.type && type < family.type + 4) { + const NSUInteger lanes = type - family.type + 1; + format = lanes == 1 ? family.scalar : (MTLVertexFormat)(family.vector2 + lanes - 2); + bytes = family.bytes * lanes; + break; + } + if (format == MTLVertexFormatInvalid) continue; + descriptor.attributes[index].format = format; + descriptor.attributes[index].bufferIndex = index; + descriptor.layouts[index].stride = bytes; + descriptor.layouts[index].stepFunction = MTLVertexStepFunctionPerVertex; + } + return descriptor; +} + // One library: every function it names, in its order; returns the log line. static std::string library(id device, const std::string &path, const std::string ¤t, const std::set &bad, - const std::string &functionsPath, const std::set &functionsDone, const std::string &build) { + const std::string &functionsPath, const std::set &functionsDone, + const std::string &refusedPath, const std::set &refusedDone, + const std::string &build, bool &complete) { + complete = false; unsigned kernels = 0, renders = 0, made = 0, refused = 0, skipped = 0, zeroed = 0, linkedMade = 0, halfMade = 0; + const unsigned earlierRefusals = native_prime_functions_refused; uint64_t waited = 0; std::string firstRefusal; // the first refused function and why, so the log tells translator gaps from unspecialized constants const uint64_t began = nowMs(); @@ -281,32 +365,31 @@ static std::string library(id device, const std::string &path, const native_prime_compiling = 0; if (!lib) return path + " library refused: " + (error ? error.localizedDescription.UTF8String : "no error"); NSArray *names = lib.functionNames; - for (NSString *name in names) { + NSArray *entries = [lib respondsToSelector:@selector(nb_primeFunctionNames)] ? [(NSObject *)lib nb_primeFunctionNames] : names; + for (NSString *name in entries) { @autoreleasepool { + if (stopping()) break; + native_prime_functions_refused = earlierRefusals + refused; + report(path, name.UTF8String); const std::string key = path + "\t" + name.UTF8String; const std::string doneKey = key + "\t" + build; native_prime_compiling = 3; - if (bad.count(key)) {++skipped; continue;} - if (functionsDone.count(doneKey)) {native_prime_functions_resumed = native_prime_functions_resumed + 1; continue;} - id function = [lib newFunctionWithName:name]; - if (!function) {++refused; continue;} + if (bad.count(doneKey)) {++skipped; continue;} + const std::string attemptKey = doneKey + "-inputs3"; + if (refusedDone.count(attemptKey)) {++refused; continue;} + if (functionsDone.count(doneKey)) {++made; native_prime_functions_resumed = native_prime_functions_resumed + 1; continue;} + id function = defaultFunction(lib, name, zeroed); + if (!function) {++refused; native_prime_functions_refused = earlierRefusals + refused; appendLine(refusedPath, attemptKey); report(path, name.UTF8String); continue;} const MTLFunctionType type = function.functionType; if (type != MTLFunctionTypeKernel && type != MTLFunctionTypeVertex) continue; waited += waitForQuiet(); - writeCurrent(current, key); + writeCurrent(current, doneKey); native_prime_compiling = 1; bool ok = false; @try { if (type == MTLFunctionTypeKernel) { ++kernels; ok = [device newComputePipelineStateWithFunction:function error:&error] != nil; - // A kernel that uses a function constant's value needs one: prime the variant with every constant zero, which - // still takes the module through translation and the driver (int-245: 2901 kernels refused for want of one). - if (!ok && [error.localizedDescription containsString:@"FunctionConstant"] && - [lib respondsToSelector:@selector(nb_newFunctionWithZeroConstants:)]) { - id zero = [(NSObject *)lib nb_newFunctionWithZeroConstants:name]; - if (zero) {error = nil; ok = [device newComputePipelineStateWithFunction:zero error:&error] != nil; if (ok) ++zeroed;} - } if (!ok && [error.localizedDescription containsString:@"LinkedFunction"]) { ok = linkedKernel(device, lib, function, &error); if (ok) ++linkedMade; @@ -315,6 +398,7 @@ static std::string library(id device, const std::string &path, const ++renders; MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; d.vertexFunction = function; + d.vertexDescriptor = vertexInputs(function); d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; // The fragment function of the same stem under the libraries' usual spellings; without one the pipeline still // rasterizes (Metal refuses an unrasterized pipeline whose vertex function returns a position, int-245: 732 refusals). @@ -323,24 +407,30 @@ static std::string library(id device, const std::string &path, const for (const auto &pair : kPairs) { NSString *fragment = [name stringByReplacingOccurrencesOfString:pair[0] withString:pair[1]]; if (![fragment isEqualToString:name] && [names containsObject:fragment]) { - d.fragmentFunction = [lib newFunctionWithName:fragment]; + d.fragmentFunction = defaultFunction(lib, fragment, zeroed); if (d.fragmentFunction) break; } } ok = [device newRenderPipelineStateWithDescriptor:d error:&error] != nil; + if (!ok && [error.localizedDescription containsString:@"RasterizedVoidVertex"]) { + d.rasterizationEnabled = NO; + error = nil; + ok = [device newRenderPipelineStateWithDescriptor:d error:&error] != nil; + } // The render server's framebuffer is half-float (RGhA) on this board: its pipelines draw into RGBA16Float, a state // of its own to Turnip, so the same pair is made for it too. if (ok) { d.colorAttachments[0].pixelFormat = MTLPixelFormatRGBA16Float; NSError *half = nil; if ([device newRenderPipelineStateWithDescriptor:d error:&half]) ++halfMade; + else {ok = false; error = half;} } } } @catch (NSException *exception) { ok = false; } native_prime_compiling = 0; - appendLine(functionsPath, doneKey); + appendLine(ok ? functionsPath : refusedPath, ok ? doneKey : attemptKey); ok ? ++made : ++refused; if (ok) native_prime_made = native_prime_made + 1; if (!ok && firstRefusal.empty()) { @@ -349,6 +439,8 @@ static std::string library(id device, const std::string &path, const if (firstRefusal.size() > 220) firstRefusal.resize(220); for (char &c : firstRefusal) if (c == '\n' || c == '\t') c = ' '; } + native_prime_functions_refused = earlierRefusals + refused; + report(path, name.UTF8String); error = nil; // Low priority already; a short pause between pipelines keeps the render server's own work ahead. usleep(20000); @@ -356,6 +448,9 @@ static std::string library(id device, const std::string &path, const } } unlink(current.c_str()); + native_prime_functions_refused = earlierRefusals + refused; + native_prime_functions_skipped = native_prime_functions_skipped + skipped; + complete = refused == 0 && skipped == 0; char line[200]; snprintf(line, sizeof line, " kernels=%u renders=%u made=%u half=%u zeroed=%u linked=%u refused=%u skipped=%u ms=%llu waited=%llu", kernels, renders, made, halfMade, zeroed, linkedMade, refused, skipped, (unsigned long long)(nowMs() - began), (unsigned long long)waited); @@ -372,53 +467,95 @@ static void run(id device, std::string root, unsigned delaySeconds, c while (root.size() > 1 && root.back() == '/') root.pop_back(); const std::string donePath = root + "/prime-done", badPath = root + "/prime-bad", current = root + "/prime-current", logPath = root + "/prime.log"; - // A function was being compiled when the process ended: it is not tried again. - for (const std::string &line : readLines(current)) appendLine(badPath, line); +#if !TARGET_OS_OSX + const std::string resume = root + "/prime-resume"; + const bool planned = access(resume.c_str(), F_OK) == 0; + if (planned) unlink(resume.c_str()); + else for (const std::string &line : readLines(current)) appendLine(badPath, line); +#endif unlink(current.c_str()); sleep(delaySeconds); +#if TARGET_OS_OSX + const std::set bad; +#else const std::set bad = readLines(badPath); +#endif std::set done = readLines(donePath); - const std::string functionsPath = root + "/prime-fn"; + const std::string functionsPath = root + "/prime-fn2", refusedPath = root + "/prime-refused2"; const std::set functionsDone = readLines(functionsPath); +#if TARGET_OS_OSX + const std::set refusedDone; +#else + const std::set refusedDone = readLines(refusedPath); +#endif const uint64_t began = nowMs(); unsigned libs = 0, finished = 0, foreign = 0; std::string note; const std::vector paths = libraries(listPath, note); - const std::string build = buildId(); - appendLine(logPath, note + " build=" + build); - native_prime_libraries_total = (uint32_t)paths.size(); + const std::string build = buildId() + "-prime2"; + const std::string boot = bootSession(); #if TARGET_OS_OSX - native_prime_scope = wholeSystem ? 2 : 1; + const unsigned scope = wholeSystem ? 2 : 1; #else - native_prime_scope = 2; // iPadOS primes every library of the list, whatever its bundle + const unsigned scope = 2; #endif + appendLine(logPath, note + " inputs=3 boot=" + boot + " scope=" + std::to_string(scope) + " build=" + build); + std::set binaryArchives; +#if TARGET_OS_OSX + if (wholeSystem) for (const std::string &path : paths) if (appleGpuArchive(path)) { + binaryArchives.insert(path); + appendLine(logPath, "archive " + path + " cpu=0x01000013"); + } +#endif + auto selected = [&](const std::string &path) { +#if TARGET_OS_OSX + return wholeSystem ? !libraryOfDriverBundle(path) && !binaryArchives.count(path) : libraryOfLoadedBundle(path); +#else + (void)path; + (void)wholeSystem; + return true; +#endif + }; + unsigned eligible = 0; + for (const std::string &path : paths) if (selected(path)) ++eligible; + native_prime_libraries_total = eligible; + native_prime_scope = scope; + report(); for (const std::string &path : paths) { + if (stopping()) break; ++libs; native_prime_libraries_done = finished; + report(path); struct stat st = {}; - if (stat(path.c_str(), &st) != 0) continue; - // The runtime's and Turnip's builds end the id: a newer one primes every library again (buildId()). - const std::string id = path + "\t" + std::to_string((long long)st.st_size) + "\t" + std::to_string((long long)st.st_mtimespec.tv_sec) + "\t" + build; - if (done.count(id)) {++finished; continue;} -#if TARGET_OS_OSX - // Counted as done for the telemetry's progress, and not listed as finished (libraryOfDriverBundle, libraryOfLoadedBundle). - if (wholeSystem ? libraryOfDriverBundle(path) : !libraryOfLoadedBundle(path)) { - ++foreign; ++finished; + if (stat(path.c_str(), &st) != 0) {native_prime_libraries_failed = native_prime_libraries_failed + 1; continue;} + if (!selected(path)) { + ++foreign; native_prime_libraries_skipped = foreign; continue; } -#endif - appendLine(logPath, library(device, path, current, bad, functionsPath, functionsDone, build)); + // The runtime's and Turnip's builds end the id: a newer one primes every library again (buildId()). + const std::string id = path + "\t" + std::to_string((long long)st.st_size) + "\t" + std::to_string((long long)st.st_mtimespec.tv_sec) + "\t" + build; + if (done.count(id)) {++finished; continue;} + bool complete = false; + appendLine(logPath, library(device, path, current, bad, functionsPath, functionsDone, refusedPath, refusedDone, build, complete)); + if (stopping()) break; native_prime_libraries_primed = native_prime_libraries_primed + 1; - appendLine(donePath, id); - done.insert(id); - ++finished; + if (complete) { + appendLine(donePath, id); + done.insert(id); + ++finished; + } else { + native_prime_libraries_failed = native_prime_libraries_failed + 1; + } } native_prime_libraries_done = finished; native_prime_compiling = 0; - char line[120]; - snprintf(line, sizeof line, "pass libraries=%u finished=%u %s=%u ms=%llu", libs, finished, wholeSystem ? "driver-bundles" : "not-loaded", - foreign, (unsigned long long)(nowMs() - began)); + native_prime_pass_complete = eligible && finished == eligible && !native_prime_libraries_failed && !native_prime_inventory_failed ? 1 : 2; + report(); + char line[512]; + snprintf(line, sizeof line, "pass boot=%s scope=%u build=%s inputs=3 libraries=%u eligible=%u finished=%u failed=%u refused=%u bad=%u missing=%u excluded=%u binary=%zu complete=%u ms=%llu", + boot.c_str(), scope, build.c_str(), libs, eligible, finished, native_prime_libraries_failed, native_prime_functions_refused, native_prime_functions_skipped, + native_prime_inventory_failed, foreign, binaryArchives.size(), native_prime_pass_complete, (unsigned long long)(nowMs() - began)); appendLine(logPath, line); } } // namespace prime diff --git a/ShaderCache/ShaderCacheApp.mm b/ShaderCache/ShaderCacheApp.mm new file mode 100644 index 0000000..f6daad9 --- /dev/null +++ b/ShaderCache/ShaderCacheApp.mm @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: MIT +#import +#import +#import +#include "ShaderCacheState.h" + +static NSString *text(NSString *english, NSString *chinese) { + return [NSLocale.preferredLanguages.firstObject hasPrefix:@"zh"] ? chinese : english; +} + +@interface ShaderCacheApp : NSObject +@property NSWindow *window; +@property NSTextField *state; +@property NSTextField *counts; +@property NSTextField *current; +@property NSProgressIndicator *progress; +@property NSButton *start; +@property NSButton *stop; +@property NSTimer *timer; +@property BOOL authorizing; +@property AuthorizationRef authorization; +@end + +@implementation ShaderCacheApp +- (void)applicationDidFinishLaunching:(NSNotification *)notification { + (void)notification; + NSMenu *menu = [NSMenu new]; + NSMenuItem *application = [NSMenuItem new]; + [menu addItem:application]; + application.submenu = [NSMenu new]; + [application.submenu addItemWithTitle:text(@"Quit Shader Cache", @"退出着色器缓存") action:@selector(terminate:) keyEquivalent:@"q"]; + NSApp.mainMenu = menu; + self.window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0,0,420,228) + styleMask:NSWindowStyleMaskTitled|NSWindowStyleMaskClosable|NSWindowStyleMaskMiniaturizable backing:NSBackingStoreBuffered defer:NO]; + self.window.title = text(@"Shader Cache", @"着色器缓存"); + self.window.releasedWhenClosed = NO; + self.state = [NSTextField labelWithString:@""]; + self.state.font = [NSFont systemFontOfSize:15 weight:NSFontWeightMedium]; + self.progress = [NSProgressIndicator new]; + self.progress.indeterminate = NO; + self.progress.minValue = 0; + self.progress.maxValue = 1; + self.counts = [NSTextField labelWithString:@""]; + self.counts.font = [NSFont monospacedDigitSystemFontOfSize:13 weight:NSFontWeightRegular]; + self.counts.maximumNumberOfLines = 2; + self.current = [NSTextField labelWithString:@""]; + self.current.lineBreakMode = NSLineBreakByTruncatingMiddle; + self.current.maximumNumberOfLines = 1; + self.current.font = [NSFont systemFontOfSize:11]; + self.current.textColor = NSColor.secondaryLabelColor; + self.start = [NSButton buttonWithTitle:text(@"Start", @"开始") target:self action:@selector(startCompilation:)]; + self.start.keyEquivalent = @"\r"; + self.stop = [NSButton buttonWithTitle:text(@"Stop", @"停止") target:self action:@selector(stopCompilation:)]; + NSStackView *buttons = [NSStackView stackViewWithViews:@[self.start,self.stop]]; + buttons.orientation = NSUserInterfaceLayoutOrientationHorizontal; + buttons.spacing = 8; + NSStackView *stack = [NSStackView stackViewWithViews:@[self.state,self.progress,self.counts,self.current,buttons]]; + stack.orientation = NSUserInterfaceLayoutOrientationVertical; + stack.alignment = NSLayoutAttributeLeading; + stack.spacing = 16; + stack.translatesAutoresizingMaskIntoConstraints = NO; + [self.window.contentView addSubview:stack]; + [NSLayoutConstraint activateConstraints:@[ + [stack.leadingAnchor constraintEqualToAnchor:self.window.contentView.leadingAnchor constant:24], + [stack.trailingAnchor constraintEqualToAnchor:self.window.contentView.trailingAnchor constant:-24], + [stack.topAnchor constraintEqualToAnchor:self.window.contentView.topAnchor constant:24], + [self.progress.widthAnchor constraintEqualToAnchor:stack.widthAnchor], + [self.current.widthAnchor constraintEqualToAnchor:stack.widthAnchor] + ]]; + self.timer = [NSTimer scheduledTimerWithTimeInterval:1 target:self selector:@selector(refresh) userInfo:nil repeats:YES]; + [self refresh]; + [self.window center]; + [self.window makeKeyAndOrderFront:nil]; + [NSApp activateIgnoringOtherApps:YES]; +} +- (void)refresh { + NSDictionary *value = SCReadState(); + NSDictionary *labels = @{ + @"idle":text(@"Not started", @"未开始"), @"running":text(@"Compiling", @"正在编译"), + @"stopping":text(@"Stopping", @"正在停止"), @"stopped":text(@"Stopped", @"已停止"), + @"interrupted":text(@"Interrupted", @"已中断"), @"complete":text(@"Complete", @"已完成"), + @"incomplete":text(@"Incomplete", @"未完成"), @"failed":text(@"Failed", @"失败"), + @"outdated":text(@"Update available", @"缓存待更新") + }; + self.state.stringValue = self.authorizing ? text(@"Authorizing", @"正在授权") : labels[value[@"state"]] ?: text(@"Not started", @"未开始"); + double total = [value[@"total"] doubleValue]; + [NSAnimationContext runAnimationGroup:^(NSAnimationContext *context) { + context.duration = 0.2; + [[self.progress animator] setDoubleValue:total ? [value[@"libraries"] doubleValue] / total : 0]; + } completionHandler:nil]; + self.counts.stringValue = [NSString stringWithFormat:text(@"Libraries %@/%@ Functions %@\nFailed libraries %@ Functions %@ Missing %@", @"库 %@/%@ 函数 %@\n失败库 %@ 失败函数 %@ 缺失 %@"), + value[@"libraries"] ?: @0,value[@"total"] ?: @0,value[@"functions"] ?: @0,value[@"failedLibraries"] ?: @0,value[@"failedFunctions"] ?: @0,value[@"missing"] ?: @0]; + self.current.stringValue = [value[@"error"] length] ? value[@"error"] : [value[@"function"] length] ? value[@"function"] : value[@"library"] ?: @""; + self.current.toolTip = self.current.stringValue; + self.start.enabled = ![value[@"running"] boolValue] && !self.authorizing; + self.stop.enabled = [value[@"running"] boolValue] && !self.authorizing && ![value[@"state"] isEqual:@"stopping"]; +} +- (void)perform:(const char *)operation { + if (self.authorizing) return; + self.authorizing = YES; + [self refresh]; + NSString *worker = [NSBundle.mainBundle.bundlePath stringByAppendingPathComponent:@"Contents/MacOS/ShaderCacheWorker"]; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + OSStatus status = errAuthorizationSuccess; + AuthorizationRef authorization = self.authorization; + if (!authorization) status = AuthorizationCreate(nullptr, kAuthorizationEmptyEnvironment, kAuthorizationFlagDefaults, &authorization); + if (status == errAuthorizationSuccess) { + AuthorizationItem item = {kAuthorizationRightExecute,0,nullptr,0}; + AuthorizationRights rights = {1,&item}; + status = AuthorizationCopyRights(authorization,&rights,kAuthorizationEmptyEnvironment, + kAuthorizationFlagInteractionAllowed|kAuthorizationFlagExtendRights|kAuthorizationFlagPreAuthorize,nullptr); + } + if (status == errAuthorizationSuccess) { + char *arguments[] = {const_cast(operation),nullptr}; + FILE *channel = nullptr; + status = AuthorizationExecuteWithPrivileges(authorization,worker.fileSystemRepresentation,kAuthorizationFlagDefaults,arguments,&channel); + if (channel) fclose(channel); + } + dispatch_async(dispatch_get_main_queue(), ^{ + self.authorization = authorization; + self.authorizing = NO; + [self refresh]; + if (status != errAuthorizationSuccess && status != errAuthorizationCanceled) { + NSAlert *alert = [NSAlert new]; + alert.messageText = text(@"Could not start the worker", @"无法启动工作进程"); + alert.informativeText = [NSString stringWithFormat:@"%d",(int)status]; + [alert beginSheetModalForWindow:self.window completionHandler:nil]; + } + }); + }); +} +- (void)startCompilation:(id)sender {(void)sender; [self perform:"--run"];} +- (void)stopCompilation:(id)sender {(void)sender; [self perform:"--stop"];} +- (BOOL)applicationShouldTerminateAfterLastWindowClosed:(NSApplication *)sender {(void)sender; return YES;} +@end + +int main(int argc, const char **argv) { + @autoreleasepool { + if (argc == 2 && !strcmp(argv[1], "--status")) { + NSMutableDictionary *value = SCReadState().mutableCopy; + [value removeObjectForKey:@"updated"]; + NSData *json = [NSJSONSerialization dataWithJSONObject:value options:NSJSONWritingPrettyPrinted error:nil]; + fwrite(json.bytes,1,json.length,stdout); + putchar('\n'); + return 0; + } + NSApplication *application = NSApplication.sharedApplication; + ShaderCacheApp *delegate = [ShaderCacheApp new]; + application.delegate = delegate; + [application setActivationPolicy:NSApplicationActivationPolicyRegular]; + [application run]; + } + return 0; +} diff --git a/ShaderCache/ShaderCacheState.h b/ShaderCache/ShaderCacheState.h new file mode 100644 index 0000000..29598bd --- /dev/null +++ b/ShaderCache/ShaderCacheState.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT +#pragma once +#import +#include +#include +#include + +static NSString *const SCStorage = @"/private/var/tmp/NativeAdrenoMetal.cache"; +static NSString *const SCPlugin = @"/Library/GPUBundles/NativeAdrenoMetal.bundle"; +static NSString *const SCApplication = @"/Library/UniXNU/Shader Cache.app"; + +static NSString *SCPath(NSString *name) {return [SCStorage stringByAppendingPathComponent:name];} +static NSString *SCInstalledBuild() { + NSData *data = [NSData dataWithContentsOfFile:[SCPlugin stringByAppendingPathComponent:@"Contents/Resources/gpu-build.json"]]; + if (!data) return @""; + NSDictionary *build = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [build[@"metal_source_sha256"] isKindOfClass:NSString.class] ? build[@"metal_source_sha256"] : @""; +} + +static BOOL SCWorkerRunning() { + int fd = open(SCPath(@"shader-cache.lock").fileSystemRepresentation, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd < 0) return NO; + const BOOL locked = flock(fd, LOCK_SH | LOCK_NB) != 0 && errno == EWOULDBLOCK; + close(fd); + return locked; +} + +static NSDictionary *SCReadState() { + NSDictionary *saved = [NSDictionary dictionaryWithContentsOfFile:SCPath(@"shader-cache.plist")]; + NSMutableDictionary *state = saved ? saved.mutableCopy : [NSMutableDictionary dictionary]; + const BOOL running = SCWorkerRunning(); + state[@"running"] = @(running); + if (running) { + state[@"state"] = access(SCPath(@"shader-cache.stop").fileSystemRepresentation, F_OK) == 0 ? @"stopping" : @"running"; + } else if ([state[@"state"] isEqual:@"running"] || [state[@"state"] isEqual:@"stopping"]) { + state[@"state"] = @"interrupted"; + } else if (!state[@"state"]) state[@"state"] = @"idle"; + if (!running && saved && ![saved[@"build"] isEqual:SCInstalledBuild()]) state[@"state"] = @"outdated"; + return state; +} diff --git a/ShaderCache/ShaderCacheWorker.mm b/ShaderCache/ShaderCacheWorker.mm new file mode 100644 index 0000000..8b88dfc --- /dev/null +++ b/ShaderCache/ShaderCacheWorker.mm @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT +#import +#import +#include "ShaderCacheState.h" +#include "../SharedShaderCache.h" +#include "../PrimeShaders.h" +#include +#include + +static void *pluginHandle; +static volatile sig_atomic_t interrupted; +static void interruptWorker(int) {interrupted = 1;} + +extern "C" const char *nmb_runtime_build() { + auto build = reinterpret_cast(dlsym(pluginHandle, "nmb_runtime_build")); + return build ? build() : "unknown"; +} +extern "C" void native_bridge_metal_quiet_problems(bool quiet) { + auto set = reinterpret_cast(dlsym(pluginHandle, "native_bridge_metal_quiet_problems")); + if (set) set(quiet); +} + +static void saveState(NSString *state, const std::string &library = {}, const std::string &function = {}, NSString *error = @"") { + NSDictionary *value = @{ + @"state":state, @"build":SCInstalledBuild(), @"updated":NSDate.date, + @"libraries":@(native_prime_libraries_done), @"total":@(native_prime_libraries_total), + @"functions":@(native_prime_made + native_prime_functions_resumed), + @"failedLibraries":@(native_prime_libraries_failed), @"failedFunctions":@(native_prime_functions_refused), + @"missing":@(native_prime_inventory_failed), @"library":@(library.c_str()), @"function":@(function.c_str()), @"error":error + }; + [value writeToFile:SCPath(@"shader-cache.plist") atomically:YES]; +} + +int main(int argc, const char **argv) { + @autoreleasepool { + if (argc != 2 || (strcmp(argv[1], "--run") && strcmp(argv[1], "--stop"))) return 64; + if (geteuid() != 0 || setuid(0) != 0 || systemcache::writerDirectory().empty()) return 77; + umask(0022); + if (!strcmp(argv[1], "--stop")) { + if (!SCWorkerRunning()) return 0; + return [NSData.data writeToFile:SCPath(@"shader-cache.stop") atomically:YES] ? 0 : 74; + } + int lock = open(SCPath(@"shader-cache.lock").fileSystemRepresentation, O_CREAT | O_RDWR | O_NOFOLLOW | O_CLOEXEC, 0644); + if (lock < 0 || flock(lock, LOCK_EX | LOCK_NB)) {if (lock >= 0) close(lock); return 75;} + freopen("/dev/null", "w", stdout); + freopen(SCPath(@"shader-cache-worker.log").fileSystemRepresentation, "a", stderr); + unlink(SCPath(@"shader-cache.stop").fileSystemRepresentation); + signal(SIGTERM, interruptWorker); + signal(SIGINT, interruptWorker); + saveState(@"running"); + setpriority(PRIO_PROCESS, 0, 10); + setenv("NATIVE_METAL_PRIMER", "1", 1); + id device = MTLCreateSystemDefaultDevice(); + NSString *executable = [SCPlugin stringByAppendingPathComponent:@"Contents/MacOS/NativeAdrenoMetal"]; + pluginHandle = dlopen(executable.fileSystemRepresentation, RTLD_NOW | RTLD_LOCAL); + if (!device || !pluginHandle || !dlsym(pluginHandle, "nmb_runtime_build") || ![NSStringFromClass([device class]) isEqual:@"NativeBridgeMetalDevice"]) { + saveState(@"failed", {}, {}, @"AdrenoMetal is unavailable"); + return 69; + } + prime::cancelled = [] {return interrupted || access(SCPath(@"shader-cache.stop").fileSystemRepresentation, F_OK) == 0;}; + prime::progress = [](const std::string &library, const std::string &function) { + saveState(prime::stopping() ? @"stopping" : @"running", library, function); + }; + prime::run(device, SCStorage.fileSystemRepresentation, 0, + "/Library/GPUBundles/NativeAdrenoMetal.primelist", true); + saveState(prime::stopping() ? @"stopped" : native_prime_pass_complete == 1 ? @"complete" : @"incomplete"); + unlink(SCPath(@"shader-cache.stop").fileSystemRepresentation); + close(lock); + return native_prime_pass_complete == 1 ? 0 : prime::stopping() ? 130 : 1; + } +} diff --git a/bridge/NativeMetalBridge.rs b/bridge/NativeMetalBridge.rs index eeea0b4..5465526 100644 --- a/bridge/NativeMetalBridge.rs +++ b/bridge/NativeMetalBridge.rs @@ -238,7 +238,8 @@ pub fn air_slice(bytes: &[u8]) -> Option<&[u8]> { return Some(bytes); } let count = word(4)? as usize; - for i in 0..count.min(16) { + if count > (bytes.len() - 8) / 20 { return None; } + for i in 0..count { let at = 8 + 20 * i; if word(at)? == AIR64 { let (offset, size) = (word(at + 8)? as usize, word(at + 12)? as usize); diff --git a/bridge/NativeMetalObjects.h b/bridge/NativeMetalObjects.h index aedfc97..902846e 100644 --- a/bridge/NativeMetalObjects.h +++ b/bridge/NativeMetalObjects.h @@ -41,9 +41,9 @@ size_t nmb_device_name(const NmbDevice *, char *out, size_t size); uint64_t nmb_device_max_buffer_length(const NmbDevice *); // Pipeline makings (count, total us, longest us, waited us, failed), sink messages (info, warning+), shader cache hits/misses/writes. bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]); -/* The shader cache's full counts in 24 words: hits, misses, function hits and misses, compute pipeline hits and misses, render pipeline hits and misses, +/* The shader cache's full counts in 24 words: hits, misses, function hits and misses, driver data found and unchanged, render pipeline hits and misses, * rejected, read failures, writes, writes queued, writes on a calling thread, bytes written, queued hits, write failures, evictions, driver cache loads, - * driver pipeline hits, driver pipeline misses, driver saves, then three spare zeros. */ + * driver pipeline hits, driver pipeline misses, driver saves, truncated driver data, mismatched driver data, then one spare zero. */ bool nmb_device_shader_cache_detail(const NmbDevice *, uint64_t out[24]); /* The read-only system cache layer of the device opened after this call: the directory at root, read only where it, every directory * on the way to an entry and the entry are owned by one of the count user ids and not writable by group or others, never through a @@ -67,13 +67,16 @@ bool nmb_device_texture_counts(const NmbDevice *, uint64_t out[8]); // Records warmed, failed and unread, driver bytes the warm pass added; scheduler passes and batches submitted on the committing thread; // render log waits, and encoder ends the lowering thread lowered. bool nmb_device_flow_counts(const NmbDevice *, uint64_t out[8]); +// Grouped submissions/batches, command pools made/reused, linear passes in place/through copies, region pipelines evicted; +// commit calls and microseconds in total/lowering/recording/pipeline wait/lock wait/scheduler pass/submission. +bool nmb_device_fast_path_counts(const NmbDevice *, uint64_t out[15]); // Passes begun as instances, split inline; instance splits for a barrier, tile memory, a memoryless write, a pending pipeline; stand-ins // made and their bytes; stand-in copies made and those made at draws. bool nmb_device_pass_counts(const NmbDevice *, uint64_t out[10]); // Bytes of pooled blocks live and placed in them, empty kept, dedicated, host imports; bytes allocated so far; pooled and dedicated // blocks live; GPU pool blocks made and released. bool nmb_device_memory_counts(const NmbDevice *, uint64_t out[10]); -// Live blocks of the runtime's own device memory (bridge patch 0031): seven size classes each (at most 16, 32, 40, 48, 56, 64 MiB, +// Live blocks of the runtime's own device memory: seven size classes each (at most 16, 32, 40, 48, 56, 64 MiB, // larger) of pooled blocks with a placement [0], pooled blocks kept empty [7], spare image blocks [14], dedicated blocks of an image // [21] and of a buffer [28]; bytes of the spare blocks [35] and of the dedicated blocks of images [36] and buffers [37]; lazily // allocated blocks with a placement [38] and kept without one [42], each as blocks, bytes, blocks the driver committed, bytes it @@ -200,6 +203,8 @@ NmbTexture *nmb_texture_new_view_swizzle(NmbTexture *, uint64_t format, uint64_t uint32_t nmb_function_constant_count(const NmbFunction *); size_t nmb_function_constant_name_size(const NmbFunction *, uint32_t at); bool nmb_function_constant_at(const NmbFunction *, uint32_t at, char *name, size_t capacity, uint64_t *index, uint64_t *data_type, bool *required); +int32_t nmb_function_attribute_count(const NmbFunction *); +size_t nmb_function_attribute_at(const NmbFunction *, uint32_t at, char *name, size_t capacity, uint64_t facts[5]); bool nmb_render_set_viewports(NmbEncoder *, const double *values, size_t count); bool nmb_render_set_scissors(NmbEncoder *, const uint64_t *values, size_t count); void nmb_texture_release(NmbTexture *); diff --git a/bridge/NativeMetalObjects.rs b/bridge/NativeMetalObjects.rs index ba9c59b..4d179c5 100644 --- a/bridge/NativeMetalObjects.rs +++ b/bridge/NativeMetalObjects.rs @@ -239,13 +239,14 @@ impl PipelineRecords { } } -/// The work placement the bridge asks of the runtime, as bits: 1 render encoder calls lowered on the device's lowering thread, -/// 2 a command buffer submitted at its commit (`commit_and_submit`), 4 the warm pass over the pipeline records of earlier -/// processes. All three by default; `NATIVE_METAL_FLOW=` (the boot argument `metal-flow=`) states the bits, 0 none. +/// Work placement: bit 1 lowers on the device's worker, bit 2 submits at commit. +/// iPadOS also supports bit 4 for warming earlier pipeline records. Defaults are +/// 3 on macOS and 7 on iPadOS; `NATIVE_METAL_FLOW` selects the bits. fn flow_bits() -> u32 { static BITS: std::sync::OnceLock = std::sync::OnceLock::new(); *BITS.get_or_init(|| { - std::env::var("NATIVE_METAL_FLOW").ok().and_then(|value| value.trim().parse::().ok()).unwrap_or(7) + let bits = std::env::var("NATIVE_METAL_FLOW").ok().and_then(|value| value.trim().parse::().ok()).unwrap_or(7); + if cfg!(target_os = "macos") { bits & 3 } else { bits } }) } @@ -263,13 +264,8 @@ pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_ options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); options.messages = Some(MessageSink::new(observe_message)); - // A texture over host pages or a buffer whose rows lie apart is an image over the bytes, rendered or written too: the window - // server's surfaces and padded framebuffers (3048x2032 BGRA8 at 12288-byte rows, int-251) were otherwise copied in and out of a - // shadow image every pass. + // Render imported surfaces and padded buffers directly under the board policy. options.policy.host_page_textures = metal2vulkan::device::HostPageTexturePolicy::InPlaceWrites; - // Metal makes a pipeline state's pipeline at its creation and never drops a draw for a pipeline being made; the runtime's - // default (PipelineWaitPolicy::Bounded) drops the draws and dispatches of a pipeline not made 4.5 s after the commit, which the - // render server showed as missing pieces of the screen while it compiled (int 590aaee2). options.policy.pipeline_waits = metal2vulkan::device::PipelineWaitPolicy::Exact; // Render encoder calls are checked at the call and lowered on the device's lowering thread, so the program's thread does not // pay for them (the render server's main thread spent most of the plugin's CPU time lowering). A refusal only the lowering @@ -284,7 +280,9 @@ pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_ // Every texture given a shadow image is announced with why no image aliases its bytes (`shadow.log`). options.announce_each.insert(metal2vulkan_runtime::Degradation::ShadowedTexture); let _ = SHADOW_LOG.set(PathBuf::from(cache.to_string_lossy().into_owned()).join("shadow.log")); + open_runtime_diagnostics(cache.to_string_lossy().as_ref()); let device = Device::system_default_with(&options).map_err(e("device"))?; + write_runtime_policy(&device); // No caller moves or detaches the pages of a no-copy buffer (the frontend has no rebind or detach): the encoders log no calls // to encode again and hold no surface's buffer per command, which every frame of the render server paid for. device.refuse_rebinds(); @@ -336,6 +334,53 @@ fn note_shadow(text: &str) { // NATIVE_METAL_WARNINGS=1: each kind of message (its id and the text's first 48 characters) the first time it comes, in full and // with its severity, to standard error, for the first 400 kinds. The telemetry keeps only the last warning, and a degradation that // draws nothing is otherwise silent. +static COMMIT_TIMINGS: [std::sync::atomic::AtomicU64; 2] = [const { std::sync::atomic::AtomicU64::new(0) }; 2]; +const TIMING_LOG_LIMIT: usize = 512 * 1024; +struct TimingLog { file: std::fs::File, bytes: usize } +static TIMING_LOG: std::sync::OnceLock> = std::sync::OnceLock::new(); +static POLICY_LOG: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn open_runtime_diagnostics(cache: &str) { + if std::env::var_os("SHENG_DIAG").is_none() { return; } + let root = PathBuf::from(cache); + let _ = POLICY_LOG.set(root.join("runtime-policy.log")); + if let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(root.join("timing.log")) { + let bytes = file.metadata().map_or(0, |m| m.len() as usize); + let _ = TIMING_LOG.set(std::sync::Mutex::new(TimingLog { file, bytes })); + } +} + +fn diagnostic_stamp() -> String { + let boot = std::env::var("NATIVE_METAL_BOOT_SESSION").unwrap_or_else(|_| "unknown".into()); + let build = option_env!("NMB_COMPONENT_ID").unwrap_or("host-working-tree"); + let uptime = metal2vulkan_runtime::stage_trace::now_nanos() / 1000; + let time = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_micros()); + format!("boot={boot} build={build} runtime={} uptime_us={uptime} time_us={time}", env!("NMB_RUNTIME_ID")) +} + +fn write_runtime_policy(device: &Device) { + let Some(path) = POLICY_LOG.get() else { return; }; + let profile = device.compile_device(); + let text = format!("{}\npolicy={:#?}\nsubmit={}\nqueue-switch={}\nin-place={:#?}\nprofile={:#?}\n", + diagnostic_stamp(), profile.policy(), device.submit_check_report(), + device.queue_switch_check_report(), device.in_place_check_report(), profile.profile()); + let text: String = text.chars().take(64 * 1024).collect(); + let _ = std::fs::write(path, text); +} + +fn note_encoder_timing(text: &str) { + use std::io::Write; + let Some(log) = TIMING_LOG.get() else { return; }; + let Ok(mut log) = log.lock() else { return; }; + let text: String = text.chars().take(4096).collect(); + let line = format!("{} {text}\n", diagnostic_stamp()); + if log.bytes + line.len() > TIMING_LOG_LIMIT { + if log.file.set_len(0).is_err() { return; } + log.bytes = 0; + } + if log.file.write_all(line.as_bytes()).is_ok() { log.bytes += line.len(); } +} + fn log_message(message: &DebugMessage) { static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); static SEEN: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); @@ -353,6 +398,7 @@ fn log_message(message: &DebugMessage) { fn observe_message(message: &DebugMessage) { use std::sync::atomic::Ordering::Relaxed; + if message.id_name == "metal2vulkan-encoder-times" { note_encoder_timing(&message.text); } if message.id_name == "shadowed-texture" { note_shadow(&message.text); } @@ -426,8 +472,8 @@ pub extern "C" fn nmb_device_shader_cache_detail(device: *const NmbDevice, out: c.misses, c.function_hits, c.function_misses, - c.compute_pipeline_hits, - c.compute_pipeline_misses, + c.driver_data_found, + c.driver_data_unchanged, c.render_pipeline_hits, c.render_pipeline_misses, c.rejected, @@ -443,6 +489,8 @@ pub extern "C" fn nmb_device_shader_cache_detail(device: *const NmbDevice, out: c.driver_pipeline_hits, c.driver_pipeline_misses, c.driver_saves, + c.driver_data_truncated, + c.driver_data_mismatched, ]; out.fill(0); out[..words.len()].copy_from_slice(&words); @@ -487,6 +535,31 @@ pub extern "C" fn nmb_device_flow_counts(device: *const NmbDevice, out: *mut u64 }) } +/// Submission grouping, command-pool reuse, in-place passes and commit costs, into the header's fifteen words. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_fast_path_counts(device: *const NmbDevice, out: *mut u64) -> bool { + guard(false, || { + if out.is_null() { + return Err("null counts".into()); + } + // SAFETY: the header's fifteen writable words. + let out = unsafe { std::slice::from_raw_parts_mut(out, 15) }; + let device = &get(device)?.0; + static SNAPSHOTS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + if POLICY_LOG.get().is_some() && SNAPSHOTS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % 32 == 0 { + write_runtime_policy(device); + } + let w = device.work_counts(); + out.fill(u64::MAX); + out[..6].copy_from_slice(&[w.grouped_submissions, w.batches_grouped, w.command_pools_made, + w.spare_command_pools_taken, w.linear_passes_in_place, w.linear_passes_through_copies]); + out[6] = 0; + out[7] = COMMIT_TIMINGS[0].load(std::sync::atomic::Ordering::Relaxed); + out[8] = COMMIT_TIMINGS[1].load(std::sync::atomic::Ordering::Relaxed) / 1000; + Ok(true) + }) +} + /// Where render passes cost the GPU more than Metal's single pass in tile memory, into the caller's ten words: passes begun as /// instances, passes split inline, then instance splits for a barrier, tile memory, a memoryless write and a pending pipeline, stand-ins /// made and their bytes, and stand-in copies made and those made at draws. @@ -525,7 +598,7 @@ pub extern "C" fn nmb_device_memory_counts(device: *const NmbDevice, out: *mut u }) } -/// The runtime's live blocks of its own device memory by what holds them and by size (bridge patch 0031), into the caller's 48 words: +/// The runtime's live blocks of its own device memory by what holds them and by size, into the caller's 48 words: /// seven size classes each (at most 16, 32, 40, 48, 56 and 64 MiB, and larger) of the pooled blocks with a placement, the pooled /// blocks kept empty, the spare image blocks, the dedicated blocks of an image and those of a buffer; then the bytes of the spare /// blocks and of the dedicated blocks of images and of buffers; then, of the blocks of a lazily allocated memory type, those with a @@ -1917,6 +1990,28 @@ pub extern "C" fn nmb_function_type(function: *const NmbFunction) -> u64 { guard(0, || Ok(get(function)?.0.function_type().map_err(e("function type"))?.as_raw() as u64)) } +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_attribute_count(function: *const NmbFunction) -> i32 { + guard(-1, || Ok(get(function)?.0.stage_input_attributes().map_err(e("function attributes"))?.map_or(0, |a| a.len() as i32))) +} + +/// # Safety +/// `name` has `capacity` writable bytes and `facts` holds five writable words. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_attribute_at(function: *const NmbFunction, at: u32, name: *mut c_char, + capacity: usize, facts: *mut u64) -> usize { + guard(0, || { + let attributes = get(function)?.0.stage_input_attributes().map_err(e("function attributes"))?.unwrap_or_default(); + let a = attributes.get(at as usize).ok_or("function attribute index")?; + if facts.is_null() { return Err("null attribute facts".into()); } + let values = [a.attribute_index() as u64, a.attribute_type().as_raw() as u64, a.is_active() as u64, + a.is_patch_data() as u64, a.is_patch_control_point_data() as u64]; + // SAFETY: the caller supplies five writable words. + unsafe { std::ptr::copy_nonoverlapping(values.as_ptr(), facts, values.len()); } + Ok(copy_out(a.name(), name, capacity)) + }) +} + /// `patchType`: 0 (none) for a function other than a post-tessellation vertex function. #[unsafe(no_mangle)] pub extern "C" fn nmb_function_patch_type(function: *const NmbFunction) -> u64 { @@ -2735,7 +2830,8 @@ pub extern "C" fn nmb_command_buffer_wait_scheduled(command_buffer: *mut NmbComm #[unsafe(no_mangle)] pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffer) -> bool { - guard(false, || { + let began = TIMING_LOG.get().map(|_| std::time::Instant::now()); + let result = guard(false, || { // The committing thread submits where no other pass is under way, with no wake of the scheduler thread between commit and GPU. let command_buffer = &get(command_buffer)?.0; if flow_bits() & 2 != 0 { @@ -2744,7 +2840,16 @@ pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffe command_buffer.commit().map_err(e("commit"))?; } Ok(true) - }) + }); + if let Some(began) = began { + let nanos = began.elapsed().as_nanos().min(u64::MAX as u128) as u64; + COMMIT_TIMINGS[0].fetch_add(1, std::sync::atomic::Ordering::Relaxed); + COMMIT_TIMINGS[1].fetch_add(nanos, std::sync::atomic::Ordering::Relaxed); + if nanos >= 8_000_000 { + note_encoder_timing(&format!("command_buffer={command_buffer:p} commit_us={} accepted={result}", nanos / 1000)); + } + } + result } #[unsafe(no_mangle)] diff --git a/bridge/RustPacBoundary.c b/bridge/RustPacBoundary.c index 7a190db..645c287 100644 --- a/bridge/RustPacBoundary.c +++ b/bridge/RustPacBoundary.c @@ -43,6 +43,11 @@ int nmb_pac_atexit(void *function) { return atexit((void (*)(void))NMB_SIGN(function)); } +extern int __cxa_atexit(void (*function)(void *), void *argument, void *shared_object); +int nmb_pac_cxa_atexit(void *function, void *argument, void *shared_object) +{ + return __cxa_atexit((void (*)(void *))NMB_SIGN(function), argument, shared_object); +} void nmb_pac_dyld_register_func_for_add_image(void *function) { _dyld_register_func_for_add_image((void (*)(const struct mach_header *, intptr_t))NMB_SIGN(function)); diff --git a/bridge/interface_regressions_host.mm b/bridge/interface_regressions_host.mm index fe96e01..75c734a 100644 --- a/bridge/interface_regressions_host.mm +++ b/bridge/interface_regressions_host.mm @@ -65,7 +65,7 @@ int main(int argc,char **argv) {@autoreleasepool { CHECK(placed && placed.heap==storage && placed.allowGPUOptimizedContents==optimized); CHECK(placed.allocatedSize<=storage.size); auto alias=[placed newTextureViewWithPixelFormat:placed.pixelFormat];CHECK(alias && alias.allowGPUOptimizedContents==optimized); - auto overflow=[storage newTextureWithDescriptor:layout offset:footprint.align];CHECK(!overflow); + auto overflow=[storage newTextureWithDescriptor:layout offset:storage.size];CHECK(!overflow); } // Zero is a real swizzle, not a marker for the identity. td.swizzle=MTLTextureSwizzleChannelsMake(MTLTextureSwizzleZero,MTLTextureSwizzleZero,MTLTextureSwizzleZero,MTLTextureSwizzleZero); diff --git a/bridge/patches/0046-main-optimizations-on-development-555b13d5.patch b/bridge/patches/0046-main-optimizations-on-development-555b13d5.patch new file mode 100644 index 0000000..8e2b834 --- /dev/null +++ b/bridge/patches/0046-main-optimizations-on-development-555b13d5.patch @@ -0,0 +1,4492 @@ +diff --git a/crates/metal2vulkan-air/src/canon/mod.rs b/crates/metal2vulkan-air/src/canon/mod.rs +index a685b0dcb..4c2dcbc68 100644 +--- a/crates/metal2vulkan-air/src/canon/mod.rs ++++ b/crates/metal2vulkan-air/src/canon/mod.rs +@@ -557,6 +557,7 @@ pub fn specialize_dynamic( + let emap = erase::erase_consts(&mut parts.consts, m)?; + erase::erase_globals(parts.globals.values_mut(), &emap); + erase_functions(&mut parts, &emap, m)?; ++ prune_entry(&mut parts, func, &fcs, &name, m)?; + let inlined = inline::inline( + &parts.funcs, + (&parts.types, &parts.symbols), +@@ -620,16 +621,20 @@ pub fn specialize_dynamic( + } = solved; + // A pointer loaded from a field of a constant global is the field's constant: a `constexpr + // sampler` that SCCP's folded select picked is its sampler state. SCCP runs again over a body +- // that changed, to fold what those constants decide. +- let (body, folded) = const_loads::fold(body, &mut parts, m)?; +- let mut body = if folded { +- let solved = solve(&body, &mut parts, &fcs, &name, m)?; +- body.release(m); ++ // that changed, to fold what those constants decide; the two alternate until a round folds ++ // nothing, since a decided branch can expose a further select of the global. ++ let mut body = body; ++ loop { ++ let (next, folded) = const_loads::fold(body, &mut parts, m)?; ++ if !folded { ++ body = next; ++ break; ++ } ++ let solved = solve(&next, &mut parts, &fcs, &name, m)?; ++ next.release(m); + undefined_reads.extend(solved.reads); +- solved.body +- } else { +- body +- }; ++ body = solved.body; ++ } + // Loops whose trip count the constants decided, unrolled where their counters decide a branch + // of their body; SCCP after each round folds what the copies decide. The unrolled body is + // kept only where it comes out no larger than the loops it replaces (`unroll::live_size`). +@@ -893,6 +898,60 @@ fn erased_entry( + } + + /// Erase every defined function's body in place, one at a time, each body replaced freed. ++/// The entry's own blocks that its function constants leave unreachable, and its loads from ++/// fields of constant globals, folded before the entry is inlined, so the inliner copies no callee ++/// only a dead block calls. An uber-shader's entry is most of the module and branches on the ++/// constants into every effect it has: QuartzCore's `fixed_frag` entries inlined 15,700 ++/// instructions of which SCCP kept 300, and building, solving and rebuilding the rest was most of a ++/// specialization. ++/// ++/// Its calls are opaque here, so what it proves unreachable is unreachable whatever they return, ++/// and the SCCP after inlining reaches the fixed point it reached over the whole body. It runs ++/// only when every declared constant is supplied, so no undefined read goes unrecorded. Any ++/// refusal here (a branch on `undef` that a call's constant result would have made dead) leaves ++/// the entry as it was, for the SCCP after inlining to judge; a budget, cancellation or limit is ++/// final. ++fn prune_entry( ++ parts: &mut ModuleParts, ++ func: FuncId, ++ fcs: &HashMap, ++ name: &DiagName, ++ m: &mut Meter, ++) -> Result<(), Error> { ++ if fcs.is_empty() || fcs.values().any(|s| !s.supplied) { ++ return Ok(()); ++ } ++ let Some(entry) = parts.funcs.get_mut(func).and_then(|f| f.body.take()) else { ++ return Ok(()); ++ }; ++ let pruned = solve(&entry, parts, fcs, name, m).and_then(|solved| { ++ let mut body = solved.body; ++ loop { ++ let (next, folded) = const_loads::fold(body, parts, m)?; ++ if !folded { ++ return Ok(next); ++ } ++ let solved = solve(&next, parts, fcs, name, m); ++ next.release(m); ++ body = solved?.body; ++ } ++ }); ++ let kept = match pruned { ++ Ok(body) => { ++ entry.release(m); ++ body ++ } ++ Err(e @ (Error::Budget(_) | Error::Cancelled | Error::LimitExceeded(_))) => return Err(e), ++ Err(_) => entry, ++ }; ++ parts ++ .funcs ++ .get_mut(func) ++ .ok_or_else(|| invariant("entry out of range"))? ++ .body = Some(kept); ++ Ok(()) ++} ++ + fn erase_functions( + parts: &mut ModuleParts, + emap: &[crate::ir::ConstId], +diff --git a/crates/metal2vulkan-conformance/src/device.rs b/crates/metal2vulkan-conformance/src/device.rs +index aae72dcd5..93b87d1a2 100644 +--- a/crates/metal2vulkan-conformance/src/device.rs ++++ b/crates/metal2vulkan-conformance/src/device.rs +@@ -260,8 +260,9 @@ fn loader() -> Option { + /// **Neither changes a native answer.** The device's native arithmetic and value answers are the + /// contract under both (AGENTS.md, rule 1): no switch emulates Metal's bits where the device has a + /// native form, so the notes that state the device's own answer fire alike under either policy +-/// ([`crate::replay::NATIVE_NOTES`]). The one arithmetic switch, `half_chains`, acts only on a +-/// device without `shaderFloat16`, which has no native `half` to answer with. ++/// ([`crate::replay::NATIVE_NOTES`]). The one arithmetic switch, `half_chains`, acts on a device ++/// without `shaderFloat16`, which has no native `half` to answer with, and on one whose 16-bit ++/// floats keep no denormals, where it chooses between two statements of the device's own `half`. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum Policy { + /// `DevicePolicy::exact_answers()`: the accurate form of each of the runtime's +diff --git a/crates/metal2vulkan-runtime/src/batch_order.rs b/crates/metal2vulkan-runtime/src/batch_order.rs +index 1fef73096..97da33d27 100644 +--- a/crates/metal2vulkan-runtime/src/batch_order.rs ++++ b/crates/metal2vulkan-runtime/src/batch_order.rs +@@ -23,6 +23,13 @@ + //! - RADV (Mesa 26.0.8) on the Radeon 780M and the RX 7900 XTX; + //! - NVIDIA's driver (615.71.09) on the RTX 5080 Laptop GPU. + //! ++//! Turnip on an Adreno runs them so by construction, read in its source and not measured: the GPU ++//! has one command processor, which executes a ring's indirect buffers in the order the kernel ++//! wrote them, and Turnip gives each `VkQueue` one kernel submit queue whose submissions go to ++//! one ring in submission order (Mesa's `tu_queue.cc`, `tu_queue_submit` to ++//! `tu_knl::queue_submit`; `drm/msm`'s `msm_gpu_submit`, or a port's own kernel driver behind ++//! `tu_knl`). No two submissions of a queue overlap there, with or without a barrier. ++//! + //! On those drivers ([`ends_order`]) the default form orders batches by their ends: every batch + //! ends with [`Barrier::to_all`], every write before every access of every later command of its + //! `VkQueue`, so the tracker plans no barrier after an access of an earlier batch there +@@ -50,7 +57,12 @@ pub(crate) fn by_ends(policy: &DevicePolicy, driver: Option) -> bool { + fn ends_order(driver: Option) -> bool { + matches!( + driver, +- Some(DriverId::INTEL_OPEN_SOURCE_MESA | DriverId::MESA_RADV | DriverId::NVIDIA_PROPRIETARY) ++ Some( ++ DriverId::INTEL_OPEN_SOURCE_MESA ++ | DriverId::MESA_RADV ++ | DriverId::NVIDIA_PROPRIETARY ++ | DriverId::MESA_TURNIP ++ ) + ) + } + +diff --git a/crates/metal2vulkan-runtime/src/buffer.rs b/crates/metal2vulkan-runtime/src/buffer.rs +index eb53071c2..b9d67e9e7 100644 +--- a/crates/metal2vulkan-runtime/src/buffer.rs ++++ b/crates/metal2vulkan-runtime/src/buffer.rs +@@ -2942,6 +2942,15 @@ impl Device { + taken + } + ++ /// Let the parked buffers go whose time is up, or every one where `all` (`Recycler::aged`): ++ /// how many went. ++ pub(super) fn age_parked_buffers(&self, all: bool) -> usize { ++ let gone = lock(&self.shared().recycled_buffers).aged(Instant::now(), all); ++ let count = gone.len(); ++ self.let_go_parked_buffers(gone); ++ count ++ } ++ + /// Destroy the parked buffers the recycler let go of, `gone`, and free their memory. + fn let_go_parked_buffers(&self, gone: recycle::Gone) { + if gone.is_empty() { +diff --git a/crates/metal2vulkan-runtime/src/buffer_texture.rs b/crates/metal2vulkan-runtime/src/buffer_texture.rs +index eea20fa68..9c2e855bd 100644 +--- a/crates/metal2vulkan-runtime/src/buffer_texture.rs ++++ b/crates/metal2vulkan-runtime/src/buffer_texture.rs +@@ -1099,10 +1099,19 @@ impl Buffer { + let texels = Texels::placed(self.clone(), placed, texel_bytes, descriptor) + .with_options(options) + .over_surface(surface); ++ // Over its planes alone where the caller asks and the texture can be ++ // (`texture/ycbcr_image.rs`). ++ let alone = ycbcr_surface.is_some_and(|facts| facts.planes_alone); + return self +- .device() +- .new_converted_texture(call, texels, descriptor, &layout, id_use, (&planes, span)) +- .map(Over::Made); ++ .device().new_converted_texture( ++ call, ++ texels, ++ descriptor, ++ &layout, ++ (id_use, alone), ++ (&planes, span), ++ ) ++ .map(Over::Made); + } + if layout.texture_type == TextureType::TextureBuffer { + // Over a surface's plane, of the descriptor's options (`Buffer::new_texture_with_iosurface_plane`): +diff --git a/crates/metal2vulkan-runtime/src/command_buffer/completion.rs b/crates/metal2vulkan-runtime/src/command_buffer/completion.rs +index 95be96b54..a06b2696c 100644 +--- a/crates/metal2vulkan-runtime/src/command_buffer/completion.rs ++++ b/crates/metal2vulkan-runtime/src/command_buffer/completion.rs +@@ -724,6 +724,11 @@ fn finish(watch: &Watch, lists: &mut FenceLists, mut finished: Vec<(usize, Submi + } = batch; + lists.gathered.extend(fence.map(|fence| (queue, fence))); + if last { ++ crate::stage_trace::mark( ++ Arc::as_ptr(&*buffer) as usize, ++ crate::stage_trace::Stage::Seen, ++ queue.min(255) as u32, ++ ); + let mut state = lock(&buffer.state); + state.times.completed = seen; + state.keep_holds(holds); +@@ -1113,6 +1118,8 @@ struct Front { + value: u64, + /// The fence its submission signals, where it has one. + fence: Option, ++ /// Its command buffer, for the stage trace (`stage_trace.rs`). ++ id: usize, + } + + /// `VkQueue` `queue`'s waiter (the module docs): it waits for its queue's first pending batch +@@ -1148,6 +1155,7 @@ fn wait_for_queue(watch: &Watch, queue: usize) { + let front = Front { + value: batch.value, + fence: batch.fence, ++ id: Arc::as_ptr(&*batch.buffer) as usize, + }; + Some((front, batch.buffer.device().clone())) + } +@@ -1169,11 +1177,18 @@ fn wait_for_queue(watch: &Watch, queue: usize) { + continue; + }; + device.shared().counts.completion_armed(slept); ++ crate::stage_trace::mark( ++ front.id, ++ crate::stage_trace::Stage::PollArmed, ++ (queue.min(255) as u32) | (u32::from(slept) << 8), ++ ); + // The wait is a section of the exit guard; once the process is exiting the waiters end, + // and what is pending stays pending, as Metal's does. + let polled = match exit_guard::enter() { + Some(busy) => { ++ let id = front.id; + let polled = poll(&device, queue, front); ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::PollDone, queue.min(255) as u32); + drop(busy); + Some(polled) + } +@@ -1360,6 +1375,7 @@ pub(crate) fn seen( + } + + /// The command-buffer error of a failed wait, recording a lost device. ++#[track_caller] + fn failure(device: &Device, error: vk::Result) -> CommandBufferError { + match error { + vk::Result::ERROR_DEVICE_LOST => { +diff --git a/crates/metal2vulkan-runtime/src/command_buffer/mod.rs b/crates/metal2vulkan-runtime/src/command_buffer/mod.rs +index 87a5686c3..6d6433032 100644 +--- a/crates/metal2vulkan-runtime/src/command_buffer/mod.rs ++++ b/crates/metal2vulkan-runtime/src/command_buffer/mod.rs +@@ -2111,6 +2111,9 @@ fn write_fields(upload: &Upload, table: u64, set: u64, count: u64) -> Result<(), + } + + impl CommandBuffer { ++ /// Stable trace identity for this command buffer until its completion. ++ pub fn diagnostic_id(&self) -> u64 { Arc::as_ptr(&self.0) as u64 } ++ + /// A new command buffer of `root`, on its place `queue`, holding `slot` of it. + pub(super) fn new(queue: &QueueGen, root: &CommandQueue, slot: Slot) -> CommandBuffer { + let footprints = Footprints::default(); +@@ -3073,9 +3076,12 @@ impl CommandBuffer { + failure: Option, + hand: Hand, + ) -> Result<(), Error> { ++ let id = Arc::as_ptr(&self.0) as usize; ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::CommitCall, 0); + // Left to wait for the last render encoder's end, where its call left it to the device's + // lowering thread and the thread has not lowered it, or for the commits left so. + if failure.is_none() && self.queue_commit(call, hand)? { ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::CommitQueued, 0); + return Ok(()); + } + // The last render encoder's end, where its call left it to the device's lowering thread. +@@ -3178,6 +3184,12 @@ impl CommandBuffer { + run: CommitRun, + hand: Hand, + ) -> Result<(), Error> { ++ let run_kind = match run { ++ CommitRun::AtCall => 0, ++ CommitRun::Queued => 1, ++ CommitRun::Refused => 2, ++ }; ++ crate::stage_trace::mark(Arc::as_ptr(&self.0) as usize, crate::stage_trace::Stage::CommitRun, run_kind); + let _for = self.notices_for(); + // The waiting fetched render targets take their storage form at this commit where it + // allows (`storage_form.rs`): one atomic read where none waits. The scheduler's thread +@@ -3432,6 +3444,11 @@ impl CommandBuffer { + return Err(error); + } + }; ++ crate::stage_trace::mark( ++ Arc::as_ptr(&self.0) as usize, ++ crate::stage_trace::Stage::Prepared, ++ prepared.len() as u32, ++ ); + // The first segment a host step records, which the device announces (a loop's check + // and a staged scatter's CPU copy record nothing). + let announced = segments +@@ -5231,6 +5248,7 @@ impl CommandBufferShared { + result: Result<(), CommandBufferError>, + ) { + let (contexts, queue) = (buffer.device.shared().contexts.clone(), buffer.queue.id()); ++ let id = Arc::as_ptr(&*buffer) as usize; + let (signallers, own): (Vec>, _) = { + let state = lock(&buffer.state); + let signallers = (state.waited.iter()) +@@ -5279,12 +5297,16 @@ impl CommandBufferShared { + let ran = contexts.run_here(queue, item, may_run); + drop(unheld); + match ran { +- Ok(()) => return, ++ Ok(()) => { ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::Finished, 1); ++ return; ++ } + Err(item) => item, + } + } + None => item, + }; ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::Finished, 2); + contexts.post(queue, item.whole()); + posted(); + } +@@ -5305,6 +5327,8 @@ impl CommandBufferShared { + /// The first part of [`complete`](Self::complete), to its completion-seen handlers: its + /// status final, they ran. What the rest needs is answered. + fn complete_seen(self: &Arc, result: Result<(), CommandBufferError>) -> Rest { ++ let trace = Arc::as_ptr(self) as usize; ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::CompleteBegin, 0); + // A thread sleeps on `changed`, as the state's lock below reads. + let parked; + // Its command pool, where it settled, whose primary the device's spares begin ahead once +@@ -5497,6 +5521,7 @@ impl CommandBufferShared { + } + // The holds taken above go here, with no lock of the command buffer's held, before any + // handler of its runs: a detach from one of them detaches (`detach.rs`). ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::CompleteState, 0); + drop(submitted_holds); + self.wake_parked(parked); + let buffer = CommandBuffer(Arc::clone(self)); +@@ -5527,6 +5552,7 @@ impl CommandBufferShared { + /// handlers ran ([`complete_seen`](Self::complete_seen)): its completed handlers, its queue's + /// slot, its releases and its gate. + fn complete_rest(self: &Arc, rest: Rest) { ++ let trace = Arc::as_ptr(self) as usize; + let Rest { + settled_pool, + released_versions, +@@ -5553,6 +5579,7 @@ impl CommandBufferShared { + exit_guard::program(|| handler(&buffer)); + } + } ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::CompleteHandlers, 0); + let handled = Instant::now(); + drop(buffer); + // Its queue's slot comes back as its last completed handler returns, as Metal's does: a +@@ -5633,6 +5660,7 @@ impl CommandBufferShared { + } + // The gate notifies its waiters only where it opened here, which this transition's own + // hold holds it from: the notify for them is that hold's, as it goes (`gate.rs`). ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::CompleteRetained, 0); + let notified = self.gate.finish(); + self.device.shared().counts.command_buffer_wake(notified); + // Its pool's primary begun ahead, where no command buffer took the pool since +@@ -5643,6 +5671,7 @@ impl CommandBufferShared { + let shared = self.device.shared(); + shared.spares.begin_ahead(shared, pool); + } ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::CompleteEnd, 0); + } + } + +diff --git a/crates/metal2vulkan-runtime/src/command_buffer/submit.rs b/crates/metal2vulkan-runtime/src/command_buffer/submit.rs +index 024ee685e..600cfae8f 100644 +--- a/crates/metal2vulkan-runtime/src/command_buffer/submit.rs ++++ b/crates/metal2vulkan-runtime/src/command_buffer/submit.rs +@@ -424,6 +424,30 @@ impl CommandBufferShared { + scheduler: &mut scheduler::State, + ) -> Result { + let _for = self.notices_for(); ++ let id = Arc::as_ptr(self) as usize; ++ let at = (index.min(0x7fff) as u32) | (u32::from(last) << 15); ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::SubmitBegin, at); ++ let submitted = self.submit_batch_now(index, last, event_waits, alone, scheduler); ++ let outcome = match &submitted { ++ Ok(Submitting::Submitted(_)) => 0, ++ Ok(Submitting::Grouped) => 1, ++ Ok(Submitting::Later) => 2, ++ Ok(Submitting::AfterGroup(_)) => 3, ++ Err(_) => 4, ++ }; ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::SubmitEnd, outcome); ++ submitted ++ } ++ ++ /// [`submit_batch`](Self::submit_batch) itself. ++ fn submit_batch_now( ++ self: &Arc, ++ index: usize, ++ last: bool, ++ event_waits: &[(vk::Semaphore, u64)], ++ alone: bool, ++ scheduler: &mut scheduler::State, ++ ) -> Result { + let device = &self.device; + if device.is_lost() { + return Err(CommandBufferError::DeviceLost); +@@ -547,6 +571,7 @@ impl CommandBufferShared { + }; + // The scrubs of emulated textures' writes, as mapped now, then every segment in its place. + self.record_scrubs(&mut state, range.clone())?; ++ crate::stage_trace::mark(Arc::as_ptr(self) as usize, crate::stage_trace::Stage::SubmitStep, 1); + state + .place(range.clone()) + .map_err(|error| recording_error(device, error))?; +@@ -753,11 +778,8 @@ impl CommandBufferShared { + barriers.first().is_some_and(Option::is_some) || !held_moves.transitions(0).is_empty(); + if entry.is_some() { + started = true; +- let globals: Vec = completed_barrier +- .take() +- .into_iter() +- .chain(entry_barrier) +- .collect(); ++ let (pair, count) = barrier_pair(completed_barrier.take(), entry_barrier); ++ let globals = &pair[..count]; + let stamps = Stamps { + start: start_stamp.filter(|_| !encoder_behind), + after: entry_barrier.is_some(), +@@ -766,7 +788,7 @@ impl CommandBufferShared { + if stamps.start.is_some() { + start_stamp = None; + } +- work.push(self.record_barriers(&mut state, &globals, &entries, stamps)?); ++ work.push(self.record_barriers(&mut state, globals, &entries, stamps)?); + } + for (offset, barrier) in barriers.iter().enumerate() { + let moves = held_moves.transitions(offset); +@@ -781,11 +803,8 @@ impl CommandBufferShared { + work.extend(&part.raws); + continue; + } +- let globals: Vec = completed_barrier +- .take() +- .into_iter() +- .chain(*barrier) +- .collect(); ++ let (pair, count) = barrier_pair(completed_barrier.take(), *barrier); ++ let globals = &pair[..count]; + if !globals.is_empty() || start_stamp.is_some() || !moves.is_empty() { + started |= offset == 0; + let stamps = Stamps { +@@ -794,7 +813,7 @@ impl CommandBufferShared { + end: None, + }; + let kept = &mut scheduler.kept; +- work.push(self.barrier_buffer(&mut state, kept, at, &globals, moves, stamps)?); ++ work.push(self.barrier_buffer(&mut state, kept, at, globals, moves, stamps)?); + } + work.extend(&state.recorded[start + offset].raws); + } +@@ -1024,6 +1043,9 @@ impl CommandBufferShared { + false => work.insert(0, lead), + } + } ++ for command in work.iter().chain(finale.iter()) { ++ crate::stage_trace::diagnostic_link(Arc::as_ptr(self) as u64, ash::vk::Handle::as_raw(*command), index as u64); ++ } + let mut work_timeline = vk::TimelineSemaphoreSubmitInfo::default() + .wait_semaphore_values(wait_values) + .signal_semaphore_values(work_signal_values); +@@ -1067,6 +1089,7 @@ impl CommandBufferShared { + let buffers = (submits.iter()) + .map(|info| info.command_buffer_count as usize) + .sum(); ++ crate::stage_trace::mark(Arc::as_ptr(self) as usize, crate::stage_trace::Stage::SubmitStep, 2); + // From here its heap set versions may be pending: nothing writes them any more, and its + // first batch takes the ticket a texture that goes from now on waits for + // (`heap_sets.rs`). One committed and not yet submitted holds no gone texture. +@@ -1128,6 +1151,7 @@ impl CommandBufferShared { + scheduler + .group + .submit(device, &mut scheduler.waited[queue], Some(after)); ++ crate::stage_trace::mark(Arc::as_ptr(self) as usize, crate::stage_trace::Stage::SubmitStep, 3); + // A test's stand-in for a scheduler thread preempted once the call returned. + #[cfg(all(test, any(m2v_gpu, m2v_cpu_icd)))] + super::completion::trace::submit_returned(); +@@ -1277,6 +1301,7 @@ impl CommandBufferShared { + finishes_after(&batches.all()[index].waits, (queue, self), waited) + }; + drop(state); ++ crate::stage_trace::mark(Arc::as_ptr(self) as usize, crate::stage_trace::Stage::SubmitStep, 4); + if rebound { + device.degrade_encoded_use_rebound(rebound_use); + } +@@ -1806,3 +1831,15 @@ fn finishes_after( + } + after + } ++ ++/// `lead` then `barrier`, those of them there are, as the first `.1` of the array: the globals ++/// of a barrier between a batch's encoders, which takes no allocation. ++fn barrier_pair(lead: Option, barrier: Option) -> ([Barrier; 2], usize) { ++ let mut pair = [Barrier::default(); 2]; ++ let mut count = 0; ++ for b in lead.into_iter().chain(barrier) { ++ pair[count] = b; ++ count += 1; ++ } ++ (pair, count) ++} +diff --git a/crates/metal2vulkan-runtime/src/creation.rs b/crates/metal2vulkan-runtime/src/creation.rs +index 12076dbdc..15a835a16 100644 +--- a/crates/metal2vulkan-runtime/src/creation.rs ++++ b/crates/metal2vulkan-runtime/src/creation.rs +@@ -116,6 +116,7 @@ pub(crate) unsafe fn allocate_memory( + return Err(vk::Result::ERROR_TOO_MANY_OBJECTS); + }; + vk_made(VkMaking::Memory, Location::caller()); ++ note_allocation_site(info.allocation_size); + let busy = allocations::Busy::of(admitted.as_deref()); + // SAFETY: the caller's. + let memory = unsafe { device.allocate_memory(info, None) }; +@@ -124,6 +125,62 @@ pub(crate) unsafe fn allocate_memory( + memory + } + ++/// The call stacks that allocated device memory of at least [`SITE_BYTES`], by the bytes they ++/// allocated: what a process's GPU memory went to, without a debugger on the device. A stack is ++/// its first [`SITE_FRAMES`] return addresses past this function's; at most [`SITE_COUNT`] ++/// stacks are kept, a new one past them counted under the last. Kept only with the diagnostics on ++/// (`stage_trace::diagnostics`). ++const SITE_BYTES: u64 = 4 << 20; ++const SITE_FRAMES: usize = 10; ++const SITE_COUNT: usize = 32; ++static SITES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); ++ ++/// One call stack's allocations ([`allocation_sites`]). ++#[derive(Clone, Copy, Debug, Default)] ++pub struct AllocationSite { ++ /// Bytes allocated so far. ++ pub bytes: u64, ++ /// Allocations so far. ++ pub count: u64, ++ /// Return addresses, innermost first; 0 past the stack's end. ++ pub frames: [usize; SITE_FRAMES], ++} ++ ++fn note_allocation_site(bytes: u64) { ++ if bytes < SITE_BYTES || !crate::stage_trace::diagnostics() { ++ return; ++ } ++ let mut raw = [std::ptr::null_mut::(); SITE_FRAMES + 2]; ++ // SAFETY: `raw` holds the count passed. ++ #[cfg(target_vendor = "apple")] ++ let depth = unsafe { libc::backtrace(raw.as_mut_ptr(), raw.len() as libc::c_int) }.max(0) as usize; ++ #[cfg(not(target_vendor = "apple"))] ++ let depth = 0usize; ++ let mut frames = [0usize; SITE_FRAMES]; ++ for (frame, address) in frames.iter_mut().zip(raw.iter().take(depth).skip(2)) { ++ // The pointer authentication bits of a return address (arm64e) are not part of it. ++ *frame = (*address as usize) & ((1usize << 47) - 1); ++ } ++ let Ok(mut sites) = SITES.lock() else { return }; ++ let at = match sites.iter().position(|site| site.frames == frames) { ++ Some(at) => at, ++ None if sites.len() < SITE_COUNT => { ++ sites.push(AllocationSite { frames, ..Default::default() }); ++ sites.len() - 1 ++ } ++ None => sites.len() - 1, ++ }; ++ sites[at].bytes += bytes; ++ sites[at].count += 1; ++} ++ ++/// The call stacks that allocated device memory of 4 MiB or more, most bytes first. ++pub fn allocation_sites() -> Vec { ++ let mut sites = SITES.lock().map(|sites| sites.clone()).unwrap_or_default(); ++ sites.sort_by_key(|site| std::cmp::Reverse(site.bytes)); ++ sites ++} ++ + /// `image` bound to `memory` at `offset`. + /// + /// # Safety +diff --git a/crates/metal2vulkan-runtime/src/device/compiles.rs b/crates/metal2vulkan-runtime/src/device/compiles.rs +index a1bbd069b..5867afb40 100644 +--- a/crates/metal2vulkan-runtime/src/device/compiles.rs ++++ b/crates/metal2vulkan-runtime/src/device/compiles.rs +@@ -292,12 +292,35 @@ pub(crate) fn holding(call: impl FnOnce() -> R) -> (R, Vec<(MessageSink, Debu + (made, messages) + } + ++/// Whether a call that may compile is made on this thread whatever its class: the host says its ++/// driver compiles inside the call, on the calling thread (`METAL2VULKAN_COMPILE_ON_ASKING_THREAD=1`), ++/// so there is no compiler service whose class would follow the asking thread's, and a thread ++/// started for the call only makes the asking thread wait for one of a lower priority than its ++/// own (a render server's main thread, which has no class, waited 40 to 170 ms for it on a busy ++/// system). A thread with less than 2 MiB of stack still hands the call over: the driver's ++/// compiler runs deep, and the runtime's thread has the room. ++#[cfg(any(target_os = "macos", target_os = "ios"))] ++fn compiles_here() -> bool { ++ static HOST: std::sync::OnceLock = std::sync::OnceLock::new(); ++ let host = *HOST.get_or_init(|| { ++ std::env::var_os("METAL2VULKAN_COMPILE_ON_ASKING_THREAD").is_some_and(|value| value == "1") ++ }); ++ // SAFETY: the calling thread's own handle. ++ host && unsafe { libc::pthread_get_stacksize_np(libc::pthread_self()) } >= 2 << 20 ++} ++ ++/// Never: no class here, so every call is made on the asking thread already. ++#[cfg(not(any(target_os = "macos", target_os = "ios")))] ++fn compiles_here() -> bool { ++ false ++} ++ + /// Run `call`, one Vulkan call that may compile, at the user-initiated quality of service or + /// above (the module docs). It runs here where this thread is at that class, and otherwise on a + /// thread of the runtime's own at it while this thread waits. It runs here too where no thread + /// starts or the process is exiting. + pub(super) fn at_user_initiated(call: impl FnOnce() -> R + Send) -> R { +- if exit_guard::at_least_user_initiated() { ++ if exit_guard::at_least_user_initiated() || compiles_here() { + return call(); + } + // The driver diagnostics this thread's call is measured to report: the driver reports them +diff --git a/crates/metal2vulkan-runtime/src/device/mod.rs b/crates/metal2vulkan-runtime/src/device/mod.rs +index cb4fe4588..6ff2064bc 100644 +--- a/crates/metal2vulkan-runtime/src/device/mod.rs ++++ b/crates/metal2vulkan-runtime/src/device/mod.rs +@@ -31,6 +31,10 @@ pub(super) mod prime; + pub(super) mod properties; + pub(crate) mod queue_quiet; + pub(crate) mod relay; ++mod room; ++ ++/// The place in the source that first recorded a lost device ([`Device::lost_at`]). ++static LOST_AT: std::sync::OnceLock<(&'static str, u32)> = std::sync::OnceLock::new(); + mod staging; + pub(super) mod submit; + pub(crate) mod submit_stalls; +@@ -667,6 +671,9 @@ pub(super) struct DeviceShared { + /// The thread that lets go of what a released state's pool pieces gave back as they ended + /// (`pipeline/reaper.rs`). + pub(super) reaper: Arc, ++ /// The thread that keeps room in the pools the encodings and commits place their own ++ /// resources in, asked by the creation calls (`device/room.rs`). ++ room: room::Room, + /// The trails its threads open between a signal and their last let-go of what they held for + /// the work, which a command buffer's waiters wait for (`trails.rs`). + pub(crate) trails: Arc, +@@ -1030,6 +1037,9 @@ impl Drop for DeviceShared { + // What the reaper is handed holds the device: none is left, and it only stops, unless + // this teardown runs on its thread. + self.reaper.stop(); ++ // The room keeper holds the device only for a pass: it stops, unless this teardown runs ++ // on its thread. ++ self.room.stop(); + // What the residency thread releases holds the device: none is left. + #[cfg(unix)] + self.residency.stop(); +@@ -1485,11 +1495,59 @@ impl Device { + /// state's, a buffer's, a texture's, a heap's), so that no encoding or commit makes a memory + /// block, whose `vkAllocateMemory` waited 2.37 s for other processes' GPU work on NVIDIA's + /// driver. Nothing in a host step, a serial point of the command buffers' stream that makes +- /// only what it needs. ++ /// only what it needs. The device's room keeper makes the blocks, so the call ++ /// returns at once (`device/room.rs`). + pub(crate) fn keep_room(&self) { + if crate::exit_guard::in_host_step() { + return; + } ++ self.shared().room.ask(self); ++ } ++ ++ /// Give back the memory kept for later placements: every spare dedicated block and every ++ /// kept empty pooled block (`memory::SPARE_FOR`), as Metal's resource cache gives memory ++ /// back when the system reports memory pressure. How many blocks went. No Metal counterpart. ++ /// ++ /// The parked images and buffers go first (`recycle.rs`): each keeps its placement, so the ++ /// blocks they lie in empty only once they went. ++ pub fn release_kept_memory(&self) -> usize { ++ let Some(_busy) = crate::exit_guard::enter() else { ++ return 0; ++ }; ++ self.age_parked_images(true); ++ self.age_parked_buffers(true); ++ let shared = self.shared(); ++ lock(&shared.memory).release_all_kept(shared.vk()) ++ } ++ ++ /// Let the parked images and buffers go whose time is up (`recycle.rs`), from the room ++ /// thread: a device that makes and releases nothing more still lets them go. ++ pub(crate) fn age_parked(&self) { ++ self.age_parked_images(false); ++ self.age_parked_buffers(false); ++ } ++ ++ /// How long a spare block of the device's memory stays before it is freed ++ /// (`memory::SPARE_FOR`): the age a consumer that gives memory back by age of its own ++ /// (resources its caller declared discardable) takes, so both go on one clock. No Metal ++ /// counterpart. ++ pub fn spare_age() -> std::time::Duration { ++ crate::memory::KEPT_EMPTY_FOR ++ } ++ ++ /// The device's live blocks of its own memory by what holds them and by size, and what its ++ /// driver committed of the lazily allocated ones (`memory::MemoryKinds`): one walk of the ++ /// blocks under the memory's lock. A call of the program's on its own thread, as ++ /// [`work_counts`](Self::work_counts) is: it takes no section of the exit guard. No Metal ++ /// counterpart. ++ pub fn memory_kinds(&self) -> crate::MemoryKinds { ++ let shared = self.shared(); ++ lock(&shared.memory).kinds(shared.vk()) ++ } ++ ++ /// One pass of the room keeper (`device/room.rs`): room kept in the pools and their blocks ++ /// made ahead while they grow (`memory::keep_room`, `memory/ready.rs`). ++ pub(crate) fn keep_room_now(&self) { + let shared = self.shared(); + let copies = || self.copies_color_on_the_host(); + let memory = &shared.memory; +@@ -1534,6 +1592,13 @@ impl Device { + Arc::ptr_eq(self.line(), other.line()) + } + ++ /// Where the runtime first recorded a lost device in this process, as its source file and ++ /// line: which of its checks or which failed Vulkan call lost it, for a consumer's log. No ++ /// Metal counterpart. ++ pub fn lost_at() -> Option<(&'static str, u32)> { ++ LOST_AT.get().copied() ++ } ++ + /// Whether the device was lost: not to a public call that took its arguments on it before + /// its loss, which the loss follows (`Device::at_serving_taking`, `line.rs`). + pub(super) fn is_lost(&self) -> bool { +@@ -1542,8 +1607,12 @@ impl Device { + } + + /// Record that the device was lost: its line recovers at the next call that needs the device, +- /// where it recovers (`line.rs`). ++ /// where it recovers (`line.rs`). The first loss's place in the source is kept for ++ /// [`Device::lost_at`]. ++ #[track_caller] + pub(super) fn mark_lost(&self) { ++ let at = std::panic::Location::caller(); ++ LOST_AT.get_or_init(|| (at.file(), at.line())); + let shared = self.shared(); + shared.lost.store(true, Ordering::Release); + shared.line.lose(shared); +@@ -2167,6 +2236,31 @@ impl Device { + // SAFETY: the device is live, of `physical`, and enables timeline semaphores, and + // `bufferDeviceAddress` where `device_address` says so; `queues` holds at least one of + // its queues, which nothing uses yet. ++ let priming_has = drawn.has(prime::Has { ++ timestamps: timestamp_bits > 0, ++ // A device the runtime created enables it (`enables.rs`). ++ host_query_reset: owned, ++ device_addresses: device_address, ++ generated_commands: generated_commands.is_some(), ++ sparse_binding: sparse.binds(), ++ push_descriptor: push_descriptor.is_some(), ++ calibration: calibrated.is_some(), ++ object_names: object_name.is_some(), ++ extended_dynamic_state: false, ++ primitive_restart: false, ++ color_write_masks: false, ++ sized_index_bind: false, ++ ycbcr_conversion: ycbcr_formats.primer().is_some(), ++ }); ++ // Raw disk/cache I/O only: starting the ordinary guarded cache or its ++ // worker before priming would change the destructor ordering it protects. ++ let priming_cache = cache.priming( ++ &unsafe { instance.raw.get_physical_device_properties(physical) }, ++ compile.profile_fingerprint(), ++ instance.settings, ++ prime::cache_key(priming_has, queue_family), ++ instance.sink().cloned(), ++ ); + if let Err(error) = unsafe { + prime::prime( + &instance.raw, +@@ -2183,27 +2277,13 @@ impl Device { + queue_family, + &queues, + Sharing::with_buffers(&queue_families, &buffer_families), +- drawn.has(prime::Has { +- timestamps: timestamp_bits > 0, +- // A device the runtime created enables it (`enables.rs`). +- host_query_reset: owned, +- device_addresses: device_address, +- generated_commands: generated_commands.is_some(), +- sparse_binding: sparse.binds(), +- push_descriptor: push_descriptor.is_some(), +- calibration: calibrated.is_some(), +- object_names: object_name.is_some(), +- extended_dynamic_state: false, +- primitive_restart: false, +- color_write_masks: false, +- sized_index_bind: false, +- ycbcr_conversion: ycbcr_formats.primer().is_some(), +- }), ++ priming_has, + generated_commands.as_ref(), + push_descriptor, + calibrated, + drawn, + ycbcr_formats.primer(), ++ priming_cache, + ) + } { + scheduler.stop(); +@@ -2771,6 +2851,7 @@ impl Device { + uncommitted: Arc::default(), + pipeline_pool: Default::default(), + reaper: Default::default(), ++ room: Default::default(), + trails: Arc::default(), + recoveries: Arc::default(), + libraries: graphics_pipeline_library.then(Default::default), +diff --git a/crates/metal2vulkan-runtime/src/device/prime.rs b/crates/metal2vulkan-runtime/src/device/prime.rs +index eada5530e..fd69af97c 100644 +--- a/crates/metal2vulkan-runtime/src/device/prime.rs ++++ b/crates/metal2vulkan-runtime/src/device/prime.rs +@@ -83,9 +83,11 @@ use crate::gpu_time::Calibration; + use crate::loader_word::DeviceWord; + use crate::memory; + use crate::release::{self, Generated, Released}; ++use crate::shader_cache::PrimingCache; + use crate::sharing::Sharing; + use crate::tracker::Barrier; + use ash::vk; ++use sha2::{Digest, Sha256}; + use std::time::{Duration, Instant}; + + use super::functions::Vk; +@@ -362,7 +364,7 @@ fn specialization() -> vk::SpecializationInfo<'static> { + /// What the device primed has: a queue family that writes timestamps, `hostQueryReset`, device + /// addresses, `VK_EXT_device_generated_commands`, sparse binding on its queues, the push command, + /// calibrated timestamps, and the render encoder's commands of [`PrimedDraws`]. +-#[derive(Clone, Copy)] ++#[derive(Clone, Copy, Debug)] + pub(crate) struct Has { + pub(crate) timestamps: bool, + pub(crate) host_query_reset: bool, +@@ -404,6 +406,16 @@ impl PrimedDraws { + } + } + ++/// Identity includes every byte of the helper implementation, enabled compile ++/// profile (in Records), instance settings, driver identity, and these choices. ++/// No pointer/handle or host path enters the persistent key. ++pub(super) fn cache_key(has: Has, queue_family: u32) -> metal2vulkan::cache::Hash256 { ++ crate::shader_cache::driver::Making::new("entry-point-priming-v1") ++ .digest(&Sha256::digest(include_bytes!("prime.rs")).into()) ++ .text(&(has, queue_family)) ++ .finish() ++} ++ + /// Prime `device` of `physical`, made from `instance`, whose queues are `queues`, the first of + /// its primary family `queue_family`, its buffers and images shared as `sharing` says, on this + /// thread; `has` says what the device has, `generated` holds the commands of +@@ -431,6 +443,7 @@ pub(super) unsafe fn prime( + calibration: Option, + drawn: PrimedDraws, + ycbcr: Option>, ++ mut disk: PrimingCache, + ) -> Result<(), Error> { + let mut primer = Primer { + device, +@@ -443,6 +456,8 @@ pub(super) unsafe fn prime( + memory: unsafe { instance.get_physical_device_memory_properties(physical) }, + calls: Calls::NONE, + held: Vec::new(), ++ disk: &mut disk, ++ pipeline_cache: vk::PipelineCache::null(), + }; + // SAFETY: the caller's. + let primed = unsafe { +@@ -461,6 +476,10 @@ pub(super) unsafe fn prime( + unsafe { + let _ = device.device_wait_idle(); + primer.made(Call::DeviceWaitIdle); ++ if primed.is_ok() && primer.pipeline_cache != vk::PipelineCache::null() { ++ primer.disk.keep(device, loader_word, primer.pipeline_cache); ++ primer.made(Call::GetPipelineCacheData); ++ } + while let Some(release) = primer.held.pop() { + let call = release(device); + primer.made(call); +@@ -548,6 +567,8 @@ struct Primer<'a> { + calls: Calls, + /// What was made, released last first. + held: Vec, ++ disk: &'a mut PrimingCache, ++ pipeline_cache: vk::PipelineCache, + } + + impl Primer<'_> { +@@ -872,7 +893,7 @@ impl Primer<'_> { + .layout(layout) + .push_next(&mut flags); + let pipeline = device +- .create_compute_pipelines(vk::PipelineCache::null(), &[info], None) ++ .create_compute_pipelines(self.pipeline_cache, &[info], None) + .map_err(|(_, error)| vulkan(error))?[0]; + self.hold(pipeline); + let pipelines = ExecutionSetPipelineInfo::new(pipeline, 1); +@@ -1208,9 +1229,8 @@ impl Primer<'_> { + let module = device.create_shader_module(&info, None).map_err(vulkan)?; + self.made(Call::CreateShaderModule); + self.hold(module); +- let cache = device +- .create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) +- .map_err(vulkan)?; ++ let cache = self.disk.open(device, self.loader_word).map_err(vulkan)?; ++ self.pipeline_cache = cache; + self.made(Call::CreatePipelineCache); + self.hold(cache); + let specialization = specialization(); +diff --git a/crates/metal2vulkan-runtime/src/device/room.rs b/crates/metal2vulkan-runtime/src/device/room.rs +new file mode 100644 +index 000000000..531abcf9e +--- /dev/null ++++ b/crates/metal2vulkan-runtime/src/device/room.rs +@@ -0,0 +1,138 @@ ++//! The device's room keeper: one thread of the device's that keeps room in the pools its encodings ++//! and commits place their own resources in (`memory::keep_room`), so the creation call that asks ++//! for room returns at once. ++//! ++//! - **Asked by the creation calls.** [`Room::ask`] marks that room is wanted and wakes the ++//! thread, which it starts unless it runs; asks that come while the thread works are one more ++//! pass, not one each. ++//! - **Why a thread.** Making a block is a `vkAllocateMemory` that a driver may make wait: on an ++//! iPadOS guest whose GPU takes the system's pages, each 64 MiB block was wired page by page as it ++//! was made, 30-110 ms in the texture creation that asked for it (Core Animation's surfaces, ++//! the widget renderer's textures). A placement that finds no room before the block is there ++//! makes its own, and the keeper's block then goes at once (`memory::keep_room`). ++//! - **Holds nothing of the device.** The thread holds a weak handle only while it waits, and the ++//! device itself only for a pass. Where that pass held its last handle, the teardown runs here, ++//! and its [`Room::stop`] does not join this thread. ++ ++use super::{lock, Device, WeakDevice}; ++use crate::exit_guard; ++use std::sync::{Arc, Condvar, Mutex, PoisonError}; ++use std::thread::JoinHandle; ++ ++/// The device's room keeper: its thread, once a creation call started it. ++#[derive(Default)] ++pub(crate) struct Room { ++ shared: Arc, ++ thread: Mutex>>, ++} ++ ++#[derive(Default)] ++struct Shared { ++ state: Mutex, ++ /// Room was asked for, or the thread is asked to stop. ++ work: Condvar, ++} ++ ++#[derive(Default)] ++struct State { ++ asked: bool, ++ stopping: bool, ++} ++ ++impl Room { ++ /// Ask for room in `device`'s pools: the thread keeps it, started unless it runs. Where the ++ /// host cannot start a thread, the room is kept here, as before there was one. ++ pub(crate) fn ask(&self, device: &Device) { ++ { ++ let mut state = lock(&self.shared.state); ++ if state.stopping { ++ return; ++ } ++ state.asked = true; ++ } ++ self.shared.work.notify_all(); ++ let mut thread = lock(&self.thread); ++ if thread.is_some() { ++ return; ++ } ++ let shared = Arc::clone(&self.shared); ++ let weak = device.downgrade(); ++ match exit_guard::spawn( ++ std::thread::Builder::new().name("metal2vulkan-room".into()), ++ move || run(&shared, &weak), ++ ) { ++ Ok(handle) => *thread = Some(handle), ++ Err(_) => { ++ drop(thread); ++ lock(&self.shared.state).asked = false; ++ keep(device); ++ } ++ } ++ } ++ ++ /// The device's teardown: the thread ends, and is joined unless this is it. ++ pub(crate) fn stop(&self) { ++ lock(&self.shared.state).stopping = true; ++ self.shared.work.notify_all(); ++ let mut thread = lock(&self.thread); ++ let own = thread ++ .as_ref() ++ .is_some_and(|handle| handle.thread().id() == std::thread::current().id()); ++ if !own { ++ if let Some(handle) = thread.take() { ++ let _ = handle.join(); ++ } ++ } ++ } ++} ++ ++/// One pass: room kept in `device`'s pools, in a section of the exit guard. ++fn keep(device: &Device) { ++ let Some(_busy) = exit_guard::enter() else { ++ return; ++ }; ++ device.age_parked(); ++ let shared = device.shared(); ++ crate::memory::age_spares(&shared.memory, shared.vk()); ++ device.keep_room_now(); ++} ++ ++/// The thread's loop: a pass for each time room was asked since the last, until stopped. ++fn run(shared: &Shared, weak: &WeakDevice) { ++ loop { ++ let requested = { ++ let mut state = lock(&shared.state); ++ while !state.asked && !state.stopping { ++ let (next, timeout) = shared ++ .work ++ .wait_timeout(state, std::time::Duration::from_millis(250)) ++ .unwrap_or_else(PoisonError::into_inner); ++ state = next; ++ if timeout.timed_out() { ++ break; ++ } ++ } ++ if state.stopping { ++ return; ++ } ++ let requested = state.asked; ++ state.asked = false; ++ requested ++ }; ++ let Some(device) = weak.upgrade() else { ++ return; ++ }; ++ if requested { ++ keep(&device); ++ } else if let Some(_busy) = exit_guard::enter() { ++ // In a section of the exit guard, as every Vulkan call of a thread of the runtime's is: ++ // a parked image or buffer, a kept empty block and a spare whose time is up go here. ++ device.age_parked(); ++ let parts = device.shared(); ++ crate::memory::age_spares(&parts.memory, parts.vk()); ++ } ++ // Outside the state's lock: it may be the device's last handle, whose teardown stops ++ // this thread. ++ drop(device); ++ } ++} +diff --git a/crates/metal2vulkan-runtime/src/device/submit.rs b/crates/metal2vulkan-runtime/src/device/submit.rs +index 0550b9f5c..a1aba6ab6 100644 +--- a/crates/metal2vulkan-runtime/src/device/submit.rs ++++ b/crates/metal2vulkan-runtime/src/device/submit.rs +@@ -503,6 +503,7 @@ impl Device { + } + + /// The error of a failed submission or wait, recording a lost device. ++ #[track_caller] + pub(crate) fn failed(&self, error: vk::Result) -> Error { + if error == vk::Result::ERROR_DEVICE_LOST { + self.mark_lost(); +diff --git a/crates/metal2vulkan-runtime/src/exit_guard.rs b/crates/metal2vulkan-runtime/src/exit_guard.rs +index 1b4b753ea..c06fbc7ad 100644 +--- a/crates/metal2vulkan-runtime/src/exit_guard.rs ++++ b/crates/metal2vulkan-runtime/src/exit_guard.rs +@@ -1225,7 +1225,7 @@ thread_local! { + /// `device/compiles.rs`). MoltenVK runs each + /// compile on a dispatch queue that takes the class of the thread asking, so the class here is the + /// compile's class in Metal's compiler service. +-#[cfg(target_os = "macos")] ++#[cfg(any(target_os = "macos", target_os = "ios"))] + pub(crate) fn raise_to_user_initiated() { + if !at_least_user_initiated() { + // SAFETY: sets the calling thread's own class; no pointer. +@@ -1235,28 +1235,28 @@ pub(crate) fn raise_to_user_initiated() { + } + } + +-/// Nothing: no quality of service class here ([`raise_to_user_initiated`] on macOS). +-#[cfg(not(target_os = "macos"))] ++/// Nothing: no quality of service class here ([`raise_to_user_initiated`] on macOS and iOS). ++#[cfg(not(any(target_os = "macos", target_os = "ios")))] + pub(crate) fn raise_to_user_initiated() {} + + /// Lower this thread to the utility quality of service: a warm pass's thread (`warm/pass.rs`), + /// whose translations nothing waits for. Its pipeline makings still compile at the + /// user-initiated class, on a thread of their own (`device/compiles.rs`), and a thread it starts + /// starts at the default class ([`spawn`]). +-#[cfg(target_os = "macos")] ++#[cfg(any(target_os = "macos", target_os = "ios"))] + pub(crate) fn lower_to_utility() { + // SAFETY: sets the calling thread's own class; no pointer. + unsafe { libc::pthread_set_qos_class_self_np(libc::qos_class_t::QOS_CLASS_UTILITY, 0) }; + } + + /// Nothing elsewhere: a Linux thread's nice value passes to the threads it starts, and the warm +-/// pass starts the pipeline pool's, which serve the program ([`lower_to_utility`] on macOS). +-#[cfg(not(target_os = "macos"))] ++/// pass starts the pipeline pool's, which serve the program ([`lower_to_utility`] on macOS and iOS). ++#[cfg(not(any(target_os = "macos", target_os = "ios")))] + pub(crate) fn lower_to_utility() {} + + /// Run this thread at the user-interactive quality of service, whatever its class: a thread of + /// the runtime's own that takes command buffers to their handlers ([`spawn_user_interactive`]). +-#[cfg(target_os = "macos")] ++#[cfg(any(target_os = "macos", target_os = "ios"))] + fn run_at_user_interactive() { + // Metal runs every command buffer's scheduled and completed handlers at the user-interactive + // class, whatever the class of the thread that committed it: measured on an M3 Max, macOS +@@ -1376,12 +1376,12 @@ impl SchedAttr { + } + + /// Nothing: no such hint here ([`run_at_user_interactive`] on macOS and Linux). +-#[cfg(not(any(target_os = "macos", target_os = "linux")))] ++#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))] + fn run_at_user_interactive() {} + + /// Whether this thread runs at the user-initiated quality of service or above: its class is + /// user-initiated or user-interactive. Always where there is no such class. +-#[cfg(target_os = "macos")] ++#[cfg(any(target_os = "macos", target_os = "ios"))] + pub(crate) fn at_least_user_initiated() -> bool { + use libc::qos_class_t as Q; + let mut class = Q::QOS_CLASS_UNSPECIFIED; +@@ -1396,8 +1396,8 @@ pub(crate) fn at_least_user_initiated() -> bool { + ) + } + +-/// Always: no quality of service class here ([`at_least_user_initiated`] on macOS). +-#[cfg(not(target_os = "macos"))] ++/// Always: no quality of service class here ([`at_least_user_initiated`] on macOS and iOS). ++#[cfg(not(any(target_os = "macos", target_os = "ios")))] + pub(crate) fn at_least_user_initiated() -> bool { + true + } +diff --git a/crates/metal2vulkan-runtime/src/in_place.rs b/crates/metal2vulkan-runtime/src/in_place.rs +index 2e597846b..e807f98de 100644 +--- a/crates/metal2vulkan-runtime/src/in_place.rs ++++ b/crates/metal2vulkan-runtime/src/in_place.rs +@@ -802,6 +802,10 @@ pub(crate) fn in_place( + if policy == HostPageTexturePolicy::Shadow { + return Err(ShadowCause::Exact); + } ++ // Trusting the driver for every usage (`HostPageTexturePolicy::InPlaceWrites`): no check. ++ if policy == HostPageTexturePolicy::InPlaceWrites { ++ return Ok(()); ++ } + let written = Access::written(texel, padding); + if Access::Writes.takes(usage) && written.is_none() { + return Err(ShadowCause::StoreWidth(texel)); +diff --git a/crates/metal2vulkan-runtime/src/lib.rs b/crates/metal2vulkan-runtime/src/lib.rs +index 9e216b6af..1e892a791 100644 +--- a/crates/metal2vulkan-runtime/src/lib.rs ++++ b/crates/metal2vulkan-runtime/src/lib.rs +@@ -29,6 +29,7 @@ + #![recursion_limit = "256"] + + mod address; ++pub mod stage_trace; + mod address_map; + mod address_pool; + mod allocations; +@@ -370,12 +371,18 @@ pub use render_values::{ + pub use resource::{ObjectId, ResourceRef}; + pub use resource_ids::{IdPlacement, ResourceId, ResourceIdKind}; + pub use sampler::{sampler_border_color, SamplerState, SamplerStateOptions}; ++pub use creation::{allocation_sites, AllocationSite}; ++pub use memory::{MemoryKinds, MEMORY_KIND_CLASSES}; ++pub use shader_cache::system::{ ++ set_cache_readable_by_all, set_system_layer, system_layer, SystemLayer, SystemLayerCounts, ++}; + pub use shader_cache::{ + ShaderCacheOptions, ShaderCacheStatistics, DEFAULT_CAPACITY, DEFAULT_MEMORY_CAPACITY, + }; + pub use sparse::device::SparseFeatures; + pub use sparse::encoder::ResourceStateCommandEncoder; + pub use sparse::{SparsePageSize, SparseProblem, SparseTextureRegionAlignmentMode}; ++pub use texture::{ycbcr_surface_counts, YCbCrSurfaceCounts}; + pub use texture::{SharedTextureHandle, Texture, VkImageLayouts, WrappedImageProblem}; + pub use texture_descriptor::{TextureDescriptor, TextureProblem, TextureSwizzleChannels}; + pub use texture_placement::{ +diff --git a/crates/metal2vulkan-runtime/src/library.rs b/crates/metal2vulkan-runtime/src/library.rs +index 8f4a05025..a3e0f21a6 100644 +--- a/crates/metal2vulkan-runtime/src/library.rs ++++ b/crates/metal2vulkan-runtime/src/library.rs +@@ -555,6 +555,15 @@ impl Function { + descriptor + } + ++ /// The function as `newFunctionWithDescriptor:` makes it with `name` as its specialized name, ++ /// for a frontend that makes the function first: `None` keeps its own name. Its definition and ++ /// constants stay its own. ++ pub fn with_specialized_name(mut self, name: Option) -> Function { ++ self.specialized_name = name; ++ self.by_descriptor = true; ++ self ++ } ++ + /// The function as a stage links it, with its constant values, under its specialized name + /// where it was made with one, specialized where it was made with constant values or by + /// [`Library::new_function_with_descriptor`], and precompiled where it was made compiled to a +diff --git a/crates/metal2vulkan-runtime/src/linear_plan.rs b/crates/metal2vulkan-runtime/src/linear_plan.rs +index bf7b0911b..8875b45df 100644 +--- a/crates/metal2vulkan-runtime/src/linear_plan.rs ++++ b/crates/metal2vulkan-runtime/src/linear_plan.rs +@@ -1356,6 +1356,26 @@ pub(super) fn holding( + )) + } + ++/// Whether a render pass that keeps a texture `width` texels wide over `layout`'s texels in tile ++/// memory stores it back within the bytes an image over them may hold, in a buffer of ++/// `buffer_length` bytes: a tile store reaching the texture's right or bottom edge writes whole ++/// 16x4-texel blocks, so up to 15 texels past a row's last and 3 rows past its last row. Every ++/// row's block must end within its pitch, and the last block within the buffer; one slice only. ++pub(super) fn tile_store_fits(layout: &BufferLayout, width: u64, buffer_length: u64) -> bool { ++ const TILE: (u64, u64) = (16, 4); ++ if layout.slices != 1 || width == 0 || !layout.row_bytes.is_multiple_of(width) { ++ return false; ++ } ++ let block_row = width.next_multiple_of(TILE.0) * (layout.row_bytes / width); ++ let last_row = layout.rows.next_multiple_of(TILE.1) - 1; ++ layout.row_pitch >= block_row ++ && layout ++ .offset ++ .saturating_add(layout.row_pitch.saturating_mul(last_row)) ++ .saturating_add(block_row) ++ <= buffer_length ++} ++ + /// Whether a linear image over `layout`'s texels, whose subresources take `size` bytes each from + /// the first texel of their slice, `array_pitch` apart, holds no byte but the texture's texels: + /// the texels lie with no byte between them ([`holding`]), and its last subresource ends at their +diff --git a/crates/metal2vulkan-runtime/src/memory.rs b/crates/metal2vulkan-runtime/src/memory.rs +index 1d1143c66..3c5cdc9ff 100644 +--- a/crates/metal2vulkan-runtime/src/memory.rs ++++ b/crates/metal2vulkan-runtime/src/memory.rs +@@ -233,6 +233,57 @@ const DEDICATED_SLACK: u64 = 4; + /// makes and frees blocks. + const LOGGED_PLACEMENT_BYTES: u64 = 4 << 20; + ++/// The first pooled block of a pool: later ones grow with the pool ([`Memory::next_block_bytes`]), ++/// doubling it up to [`BLOCK_BYTES`]. Where device memory is the system's own and every block is ++/// wired as it is made (a guest whose GPU imports its pages), a process with a few small resources ++/// otherwise held a 64 MiB block in each of its pools: 256-640 MiB each, 2 GiB in all at an iPadOS ++/// guest's home screen. ++pub(super) const FIRST_BLOCK_BYTES: u64 = 4 << 20; ++ ++/// Memory operations below this size are not traced. ++const TRACE_MIN_BYTES: u64 = 256 << 10; ++ ++fn trace_memory(pool: (Holds, u32), bytes: u64, available: u64, kind: u32) -> usize { ++ use crate::stage_trace::{self, Stage}; ++ use std::sync::atomic::{AtomicUsize, Ordering}; ++ if bytes < TRACE_MIN_BYTES || !stage_trace::diagnostics() { ++ return 0; ++ } ++ static NEXT: AtomicUsize = AtomicUsize::new(1); ++ let id = (1usize << (usize::BITS - 1)) | NEXT.fetch_add(1, Ordering::Relaxed); ++ stage_trace::mark(id, Stage::MemoryBegin, kind); ++ stage_trace::mark( ++ id, ++ Stage::MemoryPool, ++ if kind == 3 { ++ 0x80000000 | pool.1 ++ } else { ++ pool.1 * 2 + u32::from(pool.0 == Holds::Images) ++ }, ++ ); ++ stage_trace::mark( ++ id, ++ Stage::MemoryBytes, ++ (bytes >> 10).min(u64::from(u32::MAX)) as u32, ++ ); ++ stage_trace::mark( ++ id, ++ Stage::MemoryAvailable, ++ (available >> 10).min(u64::from(u32::MAX)) as u32, ++ ); ++ id ++} ++ ++fn trace_handle(id: usize, memory: vk::DeviceMemory) { ++ use crate::stage_trace::{self, Stage}; ++ use ash::vk::Handle; ++ if id != 0 { ++ let raw = memory.as_raw(); ++ stage_trace::mark(id, Stage::MemoryHandleLow, raw as u32); ++ stage_trace::mark(id, Stage::MemoryHandleHigh, (raw >> 32) as u32); ++ } ++} ++ + /// What memory a resource needs. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] + pub(super) enum MemoryClass { +@@ -568,7 +619,10 @@ pub(super) const ZERO_INITIALIZE: vk::MemoryAllocateFlags = vk::MemoryAllocateFl + /// timed, the submission's fence read after both. On ANV (Mesa 26.2.4, the UHD 770 of RPL-S) the + /// bind took 0.2 to 1.8 us and the view 1.8 to 5.1 us; on RADV (Mesa 26.0.8) 0.0 to 0.6 us and + /// 0.6 to 2.0 us on the Radeon 780M and the RX 7900 XTX; the submission still pending after both +-/// in every round, as with a fresh block. There ([`binds_wait`]) such an image shares a pooled ++/// in every round, as with a fresh block. Turnip's bind waits for nothing either, read in its ++/// source and not measured: `tu_image_bind` (Mesa's `tu_image.cc`) records the memory, the offset ++/// and the image's address and makes no kernel call and takes no lock, and `tu_CreateImageView` ++/// none either. There ([`binds_wait`]) such an image shares a pooled + /// block ([`allocate_image_while_queued`]), and every submission names one buffer object fewer + /// to the kernel for each. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] +@@ -1065,6 +1119,14 @@ impl Pool { + } + } + ++ /// The bytes of its pooled blocks. ++ pub(super) fn pooled_bytes(&self) -> u64 { ++ (self.blocks.iter().flatten()) ++ .filter(|block| !block.dedicated) ++ .map(|block| block.ranges.len()) ++ .sum() ++ } ++ + /// Whether a pooled block has room for `size` bytes at the largest alignment asked of the + /// pool so far: whether [`Pool::room`] is that long. + /// +@@ -1307,13 +1369,18 @@ impl Pool { + /// imports, whatever else they are. + fn count_into(&self, census: &mut Census, imported: impl Fn(&T) -> bool) { + for block in self.blocks.iter().flatten() { +- let count = match (imported(&block.payload), block.dedicated) { +- (true, _) => &mut census.imported, +- (false, _) if block.empty() => &mut census.empty, +- (false, true) => &mut census.dedicated, +- (false, false) => &mut census.pooled, ++ let bytes = block.ranges.len(); ++ let (count, total) = match (imported(&block.payload), block.dedicated) { ++ (true, _) => (&mut census.imported, &mut census.imported_bytes), ++ (false, _) if block.empty() => (&mut census.empty, &mut census.empty_bytes), ++ (false, true) => (&mut census.dedicated, &mut census.dedicated_bytes), ++ (false, false) => { ++ census.pooled_used_bytes += block.ranges.used(); ++ (&mut census.pooled, &mut census.pooled_bytes) ++ } + }; + *count += 1; ++ *total += bytes; + } + } + } +@@ -1353,6 +1420,53 @@ impl FreeCause { + pub(super) const COUNT: usize = 7; + } + ++/// How many size classes [`MemoryKinds`] counts a block in: at most 16, 32, 40, 48, 56 and ++/// 64 MiB, and larger. ++pub const MEMORY_KIND_CLASSES: usize = 7; ++ ++/// The size class of a block of `bytes` ([`MEMORY_KIND_CLASSES`]). ++fn kind_class(bytes: u64) -> usize { ++ const BOUNDS: [u64; MEMORY_KIND_CLASSES - 1] = ++ [16 << 20, 32 << 20, 40 << 20, 48 << 20, 56 << 20, 64 << 20]; ++ (BOUNDS.iter()) ++ .position(|&bound| bytes <= bound) ++ .unwrap_or(MEMORY_KIND_CLASSES - 1) ++} ++ ++/// A device's live blocks of its own memory by what holds them and by size ++/// ([`Device::memory_kinds`](crate::Device::memory_kinds)): where the driver backs each block ++/// with a buffer object of the system's memory as it is made, which of them make up a process's ++/// large buffer objects. The caller's host memory imported and a buffer's own pages are in none ++/// of the classes: the driver allocates nothing for them. ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++pub struct MemoryKinds { ++ /// Pooled blocks with a placement, by size class. ++ pub pooled: [u64; MEMORY_KIND_CLASSES], ++ /// Pooled blocks kept empty for the next placement. ++ pub empty: [u64; MEMORY_KIND_CLASSES], ++ /// Dedicated blocks of images kept spare for the next image. ++ pub spare: [u64; MEMORY_KIND_CLASSES], ++ /// Dedicated blocks that hold an image. ++ pub dedicated_images: [u64; MEMORY_KIND_CLASSES], ++ /// Dedicated blocks that hold a buffer. ++ pub dedicated_buffers: [u64; MEMORY_KIND_CLASSES], ++ /// The bytes of the spare blocks. ++ pub spare_bytes: u64, ++ /// The bytes of the dedicated blocks that hold an image. ++ pub dedicated_image_bytes: u64, ++ /// The bytes of the dedicated blocks that hold a buffer. ++ pub dedicated_buffer_bytes: u64, ++ /// The blocks of a lazily allocated memory type with a placement: how many, their bytes, ++ /// how many of them the driver committed memory to and the bytes it committed ++ /// (`vkGetDeviceMemoryCommitment`). A transient attachment's memory is committed where a ++ /// pass renders it outside tile memory. ++ pub lazy: [u64; 4], ++ /// The same of the lazily allocated blocks with no placement, kept empty or spare. ++ pub lazy_kept: [u64; 4], ++ /// A buffer's own pages imported as a block each: how many, and their bytes. ++ pub own_pages: [u64; 2], ++} ++ + /// A device's live `VkDeviceMemory` blocks, by kind ([`Memory::census`]): each one buffer + /// object the driver lists in every submission where it lists every allocation (ANV on i915). + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +@@ -1366,6 +1480,13 @@ pub(super) struct Census { + pub(super) dedicated: u64, + /// The caller's host memory imported ([`import_host`]), a block each. + pub(super) imported: u64, ++ /// The bytes of those blocks: pooled ones, the bytes placed in them, empty kept ones, ++ /// dedicated ones and imports. ++ pub(super) pooled_bytes: u64, ++ pub(super) pooled_used_bytes: u64, ++ pub(super) empty_bytes: u64, ++ pub(super) dedicated_bytes: u64, ++ pub(super) imported_bytes: u64, + } + + /// A host pointer the runtime may send between threads: the mapping of a block it owns. +@@ -1526,6 +1647,11 @@ struct BlockAddresses { + } + + impl Block { ++ /// Whether it is the runtime's own memory, imported from nothing. ++ fn plain(&self) -> bool { ++ !self.imported && self.pages.is_none() && self.view.is_none() ++ } ++ + /// Whether another placement may take its memory once its own went: memory the runtime + /// allocated for itself, neither an import nor a buffer's dedicated allocation + /// (`VkMemoryDedicatedAllocateInfo`, which binds that buffer alone). +@@ -2548,6 +2674,60 @@ impl Memory { + census + } + ++ /// Its live blocks of its own memory by what holds them and by size, and what `device`'s ++ /// driver committed of the lazily allocated ones ([`MemoryKinds`]). ++ pub(super) fn kinds(&self, device: &ash::Device) -> MemoryKinds { ++ let mut kinds = MemoryKinds::default(); ++ let lazy = |block: &Block| { ++ (self.types.get(block.type_index as usize)).is_some_and(|ty| { ++ (ty.property_flags).contains(vk::MemoryPropertyFlags::LAZILY_ALLOCATED) ++ }) ++ }; ++ let commitment = |into: &mut [u64; 4], block: &Block, bytes: u64| { ++ if !lazy(block) { ++ return; ++ } ++ // SAFETY: the block is live under the memory's lock, and its memory type is lazily ++ // allocated (VUID-vkGetDeviceMemoryCommitment-memory-00690). ++ let committed = unsafe { device.get_device_memory_commitment(block.memory) }; ++ into[0] += 1; ++ into[1] += bytes; ++ into[2] += u64::from(committed != 0); ++ into[3] += committed; ++ }; ++ for (at, pool) in self.pools.iter().enumerate() { ++ let images = at >= self.types.len(); ++ for block in pool.blocks.iter().flatten() { ++ let bytes = block.ranges.len(); ++ if block.payload.imported { ++ continue; ++ } ++ if !block.payload.plain() { ++ kinds.own_pages[0] += 1; ++ kinds.own_pages[1] += bytes; ++ continue; ++ } ++ let class = kind_class(bytes); ++ let kept = !block.dedicated && block.ranges.is_empty(); ++ match (block.dedicated, images) { ++ (true, true) => { ++ kinds.dedicated_images[class] += 1; ++ kinds.dedicated_image_bytes += bytes; ++ } ++ (true, false) => { ++ kinds.dedicated_buffers[class] += 1; ++ kinds.dedicated_buffer_bytes += bytes; ++ } ++ (false, _) if kept => kinds.empty[class] += 1, ++ (false, _) => kinds.pooled[class] += 1, ++ } ++ let into = if kept { &mut kinds.lazy_kept } else { &mut kinds.lazy }; ++ commitment(into, &block.payload, bytes); ++ } ++ } ++ kinds ++ } ++ + /// Whether one of the memory types `allowed` names is of `class`. + pub(super) fn has_type(&self, allowed: u32, class: MemoryClass) -> bool { + self.class_types(allowed, class) != 0 +@@ -2950,13 +3130,11 @@ impl Memory { + pool.add_as(size, dedicated, block, used, zeroed) + } + +- /// The bytes of a pooled block made for `pool`: [`Memory::block_bytes`], and a block of +- /// images' [`Memory::image_block_bytes`]. ++ /// The bytes of the next pooled block made for `pool`: as many as its pooled blocks have (the ++ /// pool doubles), at least [`FIRST_BLOCK_BYTES`], at most [`Memory::block_bytes`], and a block ++ /// of images' [`Memory::image_block_bytes`] ([`Memory::next_block_bytes`]). + fn pooled_bytes(&self, pool: (Holds, u32)) -> u64 { +- match pool.0 { +- Holds::Images => self.image_block_bytes, +- _ => self.block_bytes, +- } ++ self.next_block_bytes(pool, 1) + } + + /// The bytes of each live pooled block that holds `holds`, of every memory type, for the +@@ -3018,6 +3196,21 @@ impl Memory { + .collect() + } + ++ /// The size of the next pooled block of `pool` for a placement of `size` bytes: as many bytes ++ /// as its pooled blocks have (the pool doubles), at least [`FIRST_BLOCK_BYTES`] and the ++ /// placement's next power of two, at most [`BLOCK_BYTES`] (or the device's largest ++ /// allocation). ++ fn next_block_bytes(&self, pool: (Holds, u32), size: u64) -> u64 { ++ let have = self.pool_ref(pool.0, pool.1).pooled_bytes(); ++ let most = match pool.0 { ++ Holds::Images => self.image_block_bytes, ++ _ => self.block_bytes, ++ }; ++ (have.max(FIRST_BLOCK_BYTES).max(size.next_power_of_two())) ++ .min(most) ++ .max(size) ++ } ++ + /// What making a block takes, copied out so that the block is made with the memory's lock + /// released ([`place`], [`keep_room`]). + fn maker(&self) -> BlockMaker { +@@ -3026,7 +3219,6 @@ impl Memory { + .fold(0, |bits, (index, _)| bits | 1 << index); + BlockMaker { + block_bytes: self.block_bytes, +- image_block_bytes: self.image_block_bytes, + host_visible, + fd_types: self.fd_buffer_types.unwrap_or(0), + page_types: self.page_buffer_types.unwrap_or(0), +@@ -3222,7 +3414,9 @@ impl Memory { + let spare = self + .kept_budget(heap) + .saturating_sub(self.kept_in_heap(heap)); +- let half = self.block_bytes / 2; ++ // A dedicated block is larger than half its pool's block as it was made, and a pool's ++ // blocks grow from [`FIRST_BLOCK_BYTES`]: every dedicated one past half of that is kept. ++ let half = FIRST_BLOCK_BYTES / 2; + let reusable = |block: &Block| block.reusable() && block.size > half; + let pool = self.pool(holds, type_index); + let freed = pool.release(block, (offset, size), now, spare, reusable); +@@ -3310,6 +3504,21 @@ impl Memory { + count + } + ++ /// Free every kept empty block, pooled and dedicated: the system reports memory pressure, ++ /// the signal Metal's resource cache gives memory back on. How many went. ++ pub(super) fn release_all_kept(&mut self, device: &ash::Device) -> usize { ++ let mut blocks: Vec = Vec::new(); ++ for pool in &mut self.pools { ++ blocks.extend(pool.take_kept()); ++ } ++ let count = blocks.len(); ++ for block in blocks { ++ // SAFETY: nothing is placed in a kept empty block, and no command reads it. ++ drop(unsafe { self.let_go(device, block, FreeCause::KeptForRoom, None) }); ++ } ++ count ++ } ++ + /// Over every pool: the block slots the room queries have walked and the free ranges they + /// have visited ([`Pool::room_work`]), then what one query of each pool would walk and visit + /// where it derived per call ([`Pool::room_per_call`]). +@@ -3383,10 +3592,18 @@ impl Memory { + }; + let (heap, size) = (self.heap_of(block.type_index), block.size); + let Some(parking) = &self.parking else { ++ let id = trace_memory((Holds::Buffers, block.type_index), block.size, 0, 3); ++ trace_handle(id, block.memory); + self.say_block("freed a block", &block); + let Some(frees) = frees else { ++ if id != 0 { ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::MemoryFreeBegin, 0); ++ } + // SAFETY: the caller's. + unsafe { block.release(device) }; ++ if id != 0 { ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::MemoryFreeEnd, 0); ++ } + return after; + }; + let Block { +@@ -3403,6 +3620,9 @@ impl Memory { + size, + }; + // SAFETY: the caller's. ++ if id != 0 { ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::MemoryDeferred, (size >> 10) as u32); ++ } + return unsafe { frees.post_or_free(posted, device) }; + }; + self.say_block( +@@ -3647,8 +3867,6 @@ impl Memory { + struct BlockMaker { + /// A pooled block's size ([`BLOCK_BYTES`], or the device's largest allocation). + block_bytes: u64, +- /// As [`Memory`]'s. +- image_block_bytes: u64, + /// The host-visible memory types, as bits: a block of one is mapped whole. + host_visible: u32, + /// The memory types whose [`Holds::FdBuffers`] blocks are allocated exportable, as bits. +@@ -3668,14 +3886,6 @@ struct BlockMaker { + } + + impl BlockMaker { +- /// The bytes of a pooled block made for `pool` ([`Memory::pooled_bytes`]). +- fn pooled_bytes(&self, pool: (Holds, u32)) -> u64 { +- match pool.0 { +- Holds::Images => self.image_block_bytes, +- _ => self.block_bytes, +- } +- } +- + /// A new block of `size` bytes of memory type `type_index` that holds `holds`, with a buffer + /// of addresses where `addressed` gives the counts of such buffers made (its memory then as + /// long as that buffer requires, if longer), allocated zero-initialized where it holds images +@@ -4460,7 +4670,7 @@ pub(super) fn allocate_image_while_queued( + fn binds_wait(driver: Option) -> bool { + !matches!( + driver, +- Some(DriverId::INTEL_OPEN_SOURCE_MESA | DriverId::MESA_RADV) ++ Some(DriverId::INTEL_OPEN_SOURCE_MESA | DriverId::MESA_RADV | DriverId::MESA_TURNIP) + ) + } + +@@ -4624,13 +4834,19 @@ fn place( + let pool = (holds, type_index); + let block_size = match dedicated { + true => size, +- false => maker.pooled_bytes(pool), ++ false => lock(memory).next_block_bytes(pool, size), + }; ++ let available = lock(memory).pool(pool.0, pool.1).room(); ++ let trace = trace_memory(pool, block_size, available, 1); + if fit == Fit::Pooled { + let mut guard = lock(memory); + if let Some(placement) = + guard.place_pooled(device, pool, (size, align), addressed, class)? + { ++ trace_handle(trace, placement.memory); ++ if trace != 0 { ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::MemoryReused, 1); ++ } + ready_after_placing(&mut guard, pool, size); + return Ok(guard.mark_overflowed(placement, overflowing)); + } +@@ -4673,6 +4889,10 @@ fn place( + }; + match made { + Ok(block) => { ++ trace_handle(trace, block.memory); ++ if trace != 0 { ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::MemoryAllocated, 1); ++ } + let mut memory = lock(memory); + if !dedicated && memory.makes_ahead(pool) { + memory.ahead.made_in_place(holds); +@@ -4691,8 +4911,16 @@ fn place( + memory.say_placement("placed", &placement, Some(class)); + return Ok(memory.mark_overflowed(placement, overflowing)); + } +- Err(error @ Error::OutOfMemory(_)) => failure = error, +- Err(error) => return Err(error), ++ Err(error) => { ++ if trace != 0 { ++ crate::stage_trace::mark(trace, crate::stage_trace::Stage::MemoryAllocated, 0); ++ } ++ if matches!(error, Error::OutOfMemory(_)) { ++ failure = error; ++ } else { ++ return Err(error); ++ } ++ } + } + } + Err(failure) +@@ -4713,6 +4941,12 @@ fn make_room(memory: &Mutex, device: &ash::Device, type_index: u32) -> b + released > 0 || waited + } + ++/// Free the kept empty blocks whose time is up, from the room worker: a device that makes and ++/// releases nothing more still lets them go after [`KEPT_EMPTY_FOR`] ([`Memory::sweep_stale`]). ++pub(super) fn age_spares(memory: &Mutex, device: &ash::Device) { ++ lock(memory).sweep_stale(device, Instant::now()); ++} ++ + /// Let `memory` keep no block from now on ([`Memory::keep_none`]): those it keeps empty freed, and + /// those still being made ahead stopped, each waited for with the lock released (its making + /// reads its pages), and freed. +@@ -4789,7 +5023,7 @@ pub(super) fn keep_room( + .filter(|&&(holds, _)| holds != own && holds != Holds::FdBuffers && !page(holds)); + for &pool in first.chain(rest) { + let addressed = pool.0.buffers().then_some(&**counts); +- let bytes = maker.pooled_bytes(pool); ++ let bytes = lock(memory).pooled_bytes(pool); + keep_pool_room(memory, device, (&maker, addressed), (pool, bytes)); + ready_if_growing(&mut lock(memory), pool); + } +diff --git a/crates/metal2vulkan-runtime/src/neutral.rs b/crates/metal2vulkan-runtime/src/neutral.rs +index 526e32ebb..0090a6cb6 100644 +--- a/crates/metal2vulkan-runtime/src/neutral.rs ++++ b/crates/metal2vulkan-runtime/src/neutral.rs +@@ -37,7 +37,7 @@ use metal2vulkan::bind::NeutralResource; + use metal2vulkan::device::FormatUsage; + use metal2vulkan::vk::Format; + use std::any::Any; +-use std::collections::HashMap; ++use super::key_hash::KeyMap; + use std::sync::{Arc, Mutex, Weak}; + + /// One neutral image, its view and its memory. +@@ -99,9 +99,9 @@ impl Filled for NeutralZeros { + /// A device's neutral images, texel buffers, zero buffers and samplers, made at first use. + #[derive(Default)] + pub(super) struct Neutrals { +- images: HashMap>, ++ images: KeyMap>, + /// By format and whether the view is a storage texel buffer's. +- texels: HashMap<(Format, bool), Arc>, ++ texels: KeyMap<(Format, bool), Arc>, + /// By their length in bytes, which the plan asks for (`zero_buffer_bytes`, fixed for the + /// device): the read-only zero buffers, and how many the device has made. + /// +@@ -113,12 +113,12 @@ pub(super) struct Neutrals { + /// allocate-and-zero a draw. Zeros of a length the device's own limits fix cannot have + /// changed since the device was made, so the device makes each length once and keeps it, as + /// it keeps its neutral images, texel buffers and samplers. +- zeros: HashMap>, ++ zeros: KeyMap>, + /// How many zero buffers the device has made, for the check that draws share one + /// (`tests::neutral_zeros_shared`). + zeros_made: u64, + /// By whether depth comparison is enabled. +- samplers: HashMap, ++ samplers: KeyMap, + } + + impl Neutrals { +diff --git a/crates/metal2vulkan-runtime/src/range.rs b/crates/metal2vulkan-runtime/src/range.rs +index 1512ab0ed..1fcd2f2e9 100644 +--- a/crates/metal2vulkan-runtime/src/range.rs ++++ b/crates/metal2vulkan-runtime/src/range.rs +@@ -111,6 +111,11 @@ impl RangeAllocator { + self.free.insert(start, size); + } + ++ /// The bytes of `[base, end)`. ++ pub(super) fn len(&self) -> u64 { ++ self.end - self.base ++ } ++ + /// Whether nothing is allocated. + pub(super) fn is_empty(&self) -> bool { + self.free.len() == 1 && self.free.get(&self.base) == Some(&(self.end - self.base)) +diff --git a/crates/metal2vulkan-runtime/src/rebind.rs b/crates/metal2vulkan-runtime/src/rebind.rs +index bece8a95b..5eab3a621 100644 +--- a/crates/metal2vulkan-runtime/src/rebind.rs ++++ b/crates/metal2vulkan-runtime/src/rebind.rs +@@ -99,6 +99,8 @@ pub enum RebindProblem { + /// attachments' bytes from its open), or one committed whose commands the runtime records + /// only once earlier work completed: commit or drop it first. + EncodedUses, ++ /// The device refuses every rebind (`Device::refuse_rebinds`). ++ Refused, + } + + /// The device's buffers over host pages, by their hazard-tracking key, with the command buffers +@@ -106,6 +108,9 @@ pub enum RebindProblem { + #[derive(Default)] + pub(super) struct Rebinds { + live: AtomicUsize, ++ /// The device refuses every rebind (`Device::refuse_rebinds`): its encoders log no call and ++ /// hold no buffer over host pages, as though it had none. ++ refused: std::sync::atomic::AtomicBool, + buffers: Mutex>, + /// The keys `buffers` may hold ([`HostedKeys`]): a use of any other names no buffer over host + /// pages, which an encoder's hold tells without taking `buffers`' lock. +@@ -255,7 +260,7 @@ impl Rebinds { + + /// Whether the device has a buffer over host pages: else no use names one. + pub(super) fn any(&self) -> bool { +- self.live.load(Ordering::Relaxed) > 0 ++ self.live.load(Ordering::Relaxed) > 0 && !self.refused.load(Ordering::Relaxed) + } + + /// Note the buffer of `key` over host pages, `cell` its hold on them. +@@ -998,6 +1003,9 @@ impl Buffer { + keep_alive: Option, + ) -> Result<(), Error> { + const CALL: &str = "rebind_bytes_no_copy"; ++ if self.device().shared().rebinds.refused.load(Ordering::Relaxed) { ++ return Err(invalid(CALL, Problem::Rebind(RebindProblem::Refused))); ++ } + let Some(cell) = self.host_cell() else { + return Err(invalid(CALL, Problem::Rebind(RebindProblem::NotHostMemory))); + }; +@@ -1053,6 +1061,9 @@ impl Heap { + keep_alive: Option, + ) -> Result<(), Error> { + const CALL: &str = "rebind_bytes_no_copy"; ++ if self.device().shared().rebinds.refused.load(Ordering::Relaxed) { ++ return Err(invalid(CALL, Problem::Rebind(RebindProblem::Refused))); ++ } + let Some((own, cell)) = self + .bytes() + .and_then(|own| own.host_cell().map(|cell| (own, cell))) +@@ -1082,3 +1093,16 @@ impl Device { + self.shared().rebinds.pending(key) + } + } ++ ++impl Device { ++ /// Refuse every `rebind_bytes_no_copy` of the device's buffers and heaps from now on ++ /// (`Invalid`, [`RebindProblem::Refused`]), for a consumer that never moves the pages of what ++ /// it made with `new_buffer_with_bytes_no_copy` or `new_heap_with_bytes_no_copy`, and never ++ /// detaches them: the encoders then log no call to encode again at a rebind and hold no buffer ++ /// over host pages, which only a rebind or a detach reads, so a draw pays for neither. Call ++ /// before any buffer over host pages is made. No Metal counterpart. ++ #[doc(hidden)] ++ pub fn refuse_rebinds(&self) { ++ self.shared().rebinds.refused.store(true, Ordering::Relaxed); ++ } ++} +diff --git a/crates/metal2vulkan-runtime/src/recycle.rs b/crates/metal2vulkan-runtime/src/recycle.rs +index a5db81130..80afa1207 100644 +--- a/crates/metal2vulkan-runtime/src/recycle.rs ++++ b/crates/metal2vulkan-runtime/src/recycle.rs +@@ -315,6 +315,26 @@ impl Recycler { + Some(self.unindexed(order, parked)) + } + ++ /// The objects parked longer than the bounds' time at `now`, out of the recycler, or every ++ /// parked object where `all`: what a device lets go of when its time passes with no making or ++ /// release to notice it (the room thread, `device/room.rs`), and when the system reports ++ /// memory pressure (`Device::release_kept_memory`). A parked object keeps its placement, so ++ /// its block stays live: a process that released its last textures and then made nothing more ++ /// kept up to the bounds' bytes parked for good. ++ pub(crate) fn aged(&mut self, now: Instant, all: bool) -> Gone { ++ if !all { ++ return self.stale(now); ++ } ++ let mut gone = Gone::new(); ++ while let Some((order, parked)) = self.trial.pop_first(None) { ++ gone.push(self.unindexed(order, parked)); ++ } ++ while let Some((order, parked)) = self.proven.pop_first(None) { ++ gone.push(self.unindexed(order, parked)); ++ } ++ gone ++ } ++ + /// The objects parked longer than the bounds' time at `now`, out of the recycler. + fn stale(&mut self, now: Instant) -> Gone { + let stale = Some((now, self.bounds.parked_for)); +diff --git a/crates/metal2vulkan-runtime/src/render.rs b/crates/metal2vulkan-runtime/src/render.rs +index c068e8f47..9eb79e8ba 100644 +--- a/crates/metal2vulkan-runtime/src/render.rs ++++ b/crates/metal2vulkan-runtime/src/render.rs +@@ -12490,6 +12490,11 @@ impl Recording<'_> { + /// recorded into it bound (each recorded through it). With a pair, the pass is one + /// `render_target` made, of no views (`snapshot::Pair::record`). + unsafe fn record(&self, bound: &mut Bound) -> Recorded { ++ // A draw whose pipeline could not be created (the compile was refused) has a null pipeline; binding it crashes the driver ++ // (int-170: tu_CmdBindPipeline in the render server). The draw is dropped instead, and nothing is recorded for it. ++ if self.pipeline == vk::Pipeline::null() { ++ return Recorded::default(); ++ } + let (raw, cb) = (self.raw, self.command_buffer); + bound.follow(cb); + let mut recorded = 0; +diff --git a/crates/metal2vulkan-runtime/src/scheduler/mod.rs b/crates/metal2vulkan-runtime/src/scheduler/mod.rs +index 911fc7a1a..d12297e84 100644 +--- a/crates/metal2vulkan-runtime/src/scheduler/mod.rs ++++ b/crates/metal2vulkan-runtime/src/scheduler/mod.rs +@@ -529,6 +529,9 @@ impl SchedulerShared { + /// lock of its sleep; one that sleeps later reads what the waker wrote first), counted as + /// `woken`; else the notify, a system call, is spared. + fn woken(&self, woken: Woken, asleep: Asleep) { ++ if !matches!(asleep, Asleep::No) { ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::SchedulerWake, woken as u32); ++ } + match asleep { + // The thread alone waits on it. + Asleep::OnCondvar => { +@@ -708,6 +711,8 @@ struct Held { + /// The host nanoseconds at the first look of the scheduler's that found it held + /// ([`held_for`](Self::held_for)), until it goes ([`hold_ended`](Self::hold_ended)). + held_since: Option, ++ /// The reasons a look held it for that the stage trace recorded (`stage_trace.rs`), a bit each. ++ traced_reasons: u8, + } + + impl Held { +@@ -716,6 +721,15 @@ impl Held { + /// read from the host's clock while nothing waits. + fn held_for(&mut self, reason: HeldFor) { + (self.buffer.device().shared().counts).head_held_for(reason); ++ let bit = 1u8 << (reason as u8); ++ if self.traced_reasons & bit == 0 { ++ self.traced_reasons |= bit; ++ crate::stage_trace::mark( ++ Arc::as_ptr(&*self.buffer) as usize, ++ crate::stage_trace::Stage::Held, ++ (reason as u32 + 1) | ((self.index.min(255) as u32) << 8), ++ ); ++ } + if self.held_since.is_none() { + self.held_since = Some(gpu_time::host_nanos()); + } +@@ -859,6 +873,11 @@ impl Scheduler { + hand: Hand, + ) { + let shared = &*self.shared; ++ crate::stage_trace::mark( ++ Arc::as_ptr(buffer) as usize, ++ crate::stage_trace::Stage::Handed, ++ batches.len() as u32, ++ ); + let committed = Committed { + buffer: buffer.hold(), + queue, +@@ -1245,6 +1264,7 @@ fn run(shared: &Arc) { + && bound.is_none() + && stalls.is_none(); + let slice = bound.map_or(slice, |left| left.min(slice)); ++ let slice_ms = slice.as_millis().min(0x7fff) as u32; + let on_gpu = shared.nudge.get().filter(|_| shared.gpu_waits(&mut gpu)); + // The lock of the sleep ([`Sleep`]), under the scheduler's: a waker that found no + // pass under way wrote what it asks for before it took it, so its commit or nudge is +@@ -1255,12 +1275,15 @@ fn run(shared: &Arc) { + } + shared.sleeps.fetch_add(1, Ordering::Relaxed); + if let Some(nudge) = on_gpu { ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::SchedulerSleep, slice_ms | 0x8000); + state = wait_for_gpu(shared, (state, sleep), nudge, &mut gpu, (slice, short)); ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::SchedulerResumed, 0); + #[cfg(all(test, any(m2v_gpu, m2v_cpu_icd)))] + trace::submitting("thread woke"); + continue; + } + sleep.asleep = true; ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::SchedulerSleep, slice_ms); + // A waker that marks the state under the scheduler's lock takes it once this lets + // go of it, and then the sleep's, which the wait lets go of. + drop(state); +@@ -1277,6 +1300,7 @@ fn run(shared: &Arc) { + waited.timed_out() + } + }; ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::SchedulerResumed, u32::from(timed_out)); + sleep.asleep = false; + drop(sleep); + #[cfg(all(test, any(m2v_gpu, m2v_cpu_icd)))] +@@ -1400,8 +1424,12 @@ fn run_work(shared: &Arc, kind: Kind, work: Work) -> bool { + fn round(shared: &SchedulerShared, left: &mut Left, passed: &mut Passed) -> bool { + (shared.inbox).pass(|state, commits| { + shared.passes.fetch_add(1, Ordering::Relaxed); ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::PassBegin, 0); ++ let before = passed.spent.len(); + let taken = pass(state, commits, (&mut *passed, &mut *left), &shared.awaits); + shared.took_in(taken); ++ let spent = passed.spent.len().saturating_sub(before); ++ crate::stage_trace::mark(0, crate::stage_trace::Stage::PassEnd, spent.min(0xffff) as u32); + !passed.exiting && left.is_empty() && !passed.more + }); + passed.more = false; +@@ -1460,6 +1488,11 @@ fn take_in(state: &mut State, commits: &mut Vec, spent: &mut Spent) - + } in commits.drain(..) + { + let count = batches.len(); ++ crate::stage_trace::mark( ++ Arc::as_ptr(&*buffer) as usize, ++ crate::stage_trace::Stage::Taken, ++ count as u32, ++ ); + let State { order, binary, .. } = &mut *state; + let held = batches.into_iter().enumerate().map(|(index, batch)| Held { + buffer: buffer.hold(), +@@ -1475,6 +1508,7 @@ fn take_in(state: &mut State, commits: &mut Vec, spent: &mut Spent) - + hastened: false, + counted_held: false, + held_since: None, ++ traced_reasons: 0, + }); + order.push(queue, held); + spent.push(buffer); +diff --git a/crates/metal2vulkan-runtime/src/scheduler/steps.rs b/crates/metal2vulkan-runtime/src/scheduler/steps.rs +index 6ac057360..52b9f9a7f 100644 +--- a/crates/metal2vulkan-runtime/src/scheduler/steps.rs ++++ b/crates/metal2vulkan-runtime/src/scheduler/steps.rs +@@ -138,7 +138,10 @@ fn record(shared: &SchedulerShared, work: Work) -> bool { + }; + let (buffer, index) = &work; + let started = Instant::now(); ++ let id = Arc::as_ptr(&**buffer) as usize; ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::StepBegin, *index as u32); + buffer.host_step(*index); ++ crate::stage_trace::mark(id, crate::stage_trace::Stage::StepEnd, *index as u32); + for &call in exit_guard::HOST_STEP { + exit_guard::called(call); + } +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +index 507a38b00..dba2239b8 100644 +--- a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs ++++ b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +@@ -71,6 +71,7 @@ pub(crate) use creation::VariantStages; + pub(crate) mod driver; + mod held; + pub(super) mod store; ++pub(crate) mod system; + pub(super) mod worker; + + use super::error::{invalid, Error, Problem, ShaderCacheError}; +@@ -386,6 +387,103 @@ impl CacheSetup { + } + } + ++/// Disk access for entry-point priming, before any exit hook is registered. ++/// Deliberately no worker, exit-guard section, or ordinary driver::make call. ++/// Priming keeps its raw call/cleanup ordering and records every entry point. ++pub(super) struct PrimingCache { ++ store: Store, ++ identity: driver::Identity, ++ name: String, ++ loaded: Option>, ++} ++ ++impl CacheSetup { ++ pub(super) fn priming( ++ &self, ++ properties: &vk::PhysicalDeviceProperties, ++ profile: Hash256, ++ settings: Hash256, ++ key: Hash256, ++ sink: Option, ++ ) -> PrimingCache { ++ let identity = driver::Identity::of(properties); ++ let records = Records { ++ identity, ++ settings, ++ profile, ++ }; ++ let name = records.name(&records.key(Making::new("device-entry-point-priming").digest(&key))); ++ PrimingCache { ++ store: Store::new(self.directory.clone(), sink, None), ++ identity, ++ name, ++ loaded: None, ++ } ++ } ++} ++ ++impl PrimingCache { ++ /// The caller owns the raw cache and primes/releases its entry points. ++ /// No exit_guard::called/enter may happen here, before priming is finished. ++ pub(super) unsafe fn open( ++ &mut self, ++ raw: &ash::Device, ++ word: &crate::loader_word::DeviceWord, ++ ) -> Result { ++ let seed = match self.store.load_content(&self.name, |bytes| { ++ let data = driver::open(&self.identity, &bytes)?; ++ Some(bytes[data].to_vec()) ++ }) { ++ Lookup::Hit(bytes) => Some(bytes), ++ _ => None, ++ }; ++ if let Some(bytes) = seed { ++ let info = vk::PipelineCacheCreateInfo::default().initial_data(&bytes); ++ // SAFETY: the caller owns the live device and the info's bytes outlive the call. ++ if let Ok(cache) = ++ unsafe { word.restored(|| raw.create_pipeline_cache(&info, None)) } ++ { ++ self.loaded = Some(bytes); ++ return Ok(cache); ++ } ++ } ++ // Invalid/absent/refused data is never treated as a successful load. ++ // SAFETY: as above; the empty create info outlives the call. ++ unsafe { ++ word.restored(|| { ++ raw.create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) ++ }) ++ } ++ } ++ ++ /// Called after all priming (including graphics/generated helpers) succeeds, ++ /// before its raw cache is destroyed and before exit-hook registration. ++ pub(super) unsafe fn keep( ++ &self, ++ raw: &ash::Device, ++ word: &crate::loader_word::DeviceWord, ++ cache: vk::PipelineCache, ++ ) { ++ // SAFETY: priming owns the live device/cache and every making has returned. ++ let Ok(data) = (unsafe { word.restored(|| raw.get_pipeline_cache_data(cache)) }) else { ++ return; ++ }; ++ if self.loaded.as_deref() == Some(data.as_slice()) { ++ return; ++ } ++ if self.identity.check_vulkan_header(&data).is_err() { ++ return; ++ } ++ if self ++ .store ++ .write_record_soon(&self.name, driver::seal(&self.identity, &data)) ++ .is_err() ++ { ++ self.store.write_failed(); ++ } ++ } ++} ++ + /// One device's shader cache: its memo, its view of the directory, what names its makings' + /// records, the `VkPipelineCache` it keeps for each record, and the cache thread. + pub(super) struct ShaderCache { +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/store.rs b/crates/metal2vulkan-runtime/src/shader_cache/store.rs +index 5bccf66fa..3052b5b0b 100644 +--- a/crates/metal2vulkan-runtime/src/shader_cache/store.rs ++++ b/crates/metal2vulkan-runtime/src/shader_cache/store.rs +@@ -76,30 +76,47 @@ pub(crate) struct Directory { + root: PathBuf, + build: Hash256, + capacity: u64, ++ /// Its directories and entries are made readable by everyone (`system.rs`): the directory ++ /// of the process that fills the system layer. ++ public: bool, + } + + impl Directory { + /// Open `root` for `build` with a cap of `capacity` bytes: create it and the build's + /// directory (owner-only) and write and remove a probe file there. The error names the + /// directory that failed. ++ /// ++ /// With `METAL2VULKAN_SHADER_CACHE_OPTIONAL` in the environment a directory that cannot be ++ /// made or written opens anyway: a process that may write nowhere (a sandboxed one, as ++ /// WebKit's GPU process) still gets its device, and every read and write of the cache is a ++ /// miss, as after any later failure. + pub(crate) fn open(root: PathBuf, build: Hash256, capacity: u64) -> Result { + let directory = Directory { + root, + build, + capacity, ++ public: super::system::cache_readable_by_all(), + }; + let failed = |path: &Path| { + let path = path.to_path_buf(); + move |error| Failed { path, error } + }; +- create_private_dir_all(&directory.root).map_err(failed(&directory.root))?; +- let build_dir = directory.build_dir(); +- create_private_dir_all(&build_dir).map_err(failed(&build_dir))?; +- let probe = build_dir.join(format!(".probe.{}.tmp", unique())); +- let written = new_private_file(&probe).and_then(|mut file| file.write_all(b"probe")); +- let removed = fs::remove_file(&probe); +- written.and(removed).map_err(failed(&directory.root))?; +- Ok(directory) ++ let made = (|| { ++ directory.create_dir_all(&directory.root).map_err(failed(&directory.root))?; ++ let layout_dir = directory.layout_dir(); ++ directory.create_dir_all(&layout_dir).map_err(failed(&layout_dir))?; ++ let build_dir = directory.build_dir(); ++ directory.create_dir_all(&build_dir).map_err(failed(&build_dir))?; ++ let probe = build_dir.join(format!(".probe.{}.tmp", unique())); ++ let written = directory.new_file(&probe).and_then(|mut file| file.write_all(b"probe")); ++ let removed = fs::remove_file(&probe); ++ written.and(removed).map_err(failed(&directory.root)) ++ })(); ++ match made { ++ Err(_) if std::env::var_os("METAL2VULKAN_SHADER_CACHE_OPTIONAL").is_some() => Ok(directory), ++ Err(failed) => Err(failed), ++ Ok(()) => Ok(directory), ++ } + } + + /// The directory the consumer named or the platform's default. +@@ -122,6 +139,41 @@ impl Directory { + self.layout_dir().join(hex(&self.build)) + } + ++ /// Create `dir` and its missing parents: each only its owner may enter, or, in a directory ++ /// made for every reader (`system.rs`), everyone may read and enter, whatever the process's ++ /// file mode mask says. ++ fn create_dir_all(&self, dir: &Path) -> io::Result<()> { ++ if !self.public { ++ return create_private_dir_all(dir); ++ } ++ let mut builder = fs::DirBuilder::new(); ++ builder.recursive(true); ++ #[cfg(unix)] ++ { ++ use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; ++ builder.mode(0o755); ++ builder.create(dir)?; ++ if fs::metadata(dir)?.permissions().mode() & 0o777 != 0o755 { ++ fs::set_permissions(dir, fs::Permissions::from_mode(0o755))?; ++ } ++ Ok(()) ++ } ++ #[cfg(not(unix))] ++ builder.create(dir) ++ } ++ ++ /// Create the new file `path`, which only its owner may read, or everyone in a directory ++ /// made for every reader; fails if it exists. ++ fn new_file(&self, path: &Path) -> io::Result { ++ let file = new_private_file(path)?; ++ #[cfg(unix)] ++ if self.public { ++ use std::os::unix::fs::PermissionsExt; ++ file.set_permissions(fs::Permissions::from_mode(0o644))?; ++ } ++ Ok(file) ++ } ++ + /// The path of the entry named `name`. + pub(crate) fn entry_path(&self, name: &str) -> PathBuf { + self.build_dir().join(&name[..2]).join(name) +@@ -193,6 +245,9 @@ enum Found { + Bytes(Vec, fs::File), + /// The entry's bytes, from a write the cache thread has not made yet: no file to refresh. + Queued(Vec), ++ /// The entry's bytes, from the system layer (`system.rs`): not this directory's to refresh ++ /// or remove. ++ System(Vec), + } + + /// The counts behind [`ShaderCacheStatistics`](super::ShaderCacheStatistics). +@@ -328,11 +383,16 @@ pub(crate) struct Store { + used: AtomicU64, + /// A scan has set `used`; before that the cap is not judged. + scanned: AtomicBool, ++ /// The platform refused this process an entry's access time ([`Store::accepted`]): its hits ++ /// ask for none from then on. ++ touch_refused: AtomicBool, + /// The warnings said once each ([`Warnings`]). + warnings: Warnings, + /// The device's cache thread, which writes the entries this store is given; `None` in a + /// store with no thread, which writes them on the calling thread. + asked: Option>, ++ /// The system layer this store asks for what its directory lacks (`system.rs`). ++ system: Option>, + } + + /// What a device's store says once each, through its instance's message sink, else on stderr: +@@ -362,15 +422,24 @@ impl Store { + counters: Counters::default(), + used: AtomicU64::new(0), + scanned: AtomicBool::new(false), ++ touch_refused: AtomicBool::new(false), + warnings: Warnings { + sink, + evicted: AtomicBool::new(false), + write_failed: AtomicBool::new(false), + }, + asked, ++ system: super::system::system_layer(), + } + } + ++ /// The system layer's bytes of the file this directory would hold at `path`. ++ fn system_read(&self, path: &Path) -> Option> { ++ let system = self.system.as_ref()?; ++ let relative = path.strip_prefix(self.directory.root()).ok()?; ++ system.read(relative, self.entry_limit()) ++ } ++ + /// Count an entry not written (`ShaderCacheStatistics::write_failures`); the first is + /// announced as [`Degradation::ShaderCacheEntryNotWritten`], naming the directory and its + /// cap. Never an error: the work the entry would have saved is the directory's regardless. +@@ -460,6 +529,10 @@ impl Store { + count(&self.counters.holds_questions); + timed(&self.counters.read_nanos, || { + fs::metadata(path).is_ok_and(|metadata| metadata.len() <= self.entry_limit()) ++ || self.system.as_ref().is_some_and(|system| { ++ path.strip_prefix(self.directory.root()) ++ .is_ok_and(|relative| system.holds(relative, self.entry_limit())) ++ }) + }) + } + +@@ -499,6 +572,10 @@ impl Store { + } + Found::Bytes(bytes, file) => (bytes, Some(file)), + Found::Queued(bytes) => (bytes, None), ++ Found::System(bytes) => { ++ let length = bytes.len() as u64; ++ return self.system_verdict(decode(bytes), length); ++ } + }; + match decode(bytes) { + Some(value) => { +@@ -512,6 +589,23 @@ impl Store { + } + } + ++ /// What a decoder made of `length` bytes the system layer gave: counted there, and a ++ /// refusal in this store's rejections too. Nothing is touched or removed. ++ fn system_verdict(&self, decoded: Option, length: u64) -> Lookup { ++ let system = self.system.as_ref().expect("bytes of the system layer"); ++ match decoded { ++ Some(value) => { ++ system.accepted(length); ++ Lookup::Hit(value) ++ } ++ None => { ++ system.rejected(); ++ count(&self.counters.rejected); ++ Lookup::Rejected ++ } ++ } ++ } ++ + /// [`Store::load`] of the file at `path`. + fn load_at(&self, path: &Path, decode: impl FnOnce(&[u8]) -> Result) -> Lookup { + let (bytes, file) = match self.read(path) { +@@ -522,6 +616,9 @@ impl Store { + } + Found::Bytes(bytes, file) => (bytes, Some(file)), + Found::Queued(bytes) => (bytes, None), ++ Found::System(bytes) => { ++ return self.system_verdict(decode(&bytes).ok(), bytes.len() as u64); ++ } + }; + match decode(&bytes) { + Ok(value) => { +@@ -537,9 +634,19 @@ impl Store { + + /// The contents a decoder accepted: a hit from `file` makes its entry the most recently used; + /// a hit from the queue (no file) is counted as one the directory had not written yet. ++ /// ++ /// A process whose sandbox lets it read and add entries may still be refused their ++ /// attributes: macOS's `iconservicesagent` was, at every hit, and each refusal is a sandbox ++ /// violation the kernel logs (39133 of a boot's 45648). The first such refusal is the last ++ /// asked: the entries this process hits keep the access time their readers leave them. + fn accepted(&self, file: Option<&fs::File>) { + match file { +- Some(file) => timed(&self.counters.read_nanos, || touch(file)), ++ Some(_) if self.touch_refused.load(Ordering::Relaxed) => {} ++ Some(file) => { ++ if timed(&self.counters.read_nanos, || touch(file)) { ++ self.touch_refused.store(true, Ordering::Relaxed); ++ } ++ } + None => count(&self.counters.queued_hits), + } + } +@@ -570,7 +677,19 @@ impl Store { + return Found::Queued(bytes); + } + count(&self.counters.entry_reads); +- timed(&self.counters.read_nanos, || { ++ timed(&self.counters.read_nanos, || match self.read_own(path) { ++ // What this directory lacks or cannot give, the system layer may hold. ++ found @ (Found::Absent | Found::Failed) => match self.system_read(path) { ++ Some(bytes) => Found::System(bytes), ++ None => found, ++ }, ++ found => found, ++ }) ++ } ++ ++ /// [`Store::read`] of this directory's own file at `path`. ++ fn read_own(&self, path: &Path) -> Found { ++ { + let file = match fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Found::Absent, +@@ -593,7 +712,7 @@ impl Store { + } + Err(_) => Found::Failed, + } +- }) ++ } + } + + /// Give `bytes` to the cache thread to write as the entry named `name` (`worker.rs`): the +@@ -654,10 +773,10 @@ impl Store { + return Err(io::Error::other("the entry is larger than the cache takes")); + } + let parent = path.parent().expect("an entry has a directory"); +- create_private_dir_all(parent)?; ++ self.directory.create_dir_all(parent)?; + let temporary = parent.join(format!(".{name}.{}.tmp", unique())); + let written = (|| { +- let mut file = new_private_file(&temporary)?; ++ let mut file = self.directory.new_file(&temporary)?; + file.write_all(bytes)?; + drop(file); + fs::rename(&temporary, path)?; +@@ -849,8 +968,10 @@ fn last_use(metadata: &fs::Metadata) -> SystemTime { + } + + /// Refresh `file`'s access time: a hit makes its entry the most recently used. Best effort. +-fn touch(file: &fs::File) { +- let _ = file.set_times(fs::FileTimes::new().set_accessed(SystemTime::now())); ++/// Whether the platform refused this process the change (`EPERM`, `EACCES`). ++fn touch(file: &fs::File) -> bool { ++ let set = file.set_times(fs::FileTimes::new().set_accessed(SystemTime::now())); ++ set.is_err_and(|error| error.kind() == io::ErrorKind::PermissionDenied) + } + + /// A name no other writer uses: this process's id, a per-process sequence number, and the time. +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/system.rs b/crates/metal2vulkan-runtime/src/shader_cache/system.rs +new file mode 100644 +index 000000000..c6e5cd185 +--- /dev/null ++++ b/crates/metal2vulkan-runtime/src/shader_cache/system.rs +@@ -0,0 +1,268 @@ ++//! The system layer: a second cache directory, read only, that one trusted writer fills for every ++//! process of the machine (the consumer's render server primes the OS's Metal libraries into it). ++//! ++//! - **Laid out as a device's own directory** (`store.rs`): the same `v1///` ++//! entries and content records, so what the writer's device wrote there under its own rules is ++//! what a reader's device looks up. A lookup asks the device's own directory first and this one ++//! only for what that lacks: a record the process added to (a making's driver data) is its own ++//! from then on. ++//! - **Trusted by who may write it, at every level opened.** The directory, each directory ++//! under it on the way to an entry, and the entry are opened without following a link ++//! (`O_NOFOLLOW`, relative to the level above) and judged by `fstat`: a directory or a regular ++//! file, owned by one of the owners the consumer named, that neither its group nor others may ++//! write. The directories above the layer are judged by `lstat` when it is opened: no link, ++//! owned by the superuser or one of the owners, and written by others only where the sticky bit ++//! keeps them from replacing what is not theirs. An entry's envelope carries a checksum, not a ++//! signature, and the driver's pipeline data is parsed by the driver as it is: a process must ++//! therefore never read an entry that anyone but those owners could have put there, and a ++//! level that fails the test is refused and counted, never read. ++//! - **Never written, touched or removed from here.** A hit refreshes no access time and ++//! contents a decoder refuses stay (counted): the directory is its writer's. ++//! - **Process-wide.** [`set_system_layer`] names the layer for the devices made after it; ++//! [`set_cache_readable_by_all`] makes the directories and entries a device writes readable by ++//! everyone (`0755`, `0644`) instead of by their owner alone, for the process that fills the ++//! layer through its own directory. ++ ++use std::ffi::CString; ++use std::io::Read; ++use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; ++use std::os::unix::ffi::OsStrExt; ++use std::path::{Component, Path, PathBuf}; ++use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; ++use std::sync::{Arc, RwLock}; ++ ++/// What a system layer did for this process ([`SystemLayer::counts`]). ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++pub struct SystemLayerCounts { ++ /// Entries and records read from the layer and accepted by their decoder: work not done ++ /// in this process. ++ pub accepted: u64, ++ /// Entries read from the layer whose contents the decoder refused (left in place). ++ pub rejected: u64, ++ /// Levels (a directory or an entry) refused for their kind, owner or mode: never read. ++ pub refused: u64, ++ /// Bytes of the accepted entries. ++ pub accepted_bytes: u64, ++} ++ ++/// An opened system layer. ++#[derive(Debug)] ++pub struct SystemLayer { ++ root: OwnedFd, ++ path: PathBuf, ++ owners: Vec, ++ accepted: AtomicU64, ++ rejected: AtomicU64, ++ refused: AtomicU64, ++ accepted_bytes: AtomicU64, ++} ++ ++static LAYER: RwLock>> = RwLock::new(None); ++static READABLE_BY_ALL: AtomicBool = AtomicBool::new(false); ++ ++/// Name the system layer the devices made from now on consult, or none. ++pub fn set_system_layer(layer: Option>) { ++ *LAYER.write().unwrap_or_else(|e| e.into_inner()) = layer; ++} ++ ++/// The system layer named last. ++pub fn system_layer() -> Option> { ++ LAYER.read().unwrap_or_else(|e| e.into_inner()).clone() ++} ++ ++/// Whether the cache directories opened from now on make their directories and entries readable ++/// by everyone. ++pub fn set_cache_readable_by_all(on: bool) { ++ READABLE_BY_ALL.store(on, Ordering::Relaxed); ++} ++ ++pub(super) fn cache_readable_by_all() -> bool { ++ READABLE_BY_ALL.load(Ordering::Relaxed) ++} ++ ++fn c_path(bytes: &[u8]) -> Option { ++ CString::new(bytes).ok() ++} ++ ++fn fstat(fd: &OwnedFd) -> Option { ++ // SAFETY: a zeroed stat is a valid out value and the descriptor is open. ++ unsafe { ++ let mut st: libc::stat = std::mem::zeroed(); ++ (libc::fstat(fd.as_raw_fd(), &mut st) == 0).then_some(st) ++ } ++} ++ ++/// Open `name` under `parent` (or by its path where `parent` is `None`), never through a link. ++fn open_level(parent: Option<&OwnedFd>, name: &[u8], directory: bool) -> Result { ++ let name = c_path(name).ok_or(libc::EINVAL)?; ++ let kind = if directory { libc::O_DIRECTORY } else { libc::O_NONBLOCK }; ++ let flags = libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC | kind; ++ // SAFETY: the name is a C string and the parent descriptor is open. ++ let fd = unsafe { ++ match parent { ++ Some(parent) => libc::openat(parent.as_raw_fd(), name.as_ptr(), flags), ++ None => libc::open(name.as_ptr(), flags), ++ } ++ }; ++ if fd < 0 { ++ return Err(std::io::Error::last_os_error().raw_os_error().unwrap_or(libc::EIO)); ++ } ++ // SAFETY: a descriptor this call opened and nothing else owns. ++ Ok(unsafe { OwnedFd::from_raw_fd(fd) }) ++} ++ ++impl SystemLayer { ++ /// Open the layer at `root`, whose every level must be owned by one of `owners`. The error ++ /// says which level failed and how. ++ pub fn open(root: &Path, owners: &[u32]) -> Result { ++ if !root.is_absolute() || owners.is_empty() { ++ return Err("the layer needs an absolute path and an owner".into()); ++ } ++ // The directories above the layer: no link among them, each the superuser's or an ++ // owner's, written by others only under the sticky bit. A level the sandbox lets this ++ // process see nothing of (EPERM, which no file mode gives) is one the OS made. ++ let mut above = PathBuf::new(); ++ let mut parts = root.components().peekable(); ++ while let Some(part) = parts.next() { ++ match part { ++ Component::RootDir | Component::Normal(_) => above.push(part), ++ _ => return Err(format!("{}: not a plain path", root.display())), ++ } ++ if parts.peek().is_none() { ++ break; ++ } ++ let path = c_path(above.as_os_str().as_bytes()).ok_or("a path with a zero byte")?; ++ // SAFETY: a zeroed stat is a valid out value and the path is a C string. ++ let (result, st) = unsafe { ++ let mut st: libc::stat = std::mem::zeroed(); ++ (libc::lstat(path.as_ptr(), &mut st), st) ++ }; ++ if result != 0 { ++ let error = std::io::Error::last_os_error(); ++ if error.raw_os_error() == Some(libc::EPERM) { ++ continue; ++ } ++ return Err(format!("{}: {error}", above.display())); ++ } ++ let directory = st.st_mode & libc::S_IFMT == libc::S_IFDIR; ++ let owned = st.st_uid == 0 || owners.contains(&st.st_uid); ++ let closed = st.st_mode & 0o022 == 0 || st.st_mode & libc::S_ISVTX != 0; ++ if !directory || !owned || !closed { ++ return Err(format!( ++ "{}: above the layer, uid {} mode {:o}", ++ above.display(), ++ st.st_uid, ++ st.st_mode ++ )); ++ } ++ } ++ let fd = open_level(None, root.as_os_str().as_bytes(), true) ++ .map_err(|errno| format!("{}: errno {errno}", root.display()))?; ++ let layer = SystemLayer { ++ root: fd, ++ path: root.to_path_buf(), ++ owners: owners.to_vec(), ++ accepted: AtomicU64::new(0), ++ rejected: AtomicU64::new(0), ++ refused: AtomicU64::new(0), ++ accepted_bytes: AtomicU64::new(0), ++ }; ++ match fstat(&layer.root) { ++ Some(st) if layer.trusted(&st, true) => Ok(layer), ++ Some(st) => Err(format!( ++ "{}: uid {} mode {:o}", ++ root.display(), ++ st.st_uid, ++ st.st_mode ++ )), ++ None => Err(format!("{}: no fstat", root.display())), ++ } ++ } ++ ++ /// The layer's directory. ++ pub fn path(&self) -> &Path { ++ &self.path ++ } ++ ++ /// Whether a level of `st` is what the layer reads: a directory or a regular file, an ++ /// owner's, that neither its group nor others may write. ++ fn trusted(&self, st: &libc::stat, directory: bool) -> bool { ++ let kind = if directory { libc::S_IFDIR } else { libc::S_IFREG }; ++ st.st_mode & libc::S_IFMT == kind && self.owners.contains(&st.st_uid) && st.st_mode & 0o022 == 0 ++ } ++ ++ /// The open file of `relative` under the layer and its length, every level on the way judged. ++ /// `None` where a level is absent or refused (refusals are counted). ++ fn entry(&self, relative: &Path) -> Option<(std::fs::File, u64)> { ++ let mut parts = relative.components().peekable(); ++ let mut level: Option = None; ++ while let Some(part) = parts.next() { ++ let Component::Normal(name) = part else { ++ return None; ++ }; ++ let last = parts.peek().is_none(); ++ let parent = level.as_ref().unwrap_or(&self.root); ++ let fd = match open_level(Some(parent), name.as_bytes(), !last) { ++ Ok(fd) => fd, ++ Err(errno) => { ++ // A link in an entry's place (ELOOP), or a file where a directory belongs. ++ if errno == libc::ELOOP || errno == libc::ENOTDIR { ++ self.refused.fetch_add(1, Ordering::Relaxed); ++ } ++ return None; ++ } ++ }; ++ let st = fstat(&fd)?; ++ if !self.trusted(&st, !last) { ++ self.refused.fetch_add(1, Ordering::Relaxed); ++ return None; ++ } ++ if last { ++ return Some((std::fs::File::from(fd), st.st_size as u64)); ++ } ++ level = Some(fd); ++ } ++ None ++ } ++ ++ /// Whether the layer holds `relative`, of at most `limit` bytes. ++ pub fn holds(&self, relative: &Path, limit: u64) -> bool { ++ self.entry(relative).is_some_and(|(_, length)| length <= limit) ++ } ++ ++ /// The bytes of `relative` under the layer, at most `limit`; `None` where it is absent, ++ /// refused, longer or unreadable. The caller's decoder judges the contents and reports the ++ /// verdict ([`accepted`](Self::accepted), [`rejected`](Self::rejected)). ++ pub fn read(&self, relative: &Path, limit: u64) -> Option> { ++ let (file, length) = self.entry(relative)?; ++ if length > limit { ++ return None; ++ } ++ let mut bytes = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); ++ match file.take(limit + 1).read_to_end(&mut bytes) { ++ Ok(_) if bytes.len() as u64 <= limit => Some(bytes), ++ _ => None, ++ } ++ } ++ ++ /// Count an entry of `bytes` bytes read from the layer that its decoder accepted. ++ pub fn accepted(&self, bytes: u64) { ++ self.accepted.fetch_add(1, Ordering::Relaxed); ++ self.accepted_bytes.fetch_add(bytes, Ordering::Relaxed); ++ } ++ ++ /// Count an entry read from the layer that its decoder refused. ++ pub fn rejected(&self) { ++ self.rejected.fetch_add(1, Ordering::Relaxed); ++ } ++ ++ /// The counts so far. ++ pub fn counts(&self) -> SystemLayerCounts { ++ SystemLayerCounts { ++ accepted: self.accepted.load(Ordering::Relaxed), ++ rejected: self.rejected.load(Ordering::Relaxed), ++ refused: self.refused.load(Ordering::Relaxed), ++ accepted_bytes: self.accepted_bytes.load(Ordering::Relaxed), ++ } ++ } ++} +diff --git a/crates/metal2vulkan-runtime/src/shadow.rs b/crates/metal2vulkan-runtime/src/shadow.rs +index a7046af05..4e478c8d9 100644 +--- a/crates/metal2vulkan-runtime/src/shadow.rs ++++ b/crates/metal2vulkan-runtime/src/shadow.rs +@@ -394,6 +394,8 @@ pub(super) enum ShadowSource { + conversion: Box, + /// The texture's format, which a refusal names. + pixel_format: PixelFormat, ++ /// The image counted alive while these copies live (`texture/ycbcr_image.rs`). ++ _alive: crate::texture::ImageAlive, + }, + /// A surface's two planes in a buffer's bytes, into which the texels a function writes to a + /// 4:4:4 two-plane YCbCr texture are stored (`ycbcr.rs`): the shadow is an image of the +diff --git a/crates/metal2vulkan-runtime/src/stage_trace.rs b/crates/metal2vulkan-runtime/src/stage_trace.rs +new file mode 100644 +index 000000000..3325dd04c +--- /dev/null ++++ b/crates/metal2vulkan-runtime/src/stage_trace.rs +@@ -0,0 +1,246 @@ ++//! A process-wide ring of the stages each command buffer passes from its commit to its completion, ++//! for a host that asks where the time between them goes ([`read`]): when it was committed, left ++//! to the lowering thread, run, recorded, handed to the scheduler and taken in, what held its ++//! batches, when each was submitted, and when the completion waiter armed, saw it and finished it. ++//! The scheduler's and the waiters' own sleeps and wakes are recorded under id 0 and the queue. ++//! ++//! - **Cheap.** A record is one atomic add and three stores; nothing allocates or locks. The ring ++//! keeps the newest [`RING`] records; a reader that falls behind loses the oldest. ++//! - **One clock.** The host's uptime clock in nanoseconds (`CLOCK_UPTIME_RAW` on Apple ++//! platforms), the one the frontend's trace and the driver's rings use. ++//! - **Off unless asked for.** Records are kept only where the host's environment has ++//! `SHENG_DIAG` ([`diagnostics`]), read once; the same switch keeps the allocation sites ++//! (`creation.rs`). Without it a record costs one relaxed load. ++//! - **Ids.** A command buffer's id is the address of its shared state, alive from its commit to ++//! its completion: records of one id between a commit stage and its completion are one command ++//! buffer's. ++ ++use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; ++ ++/// The records the ring keeps. ++const RING: usize = 1 << 16; ++ ++/// What happened, the record's `stage`. ++#[derive(Clone, Copy, Debug, PartialEq, Eq)] ++#[repr(u16)] ++pub enum Stage { ++ /// `commit` was called (`arg`: 0). ++ CommitCall = 1, ++ /// The commit was left to wait for the lowering thread's end of its last render encoder. ++ CommitQueued = 2, ++ /// The commit's own work began (`arg`: 0 at the call, 1 queued, 2 refused). ++ CommitRun = 3, ++ /// Its segments were recorded or prepared for their host steps (`arg`: segments). ++ Prepared = 4, ++ /// Its batches were handed to the scheduler (`arg`: batches). ++ Handed = 5, ++ /// The scheduler took its batches into its order (`arg`: batches). ++ Taken = 6, ++ /// A look of the scheduler's held one of its batches (`arg`: the reason, `HeldFor` + 1, | ++ /// batch index << 8); recorded once per reason and batch. ++ Held = 7, ++ /// A batch's submission began (`arg`: batch index | last << 15). ++ SubmitBegin = 8, ++ /// A batch's submission returned (`arg`: 0 submitted, 1 grouped, 2 later, 3 after the group, ++ /// 4 failed). ++ SubmitEnd = 9, ++ /// Its host step began recording on the scheduler's thread (`arg`: batch index). ++ StepBegin = 10, ++ /// Its host step ended (`arg`: batch index). ++ StepEnd = 11, ++ /// A completion waiter began waiting for its batch (`arg`: queue | slept first << 8). ++ PollArmed = 12, ++ /// The waiter's wait for its batch returned (`arg`: queue). ++ PollDone = 13, ++ /// A completion turn saw it complete (`arg`: queue). ++ Seen = 14, ++ /// Its completion ran (`arg`: 1 on the waiter, 2 posted to its queue's context). ++ Finished = 15, ++ /// A step inside a batch's submission ended (`arg`: 1 the scrubs and segments placed, 2 its ++ /// waits and signals gathered, 3 the driver's `vkQueueSubmit` returned, 4 the submission's ++ /// changes made). ++ SubmitStep = 16, ++ /// The scheduler's thread went to sleep (id 0, `arg`: slice in ms, | 0x8000 on the GPU). ++ SchedulerSleep = 20, ++ /// A memory operation began (`arg`: 1 foreground placement, 2 room keeper, 3 release). ++ MemoryBegin = 40, ++ /// Its pool (`arg`: memory type * 2 + 1 for images, + 0 for buffers). ++ MemoryPool = 41, ++ /// Its requested backing bytes in KiB. ++ MemoryBytes = 42, ++ /// Its pool's available contiguous range in KiB at the decision. ++ MemoryAvailable = 43, ++ /// Its Vulkan allocation ended (`arg`: 1 success, 0 refused). ++ MemoryAllocated = 44, ++ /// Existing backing was reused (`arg`: 1 pooled placement, 2 spare block). ++ MemoryReused = 45, ++ /// Its memory handle's low 32 bits. ++ MemoryHandleLow = 46, ++ /// Its memory handle's high 32 bits. ++ MemoryHandleHigh = 47, ++ /// Its backing was retained for reuse (`arg`: total spare bytes in KiB). ++ MemoryCached = 48, ++ /// Its physical release was posted to the device's free thread (`arg`: bytes in KiB). ++ MemoryDeferred = 49, ++ /// Its physical release began. ++ MemoryFreeBegin = 50, ++ /// Its physical release ended. ++ MemoryFreeEnd = 51, ++ /// A release ran inline because the bounded queue was full. ++ MemoryQueueFull = 52, ++ /// Completion processing began. ++ CompleteBegin = 53, ++ /// Completion state and resource ownership were collected. ++ CompleteState = 54, ++ /// User completion handlers returned. ++ CompleteHandlers = 55, ++ /// Retained objects and runtime resources were released. ++ CompleteRetained = 56, ++ /// Completion gate and command-pool preparation ended. ++ CompleteEnd = 57, ++ ++ /// Something woke the scheduler's thread (id 0, `arg`: 0 commit, 1 completion, 2 host). ++ SchedulerWake = 21, ++ /// The scheduler's thread resumed after its sleep (id 0, `arg`: 1 timed out). ++ SchedulerResumed = 22, ++ /// The scheduler's thread began a pass (id 0). ++ PassBegin = 23, ++ /// The scheduler's thread ended a pass (id 0, `arg`: batches submitted, capped). ++ PassEnd = 24, ++} ++ ++struct Slot { ++ when: AtomicU64, ++ id: AtomicU64, ++ /// stage | arg << 16 | sequence << 32: a reader takes the slot only where the sequence is the ++ /// one it asked for. ++ word: AtomicU64, ++} ++ ++#[allow(clippy::declare_interior_mutable_const)] ++const EMPTY: Slot = Slot { ++ when: AtomicU64::new(0), ++ id: AtomicU64::new(0), ++ word: AtomicU64::new(0), ++}; ++static SLOTS: [Slot; RING] = [EMPTY; RING]; ++static NEXT: AtomicU64 = AtomicU64::new(1); ++ ++/// The host's uptime clock in nanoseconds. ++pub fn now_nanos() -> u64 { ++ #[cfg(any(target_os = "macos", target_os = "ios"))] ++ { ++ let mut spec = libc::timespec { ++ tv_sec: 0, ++ tv_nsec: 0, ++ }; ++ // SAFETY: a valid clock id and an owned timespec. ++ if unsafe { libc::clock_gettime(libc::CLOCK_UPTIME_RAW, &mut spec) } == 0 { ++ return spec.tv_sec as u64 * 1_000_000_000 + spec.tv_nsec as u64; ++ } ++ 0 ++ } ++ #[cfg(not(any(target_os = "macos", target_os = "ios")))] ++ { ++ crate::gpu_time::host_nanos() ++ } ++} ++ ++/// Whether the host asked for the runtime's diagnostics: `SHENG_DIAG` in the environment, read ++/// once. ++pub(crate) fn diagnostics() -> bool { ++ // 0 not read yet, 1 off, 2 on. ++ static ON: AtomicU8 = AtomicU8::new(0); ++ match ON.load(Ordering::Relaxed) { ++ 0 => { ++ let on = std::env::var_os("SHENG_DIAG").is_some(); ++ ON.store(if on { 2 } else { 1 }, Ordering::Relaxed); ++ on ++ } ++ state => state == 2, ++ } ++} ++ ++/// Record `stage` of `id` with `arg`, now, where the diagnostics are on ([`diagnostics`]). ++pub(crate) fn mark(id: usize, stage: Stage, arg: u32) { ++ // Per-command handoff and completion stages retain scheduler latency. ++ // Idle scheduler turns have no command identity and add no causal link. ++ if matches!(stage, Stage::SchedulerSleep | Stage::SchedulerWake ++ | Stage::SchedulerResumed | Stage::PassBegin | Stage::PassEnd) { ++ return; ++ } ++ if !diagnostics() { ++ return; ++ } ++ if let Some(hook) = DIAGNOSTIC_HOOK.get() { ++ hook(39, id as u64, stage as u64, arg as u64); ++ } ++ let sequence = NEXT.fetch_add(1, Ordering::Relaxed); ++ let slot = &SLOTS[(sequence as usize) & (RING - 1)]; ++ slot.word.store(0, Ordering::Release); ++ slot.when.store(now_nanos(), Ordering::Relaxed); ++ slot.id.store(id as u64, Ordering::Relaxed); ++ let word = stage as u64 | ((arg as u64 & 0xffff) << 16) | ((sequence & 0xffff_ffff) << 32); ++ slot.word.store(word, Ordering::Release); ++} ++ ++/// One record as [`read`] gives it. ++#[derive(Clone, Copy, Debug, Default)] ++#[repr(C)] ++pub struct StageRecord { ++ /// Host uptime nanoseconds. ++ pub when: u64, ++ /// The command buffer's id (0: the scheduler's thread). ++ pub id: u64, ++ /// Its place among all records. ++ pub sequence: u32, ++ /// [`Stage`] as its number. ++ pub stage: u16, ++ /// The stage's argument. ++ pub arg: u16, ++} ++ ++/// The records from sequence `from` on (the oldest kept where `from` fell out of the ring), at most ++/// `out.len()`, oldest first: how many, and the sequence to ask for next. ++pub fn read(from: u64, out: &mut [StageRecord]) -> (usize, u64) { ++ let newest = NEXT.load(Ordering::Acquire); ++ let mut sequence = from.max(1).max(newest.saturating_sub(RING as u64 - 1024)); ++ let mut written = 0; ++ while sequence < newest && written < out.len() { ++ let slot = &SLOTS[(sequence as usize) & (RING - 1)]; ++ let word = slot.word.load(Ordering::Acquire); ++ if word >> 32 == sequence & 0xffff_ffff && word & 0xffff != 0 { ++ let when = slot.when.load(Ordering::Relaxed); ++ let id = slot.id.load(Ordering::Relaxed); ++ if slot.word.load(Ordering::Acquire) == word { ++ out[written] = StageRecord { ++ when, ++ id, ++ sequence: sequence as u32, ++ stage: (word & 0xffff) as u16, ++ arg: ((word >> 16) & 0xffff) as u16, ++ }; ++ written += 1; ++ } ++ } else if word == 0 { ++ // Being written: read again at the next call. ++ break; ++ } ++ sequence += 1; ++ } ++ (written, sequence) ++} ++ ++/// Optional host hook. The installed function must remain loaded for the process lifetime. ++pub type DiagnosticHook = extern "C" fn(u32, u64, u64, u64); ++static DIAGNOSTIC_HOOK: std::sync::OnceLock = std::sync::OnceLock::new(); ++/// Install once before creating command queues. It never changes rendering state. ++pub fn set_diagnostic_hook(hook: DiagnosticHook) { ++ let _ = DIAGNOSTIC_HOOK.set(hook); ++} ++/// Link a command buffer to its Vulkan work, including batches later grouped by the scheduler. ++pub(crate) fn diagnostic_link(id: u64, command: u64, batch: u64) { ++ if let Some(hook) = DIAGNOSTIC_HOOK.get() { ++ hook(45, id, command, batch); ++ } ++} +diff --git a/crates/metal2vulkan-runtime/src/texture.rs b/crates/metal2vulkan-runtime/src/texture.rs +index d30837cdf..257b65b20 100644 +--- a/crates/metal2vulkan-runtime/src/texture.rs ++++ b/crates/metal2vulkan-runtime/src/texture.rs +@@ -85,7 +85,10 @@ use super::image_plan::{ + }; + use super::in_place::{self, PagesClass, Refusal, ShadowCause, Verdict}; + use super::key_hash::KeyMap; +-use super::linear_plan::{aliases, check_linear_limits, linear_lack, LinearImage, LinearLack}; ++use metal2vulkan::device::HostPageTexturePolicy; ++use super::linear_plan::{ ++ aliases, check_linear_limits, linear_lack, tile_store_fits, LinearImage, LinearLack, ++}; + use super::memory::{BoundMemory, MemoryClass, Placement, FD_BLOCKS_HANDLE}; + use super::nan_clear; + use super::neutral::Neutrals; +@@ -187,7 +190,8 @@ pub(super) mod wrapped; + mod ycbcr_image; + mod ycbcr_planes; + pub(crate) use decoded_twin::TwinRead; +-pub(crate) use ycbcr_image::{KeptImage, PlanesInPlace}; ++pub use ycbcr_image::{ycbcr_surface_counts, YCbCrSurfaceCounts}; ++pub(crate) use ycbcr_image::{ImageAlive, KeptImage, PlanesInPlace}; + pub(crate) use ycbcr_planes::KeptTexels; + + pub(crate) use attached_reads::feedback_candidate; +@@ -2504,6 +2508,32 @@ impl Device { + self.host_shadowed(shadowed, (usage, &plan), descriptor, pending)? + } + Route::Device { holds } => { ++ // Rows that lie apart alias the buffer's bytes, written too, under ++ // `HostPageTexturePolicy::InPlaceWrites`: the bytes between them are no texel's, ++ // and a program that pads its framebuffers (a window server's) keeps nothing ++ // there. A written texture's tile stores must stay within its rows ++ // (`linear_plan::tile_store_fits`). Rows that share bytes stay refused. ++ let policy = self.compile_device().policy().host_page_textures; ++ let writes = usage.intersects( ++ TextureUsage::RENDER_TARGET ++ | TextureUsage::SHADER_WRITE ++ | TextureUsage::SHADER_ATOMIC, ++ ); ++ let holds = match holds { ++ Err(_) ++ if policy == HostPageTexturePolicy::InPlaceWrites ++ && texels.layout.shared_bytes() == 0 ++ && (!writes ++ || tile_store_fits( ++ &texels.layout, ++ u64::from(layout.extent[0]), ++ texels.buffer.length(), ++ )) => ++ { ++ Ok(()) ++ } ++ holds => holds, ++ }; + let over = (&*texels, &holds, usage); + match self.alias_over_device(&plan, (descriptor, layout), over, &facts)? { + Ok((image, count)) => { +@@ -3886,6 +3916,15 @@ impl Device { + Ok((raw, placement)) + } + ++ /// Let the parked images go whose time is up, or every one where `all` (`Recycler::aged`): ++ /// how many went. ++ pub(super) fn age_parked_images(&self, all: bool) -> usize { ++ let gone = lock(&self.shared().textures.recycled).aged(Instant::now(), all); ++ let count = gone.len(); ++ self.let_go_parked(gone); ++ count ++ } ++ + /// Destroy the parked images the recycler let go of, `gone`. + fn let_go_parked(&self, gone: recycle::Gone) { + if gone.is_empty() { +@@ -5205,10 +5244,13 @@ impl Texture { + } + } + +- /// The buffer's bytes a texture buffer is, or `None` for a texture with an image. ++ /// The buffer's bytes a texture buffer is, or `None` for a texture with an image, and for a ++ /// two-plane YCbCr texture over its planes alone (`texture/ycbcr_image.rs`), whose texels ++ /// are the planes' bytes and which binds as a texture, not as a texture buffer. + pub(super) fn texels(&self) -> Option<&Texels> { + match self.storage() { + Storage::Image(_) | Storage::Alias { .. } | Storage::Shadow { .. } => None, ++ Storage::Texels(_) if self.image_later().is_some() => None, + Storage::Texels(texels) => Some(texels), + } + } +@@ -6089,6 +6131,30 @@ impl Texture { + self.heap_place().map(|place| place.size) + } + ++ /// The bytes placed for this texture's backing image, for a native frontend's ++ /// `MTLResource.allocatedSize` accounting. This reports Vulkan storage rather ++ /// than Apple's texture layout. Heap textures use their allocated range; ++ /// buffer textures share their buffer's length. Unowned and sparse images ++ /// return zero because their allocation is not owned by this texture. ++ pub fn backing_allocation_size(&self) -> u64 { ++ if let Some(size) = self.allocated_size() { ++ return size; ++ } ++ if let Some(texels) = self.buffer_texels() { ++ return texels.buffer.length(); ++ } ++ let Ok(image) = self.image() else { ++ return 0; ++ }; ++ match &image.owner { ++ Owner::Runtime { placement, .. } => placement.size(), ++ Owner::HeapShadow(shadow) => shadow.placement.size(), ++ Owner::HeapUnaliased(unaliased) => unaliased.placement.size(), ++ Owner::ViewShadow(shadow) => shadow.placement.size(), ++ _ => 0, ++ } ++ } ++ + /// `makeAliasable`: a heap texture's bytes may be another resource's from now on. An + /// automatic heap gives its range to the next resource it places; a placement heap's texture + /// is aliasable from the start, so nothing changes. Using the texture after is the program's +@@ -6372,6 +6438,11 @@ impl Texture { + placement: IdPlacement, + ) -> Result { + self.check_viewable(call)?; ++ // A two-plane YCbCr texture over its planes alone makes no view: its caller views the ++ // texture made with its image (`texture/ycbcr_image.rs`). ++ if self.image_later().is_some() { ++ return Err(Error::NotImplemented(RuntimeGap::Textures)); ++ } + self.device().check_texture_placement( + call, + placement, +diff --git a/crates/metal2vulkan-runtime/src/texture/heap.rs b/crates/metal2vulkan-runtime/src/texture/heap.rs +index 849ba9857..9477f0ca1 100644 +--- a/crates/metal2vulkan-runtime/src/texture/heap.rs ++++ b/crates/metal2vulkan-runtime/src/texture/heap.rs +@@ -2196,6 +2196,16 @@ impl Device { + metal_layout = tiles; + } + } ++ // An answer of the device's own (no Metal answer modelled, or one the texture is past) ++ // is the image's size in the driver's units, which need not be a multiple of the image's ++ // alignment (Turnip's 4 KiB). Metal's sizes are multiples of their alignment, so a heap ++ // sized as the sum of its textures' answers holds them in any order; with the device's ++ // size each texture placed after another pads past what the heap was sized for, and the ++ // last one finds no room (iPadOS 27's iconservicesagent: `heap texture: OutOfMemory(Heap)`). ++ // A shadow in Metal's tiles takes Metal's answer. ++ if (metal_answer.is_none() || past_metal.is_some()) && image.route != AliasRoute::Tiled { ++ size_and_align.size = size_and_align.size.next_multiple_of(size_and_align.align.max(1)); ++ } + // Only a texture an image of binds into the heap's own memory is held to the heap's base, + // and the routes decide that (`holds_heap_base`). + check_heap_base(image.route, size_and_align.align)?; +diff --git a/crates/metal2vulkan-runtime/src/texture/surface_two_planes.rs b/crates/metal2vulkan-runtime/src/texture/surface_two_planes.rs +index 0587be027..f45da0902 100644 +--- a/crates/metal2vulkan-runtime/src/texture/surface_two_planes.rs ++++ b/crates/metal2vulkan-runtime/src/texture/surface_two_planes.rs +@@ -58,6 +58,7 @@ use crate::degradation::Degradation; + use crate::error::{Error, RuntimeGap}; + use crate::image_plan::plan_image; + use crate::linear_plan::{BufferLayout, LINEAR_ALIGNMENT}; ++use crate::resource::Identity; + use crate::shadow::{ShadowBuffer, ShadowCopies, ShadowSource, ShadowTarget}; + use crate::shadow_merge::ScratchSpares; + use crate::texture_descriptor::{texture_usage, TextureDescriptor, TextureLayout}; +@@ -208,7 +209,14 @@ impl Device { + return Err(Error::NotImplemented(RuntimeGap::Textures)); + } + let hazard = texels.options().hazard_tracking_mode()?; +- let image = self.new_image(plan, layout, descriptor, hazard)?; ++ // Its copy in writes every texel of its one aspect before any command reads it, or a pass ++ // that clears it whole does (`shadow.rs`, `render/shadows.rs`), so it starts as its memory ++ // lies, as the other shadows of a buffer's bytes do (`storage_over_buffer`): zeros there ++ // would never be read, and the host's zeros copied into a full-screen surface's image ++ // were most of its making on a phone's render server. ++ let identity = Identity::new(hazard, false); ++ let image = ++ self.new_image_as_it_lies(plan, layout, descriptor, identity, Device::runtime_owner)?; + let (raw, start) = texels.buffer.at(color.offset); + let staging = HeapPlaneCopies::staging(self, &planes, start, hazard)?; + let moves = Box::new(HeapPlaneCopies::new(self, &staging, &planes, hazard)?); +diff --git a/crates/metal2vulkan-runtime/src/texture/usage_shadow.rs b/crates/metal2vulkan-runtime/src/texture/usage_shadow.rs +index b5b92e531..9e6cc697c 100644 +--- a/crates/metal2vulkan-runtime/src/texture/usage_shadow.rs ++++ b/crates/metal2vulkan-runtime/src/texture/usage_shadow.rs +@@ -37,6 +37,11 @@ + //! shadow here instead (`texture/ycbcr_image.rs`, `shadow.rs`): every storage binding of it + //! writes the shadow's view and names the shadow in its uses, and every other binding names its + //! own image. It is exact and announces nothing. ++//! - **A YCbCr texture over its planes alone** (`texture/ycbcr_image.rs`) has no image of its ++//! own: its image, made at its first slot binding that does not read its planes in place, is ++//! held here with the copies that convert the planes into it, and every such binding writes ++//! its view and names it in its uses. It is the image the texture would have been made with, ++//! and announces nothing. + + use super::{Image, Owner, Storage, Texture}; + use crate::binding_view::binding_view; +@@ -170,6 +175,10 @@ impl Texture { + storage: bool, + attached: bool, + ) -> Result>, Error> { ++ // Over its planes alone: its image is made here, at the first binding that needs it. ++ if self.image_later().is_some() { ++ return Ok(Some(self.usage_shadow(call, Need::Usage)?)); ++ } + if self.needs_usage_shadow(storage) { + let shadow = self.usage_shadow(call, Need::Usage)?; + if !shadow.announced.swap(true, Ordering::Relaxed) { +@@ -251,6 +260,16 @@ impl Texture { + } + // A converted texture's image has both shader usages: the slot holds its write shadow. + let shadow = match ycbcr_writes_stored(self.0.record.pixel_format) { ++ // But over its planes alone, where it holds the image itself and the conversion. ++ _ if self.image_later().is_some() => { ++ let (image, copies) = self.later_image(call)?; ++ UsageShadow { ++ image, ++ copies, ++ truth: "its planes", ++ announced: AtomicBool::new(true), ++ } ++ } + true => { + let (image, copies) = self.ycbcr_write_shadow(call)?; + UsageShadow { +diff --git a/crates/metal2vulkan-runtime/src/texture/ycbcr_image.rs b/crates/metal2vulkan-runtime/src/texture/ycbcr_image.rs +index 57cfc751a..32fdd2432 100644 +--- a/crates/metal2vulkan-runtime/src/texture/ycbcr_image.rs ++++ b/crates/metal2vulkan-runtime/src/texture/ycbcr_image.rs +@@ -21,9 +21,29 @@ + //! are also an image of it over their bytes (`ycbcr_planes.rs`), which a slot whose every sample + //! the device's sampler takes samples through the device's conversion instead of the texel + //! buffer. ++//! ++//! **Over its planes alone** (`PlanesInPlace::later`). Metal's own YCbCr texture is the ++//! surface's planes and nothing else, where the image above is as large as the picture in halves ++//! (63 MiB for 3840 x 2160) with a staging copy of the planes beside it, kept for the texture's ++//! life, though no function that only samples it reads either. So a texture whose planes are read ++//! in place, of `ShaderRead` usage alone, whose caller asks ++//! ([`IoSurfaceYCbCr::planes_alone`](crate::IoSurfaceYCbCr::planes_alone)), is made with no ++//! image: its texels are the planes' bytes (`Storage::Texels`), and it holds what its image ++//! is made of. A slot binding that reads the planes in place binds them as before. The first slot ++//! binding that needs the image (a `read()`, a gather, a comparison, a storage binding) makes it ++//! then, as a binding beyond a texture's usage makes its shadow (`texture/usage_shadow.rs`): the ++//! same image, view and conversion as a texture made with its image, its copies listed with the ++//! bound textures' (`shadow::texture_copies`), so it converts as that texture does. Such a ++//! texture serves slot bindings alone: it takes no resource ID (no function reads it through ++//! one), makes no view, and every other call that needs its image is refused ++//! (`NotImplemented(Textures)`), which its caller answers by making the texture with its image ++//! ([`Texture::is_planes_alone`]). Its CPU copies move the luma rows, as the other's do. The ++//! textures made each way, the images made later and the bytes of the images alive are counted ++//! ([`ycbcr_surface_counts`](crate::ycbcr_surface_counts)). + + use super::ycbcr_planes::PlanesImage; + use super::{image_error, IdUse, Image, Storage, Texture}; ++use crate::resource_ids::IdPlacement; + use crate::address_map::AddressModeMap; + use crate::binding_view::VkView; + use crate::buffer_texture::{view_usage, Texels}; +@@ -56,9 +76,70 @@ use metal2vulkan::vk::{ + ImageUsageFlags, ImageViewType, + }; + use std::ops::Range; +-use std::sync::atomic::AtomicU64; ++use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::Arc; + ++/// The two-plane YCbCr textures over surfaces a process made and the images it holds for them ++/// (the module docs' **Over its planes alone**), for a frontend's telemetry. ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++pub struct YCbCrSurfaceCounts { ++ /// Textures made over their planes alone, with no image. ++ pub planes_alone: u64, ++ /// Textures made with their image. ++ pub with_image: u64, ++ /// Images made later for a texture over its planes alone, at a slot binding that needed one. ++ pub images_later: u64, ++ /// Images alive now, of either kind (a view's own among them). ++ pub images_alive: u64, ++ /// The bytes of device memory those images and their conversions' staging buffers hold. ++ pub image_bytes_alive: u64, ++} ++ ++static PLANES_ALONE: AtomicU64 = AtomicU64::new(0); ++static WITH_IMAGE: AtomicU64 = AtomicU64::new(0); ++static IMAGES_LATER: AtomicU64 = AtomicU64::new(0); ++static IMAGES_ALIVE: AtomicU64 = AtomicU64::new(0); ++static IMAGE_BYTES_ALIVE: AtomicU64 = AtomicU64::new(0); ++ ++/// The process's counts of two-plane YCbCr textures over surfaces ([`YCbCrSurfaceCounts`]). ++pub fn ycbcr_surface_counts() -> YCbCrSurfaceCounts { ++ YCbCrSurfaceCounts { ++ planes_alone: PLANES_ALONE.load(Ordering::Relaxed), ++ with_image: WITH_IMAGE.load(Ordering::Relaxed), ++ images_later: IMAGES_LATER.load(Ordering::Relaxed), ++ images_alive: IMAGES_ALIVE.load(Ordering::Relaxed), ++ image_bytes_alive: IMAGE_BYTES_ALIVE.load(Ordering::Relaxed), ++ } ++} ++ ++/// A converted image alive, counted with its bytes until its copies go ++/// (`ShadowSource::Converted`). ++pub(crate) struct ImageAlive(u64); ++ ++impl ImageAlive { ++ fn of(bytes: u64) -> ImageAlive { ++ IMAGES_ALIVE.fetch_add(1, Ordering::Relaxed); ++ IMAGE_BYTES_ALIVE.fetch_add(bytes, Ordering::Relaxed); ++ ImageAlive(bytes) ++ } ++} ++ ++impl Drop for ImageAlive { ++ fn drop(&mut self) { ++ IMAGES_ALIVE.fetch_sub(1, Ordering::Relaxed); ++ IMAGE_BYTES_ALIVE.fetch_sub(self.0, Ordering::Relaxed); ++ } ++} ++ ++/// What the image of a texture over its planes alone is made of, at the first slot binding that ++/// needs it (the module docs): what [`Device::converted_storage`] takes. ++pub(super) struct ImageLater { ++ descriptor: TextureDescriptor, ++ layout: TextureLayout, ++ planes: YCbCrPlanes, ++ span: Range, ++} ++ + /// The formats of a texture's image and of the view the conversion writes it through, by what + /// its texels read: the converted texels as halves, every channel a multiple of 1/2048 up to 1, + /// which a half holds exactly; the sRGB twins' as floats, every channel a multiple of 1/8192, +@@ -149,6 +230,9 @@ pub(crate) struct PlanesInPlace { + pub(super) words: YCbCrInPlace, + start: u64, + pub(super) image: Option>, ++ /// Of a texture over its planes alone: what its image is made of once a slot binding needs ++ /// it (the module docs). `None` for a texture made with its image. ++ pub(super) later: Option, + } + + /// Why a two-plane YCbCr texture keeps its image alone, its planes read in place by no function +@@ -228,6 +312,7 @@ impl Device { + words, + start: span.start, + image: None, ++ later: None, + }) + } + +@@ -236,16 +321,18 @@ impl Device { + /// image of the texels Metal reads and the conversion (`ycbcr.rs`). It reads alpha 1 and maps + /// the address modes Metal samples otherwise (`ycbcr_modes`). Refused `NotImplemented` where + /// the device holds no such image, and as `new_texture` refuses the image or the view the +- /// conversion writes it through. ++ /// conversion writes it through. Where `alone` and the texture can be, it is made over its ++ /// planes alone (the module docs), refused as the image would be. + pub(crate) fn new_converted_texture( + &self, + call: &'static str, + texels: Texels, + descriptor: &TextureDescriptor, + layout: &TextureLayout, +- id_use: IdUse, ++ (id_use, alone): (IdUse, bool), + (planes, span): (&YCbCrPlanes, Range), + ) -> Result { ++ let asked_usage = descriptor.usage; + let descriptor = TextureDescriptor { + alpha_reads_one: true, + address_mode_map: ycbcr_modes(planes.codes), +@@ -267,9 +354,31 @@ impl Device { + Err(kept) => counts.ycbcr_planes_image_kept(kept), + } + } ++ // Over its planes alone: no image until a slot binding needs one, and no resource ++ // ID, which only a function reading handles would reach it through. ++ let alone = alone && asked_usage == TextureUsage::SHADER_READ; ++ let mut in_place = in_place.ok(); ++ if let (true, Some(p)) = (alone, in_place.as_mut()) { ++ // Refused now as its image would be, which is made later. ++ self.converted_plan(call, &descriptor, layout, planes)?; ++ p.later = Some(ImageLater { ++ descriptor, ++ layout: *layout, ++ planes: *planes, ++ span, ++ }); ++ PLANES_ALONE.fetch_add(1, Ordering::Relaxed); ++ let id_use = match id_use { ++ IdUse::Program(_) => IdUse::Program(IdPlacement::Outside), ++ IdUse::Runtime => IdUse::Runtime, ++ }; ++ let made = (Storage::Texels(texels), in_place); ++ return Ok(Texture::made_with_planes(self, made, record, None, id_use)); ++ } + let storage = + self.converted_storage(call, texels, &descriptor, layout, (planes, span))?; +- let made = (storage, in_place.ok()); ++ WITH_IMAGE.fetch_add(1, Ordering::Relaxed); ++ let made = (storage, in_place); + return Ok(Texture::made_with_planes(self, made, record, None, id_use)); + } + // Over a surface of a P3 matrix: an image of the codes of the code-reading format, its +@@ -286,6 +395,7 @@ impl Device { + .map_err(|_| internal("a P3 texture's descriptor past its own"))?; + let storage = + self.converted_storage(call, texels, &coded, &coded_layout, (planes, span))?; ++ WITH_IMAGE.fetch_add(1, Ordering::Relaxed); + record.read_as = Some(( + code_format, + over_read([S::Green, S::Blue, S::Red], descriptor.swizzle), +@@ -293,18 +403,16 @@ impl Device { + Ok(Texture::made(self, storage, record, None, id_use)) + } + +- /// The storage of a two-plane YCbCr texture of `descriptor` (checked as `layout`) over +- /// `texels`, for Metal's `call`, whose planes `planes` lie in `span` of the texels' buffer: +- /// its image and the conversion's shadow (`new_converted_texture`, and a view as the +- /// texture's `_sRGB` twin, `Texture::converted_view`). +- fn converted_storage( ++ /// The plan of the image of a two-plane YCbCr texture of `descriptor` (checked as `layout`) ++ /// whose planes are `planes`, for Metal's `call`, and the format of the view the conversion ++ /// writes it through. Refused `NotImplemented` where the device holds no such image. ++ fn converted_plan( + &self, + call: &'static str, +- texels: Arc, + descriptor: &TextureDescriptor, + layout: &TextureLayout, +- (planes, span): (&YCbCrPlanes, Range), +- ) -> Result { ++ planes: &YCbCrPlanes, ++ ) -> Result<(crate::image_plan::ImagePlan, Format), Error> { + let pf = layout.pixel_format; + let gap = + || Error::Unsupported(Unsupported::not_implemented(Feature::Format).naming_format(pf)); +@@ -351,8 +459,29 @@ impl Device { + plan.view_formats.push(written); + } + } ++ Ok((plan, written)) ++ } ++ ++ /// The storage of a two-plane YCbCr texture of `descriptor` (checked as `layout`) over ++ /// `texels`, for Metal's `call`, whose planes `planes` lie in `span` of the texels' buffer: ++ /// its image and the conversion's shadow (`new_converted_texture`, and a view as the ++ /// texture's `_sRGB` twin, `Texture::converted_view`). ++ fn converted_storage( ++ &self, ++ call: &'static str, ++ texels: Arc, ++ descriptor: &TextureDescriptor, ++ layout: &TextureLayout, ++ (planes, span): (&YCbCrPlanes, Range), ++ ) -> Result { ++ let pf = layout.pixel_format; ++ let (plan, written) = self.converted_plan(call, descriptor, layout, planes)?; + let hazard = texels.options().hazard_tracking_mode()?; + let image = self.new_image(plan, layout, descriptor, hazard)?; ++ let image_bytes = match &image.owner { ++ super::Owner::Runtime { placement, .. } => placement.size(), ++ _ => 0, ++ }; + // The conversion's set holds it as long as the image (`Image::own_view`). + let view = image.own_view(&VkView { + view_type: ImageViewType::TYPE_2D, +@@ -367,6 +496,7 @@ impl Device { + })?; + let (raw, at) = texels.buffer.at(span.start); + let conversion = YCbCrConversion::new(self, (planes, span.clone()), (raw, at), view); ++ let alive = ImageAlive::of(image_bytes + conversion.staging_bytes()); + let (bytes, tracked) = texels.buffer.tracked(span); + let copies = ShadowCopies { + source: ShadowSource::Converted { +@@ -376,6 +506,7 @@ impl Device { + alias: texels.buffer.alias().map(|alias| alias.key), + conversion: Box::new(conversion), + pixel_format: pf, ++ _alive: alive, + }, + shadow_key: image.identity.key, + shadow_range: UseRange::Image { +@@ -404,6 +535,47 @@ impl Device { + } + + impl Texture { ++ /// Whether this two-plane YCbCr texture is over its planes alone (the module docs): made with ++ /// [`IoSurfaceYCbCr::with_planes_alone`](crate::IoSurfaceYCbCr::with_planes_alone), it has ++ /// no image until a slot binding needs one, and serves slot bindings alone. Its caller makes ++ /// the texture again without that for any other call that names it. ++ pub fn is_planes_alone(&self) -> bool { ++ self.image_later().is_some() ++ } ++ ++ /// What the image of a texture over its planes alone is made of (the module docs); `None` ++ /// for every other texture. ++ pub(super) fn image_later(&self) -> Option<&ImageLater> { ++ self.0.planes_in_place.as_ref()?.later.as_ref() ++ } ++ ++ /// The image of a texture over its planes alone and the copies that convert the planes into ++ /// it, made at the first slot binding that needs them (`texture/usage_shadow.rs`), as ++ /// `call`'s: what a texture made with its image holds from the start. Internal for another ++ /// texture; refused as that texture's image and conversion are. ++ pub(super) fn later_image( ++ &self, ++ call: &'static str, ++ ) -> Result<(Arc, Arc), Error> { ++ let (Some(later), Storage::Texels(texels)) = (self.image_later(), &self.0.storage) else { ++ return Err(internal( ++ "an image made later for a texture made with its image", ++ )); ++ }; ++ let storage = self.device().converted_storage( ++ call, ++ Arc::clone(texels), ++ &later.descriptor, ++ &later.layout, ++ (&later.planes, later.span.clone()), ++ )?; ++ let Storage::Shadow { image, copies, .. } = storage else { ++ return Err(internal("a converted storage that is no shadow")); ++ }; ++ IMAGES_LATER.fetch_add(1, Ordering::Relaxed); ++ Ok((image, copies)) ++ } ++ + /// The uniform texel buffer view of the texture's planes read in place (the module docs), made + /// at its first bind and kept with the texels. Internal for a texture whose planes are not. + pub(crate) fn planes_view(&self) -> Result { +diff --git a/crates/metal2vulkan-runtime/src/used_resources.rs b/crates/metal2vulkan-runtime/src/used_resources.rs +index d4244e1da..ca3b10c20 100644 +--- a/crates/metal2vulkan-runtime/src/used_resources.rs ++++ b/crates/metal2vulkan-runtime/src/used_resources.rs +@@ -59,6 +59,62 @@ pub struct UsedResources { + pub heaps: Vec, + } + ++// Compare host address ranges without touching their bytes. Object identity covers private ++// memory; host ranges additionally cover distinct imports of an IOSurface or overlapping ++// no-copy buffers. Unknown arithmetic is conservative. Rebinding is not performed by this ++// consumer, and callers must not remap these ranges while the resources are in use. ++fn buffers_overlap(a: &Buffer, b: &Buffer) -> bool { ++ if a == b { ++ return true; ++ } ++ let (Some(a_start), Some(b_start)) = (a.contents(), b.contents()) else { ++ return false; ++ }; ++ let (Ok(a_len), Ok(b_len)) = (usize::try_from(a.length()), usize::try_from(b.length())) else { ++ return true; ++ }; ++ let a_start = a_start.as_ptr() as usize; ++ let b_start = b_start.as_ptr() as usize; ++ let (Some(a_end), Some(b_end)) = (a_start.checked_add(a_len), b_start.checked_add(b_len)) else { ++ return true; ++ }; ++ a_start < b_end && b_start < a_end ++} ++ ++impl UsedResources { ++ fn references_buffer(&self, buffer: &Buffer) -> bool { ++ if buffer.heap().is_some_and(|heap| self.heaps.contains(heap)) { ++ return true; ++ } ++ self.buffers.iter().any(|other| buffers_overlap(buffer, other)) ++ || self.heaps.iter().any(|heap| heap.bytes().is_some_and(|bytes| buffers_overlap(buffer, bytes))) ++ } ++ ++ /// Whether this list names a resource's storage, including a view's root, a buffer ++ /// texture's backing buffer, a heap's resources and overlapping imported host ranges. ++ /// A caller may skip waiting only while this is false. This consumer does not rebind ++ /// or remap live backing ranges. Indirect command buffers remain conservative. ++ pub fn references_resource(&self, resource: super::resource::ResourceRef<'_>) -> bool { ++ use super::resource::ResourceRef; ++ match resource { ++ ResourceRef::Buffer(buffer) => self.references_buffer(buffer), ++ ResourceRef::Texture(texture) => match texture.memory() { ++ TextureMemory::Heap(heap) => self.references_heap(heap), ++ TextureMemory::Buffer(buffer) => self.references_buffer(buffer), ++ TextureMemory::Own(root) => self.textures.contains(root), ++ }, ++ ResourceRef::IndirectCommandBuffer(_) => true, ++ } ++ } ++ ++ /// Heap storage is treated as one dependency, conservatively including its disjoint ++ /// members. Imports over its host range are dependencies too. ++ pub fn references_heap(&self, heap: &Heap) -> bool { ++ self.heaps.contains(heap) ++ || heap.bytes().is_some_and(|bytes| self.references_buffer(bytes)) ++ } ++} ++ + /// The kind of a command encoder ([`EndedEncoder::kind`]), by the Metal protocol it is. + #[non_exhaustive] + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +diff --git a/crates/metal2vulkan-runtime/src/work_counts.rs b/crates/metal2vulkan-runtime/src/work_counts.rs +index 414ffdc92..8a470ed41 100644 +--- a/crates/metal2vulkan-runtime/src/work_counts.rs ++++ b/crates/metal2vulkan-runtime/src/work_counts.rs +@@ -5053,6 +5053,16 @@ pub struct WorkCounts { + /// Its live dedicated blocks with their placement: a placement larger than half a block, one + /// placed apart, or an exportable buffer's dedicated allocation (`exported_memory.rs`). + pub dedicated_memory_blocks_live: u64, ++ /// Bytes of its live pooled blocks with a placement. ++ pub pooled_memory_bytes_live: u64, ++ /// Of those, the bytes placed. ++ pub pooled_memory_bytes_used: u64, ++ /// Bytes of its empty kept pooled blocks. ++ pub empty_memory_bytes_live: u64, ++ /// Bytes of its live dedicated blocks. ++ pub dedicated_memory_bytes_live: u64, ++ /// Bytes of its live host imports. ++ pub imported_memory_bytes_live: u64, + /// Of `pooled_memory_blocks_made`, the blocks of the pool a color texture's host-transfer + /// image places in made ready off the program's thread: allocated and every page of their + /// mapping read once by the host, then added to the pool by the first placement there that +@@ -8218,6 +8228,11 @@ impl Device { + pooled_memory_blocks_made: made, + pooled_memory_blocks_freed: freed, + dedicated_memory_blocks_live: census.dedicated, ++ pooled_memory_bytes_live: census.pooled_bytes, ++ pooled_memory_bytes_used: census.pooled_used_bytes, ++ empty_memory_bytes_live: census.empty_bytes, ++ dedicated_memory_bytes_live: census.dedicated_bytes, ++ imported_memory_bytes_live: census.imported_bytes, + host_copy_blocks_readied: host_copy.readied, + host_copy_blocks_made_in_place: host_copy.made_in_place, + buffer_blocks_made_ahead: ahead.buffers_ahead, +diff --git a/crates/metal2vulkan-runtime/src/ycbcr.rs b/crates/metal2vulkan-runtime/src/ycbcr.rs +index 195b83a42..e3b7d7027 100644 +--- a/crates/metal2vulkan-runtime/src/ycbcr.rs ++++ b/crates/metal2vulkan-runtime/src/ycbcr.rs +@@ -309,6 +309,11 @@ pub struct IoSurfaceYCbCr { + pub chroma_bytes_per_row: u64, + /// The surface's `kIOSurfaceYCbCrMatrix` as the texture is made, `None` without one. + pub matrix: Option, ++ /// Whether a texture whose planes a function reads in place, of `ShaderRead` usage alone, is ++ /// made over its planes alone (`texture/ycbcr_image.rs`): with no image until a slot binding ++ /// needs one, and for slot bindings only. `false` unless the caller asks ++ /// ([`with_planes_alone`](Self::with_planes_alone)). ++ pub planes_alone: bool, + } + + impl IoSurfaceYCbCr { +@@ -328,9 +333,18 @@ impl IoSurfaceYCbCr { + chroma_offset, + chroma_bytes_per_row, + matrix: None, ++ planes_alone: false, + } + } + ++ /// These facts for a caller that binds the texture at slots only while it is over its planes ++ /// alone ([`Texture::is_planes_alone`](crate::Texture::is_planes_alone)) and makes the ++ /// texture without this for any other call. ++ pub const fn with_planes_alone(mut self) -> IoSurfaceYCbCr { ++ self.planes_alone = true; ++ self ++ } ++ + /// These facts with the surface's `kIOSurfaceYCbCrMatrix`. + pub const fn with_matrix(mut self, matrix: SurfaceYCbCrMatrix) -> IoSurfaceYCbCr { + self.matrix = Some(matrix); +@@ -860,6 +874,12 @@ impl YCbCrConversion { + Ok(()) + } + ++ /// The bytes of its staging buffer, which it holds for the texture's life once it was made ++ /// ([`YCbCrConversion::make`]): 0 before. ++ pub(super) fn staging_bytes(&self) -> u64 { ++ self.made.get().map_or(0, |made| made.staging.length()) ++ } ++ + /// The planes it converts, and the bytes of their buffer they lie in. + pub(super) fn source(&self) -> (YCbCrPlanes, Range) { + (self.source.0, self.source.1.clone()) +diff --git a/crates/metal2vulkan/src/cache/mod.rs b/crates/metal2vulkan/src/cache/mod.rs +index e0a9a6af8..650064ca8 100644 +--- a/crates/metal2vulkan/src/cache/mod.rs ++++ b/crates/metal2vulkan/src/cache/mod.rs +@@ -60,7 +60,7 @@ mod tests; + /// unchanged version, and its bless records the version. Independent of the crate's version and of + /// [`ARTIFACT_FORMAT_VERSION`]. The commit that bumps it says what changed; this comment keeps no + /// history. +-pub const SEMANTICS_VERSION: u32 = 2020; ++pub const SEMANTICS_VERSION: u32 = 2022; + + /// The byte-layout version of encoded artifacts (`CompiledFunction::to_bytes` and the + /// [`PipelineArtifact`] encodings). Every encoded artifact carries it, and a decode under another +@@ -93,7 +93,7 @@ pub const ARTIFACT_FORMAT_VERSION: u32 = 500; + /// changes no behaviour, and then updates that hash alone with the commit saying why, or changes + /// behaviour and also bumps this version. The commit that bumps it says what changed; this comment + /// keeps no history. +-pub const RUNTIME_VERSION: u32 = 2016; ++pub const RUNTIME_VERSION: u32 = 2017; + + /// The digest of the sources this build of the crate was compiled from: SHA-256, as lowercase hex, + /// of the manifest, the build script, and exactly the files the crate roots' module tree compiles +diff --git a/crates/metal2vulkan/src/device/driver_facts.rs b/crates/metal2vulkan/src/device/driver_facts.rs +index 8d6c42df8..1231754ad 100644 +--- a/crates/metal2vulkan/src/device/driver_facts.rs ++++ b/crates/metal2vulkan/src/device/driver_facts.rs +@@ -753,6 +753,13 @@ pub(crate) const INTEL_UHD_770_RPL_S: u32 = 0xa780; + /// `intel-irisxe-rplp-anv`). + pub(crate) const INTEL_IRIS_XE_RPL_P: u32 = 0xa7a0; + ++/// Qualcomm's `vendorID`. ++const QUALCOMM_VENDOR_ID: u32 = 0x5143; ++ ++/// The `deviceID` Turnip reports for the Adreno 740: its chip id (Mesa's `tu_device.cc`, ++/// `props->deviceID = pdevice->dev_id.chip_id`; `freedreno_devices.py`, `FD740`). ++pub(crate) const ADRENO_740: u32 = 0x4305_0a01; ++ + /// The row of `fact` measured on Mesa's `driver` on the GPU `(vendor, device)`. + const fn mesa_on(fact: F, driver: DriverId, (vendor, device): (u32, u32)) -> Measured { + Measured { +@@ -1947,6 +1954,16 @@ const DYNAMIC_RESTART_WARNING: DriverDiagnostic = DriverDiagnostic { + /// row, ANV, Mesa 26.2.4, on the Iris Xe (RPL-P). Not listed: NVIDIA's 615.71.09, whose RTX 3080 + /// left a third answer; RADV on the RX 7900 XTX, clean at dd718a7905 at Mesa 26.0.8 but not + /// measured since; and every GPU not measured, which store with `STORE`. ++/// Turnip on the Adreno 740 takes it by its source, not by that measurement (as ++/// `DontCareStoreIsMetals` is read in MoltenVK's): an attachment's store is its store operation ++/// being `STORE` and nothing else (`tu_pass.cc`, `attachment_set_ops`: `att->store = (store_op == ++/// VK_ATTACHMENT_STORE_OP_STORE)`); a pass rendered in tile memory writes an attachment's memory ++/// only in its tile store, which returns before any command for one not stored ++/// (`tu_clear_blit.cc`, `tu_store_gmem_attachment`: `if (!dst->store && !dst->store_stencil) ++/// return;`), so every texel is the earlier one; a pass rendered in system memory ++/// (`tu_cmd_buffer.cc`, `tu_cmd_render_sysmem`) clears and draws into the attachment's own ++/// memory, so every texel is the one `STORE` leaves. Turnip chooses one of the two for a whole ++/// pass (`use_sysmem_rendering`). + /// + /// `InterlockOrdersDraws`, measured with no barrier between such draws on any driver (a scratch + /// build of the commit that adds it), by `runtime_lane::interlocked_draw_order` (passes of 84 +@@ -2251,6 +2268,12 @@ pub(crate) const MEASURED: &[Measured] = &[ + DriverId::INTEL_OPEN_SOURCE_MESA, + (INTEL_VENDOR_ID, INTEL_IRIS_XE_RPL_P), + ), ++ // Read in Mesa's source, not measured (the docs above). ++ mesa_on( ++ DriverFact::DontCareStoreIsOneOfMetals, ++ DriverId::MESA_TURNIP, ++ (QUALCOMM_VENDOR_ID, ADRENO_740), ++ ), + mesa_on( + DriverFact::ColorBlockDecodeIsMetals, + DriverId::MESA_RADV, +diff --git a/crates/metal2vulkan/src/device/policy.rs b/crates/metal2vulkan/src/device/policy.rs +index 50a33bc70..87e34d010 100644 +--- a/crates/metal2vulkan/src/device/policy.rs ++++ b/crates/metal2vulkan/src/device/policy.rs +@@ -327,13 +327,26 @@ pub enum HostPageTexturePolicy { + /// texels into before each use and merge back after each write, whose copies touch the + /// texels alone. + Shadow, ++ /// [`InPlace`](Self::InPlace) for every usage, a texture that renders or writes too, with no ++ /// check run and no verdict waited for, trusting the driver to touch no byte beside the ++ /// texels: a window server's surfaces, which it renders into each frame, are otherwise copied ++ /// into a shadow before each pass and merged back after it. A padded texture over a buffer of ++ /// the runtime's own memory aliases its bytes too, a written one where its tile stores stay ++ /// within its rows (`linear_plan::tile_store_fits`), where the device lays it out at the ++ /// buffer's row pitch. ++ InPlaceWrites, + } + + /// How a chain of `half` additions, subtractions and products rounds on a device without + /// `shaderFloat16`, where a module computes each in `float` on its operands held widened and + /// rounds its result by the device's `OpQuantizeToF16` (`PrecisionNote::NativeHalfQuantize`): +-/// capability emulation, as the device has no 16-bit arithmetic. A device with `shaderFloat16` +-/// computes `half` as it does under either value. Changes emitted code: the profile carries it. ++/// capability emulation, as the device has no 16-bit arithmetic. On a device with `shaderFloat16` ++/// whose 16-bit floats keep no denormals (no `DenormPreserve` for 16 bits) it chooses how a `half` ++/// operation the device has an instruction for is stated: its `float` result decorated ++/// `RelaxedPrecision`, the device's own 16-bit precision (`AtExits`), or the operation in `float` ++/// converted to 16 bits and back (`EachOperation`); `PrecisionNote::NativeHalfConversion` either ++/// way. Every other device with `shaderFloat16` computes `half` as it does under either value. ++/// Changes emitted code: the profile carries it. + #[non_exhaustive] + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +diff --git a/crates/metal2vulkan/src/lower/access/blob.rs b/crates/metal2vulkan/src/lower/access/blob.rs +index 1ed5aed39..130f80ccf 100644 +--- a/crates/metal2vulkan/src/lower/access/blob.rs ++++ b/crates/metal2vulkan/src/lower/access/blob.rs +@@ -169,13 +169,45 @@ pub fn blob_ptr(cx: &SelectCx<'_>, v: Value) -> Result { + let Value::Const(c) = v else { + return Ok(LocalPtr::Value(operand(cx, v)?)); + }; +- match cx.entry.module.consts().get(c).map(|k| &k.value) { +- Some(Const::GlobalAddr(g)) => { +- let off = global_offset(cx.memory, *g).ok_or_else(refuse)?; +- Ok(LocalPtr::Offset(u32::try_from(off).map_err(|_| refuse())?)) ++ blob_const_offset(cx, c).map(LocalPtr::Offset) ++} ++ ++/// Constant pointer expressions preserve the blob's byte offset through casts ++/// and constant GEPs. Walk the expression chain without recursive stack growth. ++fn blob_const_offset(cx: &SelectCx<'_>, mut c: crate::air::ir::ConstId) -> Result { ++ use crate::air::ir::ConstExpr; ++ let mut displacement = 0i64; ++ for _ in 0..cx.entry.module.consts().len() { ++ match cx.entry.module.consts().get(c).map(|k| &k.value) { ++ Some(Const::GlobalAddr(g)) => { ++ let base = global_offset(cx.memory, *g).ok_or_else(refuse)?; ++ let base = i64::try_from(base).map_err(|_| refuse())?; ++ return u32::try_from(base.checked_add(displacement).ok_or_else(refuse)?) ++ .map_err(|_| refuse()); ++ } ++ Some(Const::Expr(ConstExpr::Cast { ++ op: CastOp::Bitcast | CastOp::AddrSpaceCast, ++ ops, ++ })) => { ++ c = *ops.first().ok_or_else(refuse)?; ++ } ++ Some(Const::Expr(ConstExpr::Gep { src_ty, ops, .. })) => { ++ let (base, indices) = ops.split_first().ok_or_else(refuse)?; ++ let indices: Vec = indices.iter().copied().map(Value::Const).collect(); ++ let (delta, strides) = crate::memory::provenance::gep_displacement( ++ cx.entry, ++ &cx.memory.layouts, ++ *src_ty, ++ &indices, ++ )?; ++ let delta = delta.filter(|_| strides.is_empty()).ok_or_else(refuse)?; ++ displacement = displacement.checked_add(delta).ok_or_else(refuse)?; ++ c = *base; ++ } ++ _ => return Err(refuse()), + } +- _ => Err(refuse()), + } ++ Err(refuse()) + } + + /// A leaf load of AIR type `ty` with alignment `align` through blob pointer `ptr`. +diff --git a/crates/metal2vulkan/src/lower/arith/divide.rs b/crates/metal2vulkan/src/lower/arith/divide.rs +index f94247315..95503a976 100644 +--- a/crates/metal2vulkan/src/lower/arith/divide.rs ++++ b/crates/metal2vulkan/src/lower/arith/divide.rs +@@ -47,16 +47,29 @@ pub fn native_words(f: Form) -> u32 { + /// rounded to `half` (`PrecisionNote::NativeDivision`). + pub fn native(fb: &mut FunctionBuilder<'_>, f: Form, x: Value, y: Value) -> BResult { + let ay = fb.glsl(crate::spirv::ir::GlslOp::FAbs, &[y])?; +- let limit = fsplat(fb, f, 2f64.powi(126))?; +- let past = fb.binary(BinOp::FOrdGreaterThan, ay, limit)?; +- let (quarter, one) = (fsplat(fb, f, 0.25)?, fsplat(fb, f, 1.0)?); +- let scale = fb.select(past, quarter, one)?; ++ let scale = if f.float_fmt() == Some(FloatFmt::Half) && fb.module().relaxed_half() { ++ let upper = fsplat(fb, f, 2f64.powi(14))?; ++ let lower = fsplat(fb, f, 2f64.powi(-14))?; ++ let past = fb.binary(BinOp::FOrdGreaterThan, ay, upper)?; ++ let below = fb.binary(BinOp::FOrdLessThan, ay, lower)?; ++ let down = fsplat(fb, f, 0.25)?; ++ let up = fsplat(fb, f, 1024.0)?; ++ let one = fsplat(fb, f, 1.0)?; ++ let scale = fb.select(below, up, one)?; ++ fb.select(past, down, scale)? ++ } else { ++ let limit = fsplat(fb, f, 2f64.powi(126))?; ++ let past = fb.binary(BinOp::FOrdGreaterThan, ay, limit)?; ++ let quarter = fsplat(fb, f, 0.25)?; ++ let one = fsplat(fb, f, 1.0)?; ++ fb.select(past, quarter, one)? ++ }; + let x = fb.binary(BinOp::FMul, x, scale)?; + let y = fb.binary(BinOp::FMul, y, scale)?; + let q = fb.binary(BinOp::FDiv, x, y)?; + match f.float_fmt() { + Some(FloatFmt::F32) => Ok(q), +- Some(FloatFmt::Half) => narrow::round(fb, f, q), ++ Some(FloatFmt::Half) => narrow::round_native(fb, f, q), + _ => Err(BuildError::Type(TypeRule::FunctionSignature)), + } + } +diff --git a/crates/metal2vulkan/src/lower/arith/math.rs b/crates/metal2vulkan/src/lower/arith/math.rs +index 22a0042ed..33ad6b832 100644 +--- a/crates/metal2vulkan/src/lower/arith/math.rs ++++ b/crates/metal2vulkan/src/lower/arith/math.rs +@@ -1308,6 +1308,15 @@ pub fn float_fn( + // Rounded per step below. + F::Mix | F::MulAdd | F::Fract | F::FastFract => r, + F::Dot => narrow::round(fb, f.scalar_form(), r)?, ++ // One device instruction on the operands: the device's own at its 16-bit precision ++ // where the module leaves `half` results to it. ++ F::Sqrt ++ | F::FastSqrt ++ | F::Rsqrt ++ | F::FastRsqrt ++ | F::Exp2 ++ | F::Log2 ++ | F::FastLog2 => narrow::round_native(fb, f, r)?, + _ => narrow::round(fb, f, r)?, + }; + Ok(r) +@@ -1416,9 +1425,9 @@ fn float_fn_core( + F::MulAdd => { + // A narrow form takes the unfused choice, each step rounded once by the device. + let p = bin(fb, BinOp::FMul, a, arg(args, 1)?)?; +- let p = narrow::round(fb, f, p)?; ++ let p = narrow::round_native(fb, f, p)?; + let r = bin(fb, BinOp::FAdd, p, arg(args, 2)?)?; +- narrow::round(fb, f, r) ++ narrow::round_native(fb, f, r) + } + F::Sqrt => g1(fb, GlslOp::Sqrt), + F::Fma if f.float_fmt() == Some(FloatFmt::BFloat) => { +diff --git a/crates/metal2vulkan/src/lower/arith/mod.rs b/crates/metal2vulkan/src/lower/arith/mod.rs +index 1065191ea..3cc7fdfeb 100644 +--- a/crates/metal2vulkan/src/lower/arith/mod.rs ++++ b/crates/metal2vulkan/src/lower/arith/mod.rs +@@ -702,6 +702,10 @@ pub struct ArithPlan { + /// Whether [`ArithPlan::quantize`] is the device's own rounding (native half rounding, + /// [`narrow::native_rounding`]), decided with it by [`finish`]. + pub native_half: bool, ++ /// Whether the result of a `half` operation the device has an instruction for is left to ++ /// the device's own 16-bit precision ([`narrow::relaxes`], [`narrow::round_native`]), decided ++ /// by [`finish`]. ++ pub relaxed_half: bool, + /// Whether a `half`'s NaN bits are left to the device's conversion where the module rounds + /// through the conversion under the float controls ([`QuantizeForm::Controlled`]): a `half` + /// texel takes the rounding's NaN (`narrow::round_texel`). Decided by [`finish`], on every +@@ -785,6 +789,7 @@ pub fn finish(cx: &mut SelectCx<'_>) -> Result<(), Error> { + cx.families.arith.wide = legal::holds_wide(cx.profile); + cx.families.arith.quantize = narrow::quantize_form(cx.profile); + cx.families.arith.native_half = narrow::native_rounding(cx.families.arith.quantize).is_some(); ++ cx.families.arith.relaxed_half = narrow::relaxes(cx.profile); + // Under the float controls a `half`'s NaN is the device's conversion's. + cx.families.arith.native_half_nans = cx.families.arith.quantize == QuantizeForm::Controlled; + // An extended-range, padded or bfloat texel's conversion is the device's arithmetic. +diff --git a/crates/metal2vulkan/src/lower/arith/narrow.rs b/crates/metal2vulkan/src/lower/arith/narrow.rs +index 800f97607..5482f9b28 100644 +--- a/crates/metal2vulkan/src/lower/arith/narrow.rs ++++ b/crates/metal2vulkan/src/lower/arith/narrow.rs +@@ -133,6 +133,48 @@ pub fn round(fb: &mut FunctionBuilder<'_>, f: Form, x: Value) -> BResult + } + } + ++/// Whether profile `p`'s modules leave the result of a `half` operation the device has an ++/// instruction for to the device's own 16-bit precision ([`round_native`]): a profile with ++/// `shaderFloat16` whose 16-bit floats do not keep denormals (no `DenormPreserve` for 16 bits, so ++/// no float controls make its conversion Metal's and the module rounds in ++/// [`QuantizeForm::Converted`]) and whose conversion and 16-bit arithmetic no measurement ++/// makes Metal's (`DriverDeviation::MetalHalfConversion`, `MetalHalfArithmetic`: MoltenVK's stay ++/// as they are), under `HalfChainPolicy::AtExits` (the default). ++/// `HalfChainPolicy::EachOperation` (`DevicePolicy::exact_answers`) keeps the conversion pair ++/// after every operation. ++pub fn relaxes(p: &DeviceProfile) -> bool { ++ let fields = p.fields(); ++ // A device whose conversion or 16-bit arithmetic is measured to be Metal's keeps it. ++ let measured = p.takes_driver_deviation(crate::error::DriverDeviation::MetalHalfConversion) ++ || p.takes_driver_deviation(crate::error::DriverDeviation::MetalHalfArithmetic); ++ fields.half_chains == crate::device::HalfChainPolicy::AtExits ++ && !measured ++ && fields.leaf.float16 ++ && !fields.float_controls.f16.denorm_preserve ++ && quantize_form(p) == QuantizeForm::Converted ++} ++ ++/// [`round`] of `x`, the result of ONE device instruction on `half` operands held widened (a ++/// `half` `+ - * /` or a function the device has an instruction for). ++/// Where the module leaves such results to the device (`ModuleBuilder::relaxed_half`, ++/// [`relaxes`]) the result is `x` itself, recorded for the serializer to decorate ++/// `RelaxedPrecision`: the device computes the instruction at its 16-bit precision, on its ++/// 16-bit registers, in place of computing it in `float` and converting to 16 bits and back (a ++/// pair Mesa's compiler folds into nothing, the result then not rounded at all). Recorded as ++/// `PrecisionNote::NativeHalfConversion`. A `half` a routine of several instructions computes in ++/// `float`, a conversion from `float` and a texel are not this form: they take [`round`] and ++/// [`round_measured`], whose operand may be a `float` the function also reads at full precision. ++pub fn round_native(fb: &mut FunctionBuilder<'_>, f: Form, x: Value) -> BResult { ++ if f.held == Held::Float(FloatFmt::Half) ++ && fb.module().relaxed_half() ++ && fb.module().quantize_form() == QuantizeForm::Converted ++ { ++ fb.module().note_relaxed(x); ++ return Ok(x); ++ } ++ round(fb, f, x) ++} ++ + /// How profile `p`'s modules round an `f32` to `half` ([`QuantizeForm`]): the device's conversion + /// to 16 bits and back under the float controls that make it Metal's by the Vulkan spec where the + /// profile has `shaderFloat16` and them ([`QuantizeForm::Controlled`], [`controlled_modes`]), else +diff --git a/crates/metal2vulkan/src/lower/arith/ops.rs b/crates/metal2vulkan/src/lower/arith/ops.rs +index 98c9ff27f..59ad8c3ad 100644 +--- a/crates/metal2vulkan/src/lower/arith/ops.rs ++++ b/crates/metal2vulkan/src/lower/arith/ops.rs +@@ -252,7 +252,7 @@ pub fn float_binary( + _ => BinOp::FRem, + }; + let r = fb.binary_with(o, a, b, contract)?; +- narrow::round(fb, f, r) ++ narrow::round_native(fb, f, r) + } + + /// The census name of a `half` operator on the device's 16-bit arithmetic +diff --git a/crates/metal2vulkan/src/lower/emit.rs b/crates/metal2vulkan/src/lower/emit.rs +index 1ada9dd92..7b40368b9 100644 +--- a/crates/metal2vulkan/src/lower/emit.rs ++++ b/crates/metal2vulkan/src/lower/emit.rs +@@ -105,6 +105,7 @@ fn declare(mb: &mut ModuleBuilder, plan: &Plan) -> BResult { + // And every `half` rounding's quantize the device's form. + mb.set_quantize_form(plan.families.arith.quantize); + mb.set_native_half(plan.families.arith.native_half); ++ mb.set_relaxed_half(plan.families.arith.relaxed_half); + mb.set_native_half_nans(plan.families.arith.native_half_nans); + let mut d = Declared::default(); + let modes = &mut d.modes; +diff --git a/crates/metal2vulkan/src/memory/plan.rs b/crates/metal2vulkan/src/memory/plan.rs +index 2108cca50..e9164f0aa 100644 +--- a/crates/metal2vulkan/src/memory/plan.rs ++++ b/crates/metal2vulkan/src/memory/plan.rs +@@ -33,7 +33,8 @@ + //! lowered representation, an open one a buffer pointer as its 8-byte guest address. + //! - **Constant globals**: a closed, type-consistent constant is a `Private` variable with its + //! initializer, unless its type would take the function's typed objects past +-//! `alloca::MAX_UNSHARED`; any other one is placed in the **constant blob**, the byte image the ++//! `alloca::MAX_UNSHARED` or dynamic indexing reaches a table larger than 4 KiB. Other globals ++//! are placed in the **constant blob**, the byte image the + //! artifact carries (`WriteSource::ConstantBlob`). + //! - **Needs**: dynamic roots, the handle classes that need the handle table, and whether a guest + //! base is needed. +@@ -1059,28 +1060,39 @@ impl Planner<'_> { + let closed = !u.merged && !u.escapes && !u.stored && !u.cross_compare && !u.write; + // A type no typed object holds stays `Private`: selection refuses it, as before, + // which only sends a typed object past the room to the blob. +- let form = +- if closed && u.typed && self.fit(global.value_ty, room, m)? != Fit::OverRoom { +- ConstGlobalForm::Private +- } else { +- let align = self.global_align(g, global.value_ty)?; +- let size = self +- .layouts +- .alloc_size(global.value_ty) +- .ok_or_else(not_lowered)?; +- let offset = round_up(*len as u64, align).ok_or_else(not_lowered)?; +- let end = offset.checked_add(size).ok_or_else(not_lowered)?; +- let end = usize::try_from(end).map_err(|_| not_lowered())?; +- // The initializer's refusals; its bytes and its writing's work are charged +- // once every initializer is checked. +- let nodes = self.blob_nodes(init, global.value_ty, m)?; +- let at = usize::try_from(offset).map_err(|_| not_lowered())?; +- grow(placed, 1, m)?; +- placed.push((init, global.value_ty, at)); +- provenance::push_working(&mut costs, (end, nodes), &mut working, m)?; +- *len = end; +- ConstGlobalForm::Blob { offset } +- }; ++ // Dynamically indexed tables above 4 KiB are shared read-only data. ++ // Private storage would replicate the table in every shader invocation. ++ let large_indexed = u.indexed ++ && self ++ .layouts ++ .alloc_size(global.value_ty) ++ .ok_or_else(not_lowered)? ++ > 4096; ++ let form = if closed ++ && u.typed ++ && !large_indexed ++ && self.fit(global.value_ty, room, m)? != Fit::OverRoom ++ { ++ ConstGlobalForm::Private ++ } else { ++ let align = self.global_align(g, global.value_ty)?; ++ let size = self ++ .layouts ++ .alloc_size(global.value_ty) ++ .ok_or_else(not_lowered)?; ++ let offset = round_up(*len as u64, align).ok_or_else(not_lowered)?; ++ let end = offset.checked_add(size).ok_or_else(not_lowered)?; ++ let end = usize::try_from(end).map_err(|_| not_lowered())?; ++ // The initializer's refusals; its bytes and its writing's work are charged ++ // once every initializer is checked. ++ let nodes = self.blob_nodes(init, global.value_ty, m)?; ++ let at = usize::try_from(offset).map_err(|_| not_lowered())?; ++ grow(placed, 1, m)?; ++ placed.push((init, global.value_ty, at)); ++ provenance::push_working(&mut costs, (end, nodes), &mut working, m)?; ++ *len = end; ++ ConstGlobalForm::Blob { offset } ++ }; + grow(out, 1, m)?; + out.push(ConstGlobalPlan { global: g, form }); + } +diff --git a/crates/metal2vulkan/src/pipeline/compile.rs b/crates/metal2vulkan/src/pipeline/compile.rs +index 26810ac09..90c6283e7 100644 +--- a/crates/metal2vulkan/src/pipeline/compile.rs ++++ b/crates/metal2vulkan/src/pipeline/compile.rs +@@ -2808,9 +2808,13 @@ fn compile_kept( + // Metal's (`DriverDeviation::MetalHalfConversion`) and every such rounding is of a form the + // measurement covers (a `float` the function converts, or an image instruction's texel), + // which the module then requires. ++ // A `half` result left to the device's own 16-bit precision (`narrow::round_native`) is ++ // within the same note, and no measurement covers it. ++ let relaxed = !module.relaxed().is_empty(); + let converts = quantize == crate::spirv::builder::QuantizeForm::Converted +- && crate::lower::arith::narrow::rounds_by_conversion(&module, m)?; ++ && (relaxed || crate::lower::arith::narrow::rounds_by_conversion(&module, m)?); + let measured_conversion = converts ++ && !relaxed + && p.takes_driver_deviation(crate::error::DriverDeviation::MetalHalfConversion) + && !crate::lower::arith::narrow::rounds_unmeasured(&module, m)?; + // A `half` read from its bits by the device's widening, or encoded to them by its narrowing +diff --git a/crates/metal2vulkan/src/reflection/function.rs b/crates/metal2vulkan/src/reflection/function.rs +index 1be12a746..cc910cf3d 100644 +--- a/crates/metal2vulkan/src/reflection/function.rs ++++ b/crates/metal2vulkan/src/reflection/function.rs +@@ -225,9 +225,12 @@ pub(crate) fn metallib_properties( + 2 => PatchType::Quad, + _ => return Err(unmeasured()), + }; ++ // A count of 0 is `[[patch(quad, 0)]]`: a patch with no control points, whose function ++ // reads its own data by `patch_id` (GFXBench's Car Chase draws its Bezier car body so). ++ // Metal answers it as stated: the patch type and 0. + let count = match byte >> 2 { + 63 => None, +- n @ 1..=32 => Some(u32::from(n)), ++ n @ 0..=32 => Some(u32::from(n)), + _ => return Err(unmeasured()), + }; + Some((kind, count)) +diff --git a/crates/metal2vulkan/src/spirv/builder/module.rs b/crates/metal2vulkan/src/spirv/builder/module.rs +index 5ff8d5781..1edd8fb07 100644 +--- a/crates/metal2vulkan/src/spirv/builder/module.rs ++++ b/crates/metal2vulkan/src/spirv/builder/module.rs +@@ -106,6 +106,9 @@ pub struct ModuleBuilder { + /// (`ModuleBuilder::note_measured_rounding`). Nothing emitted depends on it: the compile + /// reads it to decide `PrecisionNote::NativeHalfConversion`. + pub(super) measured_roundings: BTreeSet, ++ /// The results of `half` operations the device computes at its own 16-bit precision ++ /// (`ModuleBuilder::note_relaxed`): the serializer decorates each `RelaxedPrecision`. ++ pub(super) relaxed: BTreeSet, + /// The texels the texture family's writes store (`ModuleBuilder::note_stored_texel`), whose + /// stored NaN the write's own notes account for (`lower::texture::half_write_nans` and + /// `half_storage_facts`). Nothing emitted depends on it: the compile reads it to decide +@@ -178,6 +181,9 @@ pub struct ModuleBuilder { + /// Whether a `half`'s NaN bits are the device's conversion's under the float controls + /// ([`ModuleBuilder::native_half_nans`]). + native_half_nans: bool, ++ /// Whether a `half` operation's result is left to the device's own 16-bit precision ++ /// ([`ModuleBuilder::relaxed_half`]). ++ relaxed_half: bool, + } + + /// How a float multiply the device must round on its own and never fuse is emitted +@@ -318,6 +324,28 @@ impl ModuleBuilder { + self.native_half = native; + } + ++ /// Leave the result of a `half` operation the device has an instruction for to the device's ++ /// own 16-bit precision (`narrow::round_native`): decorated `RelaxedPrecision`, with no ++ /// conversion pair. Emission sets it before any instruction, from the profile ++ /// (`narrow::relaxes`); a builder that never sets it rounds every result in its ++ /// [`QuantizeForm`]. ++ pub fn set_relaxed_half(&mut self, relaxed: bool) { ++ self.relaxed_half = relaxed; ++ } ++ ++ /// Whether a `half` operation's result is left to the device's own 16-bit precision ++ /// ([`ModuleBuilder::set_relaxed_half`]). ++ pub fn relaxed_half(&self) -> bool { ++ self.relaxed_half ++ } ++ ++ /// Record `result`, an `f32` holding the result of a `half` operation, as computed at the ++ /// device's own 16-bit precision: the serializer decorates it `RelaxedPrecision` ++ /// (`narrow::round_native`). ++ pub fn note_relaxed(&mut self, result: Value) { ++ self.relaxed.insert(result); ++ } ++ + /// Record `narrowed`, the narrowing to 16 bits of a conversion pair that rounds a `half` in a + /// form measured to answer Metal's where the driver takes + /// `DriverDeviation::MetalHalfConversion` (`narrow::round_measured`). +@@ -393,6 +421,7 @@ impl ModuleBuilder { + ) + + tree_bytes(self.non_uniform.len(), std::mem::size_of::()) + + tree_bytes(self.measured_roundings.len(), std::mem::size_of::()) ++ + tree_bytes(self.relaxed.len(), std::mem::size_of::()) + + tree_bytes(self.stored_texels.len(), std::mem::size_of::()) + + map_bytes(&self.bits) + + map_bytes(&self.lanes) +@@ -1401,6 +1430,7 @@ impl ModuleBuilder { + enabling: self.enabling, + non_uniform: self.non_uniform, + measured_roundings: self.measured_roundings, ++ relaxed: self.relaxed, + stored_texels: self.stored_texels, + #[cfg(test)] + uniform_vars: self.uniform_vars, +diff --git a/crates/metal2vulkan/src/spirv/interp/sched.rs b/crates/metal2vulkan/src/spirv/interp/sched.rs +index 95b2a23cb..c28b18c43 100644 +--- a/crates/metal2vulkan/src/spirv/interp/sched.rs ++++ b/crates/metal2vulkan/src/spirv/interp/sched.rs +@@ -588,6 +588,18 @@ impl Lane { + + /// Set the current frame's value `v`. + pub(super) fn set(&mut self, p: &Prog<'_>, v: Value, val: Val) -> IResult<()> { ++ // A result decorated `RelaxedPrecision` (`narrow::round_native`): the model of a device ++ // that computes it on 16-bit floats, its `f32` result rounded to `half` to nearest even, ++ // denormals kept. A device may flush them and round otherwise ++ // (`PrecisionNote::NativeHalfConversion`). ++ let val = if p.m.relaxed().contains(&v) { ++ super::eval::map1(&val, |x| { ++ let h = super::float::f64_to_f16(super::float::to_f64(x, 32)?); ++ super::float::from_f64(super::float::f16_to_f64(h), 32) ++ })? ++ } else { ++ val ++ }; + self.top_mut()?.define(p, v, val) + } + +diff --git a/crates/metal2vulkan/src/spirv/ir/edit.rs b/crates/metal2vulkan/src/spirv/ir/edit.rs +index 0b3ea69b4..6fe143e6f 100644 +--- a/crates/metal2vulkan/src/spirv/ir/edit.rs ++++ b/crates/metal2vulkan/src/spirv/ir/edit.rs +@@ -406,6 +406,7 @@ impl TypedModule { + let values = &self.values; + self.non_uniform.retain(|v| !removed(v, values)); + self.measured_roundings.retain(|v| !removed(v, values)); ++ self.relaxed.retain(|v| !removed(v, values)); + self.stored_texels.retain(|v| !removed(v, values)); + self.loops.retain(|l| at(&bmap, l.header.index()).is_some()); + for l in &mut self.loops { +diff --git a/crates/metal2vulkan/src/spirv/ir/module.rs b/crates/metal2vulkan/src/spirv/ir/module.rs +index 5a8926bee..f29707485 100644 +--- a/crates/metal2vulkan/src/spirv/ir/module.rs ++++ b/crates/metal2vulkan/src/spirv/ir/module.rs +@@ -1438,6 +1438,9 @@ pub struct TypedModule { + /// to answer Metal's where the driver takes `DriverDeviation::MetalHalfConversion` + /// (`ModuleBuilder::measured_roundings`), recorded as the builder emits them. + pub(in crate::spirv) measured_roundings: std::collections::BTreeSet, ++ /// The results of `half` operations left to the device's own 16-bit precision ++ /// (`ModuleBuilder::note_relaxed`), decorated `RelaxedPrecision`. ++ pub(in crate::spirv) relaxed: std::collections::BTreeSet, + /// The texels the texture family's writes store, whose stored NaN the write's own notes + /// account for (`ModuleBuilder::stored_texels`), recorded as the builder emits them. + pub(in crate::spirv) stored_texels: std::collections::BTreeSet, +@@ -1472,6 +1475,12 @@ impl TypedModule { + &self.measured_roundings + } + ++ /// The results of `half` operations left to the device's own 16-bit precision, decorated ++ /// `RelaxedPrecision` (`ModuleBuilder::note_relaxed`). ++ pub fn relaxed(&self) -> &std::collections::BTreeSet { ++ &self.relaxed ++ } ++ + /// The texels the texture family's writes store (`ModuleBuilder::note_stored_texel`). + pub fn stored_texels(&self) -> &std::collections::BTreeSet { + &self.stored_texels +diff --git a/crates/metal2vulkan/src/spirv/roll/mod.rs b/crates/metal2vulkan/src/spirv/roll/mod.rs +index 1d3a63ed7..c2dae4d3f 100644 +--- a/crates/metal2vulkan/src/spirv/roll/mod.rs ++++ b/crates/metal2vulkan/src/spirv/roll/mod.rs +@@ -1855,6 +1855,7 @@ fn verify( + if m.value(x)?.ty != m.value(y)?.ty + || m.non_uniform.contains(&x) != m.non_uniform.contains(&y) + || m.measured_roundings.contains(&x) != m.measured_roundings.contains(&y) ++ || m.relaxed.contains(&x) != m.relaxed.contains(&y) + || m.stored_texels.contains(&x) != m.stored_texels.contains(&y) + { + return Ok(None); +diff --git a/crates/metal2vulkan/src/spirv/serialize/mod.rs b/crates/metal2vulkan/src/spirv/serialize/mod.rs +index e0b02378e..f8067e12c 100644 +--- a/crates/metal2vulkan/src/spirv/serialize/mod.rs ++++ b/crates/metal2vulkan/src/spirv/serialize/mod.rs +@@ -1684,6 +1684,7 @@ impl Ser<'_> { + // A copy shares its operand's id, so one id may carry several values: each decoration is + // written once per id. + let mut non_uniform = std::collections::BTreeSet::new(); ++ let mut relaxed = std::collections::BTreeSet::new(); + for f in &m.functions { + for p in &f.params { + if self.is_psb_pointer(self.value_ty(*p)?)? { +@@ -1726,6 +1727,11 @@ impl Ser<'_> { + if m.non_uniform.contains(&r) && non_uniform.insert(self.val(r)?) { + self.decorate(w, self.val(r)?, Decoration::NonUniform, &[])?; + } ++ // The result of a `half` operation at the device's own 16-bit precision ++ // (`narrow::round_native`). ++ if m.relaxed.contains(&r) && relaxed.insert(self.val(r)?) { ++ self.decorate(w, self.val(r)?, Decoration::RelaxedPrecision, &[])?; ++ } + } + } + } +diff --git a/crates/metal2vulkan/src/spv.rs b/crates/metal2vulkan/src/spv.rs +index bc26130e3..0d469d1e7 100644 +--- a/crates/metal2vulkan/src/spv.rs ++++ b/crates/metal2vulkan/src/spv.rs +@@ -5841,6 +5841,7 @@ pub enum ImageFormat { + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] + #[repr(u32)] + pub enum Decoration { ++ RelaxedPrecision = 0, + SpecId = 1, + Block = 2, + ColMajor = 5, +diff --git a/crates/metal2vulkan/src/tests/engine_arena.rs b/crates/metal2vulkan/src/tests/engine_arena.rs +index d06163b78..aa26abc1f 100644 +--- a/crates/metal2vulkan/src/tests/engine_arena.rs ++++ b/crates/metal2vulkan/src/tests/engine_arena.rs +@@ -1828,3 +1828,48 @@ fn an_integer_threadgroup_address_outside_the_static_objects_is_refused() { + } + } + } ++ ++/// Indexed constant tables keep their values across the private/blob threshold, ++/// including a constant-expression pointer that enters halfway through the table. ++#[test] ++fn indexed_constant_tables_use_shared_storage_above_four_kib() { ++ for count in [1024, 1025, 4096] { ++ let values = (0..count) ++ .map(|i| format!("i32 {}", i * 3)) ++ .collect::>() ++ .join(", "); ++ let globals = format!( ++ "@table = internal addrspace(2) constant [{count} x i32] [{values}], align 4\n" ++ ); ++ let body = format!( ++ r#" %p = getelementptr i32, ptr addrspace(2) getelementptr ([{count} x i32], ptr addrspace(2) @table, i64 0, i64 128), i32 %t ++ %v0 = load volatile i32, ptr addrspace(2) %p, align 4 ++ %v1 = load volatile i32, ptr addrspace(2) getelementptr ([{count} x i32], ptr addrspace(2) @table, i64 0, i64 7), align 4"# ++ ); ++ let Some(bc) = assemble( ++ &format!("indexed_table_{count}"), ++ &two_word_kernel("table_read", &globals, &body), ++ ) else { ++ return; ++ }; ++ let prof = p("lavapipe"); ++ l3(&bc, "table_read", &prof, "indexed_table"); ++ let (plan, module) = typed_module(&bc, "table_read", &prof).expect("indexed table"); ++ let blob = plan.families.access.blob.as_ref(); ++ assert_eq!(blob.is_some(), count > 1024); ++ let mut res = resources(&plan, 0, 64, 32, &[]); ++ if let Some(blob) = blob { ++ res.buffers.push(Buffer { ++ slot: Some(ResourceSlot { ++ stage: Stage::Kernel, ++ kind: ResourceKind::Synthetic(SyntheticBinding::ConstantBlob), ++ index: 0, ++ }), ++ address: None, ++ bytes: blob.bytes.to_vec(), ++ }); ++ } ++ let want: Vec = (0..32).flat_map(|t| [(128 + t) * 3, 21]).collect(); ++ assert_eq!(run_both(&module, &res).expect("table output"), want); ++ } ++} +diff --git a/crates/metal2vulkan/src/tests/engine_native_half.rs b/crates/metal2vulkan/src/tests/engine_native_half.rs +index f23510ff5..026ac4aad 100644 +--- a/crates/metal2vulkan/src/tests/engine_native_half.rs ++++ b/crates/metal2vulkan/src/tests/engine_native_half.rs +@@ -1001,3 +1001,62 @@ fn a_half_read_from_memory_is_the_devices_widening_on_every_profile() { + } + assert_eq!(forms.len(), 3, "a profile of each form: {forms:?}"); + } ++ ++/// The `OpDecorate`s of `RelaxedPrecision` in `words`. ++fn relaxed_decorations(words: &[u32]) -> usize { ++ let mut at = 5; ++ let mut n = 0; ++ while let Some(first) = words.get(at) { ++ let (count, op) = ((first >> 16) as usize, first & 0xFFFF); ++ // OpDecorate %target RelaxedPrecision ++ if op == 71 && words.get(at + 2) == Some(&0) { ++ n += 1; ++ } ++ at += count.max(1); ++ } ++ n ++} ++ ++/// On lavapipe (`shaderFloat16`, no `DenormPreserve` for 16 bits, no measured conversion) a `half` ++/// sum is left to the device's own 16-bit precision by default: its `float` result is decorated ++/// `RelaxedPrecision` and no conversion pair rounds it, recorded `NativeHalfConversion` ++/// (`narrow::round_native`). Under `exact_answers` (`HalfChainPolicy::EachOperation`) the sum ++/// keeps its conversion to 16 bits and back and nothing is decorated. A conversion from `float` ++/// alone is never decorated, under either policy: its operand is a `float` the function holds. ++/// MoltenVK, whose conversion is measured to be Metal's, and RADV, whose float controls make it ++/// so, decorate nothing. ++#[test] ++fn a_half_operation_is_the_devices_16_bit_one_by_default_without_denorm_preserve() { ++ let Some(bc) = bitcode() else { ++ return; ++ }; ++ let exact = DevicePolicy::default().exact_answers(); ++ let fast = profile("lavapipe", DevicePolicy::default()).expect("profile"); ++ let kept = profile("lavapipe", exact).expect("profile"); ++ assert!(crate::lower::arith::narrow::relaxes(&fast)); ++ assert!(!crate::lower::arith::narrow::relaxes(&kept)); ++ let sum = compile(&bc, "half_sum", &fast).expect("half_sum"); ++ link_and_validate(&sum, &fast).expect("L3"); ++ assert_eq!(relaxed_decorations(&sum.module.words), 1); ++ assert!(sum.precision.contains(&PrecisionNote::NativeHalfConversion)); ++ // The two `fptrunc`s keep their pairs; the sum has none. ++ let (r, k) = ( ++ rounding(&sum.module.words), ++ rounding(&compile(&bc, "half_sum", &kept).expect("kept").module.words), ++ ); ++ assert_eq!((r.narrowings, k.narrowings), (2, 3)); ++ assert_eq!( ++ relaxed_decorations(&compile(&bc, "half_sum", &kept).expect("kept").module.words), ++ 0 ++ ); ++ for p in [&fast, &kept] { ++ let narrow = compile(&bc, "half_narrow", p).expect("half_narrow"); ++ assert_eq!(relaxed_decorations(&narrow.module.words), 0); ++ } ++ for stem in ["moltenvk-m3", "amd-780m-radv", "swiftshader"] { ++ let p = profile(stem, DevicePolicy::default()).expect("profile"); ++ assert!(!crate::lower::arith::narrow::relaxes(&p), "{stem}"); ++ let a = compile(&bc, "half_sum", &p).expect(stem); ++ assert_eq!(relaxed_decorations(&a.module.words), 0, "{stem}"); ++ } ++} +diff --git a/crates/metal2vulkan/src/tests/fast_paths.rs b/crates/metal2vulkan/src/tests/fast_paths.rs +index aa37a1f92..fd70016e4 100644 +--- a/crates/metal2vulkan/src/tests/fast_paths.rs ++++ b/crates/metal2vulkan/src/tests/fast_paths.rs +@@ -87,7 +87,7 @@ fn choices() -> Vec { + choice!(p.completion: CompletionPolicy { Grouped, Exact }), + choice!(p.batch_order: BatchOrderPolicy { Ends, Exact }), + choice!(p.render_lowering: RenderLoweringPolicy { AtCall, Worker }), +- choice!(p.host_page_textures: HostPageTexturePolicy { InPlace, Shadow }), ++ choice!(p.host_page_textures: HostPageTexturePolicy { InPlace, Shadow, InPlaceWrites }), + choice!(p.half_chains: HalfChainPolicy { AtExits, EachOperation }), + choice!(p.ycbcr_in_place_reads: YCbCrInPlacePolicy { Model, Exact }), + choice!(p.pipeline_waits: PipelineWaitPolicy { Bounded, Exact }), +diff --git a/docs/API.md b/docs/API.md +index 6a4573d91..f5389e0e7 100644 +--- a/docs/API.md ++++ b/docs/API.md +@@ -123,8 +123,14 @@ compile does. `DevicePolicy::default()` is `DevicePolicy::UNTRUSTED`: + operation whose flags hold `contract` and `reassoc` and whose result only such operations read + is left unrounded: the chain rounds where a value leaves it, one quantize per exit instead of + one per operation, recorded `FastMathRewrites`. `half_chains = HalfChainPolicy::EachOperation`, +- which `exact_answers()` selects, rounds every operation. A device with `shaderFloat16` computes +- `half` alike under either value. ++ which `exact_answers()` selects, rounds every operation. On a device with `shaderFloat16` ++ whose 16-bit floats keep no denormals (no `DenormPreserve` for 16 bits; Turnip on Adreno 6xx ++ and 7xx), the same switch chooses how a `half` operation the device has an instruction for ++ (`+ - * /`, `fma`, `dot`, `mix`, `sqrt`, `rsqrt`, `exp`, `exp2`, `log`, `log2`) is stated: under ++ `AtExits` its `float` result is decorated `RelaxedPrecision` and the device computes it at its ++ own 16-bit precision (`narrow::round_native`), recorded `NativeHalfConversion`; under ++ `EachOperation` it is computed in `float` and converted to 16 bits and back, as before. Every ++ other device with `shaderFloat16` computes `half` alike under either value. + - **Batches of a queue ordered by their ends** (`BatchOrderPolicy::Ends`, the runtime's choice; no + pipeline differs). On ANV, RADV and NVIDIA's driver, each measured to start no kernel submission + of a `VkQueue` before the one before it completed, each batch of command buffers ends with a +diff --git a/docs/FAST-PATHS.md b/docs/FAST-PATHS.md +index 1f13b41f8..32bfbf476 100644 +--- a/docs/FAST-PATHS.md ++++ b/docs/FAST-PATHS.md +@@ -35,14 +35,15 @@ three consumer pipelines: A2X (`a2x-s4`, `a2x-s96`), Xgc14 (`xgc14-bgra8`) and X + | A blend into a packed 16-bit attachment the device renders through its word view (`A1BGR5Unorm`, `ABGR4Unorm` on the RTX 5080; `WordAttachment`), which the function computes in the interlocked storage form, reads the attachment's codes, where Metal's tile holds the pass's clear and each earlier blend's result as a `half`, finer; under fast math it also sums a few states otherwise than Metal. Every device rendering such an attachment. | The in-shader blend over the codes (no switch) | **Not built yet:** the extended-range attachments' `half` shadow (`render/extended_shadow.rs`, `VariantKey::extended_shadows`) taken by word attachments: the runtime's shadow start and end, its load of the codes and the clear's halves, and the word's shadowed variant in `lower::fragment::fetch`, for a single-sampled pass and a multisampled one (an extended-range color's shadow holds per sample, from the pass's clear). Fast math's regrouped sums have no model | One code: without fast math exact over a loaded attachment (the 2,891 states of the M3 sweep, every channel), under fast math 3,527 of 94,732,288 channels a code apart; after a clear or an earlier blend in the pass, a code apart where the two readings store different codes. `DeviceColorConversion` (`device-color-conversion`) | Not measured. Expected from the shadow's form: an `R16G16B16A16_SFLOAT` image of the pass's extent per such color (8 bytes a pixel, 16.6 MB at 1920x1080), set at the draw it starts by a fill or a load dispatch, and an 8-byte storage read and write per blended fragment inside the interlock beside the word's | + | A heap's colour texture that Metal keeps losslessly compressed (`allowGPUOptimizedContents`, no `ShaderWrite`, `PixelFormatView` or `ShaderAtomic` usage, 16 texels or more a side), on a driver whose compressed images may keep their texels apart from the memory they are bound to (`DriverFact::CompressedImagesHideAliasedWrites`: ANV), is the driver's compressed optimal image, its compression and fast clears kept. A texture Metal keeps uncompressed takes the uncompressed image under either value. The runtime's choice: no pipeline differs. | `heap_compression = HeapCompressionPolicy::Compressed` | `heap_compression = HeapCompressionPolicy::Uncompressed`: an image the driver keeps uncompressed in the heap's bytes, as for a texture Metal keeps uncompressed: host-transfer usage where the device enabled `hostImageCopy` (`heap-texture-host-transfer`), else a linear image (`heap-texture-linear`), else the optimal one (`heap-texture-hides-aliased-writes`) | A write through a buffer or another texture over the texture's bytes may leave its texels as they were or change them otherwise than as written, where Metal's texture reads opaque magenta (`0xffff00ff`) in every texel of drawn content the write covers and the written bytes where it held only a clear; the texture's own writes may show in those bytes compressed. Within Metal's contract: Metal documents such a texture's layout as its own (`allowGPUOptimizedContents` lets it "adjust the private layout"), so no write through another view over its bytes has a defined result there either. `HeapTextureCompressed` (`heap-texture-compressed`), announced once per device where such a texture is made; `runtime_lane::heap_textures` | ANV (Iris Xe of RPL-P; `heap_textures::heap_compression_costs`), a 2048x2048 `RGBA8Unorm` render target in a `Private` placement heap: a pass that clears it 26.9 us against 676.2-676.5 us; a copy into another such texture 647.7-649.1 us against 673.9-693.2 us | + | Batches of command buffers committed on one queue that the scheduler finds ready together (a backlog) go to the driver in one `vkQueueSubmit`, at most four, only the last signalling the queue's timeline, on a device whose own check finds a submission that signals nothing merged into the next (`submit_check.rs`, once per driver, release and kernel: one `vkQueueSubmit` of four empty infos against four calls of one, merged where the one costs at most 0.6 of the four; Mesa 26.2.6's RADV on the Radeon 780M, the RX 7900 XTX and the RX 550 at 0.33-0.38, ANV on the Iris Xe of RPL-P at 0.39, MoltenVK 1.4.2 on the M3 Max at 0.98; a device whose check has not answered, or that does not merge, takes the ungrouped form under either value of the switch): one kernel submission, where ANV on i915 lists every `VkDeviceMemory` in each (the consumer's guest drag on the Iris Xe: about 300 entries per execbuffer, 260 of them its host imports, at 112 to 128 execbuffers a frame). A batch's event signals ride to the group's end, which signals each event's semaphore to the highest index the batches held signal it to (an event's indices rise in commit order). A batch that signals a binary semaphore, takes a fence, binds a sparse update, does first work, names the caller's wrapped image, or waits for anything not seen complete (an event, another queue's work) goes alone, after the batches held: a batch with a wait leads a group, never joins one. So that checked command buffers (the consumer checks each) are found ready together, the submission checks of up to three batches right behind a line's head that goes run with the head's, in commit order, none committed after another queue's batch: such a check runs before the previous batch's `vkQueueSubmit` rather than after it, in the scheduler pass that submits its own batch. A pass that finds no batch ready sends the batches held, unless two or more earlier submissions of their `VkQueue` are not seen complete: the GPU runs those first in any case, so the batches stay held, and those committed meanwhile join them, until one submission is left in flight (a completion turn wakes the scheduler at its value), the group fills, a later batch sends it, a head waits for the GPU, or a thread waits for one of their command buffers' completion, which sends them at once; nor are they kept where the scheduler's thread runs next a submission handler of another queue's command buffer: they go first, as Metal schedules a queue's command buffers whatever another queue's handler does (an M3 Max, probe `cross_queue_handler_waits`); a batch that ends a command buffer with a switch of storage forms riding on it, which settles by its submission's outcome, is never kept so. A batch a committing thread submits (`CommandBuffer::commit_and_submit`) goes alone where its `VkQueue` runs at most one earlier submission not seen complete, as the pass that held it would send the group at once; where two or more run it is left to the scheduler's thread, which holds it. A held batch that ends its command buffer is `Scheduled` as it is held, its submission handlers run first, then its scheduled handlers, and its kernel end time set, as an M3 Max schedules a command buffer committed behind running work 0.04 to 0.18 ms after its commit. The runtime's: no pipeline differs. | `completion = CompletionPolicy::Grouped` | `completion = CompletionPolicy::Exact`: one submission per batch, each signalling its own values; each line's head checked alone | Each command buffer of a submission but the last is seen complete (status, completed handlers, waits), and the event values it signals are reached (by the GPU waits on other queues, the host's waits, value reads and listeners), once the last completed: late by the GPU time of at most three later batches (about 0.1 to 0.2 ms each in the consumer's compositor frames on the Iris Xe; about 20 ms behind a 20 ms batch), never early, in commit order. Metal's contract states no bound on when completion is seen (an M3 Max reports a command buffer complete, its handlers, status, waits and event listeners alike, 0.04 to 0.10 ms after its GPU end), and no member but the first waits, so no wait holds a member back: a valid program sees only later completion, as of slower GPU work. A batch kept while earlier work runs is seen complete within the same bound, and, where the one submission left in flight ends before its group reaches the GPU, late besides by that gap: at most a completion turn's reaction and the group's `vkQueueSubmit`. Commands, barriers and GPU times are unchanged. `GroupedCompletion` (`grouped-completion`), announced once per device at its first such submission; `runtime_lane::grouped_submissions` | ANV (Iris Xe of RPL-P; `frame_costs::frame_costs_over_imports`, the checked device, 67 command buffers a frame, GPU at 467-783 MHz), with 520 host imports live, where the Exact form's submissions fall behind the commits as the guest's do: 32.0-33.2 kernel submissions a frame against 67.0, 3.4-3.5 ms of execbuffer ioctl a frame against 6.9-7.9 ms, the scheduler thread's CPU 61-63 us a command buffer against 121-136 us, a frame through completion 4.7-5.9 ms against 8.6-9.7 ms; in the guest's shape with 520 (a second queue taking every 20th command buffer, each signalling a shared event, every 10th waiting for one): 26.2-26.7 kernel submissions a frame against 67.0, 2.8 ms of ioctl a frame against 6.9-7.0 ms, the scheduler 52-53 us a command buffer against 122-123 us, a frame through completion 6.0 ms against 8.7 ms; with 260, where it keeps up, 64.9-66.3 kernel submissions a frame against 67.0, the CPU and wall times alike. The cost (`grouped_submissions::grouped_completion_delays`, 8 command buffers of 0.3-0.5 ms of GPU work ready together): the host saw each but a group's last complete later by the later ones' GPU time, the last 0.1-0.3 ms later than alone (the completion turn's per-command-buffer work for the group, done at its end). Keeping a group while earlier work runs (UHD 770 of RPL-S, ANV, the guest's shape with 520 host imports, a submission handler on every command buffer and a 16 MiB fill ahead of each, about 1.3 ms of GPU each; the kept form against the batches sent as found): committed every 1 ms, 18.1-18.2 groups kept a frame, 24.9-25.0 kernel submissions a frame against 57.9, 2.18-2.90 ms of execbuffer ioctl a frame against 4.64-5.43 ms, the scheduler thread's CPU 67-82 us a command buffer against 113-118 us, the GPU idle within a frame's span 0.61 ms against 0.68-0.69 ms, a frame through completion 89.5-89.6 ms alike, completion seen later (GPU end to completed handler: mean 1.82-1.86 ms against 0.43-0.46 ms, p50 1.57-1.60 against 0.11-0.13, p99 4.35-4.37 against 4.09-4.11, max 4.52-4.55 against 4.18-4.23); committed at once, 18.0 kernel submissions a frame against 23.4; with no fill, 18.0 against 20.6, the GPU idle 0.59 ms against 0.58-0.66 ms. In the consumer's guest drag on the Iris Xe (i9-13900H; three passes each way): 24.4-27.5 kernel submissions a frame against 42.4-43.8, the scheduler thread 0.4-1.7 ms of CPU a frame less, 15.7-17.3 frames a second against 16.4-17.6: the frame rate alike within noise, as the scheduler thread does not bound that frame. A batch a committing thread submits alone (RADV, Mesa 26.2.6, RX 7900 XTX, whose check finds submissions merged; two rounds each against the form that left every batch that may wait in a group to the scheduler's thread, `tests::completion_latency`): a checked fill's `commit_and_submit` to its submission handler p50 30.4-32.8 us, p90 32.3-35.9, p99 48.8-57.8 against 36.2-55.7, 56.4-176.3 and 112.1-198.5, the call returning after its submission (32-34 us against 4-8); committed 2 ms apart between two shapes of `commit`, its commit to submission p50 31.9-36.1 us, p90 102.6-135.7 against 39.2-50.1 and 152.1-169.7, and the `commit`s beside it alike (p50 32.1-47.3 us against 38.9-46.7, the scheduler thread's wake p50 4.0-4.5 us against 4.1), as the committing thread wakes that thread after a batch it submitted alone, as the batch's hand-off to it would have. On lavapipe, whose check finds no merge, the two forms alike, the scheduler's thread woken alike (a checked fill's p50 12.5-13.4 us against 12.4-13.4). | +-| Batches of command buffers on one `VkQueue`, on a driver measured to start no kernel submission of a queue before the one before it completed (ANV, RADV and NVIDIA's; `batch_order.rs`), are ordered by their ends: each ends with a barrier ordering its work before every later command of its `VkQueue` (the exact form's end orders it before the host's reads and writes alone), so a batch that depends on an earlier one there begins with no barrier of its own, often no command buffer of barriers, and its first encoder's GPU start is written at the bottom of the pipe, once everything before it completed. Metal starts an independent command buffer committed behind running work while that work runs (an M3 Max under macOS 27.0.1, probe `gpustart`); those drivers ran two batches of independent work one after the other under every shape measured (no end barrier, the host's or the widened one; in one `vkQueueSubmit` or two, merged into one kernel submission or not), where two dispatches of one command buffer ran side by side. The runtime's: no pipeline differs. | `batch_order = BatchOrderPolicy::Ends` | `batch_order = BatchOrderPolicy::Exact`: a barrier for each dependency of a batch's work on an earlier batch's of its `VkQueue`, the end's barrier to the host's reads and writes alone, the GPU start written at the top of the pipe; every other driver takes it under either value | None: on those drivers each batch starts once the one before it completed under either value, so the work's order, its results and its GPU times are the same, and a batch's GPU start is the end of the work before it, when such a driver starts it. Not announced; `runtime_lane::commit_work` and `runtime_lane::gpu_times` test both values | UHD 770 (ANV, RPL-S), the consumer's frame shape over 520 host imports, 67 command buffers a frame, Grouped completion, the widened end barrier against the exact form (a probe build of each): with no fill ahead of each command buffer, 67.0 command buffers submitted a frame against 129.3-129.7, the GPU's span a frame 1.84-1.90 ms against 1.99-2.06 ms, a frame through completion 2.13-2.22 ms against 2.27-2.38 ms; with a 1 MiB fill ahead of each, 134.0 against 262.9, 8.03-8.22 ms against 8.32-8.42 ms, 9.00-9.15 ms against 9.25-9.36 ms. Radeon 780M (RADV, Mesa 26.0.8), this switch: with no fill, 67.0 against 128.9-129.0, a frame through completion 48.3-48.6 ms against 42.6-47.0 ms; with a 16 MiB fill committed at once, 134.0 against 251.0-251.1, 50.6-50.8 ms against 50.8-57.3 ms; committed every 1 ms, 134.0 against 249.7-250.2, 71.3-83.6 ms against 106.3-113.6 ms (each kernel submission there leaves about 2.5 ms of the GPU idle over 520 host imports, which sets these frames) | ++| Batches of command buffers on one `VkQueue`, on a driver measured to start no kernel submission of a queue before the one before it completed (ANV, RADV and NVIDIA's; Turnip by its source, one command processor running a queue's ring in order; `batch_order.rs`), are ordered by their ends: each ends with a barrier ordering its work before every later command of its `VkQueue` (the exact form's end orders it before the host's reads and writes alone), so a batch that depends on an earlier one there begins with no barrier of its own, often no command buffer of barriers, and its first encoder's GPU start is written at the bottom of the pipe, once everything before it completed. Metal starts an independent command buffer committed behind running work while that work runs (an M3 Max under macOS 27.0.1, probe `gpustart`); those drivers ran two batches of independent work one after the other under every shape measured (no end barrier, the host's or the widened one; in one `vkQueueSubmit` or two, merged into one kernel submission or not), where two dispatches of one command buffer ran side by side. The runtime's: no pipeline differs. | `batch_order = BatchOrderPolicy::Ends` | `batch_order = BatchOrderPolicy::Exact`: a barrier for each dependency of a batch's work on an earlier batch's of its `VkQueue`, the end's barrier to the host's reads and writes alone, the GPU start written at the top of the pipe; every other driver takes it under either value | None: on those drivers each batch starts once the one before it completed under either value, so the work's order, its results and its GPU times are the same, and a batch's GPU start is the end of the work before it, when such a driver starts it. Not announced; `runtime_lane::commit_work` and `runtime_lane::gpu_times` test both values | UHD 770 (ANV, RPL-S), the consumer's frame shape over 520 host imports, 67 command buffers a frame, Grouped completion, the widened end barrier against the exact form (a probe build of each): with no fill ahead of each command buffer, 67.0 command buffers submitted a frame against 129.3-129.7, the GPU's span a frame 1.84-1.90 ms against 1.99-2.06 ms, a frame through completion 2.13-2.22 ms against 2.27-2.38 ms; with a 1 MiB fill ahead of each, 134.0 against 262.9, 8.03-8.22 ms against 8.32-8.42 ms, 9.00-9.15 ms against 9.25-9.36 ms. Radeon 780M (RADV, Mesa 26.0.8), this switch: with no fill, 67.0 against 128.9-129.0, a frame through completion 48.3-48.6 ms against 42.6-47.0 ms; with a 16 MiB fill committed at once, 134.0 against 251.0-251.1, 50.6-50.8 ms against 50.8-57.3 ms; committed every 1 ms, 134.0 against 249.7-250.2, 71.3-83.6 ms against 106.3-113.6 ms (each kernel submission there leaves about 2.5 ms of the GPU idle over 520 host imports, which sets these frames) | + | Command queues on a device whose GPU idles when consecutive work comes from another `VkQueue` share `VkQueue`s by choice (`queue_share.rs`): the runtime's own self-check, not a `DriverFact` row, measures that idling once per driver, release and runtime on a thread of its own and keeps its verdict (`Idles`, `DoesNotIdle`, `Unchecked`) in the shader cache's directory, as the in-place check keeps its own (`queue_switch_check.rs`: 48.7 to 51.3 us a switch on the RTX 5080 Laptop GPU, NVIDIA 615.71.09, `Idles` above 20 us). A command queue that makes a command buffer while it holds no other uncompleted one moves onto the busy `VkQueue` of its family whose oldest batch has run under 0.5 ms, the most batches in flight first, so short command buffers of several queues run with no switch between them. A queue one of whose command buffers ran on the GPU for over 2 ms, by its own timestamps (not the host's view of how long it ran as its `VkQueue`'s oldest, which also holds its wait for the GPU to start it and the waiter's lateness to see it complete), or held its `VkQueue` for over 2 ms from the submission of a batch that waited on the GPU for another `VkQueue`'s work (an `MTLEvent`'s signal, the tracker's wait, new resources' first work) to its own GPU end, is long: no queue joins its `VkQueue` while it lives, and it moves once more, at its next idle point, to an idle `VkQueue` no long or unshared queue holds. A queue whose command buffers must never wait behind another queue's (a compositor's beside an application's) is made with `Device::new_command_queue_unshared`: it never moves and no queue joins its `VkQueue`. Presenting queues and a queue that lent its `VkQueue` (`with_vk_queue`) never move. Before the verdict, and where it is not `Idles` or there are no two `VkQueue`s of one family (MoltenVK, RADV, lavapipe), every queue keeps the `VkQueue` it took at its creation; so does every queue of a device whose command buffers write no GPU timestamps it reads (`gpu_times_from_host`, or a driver whose timestamps wait for every queue's), which does not share and is counted (`WorkCounts::queue_sharing_untimed`). The runtime's: no pipeline differs. | `queue_sharing = QueueSharingPolicy::Adaptive` | `queue_sharing = QueueSharingPolicy::PerQueue`: each command queue keeps the `VkQueue` it took at its creation, round robin, and the check does not run | A command buffer waits behind the command buffers of the queues sharing its `VkQueue`, where Metal's queues wait for each other only inside one command. The residual stall: once per logical queue that goes long, for a queue that joined its `VkQueue` during its short phase, about one long command buffer's length: the command buffers committed behind that queue's first long one there wait for its whole length (and those behind its later ones while it holds work, until its next idle point). Measured on the RTX 5080 Laptop GPU beside an application that alternates draws with 30 ms command buffers: a shared compositor's commits at 4,800 a second had 64 completions over 10 ms, at most 26.4 to 30.1 ms, in each of 3 runs of each of two such shapes (one join each), against none over 10 ms under `PerQueue` in those 6 runs (at most 5.6 to 9.7 ms); with 2 ms of draws between long command buffers, none over 10 ms in 3 runs (at most 5.7 to 9.8 ms) against 7, 0 and 0 under `PerQueue` (at most 6.9 to 11.2 ms). On an unshared queue, 0 over 10 ms in every run. A command buffer's GPU start time includes the other queues' work before it. `QueueSharing` (`queue-sharing`), at the device's first join | Command buffers a second on the RTX 5080 Laptop GPU, one-draw command buffers, each thread its own queue, the default against `PerQueue`, runs after a lock hold's first: 8 threads with 3 in flight 50.7k to 54.7k against 27.4k to 28.0k (5 runs each); 8 threads with 1 in flight 47.0k to 50.1k against 13.9k to 14.3k (5); 2 threads with 3 in flight 28.0k to 53.7k against 21.4k to 28.0k (10 each, in turn, the default above `PerQueue` in each pair; no thread's queue marked long), and beside another process's 30 ms command buffers 5.1k to 6.7k against 4.0k to 4.1k (3 each; the host's view marked a thread's queue long in 1 of 3 runs, which then ran at `PerQueue`'s 4.1k); 2 threads with 1 in flight 16.9k to 20.7k against 10.1k to 11.5k (5); one thread over 16 queues in turn 21.7k to 43.3k against 10.2k to 11.1k, over 2,048 20.0k to 39.2k against 10.2k to 11.2k (4). Beside an application whose queue never shared with the compositor (isolation, 300 and 30 dispatches), both 0 over 10 ms | +-| A chain of `half` additions, subtractions and products on a device without `shaderFloat16` (vk12-floor, SwiftShader, Pascal; capability emulation: the module computes each in `float` and rounds it by the device's `OpQuantizeToF16`): an operation whose flags hold `contract` and `reassoc` and whose result only such operations read is held at `float` precision unrounded, so the chain rounds where a value leaves it (a store, an output, a conversion, a comparison, a call, a phi, a vector move, an operation without both flags), one quantize per exit instead of one per operation (`lower::arith::half_chain`). A device with `shaderFloat16` computes `half` alike under either value. | `half_chains = HalfChainPolicy::AtExits` | `half_chains = HalfChainPolicy::EachOperation`: every operation's result rounded to `half`, as the AIR computes it | The result may differ from the rounded operations' by the roundings the chain skips, each within half a unit in the last place of a `half` at its step, compounded along the chain; a value inside the chain past `half`'s range (65504) stays finite where the rounded operations reach infinity. Within the AIR's flags, which let a compiler regroup and fuse such operations. `FastMathRewrites` (`fast-math-rewrites`) | Not timed. Size on vk12-floor (the fixtures' SPIR-V words, the default against the accurate form): `hfb_filter_bank` 121,270 against 131,766 (1,024 quantizes against 2,464), `grd_half_many_samples` 82,019 against 84,627, `nan_half_fast` 9,808 against 9,832 | ++| A chain of `half` additions, subtractions and products on a device without `shaderFloat16` (vk12-floor, SwiftShader, Pascal; capability emulation: the module computes each in `float` and rounds it by the device's `OpQuantizeToF16`): an operation whose flags hold `contract` and `reassoc` and whose result only such operations read is held at `float` precision unrounded, so the chain rounds where a value leaves it (a store, an output, a conversion, a comparison, a call, a phi, a vector move, an operation without both flags), one quantize per exit instead of one per operation (`lower::arith::half_chain`). A device with `shaderFloat16` computes `half` alike under either value. Half FMA, dot, mix, exp and log keep float32 intermediates before the half conversion. Half division scales both operands by a common power of two into the native divisor range. | `half_chains = HalfChainPolicy::AtExits` | `half_chains = HalfChainPolicy::EachOperation`: every operation's result rounded to `half`, as the AIR computes it | The result may differ from the rounded operations' by the roundings the chain skips, each within half a unit in the last place of a `half` at its step, compounded along the chain; a value inside the chain past `half`'s range (65504) stays finite where the rounded operations reach infinity. Within the AIR's flags, which let a compiler regroup and fuse such operations. `FastMathRewrites` (`fast-math-rewrites`) | Not timed. Size on vk12-floor (the fixtures' SPIR-V words, the default against the accurate form): `hfb_filter_bank` 121,270 against 131,766 (1,024 quantizes against 2,464), `grd_half_many_samples` 82,019 against 84,627, `nan_half_fast` 9,808 against 9,832 | ++| A `half` operation the device has one instruction for (`+ - * /`, `fma`, `dot`, `mix`, `sqrt`, `rsqrt`, `exp`, `exp2`, `log`, `log2`) on a device with `shaderFloat16` whose 16-bit floats keep no denormals (no `DenormPreserve` for 16 bits: Turnip on Adreno 6xx and 7xx), where no float controls make the conversion to 16 bits Metal's: its `float` result is decorated `RelaxedPrecision` and left to the device's own 16-bit precision (`narrow::round_native`, `narrow::relaxes`), in place of a conversion to 16 bits and back after it. Mesa's compiler folds that pair into nothing (`nir_opt_algebraic.py`, the inexact `f2f32` of an `f2f`), so the accurate form computes every such operation in `float` unrounded, on 32-bit registers. | `half_chains = HalfChainPolicy::AtExits` | `half_chains = HalfChainPolicy::EachOperation`: the operation in `float`, then the device's conversion to 16 bits and back | Each such result is the device's 16-bit one: within one unit in the last place of a `half` (a relative 2^-10) of the `float` result at its step, compounded along a chain; a magnitude below the smallest normal half (2^-14) may be zero; a magnitude from 65520 up is infinite. These are the values Metal's own `half` takes but for the denormals. `NativeHalfConversion` (`native-half-conversion`) | Not timed on a device. ir3 for the Adreno 740 (Mesa's own statistics, scalar operations on 16-bit registers of all in the fragment's body, the default against the accurate form): QuartzCore's `glass_background_all_lph` 724 of 1,085 against 113 of 1,010, its registers 12 against 16 and its waves 8 of 128 fibers against 12 of 64; `variable_blur_frag_lph` 620 of 968 against 43 of 902; `narrow_blur_27_frag_lph` 160 of 244 against 12 of 258 | + | A texture that samples, reads, renders into it or is written through a shader (`ShaderWrite` or `ShaderAtomic` usage, of texels of 1, 2, 4, 8 or 16 bytes) over host pages (a no-copy buffer's, or a no-copy heap's buffer's, imported by their pointer or as dma-bufs; or a buffer in fresh pages; or a copied host buffer's copy of them, `copied.rs`, its image bound in the copy's memory where the check passed for the pages, and for the copy's fresh pages or, for a `DRM_FORMAT_MOD_LINEAR` image in memory the device allocates, its pooled memory, `WorkCounts::copied_textures_in_place`) is a linear image aliasing the pages in place, where the device makes the image (its format and usage linear, at the pages' pitch) and its own check shows that the operations Vulkan lets read and write every byte bound to it leave every byte alone while the CPU writes beside it: the image's first move (`VUID-VkImageCreateInfo-pNext-01443` makes such an image start `UNDEFINED`), and for one that renders a render pass's load and store operations and its moves into `ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT` and back, and for one a shader writes a kernel's storage writes of its texels (which of a linear image a driver may widen past a texel), each access checked apart (a shader's writes of 1- or 2-byte texels apart from those of wider ones, which a driver may store through a wider access), and each apart for texels that share a line of the device's memory (`nonCoherentAtomSize`, "the size and alignment in bytes that bounds concurrent access to host-mapped device memory") with bytes beside them and for texels that share none (their start, their end and, with bytes between rows, each row's start and end on a line; `in_place::RowEnds`): a byte changed beside texels that share none fails both, one changed beside texels that share one fails those alone (the Iris Xe, ANV, Mesa 26.2.6, whose clear and store, and draws, write back whole the 64-byte line that holds a row's last texels, fails renders, and writes, for rows that end inside a line, a byte between rows changed, and passes both for rows that end on one: a 3840x2160 `BGRA8Unorm` surface's 15360-byte rows are 240 lines each). A kernel's writes of a texture over a buffer's rows (no IOSurface plane's) whose rows, each to its pitch, start and end on a line, its buffer holding the last row to its pitch, take a verdict of their own from the same rounds, which a round that changed only bytes of its rows' padding (after a row's last texel, before its pitch, the last row's included) passes (`in_place::Padding`, `Access::BufferWrites`): Metal leaves those bytes to the GPU there (an Intel UHD Graphics 630 under macOS 26.7.1 wrote older values back over the CPU's writes to them under a kernel's writes of a texture over a buffer, the last row's included, where an M3 Max under macOS 27.0.1 and a Radeon Pro 5300M kept them, and all three kept them under draws, clears and stores and over IOSurfaces; `crates/metal2vulkan-conformance/probes/row_padding_writes.m`), and the rule follows the x86 Macs' envelope (docs/API.md, section 10, **Kernels**), so the Iris Xe's written textures over a buffer take the image in place whatever their rows' ends. A deliberate, scoped exception to relying on documented Vulkan behaviour alone, the owner's rulings (2026-10-26, and 2026-10-27 and 2026-11-07 for render targets and written textures: the GPU works on the caller's pages in place as a Mac's does, whatever a linear image costs it, on a discrete GPU too; `in_place.rs`). A texture over a buffer of the runtime's pooled memory whose rows the device's own linear image lays out otherwise (a padded pitch the driver does not take: ANV lays linear rows tight; rows apart only over a buffer that declares them unobserved, `BufferDescriptor::padding_unobserved`) is, by the same rule, a `DRM_FORMAT_MOD_LINEAR` image at Metal's pitch bound in the buffer's memory where the device makes one (no external memory, which Vulkan allows: `VUID-VkImageCreateInfo-tiling-02261`, `VUID-vkBindImageMemory-image-01445`), made `UNDEFINED` (`VUID-VkImageCreateInfo-initialLayout-12478`), whose first move the check's pooled class judges (`PagesClass::Pooled`: a buffer of that memory, its images at Metal's pitch alone; `WorkCounts::pooled_pitched_images`); a texture made before that class's verdict is a shadow that moves to that image once the verdict passes, as one over host pages moves in place (below; `WorkCounts::pooled_textures_moved_in_place`, `pooled_texture_shadows_freed_after_move`). A texture over a buffer whose memory the runtime exports (`exported_memory.rs`: a block that exportable buffers of one handle type share) is, by the same rule, the device's linear image or that `DRM_FORMAT_MOD_LINEAR` image made for the memory's handle type, which an image bound there must name (`VUID-vkBindImageMemory-memory-02728`), and so made `UNDEFINED` (`VUID-VkImageCreateInfo-pNext-01443`), whose first move the check's exported classes judge, one per handle type (`PagesClass::ExportedFd`, `PagesClass::ExportedDmaBuf`: an exportable buffer of that handle type, its textures taking the product's image), a texture made before their verdicts moving to it once they pass as over the pooled memory; a shadow, its notice naming why, where the memory is a buffer's dedicated allocation (`VUID-vkBindImageMemory-memory-10926`) or the device's image format query (`VkPhysicalDeviceExternalImageFormatInfo`) takes the handle type for no such image without a dedicated allocation. A texture over a buffer in a block of the runtime's buffers allocated exportable as an opaque file descriptor and never exported (`memory.rs`'s "Buffers in exportable blocks") is the device's linear image made for that handle type, made `UNDEFINED`, and never a `DRM_FORMAT_MOD_LINEAR` one (RADV asks a dedicated allocation of one made for any handle type), whose first move and accesses the check's class of such blocks judges (`PagesClass::FdBlocks`: a buffer in such a block, its textures taking the product's images, made only where such a block holds every image a texture over a plain block's buffer that keeps its padding takes in place, `texture/fd_blocks_cover.rs`); the device's buffers lie there only where those verdicts give every texture over one what it takes over a plain block's buffer (`in_place::fd_blocks_take`), in plain blocks until they are known, and a buffer that declares its padding unobserved lies in a plain block wherever, so that a texture over it whose rows lie apart keeps its `DRM_FORMAT_MOD_LINEAR` image. A texture that renders over a buffer that declares its padding unobserved, its rows each to its pitch sharing no line with bytes past them, takes the verdict of renders that lets a byte of its rows' padding change (`Access::UnobservedRenders`, `Padding::Unobserved`), which the renders' own rounds give as the writes' give theirs: the declaration lets a pass's load and store write those bytes (the Iris Xe's clear and store write back the 64-byte line that holds a row's last texels whole), and a byte past the last row's pitch, or a texel, fails it. The check runs once per driver and driver version (its record named by the driver and the check's own version, not the runtime release), per class of pages, access and row ends, on a thread of its own, the verdicts it measured kept in the shader cache's directory with those it could not check for a cause that lasts for the driver (the device lacks an extension, a feature, an import, a format or an image of the shape), while one it could not check for a cause that may pass (pages or memory not made, an import refused, its writer thread not started, a move or the stores that failed) is checked again by the next device, its reason noted; a texture made before its verdict is a shadow, made without waiting for the check (its rounds take seconds on some devices and builds; `host_texture_shadows_check_pending`), and moves in place once its class's verdicts pass, at the commit of the device's only uncommitted command buffer, with no copy (`host_textures_moved_in_place`), its shadow's image freed once the command buffer that took the move completed (`host_texture_shadows_freed_after_move`), or, where a view made before the move presents it, a shadow made over it holds it as its truth, or the texture is untracked, once those and the texture went. A texture a shader writes of texels of a width no check stores (none of 1, 2, 4, 8 and 16 bytes), a check that failed or made no image, and a format, usage or pitch no image takes keep the shadow (`shadowed-texture`, its reason named, `WorkCounts::host_texture_shadows_*`, those that render and those a shader writes counted apart in `host_texture_shadows_render_target` and `host_texture_shadows_shader_write`). On a discrete GPU a `Shared` texture and an IOSurface's are in place too, read across the bus, never copied to the device's memory for speed; a `Managed` texture over a no-copy buffer's bytes that only samples or reads, over memory that carries a log of its writes, is a device copy instead (`WorkCounts::device_copy_textures`), the GPU's copy of the two Metal keeps of a `Managed` resource on a Mac with a discrete GPU, its copies in only the rows the log reports changed. | `host_page_textures = HostPageTexturePolicy::InPlace` | `host_page_textures = HostPageTexturePolicy::Shadow`: every texture over host pages a shadow, its texels copied in before each command buffer that uses it and merged back after each write (the host-pages bullet under **Costs of matching Metal**) | Exact where the driver keeps what its check showed: the check covers texels, the bytes between rows and those after the image, of 1, 2, 4, 8 and 16-byte texels and a 3840x2160 surface, three rounds each, for an image that reads and one that renders (its pass a clear at load and a store, then a draw over its last column in a pass that loads and stores it; a resolve into it is not among the rounds), the CPU writing beside the texels sweep after sweep until each step completed, so that a device that writes back bytes it read beside them, wherever in the step, puts back an older value the check finds, and, for one a shader writes, of 4, 8 and 16-byte texels, rows apart (of widths no multiple of 4 and odd heights among them) or tight, and 1920x1080, and apart of 1- and 2-byte texels, rows apart at a 256-byte pitch (of rows that end inside a 4-byte word and odd heights) or tight, and 1024x64, each texel stored by a kernel through an unsigned-integer view (a program's atomics, and its stores through a float view, are not among the rounds); a driver whose operations changed a byte outside every round's window would undo a CPU write made meanwhile. A kernel's writes of a texture over a buffer whose padding is free may change that padding, as an x86 Mac's GPU's do (above). Not announced where the check passed. A CPU write reaches the image's reads as it reaches the buffer's (Host Write Ordering Guarantees) | See **Costs of matching Metal** (host pages): the shadow copies a sampled 3840x2160 surface in whole each command buffer, and a render target's shadow copies it in, keeps it and merges it back at each render pass instance | + | A render or compute state's creation hands the driver's making of its predicted pipeline (`pipeline/predicted.rs`, `pipeline/dispatch_making.rs`) to the runtime's pool, kept by the state as a draw's or dispatch's making is, and waits for it `pipeline_creation_wait` at most; a committed command buffer's recordings wait until its commit plus `pipeline_commit_wait` at most, one deadline its commands share, for its draws' and dispatches' (a compute encoder's, a render pass's tile dispatches) pipelines still being made (the scheduler wakes at the bound, `StepWaits::until`), an execution's replayed draws (`render/pending.rs`) among them, whose pipelines the commit's settle hands to the pool (a device replay's, one recording for every command, waits until made under either value), then record each such command as nothing: a draw's barriers and visibility query, a dispatch's barrier (a tile dispatch's copies into its scratch and back), no draw or dispatch, the command buffer running on without error. The making goes on, and the command buffers after it draw and dispatch with the pipeline once made. Metal compiles inside `newRenderPipelineState`, so no draw of a made state waits or is dropped; a host that creates a state inside a command buffer's encoding (the consumer creates a guest's state at an execution's first lookup of it) held that execution through the driver's whole compile: 11.59 s inside one execution on an M2 Pro (MoltenVK 1.4.2, a cold Metal compiler service; a sample of such a stall finds the compile's thread waiting in `MVKMetalCompiler::compile` for Metal's compiler), past the guest's 10 s FIFO check, which ended the guest's task. Each bound is 4.5 s: a creation and a commit may fall in one step of the guest's FIFO (a task's jobs run one after another, and a draw's own variant waits under its command buffer's bound), and no runtime call waits for both, so one step waits 9 s at most for makings, under that 10 s check. A creation waits on no command buffer's deadline and a `commit` on no creation's; a `commit` waits until its own deadline at most (it records there what it can, an execution of render commands the CPU writes among them), every other recording on the scheduler's thread, a host step's on one of the steps' threads. Made until made under either value, in the call that makes them: a draw that records nothing, one in a pass of custom sample positions and one recorded again into another instance of its pass (each in the draw's call or an execution's replay), a device replay's draws (in its `commit`), and a failed making made again (`PipelineWaitPolicy`'s docs). The runtime's: no pipeline differs. | `pipeline_waits = PipelineWaitPolicy::Bounded`, `pipeline_creation_wait = 4.5 s`, `pipeline_commit_wait = 4.5 s` | `pipeline_waits = PipelineWaitPolicy::Exact`: the creation makes its predicted pipeline before it returns (a render state's first key on its own thread, the rest on the pool, joined; a compute state's on its own thread), an execution's settle makes its replayed draws' pipelines at the commit, and a command buffer waits for each draw's and dispatch's pipeline until it is made | A draw whose pipeline was still being made `pipeline_commit_wait` after its command buffer's commit renders nothing in that command buffer (only there: later ones draw it), where Metal's renders: content drawn once (a layer, a glyph cache) stays without it until redrawn. A creation returns after `pipeline_creation_wait` at most with its making in flight (`WorkCounts::creations_left_making`). `DrawDroppedStillMaking` (`draw-dropped-still-making`), at each such draw where `DeviceOptions::announce_each` asks, else once per device; `CommandBuffer::dropped_draw` names the first with `VK_TIMEOUT`; `WorkCounts::draws_dropped_still_making`; `deferred_calls::a_draw_whose_pipeline_is_still_being_made_at_its_bound_runs_nothing` and `under_the_exact_wait_a_held_making_holds_its_command_buffer_until_it_lands`. A dispatch whose pipeline was still being made then writes nothing in that command buffer, where Metal's runs: `DispatchDroppedStillMaking` (`dispatch-dropped-still-making`), `WorkCounts::dispatches_dropped_still_making`, the first named by `dropped_draw`; `deferred_calls::a_dispatch_whose_pipeline_is_still_being_made_at_its_bound_runs_nothing`, `a_compute_state_s_creation_waits_its_bound_for_its_predicted_making` and `a_replayed_draw_whose_pipeline_is_still_being_made_at_its_bound_runs_nothing` | No GPU cost. The exact form holds the creating thread, and the command buffer, for the whole compile: up to 11.59 s on the M2 Pro's cold boot (that boot's 2,483 render state creations, translation included: 300 over 16 ms, 151 over 100 ms, 14 over 250 ms, 3 over 1 s, the longest 11.59 s, 5.82 s and 1.17 s), where the default holds a creation for 4.5 s at most and a command buffer for 4.5 s from its commit, which drops the draws of the compiles that outlast those (2 of that boot's 2,483 were over 4.5 s) for the command buffers committed before they land; bounds of 1 s dropped 39 draws in a cold boot on the M2 Pro, and the guest's menu bar, drawn once, stayed half drawn | + | A line Apple's `setLineWidth:` widens (`w' = floor(16 w) / 16`, other than 1), and a triangle's edge in fill mode `Lines`, is drawn by the device's own lines at `w'` wherever it draws that width (`wideLines`, `w'` in `lineWidthRange` and above 0), where Metal draws the parallelogram Vulkan's non-strict wide lines may be (`render/line_width.rs`). No device's are taken as it, announced on every device: a strict device's rectangle with square ends (lavapipe, NVIDIA), lines Vulkan does not state as parallelograms (ANV, and every width below 1), a width rounded to the granularity, and a driver's parallelograms (`strictLines` unset, `nonStrictWideLinesUseParallelogram` set), whose ends the driver places (RADV on the Radeon RX 7900 XTX draws 21 of `runtime_lane::line_widths`'s 48 scattered lines at width 2.5 a row or a column off Metal's at an end). A mesh draw's lines and edges and a tessellated patch draw's edges take the device's lines under either value, the band not drawing them (at the nearest width they draw where they do not draw `w'`, announced `lines-at-nearest-width`: a known gap below). The runtime's: no pipeline's code differs. | `wide_lines = WideLinePolicy::Device` | `wide_lines = WideLinePolicy::Exact`: a draw of lines or edges the device's lines draw is the pipeline's band variant (`VariantKey::line_band`, `pipeline::link::band`) on a device with `tessellationShader` (the device's lines where it has none, or where the translator refuses the state's band or the draw's): two triangles over each line's parallelogram, and each triangle's three edges' parallelograms facing as the triangle, through a tessellation control and evaluation stage; every other width other than 1 is the band under either value | The lines can cover pixels the parallelogram does not, and miss others, at each end and along each edge: up to half the width past each end (square ends), a rounded width, and a row or a column at an end (a driver's parallelograms). `DeviceWideLines` (`device-wide-lines`), at each such draw where `DeviceOptions::announce_each` asks, else once per device; `WorkCounts::device_wide_line_draws`, `line_band_draws`, `nearest_width_line_draws`; `runtime_lane::line_widths` (the band matches Metal's maps of lines and of triangles' edges, their facing, culling, flat inputs, primitive IDs and depth on MoltenVK under both values, which has no `wideLines`; a band of edges under a depth bias is announced `wide-edge-depth-bias`) | Not measured. Expected from the band's form: a second pipeline variant made for each render state that draws such lines or edges (its two synthesized stages), each line tessellated as a patch of two control points into four corners and each triangle as a patch of three into 21 triangles (12 degenerate), and a strip, loop or fan drawn as the list of its primitives | + | A blit or fill through a `BGRA10_XR` or `BGRA10_XR_sRGB` texture Metal keeps losslessly compressed (`allowGPUOptimizedContents`, 16 texels or more a side, neither `ShaderWrite` nor `PixelFormatView` usage, one sample, not over a buffer) is the device's copy, which keeps the bytes it moves, on every device: Metal's GPU blit into, out of or between such textures moves the texels through its compression, which writes each 16-bit field's low six bits cleared and the alpha code clamped to `[384, 894]`, and its fill clears the low six bits alone (an Apple M3 Max under macOS 27.0.1, every direction, storage mode, usage and size that decides the compression; CPU copies keep the bytes on Metal too). A pass's draws through such an attachment are the device-gap row below. The runtime's choice: no pipeline differs (`compressed_blit.rs`). | `extended_range_blits = ExtendedRangeBlitPolicy::Kept` | `extended_range_blits = ExtendedRangeBlitPolicy::Canonical`: after a blit writes such a texture's region, or a blit out of one writes a buffer's words, a texture over a buffer's or another texture's, a compute dispatch (`bind::extended_canonical_module`, `extended_canonical.rs`) rewrites the texels it wrote in Metal's encoding: in place in a buffer's bytes, an image's through a scratch buffer, a copy out and back around the dispatch; a fill's texel is encoded before it is written. A buffer whose words the device cannot bind (an offset or pitch off 4 bytes, a region past `maxStorageBufferRange`) and a texture over a buffer whose rows share bytes keep the bytes under either value, announced | Bytes no Metal write left (uploaded, CPU-copied or kernel-written texels: a field's low six bits, an alpha code outside `[384, 894]`) keep what Metal's blit would rewrite; a texel reads the same value either way, its low six bits unread and its alpha read clamped. `ExtendedRangeBitsKept` (`extended-range-bits-kept`), once per device; under `Canonical` not for the blits it makes exact; `runtime_lane::compressed_blits` (both values on MoltenVK), the conformance cases `compressed-blits-bgra10xr` and `compressed-blits-bgra10xr-srgb` | Measured, `Canonical` against `Kept`, on `Private` `BGRA10_XR` textures Metal keeps compressed (the sRGB twin and fills not measured), 256x256 and 2048x2048, one whole-texture blit a command buffer: p50 of 300 blits a run, the two values interleaved in one process, three runs a host. RTX 5080 Laptop (NVIDIA 615.71.09, the command buffer's GPU time): a buffer into the texture 23.5 against 11.5 us and 761 against 244 us from a `Private` buffer, 42 against 30 us and 1,945 against 1,176 us from a `Shared` one; the texture into a `Private` buffer 14.1 against 11.2 us and 287 against 246 us, into a `Shared` one, whose words the dispatch rewrites in system memory, 66 against 27 us and 3,386 against 1,182 us; texture into texture 22.5 against 9.7 us and 630 against 80 us. Apple M3 Max (MoltenVK 1.4.2, whose GPU times are the host's, so per blit of 16 in one command buffer), the same blits: into the texture +38 to +57 us and +536 to +548 us, out of it into a buffer +15 us and +84 to +94 us, texture into texture +46 to +48 us and +535 to +541 us. Every figure is above its `Kept` one in every run, by more than the runs' spread (under 1 us on the 5080): measurable at both sizes on both hosts. Into an image a blit adds a copy out into a scratch buffer of the region's size, the dispatch, a copy back and the barriers between them; into a buffer the dispatch and a barrier. The scratch is made without zeros, since the copy out writes every byte the dispatch and the copy back read; the figures above were taken while it was zeroed first, by a command buffer of its own (a fill on the 5080, a copy on MoltenVK). CPU from the command buffer's making through its commit, a blit alone: +1 to +3 us on the M3 Max, +1 to +127 us on the 5080, whose host other builds loaded; 16 blits into 2048x2048 images in one command buffer hold 16 scratch buffers of 32 MiB, and encoding each costs 141 to 165 us (5080) and 286 to 356 us (M3 Max), new memory blocks among it, against 2 to 6 us under `Kept`. Nothing for a texture Metal keeps uncompressed or of another format | +-| A colour attachment of a texture's own image (not over a buffer's bytes, not memoryless, resolving nothing) that a pass stores `DontCare` is stored as the driver stores it: `DONT_CARE` on a driver measured to keep one of Metal's two answers or to store as Metal's own (`DriverFact::DontCareStoreIsOneOfMetals`, `DontCareStoreIsMetals`), else `STORE`, the rendered texels (`render_target.rs`). Metal's documentation leaves the contents open (`MTLStoreAction.dontCare`: "You need to assume that GPU discarded the texture's contents"); Apple's GPU leaves the texels the attachment held before the pass, the pass's clear and draws both gone (the conformance case `pass-sequence-color-actions`, an M3 Max under macOS 27.0.1, eight runs alike), or the rendered ones where its tiler overflows. A pass MoltenVK runs as one instance is handed to Metal's own `DontCare` under either value. | `dont_care_colors = DontCareColorPolicy::Driver` | `dont_care_colors = DontCareColorPolicy::Kept`: the pass renders through a backing its command queue keeps, one image of the attachment's shape, into which a `Load` copies the texels first, and nothing is copied back, so the texture keeps its earlier texels exactly (`kept_bytes.rs`, `WorkCounts::kept_texels_passes`); where a format makes no such image, the pass stores as under `Driver`, announced | Under `Driver`, the attachment holds the rendered texels (`STORE`: NVIDIA, lavapipe) or what the driver's `DONT_CARE` leaves (RADV, ANV: measured to be one of Metal's two answers), where Metal's holds the earlier texels unless its tiler overflows. `DontCareStore` (`dont-care-store`), once per device; under `Kept` not for the passes it keeps; `runtime_lane::dont_care_stores` (both values), the conformance case `pass-sequence-color-actions` | Measured, `Kept` against the same pass stored `Store`, one pass of one draw over an `RGBA8Unorm` texture a command buffer, commit to completion, the median of 300 interleaved (a scratch measurement, `rows3_cost`): RTX 5080 Laptop (NVIDIA 615.71.09), a `Load` pass 19 µs more at 1920x1080 and 10 µs at 256x256 (16 µs and 7 µs of GPU time), a `Clear` pass 3 µs; lavapipe (llvmpipe, LLVM 23.1.4), a `Load` pass 0.5 to 0.8 ms more at 1920x1080 and 30 µs at 256x256, a `Clear` pass within the runs' spread. Memory: one backing image of the attachment's shape per command queue | ++| A colour attachment of a texture's own image (not over a buffer's bytes, not memoryless, resolving nothing) that a pass stores `DontCare` is stored as the driver stores it: `DONT_CARE` on a driver measured to keep one of Metal's two answers or to store as Metal's own (`DriverFact::DontCareStoreIsOneOfMetals`, `DontCareStoreIsMetals`), else `STORE`, the rendered texels (`render_target.rs`). Metal's documentation leaves the contents open (`MTLStoreAction.dontCare`: "You need to assume that GPU discarded the texture's contents"); Apple's GPU leaves the texels the attachment held before the pass, the pass's clear and draws both gone (the conformance case `pass-sequence-color-actions`, an M3 Max under macOS 27.0.1, eight runs alike), or the rendered ones where its tiler overflows. A pass MoltenVK runs as one instance is handed to Metal's own `DontCare` under either value. | `dont_care_colors = DontCareColorPolicy::Driver` | `dont_care_colors = DontCareColorPolicy::Kept`: the pass renders through a backing its command queue keeps, one image of the attachment's shape, into which a `Load` copies the texels first, and nothing is copied back, so the texture keeps its earlier texels exactly (`kept_bytes.rs`, `WorkCounts::kept_texels_passes`); where a format makes no such image, the pass stores as under `Driver`, announced | Under `Driver`, the attachment holds the rendered texels (`STORE`: NVIDIA, lavapipe) or what the driver's `DONT_CARE` leaves (RADV, ANV: measured to be one of Metal's two answers; Turnip on the Adreno 740: read in Mesa's source to be one, a pass in tile memory skipping the attachment's tile store and one in system memory leaving the stored texels), where Metal's holds the earlier texels unless its tiler overflows. `DontCareStore` (`dont-care-store`), once per device; under `Kept` not for the passes it keeps; `runtime_lane::dont_care_stores` (both values), the conformance case `pass-sequence-color-actions` | Measured, `Kept` against the same pass stored `Store`, one pass of one draw over an `RGBA8Unorm` texture a command buffer, commit to completion, the median of 300 interleaved (a scratch measurement, `rows3_cost`): RTX 5080 Laptop (NVIDIA 615.71.09), a `Load` pass 19 µs more at 1920x1080 and 10 µs at 256x256 (16 µs and 7 µs of GPU time), a `Clear` pass 3 µs; lavapipe (llvmpipe, LLVM 23.1.4), a `Load` pass 0.5 to 0.8 ms more at 1920x1080 and 30 µs at 256x256, a `Clear` pass within the runs' spread. Memory: one backing image of the attachment's shape per command queue | + | A linear sample of a two-plane YCbCr texture whose planes a function reads in place (`YCBCR8_420_2P`, `YCBCR10_420_2P` through a uniform texel buffer over their codes, on an integrated GPU or a CPU device; `lower::sampler::in_place`), through a sampler that addresses no border on either plane axis, with no texel offset placed by a parent's size, is Vulkan's Y′CbCr model of the codes, as the device's own conversion with a linear chroma filter computes it: the four luma codes around the point and the four chroma pairs around the same point of the chroma plane (Metal's midpoint siting) blended bilinearly, then the surface's range and matrix (Metal's coefficients) in `float`, each channel clamped into `[0, 1]`: eight fetches. A runtime sampler's word decides as the function runs. A nearest sample, a border mode, a parent-placed offset and a gather are Metal's under either value. | `ycbcr_in_place_reads = YCbCrInPlacePolicy::Model` | `ycbcr_in_place_reads = YCbCrInPlacePolicy::Exact`: every tap Metal's texel (its chroma weighted 9/3/3/1 and rounded to a code, the fixed-point conversion), and the sample Metal's sum of the four taps' steps on the grid `2048`: twenty fetches and four conversions | The model reconstructs each channel at the point where Metal converts four taps and sums them: within the `float` arithmetic's rounding of the model (checked against an `f64` model on the CPU interpreter, `engine_ycbcr_in_place`), it differs from Metal's by the chroma the taps reconstruct, by Metal's roundings to a code and to `1/2048`, and where a tap's channel leaves `[0, 1]`, which Metal clamps before it sums. `HalfSampleFilter` / `FloatSampleFilter` under the bound converted 4:2:0 format, announced `ycbcr-filter-not-exact` | Radeon 780M (RADV): a 3840 x 2160 `YCBCR10_420_2P` frame sampled linearly into a 3840 x 2160 `BGRA8Unorm` target, GPU time per frame (median of 60): 6.4-6.9 ms under the exact form (`runtime_lane::video_frames`) | + + All at once: `DevicePolicy::exact_answers()` sets every switch above; each row states its own cost. +@@ -117,6 +118,11 @@ known. The rest of the differences are docs/API.md section 9's precision notes. + + ## Costs of matching Metal + ++Dynamically indexed constant tables larger than 4 KiB use the pipeline's shared constant ++buffer. Smaller typed tables stay in private storage while they fit the private-memory limit. ++Constant GEP and pointer-cast expressions retain their byte offsets in either representation. ++This storage choice preserves values and avoids a private copy of a large table per invocation. ++ + Where matching what Metal observably does costs time, the runtime matches it and the cost is + recorded here. + +@@ -286,6 +292,12 @@ recorded here. + times covering 1.5% or 12.5%, where both builds take the snapshot and the query is the difference + (on the Radeon 780M 0.55 to 0.71 and 1.04, on the RX 7900 XTX 0.90 to 0.91 and 1.04 to 1.05). + Other devices are not measured. ++- **`host_page_textures = HostPageTexturePolicy::InPlaceWrites` holds a texture over host pages or ++ padded rows in place whatever its usage, rendered and written too,** where the policy's default ++ and `HostPageTexturePolicy::Shadow` take the shadow below. It trusts the driver: an image's first ++ move and a pass's load and store must touch the texels alone, which Vulkan does not promise. A ++ consumer sets it where its driver is known to, as Turnip with its linear host images; nothing ++ checks it, and no copy in or merge back is paid. + - **A texture over a no-copy buffer's host pages, or over a host-visible buffer longer than 32 + MiB, that a shader writes of texels of a width the check stores none of (none of 1, 2, 4, 8 and + 16 bytes; or whose class's check has not passed for an access of its, or whose format, usage or +diff --git a/scripts/regen-spirv-grammar/items.txt b/scripts/regen-spirv-grammar/items.txt +index f4222232b..9216f38c4 100644 +--- a/scripts/regen-spirv-grammar/items.txt ++++ b/scripts/regen-spirv-grammar/items.txt +@@ -102,7 +102,7 @@ enum ImageFormat + values * + + enum Decoration +-values SpecId Block ColMajor ArrayStride MatrixStride BuiltIn NoPerspective Flat Patch Centroid ++values RelaxedPrecision SpecId Block ColMajor ArrayStride MatrixStride BuiltIn NoPerspective Flat Patch Centroid + values Sample Invariant Aliased Volatile Coherent NonWritable Location Component Index Binding + values DescriptorSet Offset NoContraction InputAttachmentIndex PerPrimitiveEXT NonUniform + values AliasedPointer diff --git a/tools/audit-runtime-fast-paths.py b/tools/audit-runtime-fast-paths.py new file mode 100644 index 0000000..d399250 --- /dev/null +++ b/tools/audit-runtime-fast-paths.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Report runtime policy fields, float controls and Turnip options from their owning sources. + +Use --runtime-policy with a captured runtime-policy.log to attach the device's actual +profile and check verdicts. Source references describe gates; they do not establish +that the device executed a path. Math operations use upstream Math::float_stems(), +BinOp::ALL and PrecisionNote's existing classification and validation. +""" +import argparse +import ast +import base64 +import hashlib +import json +from pathlib import Path +import re +import subprocess +import urllib.request + +COMPONENT = Path(__file__).resolve().parents[1] +CORE = COMPONENT.parents[1] +MESA_FILES = ('src/freedreno/vulkan/tu_drirc_gen.py', + 'src/freedreno/vulkan/tu_device.cc', + 'src/freedreno/vulkan/tu_cmd_buffer.cc', + 'src/freedreno/vulkan/tu_sampler.cc', + 'src/freedreno/vulkan/tu_lrz.cc', + 'src/freedreno/ir3/ir3_compiler.c', + 'src/freedreno/ir3/ir3_compiler_nir.c') + + +def mask(text): + return re.sub(r'//[^\n]*|/\*[\s\S]*?\*/|"(?:\\.|[^"\\])*"', + lambda m: ''.join('\n' if c == '\n' else ' ' for c in m[0]), text) + + +def body(text, pattern): + match = re.search(pattern, mask(text)) + if not match: + raise ValueError('Source declaration missing: ' + pattern) + clean = mask(text) + at = clean.index('{', match.end()) + depth = 1 + end = at + 1 + while depth: + if clean[end] == '{': + depth += 1 + elif clean[end] == '}': + depth -= 1 + end += 1 + return text[at + 1:end - 1], at + 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--translator', type=Path, default=CORE / '.cache/gpu-reference/metal2vulkan') + parser.add_argument('--mesa-source', type=Path, default=CORE / '.cache/mesa-work') + parser.add_argument('--fetch-mesa', action='store_true', help='Cache only the pinned Mesa source texts this report reads') + parser.add_argument('--runtime-policy', type=Path) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + mesa_pin = json.loads((CORE / 'scripts/data/adreno-compiler-inputs.json').read_text())['mesa'] + mesa = args.mesa_source + if args.fetch_mesa: + mesa = CORE / '.cache/runtime-fast-path-audit' / mesa_pin['commit'] + for name in MESA_FILES: + path = mesa / name + if path.is_file(): + continue + url = mesa_pin['mirror'].rstrip('/') + '/+/' + mesa_pin['commit'] + '/' + name + '?format=TEXT' + with urllib.request.urlopen(url, timeout=45) as response: + data = base64.b64decode(response.read(), validate=True) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + files = {} + + def read(root, name): + path = root / name + text = path.read_text() + files[str(path)] = hashlib.sha256(text.encode()).hexdigest() + return text + + policy_path = 'crates/metal2vulkan/src/device/policy.rs' + policy = read(args.translator, policy_path) + struct, _ = body(policy, r'pub struct DevicePolicy\b') + fields = re.findall(r'^ pub (\w+): ([^,]+),', struct, re.M) + defaults, _ = body(policy, r'pub const UNTRUSTED: DevicePolicy = DevicePolicy') + defaults = dict(re.findall(r'^ (\w+): (.+),$', defaults, re.M)) + consumer = read(COMPONENT, 'bridge/NativeMetalObjects.rs') + overrides = dict(re.findall(r'options\.policy\.(\w+) = ([^;]+);', consumer)) + references = {name: [] for name, _ in fields} + match_policy = re.compile(r'\.\s*(' + '|'.join(references) + r')\b') + for folder in ('crates/metal2vulkan/src', 'crates/metal2vulkan-runtime/src'): + for path in (args.translator / folder).rglob('*.rs'): + if '/tests/' in str(path): + continue + text = path.read_text() + for line, value in enumerate(text.splitlines(), 1): + for match in match_policy.finditer(value): + references[match[1]].append(dict(file=str(path.relative_to(args.translator)), line=line, expression=value.strip())) + profile_text = read(args.translator, 'crates/metal2vulkan/src/device/profile.rs') + ledger = read(args.translator, 'docs/FAST-PATHS.md').splitlines() + rows = [] + for name, kind in fields: + pattern = r'pub enum ' + re.escape(kind) + r'\b' + owner = next((source for source in (policy, profile_text) if re.search(pattern, mask(source))), None) + declaration = body(owner, pattern)[0] if owner else '' + rows.append(dict(field=name, type=kind, upstream_default=defaults.get(name), + consumer_override=overrides.get(name), source=policy_path, + values=re.findall(r'^ ([A-Z]\w*)\s*(?:,|\{|\()', mask(declaration), re.M), + references=references[name], + accuracy_ledger=[dict(file='docs/FAST-PATHS.md', line=n, expression=line) + for n, line in enumerate(ledger, 1) + if re.search(r'\b' + name + r'\s*=', line)])) + vocabulary = {} + for name, path in ( + ('FloatWidth', 'crates/metal2vulkan/src/device/requirements.rs'), + ('FloatControl', 'crates/metal2vulkan/src/device/requirements.rs'), + ('FloatFunctions', 'crates/metal2vulkan/src/device/profile.rs'), + ('FloatWidthControls', 'crates/metal2vulkan/src/device/profile.rs'), + ('FloatControls', 'crates/metal2vulkan/src/device/profile.rs'), + ('ProfileFields', 'crates/metal2vulkan/src/device/profile.rs'), + ('LeafCaps', 'crates/metal2vulkan/src/device/profile.rs'), + ('PrecisionNote', 'crates/metal2vulkan/src/pipeline/artifact.rs'), + ): + text = read(args.translator, path) + contents, _ = body(text, r'pub (?:enum|struct) ' + name + r'\b') + members = re.findall(r'^ pub (\w+):', contents, re.M) + if not members: + members = re.findall(r'^ ([A-Z]\w*)\s*(?:,|\{|\()', mask(contents), re.M) + vocabulary[name] = dict(source=path, count=len(members), members=members) + gates = [] + for path, names in ( + ('crates/metal2vulkan/src/lower/arith/narrow.rs', ('relaxes', 'quantize_form', 'half_controls', 'controlled_modes')), + ('crates/metal2vulkan/src/lower/arith/fma.rs', ('float_route', 'half_route')), + ('crates/metal2vulkan/src/lower/arith/mod.rs', ('contraction', 'fp_env')), + ('crates/metal2vulkan-runtime/src/batch_order.rs', ('ends_order',)), + ): + text = read(args.translator, path) + for name in names: + contents, offset = body(text, r'\bfn ' + name + r'\b') + gates.append(dict(file=path, symbol=name, line=text.count('\n', 0, offset) + 1, expression=contents.strip())) + mesa_text = {} + for name in MESA_FILES: + mesa_text[name] = read(mesa, name) + registry_path = 'src/freedreno/vulkan/tu_drirc_gen.py' + registry = mesa_text[registry_path] + options = [] + for node in ast.walk(ast.parse(registry)): + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.List): + continue + group = next((target.id for target in node.targets if isinstance(target, ast.Name) and target.id.endswith('_options')), None) + if not group: + continue + for call in node.value.elts: + if not isinstance(call, ast.Call) or not call.args or not isinstance(call.args[0], ast.Constant): + continue + options.append(dict(group=group, name=call.args[0].value, + default=ast.get_source_segment(registry, call.args[1]) if len(call.args) > 1 else None, + source=registry_path, line=call.lineno, + declaration=ast.get_source_segment(registry, call))) + property_lines = [] + selectors = re.compile(r'shader(?:Float16|FloatControls2|Denorm\w+|Rounding\w+|SignedZero\w+)|driverID|vendorID|deviceID|lower_fdiv|float_mul_add\d+') + for name, text in mesa_text.items(): + for line, value in enumerate(text.splitlines(), 1): + if selectors.search(value): + property_lines.append(dict(file=name, line=line, expression=value.strip())) + plugin = read(COMPONENT, 'NativeAdrenoMetalPlugin.mm') + result = dict(schema='unixnu-runtime-fast-path-audit-1', + translator=subprocess.check_output(['git', '-C', str(args.translator), 'rev-parse', 'HEAD'], text=True).strip(), + mesa=mesa_pin['commit'], policy_field_count=len(rows), policies=rows, + float_vocabulary=vocabulary, owning_gates=gates, + mesa_option_count=len(options), mesa_options=options, mesa_property_evidence=property_lines, + consumer_turnip_configuration=[line.strip() for line in plugin.splitlines() if 'setenv("tu_' in line], + math_enumeration='Upstream Math::float_stems(), BinOp::ALL, PrecisionNote::of_math_call/of_float_operator, engine_float_call_notes and fast_paths; no separate operation list.', + evidence='Finite source declarations and gate expressions. Runtime verdicts and active paths come from runtime-policy.log and timing/FP counters.', + source_files=files, + unresolved_policy_defaults=[row['field'] for row in rows if row['upstream_default'] is None]) + if args.runtime_policy: + text = args.runtime_policy.read_text() + result['runtime_capture'] = dict(file=str(args.runtime_policy), stamp=text.splitlines()[0], contents=text) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + '\n') + print('policy-fields=%d Turnip-options=%d unresolved-policy-defaults=%d -> %s' % (len(rows), len(options), len(result['unresolved_policy_defaults']), args.output)) + + +if __name__ == '__main__': + main() diff --git a/tools/build-shader-cache-app.py b/tools/build-shader-cache-app.py new file mode 100644 index 0000000..1661699 --- /dev/null +++ b/tools/build-shader-cache-app.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Build the macOS Shader Cache app and its administrator-authorized worker.""" +import argparse +import json +import plistlib +from pathlib import Path +import re +import shutil +import subprocess + +SOURCE = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('output', type=Path) +parser.add_argument('--arch', choices=('arm64', 'arm64e'), default='arm64e') +args = parser.parse_args() +app = args.output.resolve() / 'Shader Cache.app' +if app.exists(): + shutil.rmtree(app) +code = app / 'Contents/MacOS' +code.mkdir(parents=True) +sdk = subprocess.check_output(['xcrun', '--sdk', 'macosx', '--show-sdk-path'], text=True).strip() +base = ['xcrun', 'clang++', '-target', args.arch + '-apple-macos27.0', '-isysroot', sdk, + '-std=c++17', '-fobjc-arc', '-O2', '-Wall', '-Wextra', '-Werror', '-Wno-deprecated-declarations', '-Wno-unused-function'] +hashes = {} +for name, frameworks in [('ShaderCache', ('AppKit', 'Security')), ('ShaderCacheWorker', ('Foundation', 'Metal'))]: + source = 'ShaderCacheApp.mm' if name == 'ShaderCache' else name + '.mm' + subprocess.run([*base, SOURCE / 'ShaderCache' / source, + *[part for framework in frameworks for part in ('-framework', framework)], '-o', code / name], check=True) + entitlements = args.output / (name + '-entitlements.plist') + grants = {'com.apple.security.iokit-user-client-class': ['AGXDeviceUserClient'], + 'com.apple.private.security.no-container': True} if name == 'ShaderCacheWorker' and args.arch == 'arm64e' else {} + entitlements.write_bytes(plistlib.dumps(grants)) + subprocess.run(['codesign', '--force', '--sign', '-', '--timestamp=none', '--pagesize', '4096', + '--entitlements', entitlements, code / name], check=True) + detail = subprocess.run(['codesign', '-d', '--verbose=4', code / name], capture_output=True, text=True, check=True).stderr + hashes[name] = re.search(r'^CDHash=([0-9a-f]{40})$', detail, re.M)[1] + entitlements.unlink() +(app / 'Contents/Info.plist').write_bytes(plistlib.dumps(dict( + CFBundleIdentifier='org.unixnu.ShaderCache', CFBundleName='Shader Cache', CFBundleDisplayName='Shader Cache', + CFBundleExecutable='ShaderCache', CFBundlePackageType='APPL', CFBundleVersion='1', CFBundleShortVersionString='1.0', + LSMinimumSystemVersion='27.0', NSHighResolutionCapable=True, NSPrincipalClass='NSApplication'))) +# The outer seal retains the executable's existing entitlements. +subprocess.run(['codesign', '--force', '--sign', '-', '--timestamp=none', '--pagesize', '4096', + '--preserve-metadata=entitlements', app], check=True) +for path in (app, code / 'ShaderCacheWorker'): + subprocess.run(['codesign', '--verify', '--strict', path], check=True) +detail = subprocess.run(['codesign', '-d', '--verbose=4', app], capture_output=True, text=True, check=True).stderr +hashes['ShaderCache'] = re.search(r'^CDHash=([0-9a-f]{40})$', detail, re.M)[1] +(args.output / 'shader-cache-trust.json').write_text(json.dumps(hashes, indent=2) + '\n') +print(app) diff --git a/tools/decode-command-buffer-log.py b/tools/decode-command-buffer-log.py new file mode 100644 index 0000000..f98228e --- /dev/null +++ b/tools/decode-command-buffer-log.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Reassemble NativeAdrenoMetal command-buffer messages from compact system logs.""" +import argparse +import base64 +import json +from pathlib import Path +import re +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('logs', type=Path, nargs='+', help='log files from one boot') + parser.add_argument('--functions', type=Path, help='write captured function AIR and its names to this directory') + args = parser.parse_args() + pattern = re.compile(r'(\S+)\[(\d+):[^]]+\].*NativeAdrenoMetal CB dump ' + r'(\d+) (\d+)/(\d+): (.*)') + entries = {} + raw_sequence = 0 + for path in args.logs: + for line in path.read_text(errors='replace').splitlines(): + if line.startswith('native-metal dump '): + raw_sequence += 1 + entries['stderr', 0, raw_sequence] = (1, {1: line}) + continue + if 'NativeAdrenoMetal CB dump ' not in line: + continue + match = pattern.search(line) + if not match: + continue + process, pid, sequence, part, count, text = match.groups() + key = process, int(pid), int(sequence) + part, count = int(part), int(count) + expected, chunks = entries.setdefault(key, (count, {})) + if count != expected or not 1 <= part <= count or (part in chunks and chunks[part] != text): + parser.error(f'conflicting log fragments for {process}[{pid}] entry {sequence}') + chunks[part] = text + if not entries: + parser.error('no command-buffer messages found') + incomplete = 0 + functions = [] + if args.functions: + args.functions.mkdir(parents=True, exist_ok=True) + for (process, pid, sequence), (count, chunks) in sorted(entries.items()): + missing = set(range(1, count + 1)) - chunks.keys() + if missing: + incomplete += 1 + print(f'{process}[{pid}] entry {sequence}: missing fragments {sorted(missing)}', file=sys.stderr) + continue + text = ''.join(chunks[part] for part in range(1, count + 1)) + if text.strip() == 'native-metal dump capture limit reached': + incomplete += 1 + print('command-buffer capture reached its file limit', file=sys.stderr) + function = re.fullmatch(r'native-metal dump function (\d+) (\S+): (.*)', text) + if function and args.functions: + index, name, encoded = function.groups() + try: + data = base64.b64decode(encoded, validate=True) + except ValueError: + incomplete += 1 + print(f'{process}[{pid}] function {name}: {encoded[:120]}', file=sys.stderr) + continue + filename = f'function-{pid}-{index}.air' + (args.functions / filename).write_bytes(data) + functions.append(dict(process=process, pid=pid, name=name, file=filename)) + print(f'{process}[{pid}] function {name}: {filename}, {len(data)} bytes') + else: + print(f'{process}[{pid}] ' + text) + if args.functions: + (args.functions / 'functions.json').write_text(json.dumps(functions, indent=2) + '\n') + print(f'command-buffer entries={len(entries)} incomplete={incomplete}', file=sys.stderr) + return bool(incomplete) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/gpu-acceptance/build.py b/tools/gpu-acceptance/build.py new file mode 100644 index 0000000..400358f --- /dev/null +++ b/tools/gpu-acceptance/build.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Build the bounded A740 GPU acceptance executable with embedded AIR and native SPIR-V controls.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess + +COMPONENT = Path(__file__).resolve().parents[2] +CORE = COMPONENT.parents[1] +HERE = Path(__file__).resolve().parent + + +def run(args, **kwargs): + return subprocess.run([str(x) for x in args], check=True, **kwargs) + + +def mask(source): + result = list(source) + i = 0 + while i < len(source): + end = i + if source.startswith('//', i): + end = source.find('\n', i) + if end < 0: end = len(source) + elif source.startswith('/*', i): + depth, end = 1, i + 2 + while depth: + if source.startswith('/*', end): depth += 1; end += 2 + elif source.startswith('*/', end): depth -= 1; end += 2 + else: end += 1 + elif source[i] == '"': + end = i + 1 + while end < len(source): + if source[end] == '\\': end += 2 + elif source[end] == '"': end += 1; break + else: end += 1 + if end > i: + for at in range(i, end): + if result[at] != '\n': result[at] = ' ' + i = end + else: i += 1 + return ''.join(result) + + +def item(source, name, kind='fn'): + clean = mask(source) + start = re.search(r'^' + kind + r' ' + name + r'\b', clean, re.M).start() + first = clean.index('{', start) + depth, end = 1, first + 1 + while depth: + if clean[end] == '{': depth += 1 + elif clean[end] == '}': depth -= 1 + end += 1 + return source[start:end] + '\n' + + +def arithmetic_air(half): + f = 'half' if half else 'float' + v = '<4 x ' + f + '>' + a = '0xH3BFE' if half else '9.990234375e-01' + b = '0xH1400' if half else '9.765625e-04' + vec = lambda value: '<' + ', '.join(f + ' ' + value for _ in range(4)) + '>' + narrow = '%initial = fptrunc <4 x float> %loaded to <4 x half>' if half else '%initial = fadd <4 x float> %loaded, zeroinitializer' + widen = '%answer = fpext <4 x half> %next to <4 x float>' if half else '%answer = fadd <4 x float> %next, zeroinitializer' + return '''target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32" +target triple = "air64_v28-apple-macosx26.0.0" +define void @arith(ptr addrspace(1) noundef "air-buffer-no-alias" %in, ptr addrspace(1) noundef "air-buffer-no-alias" %out, i32 noundef %gid) { +entry: + %g = zext i32 %gid to i64 + %src = getelementptr inbounds <4 x float>, ptr addrspace(1) %in, i64 %g + %loaded = load <4 x float>, ptr addrspace(1) %src, align 16 + ''' + narrow + ''' + br label %loop +loop: + %i = phi i32 [0, %entry], [%j, %loop] + %acc = phi ''' + v + ''' [%initial, %entry], [%next, %loop] + %product = fmul fast ''' + v + ' %acc, ' + vec(a) + ''' + %next = fadd fast ''' + v + ' %product, ' + vec(b) + ''' + %j = add i32 %i, 1 + %again = icmp ult i32 %j, 256 + br i1 %again, label %loop, label %exit +exit: + ''' + widen + ''' + %dst = getelementptr inbounds <4 x float>, ptr addrspace(1) %out, i64 %g + store <4 x float> %answer, ptr addrspace(1) %dst, align 16 + ret void +} +!air.kernel = !{!0} +!air.version = !{!6} +!air.language_version = !{!7} +!0 = !{ptr @arith, !1, !2} +!1 = !{} +!2 = !{!3, !4, !5} +!3 = !{i32 0, !"air.buffer", !"air.location_index", i32 0, i32 1, !"air.read", !"air.address_space", i32 1, !"air.arg_type_size", i32 16, !"air.arg_type_align_size", i32 16, !"air.arg_type_name", !"float4", !"air.arg_name", !"in"} +!4 = !{i32 1, !"air.buffer", !"air.location_index", i32 1, i32 1, !"air.read_write", !"air.address_space", i32 1, !"air.arg_type_size", i32 16, !"air.arg_type_align_size", i32 16, !"air.arg_type_name", !"float4", !"air.arg_name", !"out"} +!5 = !{i32 2, !"air.thread_position_in_grid", !"air.arg_type_name", !"uint", !"air.arg_name", !"gid"} +!6 = !{i32 2, i32 8, i32 0} +!7 = !{!"Metal", i32 4, i32 0, i32 0} +''' + + +def control_spirv(half=False, scale=False): + # The arithmetic control has the AIR's four independent lanes and bounded loop. The + # bandwidth control is runtime_examples.ll's scale_and_offset: out[i] = in[i] * scale + i. + text = '''OpCapability Shader +''' + if half: text += 'OpCapability Float16\n' + text += '''OpMemoryModel Logical GLSL450 +OpEntryPoint GLCompute %main "main" %gid %input %output''' + (' %scale' if scale else '') + ''' +OpExecutionMode %main LocalSize 64 1 1 +OpDecorate %gid BuiltIn GlobalInvocationId +OpDecorate %arr ArrayStride ''' + ('4' if scale else '16') + ''' +OpMemberDecorate %Block 0 Offset 0 +OpDecorate %Block Block +OpDecorate %input DescriptorSet 0 +OpDecorate %input Binding 0 +OpDecorate %output DescriptorSet 0 +OpDecorate %output Binding 1 +''' + if scale: text += '''OpDecorate %scale DescriptorSet 0 +OpDecorate %scale Binding 2 +''' + text += '''%void = OpTypeVoid +%fn = OpTypeFunction %void +%uint = OpTypeInt 32 0 +%bool = OpTypeBool +%v3uint = OpTypeVector %uint 3 +%ptr_gid = OpTypePointer Input %v3uint +%gid = OpVariable %ptr_gid Input +%float = OpTypeFloat 32 +%v4float = OpTypeVector %float 4 +''' + if half: text += '%half = OpTypeFloat 16\n%v4half = OpTypeVector %half 4\n' + value = '%float' if scale else '%v4float' + arithmetic = '%v4half' if half else '%v4float' + text += '%arr = OpTypeRuntimeArray ' + value + ''' +%Block = OpTypeStruct %arr +%ptr_block = OpTypePointer StorageBuffer %Block +%ptr_value = OpTypePointer StorageBuffer ''' + value + ''' +%input = OpVariable %ptr_block StorageBuffer +%output = OpVariable %ptr_block StorageBuffer +''' + if scale: text += '%scale = OpVariable %ptr_block StorageBuffer\n' + text += '''%zero = OpConstant %uint 0 +%one = OpConstant %uint 1 +%limit = OpConstant %uint 256 +''' + if not scale: + scalar = '%half' if half else '%float' + text += '%a = OpConstant ' + scalar + ' 0.9990234375\n%b = OpConstant ' + scalar + ' 0.0009765625\n' + text += '%va = OpConstantComposite ' + arithmetic + ' %a %a %a %a\n%vb = OpConstantComposite ' + arithmetic + ' %b %b %b %b\n' + text += '''%main = OpFunction %void None %fn +%entry = OpLabel +%ids = OpLoad %v3uint %gid +%index = OpCompositeExtract %uint %ids 0 +%src = OpAccessChain %ptr_value %input %zero %index +%dst = OpAccessChain %ptr_value %output %zero %index +%loaded = OpLoad ''' + value + ''' %src +''' + if scale: + text += '''%sf = OpAccessChain %ptr_value %scale %zero %zero +%factor = OpLoad %float %sf +%product = OpFMul %float %loaded %factor +%fi = OpConvertUToF %float %index +%answer = OpFAdd %float %product %fi +OpStore %dst %answer +OpReturn +OpFunctionEnd +''' + else: + text += ('%initial = OpFConvert %v4half %loaded\n' if half else '%initial = OpCopyObject %v4float %loaded\n') + text += '''OpBranch %loop +%loop = OpLabel +%i = OpPhi %uint %zero %entry %j %body +%acc = OpPhi ''' + arithmetic + ''' %initial %entry %next %body +OpLoopMerge %exit %body None +OpBranch %body +%body = OpLabel +%product = OpFMul ''' + arithmetic + ''' %acc %va +%next = OpFAdd ''' + arithmetic + ''' %product %vb +%j = OpIAdd %uint %i %one +%again = OpULessThan %bool %j %limit +OpBranchConditional %again %loop %exit +%exit = OpLabel +''' + text += ('%answer = OpFConvert %v4float %next\n' if half else '%answer = OpCopyObject %v4float %next\n') + text += 'OpStore %dst %answer\nOpReturn\nOpFunctionEnd\n' + return text + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--translator', type=Path, default=CORE / '.cache/gpu-reference/metal2vulkan') + parser.add_argument('--target', choices=['host', 'macos27'], default='macos27') + parser.add_argument('--vulkan-include', type=Path, default=CORE / '.cache/mesa-work/include') + parser.add_argument('--output', type=Path, default=CORE / 'build/gpu-acceptance') + args = parser.parse_args() + out = args.output.resolve(); out.mkdir(parents=True, exist_ok=True) + source = args.translator.resolve() + pin = subprocess.check_output(['git', '-C', source, 'rev-parse', 'HEAD'], text=True).strip() + expected = json.loads((CORE / 'scripts/data/gpu-metal-vulkan-inputs.json').read_text())['translator']['commit'] + if pin != expected: raise SystemExit('Acceptance translator differs from the product source pin') + upstream = source / 'crates/metal2vulkan-validation/src/runtime_lane/native_maths.rs' + maths = upstream.read_text() + helpers = ['operands','ulp','half_ulp','excess','broken','to_half','metal_half','half_operands','half_operand','half_within','half_within_sum'] + constants = ''.join(re.search(r'^const ' + name + r':[^\n]+', maths, re.M)[0]+'\n' for name in ['IN','OUT','COUNT','PAYLOAD_NAN']) + stream_start = maths.index('struct Stream(') + stream = maths[stream_start:maths.index('const PAYLOAD_NAN', stream_start)] + numerical = constants + stream + ''.join(item(maths, name).replace('fn '+name, 'pub fn '+name, 1) if name in ['operands','broken','metal_half','half_operands','half_within','half_within_sum','half_ulp'] else item(maths,name) for name in helpers) + (out/'numerical.rs').write_text(numerical) + kernels = ['kernel','half_kernel','half_arithmetic_kernel'] + generator = constants + ''.join(item(maths, name) for name in kernels) + generator += '\nfn main() { let root=std::path::PathBuf::from(std::env::args().nth(1).unwrap());' + for f,name in zip(kernels,['native_maths','native_half_rounding','native_half_arithmetic']): + generator += 'std::fs::write(root.join("'+name+'.ll"), '+f+'()).unwrap();' + generator += '}\n' + (out/'kernels.rs').write_text(generator) + run(['rustc','+1.98.0','--edition=2021','-A','dead_code',out/'kernels.rs','-o',out/'kernels']) + run([out/'kernels',out]) + (out/'native_half_division.ll').write_text((out/'native_half_rounding.ll').read_text().replace('%hs = fadd half', '%hs = fdiv half')) + (out/'arith32.ll').write_text(arithmetic_air(False));(out/'arith16.ll').write_text(arithmetic_air(True)) + llvm = shutil.which('llvm-as') or '/opt/homebrew/opt/llvm/bin/llvm-as' + for name in ['native_maths','native_half_rounding','native_half_arithmetic','native_half_division','arith32','arith16']: + run([llvm,out/(name+'.ll'),'-o',out/(name+'.air')]) + run([llvm,source/'crates/metal2vulkan-validation/fixtures/public/examples/runtime_examples.ll','-o',out/'runtime_examples.air']) + for name,half,scale in [('arith32',False,False),('arith16',True,False),('scale',False,True)]: + (out/(name+'.spvasm')).write_text(control_spirv(half,scale)) + run(['spirv-as','--target-env','vulkan1.2',out/(name+'.spvasm'),'-o',out/(name+'.spv')]) + run(['spirv-val','--target-env','vulkan1.2',out/(name+'.spv')]) + component = subprocess.check_output(['git','-C',COMPONENT,'rev-parse','HEAD'],text=True).strip() + manifest = '\n'.join(['[package]','name="gpu-acceptance"','version="0.1.0"','edition="2024"','[workspace]', + '[lib]','crate-type=["staticlib"]','path='+json.dumps(str(HERE/'lib.rs')),'[dependencies]', + 'metal2vulkan={path='+json.dumps(str(source/'crates/metal2vulkan'))+',features=["ash"]}', + 'metal2vulkan-runtime={path='+json.dumps(str(source/'crates/metal2vulkan-runtime'))+'}', + 'ash={version="0.38",default-features=false,features=["loaded"]}','serde_json="1"','libc="0.2"', + '[profile.release]','panic="unwind"','debug=1','lto="fat"','codegen-units=1','']) + (out/'Cargo.toml').write_text(manifest) + triple = 'arm64e-apple-darwin' if args.target=='macos27' else 'aarch64-apple-darwin' + env=dict(os.environ,NMB_ACCEPTANCE_NUMERIC=str(out/'numerical.rs'),NMB_ACCEPTANCE_INPUTS=str(out), + NMB_COMPONENT_ID=component,NMB_TRANSLATOR_ID=pin,MACOSX_DEPLOYMENT_TARGET='27.0' if args.target=='macos27' else '14.0') + cargo=['cargo','+1.98.0'] + if args.target=='macos27': env['RUSTC_BOOTSTRAP']='1';cargo+=['-Z','build-std=std,panic_unwind'] + with (out/'build.log').open('w') as stream: + run(cargo+['build','--release','--manifest-path',out/'Cargo.toml','--target',triple,'--target-dir',CORE/'build/m2v-target'],env=env,cwd=CORE,stdout=stream,stderr=subprocess.STDOUT) + archive=CORE/'build/m2v-target'/triple/'release/libgpu_acceptance.a' + sdk=subprocess.check_output(['xcrun','--sdk','macosx','--show-sdk-path'],text=True).strip() + cc=['xcrun','clang','-target','arm64e-apple-macos27.0' if args.target=='macos27' else 'arm64-apple-macos14.0','-isysroot',sdk,'-O2','-Wall','-Wextra','-Werror'] + extra=[] + if args.target=='macos27': + renames={'_pthread_create':'_nmb_pac_pthread_create','__tlv_atexit':'_nmb_pac_tlv_atexit','_atexit':'_nmb_pac_atexit','___cxa_atexit':'_nmb_pac_cxa_atexit','__dyld_register_func_for_add_image':'_nmb_pac_dyld_register_func_for_add_image','__Unwind_Backtrace':'_nmb_pac_Unwind_Backtrace','_dlsym':'_nmb_pac_dlsym'} + objcopy=shutil.which('llvm-objcopy') or '/opt/homebrew/opt/llvm/bin/llvm-objcopy' + pac=out/'libgpu_acceptance_pac.a' + run([objcopy,*['--redefine-sym='+a+'='+b for a,b in renames.items()],archive,pac]);archive=pac + run(cc+['-std=c11','-I',args.vulkan_include,'-c',COMPONENT/'bridge/RustPacBoundary.c','-o',out/'RustPacBoundary.o']) + extra=[out/'RustPacBoundary.o','-Wl,-map,'+str(out/'gpu-acceptance.map')] + exe=out/'gpu-acceptance' + run(cc+['-std=c11',HERE/'main.c',archive,*extra,'-framework','Foundation','-framework','Metal','-framework','IOSurface','-lobjc','-liconv','-Wl,-dead_strip','-o',exe]) + if args.target=='macos27':run(['python3',CORE/'scripts/rust-pac-rewrite.py',exe,out/'gpu-acceptance.map',archive.name]) + run(['strip','-x',exe]) + run(['codesign','--force','--sign','-','--pagesize','4096',exe]) + if args.target=='host': run([exe,'--describe']) + (out/'gpu-acceptance.json').write_text(json.dumps(dict(component=component,translator=pin,numerical_source_sha256=hashlib.sha256(upstream.read_bytes()).hexdigest(),target=triple,sha256=hashlib.sha256(exe.read_bytes()).hexdigest()),indent=2)+'\n') + package=out/'files' + (package/'bin').mkdir(parents=True,exist_ok=True) + (package/'share/gpu-acceptance').mkdir(parents=True,exist_ok=True) + shutil.copy2(exe,package/'bin/gpu-acceptance') + shutil.copyfile(out/'gpu-acceptance.json',package/'share/gpu-acceptance/build.json') + shutil.copyfile(COMPONENT/'LICENSE',package/'share/gpu-acceptance/LICENSE') + for license in source.glob('LICENSE*'): + if license.is_file():shutil.copyfile(license,package/'share/gpu-acceptance'/('metal2vulkan-'+license.name)) + print(exe) + + +if __name__=='__main__': main() diff --git a/tools/gpu-acceptance/lib.rs b/tools/gpu-acceptance/lib.rs new file mode 100644 index 0000000..987a220 --- /dev/null +++ b/tools/gpu-acceptance/lib.rs @@ -0,0 +1,825 @@ +// SPDX-License-Identifier: MIT +use ash::vk; +use metal2vulkan_runtime::{ + Buffer, CommandBuffer, CommandBufferStatus, CommandQueue, ComputePipelineState, Device, + DeviceOptions, MessageSeverity, MessageSink, PipelineOptions, ResourceOptions, + ShaderCacheOptions, Size, +}; +use serde_json::json; +use std::ffi::{CStr, c_char}; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +mod numerical { + include!(env!("NMB_ACCEPTANCE_NUMERIC")); +} +const MATH: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/native_maths.air")); +const HALF: &[u8] = include_bytes!(concat!( + env!("NMB_ACCEPTANCE_INPUTS"), + "/native_half_rounding.air" +)); +const HALF_DIVISION: &[u8] = include_bytes!(concat!( + env!("NMB_ACCEPTANCE_INPUTS"), + "/native_half_division.air" +)); +const EXAMPLES: &[u8] = include_bytes!(concat!( + env!("NMB_ACCEPTANCE_INPUTS"), + "/runtime_examples.air" +)); +const FP32: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith32.air")); +const FP16: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith16.air")); +const SPV32: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith32.spv")); +const SPV16: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith16.spv")); +const SPV_SCALE: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/scale.spv")); +const ITERATIONS: u64 = 256; +const WARM: usize = 2; +const SAMPLES: usize = 9; +const WAIT: Duration = Duration::from_secs(10); +type Result = std::result::Result; +fn err(value: impl std::fmt::Debug) -> String { + format!("{value:?}") +} + +fn emit(value: serde_json::Value) { + println!("{value}"); +} +fn stamp() -> serde_json::Value { + json!({"boot":std::env::var("NATIVE_METAL_BOOT_SESSION").unwrap_or_default(), + "component":env!("NMB_COMPONENT_ID"), "translator":env!("NMB_TRANSLATOR_ID"), + "uptime_us":metal2vulkan_runtime::stage_trace::now_nanos()/1000}) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_acceptance_describe() { + emit(json!({"schema":"unixnu-gpu-acceptance-1", "stamp":stamp(), + "numeric":"upstream native_maths/half_rounding references and bounds, 4096 threads", + "arithmetic":"FP32/FP16 vec4 independent multiply/add chains, 256 iterations, paired native SPIR-V on the same Vulkan device and queue", + "bandwidth":"native Shared buffers, scale_and_offset and blit copy, paired native Vulkan dispatch/copy", + "samples":SAMPLES,"warmups":WARM,"ratio_required":0.95,"runtime_llvm":false})); +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_acceptance( + loader: *const c_char, + cache: *const c_char, + mode: *const c_char, +) -> i32 { + let run = std::panic::catch_unwind(|| { + // The C launcher owns these NUL-terminated strings for this call. + let (loader, cache, mode) = unsafe { + ( + CStr::from_ptr(loader), + CStr::from_ptr(cache), + CStr::from_ptr(mode), + ) + }; + execute(loader, cache, mode.to_str().map_err(err)?) + }); + match run { + Ok(Ok(pass)) => { + emit( + json!({"event":"complete","measurement_complete":true,"correct":pass,"stamp":stamp()}), + ); + if pass { 0 } else { 1 } + } + Ok(Err(error)) => { + emit(json!({"event":"error","error":error,"stamp":stamp()})); + 70 + } + Err(_) => { + emit(json!({"event":"error","error":"panic","stamp":stamp()})); + 70 + } + } +} + +fn execute(loader: &CStr, cache: &CStr, mode: &str) -> Result { + nmb_acceptance_describe(); + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + options.shader_cache = ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + if mode == "numeric" { + options + .layers + .push(c"VK_LAYER_KHRONOS_validation".to_owned()); + } + let validation_errors = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let errors = validation_errors.clone(); + options.messages = Some(MessageSink::new(move |m| { + if m.id_name.ends_with("device-recovered") { + fatal_gpu("Vulkan device recovered during acceptance"); + } + if m.severity >= MessageSeverity::Error { + errors.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + eprintln!("gpu-acceptance {} {}", m.id_name, m.text) + })); + let device = Device::system_default_with(&options).map_err(err)?; + let p = device.properties(); + emit( + json!({"event":"device","name":device.name(),"vendor_id":p.vendor_id,"device_id":p.device_id, + "policy":format!("{:?}", device.compile_device().policy()), + "profile":format!("{:?}", device.compile_device().profile()),"stamp":stamp()}), + ); + if p.vendor_id != 0x5143 || p.device_id != 0x4305_0a01 { + return Err("This calibration requires the sheng Adreno 740".into()); + } + let queue = device.new_command_queue().map_err(err)?; + let mut pass = true; + if mode != "calibrate" { + pass &= numeric(&device, &queue)?; + } + if mode != "numeric" { + pass &= calibrate(&device, &queue)?; + } + let w = device.work_counts(); + emit( + json!({"event":"runtime-counters","timestamp_slots":w.timestamp_slots_taken, + "calibrations":w.calibrations,"calibration_disagreements":w.calibration_disagreements,"validation_errors":validation_errors.load(std::sync::atomic::Ordering::Relaxed),"stamp":stamp()}), + ); + Ok(pass && validation_errors.load(std::sync::atomic::Ordering::Relaxed) == 0) +} + +fn state(device: &Device, air: &[u8], name: &str) -> Result { + let lib = device.new_library_with_compile_library( + metal2vulkan::library::Library::from_air(air).map_err(err)?, + ); + let f = lib.new_function_with_name(name).map_err(err)?; + device + .new_compute_pipeline_state_with_function(&f, &PipelineOptions::default()) + .map_err(err) +} +fn buffer(device: &Device, bytes: &[u8]) -> Result { + device + .new_buffer_with_bytes(bytes, ResourceOptions::shared()) + .map_err(err) +} +fn floats(buffer: &Buffer) -> Result<&[f32]> { + let ptr = buffer.contents().ok_or("Shared buffer has no contents")?; + // Every caller has waited for GPU completion and all lengths are multiples of four. + Ok(unsafe { std::slice::from_raw_parts(ptr.as_ptr().cast(), buffer.length() as usize / 4) }) +} +fn timed(cb: &CommandBuffer, device: &Device) -> Result<(f64, f64)> { + let slots = device.work_counts().timestamp_slots_taken; + let wall = Instant::now(); + cb.commit().map_err(err)?; + if cb.wait_until_completed_for(WAIT).is_err() { + fatal_gpu("runtime completion exceeded 10 seconds"); + } + if cb.status() != CommandBufferStatus::Completed { + return Err(format!( + "command buffer {:?}: {:?}", + cb.status(), + cb.error() + )); + } + let us = (cb.gpu_end_time().map_err(err)? - cb.gpu_start_time().map_err(err)?) * 1e6; + if us <= 0.0 || !us.is_finite() { + return Err("GPU timestamp interval unavailable".into()); + } + // The runtime allocates timestamp slots while encoding, before commit. The physical queue + // timestamp support is checked by Native::new; the total proves timing was in use. + if slots == 0 { + return Err("Runtime supplied no GPU timestamp slots".into()); + } + Ok((us, wall.elapsed().as_secs_f64() * 1e6)) +} +fn dispatch( + device: &Device, + queue: &CommandQueue, + state: &ComputePipelineState, + buffers: &[&Buffer], + count: u64, +) -> Result<(f64, f64)> { + let cb = queue.command_buffer().map_err(err)?; + let mut enc = cb.compute_command_encoder().map_err(err)?; + enc.set_compute_pipeline_state(state).map_err(err)?; + let raw = unsafe { device.raw() }; + let limits = unsafe { + raw.instance + .get_physical_device_properties(raw.physical_device) + } + .limits; + let chunk = limits.max_compute_work_group_count[0] as u64 * 64; + let stride = buffers[0].length() / count; + for first in (0..count).step_by(chunk as usize) { + let units = (count - first).min(chunk); + for (i, b) in buffers.iter().enumerate() { + enc.set_buffer(Some(b), if i < 2 { first * stride } else { 0 }, i as u32) + .map_err(err)?; + } + enc.dispatch_threadgroups(Size::new(units / 64, 1, 1), Size::new(64, 1, 1)) + .map_err(err)?; + } + enc.end_encoding().map_err(err)?; + timed(&cb, device) +} +fn copy( + device: &Device, + queue: &CommandQueue, + input: &Buffer, + output: &Buffer, +) -> Result<(f64, f64)> { + let cb = queue.command_buffer().map_err(err)?; + let mut enc = cb.blit_command_encoder().map_err(err)?; + enc.copy_from_buffer_to_buffer(input, 0, output, 0, input.length()) + .map_err(err)?; + enc.end_encoding().map_err(err)?; + timed(&cb, device) +} +fn numeric(device: &Device, queue: &CommandQueue) -> Result { + let inputs = numerical::operands(); + let bytes: Vec = inputs + .iter() + .flatten() + .flat_map(|v| v.to_le_bytes()) + .collect(); + let input = buffer(device, &bytes)?; + let output = device + .new_buffer(4096 * 14 * 4, ResourceOptions::shared()) + .map_err(err)?; + emit(json!({"event":"begin","case":"native-maths","stamp":stamp()})); + let pipeline = state(device, MATH, "nm")?; + dispatch(device, queue, &pipeline, &[&input, &output], 4096)?; + let results = floats(&output)?; + let mut failures = 0; + let mut examples = Vec::new(); + for (x, r) in inputs.iter().zip(results.as_chunks::<14>().0) { + for problem in numerical::broken(x, r, true) { + failures += 1; + if examples.len() < 12 { + examples.push(problem); + } + } + } + emit( + json!({"event":"numeric","case":"native-maths","words":4096*14,"failures":failures,"examples":examples,"stamp":stamp()}), + ); + let mut half_failures = 0; + let inputs = numerical::half_operands(); + let bytes: Vec = inputs + .iter() + .flatten() + .flat_map(|v| v.to_le_bytes()) + .collect(); + let input = buffer(device, &bytes)?; + let output = device + .new_buffer(4096 * 2 * 4, ResourceOptions::shared()) + .map_err(err)?; + emit(json!({"event":"begin","case":"native-half-rounding","stamp":stamp()})); + let pipeline = state(device, HALF, "hr")?; + dispatch(device, queue, &pipeline, &[&input, &output], 4096)?; + for (&[a, b], r) in inputs.iter().zip(floats(&output)?.as_chunks::<2>().0) { + // Bounds and allowed dropped-rounding results are the upstream half_rounding lane's. + let ha = numerical::metal_half(a as f64); + let hb = numerical::metal_half(b as f64); + let sum = numerical::metal_half(ha + hb); + let allowed = [ + a + b, + ha as f32 + hb as f32, + numerical::metal_half((a + b) as f64) as f32, + ]; + if !(r[0].to_bits() == (sum as f32).to_bits() + || numerical::half_within_sum(r[0], a as f64, b as f64) + || allowed.contains(&r[0])) + { + half_failures += 1; + } + if !(r[1].to_bits() == (ha as f32).to_bits() + || numerical::half_within(r[1], a as f64) + || r[1].to_bits() == a.to_bits()) + { + half_failures += 1; + } + } + emit( + json!({"event":"numeric","case":"native-half-rounding","words":4096*2,"failures":half_failures,"stamp":stamp()}), + ); + // These normal, exactly representable operands cover the native16 divisor limit. + // The large divisors need normalization before a half reciprocal can stay normal. + let cases = [ + [65504.0f32, 65504.0], + [65504.0, 32768.0], + [32768.0, 65504.0], + [16384.0, 16384.0], + [16392.0, 16392.0], + [-65504.0, 65504.0], + [65504.0, -32768.0], + [1.0, 1.0], + [0.00006103515625, 0.00006103515625], + ]; + let inputs: Vec<[f32; 2]> = (0..4096).map(|i| cases[i % cases.len()]).collect(); + let bytes: Vec = inputs + .iter() + .flatten() + .flat_map(|v| v.to_le_bytes()) + .collect(); + let input = buffer(device, &bytes)?; + let output = device + .new_buffer(4096 * 2 * 4, ResourceOptions::shared()) + .map_err(err)?; + emit(json!({"event":"begin","case":"native-half-division-range","stamp":stamp()})); + let pipeline = state(device, HALF_DIVISION, "hr")?; + dispatch(device, queue, &pipeline, &[&input, &output], 4096)?; + let mut division_failures = 0; + let mut max_error_ulp = 0.0f64; + for (&[a, b], r) in inputs.iter().zip(floats(&output)?.as_chunks::<2>().0) { + let want = a as f64 / b as f64; + let error = (r[0] as f64 - want).abs() / numerical::half_ulp(want); + if !error.is_finite() || error > 2.5 { + division_failures += 1; + } + max_error_ulp = max_error_ulp.max(error); + } + emit( + json!({"event":"numeric","case":"native-half-division-range","words":4096, + "failures":division_failures,"max_error_half_ulp":max_error_ulp,"bound_half_ulp":2.5,"stamp":stamp()}), + ); + Ok(failures == 0 && half_failures == 0 && division_failures == 0) +} +fn median(values: &mut [f64]) -> f64 { + values.sort_by(f64::total_cmp); + values[values.len() / 2] +} +fn paired( + mut translated: impl FnMut() -> Result<(f64, f64)>, + mut native: impl FnMut() -> Result<(f64, f64)>, +) -> Result<(f64, f64, f64, f64)> { + let (mut t, mut n, mut tw, mut nw) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for i in 0..WARM + SAMPLES { + let (a, b) = if i % 2 == 0 { + (translated()?, native()?) + } else { + let b = native()?; + (translated()?, b) + }; + if i >= WARM { + t.push(a.0); + tw.push(a.1); + n.push(b.0); + nw.push(b.1); + } + } + Ok(( + median(&mut t), + median(&mut n), + median(&mut tw), + median(&mut nw), + )) +} +fn row( + kind: &str, + precision: &str, + bytes: u64, + operations: u64, + times: (f64, f64, f64, f64), + difference: f64, +) -> bool { + let (t, n, tw, nw) = times; + let ratio = n / t; + let limit = match (kind, precision) { + ("arithmetic", "float16") => 0.5, + ("arithmetic", _) => 1e-4, + _ => 0.0, + }; + let correct = difference.is_finite() && difference <= limit; + let target_met = ratio >= 0.95; + emit( + json!({"event":"calibration","kind":kind,"precision":precision,"bytes":bytes,"operations":operations, + "translator_gpu_us":t,"native_gpu_us":n,"translator_wall_us":tw,"native_wall_us":nw, + "native_fraction":ratio,"required_fraction":0.95,"translator_gflops":operations as f64/t/1000.0, + "translator_gbps":bytes as f64/t/1000.0,"max_output_difference":difference,"comparison_limit":limit,"correct":correct,"target_met":target_met,"stamp":stamp()}), + ); + correct +} +fn calibrate(device: &Device, queue: &CommandQueue) -> Result { + let mut pass = true; + for (name, air, spv) in [("float32", FP32, SPV32), ("float16", FP16, SPV16)] { + if name == "float16" + && !device + .compile_device() + .profile() + .enabled() + .features + .contains(&metal2vulkan::device::VkFeature::ShaderFloat16) + { + emit( + json!({"event":"unsupported","case":"float16","reason":"shaderFloat16 is not enabled","stamp":stamp()}), + ); + pass = false; + continue; + } + emit(json!({"event":"begin","case":"arithmetic","precision":name,"stamp":stamp()})); + let pipeline = state(device, air, "arith")?; + for count in [131072u64, 524288] { + let data: Vec = (0..count * 4) + .flat_map(|i| ((i % 1024) as f32 / 2048.0).to_le_bytes()) + .collect(); + let input = buffer(device, &data)?; + let output = device + .new_buffer(data.len() as u64, ResourceOptions::shared()) + .map_err(err)?; + let control = device + .new_buffer(data.len() as u64, ResourceOptions::shared()) + .map_err(err)?; + let native = Native::new(device, queue, spv, &[&input, &control], count, 16)?; + let times = paired( + || dispatch(device, queue, &pipeline, &[&input, &output], count), + || native.run(queue, false), + )?; + let difference = + floats(&output)? + .iter() + .zip(floats(&control)?) + .fold(0.0f64, |v, (a, b)| { + if a.is_finite() && b.is_finite() { + v.max((*a as f64 - *b as f64).abs()) + } else { + f64::INFINITY + } + }); + pass &= row( + "arithmetic", + name, + 0, + count * 4 * ITERATIONS * 2, + times, + difference, + ); + } + } + let pipeline = state(device, EXAMPLES, "scale_and_offset")?; + let scale = buffer(device, &2.0f32.to_le_bytes())?; + for mib in [16u64, 64, 256] { + emit(json!({"event":"begin","case":"shared-bandwidth","mib":mib,"stamp":stamp()})); + let length = mib << 20; + let count = length / 4; + let input = device + .new_buffer(length, ResourceOptions::shared()) + .map_err(err)?; + if let Some(at) = input.contents() { + unsafe { + std::ptr::write_bytes(at.as_ptr(), 0x3e, length as usize); + } + } + let output = device + .new_buffer(length, ResourceOptions::shared()) + .map_err(err)?; + let control = device + .new_buffer(length, ResourceOptions::shared()) + .map_err(err)?; + let native = Native::new( + device, + queue, + SPV_SCALE, + &[&input, &control, &scale], + count, + 4, + )?; + let times = paired( + || dispatch(device, queue, &pipeline, &[&input, &output, &scale], count), + || native.run(queue, false), + )?; + let difference = floats(&output)? + .iter() + .zip(floats(&control)?) + .fold(0.0f64, |v, (a, b)| v.max((*a as f64 - *b as f64).abs())); + pass &= row( + "shared-kernel-read-write", + "float32", + length * 2, + 0, + times, + difference, + ); + let times = paired( + || copy(device, queue, &input, &output), + || native.run(queue, true), + )?; + let difference = floats(&output)? + .iter() + .zip(floats(&control)?) + .fold(0.0f64, |v, (a, b)| v.max((*a as f64 - *b as f64).abs())); + pass &= row( + "shared-blit-copy", + "bytes", + length * 2, + 0, + times, + difference, + ); + } + Ok(pass) +} + +fn fatal_gpu(why: &str) -> ! { + emit(json!({"event":"gpu-timeout","error":why,"reboot_required":true,"stamp":stamp()})); + use std::io::Write; + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); + // A timed-out queue may still own every resource; exit without running Vulkan destructors. + unsafe { libc::_exit(75) } +} + +struct Native<'a> { + device: &'a Device, + owner: vk::Device, + pool: vk::CommandPool, + commands: [vk::CommandBuffer; 2], + queries: vk::QueryPool, + fence: vk::Fence, + layout: vk::PipelineLayout, + set_layout: vk::DescriptorSetLayout, + descriptors: vk::DescriptorPool, + pipeline: vk::Pipeline, + module: vk::ShaderModule, + period: f64, + bits: u32, +} +impl<'a> Native<'a> { + fn new( + device: &'a Device, + queue: &CommandQueue, + spv: &[u8], + buffers: &[&Buffer], + count: u64, + stride: u64, + ) -> Result { + // All borrowed objects outlive native submissions, which finish before this object drops. + unsafe { + let raw = device.raw(); + let vk = raw.device; + let owner = vk.handle(); + let family = queue.with_vk_queue(|d, _, f| { + if d.handle() != owner { + fatal_gpu("Vulkan device changed before native setup"); + } + f + }); + let families = raw + .instance + .get_physical_device_queue_family_properties(raw.physical_device); + let bits = families[family as usize].timestamp_valid_bits; + if bits == 0 { + return Err("Native calibration requires hardware timestamps".into()); + } + let limits = raw + .instance + .get_physical_device_properties(raw.physical_device) + .limits; + let period = limits.timestamp_period as f64; + let chunk = limits.max_compute_work_group_count[0] as u64 * 64; + let pieces = count.div_ceil(chunk) as usize; + let pool = vk + .create_command_pool( + &vk::CommandPoolCreateInfo::default().queue_family_index(family), + None, + ) + .map_err(err)?; + let cmds = device + .with_loader_word(|| { + vk.allocate_command_buffers( + &vk::CommandBufferAllocateInfo::default() + .command_pool(pool) + .level(vk::CommandBufferLevel::PRIMARY) + .command_buffer_count(2), + ) + }) + .map_err(err)?; + let commands = [cmds[0], cmds[1]]; + let queries = vk + .create_query_pool( + &vk::QueryPoolCreateInfo::default() + .query_type(vk::QueryType::TIMESTAMP) + .query_count(2), + None, + ) + .map_err(err)?; + let fence = vk + .create_fence(&vk::FenceCreateInfo::default(), None) + .map_err(err)?; + let bindings: Vec<_> = (0..buffers.len()) + .map(|i| { + vk::DescriptorSetLayoutBinding::default() + .binding(i as u32) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .descriptor_count(1) + .stage_flags(vk::ShaderStageFlags::COMPUTE) + }) + .collect(); + let set_layout = vk + .create_descriptor_set_layout( + &vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings), + None, + ) + .map_err(err)?; + let layout = vk + .create_pipeline_layout( + &vk::PipelineLayoutCreateInfo::default().set_layouts(&[set_layout]), + None, + ) + .map_err(err)?; + let sizes = [vk::DescriptorPoolSize { + ty: vk::DescriptorType::STORAGE_BUFFER, + descriptor_count: (buffers.len() * pieces) as u32, + }]; + let descriptors = vk + .create_descriptor_pool( + &vk::DescriptorPoolCreateInfo::default() + .max_sets(pieces as u32) + .pool_sizes(&sizes), + None, + ) + .map_err(err)?; + let layouts = vec![set_layout; pieces]; + let sets = vk + .allocate_descriptor_sets( + &vk::DescriptorSetAllocateInfo::default() + .descriptor_pool(descriptors) + .set_layouts(&layouts), + ) + .map_err(err)?; + for (piece, &set) in sets.iter().enumerate() { + let first = piece as u64 * chunk; + let units = (count - first).min(chunk); + let infos: Vec<_> = buffers + .iter() + .enumerate() + .map(|(i, b)| { + let (buffer, offset) = b.vk_buffer(); + [vk::DescriptorBufferInfo::default() + .buffer(buffer) + .offset(offset + if i < 2 { first * stride } else { 0 }) + .range(if i < 2 { units * stride } else { b.length() })] + }) + .collect(); + let writes: Vec<_> = infos + .iter() + .enumerate() + .map(|(i, b)| { + vk::WriteDescriptorSet::default() + .dst_set(set) + .dst_binding(i as u32) + .descriptor_type(vk::DescriptorType::STORAGE_BUFFER) + .buffer_info(b) + }) + .collect(); + vk.update_descriptor_sets(&writes, &[]); + } + let words: Vec = spv + .chunks_exact(4) + .map(|b| u32::from_le_bytes(b.try_into().unwrap())) + .collect(); + let module = vk + .create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&words), None) + .map_err(err)?; + let stage = vk::PipelineShaderStageCreateInfo::default() + .stage(vk::ShaderStageFlags::COMPUTE) + .module(module) + .name(c"main"); + let pipeline = vk + .create_compute_pipelines( + vk::PipelineCache::null(), + &[vk::ComputePipelineCreateInfo::default() + .stage(stage) + .layout(layout)], + None, + ) + .map_err(|(_, e)| err(e))?[0]; + let barriers = [vk::MemoryBarrier::default() + .src_access_mask(vk::AccessFlags::MEMORY_WRITE | vk::AccessFlags::HOST_WRITE) + .dst_access_mask(vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE)]; + for (i, &cb) in commands.iter().enumerate() { + vk.begin_command_buffer(cb, &vk::CommandBufferBeginInfo::default()) + .map_err(err)?; + vk.cmd_reset_query_pool(cb, queries, 0, 2); + vk.cmd_pipeline_barrier( + cb, + vk::PipelineStageFlags::ALL_COMMANDS | vk::PipelineStageFlags::HOST, + vk::PipelineStageFlags::ALL_COMMANDS, + vk::DependencyFlags::empty(), + &barriers, + &[], + &[], + ); + vk.cmd_write_timestamp(cb, vk::PipelineStageFlags::BOTTOM_OF_PIPE, queries, 0); + if i == 0 { + vk.cmd_bind_pipeline(cb, vk::PipelineBindPoint::COMPUTE, pipeline); + for (piece, &set) in sets.iter().enumerate() { + vk.cmd_bind_descriptor_sets( + cb, + vk::PipelineBindPoint::COMPUTE, + layout, + 0, + &[set], + &[], + ); + let units = (count - piece as u64 * chunk).min(chunk); + vk.cmd_dispatch(cb, (units / 64) as u32, 1, 1); + } + } else { + let (a, ao) = buffers[0].vk_buffer(); + let (b, bo) = buffers[1].vk_buffer(); + vk.cmd_copy_buffer( + cb, + a, + b, + &[vk::BufferCopy { + src_offset: ao, + dst_offset: bo, + size: buffers[0].length(), + }], + ); + } + vk.cmd_pipeline_barrier( + cb, + vk::PipelineStageFlags::ALL_COMMANDS, + vk::PipelineStageFlags::HOST, + vk::DependencyFlags::empty(), + &[vk::MemoryBarrier::default() + .src_access_mask(vk::AccessFlags::MEMORY_WRITE) + .dst_access_mask(vk::AccessFlags::HOST_READ)], + &[], + &[], + ); + vk.cmd_write_timestamp(cb, vk::PipelineStageFlags::BOTTOM_OF_PIPE, queries, 1); + vk.end_command_buffer(cb).map_err(err)?; + } + Ok(Self { + device, + owner, + pool, + commands, + queries, + fence, + layout, + set_layout, + descriptors, + pipeline, + module, + period, + bits, + }) + } + } + fn run(&self, queue: &CommandQueue, copy: bool) -> Result<(f64, f64)> { + let wall = Instant::now(); + unsafe { + let raw = self.device.raw(); + let vk = raw.device; + if vk.handle() != self.owner { + fatal_gpu("Vulkan device changed during native calibration"); + } + vk.reset_fences(&[self.fence]).map_err(err)?; + queue + .with_vk_queue(|d, q, _| { + if d.handle() != self.owner { + fatal_gpu("Vulkan queue changed during native calibration"); + } + d.queue_submit( + q, + &[vk::SubmitInfo::default() + .command_buffers(&[self.commands[usize::from(copy)]])], + self.fence, + ) + }) + .map_err(err)?; + match vk.wait_for_fences(&[self.fence], true, 10_000_000_000) { + Err(vk::Result::TIMEOUT) => fatal_gpu("native completion exceeded 10 seconds"), + other => other.map_err(err)?, + } + let mut times = [0u64; 2]; + vk.get_query_pool_results(self.queries, 0, &mut times, vk::QueryResultFlags::TYPE_64) + .map_err(err)?; + let mask = if self.bits == 64 { + u64::MAX + } else { + (1u64 << self.bits) - 1 + }; + let us = (times[1].wrapping_sub(times[0]) & mask) as f64 * self.period / 1000.0; + if us <= 0.0 { + return Err("Native timestamp interval is zero".into()); + } + Ok((us, wall.elapsed().as_secs_f64() * 1e6)) + } + } +} +impl Drop for Native<'_> { + fn drop(&mut self) { + unsafe { + let raw = self.device.raw(); + let vk = raw.device; + if vk.handle() != self.owner { + fatal_gpu("Vulkan device changed during native calibration"); + } + vk.destroy_pipeline(self.pipeline, None); + vk.destroy_shader_module(self.module, None); + vk.destroy_descriptor_pool(self.descriptors, None); + vk.destroy_pipeline_layout(self.layout, None); + vk.destroy_descriptor_set_layout(self.set_layout, None); + vk.destroy_fence(self.fence, None); + vk.destroy_query_pool(self.queries, None); + vk.destroy_command_pool(self.pool, None); + } + } +} diff --git a/tools/gpu-acceptance/main.c b/tools/gpu-acceptance/main.c new file mode 100644 index 0000000..91fe4a8 --- /dev/null +++ b/tools/gpu-acceptance/main.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MIT +#include +#include +#include +#include + +extern int nmb_acceptance(const char *, const char *, const char *); +extern void nmb_acceptance_describe(void); + +static int primed(const char *path) { + char boot[64] = {0}, token[80], *line = NULL; + size_t size = sizeof boot, capacity = 0; + if (sysctlbyname("kern.bootsessionuuid", boot, &size, NULL, 0)) return 0; + boot[sizeof boot - 1] = 0; + snprintf(token, sizeof token, " boot=%s ", boot); + FILE *file = fopen(path, "r"); + if (!file) return 0; + int region = 0, complete = 0; + while (getline(&line, &capacity, file) >= 0) { + if (!strncmp(line, "list ", 5)) { + region = strstr(line, token) && strstr(line, " scope=2 "); + complete = 0; + } else if (!strncmp(line, "pass ", 5)) { + complete = region && strstr(line, token) && strstr(line, " scope=2 ") && strstr(line, " complete=1 "); + } + } + free(line); + fclose(file); + if (complete) setenv("NATIVE_METAL_BOOT_SESSION", boot, 1); + return complete; +} + +int main(int argc, char **argv) { + const char *loader = NULL, *cache = NULL, *prime = NULL, *mode = "all"; + if (argc == 2 && !strcmp(argv[1], "--describe")) {nmb_acceptance_describe(); return 0;} + for (int i = 1; i < argc; ++i) { + if (i + 1 >= argc) goto usage; + if (!strcmp(argv[i], "--loader")) loader = argv[++i]; + else if (!strcmp(argv[i], "--cache")) cache = argv[++i]; + else if (!strcmp(argv[i], "--prime-log")) prime = argv[++i]; + else if (!strcmp(argv[i], "--mode")) mode = argv[++i]; + else goto usage; + } + if (!loader || !cache || !prime) goto usage; + if (strcmp(mode, "all") && strcmp(mode, "numeric") && strcmp(mode, "calibrate")) goto usage; + if (!primed(prime)) { + fprintf(stderr, "Current boot whole-system shader priming is incomplete: %s\n", prime); + return 69; + } + if (!strcmp(mode, "all")) { + int result = nmb_acceptance(loader, cache, "numeric"); + if (result) return result; + return nmb_acceptance(loader, cache, "calibrate"); + } + return nmb_acceptance(loader, cache, mode); +usage: + fprintf(stderr, "gpu-acceptance --loader PATH --cache DIR --prime-log PATH [--mode numeric|calibrate|all]\n"); + return 64; +} diff --git a/tools/replay-renderbox-alpha.mm b/tools/replay-renderbox-alpha.mm new file mode 100644 index 0000000..0c25efc --- /dev/null +++ b/tools/replay-renderbox-alpha.mm @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT +// Replay the captured macOS 27.3 RenderBox alpha stage with controlled framebuffer values. +#import +#import +#include +#include +#include +#include +#include +static id (*libraryWithAir)(const void *,size_t); +#include +static id air(const char *path) { + NSData *d=[NSData dataWithContentsOfFile:@(path)];uint32_t offset=0,length=0;if(d.length<20)return nil; + memcpy(&offset,(const uint8_t *)d.bytes+8,4);memcpy(&length,(const uint8_t *)d.bytes+12,4); + if(offset>d.length || length>d.length-offset)return nil; + return libraryWithAir((const uint8_t *)d.bytes+offset,length); +} +int main(int argc,char **argv) {@autoreleasepool { + if(argc!=5)return 2;id device=MTLCreateSystemDefaultDevice();if(!device)return 2; + void *plugin=dlopen("/Library/GPUBundles/NativeAdrenoMetal.bundle/Contents/MacOS/NativeAdrenoMetal",RTLD_LAZY|RTLD_NOLOAD); + libraryWithAir=(id (*)(const void *,size_t))dlsym(plugin?plugin:RTLD_DEFAULT,"native_bridge_metal_library_with_air");if(!libraryWithAir){fprintf(stderr,"Metal bridge AIR entry unavailable\n");return 2;} + printf("RenderBox alpha control device=%s\n",device.name.UTF8String); + auto vl=air(argv[1]),fl=air(argv[2]);if(!vl || !fl){fprintf(stderr,"AIR creation failed\n");return 1;} + NSData *globals=[NSData dataWithContentsOfFile:@(argv[3])],*gradient=[NSData dataWithContentsOfFile:@(argv[4])]; + uint32_t state[4]={0xe80524,0x24000,0xf473,0x6c};auto fc=[MTLFunctionConstantValues new];[fc setConstantValue:state type:MTLDataTypeUInt4 atIndex:0]; + NSError *error=nil;auto pd=[MTLRenderPipelineDescriptor new];pd.vertexFunction=[vl newFunctionWithName:@"alpha_effect_vertex"];pd.fragmentFunction=[fl newFunctionWithName:@"alpha_effect_fragment" constantValues:fc error:&error];pd.colorAttachments[0].pixelFormat=MTLPixelFormatRGBA16Float;pd.colorAttachments[1].pixelFormat=MTLPixelFormatRG32Float;pd.colorAttachments[1].writeMask=MTLColorWriteMaskNone; + auto pipeline=[device newRenderPipelineStateWithDescriptor:pd error:&error];if(!pipeline){fprintf(stderr,"pipeline: %s | %s\n",error.description.UTF8String,"");return 1;} + auto td=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA16Float width:230 height:124 mipmapped:NO];td.usage=MTLTextureUsageRenderTarget|MTLTextureUsageShaderRead;auto color=[device newTextureWithDescriptor:td];td.pixelFormat=MTLPixelFormatRG32Float;auto coverage=[device newTextureWithDescriptor:td]; + td.width=td.height=32;td.pixelFormat=MTLPixelFormatRGBA8Unorm;auto noise=[device newTextureWithDescriptor:td];uint8_t n[32*32*4];memset(n,127,sizeof n);[noise replaceRegion:MTLRegionMake2D(0,0,32,32) mipmapLevel:0 withBytes:n bytesPerRow:128]; + uint16_t indices[4]={1,0,2,3};auto index=[device newBufferWithBytes:indices length:sizeof indices options:MTLResourceStorageModeShared];auto queue=[device newCommandQueue]; + unsigned failed=0; + for(double red : {-0.25,0.,0.05,0.15,0.5,1.,1.5}) { + auto cb=[queue commandBuffer];auto pass=[MTLRenderPassDescriptor renderPassDescriptor];pass.colorAttachments[0].texture=color;pass.colorAttachments[0].loadAction=MTLLoadActionClear;pass.colorAttachments[0].storeAction=MTLStoreActionStore;pass.colorAttachments[0].clearColor=MTLClearColorMake(red,.2,.3,1);pass.colorAttachments[1].texture=coverage;pass.colorAttachments[1].loadAction=MTLLoadActionClear;pass.colorAttachments[1].storeAction=MTLStoreActionStore;pass.colorAttachments[1].clearColor=MTLClearColorMake(1,.5,0,0); + auto e=[cb renderCommandEncoderWithDescriptor:pass];[e setRenderPipelineState:pipeline];[e setScissorRect:(MTLScissorRect){124,0,106,106}];[e setVertexBytes:globals.bytes length:globals.length atIndex:0];[e setFragmentBytes:globals.bytes length:globals.length atIndex:0];[e setFragmentBytes:gradient.bytes length:gradient.length atIndex:3];[e setFragmentTexture:noise atIndex:3];[e drawIndexedPrimitives:MTLPrimitiveTypeTriangleStrip indexCount:4 indexType:MTLIndexTypeUInt16 indexBuffer:index indexBufferOffset:0];[e endEncoding];[cb commit];[cb waitUntilCompleted];if(cb.status!=MTLCommandBufferStatusCompleted){fprintf(stderr,"render: %s\n",cb.error.description.UTF8String);return 1;} + std::vector<__fp16> pixels(230*124*4);[color getBytes:pixels.data() bytesPerRow:230*8 fromRegion:MTLRegionMake2D(0,0,230,124) mipmapLevel:0];unsigned bad=0;for(__fp16 v:pixels)bad+=!std::isfinite(float(v));failed+=bad!=0;size_t at=(50*230+150)*4;printf("source=%g nonfinite=%u sample=%g,%g,%g,%g\n",red,bad,float(pixels[at]),float(pixels[at+1]),float(pixels[at+2]),float(pixels[at+3])); + } + return failed?1:0; +}}