perf(metal): integrate AdrenoMetal runtime changes

This commit is contained in:
rponeawa
2026-11-16 10:03:58 +08:00
Unverified
parent a87ff47dee
commit 9f336eb4a4
19 changed files with 6905 additions and 128 deletions
+54 -8
View File
@@ -33,7 +33,9 @@
#include "NativeGpuTrace.h"
#include "NativeMetalBridgePhase.h"
#include "CacheExport.h"
#if !TARGET_OS_OSX
#include "PrimeShaders.h"
#endif
#include "ExceptionDiag.h"
#if TARGET_OS_OSX
@@ -224,6 +226,14 @@ static id clientDevice(unsigned port) {
}
// IOSurfaces this GPU reads or writes must be plain rows, as in the render server.
CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication);
#if TARGET_OS_OSX
if (geteuid() == 0 && getenv("NATIVE_METAL_PRIMER")) {
const std::string root = systemcache::writerDirectory();
if (root.empty()) return nil;
native_bridge_metal_system_cache(nullptr, nullptr, 0, true);
return native_bridge_metal_open(turnip, root.c_str(), registryID, nullptr);
}
#endif
char candidates[3][512] = {};
if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0;
if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0;
@@ -348,14 +358,21 @@ static id deviceForPort(unsigned port) {
// draws on a small part of its target otherwise pays for whole. The colour order of GMEM loads and stores is the image's own
// swap on the memory side (patches/mesa/turnip-gmem-memory-side-swap.patch). The RM row of the memory log counts both kinds;
// tu-debug= (for instance gmem,nocb: every pass in tile memory, no concurrent binning) overrides this.
// Concurrent binning is a Turnip option that is off unless asked for (tu_allow_concurrent_binning, read from the environment
// by Mesa's option parser): with it the BV thread bins a pass while BR still renders the one before. tu-cb=1 turns it on.
// It stays at Mesa's default, off: what it gains here was never measured.
if (bootArgSet("tu-cb=1")) setenv("tu_allow_concurrent_binning", "true", 0);
// A7xx bins on BV while BR renders; Turnip retains its per-pass LRZ, query and hardware-binning checks.
setenv("tu_allow_concurrent_binning", bootArgSet("tu-cb=0") ? "false" : "true", 1);
// A740 has the sampler's built-in border constants; custom colors retain Turnip's table route.
setenv("tu_enable_fast_border_color_for_undefined_formats", bootArgSet("tu-fbc=0") ? "false" : "true", 1);
// diag=1 turns on every diagnostic of the GPU stack in this process: the frontend's frame trace, Turnip's pass stamps and
// submission ring, the runtime's stage ring, and in the render server the status and telemetry lines. Without it none of them runs.
if (bootArgSet("diag=1") || bootArgSet("gpu-trace=1")) setenv("SHENG_DIAG", "1", 1);
else unsetenv("SHENG_DIAG");
if (getenv("SHENG_DIAG")) {
char boot[64] = {};
size_t size = sizeof boot - 1;
if (!sysctlbyname("kern.bootsessionuuid", boot, &size, nullptr, 0)) setenv("NATIVE_METAL_BOOT_SESSION", boot, 1);
setenv("METAL2VULKAN_ENCODER_TIMES", "8ms", 1);
setenv("METAL2VULKAN_QUEUE_TIMES", "1", 1);
}
if (bootArgSet("gpu-trace=1")) setenv("SHENG_GPU_TRACE", "1", 1);
else unsetenv("SHENG_GPU_TRACE");
// The bridge's work placement (metal-flow=<bits>, read below). Any value logs the FL counts
@@ -389,6 +406,35 @@ static id deviceForPort(unsigned port) {
// log-gpu=1: the GPU stack's log lines (NSLog, the pipeline journal); off without it.
if (bootArgSet("log-gpu=1")) setenv("SHENG_LOG_GPU", "1", 1);
else unsetenv("SHENG_LOG_GPU");
#if TARGET_OS_OSX
// ci-mps=0 selects Core Image's own blur kernels. Core Image reads this at its first blur;
// an explicit environment value takes precedence.
if (bootArgSet("ci-mps=0")) setenv("CI_ENABLE_MPS", "0", 0);
if (!strcmp(getprogname(), "iconservicesagent")) {
char args[1100] = {};
size_t size = sizeof args - 1;
if (sysctlbyname("kern.bootargs", args, &size, nullptr, 0) == 0)
if (const char *at = bootArgToken(args, "icon-cb=")) {
unsigned first = 0, count = 0, pass = 0, skipFirst = 0, skipLast = 0;
const int fields = sscanf(at + strlen("icon-cb="), "%u:%u:%u:%u:%u", &first, &count, &pass, &skipFirst, &skipLast);
if ((fields == 2 || fields == 5) && first && count && count <= 32) {
char range[32];snprintf(range, sizeof range, "%u:%u", first, count);
setenv("NATIVE_METAL_DUMP_CB", range, 0);
setenv("NATIVE_METAL_DUMP_LOG", "1", 0);
setenv("NATIVE_METAL_DUMP_FILE", "1", 0);
setenv("NATIVE_METAL_DUMP_PASS", "1", 0);
setenv("NATIVE_METAL_DUMP_INPUTS", "1", 0);
setenv("NATIVE_METAL_DUMP_DRAWS", "1:64", 0);
setenv("NATIVE_METAL_DUMP_WORDS", "32", 0);
setenv("NATIVE_METAL_DUMP_FUNCTIONS", "alpha_effect,accumulator_color,stroke_lines,plane_color,primitive_image", 0);
if (fields == 5 && pass && skipFirst && skipLast >= skipFirst) {
char skip[48];snprintf(skip, sizeof skip, "%u:%u:%u", pass, skipFirst, skipLast);
setenv("NATIVE_METAL_SKIP_DRAWS", skip, 0);
}
}
}
}
#endif
#if TARGET_OS_OSX
// macOS: Turnip opens the accelerator by class (AGXAccelerator here); the display takes plain rows (no Apple framebuffer or IOSurface
// compression). The runtime keeps its default block size: the virtual-machine board shrank it for a 512 MiB GPU pool, which this
@@ -608,12 +654,11 @@ static id deviceForPort(unsigned port) {
return nullptr;
}, gpu);
}
// Every Metal library of the OS through the runtime, 90 s after the render server opened the device (PrimeShaders.h); the boots that froze
// or crashed before three minutes never primed anything. Progress is kept in the Data cache directory, so later boots resume.
// The translator recurses deeply on large functions: give it the stack of a main thread.
#if !TARGET_OS_OSX
// The iPadOS primer uses an 8 MiB stack for large translated functions.
pthread_attr_setstacksize(&attributes, 8u << 20);
pthread_t primer;
pthread_create(&primer, &attributes, [](void *) -> void * {
if (!bootArgSet("gpu-prime=0")) pthread_create(&primer, &attributes, [](void *) -> void * {
pthread_setname_np("native-metal-prime"); // a process sample and the telemetry's TH rows name its CPU time
char list[1024];
prime::run((id<MTLDevice>)device, gCacheDirectory, 90,
@@ -621,6 +666,7 @@ static id deviceForPort(unsigned port) {
gFillsSystemCache);
return nullptr;
}, nullptr);
#endif
pthread_attr_destroy(&attributes);
// The GMEM colour order test (NativeTurnipSwapTest.h), once, 20 s after the device is published: each result line goes to the
// kernel's GPU trace as a problem record (event 65).
+249 -40
View File
@@ -5,7 +5,6 @@
// Every result comes from the runtime and the GPU: a refused call returns nil/NO with the runtime's reason, an encoder call the runtime
// refuses fails its command buffer (status Error, handlers run) instead of being dropped. A selector this frontend does not implement is
// reported through the missing-selector hook before the usual exception, so a hardware run names the gap.
// Shared events work within this process (no cross-process handle yet). Not yet implemented here: heaps, argument buffers, indirect commands, drawables/presentation.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import <IOSurface/IOSurfaceRef.h>
@@ -19,6 +18,7 @@
#include <deque>
#include <condition_variable>
#include <cmath>
#include <cstdarg>
#include <atomic>
#include <optional>
#include <mutex>
@@ -580,7 +580,7 @@ static bool isRenderServer() {return nativeIsRenderServer();}
static void endRenderServerAfterGpuLoss() {
static std::atomic<bool> ending{false};
if(!isRenderServer() || ending.exchange(true))return;
NBLOG(@"native-metal: the render server's GPU context is lost; ending backboardd so launchd starts it again");
NBLOG(@"native-metal: the render server's GPU context is lost; ending %s so launchd starts it again",NATIVE_RENDER_SERVER);
dispatch_after(dispatch_time(DISPATCH_TIME_NOW,(int64_t)(500*NSEC_PER_MSEC)),dispatch_get_global_queue(QOS_CLASS_USER_INTERACTIVE,0),^{
kill(getpid(),SIGTERM);
});
@@ -624,6 +624,7 @@ static void logTexture(const char *kind,MTLTextureDescriptor *d,IOSurfaceRef sur
// planes, requested plane, plane offset, plane row bytes and allocation, and the descriptor's size and pixel format.
struct SurfaceRefusal {uint32_t step,format,planes,plane,pixelFormat;uint64_t offset,row,allocation,width,height,surfaceWidth,surfaceHeight,rotation;bool framebufferOnly;};
static SurfaceRefusal gSurfaceLastRefusal;
#if !TARGET_OS_OSX
// Shader priming's progress (PrimeShaders.h, in the plugin), for the PR row. Weak here: the plugin's own definition wins, and a host
// binary that links the frontend without the plugin (every check-*-host.py test) has zeroes instead of an undefined symbol.
extern "C" {
@@ -635,8 +636,14 @@ __attribute__((weak)) volatile uint32_t native_prime_rate;
__attribute__((weak)) volatile uint32_t native_prime_functions_resumed;
__attribute__((weak)) volatile uint32_t native_prime_libraries_primed; // libraries this process's passes compiled
__attribute__((weak)) volatile uint32_t native_prime_libraries_skipped; // other GPUs' driver bundles (whole system) or not loaded here
__attribute__((weak)) volatile uint32_t native_prime_pass_complete;
__attribute__((weak)) volatile uint32_t native_prime_libraries_failed;
__attribute__((weak)) volatile uint32_t native_prime_functions_refused;
__attribute__((weak)) volatile uint32_t native_prime_functions_skipped;
__attribute__((weak)) volatile uint32_t native_prime_inventory_failed;
__attribute__((weak)) volatile uint32_t native_prime_scope; // 0 no pass yet, 1 loaded bundles only, 2 the whole system
}
#endif
static void refuseSurface(uint32_t step,IOSurfaceRef surface,MTLTextureDescriptor *d,NSUInteger plane,uint64_t offset,uint64_t row) {
gSurfaceRefusals++;
SurfaceRefusal r={step,surface?IOSurfaceGetPixelFormat(surface):0,surface?(uint32_t)IOSurfaceGetPlaneCount(surface):0,(uint32_t)plane,
@@ -872,7 +879,7 @@ static NSData *compileMetalSource(NSString *source,NSString *arguments,NSString
memcpy(request.data()+16,text.bytes,text.length);memcpy(request.data()+16+sourceBytes,options,optionBytes);
return codeGenRequest(13,request,warnings,failure);
}
// The front end's arguments for a set of compile options, as Metal passes them: the language version, fast math (Metal's default),
// The front end's arguments for a set of compile options, as Metal passes them: the language version, math mode and FP32 functions,
// invariance, size optimization and the preprocessor macros; module caches in this process's temporary directory, which a sandboxed
// client may write.
static NSString *compileArgument(NSString *text) {
@@ -885,8 +892,26 @@ static NSString *compileArguments(MTLCompileOptions *options) {
const NSUInteger version=options && options.languageVersion?(NSUInteger)options.languageVersion:(NSUInteger)MTLLanguageVersion3_1;
const unsigned major=(unsigned)(version>>16),minor=(unsigned)(version&0xffff);
[arguments addObject:[NSString stringWithFormat:major>=3?@"-std=metal%u.%u":@"-std=ios-metal%u.%u",major,minor]];
const BOOL fast=options?((BOOL (*)(id,SEL))objc_msgSend)(options,sel_registerName("fastMathEnabled")):YES;
[arguments addObject:fast?@"-ffast-math":@"-fno-fast-math"];
const SEL modeSelector=sel_registerName("mathMode"),functionsSelector=sel_registerName("mathFloatingPointFunctions");
NSString *modeName=nil,*functionsName=nil;
if([options respondsToSelector:modeSelector] && [options respondsToSelector:functionsSelector]) {
switch(((NSInteger (*)(id,SEL))objc_msgSend)(options,modeSelector)) {
case MTLMathModeSafe:modeName=@"safe";break;
case MTLMathModeRelaxed:modeName=@"relaxed";break;
case MTLMathModeFast:modeName=@"fast";break;
}
switch(((NSInteger (*)(id,SEL))objc_msgSend)(options,functionsSelector)) {
case MTLMathFloatingPointFunctionsFast:functionsName=@"fast";break;
case MTLMathFloatingPointFunctionsPrecise:functionsName=@"precise";break;
}
}
if(modeName && functionsName) {
[arguments addObject:[@"-fmetal-math-mode=" stringByAppendingString:modeName]];
[arguments addObject:[@"-fmetal-math-fp32-functions=" stringByAppendingString:functionsName]];
} else {
const BOOL fast=options?((BOOL (*)(id,SEL))objc_msgSend)(options,sel_registerName("fastMathEnabled")):YES;
[arguments addObject:fast?@"-ffast-math":@"-fno-fast-math"];
}
if(options.preserveInvariance)[arguments addObject:@"-fpreserve-invariance"];
if(options.optimizationLevel==MTLLibraryOptimizationLevelSize)[arguments addObject:@"-Os"];
[options.preprocessorMacros enumerateKeysAndObjectsUsingBlock:^(NSString *name,NSObject *value,BOOL *) {
@@ -1584,7 +1609,46 @@ static void noteUnalignedTexture(MTLTextureDescriptor *d,NSUInteger offset,NSUIn
@end
@implementation NBFunctionConstant
@end
@interface NBFunctionAttribute : NSObject
@property(nonatomic,copy) NSString *name;
@property(nonatomic) NSUInteger attributeIndex;
@property(nonatomic) MTLDataType attributeType;
@property(nonatomic,getter=isActive) BOOL active;
@property(nonatomic,getter=isPatchData) BOOL patchData;
@property(nonatomic,getter=isPatchControlPointData) BOOL patchControlPointData;
@end
@implementation NBFunctionAttribute
@end
static void libraryAirBytes(NBLibrary *library,const void *bytes,size_t length) {
const uint8_t *file=(const uint8_t *)bytes;
library.bytes=file;library.length=length;
if(length<8 || memcmp(file,"\xca\xfe\xba\xbe",4))return;
auto word=[&](size_t at) {return ((uint32_t)file[at]<<24)|((uint32_t)file[at+1]<<16)|((uint32_t)file[at+2]<<8)|file[at+3];};
const uint32_t count=word(4);
if(count>(length-8)/20)return;
for(uint32_t i=0;i<count;i++) {
const size_t at=8+20*(size_t)i;if(at>length || length-at<20)return;
if(word(at)!=0x01000017)continue;
const size_t offset=word(at+8),size=word(at+12);
if(offset>length || size>length-offset)return;
library.bytes=file+offset;library.length=size;return;
}
}
@implementation NBFunction
- (NSArray *)stageInputAttributes {
if(!_handle)return nil;
int32_t count=nmb_function_attribute_count(_handle);if(count<=0)return nil;
NSMutableArray *attributes=[NSMutableArray arrayWithCapacity:count];
for(int32_t i=0;i<count;i++) {
char name[1024];uint64_t facts[5]={};
size_t length=nmb_function_attribute_at(_handle,(uint32_t)i,name,sizeof name,facts);
if(!length || length>sizeof name)return nil;
NBFunctionAttribute *a=[NBFunctionAttribute new];a.name=@(name);a.attributeIndex=facts[0];a.attributeType=(MTLDataType)facts[1];
a.active=facts[2];a.patchData=facts[3];a.patchControlPointData=facts[4];[attributes addObject:a];
}
return attributes;
}
- (NSArray *)vertexAttributes {return self.functionType==MTLFunctionTypeVertex?[self stageInputAttributes]:nil;}
- (NSDictionary<NSString *,MTLFunctionConstant *> *)functionConstantsDictionary {
NSMutableDictionary *result=[NSMutableDictionary dictionary];
if(!_handle)return result;
@@ -1759,6 +1823,14 @@ static NSError *functionLookupError(NBLibrary *library, NSString *name, NSString
}
return names;
}
- (NSArray<NSString *> *)nb_primeFunctionNames {
NSMutableArray<NSString *> *names=[NSMutableArray array];
if(!forEachLibraryEntry(self,[&](const std::string &entry,int type) {
if((type==0 || type==2) && !entry.empty())[names addObject:@(entry.c_str())];
return true;
}))return self.functionNames;
return names;
}
// _MTLLibrary's externFunctionNames: the entries of external linkage (TYPE 5), an empty array for a library without any. iPadOS 27's Core
// Image collects the precompiled kernels' names so (DAGPrecompiledKernels: [CIKernelLibrary externFunctionNames] of ci_stdlib and
// ci_filters) and links a kernel into its DAG only when its name is there or in a stitchable library; without the answer (nil, an unknown
@@ -1789,6 +1861,7 @@ static NSError *functionLookupError(NBLibrary *library, NSString *name, NSString
NmbFunction *handle=name?nmb_function_new_zero_constants(_handle,name.UTF8String):nullptr;
if(!handle)return nil;
NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.sourceName=name;function.functionType=(MTLFunctionType)nmb_function_type(handle);
function.library=self;
return (id<MTLFunction>)function;
}
@end
@@ -2067,8 +2140,10 @@ static void dumpSurfacesOnce(NSSet *textures,NSSet *owners) {
// NATIVE_METAL_DUMP_CB: the textures this command buffer wrote, met while it was encoded (dumpTargets), the render pass's attachments
// waiting for their encoder (dumpPending), how many passes and kernels it encoded so far, and whether it is the plugin's own readback.
@property(nonatomic,strong) NSMutableArray *dumpTargets,*dumpPending;
@property(nonatomic,strong) NSMutableArray *dumpPassReads;
@property(nonatomic) uint64_t dumpAppIndex;
@property(nonatomic) uint32_t dumpPasses,dumpKernels;
@property(nonatomic) BOOL dumpReadback;
@property(nonatomic) BOOL dumpReadback,dumpCapture;
@property(nonatomic,strong) NBTexture *wallpaperOutput;
@property(nonatomic,strong) NSMutableArray<NSString *> *dumpLog; // its passes and first draws, as encoded (NBEncoder dumpDraw:)
- (void)refuse:(NSString *)what;
@@ -2125,9 +2200,18 @@ static std::pair<uint32_t,uint32_t> dumpRange() {
static bool dumpMaybe(NBCommandBuffer *commandBuffer) {
const auto range=dumpRange();
if(!range.first || commandBuffer.dumpReadback)return false;
if(commandBuffer.dumpCapture)return true;
const uint64_t next=gAppCommits.load()+1;
return next+16>=range.first && next<(uint64_t)range.first+range.second;
const bool capture=next+16>=range.first && next<(uint64_t)range.first+range.second;
// Keep a selected buffer's complete encoding when other buffers commit before it.
if(capture)commandBuffer.dumpCapture=YES;
return capture;
}
static bool dumpAtPassEnd() {
static const bool enabled=getenv("NATIVE_METAL_DUMP_PASS")!=nullptr;
return enabled;
}
static void dumpPassSnapshot(NBCommandBuffer *commands,NSArray *targets);
// A line of the command buffer's NATIVE_METAL_DUMP_CB log (its passes, first draws and their state), printed before its readback.
// NATIVE_METAL_DUMP_ONLY=1: only the NATIVE_METAL_DUMP_IMAGE targets are read back and written, with no log, so a long range of
// command buffers (a small texture over hundreds of frames) costs little.
@@ -2143,6 +2227,8 @@ static NSString *dumpWords(const uint8_t *p,size_t n) {
static const size_t words=[]{const char *v=getenv("NATIVE_METAL_DUMP_WORDS");return v && atoi(v)>0?size_t(atoi(v)):size_t(16);}();
NSMutableString *s=[NSMutableString string];
for(size_t i=0;i<std::min<size_t>(n/4,words);i++){float f;memcpy(&f,p+4*i,4);[s appendFormat:@" %.4g",f];}
[s appendString:@" hex"];
for(size_t i=0;i<std::min<size_t>(n/4,words);i++){uint32_t v;memcpy(&v,p+4*i,4);[s appendFormat:@" %08x",v];}
size_t zero=0,nan=0;const size_t all=std::min<size_t>(n/4,256);
for(size_t i=0;i<all;i++){uint32_t w;memcpy(&w,p+4*i,4);float f;memcpy(&f,&w,4);zero+=!w;nan+=std::isnan(f);}
[s appendFormat:@" (of %zu words %zu zero, %zu NaN)",all,zero,nan];
@@ -2166,7 +2252,7 @@ static bool dumpMatched(NSString *pipeline) {
}
static void dumpNote(NBCommandBuffer *commandBuffer,NSMutableArray *into,id texture,NSUInteger level,NSUInteger slice,NSString *where) {
if(!texture || ![texture isKindOfClass:[NBTexture class]])return;
for(NSDictionary *seen in commandBuffer.dumpTargets)
if(!dumpAtPassEnd())for(NSDictionary *seen in commandBuffer.dumpTargets)
if(seen[@"texture"]==texture && [seen[@"level"] unsignedIntegerValue]==level && [seen[@"slice"] unsignedIntegerValue]==slice)return;
for(NSDictionary *seen in commandBuffer.dumpPending)
if(seen[@"texture"]==texture && [seen[@"level"] unsignedIntegerValue]==level && [seen[@"slice"] unsignedIntegerValue]==slice)return;
@@ -2501,12 +2587,15 @@ static uint64_t encodeCommandOf(id object) {
- (void)texture:(id<MTLTexture>)texture index:(NSUInteger)index stage:(uint32_t)stage {
if(texture && ![(id)texture isKindOfClass:[NBTexture class]]){[_commandBuffer refuse:@"foreign texture"];return;}
if(gDrawStats)[self noteState:kDSTexture key:stage<<8|(uint32_t)index object:(__bridge const void *)texture value:0];
if(texture && stage==0 && (texture.usage&MTLTextureUsageShaderWrite) && dumpMaybe(_commandBuffer)) {
static const bool captureInputs=getenv("NATIVE_METAL_DUMP_INPUTS")!=nullptr;
const bool captureInput=stage==2 && captureInputs;
if(texture && ((stage==0 && (texture.usage&MTLTextureUsageShaderWrite)) || captureInput) && dumpMaybe(_commandBuffer)) {
if(!_dumpEntries)_dumpEntries=[NSMutableArray array];
if(!_commandBuffer.dumpTargets)_commandBuffer.dumpTargets=[NSMutableArray array];
const NSUInteger before=_commandBuffer.dumpTargets.count;
dumpNote(_commandBuffer,_commandBuffer.dumpTargets,texture,0,0,
[NSString stringWithFormat:@"kernel %u texture %lu",_commandBuffer.dumpKernels,(unsigned long)index]);
captureInput?[NSString stringWithFormat:@"pass %u input texture %lu",_commandBuffer.dumpPasses,(unsigned long)index]:
[NSString stringWithFormat:@"kernel %u texture %lu",_commandBuffer.dumpKernels,(unsigned long)index]);
if(_commandBuffer.dumpTargets.count>before)[_dumpEntries addObject:_commandBuffer.dumpTargets.lastObject];
}
if(texture){[_commandBuffer own:texture];}
@@ -2632,6 +2721,7 @@ static uint64_t encodeCommandOf(id object) {
_targetW=0;
}
[super endEncoding];
if(dumpAtPassEnd() && dumpMaybe(self.commandBuffer))dumpPassSnapshot(self.commandBuffer,self.dumpEntries);
}
- (void)setRenderPipelineState:(id<MTLRenderPipelineState>)state {
if(![(id)state isKindOfClass:[NBRenderPipeline class]]){[self.commandBuffer refuse:@"foreign render pipeline"];return;}
@@ -3024,19 +3114,80 @@ static unsigned dumpTexel(uint32_t format,const uint8_t *p,float c[4],unsigned *
default: return 0;
}
}
static void dumpWrite(const char *format,...) {
va_list args;va_start(args,format);char *text=nullptr;
const int length=vasprintf(&text,format,args);va_end(args);
if(length<0 || !text)return;
fwrite(text,1,(size_t)length,stderr);
static const bool file=getenv("NATIVE_METAL_DUMP_FILE")!=nullptr;
if(file && !gCacheRoot.empty()) {
static std::mutex lock;
static int fd=-1;
static size_t written=0;
static bool stopped=false;
std::lock_guard<std::mutex> hold(lock);
if(!stopped) {
if(fd<0)fd=open((gCacheRoot+"/command-buffer.log").c_str(),O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOFOLLOW,0600);
constexpr size_t limit=2*1024*1024;
const char capped[]="native-metal dump capture limit reached\n";
const bool fits=(size_t)length<=limit-sizeof capped-written;
const char *data=fits?text:capped;
size_t left=fits?(size_t)length:sizeof capped-1;
while(fd>=0 && left) {
const ssize_t n=write(fd,data,left);
if(n<0 && errno==EINTR)continue;
if(n<=0){close(fd);fd=-1;stopped=true;break;}
written+=(size_t)n;data+=n;left-=(size_t)n;
}
if(!fits && fd>=0){close(fd);fd=-1;stopped=true;}
}
}
static const bool unified=[]{const char *value=getenv("NATIVE_METAL_DUMP_LOG");return value && !strcmp(value,"1");}();
if(unified) {
static std::atomic<uint64_t> sequence;
const uint64_t entry=sequence.fetch_add(1)+1;
const size_t parts=((size_t)length+511)/512;
for(size_t part=0;part<parts;++part) {
const size_t at=part*512;
std::string chunk(text+at,std::min<size_t>(512,(size_t)length-at));
os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal CB dump %llu %zu/%zu: %{public}s",(unsigned long long)entry,part+1,parts,chunk.c_str());
}
}
free(text);
}
static void dumpShaderFunction(NBFunction *function) {
static NSArray<NSString *> *wanted=[]{
const char *value=getenv("NATIVE_METAL_DUMP_FUNCTIONS");
return value?[@(value) componentsSeparatedByString:@","]:@[];
}();
NSString *name=function.sourceName?:function.name;
bool selected=false;
for(NSString *prefix in wanted)if(prefix.length && ([prefix isEqualToString:@"all"] || [name containsString:prefix])){selected=true;break;}
if(!selected)return;
static std::mutex lock;static std::unordered_set<std::string> seen;static uint64_t sequence;
const std::string key=std::to_string((uintptr_t)function.library.bytes)+":"+(name.UTF8String?:"");
uint64_t entry;
{std::lock_guard<std::mutex> guard(lock);if(!seen.insert(key).second)return;entry=++sequence;}
NSData *data=functionBitcode(function);
if(!data.length || data.length>512*1024){dumpWrite("native-metal dump function %llu %s: unavailable (%lu bytes)\n",
(unsigned long long)entry,name.UTF8String,(unsigned long)data.length);return;}
NSString *encoded=[data base64EncodedStringWithOptions:0];
dumpWrite("native-metal dump function %llu %s: %s\n",(unsigned long long)entry,name.UTF8String,encoded.UTF8String);
}
static void dumpPrint(uint64_t appIndex,NSDictionary *read) {
NSDictionary *t=read[@"target"];NBTexture *texture=t[@"texture"];const NmbTextureDescriptor d=texture.descriptor;
NSString *encoder=t[@"encoder"];
char head[400];
snprintf(head,sizeof head,"native-metal dump cb %llu: %s%s%s%s texture %s %llux%llu f%llu L%lu S%lu",(unsigned long long)appIndex,
snprintf(head,sizeof head,"native-metal dump cb %llu: %s%s%s%s texture %s@%p %llux%llu f%llu L%lu S%lu",(unsigned long long)appIndex,
[t[@"where"] UTF8String],encoder?" (":"",encoder?encoder.UTF8String:"",encoder?")":"",texture.label.length?texture.label.UTF8String:"-",
(__bridge void *)texture,
(unsigned long long)d.width,(unsigned long long)d.height,(unsigned long long)d.format,
(unsigned long)[t[@"level"] unsignedIntegerValue],(unsigned long)[t[@"slice"] unsignedIntegerValue]);
if(read[@"skip"]){fprintf(stderr,"%s: %s\n",head,[read[@"skip"] UTF8String]);return;}
if(read[@"skip"]){dumpWrite("%s: %s\n",head,[read[@"skip"] UTF8String]);return;}
const unsigned w=[read[@"w"] unsignedIntValue],rows=[read[@"rows"] unsignedIntValue],bpp=[read[@"bpp"] unsignedIntValue];
const uint32_t format=[read[@"format"] unsignedIntValue];
const uint8_t *bytes=(const uint8_t *)[(id<MTLBuffer>)read[@"buffer"] contents];
if(!bytes){fprintf(stderr,"%s: no readback contents\n",head);return;}
if(!bytes){dumpWrite("%s: no readback contents\n",head);return;}
double sum[4]={},lo[4]={INFINITY,INFINITY,INFINITY,INFINITY},hi[4]={-INFINITY,-INFINITY,-INFINITY,-INFINITY};
uint64_t finite[4]={},nans=0,infs=0,black=0,texels=0;unsigned channels=1;
double cell[8][32]={};uint32_t cellCount[8][32]={},cellNan[8][32]={},cellLit[8][32]={};
@@ -3062,21 +3213,21 @@ static void dumpPrint(uint64_t appIndex,NSDictionary *read) {
}
snprintf(part,sizeof part,"; nan %llu inf %llu black %.1f%% of %llu",(unsigned long long)nans,(unsigned long long)infs,
texels?100.0*black/texels:0.0,(unsigned long long)texels);line+=part;
fprintf(stderr,"%s\n",line.c_str());
dumpWrite("%s\n",line.c_str());
for(unsigned gy=0;gy<8;gy++) {
char grid[33]={};
for(unsigned gx=0;gx<32;gx++) {
const uint32_t n=cellCount[gy][gx];const double mean=n?cell[gy][gx]/n:0;
grid[gx]=!n?' ':cellNan[gy][gx]?'!':!cellLit[gy][gx]?'.':mean>1?'+':mean<0?'-':(char)('0'+std::min(9,(int)(mean*10)));
}
fprintf(stderr,"native-metal dump cb %llu: |%s|\n",(unsigned long long)appIndex,grid);
dumpWrite("native-metal dump cb %llu: |%s|\n",(unsigned long long)appIndex,grid);
}
// NATIVE_METAL_DUMP_IMAGE: the whole texture, its rows top down as the format stores them, in base64 lines of 6 KiB.
if([read[@"image"] boolValue]) {
NSString *text=[[NSData dataWithBytesNoCopy:(void *)bytes length:(NSUInteger)rows*w*bpp freeWhenDone:NO] base64EncodedStringWithOptions:0];
const NSUInteger part=8192,parts=(text.length+part-1)/part;
for(NSUInteger i=0;i<parts;i++)
fprintf(stderr,"native-metal dump image cb %llu: %s|%ux%u f%u bpp %u|%lu/%lu|%s\n",(unsigned long long)appIndex,[t[@"where"] UTF8String],w,rows,format,bpp,
dumpWrite("native-metal dump image cb %llu: %s|%ux%u f%u bpp %u|%lu/%lu|%s\n",(unsigned long long)appIndex,[t[@"where"] UTF8String],w,rows,format,bpp,
(unsigned long)i+1,(unsigned long)parts,[text substringWithRange:NSMakeRange(i*part,std::min(part,text.length-i*part))].UTF8String);
}
}
@@ -3090,20 +3241,8 @@ static bool dumpImageWanted(NSString *where) {
for(NSString *w in wanted)if(w.length && ([w isEqualToString:@"all"] || [where containsString:w]))return true;
return false;
}
// The readback of `source`'s targets, by a command buffer of the plugin's own committed after it on its queue.
static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) {
NSArray *targets=[source.dumpTargets copy];
if(dumpOnly())targets=[targets filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(NSDictionary *t,NSDictionary *bindings){(void)bindings;
return dumpImageWanted(t[@"where"]);
}]];
else fprintf(stderr,"native-metal dump cb %llu: %lu targets, %u passes, %u kernels\n",(unsigned long long)appIndex,
(unsigned long)targets.count,source.dumpPasses,source.dumpKernels);
for(NSString *line in source.dumpLog)fprintf(stderr,"native-metal dump cb %llu: %s\n",(unsigned long long)appIndex,line.UTF8String);
if(!targets.count)return;
NBCommandBuffer *readback=(NBCommandBuffer *)[(id<MTLCommandQueue>)source.queue commandBuffer];
if(!readback)return;
readback.dumpReadback=YES;
id<MTLBlitCommandEncoder> blit=[(id<MTLCommandBuffer>)readback blitCommandEncoder];
static NSArray *dumpEncodeReads(NBCommandBuffer *commands,NSArray *targets) {
id<MTLBlitCommandEncoder> blit=[(id<MTLCommandBuffer>)commands blitCommandEncoder];
NSMutableArray *reads=[NSMutableArray array];
for(NSDictionary *t in targets) {
NBTexture *texture=t[@"texture"];const NmbTextureDescriptor d=texture.descriptor;
@@ -3111,7 +3250,8 @@ static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) {
float c[4];unsigned channels=0;const uint8_t zero[16]={};
const unsigned bpp=texture.pvrtc?0:dumpTexel((uint32_t)d.format,zero,c,&channels);
NSString *skip=!bpp?@"format not read back":d.samples>1?@"multisampled":d.type==MTLTextureType3D?@"3D":
texture.storageMode==MTLStorageModeMemoryless?@"memoryless":nil;
texture.storageMode==MTLStorageModeMemoryless?@"memoryless":
dumpAtPassEnd() && t[@"store"] && [t[@"store"] unsignedIntegerValue]==MTLStoreActionDontCare?@"store action DontCare":nil;
if(skip){[reads addObject:@{@"target":t,@"skip":skip}];continue;}
const NSUInteger w=std::max<NSUInteger>(1,(NSUInteger)d.width>>level),h=std::max<NSUInteger>(1,(NSUInteger)d.height>>level);
const bool image=dumpImageWanted(t[@"where"]);
@@ -3125,8 +3265,33 @@ static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) {
[reads addObject:@{@"target":t,@"buffer":buffer,@"w":@(w),@"rows":@(rows),@"bpp":@(bpp),@"format":@((uint32_t)d.format),@"image":@(image)}];
}
[blit endEncoding];
return reads;
}
// Snapshot a stored attachment immediately after its encoder, before heap reuse or a later pass can overwrite it.
static void dumpPassSnapshot(NBCommandBuffer *commands,NSArray *targets) {
if(dumpOnly())targets=[targets filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(NSDictionary *t,NSDictionary *bindings){(void)bindings;
return dumpImageWanted(t[@"where"]);
}]];
if(!targets.count)return;
if(!commands.dumpPassReads)commands.dumpPassReads=[NSMutableArray array];
[commands.dumpPassReads addObjectsFromArray:dumpEncodeReads(commands,targets)];
}
// The readback of `source`'s targets, by a command buffer of the plugin's own committed after it on its queue.
static void dumpReadback(NBCommandBuffer *source,uint64_t appIndex) {
NSArray *targets=[source.dumpTargets copy];
if(dumpOnly())targets=[targets filteredArrayUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(NSDictionary *t,NSDictionary *bindings){(void)bindings;
return dumpImageWanted(t[@"where"]);
}]];
else dumpWrite("native-metal dump cb %llu: %lu targets, %u passes, %u kernels%s\n",(unsigned long long)appIndex,
(unsigned long)targets.count,source.dumpPasses,source.dumpKernels,dumpAtPassEnd()?"; pass snapshots":"");
for(NSString *line in source.dumpLog)dumpWrite("native-metal dump cb %llu: %s\n",(unsigned long long)appIndex,line.UTF8String);
if(dumpAtPassEnd() || !targets.count)return;
NBCommandBuffer *readback=(NBCommandBuffer *)[(id<MTLCommandQueue>)source.queue commandBuffer];
if(!readback)return;
readback.dumpReadback=YES;
NSArray *reads=dumpEncodeReads(readback,targets);
[(id<MTLCommandBuffer>)readback addCompletedHandler:^(id<MTLCommandBuffer> done) {
if(done.error){fprintf(stderr,"native-metal dump cb %llu: readback failed: %s\n",(unsigned long long)appIndex,
if(done.error){dumpWrite("native-metal dump cb %llu: readback failed: %s\n",(unsigned long long)appIndex,
done.error.localizedDescription.UTF8String);return;}
for(NSDictionary *read in reads)dumpPrint(appIndex,read);
}];
@@ -3180,6 +3345,8 @@ static char kUserDictionaryKey;
- (void)setProtectionOptions:(uint64_t)options {if(options)[self refuse:[NSString stringWithFormat:@"protection options 0x%llx",options]];}
- (void)refuse:(NSString *)what {
if(!self.encodeFailure){self.encodeFailure=what;NBLOG(@"native-metal: command buffer refused: %@",what);}
// Recovery creates a new runtime device; launchd restarts a render server whose retained objects belong to the previous device.
if([what rangeOfString:@"MadeBeforeRecovery"].location!=NSNotFound)endRenderServerAfterGpuLoss();
}
- (MTLCommandBufferStatus)status {
if(self.failedLocally)return MTLCommandBufferStatusError;
@@ -3255,9 +3422,11 @@ static char kUserDictionaryKey;
_dumpPending=[NSMutableArray array];
for(unsigned i=0;i<8;i++) {
MTLRenderPassColorAttachmentDescriptor *c=d.colorAttachments[i];
const NSUInteger before=_dumpPending.count;
if(c.resolveTexture)dumpNote(self,_dumpPending,c.resolveTexture,c.resolveLevel,c.resolveSlice,
[NSString stringWithFormat:@"pass %u resolve %u",_dumpPasses,i]);
else dumpNote(self,_dumpPending,c.texture,c.level,c.slice,[NSString stringWithFormat:@"pass %u color %u",_dumpPasses,i]);
if(_dumpPending.count>before)_dumpPending.lastObject[@"store"]=@(c.storeAction);
}
dumpNote(self,_dumpPending,d.depthAttachment.texture,d.depthAttachment.level,d.depthAttachment.slice,
[NSString stringWithFormat:@"pass %u depth",_dumpPasses]);
@@ -3418,6 +3587,11 @@ static char kUserDictionaryKey;
frontStage(traceNumber,self.commitIndex,8);
const uint64_t flushed=flushSurfaces(_surfaces);
frontStage(traceNumber,self.commitIndex,9,flushed,(uint32_t)_surfaces.count);
if(_dumpAppIndex) {
if(self.status==MTLCommandBufferStatusCompleted)for(NSDictionary *read in _dumpPassReads)dumpPrint(_dumpAppIndex,read);
else dumpWrite("native-metal dump cb %llu: pass snapshots failed: %s\n",(unsigned long long)_dumpAppIndex,self.error.localizedDescription.UTF8String?:"command refused");
_dumpPassReads=nil;
}
if(freezeWallpaperOn() && !self.failedLocally && self.status==MTLCommandBufferStatusCompleted &&
self.wallpaperOutput && encodedWork.draws) {
std::lock_guard<std::recursive_mutex> guard(gWallpaperLock);
@@ -3519,13 +3693,15 @@ static char kUserDictionaryKey;
// Completion owns this reference; the runtime orders the non-owning scheduled callback before it.
void *context=(__bridge_retained void *)self;
const uint64_t appIndex=self.dumpReadback?0:gAppCommits.fetch_add(1)+1;
const auto range=dumpRange();
const bool capture=appIndex && range.first && appIndex>=range.first && appIndex<(uint64_t)range.first+range.second;
if(capture && dumpAtPassEnd()){self.dumpAppIndex=appIndex;dumpReadback(self,appIndex);}
frontStage(traceNumber,self.commitIndex,6);
if(nmb_command_buffer_on_completed(_handle,completed,context) &&
nmb_command_buffer_on_scheduled(_handle,scheduled,context) && nmb_command_buffer_commit(_handle)) {
frontStage(traceNumber,self.commitIndex,7);
const auto range=dumpRange();
self.submitted=YES;
if(appIndex && range.first && appIndex>=range.first && appIndex<(uint64_t)range.first+range.second)dumpReadback(self,appIndex);
if(capture && !dumpAtPassEnd())dumpReadback(self,appIndex);
[self fireEarly];return;
}
(void)CFBridgingRelease(context); // refused registration/commit cannot call the registered handlers
@@ -3859,6 +4035,14 @@ static bool renderDescriptor(MTLRenderPipelineDescriptor *d,NBRenderDescriptor *
// NATIVE_METAL_DUMP_CB: a render pipeline's functions, color formats (b blends, m its write mask), depth format and vertex layout
// (attribute=format@offset/buffer, layout=stride/step/rate), for its draws' lines.
static NSString *renderDumpInfo(MTLRenderPipelineDescriptor *d) {
if(dumpRange().first) {
NBFunction *vertex=[(id)d.vertexFunction isKindOfClass:[NBFunction class]]?(NBFunction *)d.vertexFunction:nil;
NBFunction *fragment=[(id)d.fragmentFunction isKindOfClass:[NBFunction class]]?(NBFunction *)d.fragmentFunction:nil;
dumpWrite("native-metal dump pipeline %s | v %s source %s %s | f %s source %s %s\n",d.label.UTF8String?:"-",
vertex.name.UTF8String?:"-",vertex.sourceName.UTF8String?:"-",vertex?vertex.constantsText.c_str():"",
fragment.name.UTF8String?:"-",fragment.sourceName.UTF8String?:"-",fragment?fragment.constantsText.c_str():"");
if(vertex)dumpShaderFunction(vertex);if(fragment)dumpShaderFunction(fragment);
}
NSMutableString *s=[NSMutableString stringWithFormat:@"%@ v=%@ f=%@",d.label?:@"-",d.vertexFunction.name?:@"-",d.fragmentFunction.name?:@"-"];
for(unsigned i=0;i<8;i++) {
MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];if(a.pixelFormat==MTLPixelFormatInvalid)continue;
@@ -4220,10 +4404,11 @@ static char dvmMemoryReleaseKey;
NSString *why=lastError();
uint64_t answer[2]={0,0};
(void)nmb_device_heap_texture_size_and_align(gDevice,&descriptor,answer);
GLOG(@"native-metal: heap texture refused: %@ | fmt %lu %lux%lu L%lu type %lu needs %#llx@%#llx heap %#lx used %#lx room %#lx",why,
GLOG(@"native-metal: heap texture refused: %@ | fmt %lu %lux%lu L%lu type %lu needs %#llx@%#llx heap %#lx used %#lx room %#lx usage %#lx options %#lx optimized %d descriptor %p",why,
(unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount,
(unsigned long)_heapType,(unsigned long long)answer[0],(unsigned long long)answer[1],(unsigned long)nmb_heap_size(_handle),
(unsigned long)nmb_heap_used_size(_handle),(unsigned long)nmb_heap_max_available_size(_handle,answer[1]?answer[1]:1));
(unsigned long)nmb_heap_used_size(_handle),(unsigned long)nmb_heap_max_available_size(_handle,answer[1]?answer[1]:1),
(unsigned long)d.usage,(unsigned long)d.resourceOptions,(int)d.allowGPUOptimizedContents,d);
if(offset!=UINT64_MAX)return nil;
handle=nmb_texture_new(gDevice,&descriptor);
if(!handle){NBLOG(@"native-metal: texture (fmt %lu %lux%lu, %lu levels) refused: %@",(unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount,lastError());return nil;}
@@ -4432,6 +4617,13 @@ static void fillBaseLimits(id device);
NmbTextureDescriptor descriptor;uint64_t answer[2]={0,0};
if(textureDescriptor(d,&descriptor)&&!nmb_device_heap_texture_size_and_align(gDevice,&descriptor,answer))
NBLOG(@"native-metal: heap texture size (fmt %lu %lux%lu, %lu levels) refused: %@",(unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount,lastError());
if(iconMetalTrace()) {
static std::atomic<unsigned> queries{0};
if(queries.fetch_add(1)<64)GLOG(@"native-metal: heap query fmt %lu %lux%lu L%lu usage %#lx options %#lx optimized %d -> %#llx@%#llx descriptor %p",
(unsigned long)d.pixelFormat,(unsigned long)d.width,(unsigned long)d.height,(unsigned long)d.mipmapLevelCount,
(unsigned long)d.usage,(unsigned long)d.resourceOptions,(int)d.allowGPUOptimizedContents,
(unsigned long long)answer[0],(unsigned long long)answer[1],d);
}
return (MTLSizeAndAlign){answer[0],answer[1]};
}
- (MTLSizeAndAlign)heapBufferSizeAndAlignWithLength:(NSUInteger)length options:(MTLResourceOptions)options {
@@ -4539,7 +4731,7 @@ static void fillBaseLimits(id device);
const uint64_t libraryBegan=nowUs();
NmbLibrary *handle=nmb_library_new(gDevice,bytes,size,false);gTimeLibrary.add(libraryBegan);
if(!handle)return failWith<id<MTLLibrary>>(error,@"library",nil);
NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=mapped;library.bytes=(const uint8_t *)bytes;library.length=size;
NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=mapped;libraryAirBytes(library,bytes,size);
if(error)*error=nil;return (id<MTLLibrary>)library;
}
// A library file that cannot be read (macOS run36: QuartzCore's default.metallib failed with NSCocoaErrorDomain 260 on a fresh Data
@@ -4564,7 +4756,7 @@ static void reportUnreadableLibrary(NSString *path) {
const uint64_t libraryBegan=nowUs();
NmbLibrary *handle=nmb_library_new(gDevice,bytes.bytes,bytes.length,false);gTimeLibrary.add(libraryBegan);
if(!handle)return failWith<id<MTLLibrary>>(error,@"library",nil);
NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=bytes;library.bytes=(const uint8_t *)bytes.bytes;library.length=bytes.length;
NBLibrary *library=[NBLibrary new];library.handle=handle;library.storage=bytes;libraryAirBytes(library,bytes.bytes,bytes.length);
if(error)*error=nil;return (id<MTLLibrary>)library;
}
- (id<MTLLibrary>)newLibraryWithURL:(NSURL *)url error:(NSError **)error {
@@ -5495,18 +5687,23 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) {
row("V vkpipe n=%llu total=%llums max=%llums waited=%llums failed=%llu",(ull)work[0],ms(work[1]),ms(work[2]),ms(work[3]),(ull)work[4]);
row("V cache hit%llu miss%llu wr%llu | msg i%llu w%llu",(ull)work[7],(ull)work[8],(ull)work[9],(ull)work[5],(ull)work[6]);
}
// The shader cache's full counts (nmb_device_shader_cache_detail): functions found/translated, compute and render pipelines found/compiled, entries
// The shader cache's full counts (nmb_device_shader_cache_detail): functions found/translated, driver data found/unchanged, render pipelines found/compiled, entries
// rejected, unreadable, handed to the directory (queued, written on the asking thread), bytes written, failed writes, evictions, the driver's own
// pipeline cache: loads, saves, hits, misses.
uint64_t sc[24]={};
if(nmb_device_shader_cache_detail(gDevice,sc)) {
row("SC fn %llu/%llu cp %llu/%llu rp %llu/%llu rej %llu rd-fail %llu",(ull)sc[2],(ull)sc[3],(ull)sc[4],(ull)sc[5],(ull)sc[6],(ull)sc[7],
row("SC fn %llu/%llu data %llu/%llu rp %llu/%llu rej %llu rd-fail %llu",(ull)sc[2],(ull)sc[3],(ull)sc[4],(ull)sc[5],(ull)sc[6],(ull)sc[7],
(ull)sc[8],(ull)sc[9]);
row("SCb wr %llu q %llu thr %llu %lluK wfail %llu evict %llu qhit %llu | drv load %llu save %llu hit %llu miss %llu",(ull)sc[10],(ull)sc[11],
(ull)sc[12],(ull)(sc[13]>>10),(ull)sc[15],(ull)sc[16],(ull)sc[14],(ull)sc[17],(ull)sc[20],(ull)sc[18],(ull)sc[19]);
row("SCc drv truncated %llu mismatched %llu",(ull)sc[21],(ull)sc[22]);
}
}
#if !TARGET_OS_OSX
row("PR compiling=%u libs=%u/%u made=%u rate=%u resumed=%u",native_prime_compiling,native_prime_libraries_done,native_prime_libraries_total,native_prime_made,native_prime_rate,native_prime_functions_resumed);
row("PRc state=%u failed=%u refused=%u bad=%u missing=%u",native_prime_pass_complete,native_prime_libraries_failed,
native_prime_functions_refused,native_prime_functions_skipped,native_prime_inventory_failed);
#endif
// Priming's reach and the system shader cache. PRb: the pass's scope (1 the bundles loaded here, 2 the whole system), the libraries
// it compiled itself, those it leaves out (other GPUs' driver bundles, or not loaded here), the system cache's state here (0 not
// named, 1 read, 2 filled by this process, 3 refused) and what this process published: the entries and KiB its cache directory
@@ -5516,12 +5713,24 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) {
uint64_t sc[24]={},sy[4]={};
if(gDevice)(void)nmb_device_shader_cache_detail(gDevice,sc);
(void)nmb_system_cache_counts(sy);
#if !TARGET_OS_OSX
row("PRb scope=%u primed=%u left=%u | system %d pub %llu/%lluK",native_prime_scope,native_prime_libraries_primed,
native_prime_libraries_skipped,gSystemCacheState,(ull)(gSystemCacheState==2?sc[11]+sc[12]:0),(ull)(gSystemCacheState==2?sc[13]>>10:0));
#else
row("SY system %d pub %llu/%lluK",gSystemCacheState,(ull)(gSystemCacheState==2?sc[11]+sc[12]:0),(ull)(gSystemCacheState==2?sc[13]>>10:0));
#endif
if(gSystemCacheState==1 || gSystemCacheState==3)
row("SY read %llu/%lluK rej %llu refused %llu replies %llu",(ull)sy[0],(ull)(sy[3]>>10),(ull)sy[1],(ull)sy[2],(ull)gCompileSystem.load());
}
if(gDevice) {
uint64_t fast[15]={};
if(nmb_device_fast_path_counts(gDevice,fast)) {
row("FP group %llu/%llu pools %llu/%llu linear %llu/%llu evict %llu",(ull)fast[0],(ull)fast[1],(ull)fast[2],(ull)fast[3],
(ull)fast[4],(ull)fast[5],(ull)fast[6]);
if(fast[9]==UINT64_MAX)row("CM calls %llu total %lluus",(ull)fast[7],(ull)fast[8]);
else row("CM calls %llu us %llu/%llu/%llu/%llu/%llu/%llu/%llu",(ull)fast[7],(ull)fast[8],(ull)fast[9],(ull)fast[10],(ull)fast[11],
(ull)fast[12],(ull)fast[13],(ull)fast[14]);
}
// The warm pass (records warmed, failed, unread, KiB of driver data) and where work ran: scheduler passes and batches at commit, render
// log waits, encoder ends on the lowering thread.
uint64_t flow[8]={};
+197 -60
View File
@@ -1,15 +1,5 @@
// SPDX-License-Identifier: MIT
// Shader priming in the render server: once the desktop has settled, every Metal library of the OS volume goes through the runtime on a
// background thread, so its functions are translated and compiled into the persistent cache directory before anything asks for them.
// Each kernel function becomes a compute pipeline; each vertex function becomes a render pipeline with the library's fragment function of
// the same stem (`_vert_` -> `_frag_`, or a fragment-less pipeline) and one BGRA8 attachment. Pipelines a caller makes with other constants
// or attachments still compile then, but their functions' modules are decoded and the translator has seen every function once.
// State in the cache directory: prime-done lists finished libraries (path, size, mtime), prime-bad the functions that ended the process
// (prime-current names the function being compiled; a start that finds it moves it to prime-bad), prime.log one line per library, and
// prime-fn the functions finished inside the libraries not finished yet (path, name and the build that made them).
// A library listed as finished is never read again, and a function listed in prime-fn is not compiled again, so a restart resumes where
// the previous run stopped even inside a library with thousands of functions (the one that held priming at 162 of 280 libraries for
// every boot: each boot ended before it did and began it again).
// Inventory compilation shared by Shader Cache on macOS and the iPadOS render server.
#pragma once
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
@@ -24,15 +14,18 @@
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <unistd.h>
#include <set>
#include <string>
#include <vector>
#include <functional>
extern "C" unsigned native_bridge_metal_recent_commits(uint64_t windowUs);
@interface NSObject (NBPrimeZeroConstants)
- (id<MTLFunction>)nb_newFunctionWithZeroConstants:(NSString *)name;
- (NSArray<NSString *> *)nb_primeFunctionNames;
@end
// Priming's progress for the telemetry table (the PR row): what the thread does (1 a pipeline compiles, 2 it waits for the desktop to
@@ -42,10 +35,18 @@ extern "C" {volatile uint32_t native_prime_compiling, native_prime_libraries_don
native_prime_functions_resumed;
// For the PRb row: the libraries this process's passes compiled, those a pass leaves out, and the pass's scope (1 the bundles loaded
// in this process, 2 the whole system).
volatile uint32_t native_prime_libraries_primed, native_prime_libraries_skipped, native_prime_scope;}
volatile uint32_t native_prime_libraries_primed, native_prime_libraries_skipped, native_prime_scope;
volatile uint32_t native_prime_pass_complete, native_prime_libraries_failed, native_prime_functions_refused,
native_prime_functions_skipped, native_prime_inventory_failed;}
extern "C" void native_bridge_metal_quiet_problems(bool);
namespace prime {
static std::function<void(const std::string &, const std::string &)> progress;
static std::function<bool()> cancelled;
static bool stopping() {return cancelled && cancelled();}
static void report(const std::string &library = {}, const std::string &function = {}) {
if (progress) progress(library, function);
}
#if TARGET_OS_OSX
// macOS: the system apps are in /System/Applications; the OS cryptex is reached through /System/Cryptexes/OS as on iPadOS.
static const char *const kRoots[] = {"/System/Library", "/System/Applications", "/Applications", "/System/Cryptexes/OS/System/Library"};
@@ -91,6 +92,14 @@ static void writeCurrent(const std::string &path, const std::string &line) {
}
static uint64_t nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1000000;}
static std::string bootSession() {
char uuid[64] = {};
size_t size = sizeof uuid;
if (sysctlbyname("kern.bootsessionuuid", uuid, &size, nullptr, 0) != 0) return "unknown";
uuid[sizeof uuid - 1] = 0;
return uuid;
}
// The libraries: first the paths of the list file beside the bundle (NativeAdrenoMetal.primelist, every .metallib of the OS image, one
// absolute path per line), because the render server's sandbox refuses to list most directories (a scan alone found 8 of 166). Only
// without a list does the scan run: it walks every directory it can reach and held a core for 8 s with frames stalling (i557u, 120 s
@@ -99,22 +108,25 @@ static uint64_t nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 100000
static std::vector<std::string> libraries(const std::string &listPath, std::string &note) {
std::vector<std::string> found;
std::set<std::pair<dev_t, ino_t>> seen;
unsigned lines = 0, failed = 0;
unsigned lines = 0, failed = 0, invalid = 0;
int lastErrno = 0;
std::set<std::string> list = listPath.empty() ? std::set<std::string>() : readLines(listPath);
const char *source = listPath.empty() ? "(none)" : listPath.c_str();
if (list.empty()) {list = embeddedLines(); source = "embedded";}
const bool inventory = !list.empty();
for (const std::string &path : list) {
++lines;
struct stat st = {};
if (path.empty() || path[0] != '/') continue;
if (path.empty() || path[0] != '/') {++invalid; continue;}
if (stat(path.c_str(), &st) != 0) {++failed; lastErrno = errno; continue;}
if (S_ISREG(st.st_mode) && seen.insert({st.st_dev, st.st_ino}).second) found.push_back(path);
if (!S_ISREG(st.st_mode)) {++invalid; continue;}
if (seen.insert({st.st_dev, st.st_ino}).second) found.push_back(path);
}
char line[200];
snprintf(line, sizeof line, "list %s lines=%u kept=%zu stat-failed=%u errno=%d", source, lines,
found.size(), failed, lastErrno);
snprintf(line, sizeof line, "list %s lines=%u kept=%zu stat-failed=%u errno=%d invalid=%u inventory=%u", source, lines,
found.size(), failed, lastErrno, invalid, inventory ? 1u : 0u);
note = line;
native_prime_inventory_failed = failed + invalid + (inventory ? 0u : 1u);
for (const char *root : kRoots) {
if (!found.empty()) break;
char *paths[] = {const_cast<char *>(root), nullptr};
@@ -139,12 +151,13 @@ static std::vector<std::string> libraries(const std::string &listPath, std::stri
// a quarter of a run's two-second windows, and a desktop in use ran at 18 to 20 submissions a second, under the bar above, so priming
// compiled through it (in four runs' process samples the priming thread was compiling or writing its progress in 88 to 94 % of the
// samples, in two of them beside the encoding thread's draws). There any command buffer in the window is the render server at work.
#if TARGET_OS_OSX
static constexpr unsigned kQuietCommits = 0;
#else
#if !TARGET_OS_OSX
static constexpr unsigned kQuietCommits = 8;
#endif
static uint64_t waitForQuiet() {
#if TARGET_OS_OSX
return 0;
#else
const uint64_t began = nowMs();
uint64_t quietSince = 0;
bool busy = false;
@@ -161,6 +174,7 @@ static uint64_t waitForQuiet() {
else if (now - quietSince >= 2000) return now - began;
usleep(100000);
}
#endif
}
#if TARGET_OS_OSX
@@ -201,6 +215,26 @@ static bool libraryOfDriverBundle(const std::string &path) {
const char *slash = strchr(plain + sizeof kExtensions - 1, '/');
return slash && slash - plain > 7 && strncmp(slash - 7, ".bundle", 7) == 0;
}
// A universal archive whose slices all contain Apple GPU machine code.
static bool appleGpuArchive(const std::string &path) {
FILE *file = fopen(path.c_str(), "rb");
if (!file) return false;
struct stat st = {};
uint8_t header[8];
auto word = [](const uint8_t *p) {return (uint32_t)p[0] << 24 | (uint32_t)p[1] << 16 | (uint32_t)p[2] << 8 | p[3];};
bool archive = fstat(fileno(file), &st) == 0 && fread(header, 1, sizeof header, file) == sizeof header &&
!memcmp(header, "\xca\xfe\xba\xbe", 4);
const uint32_t count = archive ? word(header + 4) : 0;
archive = archive && count && st.st_size >= 8 && count <= (uint64_t)(st.st_size - 8) / 20;
for (uint32_t i = 0; archive && i < count; ++i) {
uint8_t slice[20];
if (fread(slice, 1, sizeof slice, file) != sizeof slice) {archive = false; break;}
const uint64_t offset = word(slice + 8), size = word(slice + 12);
archive = word(slice) == 0x01000013 && size && offset <= (uint64_t)st.st_size && size <= (uint64_t)st.st_size - offset;
}
fclose(file);
return archive;
}
#endif
extern "C" const char *nmb_runtime_build(void);
@@ -253,6 +287,9 @@ static bool linkedKernel(id<MTLDevice> device, id<MTLLibrary> lib, id<MTLFunctio
[symbols addObject:symbol];
id<MTLFunction> callee = [lib newFunctionWithName:symbol];
if (!callee) return false;
if (callee.functionConstantsDictionary.count && [lib respondsToSelector:@selector(nb_newFunctionWithZeroConstants:)])
callee = [(NSObject *)lib nb_newFunctionWithZeroConstants:symbol];
if (!callee) return false;
[linked addObject:callee];
MTLComputePipelineDescriptor *d = [MTLComputePipelineDescriptor new];
d.computeFunction = kernel;
@@ -265,10 +302,57 @@ static bool linkedKernel(id<MTLDevice> device, id<MTLLibrary> lib, id<MTLFunctio
return false;
}
static id<MTLFunction> defaultFunction(id<MTLLibrary> lib, NSString *name, unsigned &zeroed) {
id<MTLFunction> function = [lib newFunctionWithName:name];
if (function.functionConstantsDictionary.count && [lib respondsToSelector:@selector(nb_newFunctionWithZeroConstants:)]) {
function = [(NSObject *)lib nb_newFunctionWithZeroConstants:name];
if (function) ++zeroed;
}
return function;
}
// Canonical packed inputs follow the function's reflected scalar type and lane count.
static MTLVertexDescriptor *vertexInputs(id<MTLFunction> function) {
MTLVertexDescriptor *descriptor = [MTLVertexDescriptor vertexDescriptor];
for (MTLVertexAttribute *attribute in function.vertexAttributes) {
const NSUInteger index = attribute.attributeIndex, type = attribute.attributeType;
if (index >= 31 || !attribute.isActive) continue;
MTLVertexFormat format = MTLVertexFormatInvalid;
NSUInteger bytes = 0;
struct Family {NSUInteger type, bytes; MTLVertexFormat scalar, vector2;};
static const Family families[] = {
{MTLDataTypeFloat,4,MTLVertexFormatFloat,MTLVertexFormatFloat2},
{MTLDataTypeHalf,2,MTLVertexFormatHalf,MTLVertexFormatHalf2},
{MTLDataTypeInt,4,MTLVertexFormatInt,MTLVertexFormatInt2},
{MTLDataTypeUInt,4,MTLVertexFormatUInt,MTLVertexFormatUInt2},
{MTLDataTypeShort,2,MTLVertexFormatShort,MTLVertexFormatShort2},
{MTLDataTypeUShort,2,MTLVertexFormatUShort,MTLVertexFormatUShort2},
{MTLDataTypeChar,1,MTLVertexFormatChar,MTLVertexFormatChar2},
{MTLDataTypeUChar,1,MTLVertexFormatUChar,MTLVertexFormatUChar2},
};
for (const Family &family : families) if (type >= family.type && type < family.type + 4) {
const NSUInteger lanes = type - family.type + 1;
format = lanes == 1 ? family.scalar : (MTLVertexFormat)(family.vector2 + lanes - 2);
bytes = family.bytes * lanes;
break;
}
if (format == MTLVertexFormatInvalid) continue;
descriptor.attributes[index].format = format;
descriptor.attributes[index].bufferIndex = index;
descriptor.layouts[index].stride = bytes;
descriptor.layouts[index].stepFunction = MTLVertexStepFunctionPerVertex;
}
return descriptor;
}
// One library: every function it names, in its order; returns the log line.
static std::string library(id<MTLDevice> device, const std::string &path, const std::string &current, const std::set<std::string> &bad,
const std::string &functionsPath, const std::set<std::string> &functionsDone, const std::string &build) {
const std::string &functionsPath, const std::set<std::string> &functionsDone,
const std::string &refusedPath, const std::set<std::string> &refusedDone,
const std::string &build, bool &complete) {
complete = false;
unsigned kernels = 0, renders = 0, made = 0, refused = 0, skipped = 0, zeroed = 0, linkedMade = 0, halfMade = 0;
const unsigned earlierRefusals = native_prime_functions_refused;
uint64_t waited = 0;
std::string firstRefusal; // the first refused function and why, so the log tells translator gaps from unspecialized constants
const uint64_t began = nowMs();
@@ -281,32 +365,31 @@ static std::string library(id<MTLDevice> device, const std::string &path, const
native_prime_compiling = 0;
if (!lib) return path + " library refused: " + (error ? error.localizedDescription.UTF8String : "no error");
NSArray<NSString *> *names = lib.functionNames;
for (NSString *name in names) {
NSArray<NSString *> *entries = [lib respondsToSelector:@selector(nb_primeFunctionNames)] ? [(NSObject *)lib nb_primeFunctionNames] : names;
for (NSString *name in entries) {
@autoreleasepool {
if (stopping()) break;
native_prime_functions_refused = earlierRefusals + refused;
report(path, name.UTF8String);
const std::string key = path + "\t" + name.UTF8String;
const std::string doneKey = key + "\t" + build;
native_prime_compiling = 3;
if (bad.count(key)) {++skipped; continue;}
if (functionsDone.count(doneKey)) {native_prime_functions_resumed = native_prime_functions_resumed + 1; continue;}
id<MTLFunction> function = [lib newFunctionWithName:name];
if (!function) {++refused; continue;}
if (bad.count(doneKey)) {++skipped; continue;}
const std::string attemptKey = doneKey + "-inputs3";
if (refusedDone.count(attemptKey)) {++refused; continue;}
if (functionsDone.count(doneKey)) {++made; native_prime_functions_resumed = native_prime_functions_resumed + 1; continue;}
id<MTLFunction> function = defaultFunction(lib, name, zeroed);
if (!function) {++refused; native_prime_functions_refused = earlierRefusals + refused; appendLine(refusedPath, attemptKey); report(path, name.UTF8String); continue;}
const MTLFunctionType type = function.functionType;
if (type != MTLFunctionTypeKernel && type != MTLFunctionTypeVertex) continue;
waited += waitForQuiet();
writeCurrent(current, key);
writeCurrent(current, doneKey);
native_prime_compiling = 1;
bool ok = false;
@try {
if (type == MTLFunctionTypeKernel) {
++kernels;
ok = [device newComputePipelineStateWithFunction:function error:&error] != nil;
// A kernel that uses a function constant's value needs one: prime the variant with every constant zero, which
// still takes the module through translation and the driver (int-245: 2901 kernels refused for want of one).
if (!ok && [error.localizedDescription containsString:@"FunctionConstant"] &&
[lib respondsToSelector:@selector(nb_newFunctionWithZeroConstants:)]) {
id<MTLFunction> zero = [(NSObject *)lib nb_newFunctionWithZeroConstants:name];
if (zero) {error = nil; ok = [device newComputePipelineStateWithFunction:zero error:&error] != nil; if (ok) ++zeroed;}
}
if (!ok && [error.localizedDescription containsString:@"LinkedFunction"]) {
ok = linkedKernel(device, lib, function, &error);
if (ok) ++linkedMade;
@@ -315,6 +398,7 @@ static std::string library(id<MTLDevice> device, const std::string &path, const
++renders;
MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new];
d.vertexFunction = function;
d.vertexDescriptor = vertexInputs(function);
d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
// The fragment function of the same stem under the libraries' usual spellings; without one the pipeline still
// rasterizes (Metal refuses an unrasterized pipeline whose vertex function returns a position, int-245: 732 refusals).
@@ -323,24 +407,30 @@ static std::string library(id<MTLDevice> device, const std::string &path, const
for (const auto &pair : kPairs) {
NSString *fragment = [name stringByReplacingOccurrencesOfString:pair[0] withString:pair[1]];
if (![fragment isEqualToString:name] && [names containsObject:fragment]) {
d.fragmentFunction = [lib newFunctionWithName:fragment];
d.fragmentFunction = defaultFunction(lib, fragment, zeroed);
if (d.fragmentFunction) break;
}
}
ok = [device newRenderPipelineStateWithDescriptor:d error:&error] != nil;
if (!ok && [error.localizedDescription containsString:@"RasterizedVoidVertex"]) {
d.rasterizationEnabled = NO;
error = nil;
ok = [device newRenderPipelineStateWithDescriptor:d error:&error] != nil;
}
// The render server's framebuffer is half-float (RGhA) on this board: its pipelines draw into RGBA16Float, a state
// of its own to Turnip, so the same pair is made for it too.
if (ok) {
d.colorAttachments[0].pixelFormat = MTLPixelFormatRGBA16Float;
NSError *half = nil;
if ([device newRenderPipelineStateWithDescriptor:d error:&half]) ++halfMade;
else {ok = false; error = half;}
}
}
} @catch (NSException *exception) {
ok = false;
}
native_prime_compiling = 0;
appendLine(functionsPath, doneKey);
appendLine(ok ? functionsPath : refusedPath, ok ? doneKey : attemptKey);
ok ? ++made : ++refused;
if (ok) native_prime_made = native_prime_made + 1;
if (!ok && firstRefusal.empty()) {
@@ -349,6 +439,8 @@ static std::string library(id<MTLDevice> device, const std::string &path, const
if (firstRefusal.size() > 220) firstRefusal.resize(220);
for (char &c : firstRefusal) if (c == '\n' || c == '\t') c = ' ';
}
native_prime_functions_refused = earlierRefusals + refused;
report(path, name.UTF8String);
error = nil;
// Low priority already; a short pause between pipelines keeps the render server's own work ahead.
usleep(20000);
@@ -356,6 +448,9 @@ static std::string library(id<MTLDevice> device, const std::string &path, const
}
}
unlink(current.c_str());
native_prime_functions_refused = earlierRefusals + refused;
native_prime_functions_skipped = native_prime_functions_skipped + skipped;
complete = refused == 0 && skipped == 0;
char line[200];
snprintf(line, sizeof line, " kernels=%u renders=%u made=%u half=%u zeroed=%u linked=%u refused=%u skipped=%u ms=%llu waited=%llu", kernels, renders,
made, halfMade, zeroed, linkedMade, refused, skipped, (unsigned long long)(nowMs() - began), (unsigned long long)waited);
@@ -372,53 +467,95 @@ static void run(id<MTLDevice> device, std::string root, unsigned delaySeconds, c
while (root.size() > 1 && root.back() == '/') root.pop_back();
const std::string donePath = root + "/prime-done", badPath = root + "/prime-bad", current = root + "/prime-current",
logPath = root + "/prime.log";
// A function was being compiled when the process ended: it is not tried again.
for (const std::string &line : readLines(current)) appendLine(badPath, line);
#if !TARGET_OS_OSX
const std::string resume = root + "/prime-resume";
const bool planned = access(resume.c_str(), F_OK) == 0;
if (planned) unlink(resume.c_str());
else for (const std::string &line : readLines(current)) appendLine(badPath, line);
#endif
unlink(current.c_str());
sleep(delaySeconds);
#if TARGET_OS_OSX
const std::set<std::string> bad;
#else
const std::set<std::string> bad = readLines(badPath);
#endif
std::set<std::string> done = readLines(donePath);
const std::string functionsPath = root + "/prime-fn";
const std::string functionsPath = root + "/prime-fn2", refusedPath = root + "/prime-refused2";
const std::set<std::string> functionsDone = readLines(functionsPath);
#if TARGET_OS_OSX
const std::set<std::string> refusedDone;
#else
const std::set<std::string> refusedDone = readLines(refusedPath);
#endif
const uint64_t began = nowMs();
unsigned libs = 0, finished = 0, foreign = 0;
std::string note;
const std::vector<std::string> paths = libraries(listPath, note);
const std::string build = buildId();
appendLine(logPath, note + " build=" + build);
native_prime_libraries_total = (uint32_t)paths.size();
const std::string build = buildId() + "-prime2";
const std::string boot = bootSession();
#if TARGET_OS_OSX
native_prime_scope = wholeSystem ? 2 : 1;
const unsigned scope = wholeSystem ? 2 : 1;
#else
native_prime_scope = 2; // iPadOS primes every library of the list, whatever its bundle
const unsigned scope = 2;
#endif
appendLine(logPath, note + " inputs=3 boot=" + boot + " scope=" + std::to_string(scope) + " build=" + build);
std::set<std::string> binaryArchives;
#if TARGET_OS_OSX
if (wholeSystem) for (const std::string &path : paths) if (appleGpuArchive(path)) {
binaryArchives.insert(path);
appendLine(logPath, "archive " + path + " cpu=0x01000013");
}
#endif
auto selected = [&](const std::string &path) {
#if TARGET_OS_OSX
return wholeSystem ? !libraryOfDriverBundle(path) && !binaryArchives.count(path) : libraryOfLoadedBundle(path);
#else
(void)path;
(void)wholeSystem;
return true;
#endif
};
unsigned eligible = 0;
for (const std::string &path : paths) if (selected(path)) ++eligible;
native_prime_libraries_total = eligible;
native_prime_scope = scope;
report();
for (const std::string &path : paths) {
if (stopping()) break;
++libs;
native_prime_libraries_done = finished;
report(path);
struct stat st = {};
if (stat(path.c_str(), &st) != 0) continue;
// The runtime's and Turnip's builds end the id: a newer one primes every library again (buildId()).
const std::string id = path + "\t" + std::to_string((long long)st.st_size) + "\t" + std::to_string((long long)st.st_mtimespec.tv_sec) + "\t" + build;
if (done.count(id)) {++finished; continue;}
#if TARGET_OS_OSX
// Counted as done for the telemetry's progress, and not listed as finished (libraryOfDriverBundle, libraryOfLoadedBundle).
if (wholeSystem ? libraryOfDriverBundle(path) : !libraryOfLoadedBundle(path)) {
++foreign; ++finished;
if (stat(path.c_str(), &st) != 0) {native_prime_libraries_failed = native_prime_libraries_failed + 1; continue;}
if (!selected(path)) {
++foreign;
native_prime_libraries_skipped = foreign;
continue;
}
#endif
appendLine(logPath, library(device, path, current, bad, functionsPath, functionsDone, build));
// The runtime's and Turnip's builds end the id: a newer one primes every library again (buildId()).
const std::string id = path + "\t" + std::to_string((long long)st.st_size) + "\t" + std::to_string((long long)st.st_mtimespec.tv_sec) + "\t" + build;
if (done.count(id)) {++finished; continue;}
bool complete = false;
appendLine(logPath, library(device, path, current, bad, functionsPath, functionsDone, refusedPath, refusedDone, build, complete));
if (stopping()) break;
native_prime_libraries_primed = native_prime_libraries_primed + 1;
appendLine(donePath, id);
done.insert(id);
++finished;
if (complete) {
appendLine(donePath, id);
done.insert(id);
++finished;
} else {
native_prime_libraries_failed = native_prime_libraries_failed + 1;
}
}
native_prime_libraries_done = finished;
native_prime_compiling = 0;
char line[120];
snprintf(line, sizeof line, "pass libraries=%u finished=%u %s=%u ms=%llu", libs, finished, wholeSystem ? "driver-bundles" : "not-loaded",
foreign, (unsigned long long)(nowMs() - began));
native_prime_pass_complete = eligible && finished == eligible && !native_prime_libraries_failed && !native_prime_inventory_failed ? 1 : 2;
report();
char line[512];
snprintf(line, sizeof line, "pass boot=%s scope=%u build=%s inputs=3 libraries=%u eligible=%u finished=%u failed=%u refused=%u bad=%u missing=%u excluded=%u binary=%zu complete=%u ms=%llu",
boot.c_str(), scope, build.c_str(), libs, eligible, finished, native_prime_libraries_failed, native_prime_functions_refused, native_prime_functions_skipped,
native_prime_inventory_failed, foreign, binaryArchives.size(), native_prime_pass_complete, (unsigned long long)(nowMs() - began));
appendLine(logPath, line);
}
} // namespace prime
+154
View File
@@ -0,0 +1,154 @@
// SPDX-License-Identifier: MIT
#import <AppKit/AppKit.h>
#import <Security/Authorization.h>
#import <Security/AuthorizationTags.h>
#include "ShaderCacheState.h"
static NSString *text(NSString *english, NSString *chinese) {
return [NSLocale.preferredLanguages.firstObject hasPrefix:@"zh"] ? chinese : english;
}
@interface ShaderCacheApp : NSObject <NSApplicationDelegate>
@property NSWindow *window;
@property NSTextField *state;
@property NSTextField *counts;
@property NSTextField *current;
@property NSProgressIndicator *progress;
@property NSButton *start;
@property NSButton *stop;
@property NSTimer *timer;
@property BOOL authorizing;
@property AuthorizationRef authorization;
@end
@implementation ShaderCacheApp
- (void)applicationDidFinishLaunching:(NSNotification *)notification {
(void)notification;
NSMenu *menu = [NSMenu new];
NSMenuItem *application = [NSMenuItem new];
[menu addItem:application];
application.submenu = [NSMenu new];
[application.submenu addItemWithTitle:text(@"Quit Shader Cache", @"退出着色器缓存") action:@selector(terminate:) keyEquivalent:@"q"];
NSApp.mainMenu = menu;
self.window = [[NSWindow alloc] initWithContentRect:NSMakeRect(0,0,420,228)
styleMask:NSWindowStyleMaskTitled|NSWindowStyleMaskClosable|NSWindowStyleMaskMiniaturizable backing:NSBackingStoreBuffered defer:NO];
self.window.title = text(@"Shader Cache", @"着色器缓存");
self.window.releasedWhenClosed = NO;
self.state = [NSTextField labelWithString:@""];
self.state.font = [NSFont systemFontOfSize:15 weight:NSFontWeightMedium];
self.progress = [NSProgressIndicator new];
self.progress.indeterminate = NO;
self.progress.minValue = 0;
self.progress.maxValue = 1;
self.counts = [NSTextField labelWithString:@""];
self.counts.font = [NSFont monospacedDigitSystemFontOfSize:13 weight:NSFontWeightRegular];
self.counts.maximumNumberOfLines = 2;
self.current = [NSTextField labelWithString:@""];
self.current.lineBreakMode = NSLineBreakByTruncatingMiddle;
self.current.maximumNumberOfLines = 1;
self.current.font = [NSFont systemFontOfSize:11];
self.current.textColor = NSColor.secondaryLabelColor;
self.start = [NSButton buttonWithTitle:text(@"Start", @"开始") target:self action:@selector(startCompilation:)];
self.start.keyEquivalent = @"\r";
self.stop = [NSButton buttonWithTitle:text(@"Stop", @"停止") target:self action:@selector(stopCompilation:)];
NSStackView *buttons = [NSStackView stackViewWithViews:@[self.start,self.stop]];
buttons.orientation = NSUserInterfaceLayoutOrientationHorizontal;
buttons.spacing = 8;
NSStackView *stack = [NSStackView stackViewWithViews:@[self.state,self.progress,self.counts,self.current,buttons]];
stack.orientation = NSUserInterfaceLayoutOrientationVertical;
stack.alignment = NSLayoutAttributeLeading;
stack.spacing = 16;
stack.translatesAutoresizingMaskIntoConstraints = NO;
[self.window.contentView addSubview:stack];
[NSLayoutConstraint activateConstraints:@[
[stack.leadingAnchor constraintEqualToAnchor:self.window.contentView.leadingAnchor constant:24],
[stack.trailingAnchor constraintEqualToAnchor:self.window.contentView.trailingAnchor constant:-24],
[stack.topAnchor constraintEqualToAnchor:self.window.contentView.topAnchor constant:24],
[self.progress.widthAnchor constraintEqualToAnchor:stack.widthAnchor],
[self.current.widthAnchor constraintEqualToAnchor:stack.widthAnchor]
]];
self.timer = [NSTimer scheduledTimerWithTimeInterval:1 target:self selector:@selector(refresh) userInfo:nil repeats:YES];
[self refresh];
[self.window center];
[self.window makeKeyAndOrderFront:nil];
[NSApp activateIgnoringOtherApps:YES];
}
- (void)refresh {
NSDictionary *value = SCReadState();
NSDictionary *labels = @{
@"idle":text(@"Not started", @"未开始"), @"running":text(@"Compiling", @"正在编译"),
@"stopping":text(@"Stopping", @"正在停止"), @"stopped":text(@"Stopped", @"已停止"),
@"interrupted":text(@"Interrupted", @"已中断"), @"complete":text(@"Complete", @"已完成"),
@"incomplete":text(@"Incomplete", @"未完成"), @"failed":text(@"Failed", @"失败"),
@"outdated":text(@"Update available", @"缓存待更新")
};
self.state.stringValue = self.authorizing ? text(@"Authorizing", @"正在授权") : labels[value[@"state"]] ?: text(@"Not started", @"未开始");
double total = [value[@"total"] doubleValue];
[NSAnimationContext runAnimationGroup:^(NSAnimationContext *context) {
context.duration = 0.2;
[[self.progress animator] setDoubleValue:total ? [value[@"libraries"] doubleValue] / total : 0];
} completionHandler:nil];
self.counts.stringValue = [NSString stringWithFormat:text(@"Libraries %@/%@ Functions %@\nFailed libraries %@ Functions %@ Missing %@", @"库 %@/%@ 函数 %@\n失败库 %@ 失败函数 %@ 缺失 %@"),
value[@"libraries"] ?: @0,value[@"total"] ?: @0,value[@"functions"] ?: @0,value[@"failedLibraries"] ?: @0,value[@"failedFunctions"] ?: @0,value[@"missing"] ?: @0];
self.current.stringValue = [value[@"error"] length] ? value[@"error"] : [value[@"function"] length] ? value[@"function"] : value[@"library"] ?: @"";
self.current.toolTip = self.current.stringValue;
self.start.enabled = ![value[@"running"] boolValue] && !self.authorizing;
self.stop.enabled = [value[@"running"] boolValue] && !self.authorizing && ![value[@"state"] isEqual:@"stopping"];
}
- (void)perform:(const char *)operation {
if (self.authorizing) return;
self.authorizing = YES;
[self refresh];
NSString *worker = [NSBundle.mainBundle.bundlePath stringByAppendingPathComponent:@"Contents/MacOS/ShaderCacheWorker"];
dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{
OSStatus status = errAuthorizationSuccess;
AuthorizationRef authorization = self.authorization;
if (!authorization) status = AuthorizationCreate(nullptr, kAuthorizationEmptyEnvironment, kAuthorizationFlagDefaults, &authorization);
if (status == errAuthorizationSuccess) {
AuthorizationItem item = {kAuthorizationRightExecute,0,nullptr,0};
AuthorizationRights rights = {1,&item};
status = AuthorizationCopyRights(authorization,&rights,kAuthorizationEmptyEnvironment,
kAuthorizationFlagInteractionAllowed|kAuthorizationFlagExtendRights|kAuthorizationFlagPreAuthorize,nullptr);
}
if (status == errAuthorizationSuccess) {
char *arguments[] = {const_cast<char *>(operation),nullptr};
FILE *channel = nullptr;
status = AuthorizationExecuteWithPrivileges(authorization,worker.fileSystemRepresentation,kAuthorizationFlagDefaults,arguments,&channel);
if (channel) fclose(channel);
}
dispatch_async(dispatch_get_main_queue(), ^{
self.authorization = authorization;
self.authorizing = NO;
[self refresh];
if (status != errAuthorizationSuccess && status != errAuthorizationCanceled) {
NSAlert *alert = [NSAlert new];
alert.messageText = text(@"Could not start the worker", @"无法启动工作进程");
alert.informativeText = [NSString stringWithFormat:@"%d",(int)status];
[alert beginSheetModalForWindow:self.window completionHandler:nil];
}
});
});
}
- (void)startCompilation:(id)sender {(void)sender; [self perform:"--run"];}
- (void)stopCompilation:(id)sender {(void)sender; [self perform:"--stop"];}
- (BOOL)applicationShouldTerminateAfterLastWindowClosed:(NSApplication *)sender {(void)sender; return YES;}
@end
int main(int argc, const char **argv) {
@autoreleasepool {
if (argc == 2 && !strcmp(argv[1], "--status")) {
NSMutableDictionary *value = SCReadState().mutableCopy;
[value removeObjectForKey:@"updated"];
NSData *json = [NSJSONSerialization dataWithJSONObject:value options:NSJSONWritingPrettyPrinted error:nil];
fwrite(json.bytes,1,json.length,stdout);
putchar('\n');
return 0;
}
NSApplication *application = NSApplication.sharedApplication;
ShaderCacheApp *delegate = [ShaderCacheApp new];
application.delegate = delegate;
[application setActivationPolicy:NSApplicationActivationPolicyRegular];
[application run];
}
return 0;
}
+40
View File
@@ -0,0 +1,40 @@
// SPDX-License-Identifier: MIT
#pragma once
#import <Foundation/Foundation.h>
#include <fcntl.h>
#include <sys/file.h>
#include <unistd.h>
static NSString *const SCStorage = @"/private/var/tmp/NativeAdrenoMetal.cache";
static NSString *const SCPlugin = @"/Library/GPUBundles/NativeAdrenoMetal.bundle";
static NSString *const SCApplication = @"/Library/UniXNU/Shader Cache.app";
static NSString *SCPath(NSString *name) {return [SCStorage stringByAppendingPathComponent:name];}
static NSString *SCInstalledBuild() {
NSData *data = [NSData dataWithContentsOfFile:[SCPlugin stringByAppendingPathComponent:@"Contents/Resources/gpu-build.json"]];
if (!data) return @"";
NSDictionary *build = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil];
return [build[@"metal_source_sha256"] isKindOfClass:NSString.class] ? build[@"metal_source_sha256"] : @"";
}
static BOOL SCWorkerRunning() {
int fd = open(SCPath(@"shader-cache.lock").fileSystemRepresentation, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
if (fd < 0) return NO;
const BOOL locked = flock(fd, LOCK_SH | LOCK_NB) != 0 && errno == EWOULDBLOCK;
close(fd);
return locked;
}
static NSDictionary *SCReadState() {
NSDictionary *saved = [NSDictionary dictionaryWithContentsOfFile:SCPath(@"shader-cache.plist")];
NSMutableDictionary *state = saved ? saved.mutableCopy : [NSMutableDictionary dictionary];
const BOOL running = SCWorkerRunning();
state[@"running"] = @(running);
if (running) {
state[@"state"] = access(SCPath(@"shader-cache.stop").fileSystemRepresentation, F_OK) == 0 ? @"stopping" : @"running";
} else if ([state[@"state"] isEqual:@"running"] || [state[@"state"] isEqual:@"stopping"]) {
state[@"state"] = @"interrupted";
} else if (!state[@"state"]) state[@"state"] = @"idle";
if (!running && saved && ![saved[@"build"] isEqual:SCInstalledBuild()]) state[@"state"] = @"outdated";
return state;
}
+71
View File
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: MIT
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include "ShaderCacheState.h"
#include "../SharedShaderCache.h"
#include "../PrimeShaders.h"
#include <signal.h>
#include <sys/resource.h>
static void *pluginHandle;
static volatile sig_atomic_t interrupted;
static void interruptWorker(int) {interrupted = 1;}
extern "C" const char *nmb_runtime_build() {
auto build = reinterpret_cast<const char *(*)()>(dlsym(pluginHandle, "nmb_runtime_build"));
return build ? build() : "unknown";
}
extern "C" void native_bridge_metal_quiet_problems(bool quiet) {
auto set = reinterpret_cast<void (*)(bool)>(dlsym(pluginHandle, "native_bridge_metal_quiet_problems"));
if (set) set(quiet);
}
static void saveState(NSString *state, const std::string &library = {}, const std::string &function = {}, NSString *error = @"") {
NSDictionary *value = @{
@"state":state, @"build":SCInstalledBuild(), @"updated":NSDate.date,
@"libraries":@(native_prime_libraries_done), @"total":@(native_prime_libraries_total),
@"functions":@(native_prime_made + native_prime_functions_resumed),
@"failedLibraries":@(native_prime_libraries_failed), @"failedFunctions":@(native_prime_functions_refused),
@"missing":@(native_prime_inventory_failed), @"library":@(library.c_str()), @"function":@(function.c_str()), @"error":error
};
[value writeToFile:SCPath(@"shader-cache.plist") atomically:YES];
}
int main(int argc, const char **argv) {
@autoreleasepool {
if (argc != 2 || (strcmp(argv[1], "--run") && strcmp(argv[1], "--stop"))) return 64;
if (geteuid() != 0 || setuid(0) != 0 || systemcache::writerDirectory().empty()) return 77;
umask(0022);
if (!strcmp(argv[1], "--stop")) {
if (!SCWorkerRunning()) return 0;
return [NSData.data writeToFile:SCPath(@"shader-cache.stop") atomically:YES] ? 0 : 74;
}
int lock = open(SCPath(@"shader-cache.lock").fileSystemRepresentation, O_CREAT | O_RDWR | O_NOFOLLOW | O_CLOEXEC, 0644);
if (lock < 0 || flock(lock, LOCK_EX | LOCK_NB)) {if (lock >= 0) close(lock); return 75;}
freopen("/dev/null", "w", stdout);
freopen(SCPath(@"shader-cache-worker.log").fileSystemRepresentation, "a", stderr);
unlink(SCPath(@"shader-cache.stop").fileSystemRepresentation);
signal(SIGTERM, interruptWorker);
signal(SIGINT, interruptWorker);
saveState(@"running");
setpriority(PRIO_PROCESS, 0, 10);
setenv("NATIVE_METAL_PRIMER", "1", 1);
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
NSString *executable = [SCPlugin stringByAppendingPathComponent:@"Contents/MacOS/NativeAdrenoMetal"];
pluginHandle = dlopen(executable.fileSystemRepresentation, RTLD_NOW | RTLD_LOCAL);
if (!device || !pluginHandle || !dlsym(pluginHandle, "nmb_runtime_build") || ![NSStringFromClass([device class]) isEqual:@"NativeBridgeMetalDevice"]) {
saveState(@"failed", {}, {}, @"AdrenoMetal is unavailable");
return 69;
}
prime::cancelled = [] {return interrupted || access(SCPath(@"shader-cache.stop").fileSystemRepresentation, F_OK) == 0;};
prime::progress = [](const std::string &library, const std::string &function) {
saveState(prime::stopping() ? @"stopping" : @"running", library, function);
};
prime::run(device, SCStorage.fileSystemRepresentation, 0,
"/Library/GPUBundles/NativeAdrenoMetal.primelist", true);
saveState(prime::stopping() ? @"stopped" : native_prime_pass_complete == 1 ? @"complete" : @"incomplete");
unlink(SCPath(@"shader-cache.stop").fileSystemRepresentation);
close(lock);
return native_prime_pass_complete == 1 ? 0 : prime::stopping() ? 130 : 1;
}
}
+2 -1
View File
@@ -238,7 +238,8 @@ pub fn air_slice(bytes: &[u8]) -> Option<&[u8]> {
return Some(bytes);
}
let count = word(4)? as usize;
for i in 0..count.min(16) {
if count > (bytes.len() - 8) / 20 { return None; }
for i in 0..count {
let at = 8 + 20 * i;
if word(at)? == AIR64 {
let (offset, size) = (word(at + 8)? as usize, word(at + 12)? as usize);
+8 -3
View File
@@ -41,9 +41,9 @@ size_t nmb_device_name(const NmbDevice *, char *out, size_t size);
uint64_t nmb_device_max_buffer_length(const NmbDevice *);
// Pipeline makings (count, total us, longest us, waited us, failed), sink messages (info, warning+), shader cache hits/misses/writes.
bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]);
/* The shader cache's full counts in 24 words: hits, misses, function hits and misses, compute pipeline hits and misses, render pipeline hits and misses,
/* The shader cache's full counts in 24 words: hits, misses, function hits and misses, driver data found and unchanged, render pipeline hits and misses,
* rejected, read failures, writes, writes queued, writes on a calling thread, bytes written, queued hits, write failures, evictions, driver cache loads,
* driver pipeline hits, driver pipeline misses, driver saves, then three spare zeros. */
* driver pipeline hits, driver pipeline misses, driver saves, truncated driver data, mismatched driver data, then one spare zero. */
bool nmb_device_shader_cache_detail(const NmbDevice *, uint64_t out[24]);
/* The read-only system cache layer of the device opened after this call: the directory at root, read only where it, every directory
* on the way to an entry and the entry are owned by one of the count user ids and not writable by group or others, never through a
@@ -67,13 +67,16 @@ bool nmb_device_texture_counts(const NmbDevice *, uint64_t out[8]);
// Records warmed, failed and unread, driver bytes the warm pass added; scheduler passes and batches submitted on the committing thread;
// render log waits, and encoder ends the lowering thread lowered.
bool nmb_device_flow_counts(const NmbDevice *, uint64_t out[8]);
// Grouped submissions/batches, command pools made/reused, linear passes in place/through copies, region pipelines evicted;
// commit calls and microseconds in total/lowering/recording/pipeline wait/lock wait/scheduler pass/submission.
bool nmb_device_fast_path_counts(const NmbDevice *, uint64_t out[15]);
// Passes begun as instances, split inline; instance splits for a barrier, tile memory, a memoryless write, a pending pipeline; stand-ins
// made and their bytes; stand-in copies made and those made at draws.
bool nmb_device_pass_counts(const NmbDevice *, uint64_t out[10]);
// Bytes of pooled blocks live and placed in them, empty kept, dedicated, host imports; bytes allocated so far; pooled and dedicated
// blocks live; GPU pool blocks made and released.
bool nmb_device_memory_counts(const NmbDevice *, uint64_t out[10]);
// Live blocks of the runtime's own device memory (bridge patch 0031): seven size classes each (at most 16, 32, 40, 48, 56, 64 MiB,
// Live blocks of the runtime's own device memory: seven size classes each (at most 16, 32, 40, 48, 56, 64 MiB,
// larger) of pooled blocks with a placement [0], pooled blocks kept empty [7], spare image blocks [14], dedicated blocks of an image
// [21] and of a buffer [28]; bytes of the spare blocks [35] and of the dedicated blocks of images [36] and buffers [37]; lazily
// allocated blocks with a placement [38] and kept without one [42], each as blocks, bytes, blocks the driver committed, bytes it
@@ -200,6 +203,8 @@ NmbTexture *nmb_texture_new_view_swizzle(NmbTexture *, uint64_t format, uint64_t
uint32_t nmb_function_constant_count(const NmbFunction *);
size_t nmb_function_constant_name_size(const NmbFunction *, uint32_t at);
bool nmb_function_constant_at(const NmbFunction *, uint32_t at, char *name, size_t capacity, uint64_t *index, uint64_t *data_type, bool *required);
int32_t nmb_function_attribute_count(const NmbFunction *);
size_t nmb_function_attribute_at(const NmbFunction *, uint32_t at, char *name, size_t capacity, uint64_t facts[5]);
bool nmb_render_set_viewports(NmbEncoder *, const double *values, size_t count);
bool nmb_render_set_scissors(NmbEncoder *, const uint64_t *values, size_t count);
void nmb_texture_release(NmbTexture *);
+120 -15
View File
@@ -239,13 +239,14 @@ impl PipelineRecords {
}
}
/// The work placement the bridge asks of the runtime, as bits: 1 render encoder calls lowered on the device's lowering thread,
/// 2 a command buffer submitted at its commit (`commit_and_submit`), 4 the warm pass over the pipeline records of earlier
/// processes. All three by default; `NATIVE_METAL_FLOW=<number>` (the boot argument `metal-flow=<number>`) states the bits, 0 none.
/// Work placement: bit 1 lowers on the device's worker, bit 2 submits at commit.
/// iPadOS also supports bit 4 for warming earlier pipeline records. Defaults are
/// 3 on macOS and 7 on iPadOS; `NATIVE_METAL_FLOW` selects the bits.
fn flow_bits() -> u32 {
static BITS: std::sync::OnceLock<u32> = std::sync::OnceLock::new();
*BITS.get_or_init(|| {
std::env::var("NATIVE_METAL_FLOW").ok().and_then(|value| value.trim().parse::<u32>().ok()).unwrap_or(7)
let bits = std::env::var("NATIVE_METAL_FLOW").ok().and_then(|value| value.trim().parse::<u32>().ok()).unwrap_or(7);
if cfg!(target_os = "macos") { bits & 3 } else { bits }
})
}
@@ -263,13 +264,8 @@ pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_
options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned()));
options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned());
options.messages = Some(MessageSink::new(observe_message));
// A texture over host pages or a buffer whose rows lie apart is an image over the bytes, rendered or written too: the window
// server's surfaces and padded framebuffers (3048x2032 BGRA8 at 12288-byte rows, int-251) were otherwise copied in and out of a
// shadow image every pass.
// Render imported surfaces and padded buffers directly under the board policy.
options.policy.host_page_textures = metal2vulkan::device::HostPageTexturePolicy::InPlaceWrites;
// Metal makes a pipeline state's pipeline at its creation and never drops a draw for a pipeline being made; the runtime's
// default (PipelineWaitPolicy::Bounded) drops the draws and dispatches of a pipeline not made 4.5 s after the commit, which the
// render server showed as missing pieces of the screen while it compiled (int 590aaee2).
options.policy.pipeline_waits = metal2vulkan::device::PipelineWaitPolicy::Exact;
// Render encoder calls are checked at the call and lowered on the device's lowering thread, so the program's thread does not
// pay for them (the render server's main thread spent most of the plugin's CPU time lowering). A refusal only the lowering
@@ -284,7 +280,9 @@ pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_
// Every texture given a shadow image is announced with why no image aliases its bytes (`shadow.log`).
options.announce_each.insert(metal2vulkan_runtime::Degradation::ShadowedTexture);
let _ = SHADOW_LOG.set(PathBuf::from(cache.to_string_lossy().into_owned()).join("shadow.log"));
open_runtime_diagnostics(cache.to_string_lossy().as_ref());
let device = Device::system_default_with(&options).map_err(e("device"))?;
write_runtime_policy(&device);
// No caller moves or detaches the pages of a no-copy buffer (the frontend has no rebind or detach): the encoders log no calls
// to encode again and hold no surface's buffer per command, which every frame of the render server paid for.
device.refuse_rebinds();
@@ -336,6 +334,53 @@ fn note_shadow(text: &str) {
// NATIVE_METAL_WARNINGS=1: each kind of message (its id and the text's first 48 characters) the first time it comes, in full and
// with its severity, to standard error, for the first 400 kinds. The telemetry keeps only the last warning, and a degradation that
// draws nothing is otherwise silent.
static COMMIT_TIMINGS: [std::sync::atomic::AtomicU64; 2] = [const { std::sync::atomic::AtomicU64::new(0) }; 2];
const TIMING_LOG_LIMIT: usize = 512 * 1024;
struct TimingLog { file: std::fs::File, bytes: usize }
static TIMING_LOG: std::sync::OnceLock<std::sync::Mutex<TimingLog>> = std::sync::OnceLock::new();
static POLICY_LOG: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
fn open_runtime_diagnostics(cache: &str) {
if std::env::var_os("SHENG_DIAG").is_none() { return; }
let root = PathBuf::from(cache);
let _ = POLICY_LOG.set(root.join("runtime-policy.log"));
if let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(root.join("timing.log")) {
let bytes = file.metadata().map_or(0, |m| m.len() as usize);
let _ = TIMING_LOG.set(std::sync::Mutex::new(TimingLog { file, bytes }));
}
}
fn diagnostic_stamp() -> String {
let boot = std::env::var("NATIVE_METAL_BOOT_SESSION").unwrap_or_else(|_| "unknown".into());
let build = option_env!("NMB_COMPONENT_ID").unwrap_or("host-working-tree");
let uptime = metal2vulkan_runtime::stage_trace::now_nanos() / 1000;
let time = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_micros());
format!("boot={boot} build={build} runtime={} uptime_us={uptime} time_us={time}", env!("NMB_RUNTIME_ID"))
}
fn write_runtime_policy(device: &Device) {
let Some(path) = POLICY_LOG.get() else { return; };
let profile = device.compile_device();
let text = format!("{}\npolicy={:#?}\nsubmit={}\nqueue-switch={}\nin-place={:#?}\nprofile={:#?}\n",
diagnostic_stamp(), profile.policy(), device.submit_check_report(),
device.queue_switch_check_report(), device.in_place_check_report(), profile.profile());
let text: String = text.chars().take(64 * 1024).collect();
let _ = std::fs::write(path, text);
}
fn note_encoder_timing(text: &str) {
use std::io::Write;
let Some(log) = TIMING_LOG.get() else { return; };
let Ok(mut log) = log.lock() else { return; };
let text: String = text.chars().take(4096).collect();
let line = format!("{} {text}\n", diagnostic_stamp());
if log.bytes + line.len() > TIMING_LOG_LIMIT {
if log.file.set_len(0).is_err() { return; }
log.bytes = 0;
}
if log.file.write_all(line.as_bytes()).is_ok() { log.bytes += line.len(); }
}
fn log_message(message: &DebugMessage) {
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
static SEEN: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
@@ -353,6 +398,7 @@ fn log_message(message: &DebugMessage) {
fn observe_message(message: &DebugMessage) {
use std::sync::atomic::Ordering::Relaxed;
if message.id_name == "metal2vulkan-encoder-times" { note_encoder_timing(&message.text); }
if message.id_name == "shadowed-texture" {
note_shadow(&message.text);
}
@@ -426,8 +472,8 @@ pub extern "C" fn nmb_device_shader_cache_detail(device: *const NmbDevice, out:
c.misses,
c.function_hits,
c.function_misses,
c.compute_pipeline_hits,
c.compute_pipeline_misses,
c.driver_data_found,
c.driver_data_unchanged,
c.render_pipeline_hits,
c.render_pipeline_misses,
c.rejected,
@@ -443,6 +489,8 @@ pub extern "C" fn nmb_device_shader_cache_detail(device: *const NmbDevice, out:
c.driver_pipeline_hits,
c.driver_pipeline_misses,
c.driver_saves,
c.driver_data_truncated,
c.driver_data_mismatched,
];
out.fill(0);
out[..words.len()].copy_from_slice(&words);
@@ -487,6 +535,31 @@ pub extern "C" fn nmb_device_flow_counts(device: *const NmbDevice, out: *mut u64
})
}
/// Submission grouping, command-pool reuse, in-place passes and commit costs, into the header's fifteen words.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_fast_path_counts(device: *const NmbDevice, out: *mut u64) -> bool {
guard(false, || {
if out.is_null() {
return Err("null counts".into());
}
// SAFETY: the header's fifteen writable words.
let out = unsafe { std::slice::from_raw_parts_mut(out, 15) };
let device = &get(device)?.0;
static SNAPSHOTS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
if POLICY_LOG.get().is_some() && SNAPSHOTS.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % 32 == 0 {
write_runtime_policy(device);
}
let w = device.work_counts();
out.fill(u64::MAX);
out[..6].copy_from_slice(&[w.grouped_submissions, w.batches_grouped, w.command_pools_made,
w.spare_command_pools_taken, w.linear_passes_in_place, w.linear_passes_through_copies]);
out[6] = 0;
out[7] = COMMIT_TIMINGS[0].load(std::sync::atomic::Ordering::Relaxed);
out[8] = COMMIT_TIMINGS[1].load(std::sync::atomic::Ordering::Relaxed) / 1000;
Ok(true)
})
}
/// Where render passes cost the GPU more than Metal's single pass in tile memory, into the caller's ten words: passes begun as
/// instances, passes split inline, then instance splits for a barrier, tile memory, a memoryless write and a pending pipeline, stand-ins
/// made and their bytes, and stand-in copies made and those made at draws.
@@ -525,7 +598,7 @@ pub extern "C" fn nmb_device_memory_counts(device: *const NmbDevice, out: *mut u
})
}
/// The runtime's live blocks of its own device memory by what holds them and by size (bridge patch 0031), into the caller's 48 words:
/// The runtime's live blocks of its own device memory by what holds them and by size, into the caller's 48 words:
/// seven size classes each (at most 16, 32, 40, 48, 56 and 64 MiB, and larger) of the pooled blocks with a placement, the pooled
/// blocks kept empty, the spare image blocks, the dedicated blocks of an image and those of a buffer; then the bytes of the spare
/// blocks and of the dedicated blocks of images and of buffers; then, of the blocks of a lazily allocated memory type, those with a
@@ -1917,6 +1990,28 @@ pub extern "C" fn nmb_function_type(function: *const NmbFunction) -> u64 {
guard(0, || Ok(get(function)?.0.function_type().map_err(e("function type"))?.as_raw() as u64))
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_function_attribute_count(function: *const NmbFunction) -> i32 {
guard(-1, || Ok(get(function)?.0.stage_input_attributes().map_err(e("function attributes"))?.map_or(0, |a| a.len() as i32)))
}
/// # Safety
/// `name` has `capacity` writable bytes and `facts` holds five writable words.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_function_attribute_at(function: *const NmbFunction, at: u32, name: *mut c_char,
capacity: usize, facts: *mut u64) -> usize {
guard(0, || {
let attributes = get(function)?.0.stage_input_attributes().map_err(e("function attributes"))?.unwrap_or_default();
let a = attributes.get(at as usize).ok_or("function attribute index")?;
if facts.is_null() { return Err("null attribute facts".into()); }
let values = [a.attribute_index() as u64, a.attribute_type().as_raw() as u64, a.is_active() as u64,
a.is_patch_data() as u64, a.is_patch_control_point_data() as u64];
// SAFETY: the caller supplies five writable words.
unsafe { std::ptr::copy_nonoverlapping(values.as_ptr(), facts, values.len()); }
Ok(copy_out(a.name(), name, capacity))
})
}
/// `patchType`: 0 (none) for a function other than a post-tessellation vertex function.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_function_patch_type(function: *const NmbFunction) -> u64 {
@@ -2735,7 +2830,8 @@ pub extern "C" fn nmb_command_buffer_wait_scheduled(command_buffer: *mut NmbComm
#[unsafe(no_mangle)]
pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffer) -> bool {
guard(false, || {
let began = TIMING_LOG.get().map(|_| std::time::Instant::now());
let result = guard(false, || {
// The committing thread submits where no other pass is under way, with no wake of the scheduler thread between commit and GPU.
let command_buffer = &get(command_buffer)?.0;
if flow_bits() & 2 != 0 {
@@ -2744,7 +2840,16 @@ pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffe
command_buffer.commit().map_err(e("commit"))?;
}
Ok(true)
})
});
if let Some(began) = began {
let nanos = began.elapsed().as_nanos().min(u64::MAX as u128) as u64;
COMMIT_TIMINGS[0].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
COMMIT_TIMINGS[1].fetch_add(nanos, std::sync::atomic::Ordering::Relaxed);
if nanos >= 8_000_000 {
note_encoder_timing(&format!("command_buffer={command_buffer:p} commit_us={} accepted={result}", nanos / 1000));
}
}
result
}
#[unsafe(no_mangle)]
+5
View File
@@ -43,6 +43,11 @@ int nmb_pac_atexit(void *function)
{
return atexit((void (*)(void))NMB_SIGN(function));
}
extern int __cxa_atexit(void (*function)(void *), void *argument, void *shared_object);
int nmb_pac_cxa_atexit(void *function, void *argument, void *shared_object)
{
return __cxa_atexit((void (*)(void *))NMB_SIGN(function), argument, shared_object);
}
void nmb_pac_dyld_register_func_for_add_image(void *function)
{
_dyld_register_func_for_add_image((void (*)(const struct mach_header *, intptr_t))NMB_SIGN(function));
+1 -1
View File
@@ -65,7 +65,7 @@ int main(int argc,char **argv) {@autoreleasepool {
CHECK(placed && placed.heap==storage && placed.allowGPUOptimizedContents==optimized);
CHECK(placed.allocatedSize<=storage.size);
auto alias=[placed newTextureViewWithPixelFormat:placed.pixelFormat];CHECK(alias && alias.allowGPUOptimizedContents==optimized);
auto overflow=[storage newTextureWithDescriptor:layout offset:footprint.align];CHECK(!overflow);
auto overflow=[storage newTextureWithDescriptor:layout offset:storage.size];CHECK(!overflow);
}
// Zero is a real swizzle, not a marker for the identity.
td.swizzle=MTLTextureSwizzleChannelsMake(MTLTextureSwizzleZero,MTLTextureSwizzleZero,MTLTextureSwizzleZero,MTLTextureSwizzleZero);
File diff suppressed because one or more lines are too long
+186
View File
@@ -0,0 +1,186 @@
#!/usr/bin/env python3
"""Report runtime policy fields, float controls and Turnip options from their owning sources.
Use --runtime-policy with a captured runtime-policy.log to attach the device's actual
profile and check verdicts. Source references describe gates; they do not establish
that the device executed a path. Math operations use upstream Math::float_stems(),
BinOp::ALL and PrecisionNote's existing classification and validation.
"""
import argparse
import ast
import base64
import hashlib
import json
from pathlib import Path
import re
import subprocess
import urllib.request
COMPONENT = Path(__file__).resolve().parents[1]
CORE = COMPONENT.parents[1]
MESA_FILES = ('src/freedreno/vulkan/tu_drirc_gen.py',
'src/freedreno/vulkan/tu_device.cc',
'src/freedreno/vulkan/tu_cmd_buffer.cc',
'src/freedreno/vulkan/tu_sampler.cc',
'src/freedreno/vulkan/tu_lrz.cc',
'src/freedreno/ir3/ir3_compiler.c',
'src/freedreno/ir3/ir3_compiler_nir.c')
def mask(text):
return re.sub(r'//[^\n]*|/\*[\s\S]*?\*/|"(?:\\.|[^"\\])*"',
lambda m: ''.join('\n' if c == '\n' else ' ' for c in m[0]), text)
def body(text, pattern):
match = re.search(pattern, mask(text))
if not match:
raise ValueError('Source declaration missing: ' + pattern)
clean = mask(text)
at = clean.index('{', match.end())
depth = 1
end = at + 1
while depth:
if clean[end] == '{':
depth += 1
elif clean[end] == '}':
depth -= 1
end += 1
return text[at + 1:end - 1], at + 1
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--translator', type=Path, default=CORE / '.cache/gpu-reference/metal2vulkan')
parser.add_argument('--mesa-source', type=Path, default=CORE / '.cache/mesa-work')
parser.add_argument('--fetch-mesa', action='store_true', help='Cache only the pinned Mesa source texts this report reads')
parser.add_argument('--runtime-policy', type=Path)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
mesa_pin = json.loads((CORE / 'scripts/data/adreno-compiler-inputs.json').read_text())['mesa']
mesa = args.mesa_source
if args.fetch_mesa:
mesa = CORE / '.cache/runtime-fast-path-audit' / mesa_pin['commit']
for name in MESA_FILES:
path = mesa / name
if path.is_file():
continue
url = mesa_pin['mirror'].rstrip('/') + '/+/' + mesa_pin['commit'] + '/' + name + '?format=TEXT'
with urllib.request.urlopen(url, timeout=45) as response:
data = base64.b64decode(response.read(), validate=True)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
files = {}
def read(root, name):
path = root / name
text = path.read_text()
files[str(path)] = hashlib.sha256(text.encode()).hexdigest()
return text
policy_path = 'crates/metal2vulkan/src/device/policy.rs'
policy = read(args.translator, policy_path)
struct, _ = body(policy, r'pub struct DevicePolicy\b')
fields = re.findall(r'^ pub (\w+): ([^,]+),', struct, re.M)
defaults, _ = body(policy, r'pub const UNTRUSTED: DevicePolicy = DevicePolicy')
defaults = dict(re.findall(r'^ (\w+): (.+),$', defaults, re.M))
consumer = read(COMPONENT, 'bridge/NativeMetalObjects.rs')
overrides = dict(re.findall(r'options\.policy\.(\w+) = ([^;]+);', consumer))
references = {name: [] for name, _ in fields}
match_policy = re.compile(r'\.\s*(' + '|'.join(references) + r')\b')
for folder in ('crates/metal2vulkan/src', 'crates/metal2vulkan-runtime/src'):
for path in (args.translator / folder).rglob('*.rs'):
if '/tests/' in str(path):
continue
text = path.read_text()
for line, value in enumerate(text.splitlines(), 1):
for match in match_policy.finditer(value):
references[match[1]].append(dict(file=str(path.relative_to(args.translator)), line=line, expression=value.strip()))
profile_text = read(args.translator, 'crates/metal2vulkan/src/device/profile.rs')
ledger = read(args.translator, 'docs/FAST-PATHS.md').splitlines()
rows = []
for name, kind in fields:
pattern = r'pub enum ' + re.escape(kind) + r'\b'
owner = next((source for source in (policy, profile_text) if re.search(pattern, mask(source))), None)
declaration = body(owner, pattern)[0] if owner else ''
rows.append(dict(field=name, type=kind, upstream_default=defaults.get(name),
consumer_override=overrides.get(name), source=policy_path,
values=re.findall(r'^ ([A-Z]\w*)\s*(?:,|\{|\()', mask(declaration), re.M),
references=references[name],
accuracy_ledger=[dict(file='docs/FAST-PATHS.md', line=n, expression=line)
for n, line in enumerate(ledger, 1)
if re.search(r'\b' + name + r'\s*=', line)]))
vocabulary = {}
for name, path in (
('FloatWidth', 'crates/metal2vulkan/src/device/requirements.rs'),
('FloatControl', 'crates/metal2vulkan/src/device/requirements.rs'),
('FloatFunctions', 'crates/metal2vulkan/src/device/profile.rs'),
('FloatWidthControls', 'crates/metal2vulkan/src/device/profile.rs'),
('FloatControls', 'crates/metal2vulkan/src/device/profile.rs'),
('ProfileFields', 'crates/metal2vulkan/src/device/profile.rs'),
('LeafCaps', 'crates/metal2vulkan/src/device/profile.rs'),
('PrecisionNote', 'crates/metal2vulkan/src/pipeline/artifact.rs'),
):
text = read(args.translator, path)
contents, _ = body(text, r'pub (?:enum|struct) ' + name + r'\b')
members = re.findall(r'^ pub (\w+):', contents, re.M)
if not members:
members = re.findall(r'^ ([A-Z]\w*)\s*(?:,|\{|\()', mask(contents), re.M)
vocabulary[name] = dict(source=path, count=len(members), members=members)
gates = []
for path, names in (
('crates/metal2vulkan/src/lower/arith/narrow.rs', ('relaxes', 'quantize_form', 'half_controls', 'controlled_modes')),
('crates/metal2vulkan/src/lower/arith/fma.rs', ('float_route', 'half_route')),
('crates/metal2vulkan/src/lower/arith/mod.rs', ('contraction', 'fp_env')),
('crates/metal2vulkan-runtime/src/batch_order.rs', ('ends_order',)),
):
text = read(args.translator, path)
for name in names:
contents, offset = body(text, r'\bfn ' + name + r'\b')
gates.append(dict(file=path, symbol=name, line=text.count('\n', 0, offset) + 1, expression=contents.strip()))
mesa_text = {}
for name in MESA_FILES:
mesa_text[name] = read(mesa, name)
registry_path = 'src/freedreno/vulkan/tu_drirc_gen.py'
registry = mesa_text[registry_path]
options = []
for node in ast.walk(ast.parse(registry)):
if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.List):
continue
group = next((target.id for target in node.targets if isinstance(target, ast.Name) and target.id.endswith('_options')), None)
if not group:
continue
for call in node.value.elts:
if not isinstance(call, ast.Call) or not call.args or not isinstance(call.args[0], ast.Constant):
continue
options.append(dict(group=group, name=call.args[0].value,
default=ast.get_source_segment(registry, call.args[1]) if len(call.args) > 1 else None,
source=registry_path, line=call.lineno,
declaration=ast.get_source_segment(registry, call)))
property_lines = []
selectors = re.compile(r'shader(?:Float16|FloatControls2|Denorm\w+|Rounding\w+|SignedZero\w+)|driverID|vendorID|deviceID|lower_fdiv|float_mul_add\d+')
for name, text in mesa_text.items():
for line, value in enumerate(text.splitlines(), 1):
if selectors.search(value):
property_lines.append(dict(file=name, line=line, expression=value.strip()))
plugin = read(COMPONENT, 'NativeAdrenoMetalPlugin.mm')
result = dict(schema='unixnu-runtime-fast-path-audit-1',
translator=subprocess.check_output(['git', '-C', str(args.translator), 'rev-parse', 'HEAD'], text=True).strip(),
mesa=mesa_pin['commit'], policy_field_count=len(rows), policies=rows,
float_vocabulary=vocabulary, owning_gates=gates,
mesa_option_count=len(options), mesa_options=options, mesa_property_evidence=property_lines,
consumer_turnip_configuration=[line.strip() for line in plugin.splitlines() if 'setenv("tu_' in line],
math_enumeration='Upstream Math::float_stems(), BinOp::ALL, PrecisionNote::of_math_call/of_float_operator, engine_float_call_notes and fast_paths; no separate operation list.',
evidence='Finite source declarations and gate expressions. Runtime verdicts and active paths come from runtime-policy.log and timing/FP counters.',
source_files=files,
unresolved_policy_defaults=[row['field'] for row in rows if row['upstream_default'] is None])
if args.runtime_policy:
text = args.runtime_policy.read_text()
result['runtime_capture'] = dict(file=str(args.runtime_policy), stamp=text.splitlines()[0], contents=text)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2) + '\n')
print('policy-fields=%d Turnip-options=%d unresolved-policy-defaults=%d -> %s' % (len(rows), len(options), len(result['unresolved_policy_defaults']), args.output))
if __name__ == '__main__':
main()
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Build the macOS Shader Cache app and its administrator-authorized worker."""
import argparse
import json
import plistlib
from pathlib import Path
import re
import shutil
import subprocess
SOURCE = Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('output', type=Path)
parser.add_argument('--arch', choices=('arm64', 'arm64e'), default='arm64e')
args = parser.parse_args()
app = args.output.resolve() / 'Shader Cache.app'
if app.exists():
shutil.rmtree(app)
code = app / 'Contents/MacOS'
code.mkdir(parents=True)
sdk = subprocess.check_output(['xcrun', '--sdk', 'macosx', '--show-sdk-path'], text=True).strip()
base = ['xcrun', 'clang++', '-target', args.arch + '-apple-macos27.0', '-isysroot', sdk,
'-std=c++17', '-fobjc-arc', '-O2', '-Wall', '-Wextra', '-Werror', '-Wno-deprecated-declarations', '-Wno-unused-function']
hashes = {}
for name, frameworks in [('ShaderCache', ('AppKit', 'Security')), ('ShaderCacheWorker', ('Foundation', 'Metal'))]:
source = 'ShaderCacheApp.mm' if name == 'ShaderCache' else name + '.mm'
subprocess.run([*base, SOURCE / 'ShaderCache' / source,
*[part for framework in frameworks for part in ('-framework', framework)], '-o', code / name], check=True)
entitlements = args.output / (name + '-entitlements.plist')
grants = {'com.apple.security.iokit-user-client-class': ['AGXDeviceUserClient'],
'com.apple.private.security.no-container': True} if name == 'ShaderCacheWorker' and args.arch == 'arm64e' else {}
entitlements.write_bytes(plistlib.dumps(grants))
subprocess.run(['codesign', '--force', '--sign', '-', '--timestamp=none', '--pagesize', '4096',
'--entitlements', entitlements, code / name], check=True)
detail = subprocess.run(['codesign', '-d', '--verbose=4', code / name], capture_output=True, text=True, check=True).stderr
hashes[name] = re.search(r'^CDHash=([0-9a-f]{40})$', detail, re.M)[1]
entitlements.unlink()
(app / 'Contents/Info.plist').write_bytes(plistlib.dumps(dict(
CFBundleIdentifier='org.unixnu.ShaderCache', CFBundleName='Shader Cache', CFBundleDisplayName='Shader Cache',
CFBundleExecutable='ShaderCache', CFBundlePackageType='APPL', CFBundleVersion='1', CFBundleShortVersionString='1.0',
LSMinimumSystemVersion='27.0', NSHighResolutionCapable=True, NSPrincipalClass='NSApplication')))
# The outer seal retains the executable's existing entitlements.
subprocess.run(['codesign', '--force', '--sign', '-', '--timestamp=none', '--pagesize', '4096',
'--preserve-metadata=entitlements', app], check=True)
for path in (app, code / 'ShaderCacheWorker'):
subprocess.run(['codesign', '--verify', '--strict', path], check=True)
detail = subprocess.run(['codesign', '-d', '--verbose=4', app], capture_output=True, text=True, check=True).stderr
hashes['ShaderCache'] = re.search(r'^CDHash=([0-9a-f]{40})$', detail, re.M)[1]
(args.output / 'shader-cache-trust.json').write_text(json.dumps(hashes, indent=2) + '\n')
print(app)
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Reassemble NativeAdrenoMetal command-buffer messages from compact system logs."""
import argparse
import base64
import json
from pathlib import Path
import re
import sys
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('logs', type=Path, nargs='+', help='log files from one boot')
parser.add_argument('--functions', type=Path, help='write captured function AIR and its names to this directory')
args = parser.parse_args()
pattern = re.compile(r'(\S+)\[(\d+):[^]]+\].*NativeAdrenoMetal CB dump '
r'(\d+) (\d+)/(\d+): (.*)')
entries = {}
raw_sequence = 0
for path in args.logs:
for line in path.read_text(errors='replace').splitlines():
if line.startswith('native-metal dump '):
raw_sequence += 1
entries['stderr', 0, raw_sequence] = (1, {1: line})
continue
if 'NativeAdrenoMetal CB dump ' not in line:
continue
match = pattern.search(line)
if not match:
continue
process, pid, sequence, part, count, text = match.groups()
key = process, int(pid), int(sequence)
part, count = int(part), int(count)
expected, chunks = entries.setdefault(key, (count, {}))
if count != expected or not 1 <= part <= count or (part in chunks and chunks[part] != text):
parser.error(f'conflicting log fragments for {process}[{pid}] entry {sequence}')
chunks[part] = text
if not entries:
parser.error('no command-buffer messages found')
incomplete = 0
functions = []
if args.functions:
args.functions.mkdir(parents=True, exist_ok=True)
for (process, pid, sequence), (count, chunks) in sorted(entries.items()):
missing = set(range(1, count + 1)) - chunks.keys()
if missing:
incomplete += 1
print(f'{process}[{pid}] entry {sequence}: missing fragments {sorted(missing)}', file=sys.stderr)
continue
text = ''.join(chunks[part] for part in range(1, count + 1))
if text.strip() == 'native-metal dump capture limit reached':
incomplete += 1
print('command-buffer capture reached its file limit', file=sys.stderr)
function = re.fullmatch(r'native-metal dump function (\d+) (\S+): (.*)', text)
if function and args.functions:
index, name, encoded = function.groups()
try:
data = base64.b64decode(encoded, validate=True)
except ValueError:
incomplete += 1
print(f'{process}[{pid}] function {name}: {encoded[:120]}', file=sys.stderr)
continue
filename = f'function-{pid}-{index}.air'
(args.functions / filename).write_bytes(data)
functions.append(dict(process=process, pid=pid, name=name, file=filename))
print(f'{process}[{pid}] function {name}: {filename}, {len(data)} bytes')
else:
print(f'{process}[{pid}] ' + text)
if args.functions:
(args.functions / 'functions.json').write_text(json.dumps(functions, indent=2) + '\n')
print(f'command-buffer entries={len(entries)} incomplete={incomplete}', file=sys.stderr)
return bool(incomplete)
if __name__ == '__main__':
sys.exit(main())
+278
View File
@@ -0,0 +1,278 @@
#!/usr/bin/env python3
"""Build the bounded A740 GPU acceptance executable with embedded AIR and native SPIR-V controls."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
COMPONENT = Path(__file__).resolve().parents[2]
CORE = COMPONENT.parents[1]
HERE = Path(__file__).resolve().parent
def run(args, **kwargs):
return subprocess.run([str(x) for x in args], check=True, **kwargs)
def mask(source):
result = list(source)
i = 0
while i < len(source):
end = i
if source.startswith('//', i):
end = source.find('\n', i)
if end < 0: end = len(source)
elif source.startswith('/*', i):
depth, end = 1, i + 2
while depth:
if source.startswith('/*', end): depth += 1; end += 2
elif source.startswith('*/', end): depth -= 1; end += 2
else: end += 1
elif source[i] == '"':
end = i + 1
while end < len(source):
if source[end] == '\\': end += 2
elif source[end] == '"': end += 1; break
else: end += 1
if end > i:
for at in range(i, end):
if result[at] != '\n': result[at] = ' '
i = end
else: i += 1
return ''.join(result)
def item(source, name, kind='fn'):
clean = mask(source)
start = re.search(r'^' + kind + r' ' + name + r'\b', clean, re.M).start()
first = clean.index('{', start)
depth, end = 1, first + 1
while depth:
if clean[end] == '{': depth += 1
elif clean[end] == '}': depth -= 1
end += 1
return source[start:end] + '\n'
def arithmetic_air(half):
f = 'half' if half else 'float'
v = '<4 x ' + f + '>'
a = '0xH3BFE' if half else '9.990234375e-01'
b = '0xH1400' if half else '9.765625e-04'
vec = lambda value: '<' + ', '.join(f + ' ' + value for _ in range(4)) + '>'
narrow = '%initial = fptrunc <4 x float> %loaded to <4 x half>' if half else '%initial = fadd <4 x float> %loaded, zeroinitializer'
widen = '%answer = fpext <4 x half> %next to <4 x float>' if half else '%answer = fadd <4 x float> %next, zeroinitializer'
return '''target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-n8:16:32"
target triple = "air64_v28-apple-macosx26.0.0"
define void @arith(ptr addrspace(1) noundef "air-buffer-no-alias" %in, ptr addrspace(1) noundef "air-buffer-no-alias" %out, i32 noundef %gid) {
entry:
%g = zext i32 %gid to i64
%src = getelementptr inbounds <4 x float>, ptr addrspace(1) %in, i64 %g
%loaded = load <4 x float>, ptr addrspace(1) %src, align 16
''' + narrow + '''
br label %loop
loop:
%i = phi i32 [0, %entry], [%j, %loop]
%acc = phi ''' + v + ''' [%initial, %entry], [%next, %loop]
%product = fmul fast ''' + v + ' %acc, ' + vec(a) + '''
%next = fadd fast ''' + v + ' %product, ' + vec(b) + '''
%j = add i32 %i, 1
%again = icmp ult i32 %j, 256
br i1 %again, label %loop, label %exit
exit:
''' + widen + '''
%dst = getelementptr inbounds <4 x float>, ptr addrspace(1) %out, i64 %g
store <4 x float> %answer, ptr addrspace(1) %dst, align 16
ret void
}
!air.kernel = !{!0}
!air.version = !{!6}
!air.language_version = !{!7}
!0 = !{ptr @arith, !1, !2}
!1 = !{}
!2 = !{!3, !4, !5}
!3 = !{i32 0, !"air.buffer", !"air.location_index", i32 0, i32 1, !"air.read", !"air.address_space", i32 1, !"air.arg_type_size", i32 16, !"air.arg_type_align_size", i32 16, !"air.arg_type_name", !"float4", !"air.arg_name", !"in"}
!4 = !{i32 1, !"air.buffer", !"air.location_index", i32 1, i32 1, !"air.read_write", !"air.address_space", i32 1, !"air.arg_type_size", i32 16, !"air.arg_type_align_size", i32 16, !"air.arg_type_name", !"float4", !"air.arg_name", !"out"}
!5 = !{i32 2, !"air.thread_position_in_grid", !"air.arg_type_name", !"uint", !"air.arg_name", !"gid"}
!6 = !{i32 2, i32 8, i32 0}
!7 = !{!"Metal", i32 4, i32 0, i32 0}
'''
def control_spirv(half=False, scale=False):
# The arithmetic control has the AIR's four independent lanes and bounded loop. The
# bandwidth control is runtime_examples.ll's scale_and_offset: out[i] = in[i] * scale + i.
text = '''OpCapability Shader
'''
if half: text += 'OpCapability Float16\n'
text += '''OpMemoryModel Logical GLSL450
OpEntryPoint GLCompute %main "main" %gid %input %output''' + (' %scale' if scale else '') + '''
OpExecutionMode %main LocalSize 64 1 1
OpDecorate %gid BuiltIn GlobalInvocationId
OpDecorate %arr ArrayStride ''' + ('4' if scale else '16') + '''
OpMemberDecorate %Block 0 Offset 0
OpDecorate %Block Block
OpDecorate %input DescriptorSet 0
OpDecorate %input Binding 0
OpDecorate %output DescriptorSet 0
OpDecorate %output Binding 1
'''
if scale: text += '''OpDecorate %scale DescriptorSet 0
OpDecorate %scale Binding 2
'''
text += '''%void = OpTypeVoid
%fn = OpTypeFunction %void
%uint = OpTypeInt 32 0
%bool = OpTypeBool
%v3uint = OpTypeVector %uint 3
%ptr_gid = OpTypePointer Input %v3uint
%gid = OpVariable %ptr_gid Input
%float = OpTypeFloat 32
%v4float = OpTypeVector %float 4
'''
if half: text += '%half = OpTypeFloat 16\n%v4half = OpTypeVector %half 4\n'
value = '%float' if scale else '%v4float'
arithmetic = '%v4half' if half else '%v4float'
text += '%arr = OpTypeRuntimeArray ' + value + '''
%Block = OpTypeStruct %arr
%ptr_block = OpTypePointer StorageBuffer %Block
%ptr_value = OpTypePointer StorageBuffer ''' + value + '''
%input = OpVariable %ptr_block StorageBuffer
%output = OpVariable %ptr_block StorageBuffer
'''
if scale: text += '%scale = OpVariable %ptr_block StorageBuffer\n'
text += '''%zero = OpConstant %uint 0
%one = OpConstant %uint 1
%limit = OpConstant %uint 256
'''
if not scale:
scalar = '%half' if half else '%float'
text += '%a = OpConstant ' + scalar + ' 0.9990234375\n%b = OpConstant ' + scalar + ' 0.0009765625\n'
text += '%va = OpConstantComposite ' + arithmetic + ' %a %a %a %a\n%vb = OpConstantComposite ' + arithmetic + ' %b %b %b %b\n'
text += '''%main = OpFunction %void None %fn
%entry = OpLabel
%ids = OpLoad %v3uint %gid
%index = OpCompositeExtract %uint %ids 0
%src = OpAccessChain %ptr_value %input %zero %index
%dst = OpAccessChain %ptr_value %output %zero %index
%loaded = OpLoad ''' + value + ''' %src
'''
if scale:
text += '''%sf = OpAccessChain %ptr_value %scale %zero %zero
%factor = OpLoad %float %sf
%product = OpFMul %float %loaded %factor
%fi = OpConvertUToF %float %index
%answer = OpFAdd %float %product %fi
OpStore %dst %answer
OpReturn
OpFunctionEnd
'''
else:
text += ('%initial = OpFConvert %v4half %loaded\n' if half else '%initial = OpCopyObject %v4float %loaded\n')
text += '''OpBranch %loop
%loop = OpLabel
%i = OpPhi %uint %zero %entry %j %body
%acc = OpPhi ''' + arithmetic + ''' %initial %entry %next %body
OpLoopMerge %exit %body None
OpBranch %body
%body = OpLabel
%product = OpFMul ''' + arithmetic + ''' %acc %va
%next = OpFAdd ''' + arithmetic + ''' %product %vb
%j = OpIAdd %uint %i %one
%again = OpULessThan %bool %j %limit
OpBranchConditional %again %loop %exit
%exit = OpLabel
'''
text += ('%answer = OpFConvert %v4float %next\n' if half else '%answer = OpCopyObject %v4float %next\n')
text += 'OpStore %dst %answer\nOpReturn\nOpFunctionEnd\n'
return text
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--translator', type=Path, default=CORE / '.cache/gpu-reference/metal2vulkan')
parser.add_argument('--target', choices=['host', 'macos27'], default='macos27')
parser.add_argument('--vulkan-include', type=Path, default=CORE / '.cache/mesa-work/include')
parser.add_argument('--output', type=Path, default=CORE / 'build/gpu-acceptance')
args = parser.parse_args()
out = args.output.resolve(); out.mkdir(parents=True, exist_ok=True)
source = args.translator.resolve()
pin = subprocess.check_output(['git', '-C', source, 'rev-parse', 'HEAD'], text=True).strip()
expected = json.loads((CORE / 'scripts/data/gpu-metal-vulkan-inputs.json').read_text())['translator']['commit']
if pin != expected: raise SystemExit('Acceptance translator differs from the product source pin')
upstream = source / 'crates/metal2vulkan-validation/src/runtime_lane/native_maths.rs'
maths = upstream.read_text()
helpers = ['operands','ulp','half_ulp','excess','broken','to_half','metal_half','half_operands','half_operand','half_within','half_within_sum']
constants = ''.join(re.search(r'^const ' + name + r':[^\n]+', maths, re.M)[0]+'\n' for name in ['IN','OUT','COUNT','PAYLOAD_NAN'])
stream_start = maths.index('struct Stream(')
stream = maths[stream_start:maths.index('const PAYLOAD_NAN', stream_start)]
numerical = constants + stream + ''.join(item(maths, name).replace('fn '+name, 'pub fn '+name, 1) if name in ['operands','broken','metal_half','half_operands','half_within','half_within_sum','half_ulp'] else item(maths,name) for name in helpers)
(out/'numerical.rs').write_text(numerical)
kernels = ['kernel','half_kernel','half_arithmetic_kernel']
generator = constants + ''.join(item(maths, name) for name in kernels)
generator += '\nfn main() { let root=std::path::PathBuf::from(std::env::args().nth(1).unwrap());'
for f,name in zip(kernels,['native_maths','native_half_rounding','native_half_arithmetic']):
generator += 'std::fs::write(root.join("'+name+'.ll"), '+f+'()).unwrap();'
generator += '}\n'
(out/'kernels.rs').write_text(generator)
run(['rustc','+1.98.0','--edition=2021','-A','dead_code',out/'kernels.rs','-o',out/'kernels'])
run([out/'kernels',out])
(out/'native_half_division.ll').write_text((out/'native_half_rounding.ll').read_text().replace('%hs = fadd half', '%hs = fdiv half'))
(out/'arith32.ll').write_text(arithmetic_air(False));(out/'arith16.ll').write_text(arithmetic_air(True))
llvm = shutil.which('llvm-as') or '/opt/homebrew/opt/llvm/bin/llvm-as'
for name in ['native_maths','native_half_rounding','native_half_arithmetic','native_half_division','arith32','arith16']:
run([llvm,out/(name+'.ll'),'-o',out/(name+'.air')])
run([llvm,source/'crates/metal2vulkan-validation/fixtures/public/examples/runtime_examples.ll','-o',out/'runtime_examples.air'])
for name,half,scale in [('arith32',False,False),('arith16',True,False),('scale',False,True)]:
(out/(name+'.spvasm')).write_text(control_spirv(half,scale))
run(['spirv-as','--target-env','vulkan1.2',out/(name+'.spvasm'),'-o',out/(name+'.spv')])
run(['spirv-val','--target-env','vulkan1.2',out/(name+'.spv')])
component = subprocess.check_output(['git','-C',COMPONENT,'rev-parse','HEAD'],text=True).strip()
manifest = '\n'.join(['[package]','name="gpu-acceptance"','version="0.1.0"','edition="2024"','[workspace]',
'[lib]','crate-type=["staticlib"]','path='+json.dumps(str(HERE/'lib.rs')),'[dependencies]',
'metal2vulkan={path='+json.dumps(str(source/'crates/metal2vulkan'))+',features=["ash"]}',
'metal2vulkan-runtime={path='+json.dumps(str(source/'crates/metal2vulkan-runtime'))+'}',
'ash={version="0.38",default-features=false,features=["loaded"]}','serde_json="1"','libc="0.2"',
'[profile.release]','panic="unwind"','debug=1','lto="fat"','codegen-units=1',''])
(out/'Cargo.toml').write_text(manifest)
triple = 'arm64e-apple-darwin' if args.target=='macos27' else 'aarch64-apple-darwin'
env=dict(os.environ,NMB_ACCEPTANCE_NUMERIC=str(out/'numerical.rs'),NMB_ACCEPTANCE_INPUTS=str(out),
NMB_COMPONENT_ID=component,NMB_TRANSLATOR_ID=pin,MACOSX_DEPLOYMENT_TARGET='27.0' if args.target=='macos27' else '14.0')
cargo=['cargo','+1.98.0']
if args.target=='macos27': env['RUSTC_BOOTSTRAP']='1';cargo+=['-Z','build-std=std,panic_unwind']
with (out/'build.log').open('w') as stream:
run(cargo+['build','--release','--manifest-path',out/'Cargo.toml','--target',triple,'--target-dir',CORE/'build/m2v-target'],env=env,cwd=CORE,stdout=stream,stderr=subprocess.STDOUT)
archive=CORE/'build/m2v-target'/triple/'release/libgpu_acceptance.a'
sdk=subprocess.check_output(['xcrun','--sdk','macosx','--show-sdk-path'],text=True).strip()
cc=['xcrun','clang','-target','arm64e-apple-macos27.0' if args.target=='macos27' else 'arm64-apple-macos14.0','-isysroot',sdk,'-O2','-Wall','-Wextra','-Werror']
extra=[]
if args.target=='macos27':
renames={'_pthread_create':'_nmb_pac_pthread_create','__tlv_atexit':'_nmb_pac_tlv_atexit','_atexit':'_nmb_pac_atexit','___cxa_atexit':'_nmb_pac_cxa_atexit','__dyld_register_func_for_add_image':'_nmb_pac_dyld_register_func_for_add_image','__Unwind_Backtrace':'_nmb_pac_Unwind_Backtrace','_dlsym':'_nmb_pac_dlsym'}
objcopy=shutil.which('llvm-objcopy') or '/opt/homebrew/opt/llvm/bin/llvm-objcopy'
pac=out/'libgpu_acceptance_pac.a'
run([objcopy,*['--redefine-sym='+a+'='+b for a,b in renames.items()],archive,pac]);archive=pac
run(cc+['-std=c11','-I',args.vulkan_include,'-c',COMPONENT/'bridge/RustPacBoundary.c','-o',out/'RustPacBoundary.o'])
extra=[out/'RustPacBoundary.o','-Wl,-map,'+str(out/'gpu-acceptance.map')]
exe=out/'gpu-acceptance'
run(cc+['-std=c11',HERE/'main.c',archive,*extra,'-framework','Foundation','-framework','Metal','-framework','IOSurface','-lobjc','-liconv','-Wl,-dead_strip','-o',exe])
if args.target=='macos27':run(['python3',CORE/'scripts/rust-pac-rewrite.py',exe,out/'gpu-acceptance.map',archive.name])
run(['strip','-x',exe])
run(['codesign','--force','--sign','-','--pagesize','4096',exe])
if args.target=='host': run([exe,'--describe'])
(out/'gpu-acceptance.json').write_text(json.dumps(dict(component=component,translator=pin,numerical_source_sha256=hashlib.sha256(upstream.read_bytes()).hexdigest(),target=triple,sha256=hashlib.sha256(exe.read_bytes()).hexdigest()),indent=2)+'\n')
package=out/'files'
(package/'bin').mkdir(parents=True,exist_ok=True)
(package/'share/gpu-acceptance').mkdir(parents=True,exist_ok=True)
shutil.copy2(exe,package/'bin/gpu-acceptance')
shutil.copyfile(out/'gpu-acceptance.json',package/'share/gpu-acceptance/build.json')
shutil.copyfile(COMPONENT/'LICENSE',package/'share/gpu-acceptance/LICENSE')
for license in source.glob('LICENSE*'):
if license.is_file():shutil.copyfile(license,package/'share/gpu-acceptance'/('metal2vulkan-'+license.name))
print(exe)
if __name__=='__main__': main()
+825
View File
@@ -0,0 +1,825 @@
// SPDX-License-Identifier: MIT
use ash::vk;
use metal2vulkan_runtime::{
Buffer, CommandBuffer, CommandBufferStatus, CommandQueue, ComputePipelineState, Device,
DeviceOptions, MessageSeverity, MessageSink, PipelineOptions, ResourceOptions,
ShaderCacheOptions, Size,
};
use serde_json::json;
use std::ffi::{CStr, c_char};
use std::path::PathBuf;
use std::time::{Duration, Instant};
mod numerical {
include!(env!("NMB_ACCEPTANCE_NUMERIC"));
}
const MATH: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/native_maths.air"));
const HALF: &[u8] = include_bytes!(concat!(
env!("NMB_ACCEPTANCE_INPUTS"),
"/native_half_rounding.air"
));
const HALF_DIVISION: &[u8] = include_bytes!(concat!(
env!("NMB_ACCEPTANCE_INPUTS"),
"/native_half_division.air"
));
const EXAMPLES: &[u8] = include_bytes!(concat!(
env!("NMB_ACCEPTANCE_INPUTS"),
"/runtime_examples.air"
));
const FP32: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith32.air"));
const FP16: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith16.air"));
const SPV32: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith32.spv"));
const SPV16: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/arith16.spv"));
const SPV_SCALE: &[u8] = include_bytes!(concat!(env!("NMB_ACCEPTANCE_INPUTS"), "/scale.spv"));
const ITERATIONS: u64 = 256;
const WARM: usize = 2;
const SAMPLES: usize = 9;
const WAIT: Duration = Duration::from_secs(10);
type Result<T> = std::result::Result<T, String>;
fn err(value: impl std::fmt::Debug) -> String {
format!("{value:?}")
}
fn emit(value: serde_json::Value) {
println!("{value}");
}
fn stamp() -> serde_json::Value {
json!({"boot":std::env::var("NATIVE_METAL_BOOT_SESSION").unwrap_or_default(),
"component":env!("NMB_COMPONENT_ID"), "translator":env!("NMB_TRANSLATOR_ID"),
"uptime_us":metal2vulkan_runtime::stage_trace::now_nanos()/1000})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_acceptance_describe() {
emit(json!({"schema":"unixnu-gpu-acceptance-1", "stamp":stamp(),
"numeric":"upstream native_maths/half_rounding references and bounds, 4096 threads",
"arithmetic":"FP32/FP16 vec4 independent multiply/add chains, 256 iterations, paired native SPIR-V on the same Vulkan device and queue",
"bandwidth":"native Shared buffers, scale_and_offset and blit copy, paired native Vulkan dispatch/copy",
"samples":SAMPLES,"warmups":WARM,"ratio_required":0.95,"runtime_llvm":false}));
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_acceptance(
loader: *const c_char,
cache: *const c_char,
mode: *const c_char,
) -> i32 {
let run = std::panic::catch_unwind(|| {
// The C launcher owns these NUL-terminated strings for this call.
let (loader, cache, mode) = unsafe {
(
CStr::from_ptr(loader),
CStr::from_ptr(cache),
CStr::from_ptr(mode),
)
};
execute(loader, cache, mode.to_str().map_err(err)?)
});
match run {
Ok(Ok(pass)) => {
emit(
json!({"event":"complete","measurement_complete":true,"correct":pass,"stamp":stamp()}),
);
if pass { 0 } else { 1 }
}
Ok(Err(error)) => {
emit(json!({"event":"error","error":error,"stamp":stamp()}));
70
}
Err(_) => {
emit(json!({"event":"error","error":"panic","stamp":stamp()}));
70
}
}
}
fn execute(loader: &CStr, cache: &CStr, mode: &str) -> Result<bool> {
nmb_acceptance_describe();
let mut options = DeviceOptions::default();
options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned()));
options.shader_cache = ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned());
if mode == "numeric" {
options
.layers
.push(c"VK_LAYER_KHRONOS_validation".to_owned());
}
let validation_errors = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let errors = validation_errors.clone();
options.messages = Some(MessageSink::new(move |m| {
if m.id_name.ends_with("device-recovered") {
fatal_gpu("Vulkan device recovered during acceptance");
}
if m.severity >= MessageSeverity::Error {
errors.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
eprintln!("gpu-acceptance {} {}", m.id_name, m.text)
}));
let device = Device::system_default_with(&options).map_err(err)?;
let p = device.properties();
emit(
json!({"event":"device","name":device.name(),"vendor_id":p.vendor_id,"device_id":p.device_id,
"policy":format!("{:?}", device.compile_device().policy()),
"profile":format!("{:?}", device.compile_device().profile()),"stamp":stamp()}),
);
if p.vendor_id != 0x5143 || p.device_id != 0x4305_0a01 {
return Err("This calibration requires the sheng Adreno 740".into());
}
let queue = device.new_command_queue().map_err(err)?;
let mut pass = true;
if mode != "calibrate" {
pass &= numeric(&device, &queue)?;
}
if mode != "numeric" {
pass &= calibrate(&device, &queue)?;
}
let w = device.work_counts();
emit(
json!({"event":"runtime-counters","timestamp_slots":w.timestamp_slots_taken,
"calibrations":w.calibrations,"calibration_disagreements":w.calibration_disagreements,"validation_errors":validation_errors.load(std::sync::atomic::Ordering::Relaxed),"stamp":stamp()}),
);
Ok(pass && validation_errors.load(std::sync::atomic::Ordering::Relaxed) == 0)
}
fn state(device: &Device, air: &[u8], name: &str) -> Result<ComputePipelineState> {
let lib = device.new_library_with_compile_library(
metal2vulkan::library::Library::from_air(air).map_err(err)?,
);
let f = lib.new_function_with_name(name).map_err(err)?;
device
.new_compute_pipeline_state_with_function(&f, &PipelineOptions::default())
.map_err(err)
}
fn buffer(device: &Device, bytes: &[u8]) -> Result<Buffer> {
device
.new_buffer_with_bytes(bytes, ResourceOptions::shared())
.map_err(err)
}
fn floats(buffer: &Buffer) -> Result<&[f32]> {
let ptr = buffer.contents().ok_or("Shared buffer has no contents")?;
// Every caller has waited for GPU completion and all lengths are multiples of four.
Ok(unsafe { std::slice::from_raw_parts(ptr.as_ptr().cast(), buffer.length() as usize / 4) })
}
fn timed(cb: &CommandBuffer, device: &Device) -> Result<(f64, f64)> {
let slots = device.work_counts().timestamp_slots_taken;
let wall = Instant::now();
cb.commit().map_err(err)?;
if cb.wait_until_completed_for(WAIT).is_err() {
fatal_gpu("runtime completion exceeded 10 seconds");
}
if cb.status() != CommandBufferStatus::Completed {
return Err(format!(
"command buffer {:?}: {:?}",
cb.status(),
cb.error()
));
}
let us = (cb.gpu_end_time().map_err(err)? - cb.gpu_start_time().map_err(err)?) * 1e6;
if us <= 0.0 || !us.is_finite() {
return Err("GPU timestamp interval unavailable".into());
}
// The runtime allocates timestamp slots while encoding, before commit. The physical queue
// timestamp support is checked by Native::new; the total proves timing was in use.
if slots == 0 {
return Err("Runtime supplied no GPU timestamp slots".into());
}
Ok((us, wall.elapsed().as_secs_f64() * 1e6))
}
fn dispatch(
device: &Device,
queue: &CommandQueue,
state: &ComputePipelineState,
buffers: &[&Buffer],
count: u64,
) -> Result<(f64, f64)> {
let cb = queue.command_buffer().map_err(err)?;
let mut enc = cb.compute_command_encoder().map_err(err)?;
enc.set_compute_pipeline_state(state).map_err(err)?;
let raw = unsafe { device.raw() };
let limits = unsafe {
raw.instance
.get_physical_device_properties(raw.physical_device)
}
.limits;
let chunk = limits.max_compute_work_group_count[0] as u64 * 64;
let stride = buffers[0].length() / count;
for first in (0..count).step_by(chunk as usize) {
let units = (count - first).min(chunk);
for (i, b) in buffers.iter().enumerate() {
enc.set_buffer(Some(b), if i < 2 { first * stride } else { 0 }, i as u32)
.map_err(err)?;
}
enc.dispatch_threadgroups(Size::new(units / 64, 1, 1), Size::new(64, 1, 1))
.map_err(err)?;
}
enc.end_encoding().map_err(err)?;
timed(&cb, device)
}
fn copy(
device: &Device,
queue: &CommandQueue,
input: &Buffer,
output: &Buffer,
) -> Result<(f64, f64)> {
let cb = queue.command_buffer().map_err(err)?;
let mut enc = cb.blit_command_encoder().map_err(err)?;
enc.copy_from_buffer_to_buffer(input, 0, output, 0, input.length())
.map_err(err)?;
enc.end_encoding().map_err(err)?;
timed(&cb, device)
}
fn numeric(device: &Device, queue: &CommandQueue) -> Result<bool> {
let inputs = numerical::operands();
let bytes: Vec<u8> = inputs
.iter()
.flatten()
.flat_map(|v| v.to_le_bytes())
.collect();
let input = buffer(device, &bytes)?;
let output = device
.new_buffer(4096 * 14 * 4, ResourceOptions::shared())
.map_err(err)?;
emit(json!({"event":"begin","case":"native-maths","stamp":stamp()}));
let pipeline = state(device, MATH, "nm")?;
dispatch(device, queue, &pipeline, &[&input, &output], 4096)?;
let results = floats(&output)?;
let mut failures = 0;
let mut examples = Vec::new();
for (x, r) in inputs.iter().zip(results.as_chunks::<14>().0) {
for problem in numerical::broken(x, r, true) {
failures += 1;
if examples.len() < 12 {
examples.push(problem);
}
}
}
emit(
json!({"event":"numeric","case":"native-maths","words":4096*14,"failures":failures,"examples":examples,"stamp":stamp()}),
);
let mut half_failures = 0;
let inputs = numerical::half_operands();
let bytes: Vec<u8> = inputs
.iter()
.flatten()
.flat_map(|v| v.to_le_bytes())
.collect();
let input = buffer(device, &bytes)?;
let output = device
.new_buffer(4096 * 2 * 4, ResourceOptions::shared())
.map_err(err)?;
emit(json!({"event":"begin","case":"native-half-rounding","stamp":stamp()}));
let pipeline = state(device, HALF, "hr")?;
dispatch(device, queue, &pipeline, &[&input, &output], 4096)?;
for (&[a, b], r) in inputs.iter().zip(floats(&output)?.as_chunks::<2>().0) {
// Bounds and allowed dropped-rounding results are the upstream half_rounding lane's.
let ha = numerical::metal_half(a as f64);
let hb = numerical::metal_half(b as f64);
let sum = numerical::metal_half(ha + hb);
let allowed = [
a + b,
ha as f32 + hb as f32,
numerical::metal_half((a + b) as f64) as f32,
];
if !(r[0].to_bits() == (sum as f32).to_bits()
|| numerical::half_within_sum(r[0], a as f64, b as f64)
|| allowed.contains(&r[0]))
{
half_failures += 1;
}
if !(r[1].to_bits() == (ha as f32).to_bits()
|| numerical::half_within(r[1], a as f64)
|| r[1].to_bits() == a.to_bits())
{
half_failures += 1;
}
}
emit(
json!({"event":"numeric","case":"native-half-rounding","words":4096*2,"failures":half_failures,"stamp":stamp()}),
);
// These normal, exactly representable operands cover the native16 divisor limit.
// The large divisors need normalization before a half reciprocal can stay normal.
let cases = [
[65504.0f32, 65504.0],
[65504.0, 32768.0],
[32768.0, 65504.0],
[16384.0, 16384.0],
[16392.0, 16392.0],
[-65504.0, 65504.0],
[65504.0, -32768.0],
[1.0, 1.0],
[0.00006103515625, 0.00006103515625],
];
let inputs: Vec<[f32; 2]> = (0..4096).map(|i| cases[i % cases.len()]).collect();
let bytes: Vec<u8> = inputs
.iter()
.flatten()
.flat_map(|v| v.to_le_bytes())
.collect();
let input = buffer(device, &bytes)?;
let output = device
.new_buffer(4096 * 2 * 4, ResourceOptions::shared())
.map_err(err)?;
emit(json!({"event":"begin","case":"native-half-division-range","stamp":stamp()}));
let pipeline = state(device, HALF_DIVISION, "hr")?;
dispatch(device, queue, &pipeline, &[&input, &output], 4096)?;
let mut division_failures = 0;
let mut max_error_ulp = 0.0f64;
for (&[a, b], r) in inputs.iter().zip(floats(&output)?.as_chunks::<2>().0) {
let want = a as f64 / b as f64;
let error = (r[0] as f64 - want).abs() / numerical::half_ulp(want);
if !error.is_finite() || error > 2.5 {
division_failures += 1;
}
max_error_ulp = max_error_ulp.max(error);
}
emit(
json!({"event":"numeric","case":"native-half-division-range","words":4096,
"failures":division_failures,"max_error_half_ulp":max_error_ulp,"bound_half_ulp":2.5,"stamp":stamp()}),
);
Ok(failures == 0 && half_failures == 0 && division_failures == 0)
}
fn median(values: &mut [f64]) -> f64 {
values.sort_by(f64::total_cmp);
values[values.len() / 2]
}
fn paired(
mut translated: impl FnMut() -> Result<(f64, f64)>,
mut native: impl FnMut() -> Result<(f64, f64)>,
) -> Result<(f64, f64, f64, f64)> {
let (mut t, mut n, mut tw, mut nw) = (Vec::new(), Vec::new(), Vec::new(), Vec::new());
for i in 0..WARM + SAMPLES {
let (a, b) = if i % 2 == 0 {
(translated()?, native()?)
} else {
let b = native()?;
(translated()?, b)
};
if i >= WARM {
t.push(a.0);
tw.push(a.1);
n.push(b.0);
nw.push(b.1);
}
}
Ok((
median(&mut t),
median(&mut n),
median(&mut tw),
median(&mut nw),
))
}
fn row(
kind: &str,
precision: &str,
bytes: u64,
operations: u64,
times: (f64, f64, f64, f64),
difference: f64,
) -> bool {
let (t, n, tw, nw) = times;
let ratio = n / t;
let limit = match (kind, precision) {
("arithmetic", "float16") => 0.5,
("arithmetic", _) => 1e-4,
_ => 0.0,
};
let correct = difference.is_finite() && difference <= limit;
let target_met = ratio >= 0.95;
emit(
json!({"event":"calibration","kind":kind,"precision":precision,"bytes":bytes,"operations":operations,
"translator_gpu_us":t,"native_gpu_us":n,"translator_wall_us":tw,"native_wall_us":nw,
"native_fraction":ratio,"required_fraction":0.95,"translator_gflops":operations as f64/t/1000.0,
"translator_gbps":bytes as f64/t/1000.0,"max_output_difference":difference,"comparison_limit":limit,"correct":correct,"target_met":target_met,"stamp":stamp()}),
);
correct
}
fn calibrate(device: &Device, queue: &CommandQueue) -> Result<bool> {
let mut pass = true;
for (name, air, spv) in [("float32", FP32, SPV32), ("float16", FP16, SPV16)] {
if name == "float16"
&& !device
.compile_device()
.profile()
.enabled()
.features
.contains(&metal2vulkan::device::VkFeature::ShaderFloat16)
{
emit(
json!({"event":"unsupported","case":"float16","reason":"shaderFloat16 is not enabled","stamp":stamp()}),
);
pass = false;
continue;
}
emit(json!({"event":"begin","case":"arithmetic","precision":name,"stamp":stamp()}));
let pipeline = state(device, air, "arith")?;
for count in [131072u64, 524288] {
let data: Vec<u8> = (0..count * 4)
.flat_map(|i| ((i % 1024) as f32 / 2048.0).to_le_bytes())
.collect();
let input = buffer(device, &data)?;
let output = device
.new_buffer(data.len() as u64, ResourceOptions::shared())
.map_err(err)?;
let control = device
.new_buffer(data.len() as u64, ResourceOptions::shared())
.map_err(err)?;
let native = Native::new(device, queue, spv, &[&input, &control], count, 16)?;
let times = paired(
|| dispatch(device, queue, &pipeline, &[&input, &output], count),
|| native.run(queue, false),
)?;
let difference =
floats(&output)?
.iter()
.zip(floats(&control)?)
.fold(0.0f64, |v, (a, b)| {
if a.is_finite() && b.is_finite() {
v.max((*a as f64 - *b as f64).abs())
} else {
f64::INFINITY
}
});
pass &= row(
"arithmetic",
name,
0,
count * 4 * ITERATIONS * 2,
times,
difference,
);
}
}
let pipeline = state(device, EXAMPLES, "scale_and_offset")?;
let scale = buffer(device, &2.0f32.to_le_bytes())?;
for mib in [16u64, 64, 256] {
emit(json!({"event":"begin","case":"shared-bandwidth","mib":mib,"stamp":stamp()}));
let length = mib << 20;
let count = length / 4;
let input = device
.new_buffer(length, ResourceOptions::shared())
.map_err(err)?;
if let Some(at) = input.contents() {
unsafe {
std::ptr::write_bytes(at.as_ptr(), 0x3e, length as usize);
}
}
let output = device
.new_buffer(length, ResourceOptions::shared())
.map_err(err)?;
let control = device
.new_buffer(length, ResourceOptions::shared())
.map_err(err)?;
let native = Native::new(
device,
queue,
SPV_SCALE,
&[&input, &control, &scale],
count,
4,
)?;
let times = paired(
|| dispatch(device, queue, &pipeline, &[&input, &output, &scale], count),
|| native.run(queue, false),
)?;
let difference = floats(&output)?
.iter()
.zip(floats(&control)?)
.fold(0.0f64, |v, (a, b)| v.max((*a as f64 - *b as f64).abs()));
pass &= row(
"shared-kernel-read-write",
"float32",
length * 2,
0,
times,
difference,
);
let times = paired(
|| copy(device, queue, &input, &output),
|| native.run(queue, true),
)?;
let difference = floats(&output)?
.iter()
.zip(floats(&control)?)
.fold(0.0f64, |v, (a, b)| v.max((*a as f64 - *b as f64).abs()));
pass &= row(
"shared-blit-copy",
"bytes",
length * 2,
0,
times,
difference,
);
}
Ok(pass)
}
fn fatal_gpu(why: &str) -> ! {
emit(json!({"event":"gpu-timeout","error":why,"reboot_required":true,"stamp":stamp()}));
use std::io::Write;
let _ = std::io::stdout().flush();
let _ = std::io::stderr().flush();
// A timed-out queue may still own every resource; exit without running Vulkan destructors.
unsafe { libc::_exit(75) }
}
struct Native<'a> {
device: &'a Device,
owner: vk::Device,
pool: vk::CommandPool,
commands: [vk::CommandBuffer; 2],
queries: vk::QueryPool,
fence: vk::Fence,
layout: vk::PipelineLayout,
set_layout: vk::DescriptorSetLayout,
descriptors: vk::DescriptorPool,
pipeline: vk::Pipeline,
module: vk::ShaderModule,
period: f64,
bits: u32,
}
impl<'a> Native<'a> {
fn new(
device: &'a Device,
queue: &CommandQueue,
spv: &[u8],
buffers: &[&Buffer],
count: u64,
stride: u64,
) -> Result<Self> {
// All borrowed objects outlive native submissions, which finish before this object drops.
unsafe {
let raw = device.raw();
let vk = raw.device;
let owner = vk.handle();
let family = queue.with_vk_queue(|d, _, f| {
if d.handle() != owner {
fatal_gpu("Vulkan device changed before native setup");
}
f
});
let families = raw
.instance
.get_physical_device_queue_family_properties(raw.physical_device);
let bits = families[family as usize].timestamp_valid_bits;
if bits == 0 {
return Err("Native calibration requires hardware timestamps".into());
}
let limits = raw
.instance
.get_physical_device_properties(raw.physical_device)
.limits;
let period = limits.timestamp_period as f64;
let chunk = limits.max_compute_work_group_count[0] as u64 * 64;
let pieces = count.div_ceil(chunk) as usize;
let pool = vk
.create_command_pool(
&vk::CommandPoolCreateInfo::default().queue_family_index(family),
None,
)
.map_err(err)?;
let cmds = device
.with_loader_word(|| {
vk.allocate_command_buffers(
&vk::CommandBufferAllocateInfo::default()
.command_pool(pool)
.level(vk::CommandBufferLevel::PRIMARY)
.command_buffer_count(2),
)
})
.map_err(err)?;
let commands = [cmds[0], cmds[1]];
let queries = vk
.create_query_pool(
&vk::QueryPoolCreateInfo::default()
.query_type(vk::QueryType::TIMESTAMP)
.query_count(2),
None,
)
.map_err(err)?;
let fence = vk
.create_fence(&vk::FenceCreateInfo::default(), None)
.map_err(err)?;
let bindings: Vec<_> = (0..buffers.len())
.map(|i| {
vk::DescriptorSetLayoutBinding::default()
.binding(i as u32)
.descriptor_type(vk::DescriptorType::STORAGE_BUFFER)
.descriptor_count(1)
.stage_flags(vk::ShaderStageFlags::COMPUTE)
})
.collect();
let set_layout = vk
.create_descriptor_set_layout(
&vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings),
None,
)
.map_err(err)?;
let layout = vk
.create_pipeline_layout(
&vk::PipelineLayoutCreateInfo::default().set_layouts(&[set_layout]),
None,
)
.map_err(err)?;
let sizes = [vk::DescriptorPoolSize {
ty: vk::DescriptorType::STORAGE_BUFFER,
descriptor_count: (buffers.len() * pieces) as u32,
}];
let descriptors = vk
.create_descriptor_pool(
&vk::DescriptorPoolCreateInfo::default()
.max_sets(pieces as u32)
.pool_sizes(&sizes),
None,
)
.map_err(err)?;
let layouts = vec![set_layout; pieces];
let sets = vk
.allocate_descriptor_sets(
&vk::DescriptorSetAllocateInfo::default()
.descriptor_pool(descriptors)
.set_layouts(&layouts),
)
.map_err(err)?;
for (piece, &set) in sets.iter().enumerate() {
let first = piece as u64 * chunk;
let units = (count - first).min(chunk);
let infos: Vec<_> = buffers
.iter()
.enumerate()
.map(|(i, b)| {
let (buffer, offset) = b.vk_buffer();
[vk::DescriptorBufferInfo::default()
.buffer(buffer)
.offset(offset + if i < 2 { first * stride } else { 0 })
.range(if i < 2 { units * stride } else { b.length() })]
})
.collect();
let writes: Vec<_> = infos
.iter()
.enumerate()
.map(|(i, b)| {
vk::WriteDescriptorSet::default()
.dst_set(set)
.dst_binding(i as u32)
.descriptor_type(vk::DescriptorType::STORAGE_BUFFER)
.buffer_info(b)
})
.collect();
vk.update_descriptor_sets(&writes, &[]);
}
let words: Vec<u32> = spv
.chunks_exact(4)
.map(|b| u32::from_le_bytes(b.try_into().unwrap()))
.collect();
let module = vk
.create_shader_module(&vk::ShaderModuleCreateInfo::default().code(&words), None)
.map_err(err)?;
let stage = vk::PipelineShaderStageCreateInfo::default()
.stage(vk::ShaderStageFlags::COMPUTE)
.module(module)
.name(c"main");
let pipeline = vk
.create_compute_pipelines(
vk::PipelineCache::null(),
&[vk::ComputePipelineCreateInfo::default()
.stage(stage)
.layout(layout)],
None,
)
.map_err(|(_, e)| err(e))?[0];
let barriers = [vk::MemoryBarrier::default()
.src_access_mask(vk::AccessFlags::MEMORY_WRITE | vk::AccessFlags::HOST_WRITE)
.dst_access_mask(vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE)];
for (i, &cb) in commands.iter().enumerate() {
vk.begin_command_buffer(cb, &vk::CommandBufferBeginInfo::default())
.map_err(err)?;
vk.cmd_reset_query_pool(cb, queries, 0, 2);
vk.cmd_pipeline_barrier(
cb,
vk::PipelineStageFlags::ALL_COMMANDS | vk::PipelineStageFlags::HOST,
vk::PipelineStageFlags::ALL_COMMANDS,
vk::DependencyFlags::empty(),
&barriers,
&[],
&[],
);
vk.cmd_write_timestamp(cb, vk::PipelineStageFlags::BOTTOM_OF_PIPE, queries, 0);
if i == 0 {
vk.cmd_bind_pipeline(cb, vk::PipelineBindPoint::COMPUTE, pipeline);
for (piece, &set) in sets.iter().enumerate() {
vk.cmd_bind_descriptor_sets(
cb,
vk::PipelineBindPoint::COMPUTE,
layout,
0,
&[set],
&[],
);
let units = (count - piece as u64 * chunk).min(chunk);
vk.cmd_dispatch(cb, (units / 64) as u32, 1, 1);
}
} else {
let (a, ao) = buffers[0].vk_buffer();
let (b, bo) = buffers[1].vk_buffer();
vk.cmd_copy_buffer(
cb,
a,
b,
&[vk::BufferCopy {
src_offset: ao,
dst_offset: bo,
size: buffers[0].length(),
}],
);
}
vk.cmd_pipeline_barrier(
cb,
vk::PipelineStageFlags::ALL_COMMANDS,
vk::PipelineStageFlags::HOST,
vk::DependencyFlags::empty(),
&[vk::MemoryBarrier::default()
.src_access_mask(vk::AccessFlags::MEMORY_WRITE)
.dst_access_mask(vk::AccessFlags::HOST_READ)],
&[],
&[],
);
vk.cmd_write_timestamp(cb, vk::PipelineStageFlags::BOTTOM_OF_PIPE, queries, 1);
vk.end_command_buffer(cb).map_err(err)?;
}
Ok(Self {
device,
owner,
pool,
commands,
queries,
fence,
layout,
set_layout,
descriptors,
pipeline,
module,
period,
bits,
})
}
}
fn run(&self, queue: &CommandQueue, copy: bool) -> Result<(f64, f64)> {
let wall = Instant::now();
unsafe {
let raw = self.device.raw();
let vk = raw.device;
if vk.handle() != self.owner {
fatal_gpu("Vulkan device changed during native calibration");
}
vk.reset_fences(&[self.fence]).map_err(err)?;
queue
.with_vk_queue(|d, q, _| {
if d.handle() != self.owner {
fatal_gpu("Vulkan queue changed during native calibration");
}
d.queue_submit(
q,
&[vk::SubmitInfo::default()
.command_buffers(&[self.commands[usize::from(copy)]])],
self.fence,
)
})
.map_err(err)?;
match vk.wait_for_fences(&[self.fence], true, 10_000_000_000) {
Err(vk::Result::TIMEOUT) => fatal_gpu("native completion exceeded 10 seconds"),
other => other.map_err(err)?,
}
let mut times = [0u64; 2];
vk.get_query_pool_results(self.queries, 0, &mut times, vk::QueryResultFlags::TYPE_64)
.map_err(err)?;
let mask = if self.bits == 64 {
u64::MAX
} else {
(1u64 << self.bits) - 1
};
let us = (times[1].wrapping_sub(times[0]) & mask) as f64 * self.period / 1000.0;
if us <= 0.0 {
return Err("Native timestamp interval is zero".into());
}
Ok((us, wall.elapsed().as_secs_f64() * 1e6))
}
}
}
impl Drop for Native<'_> {
fn drop(&mut self) {
unsafe {
let raw = self.device.raw();
let vk = raw.device;
if vk.handle() != self.owner {
fatal_gpu("Vulkan device changed during native calibration");
}
vk.destroy_pipeline(self.pipeline, None);
vk.destroy_shader_module(self.module, None);
vk.destroy_descriptor_pool(self.descriptors, None);
vk.destroy_pipeline_layout(self.layout, None);
vk.destroy_descriptor_set_layout(self.set_layout, None);
vk.destroy_fence(self.fence, None);
vk.destroy_query_pool(self.queries, None);
vk.destroy_command_pool(self.pool, None);
}
}
}
+59
View File
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: MIT
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/sysctl.h>
extern int nmb_acceptance(const char *, const char *, const char *);
extern void nmb_acceptance_describe(void);
static int primed(const char *path) {
char boot[64] = {0}, token[80], *line = NULL;
size_t size = sizeof boot, capacity = 0;
if (sysctlbyname("kern.bootsessionuuid", boot, &size, NULL, 0)) return 0;
boot[sizeof boot - 1] = 0;
snprintf(token, sizeof token, " boot=%s ", boot);
FILE *file = fopen(path, "r");
if (!file) return 0;
int region = 0, complete = 0;
while (getline(&line, &capacity, file) >= 0) {
if (!strncmp(line, "list ", 5)) {
region = strstr(line, token) && strstr(line, " scope=2 ");
complete = 0;
} else if (!strncmp(line, "pass ", 5)) {
complete = region && strstr(line, token) && strstr(line, " scope=2 ") && strstr(line, " complete=1 ");
}
}
free(line);
fclose(file);
if (complete) setenv("NATIVE_METAL_BOOT_SESSION", boot, 1);
return complete;
}
int main(int argc, char **argv) {
const char *loader = NULL, *cache = NULL, *prime = NULL, *mode = "all";
if (argc == 2 && !strcmp(argv[1], "--describe")) {nmb_acceptance_describe(); return 0;}
for (int i = 1; i < argc; ++i) {
if (i + 1 >= argc) goto usage;
if (!strcmp(argv[i], "--loader")) loader = argv[++i];
else if (!strcmp(argv[i], "--cache")) cache = argv[++i];
else if (!strcmp(argv[i], "--prime-log")) prime = argv[++i];
else if (!strcmp(argv[i], "--mode")) mode = argv[++i];
else goto usage;
}
if (!loader || !cache || !prime) goto usage;
if (strcmp(mode, "all") && strcmp(mode, "numeric") && strcmp(mode, "calibrate")) goto usage;
if (!primed(prime)) {
fprintf(stderr, "Current boot whole-system shader priming is incomplete: %s\n", prime);
return 69;
}
if (!strcmp(mode, "all")) {
int result = nmb_acceptance(loader, cache, "numeric");
if (result) return result;
return nmb_acceptance(loader, cache, "calibrate");
}
return nmb_acceptance(loader, cache, mode);
usage:
fprintf(stderr, "gpu-acceptance --loader PATH --cache DIR --prime-log PATH [--mode numeric|calibrate|all]\n");
return 64;
}
+38
View File
@@ -0,0 +1,38 @@
// SPDX-License-Identifier: MIT
// Replay the captured macOS 27.3 RenderBox alpha stage with controlled framebuffer values.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <dlfcn.h>
#include <cstdint>
#include <cstring>
#include <cstdio>
#include <vector>
static id (*libraryWithAir)(const void *,size_t);
#include <cmath>
static id<MTLLibrary> air(const char *path) {
NSData *d=[NSData dataWithContentsOfFile:@(path)];uint32_t offset=0,length=0;if(d.length<20)return nil;
memcpy(&offset,(const uint8_t *)d.bytes+8,4);memcpy(&length,(const uint8_t *)d.bytes+12,4);
if(offset>d.length || length>d.length-offset)return nil;
return libraryWithAir((const uint8_t *)d.bytes+offset,length);
}
int main(int argc,char **argv) {@autoreleasepool {
if(argc!=5)return 2;id<MTLDevice> device=MTLCreateSystemDefaultDevice();if(!device)return 2;
void *plugin=dlopen("/Library/GPUBundles/NativeAdrenoMetal.bundle/Contents/MacOS/NativeAdrenoMetal",RTLD_LAZY|RTLD_NOLOAD);
libraryWithAir=(id (*)(const void *,size_t))dlsym(plugin?plugin:RTLD_DEFAULT,"native_bridge_metal_library_with_air");if(!libraryWithAir){fprintf(stderr,"Metal bridge AIR entry unavailable\n");return 2;}
printf("RenderBox alpha control device=%s\n",device.name.UTF8String);
auto vl=air(argv[1]),fl=air(argv[2]);if(!vl || !fl){fprintf(stderr,"AIR creation failed\n");return 1;}
NSData *globals=[NSData dataWithContentsOfFile:@(argv[3])],*gradient=[NSData dataWithContentsOfFile:@(argv[4])];
uint32_t state[4]={0xe80524,0x24000,0xf473,0x6c};auto fc=[MTLFunctionConstantValues new];[fc setConstantValue:state type:MTLDataTypeUInt4 atIndex:0];
NSError *error=nil;auto pd=[MTLRenderPipelineDescriptor new];pd.vertexFunction=[vl newFunctionWithName:@"alpha_effect_vertex"];pd.fragmentFunction=[fl newFunctionWithName:@"alpha_effect_fragment" constantValues:fc error:&error];pd.colorAttachments[0].pixelFormat=MTLPixelFormatRGBA16Float;pd.colorAttachments[1].pixelFormat=MTLPixelFormatRG32Float;pd.colorAttachments[1].writeMask=MTLColorWriteMaskNone;
auto pipeline=[device newRenderPipelineStateWithDescriptor:pd error:&error];if(!pipeline){fprintf(stderr,"pipeline: %s | %s\n",error.description.UTF8String,"");return 1;}
auto td=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA16Float width:230 height:124 mipmapped:NO];td.usage=MTLTextureUsageRenderTarget|MTLTextureUsageShaderRead;auto color=[device newTextureWithDescriptor:td];td.pixelFormat=MTLPixelFormatRG32Float;auto coverage=[device newTextureWithDescriptor:td];
td.width=td.height=32;td.pixelFormat=MTLPixelFormatRGBA8Unorm;auto noise=[device newTextureWithDescriptor:td];uint8_t n[32*32*4];memset(n,127,sizeof n);[noise replaceRegion:MTLRegionMake2D(0,0,32,32) mipmapLevel:0 withBytes:n bytesPerRow:128];
uint16_t indices[4]={1,0,2,3};auto index=[device newBufferWithBytes:indices length:sizeof indices options:MTLResourceStorageModeShared];auto queue=[device newCommandQueue];
unsigned failed=0;
for(double red : {-0.25,0.,0.05,0.15,0.5,1.,1.5}) {
auto cb=[queue commandBuffer];auto pass=[MTLRenderPassDescriptor renderPassDescriptor];pass.colorAttachments[0].texture=color;pass.colorAttachments[0].loadAction=MTLLoadActionClear;pass.colorAttachments[0].storeAction=MTLStoreActionStore;pass.colorAttachments[0].clearColor=MTLClearColorMake(red,.2,.3,1);pass.colorAttachments[1].texture=coverage;pass.colorAttachments[1].loadAction=MTLLoadActionClear;pass.colorAttachments[1].storeAction=MTLStoreActionStore;pass.colorAttachments[1].clearColor=MTLClearColorMake(1,.5,0,0);
auto e=[cb renderCommandEncoderWithDescriptor:pass];[e setRenderPipelineState:pipeline];[e setScissorRect:(MTLScissorRect){124,0,106,106}];[e setVertexBytes:globals.bytes length:globals.length atIndex:0];[e setFragmentBytes:globals.bytes length:globals.length atIndex:0];[e setFragmentBytes:gradient.bytes length:gradient.length atIndex:3];[e setFragmentTexture:noise atIndex:3];[e drawIndexedPrimitives:MTLPrimitiveTypeTriangleStrip indexCount:4 indexType:MTLIndexTypeUInt16 indexBuffer:index indexBufferOffset:0];[e endEncoding];[cb commit];[cb waitUntilCompleted];if(cb.status!=MTLCommandBufferStatusCompleted){fprintf(stderr,"render: %s\n",cb.error.description.UTF8String);return 1;}
std::vector<__fp16> pixels(230*124*4);[color getBytes:pixels.data() bytesPerRow:230*8 fromRegion:MTLRegionMake2D(0,0,230,124) mipmapLevel:0];unsigned bad=0;for(__fp16 v:pixels)bad+=!std::isfinite(float(v));failed+=bad!=0;size_t at=(50*230+150)*4;printf("source=%g nonfinite=%u sample=%g,%g,%g,%g\n",red,bad,float(pixels[at]),float(pixels[at+1]),float(pixels[at+2]),float(pixels[at+3]));
}
return failed?1:0;
}}