diff --git a/NativeAdrenoMetalPlugin.mm b/NativeAdrenoMetalPlugin.mm index 50852ff..44f0c2d 100644 --- a/NativeAdrenoMetalPlugin.mm +++ b/NativeAdrenoMetalPlugin.mm @@ -350,6 +350,10 @@ static id deviceForPort(unsigned port) { else unsetenv("SHENG_GPU_TRACE"); // metal-flow=1 enables the lowering thread, commit submission and the warm pass; =0 states the default. Either logs the FL counts // (native_bridge_metal_open), so two runs compare without diag=1's own cost. + // gpu-purge=0: the bridge discards nothing of what callers declare Volatile (NativeMetalPurge.rs), for a comparison on the + // same image. Read by the bridge as the device opens. + if (bootArgSet("gpu-purge=0")) setenv("NATIVE_METAL_PURGE", "0", 1); + else unsetenv("NATIVE_METAL_PURGE"); if (bootArgSet("metal-flow=0")) setenv("NATIVE_METAL_FLOW", "0", 1); else if (bootArgSet("metal-flow=1")) setenv("NATIVE_METAL_FLOW", "1", 1); else unsetenv("NATIVE_METAL_FLOW"); diff --git a/NativeBridgeMetalFrontend.h b/NativeBridgeMetalFrontend.h index 25ed8e5..9a91ea7 100644 --- a/NativeBridgeMetalFrontend.h +++ b/NativeBridgeMetalFrontend.h @@ -32,5 +32,14 @@ bool native_bridge_metal_work_stats(uint64_t out[10]); size_t native_bridge_metal_benchmark(id device, const char *metallib, unsigned pairs, char *out, size_t size); // Lets go of every held IOSurface import whose surface only the import still retains (also done at each command buffer's completion). void native_bridge_metal_trim_surface_imports(void); +// Purgeable memory (bridge/NativeMetalPurge.rs). Discards the buffers, textures and heaps their caller declared Volatile or Empty +// that nothing uses: cause 0 all of them (what the memory pressure handler does), cause 1 those idle for idleMs (what the age timer +// does, with the runtime's spare block age). Returns the bytes discarded; nothing with the boot argument gpu-purge=0. +uint64_t native_bridge_metal_purge(uint32_t cause, uint64_t idleMs); +// The bridge's sixteen purge counts (nmb_device_purge_counts). +bool native_bridge_metal_purge_counts(uint64_t out[16]); +// What a memory pressure event does: held surface imports, discardable resources, then the runtime's kept memory. Returns the bytes +// of the resources discarded. +uint64_t native_bridge_metal_relieve_memory(void); } #endif diff --git a/NativeBridgeMetalFrontend.mm b/NativeBridgeMetalFrontend.mm index eb92969..b85cdeb 100644 --- a/NativeBridgeMetalFrontend.mm +++ b/NativeBridgeMetalFrontend.mm @@ -893,33 +893,49 @@ static void answerMissing(NSInvocation *invocation) { @interface NBObject : NSObject @property(atomic,copy) NSString *label; +// A command buffer names the object from now on (YES) until it completed or went (NO): a buffer or a texture is not discarded +// meanwhile (nmb_buffer_hold). Nothing for any other object. +- (void)purgeHold:(BOOL)hold; @end // What the caller asked of setPurgeableState: (MTLResource, MTLHeap), for the telemetry's PG row: how much memory the process has declared -// discardable. The runtime keeps the state and never discards (its attributes.rs), so these bytes are what a purge could give back. -// [0..2] requests on buffers, textures, heaps; [3] bytes volatile now, [4] bytes empty now (an object freed in either state stays -// counted: an upper bound); [5] bytes ever made volatile or empty, [6] bytes made non-volatile again, [7] the largest such object. +// discardable. [0..2] requests on buffers, textures, heaps; [3] bytes of the objects whose state is Volatile now, [4] of those whose +// state is Empty now; [5] bytes ever made volatile or empty, [6] bytes made non-volatile again, [7] the largest such object. An +// object carries which of [3] and [4] counts it (`counted`), so the two are exact for the states the caller was last told: one the +// bridge discarded since (NativeMetalPurge.rs) stays under Volatile until the caller asks again, and the bytes discarded now are the +// PGb row's. static std::atomic gPurgeable[8]; -static void notePurgeable(unsigned kind,MTLPurgeableState before,MTLPurgeableState asked,uint64_t bytes) { - gPurgeable[kind].fetch_add(1,std::memory_order_relaxed); - if(asked==MTLPurgeableStateKeepCurrent)return; - const MTLPurgeableState after=before==MTLPurgeableStateEmpty && asked==MTLPurgeableStateVolatile?MTLPurgeableStateEmpty:asked; - if(after==before)return; - const auto slot=[](MTLPurgeableState state){return state==MTLPurgeableStateVolatile?3:state==MTLPurgeableStateEmpty?4:-1;}; - if(slot(before)>=0) { - // never below zero: the count is an estimate (see above) - uint64_t now=gPurgeable[slot(before)].load(std::memory_order_relaxed); - while(!gPurgeable[slot(before)].compare_exchange_weak(now,now>bytes?now-bytes:0,std::memory_order_relaxed)){} - } - if(slot(after)>=0) { - gPurgeable[slot(after)].fetch_add(bytes,std::memory_order_relaxed); - if(slot(before)<0)gPurgeable[5].fetch_add(bytes,std::memory_order_relaxed); +static void countPurgeable(uint8_t &counted,uint8_t slot,uint64_t bytes) { + if(counted==slot)return; + if(counted)gPurgeable[counted].fetch_sub(bytes,std::memory_order_relaxed); + if(slot)gPurgeable[slot].fetch_add(bytes,std::memory_order_relaxed); + if(slot && !counted) { + gPurgeable[5].fetch_add(bytes,std::memory_order_relaxed); uint64_t big=gPurgeable[7].load(std::memory_order_relaxed); while(bytes>big && !gPurgeable[7].compare_exchange_weak(big,bytes,std::memory_order_relaxed)){} - } else if(slot(before)>=0) gPurgeable[6].fetch_add(bytes,std::memory_order_relaxed); + } else if(!slot && counted) gPurgeable[6].fetch_add(bytes,std::memory_order_relaxed); + counted=slot; +} +static void armPurgeAge(); +// The bridge's answer to setPurgeableState: (0: the call failed) as Metal's, counted. `before` is the state before the call (the +// state now for a query); the state after is the one asked, but Volatile asked of an Empty object leaves it Empty, as Metal does. +static MTLPurgeableState notePurgeable(unsigned kind,uint64_t answer,MTLPurgeableState asked,uint64_t bytes,uint8_t &counted,const char *what) { + if(!answer){NBLOG(@"native-metal: %s setPurgeableState refused: %@",what,lastError());return (MTLPurgeableState)0;} + // The contents were discarded and the object could not be placed again: it stays without memory, the answer is still Empty, and + // its next use tries again and fails as an allocation fails. + if(answer&NMB_PURGE_AGAIN_FAILED)NBLOG(@"native-metal: %s could not be placed again after its discard: %@",what,lastError()); + const MTLPurgeableState before=(MTLPurgeableState)(answer&0xff); + gPurgeable[kind].fetch_add(1,std::memory_order_relaxed); + const MTLPurgeableState after=asked==MTLPurgeableStateKeepCurrent || (answer&NMB_PURGE_AGAIN_FAILED) || + (before==MTLPurgeableStateEmpty && asked==MTLPurgeableStateVolatile)?before:asked; + countPurgeable(counted,after==MTLPurgeableStateVolatile?3:after==MTLPurgeableStateEmpty?4:0,bytes); + // An object that may be discarded by age: the timer runs while there is one. + if(after==MTLPurgeableStateVolatile || after==MTLPurgeableStateEmpty)armPurgeAge(); + return before; } @implementation NBObject - (id)device {return gDeviceObject;} +- (void)purgeHold:(BOOL)hold {(void)hold;} - (NSMethodSignature *)methodSignatureForSelector:(SEL)selector { NSMethodSignature *signature=[super methodSignatureForSelector:selector]; return signature?signature:missingSignature(self,selector); @@ -976,13 +992,19 @@ static void releaseBlock(void *context,void *pointer,uint64_t length) { void (^deallocator)(void *,NSUInteger)=(__bridge_transfer void (^)(void *,NSUInteger))context;deallocator(pointer,(NSUInteger)length); } static void releaseSurface(void *context,void *,uint64_t) {DetailSpan timing(7,(uintptr_t)context);gSurfaceImportsReleased++;CFRelease((IOSurfaceRef)context);} -@implementation NBBuffer +@implementation NBBuffer { + uint8_t _purgeCounted; // which of gPurgeable[3] and [4] counts its bytes (0: neither) +} - (BOOL)isAliasable {return nmb_buffer_is_aliasable(_handle);} - (void)makeAliasable {if(!nmb_buffer_make_aliasable(_handle))NBLOG(@"native-metal: buffer makeAliasable: %@",lastError());} - (void)dealloc { DetailSpan timing(4,(uintptr_t)_handle); + if(_purgeCounted)countPurgeable(_purgeCounted,0,nmb_buffer_length(_handle)); nmb_buffer_release(_handle); + // A heap whose last resource went may be discarded now: the age timer runs again if the heap waits for it. + if(_parentHeap)armPurgeAge(); } +- (void)purgeHold:(BOOL)hold {if(hold)nmb_buffer_hold(_handle);else nmb_buffer_unhold(_handle);} - (NSUInteger)length {return nmb_buffer_length(_handle);} - (void *)contents {return nmb_buffer_contents(_handle);} - (uint64_t)gpuAddress {return nmb_buffer_gpu_address(_handle);} @@ -992,10 +1014,7 @@ static void releaseSurface(void *context,void *,uint64_t) {DetailSpan timing(7,( - (MTLHazardTrackingMode)hazardTrackingMode {return (MTLHazardTrackingMode)((_options&MTLResourceHazardTrackingModeMask)>>MTLResourceHazardTrackingModeShift);} - (void)didModifyRange:(NSRange)range {(void)range;} - (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { - const uint64_t answer=nmb_buffer_set_purgeable_state(_handle,state); - if(!answer)NBLOG(@"native-metal: buffer setPurgeableState refused: %@",lastError()); - else notePurgeable(0,(MTLPurgeableState)answer,state,[self allocatedSize]); - return (MTLPurgeableState)answer; + return notePurgeable(0,nmb_buffer_set_purgeable_state(_handle,state),state,[self allocatedSize],_purgeCounted,"buffer"); } - (NSUInteger)allocatedSize {return nmb_buffer_length(_handle);} - (IOSurfaceRef)iosurface {return nullptr;} // a buffer is never the backing of a surface here (surfaces back textures) @@ -1037,6 +1056,51 @@ extern "C" void native_bridge_metal_trim_surface_imports(void) {trimHeldImports( // The system's memory pressure events this process got, for the telemetry's PE row: [0] warnings, [1] critical ones, [2] the blocks // the runtime gave back at them. static std::atomic gPressureEvents[3]; +// Discards the buffers, textures and heaps the caller declared Volatile or Empty that nothing uses (NativeMetalPurge.rs): all of +// them for cause 0, the system's memory pressure signal, as the kernel purges volatile memory when it needs pages; for cause 1 those +// idle for idleMs. The bytes discarded. Nothing with the boot argument gpu-purge=0. +// With diag=1, a process other than the render server (whose table has the PGb and PGc rows) writes a line for each pass that +// discarded something: the pass's MiB and cause, then the process's totals (nmb_device_purge_counts): MiB discarded now, at +// pressure and by age, placed again by a NonVolatile request and by another use (a caller that used a discarded resource without +// setting it NonVolatile first), Empty answers, MiB never discardable because a pointer or address was handed out, placements again +// that failed. At most one line a pass, and the age timer passes twice an age. +extern "C" uint64_t native_bridge_metal_purge(uint32_t cause,uint64_t idleMs) { + if(!gDevice)return 0; + const uint64_t discarded=nmb_device_purge(gDevice,cause,idleMs); + if(discarded && diagOn() && !isRenderServer()) { + uint64_t pg[16]={}; + (void)nmb_device_purge_counts(gDevice,pg); + os_log(OS_LOG_DEFAULT,"NativeAdrenoMetal purge: %lluK by %{public}s | out %lluM pressure %lluM age %lluM | again %lluM use %lluM empty %llu | " + "pinned %lluM fail %llu",(unsigned long long)(discarded>>10),cause?"age":"pressure",(unsigned long long)(pg[9]>>20), + (unsigned long long)(pg[1]>>20),(unsigned long long)(pg[3]>>20),(unsigned long long)(pg[5]>>20),(unsigned long long)(pg[7]>>20), + (unsigned long long)pg[8],(unsigned long long)(pg[11]>>20),(unsigned long long)pg[15]); + } + return discarded; +} +extern "C" bool native_bridge_metal_purge_counts(uint64_t out[16]) {return gDevice && nmb_device_purge_counts(gDevice,out);} +// What a memory pressure event gives back: the surface imports held between frames, what the caller declared discardable (first, +// so the blocks it empties go with the rest), then the runtime's parked objects and its spare and kept empty blocks (bridge patches +// 0021 and 0034), as Metal's resource cache gives memory back. The bytes discarded. +extern "C" uint64_t native_bridge_metal_relieve_memory(void) { + trimHeldImports(true); + const uint64_t discarded=native_bridge_metal_purge(0,0); + if(gDevice)gPressureEvents[2]+=nmb_device_release_kept_memory(gDevice); + return discarded; +} +// Pressure events are rare on a system whose GPU memory is wired (one warning in the five boots of run61's log archive, with 3.1 of +// 4.72 GiB wired), so what stays Volatile or Empty and idle for the runtime's spare block age goes by age too. The timer runs only +// while something may still go: armed by a setPurgeableState: that leaves an object Volatile or Empty, and again by a pass that +// left some. Two passes an age, so an object goes at most half an age late. +static void armPurgeAge() { + static std::atomic armed; + if(!gDevice || !nmb_device_purge_pending(gDevice) || armed.exchange(true))return; + const uint64_t age=nmb_purge_age_ms(); + dispatch_after(dispatch_time(DISPATCH_TIME_NOW,(int64_t)(age/2)*(int64_t)NSEC_PER_MSEC),dispatch_get_global_queue(QOS_CLASS_UTILITY,0),^{ + armed.store(false); + native_bridge_metal_purge(1,age); + armPurgeAge(); + }); +} static void watchImportPressure() { static dispatch_once_t once; // The source lives as long as the process: a source whose last reference goes is taken off its event queue, and this file's @@ -1046,11 +1110,9 @@ static void watchImportPressure() { source=dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE,0, DISPATCH_MEMORYPRESSURE_WARN|DISPATCH_MEMORYPRESSURE_CRITICAL,dispatch_get_global_queue(QOS_CLASS_UTILITY,0)); if(!source)return; - // The runtime's spare and kept empty memory blocks go too (bridge patch 0021), as Metal's resource cache gives memory back. dispatch_source_set_event_handler(source,^{ gPressureEvents[dispatch_source_get_data(source)&DISPATCH_MEMORYPRESSURE_CRITICAL?1:0]++; - trimHeldImports(true); - if(gDevice)gPressureEvents[2]+=nmb_device_release_kept_memory(gDevice); + native_bridge_metal_relieve_memory(); }); dispatch_activate(source); }); @@ -1203,7 +1265,10 @@ static bool textureDescriptor(MTLTextureDescriptor *d,NmbTextureDescriptor *out) } return true; } -@implementation NBTexture +@implementation NBTexture { + uint8_t _purgeCounted; // as NBBuffer's +} +- (void)purgeHold:(BOOL)hold {if(hold)nmb_texture_hold(_handle);else nmb_texture_unhold(_handle);} - (NSUInteger)allocatedSize {return nmb_texture_allocated_size(_handle);} - (BOOL)isAliasable {return nmb_texture_is_aliasable(_handle);} - (void)makeAliasable {if(!nmb_texture_make_aliasable(_handle))NBLOG(@"native-metal: texture makeAliasable: %@",lastError());} @@ -1211,7 +1276,13 @@ static bool textureDescriptor(MTLTextureDescriptor *d,NmbTextureDescriptor *out) const uint64_t s=_descriptor.swizzle; return MTLTextureSwizzleChannelsMake((MTLTextureSwizzle)(s&255),(MTLTextureSwizzle)((s>>8)&255),(MTLTextureSwizzle)((s>>16)&255),(MTLTextureSwizzle)((s>>24)&255)); } -- (void)dealloc {DetailSpan timing(5,(uintptr_t)_handle);nmb_texture_release(_handle);if(_surface){if(_counted)IOSurfaceDecrementUseCount(_surface);CFRelease(_surface);}} +- (void)dealloc { + DetailSpan timing(5,(uintptr_t)_handle); + if(_purgeCounted)countPurgeable(_purgeCounted,0,nmb_texture_allocated_size(_handle)); + nmb_texture_release(_handle);if(_surface){if(_counted)IOSurfaceDecrementUseCount(_surface);CFRelease(_surface);} + // What it was made of (a viewed texture, a buffer, a heap) may be discarded now: the age timer runs again if that waits for it. + if(_parent || _parentHeap || _buffer)armPurgeAge(); +} - (void)countUse {if(_surface && !_counted && gCountsSurfaceUse){IOSurfaceIncrementUseCount(_surface);_counted=YES;}} - (MTLTextureType)textureType {return (MTLTextureType)_descriptor.type;} - (MTLPixelFormat)pixelFormat {return (MTLPixelFormat)(_pvrtc?_pvrtc:_descriptor.format);} @@ -1235,10 +1306,7 @@ static bool textureDescriptor(MTLTextureDescriptor *d,NmbTextureDescriptor *out) - (id)parentTexture {return _parent;} - (NSUInteger)rotation {return _descriptor.rotation;} - (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { - const uint64_t answer=nmb_texture_set_purgeable_state(_handle,state); - if(!answer)NBLOG(@"native-metal: texture setPurgeableState refused: %@",lastError()); - else notePurgeable(1,(MTLPurgeableState)answer,state,[self allocatedSize]); - return (MTLPurgeableState)answer; + return notePurgeable(1,nmb_texture_set_purgeable_state(_handle,state),state,[self allocatedSize],_purgeCounted,"texture"); } NB_RESOURCE_SPI - (id)view:(NmbTexture *)handle { @@ -1886,6 +1954,9 @@ static void dumpSurfacesOnce(NSSet *textures,NSSet *owners) { // Runtime handles retain GPU storage, but only the Objective-C owner retains // a no-copy deallocator, its memory owner and texture/heap ancestry. @property(nonatomic,strong) NSMutableSet *resourceOwners; +// Adds a resource a command names to resourceOwners, and holds it against a discard until the command buffer completed or went +// (purgeHold:). Called on the thread that makes the call, before the command reaches the runtime. +- (void)own:(id)resource; // The sampled sources watched for a rewrite while the GPU could read them (gWatchSources), kept alive until completion. @property(nonatomic,strong) NSMutableSet *sampled; - (void)watchTexture:(NBTexture *)texture; @@ -2320,7 +2391,7 @@ static uint64_t encodeCommandOf(id object) { // Stage 0 compute, 1 vertex, 2 fragment (the bridge's numbering). - (void)buffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { if(buffer && ![(id)buffer isKindOfClass:[NBBuffer class]]){[_commandBuffer refuse:@"foreign buffer"];return;} - if(buffer)[_commandBuffer.resourceOwners addObject:buffer]; + if(buffer)[_commandBuffer own:buffer]; if(gDrawStats)[self noteState:kDSBuffer key:stage<<8|(uint32_t)index object:(__bridge const void *)buffer value:offset]; if(dumpMaybe(_commandBuffer))[self dumpBind:buffer?@[buffer,@(offset)]:nil index:index stage:stage]; ENCODE([self check:nmb_encoder_set_buffer(self->_handle,stage,buffer?[(NBBuffer *)buffer handle]:nullptr,offset,(uint32_t)index) what:"setBuffer"]); @@ -2336,7 +2407,7 @@ static uint64_t encodeCommandOf(id object) { [NSString stringWithFormat:@"kernel %u texture %lu",_commandBuffer.dumpKernels,(unsigned long)index]); if(_commandBuffer.dumpTargets.count>before)[_dumpEntries addObject:_commandBuffer.dumpTargets.lastObject]; } - if(texture){[_commandBuffer.resourceOwners addObject:texture];} + if(texture){[_commandBuffer own:texture];} [self dumpBind:texture index:index stage:stage+4]; if(texture && [(NBTexture *)texture surface])[_commandBuffer.surfaces addObject:texture]; if(texture && gWatchSources)[_commandBuffer watchTexture:(NBTexture *)texture]; @@ -2415,7 +2486,7 @@ static uint64_t encodeCommandOf(id object) { - (void)dispatchThreadgroups:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:false];} - (void)dispatchThreadgroupsWithIndirectBuffer:(id)buffer indirectBufferOffset:(NSUInteger)offset threadsPerThreadgroup:(MTLSize)group { if(![(id)buffer isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:@"foreign indirect buffer"];return;} - [self.commandBuffer.resourceOwners addObject:buffer]; + [self.commandBuffer own:buffer]; gDispatches++;self.commandBuffer->encodedWork.dispatches++; const std::array g={group.width,group.height,group.depth}; ENCODE([self check:nmb_compute_dispatch_indirect(self.handle,[(NBBuffer *)buffer handle],offset,g.data()) what:"indirect dispatch"]); @@ -2567,7 +2638,7 @@ static uint64_t encodeCommandOf(id object) { - (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o instanceCount:(NSUInteger)i baseVertex:(NSInteger)v baseInstance:(NSUInteger)bi { if(![(id)b isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:@"foreign index buffer"];return;} - [self.commandBuffer.resourceOwners addObject:b]; + [self.commandBuffer own:b]; if([self skipDraw])return; gDraws++;self.commandBuffer->encodedWork.draws++;if(gDrawStats)[self noteDraw]; if(dumpMaybe(self.commandBuffer) && dumpShown(self.dumpDraws+1)) { @@ -2594,7 +2665,7 @@ static uint64_t encodeCommandOf(id object) { - (BOOL)own:(id)b what:(NSString *)what { if(!b)return YES; if(![(id)b isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:[@"foreign " stringByAppendingString:what]];return NO;} - [self.commandBuffer.resourceOwners addObject:b];return YES; + [self.commandBuffer own:b];return YES; } - (void)setTessellationFactorBuffer:(id)b offset:(NSUInteger)o instanceStride:(NSUInteger)stride { if(![self own:b what:@"tessellation factor buffer"])return; @@ -2647,14 +2718,14 @@ static uint64_t encodeCommandOf(id object) { #define NB_OBJECTS(...) ((__unsafe_unretained id[]){__VA_ARGS__}) - (bool)buffers:(__unsafe_unretained id *)objects count:(size_t)count { for(size_t i=0;i)bufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options offset:(uint64_t)offset { NmbBuffer *handle=nmb_heap_new_buffer(_handle,length,options,offset); @@ -4738,8 +4818,10 @@ static void logClientWork() { // the process holds: the bytes of the pooled blocks against the bytes placed in them (the rest is room the pools keep and // fragmentation), the kept empty blocks, the dedicated blocks and the imports; then what the process declared discardable // (setPurgeableState:, notePurgeable): requests on buffers, textures and heaps, the bytes volatile and empty now, ever made so, -// made non-volatile again, and the largest such object. RenderBox keeps its recycled buffers, textures and heaps volatile, so the -// volatile bytes are what a purge could give back. At most one line per doubling: a 250 ms check that writes nothing else. +// made non-volatile again, and the largest such object; then what was done with it (nmb_device_purge_counts): the bytes discarded +// now, discarded at memory pressure and by age so far, placed again by a NonVolatile request and by another use, and those never +// discardable because a CPU pointer or GPU address was handed out. RenderBox keeps its recycled buffers, textures and heaps volatile. At most one line per doubling: a +// 250 ms check that writes nothing else. static void logClientMemory() { if(!diagOn() || isRenderServer())return; static dispatch_source_t timer; @@ -4747,19 +4829,23 @@ static void logClientMemory() { dispatch_source_set_timer(timer,dispatch_time(DISPATCH_TIME_NOW,250*NSEC_PER_MSEC),250*NSEC_PER_MSEC,100*NSEC_PER_MSEC); dispatch_source_set_event_handler(timer,^{ static uint64_t next=64ull<<20; - uint64_t mc[10]={}; + uint64_t mc[10]={},pg[16]={}; if(!nmb_device_memory_counts(gDevice,mc))return; const uint64_t blocks=mc[0]+mc[2]+mc[3]; if(blocks>20),(unsigned long long)(mc[1]>>20),(unsigned long long)(mc[2]>>20),(unsigned long long)(mc[3]>>20), (unsigned long long)(mc[4]>>20),(unsigned long long)mc[6],(unsigned long long)mc[7], (unsigned long long)gPurgeable[0].load(),(unsigned long long)gPurgeable[1].load(),(unsigned long long)gPurgeable[2].load(), (unsigned long long)(gPurgeable[3].load()>>20),(unsigned long long)(gPurgeable[4].load()>>20), (unsigned long long)(gPurgeable[5].load()>>20),(unsigned long long)(gPurgeable[6].load()>>20), - (unsigned long long)(gPurgeable[7].load()>>10)); + (unsigned long long)(gPurgeable[7].load()>>10), + (unsigned long long)(pg[9]>>20),(unsigned long long)(pg[1]>>20),(unsigned long long)(pg[3]>>20), + (unsigned long long)(pg[5]>>20),(unsigned long long)(pg[7]>>20),(unsigned long long)(pg[11]>>20)); }); dispatch_resume(timer); } @@ -5042,6 +5128,19 @@ extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) { row("PG req %llu/%llu/%llu | vol %lluM empty %lluM | to %lluM back %lluM big %lluM",(ull)gPurgeable[0].load(), (ull)gPurgeable[1].load(),(ull)gPurgeable[2].load(),(ull)(gPurgeable[3].load()>>20),(ull)(gPurgeable[4].load()>>20), (ull)(gPurgeable[5].load()>>20),(ull)(gPurgeable[6].load()>>20),(ull)(gPurgeable[7].load()>>20)); + // What was done with it (nmb_device_purge_counts): MiB discarded and not placed again now; MiB discarded at memory pressure and + // by age so far; MiB placed again by a NonVolatile request and by another use; answers of Empty for discarded contents. PGc: of + // the MiB that became Volatile or Empty, those never discardable because a CPU pointer or GPU address was handed out; what the + // last pass left, in MiB, because it was in use, had a view or a placed resource, or was not idle long enough; placements again + // that failed. All zero with the boot argument gpu-purge=0. + { + uint64_t pg[16]={}; + (void)nmb_device_purge_counts(gDevice,pg); + row("PGb out %lluM | pres %lluM age %lluM | again %lluM use %lluM | empty %llu",(ull)(pg[9]>>20),(ull)(pg[1]>>20),(ull)(pg[3]>>20), + (ull)(pg[5]>>20),(ull)(pg[7]>>20),(ull)pg[8]); + row("PGc decl %lluM pin %lluM | left use %lluM tie %lluM young %lluM | fail %llu",(ull)(pg[10]>>20),(ull)(pg[11]>>20), + (ull)(pg[12]>>20),(ull)(pg[13]>>20),(ull)(pg[14]>>20),(ull)pg[15]); + } // The system's memory pressure events this process got (warnings, critical ones) and the blocks the runtime gave back at them; // then the surface imports held between frames and their MiB, which go back at such an event, and how often Turnip emptied its BO // cache (at its own pressure events and when an allocation failed). diff --git a/bridge/NativeMetalBridge.rs b/bridge/NativeMetalBridge.rs index 8947759..eeea0b4 100644 --- a/bridge/NativeMetalBridge.rs +++ b/bridge/NativeMetalBridge.rs @@ -10,6 +10,8 @@ #[path = "NativeMetalObjects.rs"] pub mod objects; +#[path = "NativeMetalPurge.rs"] +pub mod purge; // The system allocator: mimalloc reserved its arenas with large mmaps, and on iPadOS 27 the kernel stopped the render server and // SpringBoard with an exception inside that mmap (unix_mmap_prim_aligned from mi_arena_reserve), so the shell resprung. diff --git a/bridge/NativeMetalObjects.h b/bridge/NativeMetalObjects.h index cc424cf..124c631 100644 --- a/bridge/NativeMetalObjects.h +++ b/bridge/NativeMetalObjects.h @@ -73,6 +73,21 @@ bool nmb_device_pipeline_counts(const NmbDevice *, uint64_t out[8]); size_t nmb_stage_ring_read(uint64_t from, void *out, size_t max, uint64_t *next); /* Frees the runtime's spare dedicated blocks and kept empty pooled blocks (memory pressure); returns how many went. */ size_t nmb_device_release_kept_memory(const NmbDevice *); +/* Purgeable memory (NativeMetalPurge.rs). A buffer or texture the device made of its own memory, or a heap, whose state is Volatile or + * Empty is discarded while nothing uses it: no command buffer that named it is recording or in flight (the holds below), nothing made + * of it lives (a view, a texture over it, a resource placed in it), neither its CPU pointer nor its GPU address was handed out, and no + * call is inside it. Its handle stays valid: NonVolatile places it again and answers Empty, and any other use of it places it again + * with zeroed memory. NATIVE_METAL_PURGE=0 in the environment when the device opens discards nothing. + * nmb_device_purge: cause 0 (memory pressure) discards every such resource, cause 1 (age) those idle for idle_ms; the bytes discarded. + * nmb_device_purge_pending: the resources a later call may still discard. nmb_purge_age_ms: the age to pass (the runtime's spare + * block age). nmb_device_purge_counts, sixteen words: resources and bytes discarded by pressure, then by age; resources and bytes + * placed again by a NonVolatile request, then by another use; Empty answers for discarded contents; bytes discarded now; bytes that + * became Volatile or Empty so far and those of them never discardable (pointer or address handed out); the bytes the last call left + * because they were in use, tied to what was made of them, or not idle long enough; placements again that failed. */ +uint64_t nmb_device_purge(const NmbDevice *, uint32_t cause, uint64_t idle_ms); +uint64_t nmb_device_purge_pending(const NmbDevice *); +uint64_t nmb_purge_age_ms(void); +bool nmb_device_purge_counts(const NmbDevice *, uint64_t out[16]); // Call stacks that allocated device memory of 4 MiB or more, most bytes first: records of twelve words (bytes, allocations, ten return // addresses); returns how many were written, at most max. size_t nmb_allocation_sites(uint64_t *out, size_t max); @@ -108,9 +123,14 @@ NmbBuffer *nmb_buffer_new_no_copy(NmbDevice *, void *pointer, uint64_t length, u NmbBuffer *nmb_buffer_new_no_copy_releasing(NmbDevice *, void *pointer, uint64_t length, uint64_t options, void (*release)(void *context, void *pointer, uint64_t length), void *context); void nmb_buffer_release(NmbBuffer *); +/* The CPU pointer and the GPU address are the buffer's for its life: once either is handed out the buffer is never discarded. */ void *nmb_buffer_contents(const NmbBuffer *); uint64_t nmb_buffer_length(const NmbBuffer *); uint64_t nmb_buffer_gpu_address(const NmbBuffer *); +/* A command buffer names the resource from the hold until the unhold (its completion, or its release without a commit): the + * resource is not discarded meanwhile. The hold comes before the command that names the resource is encoded. */ +void nmb_buffer_hold(const NmbBuffer *); +void nmb_buffer_unhold(const NmbBuffer *); struct NmbTextureDescriptor { uint64_t type, format, width, height, depth, levels, samples, array_length, usage, options; @@ -139,9 +159,15 @@ NmbTexture *nmb_texture_new_on_surface(NmbBuffer *, const struct NmbTextureDescr NmbTexture *nmb_texture_new_linear(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image); /* whole: a view of every level and slice as format; else levels and slices are {start, count} and kind the view's texture type. */ NmbTexture *nmb_texture_new_view(NmbTexture *, uint64_t format, uint64_t kind, const uint64_t levels[2], const uint64_t slices[2], bool whole); -/* setPurgeableState: with Metal's raw state; the answer's raw state, 0 where it fails. */ +/* setPurgeableState: with Metal's raw state; the answer's raw state, 0 where it fails. A discarded resource answers Empty: to + * NonVolatile after it is placed again, to a query and to Volatile without being placed. Bit 8 (NMB_PURGE_AGAIN_FAILED) is set where + * placing it again failed: the low byte is still Empty, the reason is nmb_last_error, and the next use tries again. The heap's + * nmb_heap_set_purgeable_state answers the same way. */ +#define NMB_PURGE_AGAIN_FAILED 0x100ull uint64_t nmb_texture_set_purgeable_state(const NmbTexture *, uint64_t state); uint64_t nmb_buffer_set_purgeable_state(const NmbBuffer *, uint64_t state); +void nmb_texture_hold(const NmbTexture *); +void nmb_texture_unhold(const NmbTexture *); uint64_t nmb_texture_allocated_size(const NmbTexture *); uint64_t nmb_device_current_allocated_size(const NmbDevice *); bool nmb_texture_make_aliasable(const NmbTexture *); diff --git a/bridge/NativeMetalObjects.rs b/bridge/NativeMetalObjects.rs index 5cec1da..c7e3056 100644 --- a/bridge/NativeMetalObjects.rs +++ b/bridge/NativeMetalObjects.rs @@ -23,6 +23,7 @@ use metal2vulkan_runtime::{ Origin, PipelineOption, PipelineOptions, PurgeableState, PrivateCount, PrivateFlag, PrivateProperties, MAX_COLOR_ATTACHMENTS, Region, RenderCommandEncoder, RenderPassDescriptor, RenderPipelineState, ResourceOptions, SamplerState, ScissorRect, SharedEvent, SharedEventListener, Size, StoreAction, Texture, TextureDescriptor, Viewport, VisibilityResultMode, }; +use crate::purge::{Cause, Purge, Resource}; use std::cell::RefCell; use std::ffi::{c_char, c_void, CStr}; use std::path::PathBuf; @@ -34,7 +35,7 @@ thread_local! { static LAST: RefCell> = const { RefCell::new(Vec::new()) }; } -fn set_error(why: String) { +pub(crate) fn set_error(why: String) { LAST.with(|last| { let mut last = last.borrow_mut(); last.clear(); @@ -156,7 +157,7 @@ pub unsafe extern "C" fn nmb_function_constant_air(length: *mut usize) -> *const // ---- device ---- -pub struct NmbDevice(Device, std::sync::OnceLock, std::sync::Mutex); +pub struct NmbDevice(Device, std::sync::OnceLock, std::sync::Mutex, std::sync::Arc); impl NmbDevice { /// The runtime judges the private properties by the family rows once. @@ -278,7 +279,8 @@ pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_ if let Some(records) = records { device.warm(records.take_earlier(), u64::MAX); } - Ok(boxed(NmbDevice(device, std::sync::OnceLock::new(), std::sync::Mutex::default()))) + let purge = Purge::new(&device); + Ok(boxed(NmbDevice(device, std::sync::OnceLock::new(), std::sync::Mutex::default(), purge))) }) } @@ -577,6 +579,70 @@ pub extern "C" fn nmb_device_release_kept_memory(device: *const NmbDevice) -> us guard(0, || Ok(get(device)?.0.release_kept_memory())) } +/// Discard the device's resources that are Volatile or Empty and idle (NativeMetalPurge.rs): `cause` 0 for the system's memory +/// pressure signal, which takes every such resource, 1 for the age timer, which takes those idle for `idle_ms`. The bytes discarded. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_purge(device: *const NmbDevice, cause: u32, idle_ms: u64) -> u64 { + guard(0, || { + let cause = match cause { + 0 => Cause::Pressure, + 1 => Cause::Age, + _ => return Err(format!("purge cause {cause}")), + }; + Ok(get(device)?.3.sweep(cause, Duration::from_millis(idle_ms))) + }) +} + +/// How many of the device's resources a later sweep may still discard: Volatile or Empty with memory, in use or not idle long enough. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_purge_pending(device: *const NmbDevice) -> u64 { + guard(0, || Ok(get(device)?.3.pending() as u64)) +} + +/// The age an idle Volatile or Empty resource is discarded at, in milliseconds: the runtime's spare block age. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_purge_age_ms() -> u64 { + Purge::age().as_millis() as u64 +} + +/// What the device discarded and placed again, into the caller's sixteen words (`Purge::counts`). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_purge_counts(device: *const NmbDevice, out: *mut u64) -> bool { + guard(false, || { + if out.is_null() { + return Err("null counts".into()); + } + // SAFETY: the header's sixteen words. + get(device)?.3.counts(unsafe { std::slice::from_raw_parts_mut(out, 16) }); + Ok(true) + }) +} + +/// A command buffer names the buffer from now until `nmb_buffer_unhold`: it is not discarded meanwhile. Called before the command +/// that names it is encoded. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_hold(buffer: *const NmbBuffer) { + guard((), || Ok(get(buffer)?.0.hold())) +} + +/// The command buffer that held the buffer completed, or went without a commit. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_unhold(buffer: *const NmbBuffer) { + guard((), || Ok(get(buffer)?.0.unhold())) +} + +/// `nmb_buffer_hold` for a texture. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_hold(texture: *const NmbTexture) { + guard((), || Ok(get(texture)?.0.hold())) +} + +/// `nmb_buffer_unhold` for a texture. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_unhold(texture: *const NmbTexture) { + guard((), || Ok(get(texture)?.0.unhold())) +} + /// The call stacks that allocated device memory of 4 MiB or more (bridge patch 0020), most bytes first, into the caller's `max` records /// of twelve words: bytes, allocations, then ten return addresses (0 past the stack's end). Returns the records written. #[unsafe(no_mangle)] @@ -878,23 +944,31 @@ pub extern "C" fn nmb_device_vendor_name(device: *const NmbDevice, out: *mut c_c // ---- buffers ---- -pub struct NmbBuffer(Buffer); +/// A buffer: the runtime's, behind what keeps its purgeable state (NativeMetalPurge.rs), and whether it has a CPU pointer. +pub struct NmbBuffer(Resource, bool); + +impl NmbBuffer { + fn of(buffer: Resource) -> Result { + let cpu = buffer.live()?.contents().is_some(); + Ok(NmbBuffer(buffer, cpu)) + } +} /// # Safety /// `bytes` is null or holds `length` bytes. #[unsafe(no_mangle)] pub unsafe extern "C" fn nmb_buffer_new(device: *mut NmbDevice, length: u64, options_word: u64, bytes: *const c_void) -> *mut NmbBuffer { guard(std::ptr::null_mut(), || { - let device = &get(device)?.0; + let device = get(device)?; let options = options(options_word)?; let buffer = if bytes.is_null() { - device.new_buffer(length, options) + device.0.new_buffer(length, options) } else { // SAFETY: the caller's contract. let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; - device.new_buffer_with_bytes(bytes, options) + device.0.new_buffer_with_bytes(bytes, options) }; - Ok(boxed(NmbBuffer(buffer.map_err(e("buffer"))?))) + Ok(boxed(NmbBuffer::of(Resource::kept(&device.3, buffer.map_err(e("buffer"))?, (length, options)))?)) }) } @@ -1071,7 +1145,7 @@ unsafe fn no_copy(device: *mut NmbDevice, pointer: *mut c_void, length: u64, opt } buffer }; - Ok(boxed(NmbBuffer(buffer.map_err(e("no-copy buffer"))?))) + Ok(boxed(NmbBuffer::of(Resource::fixed(buffer.map_err(e("no-copy buffer"))?))?)) }) } @@ -1082,34 +1156,47 @@ pub extern "C" fn nmb_buffer_release(buffer: *mut NmbBuffer) { #[unsafe(no_mangle)] pub extern "C" fn nmb_buffer_contents(buffer: *const NmbBuffer) -> *mut c_void { - guard(std::ptr::null_mut(), || Ok(get(buffer)?.0.contents().map_or(std::ptr::null_mut(), |p| p.as_ptr() as *mut c_void))) + guard(std::ptr::null_mut(), || { + let buffer = get(buffer)?; + // The pointer is the buffer's for its life, as Metal's is: one handed out keeps the buffer's memory where it is. + if buffer.1 { + buffer.0.pin(); + } + Ok(buffer.0.live()?.contents().map_or(std::ptr::null_mut(), |p| p.as_ptr() as *mut c_void)) + }) } #[unsafe(no_mangle)] pub extern "C" fn nmb_buffer_length(buffer: *const NmbBuffer) -> u64 { - guard(0, || Ok(get(buffer)?.0.length())) + guard(0, || Ok(get(buffer)?.0.bytes())) } #[unsafe(no_mangle)] pub extern "C" fn nmb_buffer_gpu_address(buffer: *const NmbBuffer) -> u64 { - guard(0, || Ok(get(buffer)?.0.gpu_address().0)) + guard(0, || { + let buffer = &get(buffer)?.0; + // The address is the buffer's for its life too. + buffer.pin(); + Ok(buffer.live()?.gpu_address().0) + }) } // ---- heaps ---- -pub struct NmbHeap(Heap); +pub struct NmbHeap(Resource); /// `newHeapWithDescriptor:`: type (0 automatic, 1 placement), size, resource options. Sparse heaps are not made here. #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_new(device: *mut NmbDevice, heap_type: u64, size: u64, options_word: u64) -> *mut NmbHeap { guard(std::ptr::null_mut(), || { - let device = &get(device)?.0; + let device = get(device)?; let heap_type = HeapType::from_raw(heap_type).map_err(e("heap type"))?; if heap_type == HeapType::Sparse { return Err("sparse heaps are not implemented".into()); } - let descriptor = HeapDescriptor::new(heap_type, size, options(options_word)?); - Ok(boxed(NmbHeap(device.new_heap(&descriptor).map_err(e("heap"))?))) + let options = options(options_word)?; + let heap = device.0.new_heap(&HeapDescriptor::new(heap_type, size, options)).map_err(e("heap"))?; + Ok(boxed(NmbHeap(Resource::kept(&device.3, heap, (heap_type, size, options))))) }) } @@ -1120,29 +1207,42 @@ pub extern "C" fn nmb_heap_release(heap: *mut NmbHeap) { #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_size(heap: *const NmbHeap) -> u64 { - guard(0, || Ok(get(heap)?.0.size())) + guard(0, || Ok(get(heap)?.0.bytes())) } #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_used_size(heap: *const NmbHeap) -> u64 { - guard(0, || Ok(get(heap)?.0.used_size())) + guard(0, || Ok(get(heap)?.0.peek().map_or(0, |heap| heap.used_size()))) } #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_allocated_size(heap: *const NmbHeap) -> u64 { - guard(0, || Ok(get(heap)?.0.current_allocated_size())) + guard(0, || Ok(get(heap)?.0.bytes())) } #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_max_available_size(heap: *const NmbHeap, alignment: u64) -> u64 { - guard(0, || Ok(get(heap)?.0.max_available_size(alignment).map_err(e("heap available size"))?)) + guard(0, || { + let heap = &get(heap)?.0; + if let Some(heap) = heap.peek() { + return heap.max_available_size(alignment).map_err(e("heap available size")); + } + // A discarded heap holds nothing (no resource placed in it lives): an empty heap's answer, placing nothing. + if alignment != 0 && !alignment.is_power_of_two() { + return Err(format!("heap available size: alignment {alignment} is not a power of two")); + } + Ok(match heap.recipe() { + Some((HeapType::Automatic, ..)) => heap.bytes(), + _ => 0, + }) + }) } #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_set_purgeable_state(heap: *const NmbHeap, state: u64) -> u64 { guard(0, || { let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; - Ok(get(heap)?.0.set_purgeable_state(state).map_err(e("heap purgeable state"))?.raw()) + get(heap)?.0.set_state(state) }) } @@ -1150,10 +1250,11 @@ pub extern "C" fn nmb_heap_set_purgeable_state(heap: *const NmbHeap, state: u64) #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_new_buffer(heap: *const NmbHeap, length: u64, options_word: u64, offset: u64) -> *mut NmbBuffer { guard(std::ptr::null_mut(), || { - let heap = &get(heap)?.0; + // The tie comes first: a heap that holds a resource is not discarded. + let (tie, heap) = (get(heap)?.0.tie(), get(heap)?.0.live()?); let options = options(options_word)?; let buffer = if offset == u64::MAX { heap.new_buffer(length, options) } else { heap.new_buffer_at_offset(length, options, offset) }; - Ok(boxed(NmbBuffer(buffer.map_err(e("heap buffer"))?))) + Ok(boxed(NmbBuffer::of(Resource::tied(buffer.map_err(e("heap buffer"))?, tie))?)) }) } @@ -1161,10 +1262,10 @@ pub extern "C" fn nmb_heap_new_buffer(heap: *const NmbHeap, length: u64, options #[unsafe(no_mangle)] pub extern "C" fn nmb_heap_new_texture(heap: *const NmbHeap, descriptor: *const NmbTextureDescriptor, offset: u64) -> *mut NmbTexture { guard(std::ptr::null_mut(), || { - let heap = &get(heap)?.0; + let (tie, heap) = (get(heap)?.0.tie(), get(heap)?.0.live()?); let descriptor = texture_descriptor(descriptor)?; let texture = if offset == u64::MAX { heap.new_texture(&descriptor) } else { heap.new_texture_at_offset(&descriptor, offset) }; - Ok(boxed(NmbTexture(texture.map_err(e("heap texture"))?))) + Ok(boxed(NmbTexture(Resource::tied(texture.map_err(e("heap texture"))?, tie)))) }) } @@ -1220,7 +1321,7 @@ fn swizzle_of(packed: u64) -> Result, String> { fn swizzle_packed(s: TextureSwizzleChannels) -> u64 { (1u64 << 32) | s.red.as_raw() as u64 | (s.green.as_raw() as u64) << 8 | (s.blue.as_raw() as u64) << 16 | (s.alpha.as_raw() as u64) << 24 } -pub struct NmbTexture(Texture); +pub struct NmbTexture(Resource); fn texture_descriptor(d: *const NmbTextureDescriptor) -> Result { let d = get(d)?; @@ -1246,7 +1347,13 @@ fn texture_descriptor(d: *const NmbTextureDescriptor) -> Result *mut NmbTexture { guard(std::ptr::null_mut(), || { let descriptor = texture_descriptor(descriptor)?; - Ok(boxed(NmbTexture(get(device)?.0.new_texture(&descriptor).map_err(e("texture"))?))) + let device = get(device)?; + let texture = device.0.new_texture(&descriptor).map_err(e("texture"))?; + // A drawable's texture is the display's: never discarded. + Ok(boxed(NmbTexture(match descriptor.is_drawable { + true => Resource::fixed(texture), + false => Resource::kept(&device.3, texture, descriptor), + }))) }) } @@ -1255,13 +1362,14 @@ pub extern "C" fn nmb_texture_new_on_buffer(buffer: *mut NmbBuffer, descriptor: bytes_per_row: u64, iosurface: bool) -> *mut NmbTexture { guard(std::ptr::null_mut(), || { let descriptor = texture_descriptor(descriptor)?; - let buffer = &get(buffer)?.0; + // The tie comes first: a buffer with a texture over its bytes is not discarded. + let (tie, buffer) = (get(buffer)?.0.tie(), get(buffer)?.0.live()?); let texture = if iosurface { buffer.new_texture_with_iosurface_plane(&descriptor, IoSurfacePlane::new(offset, bytes_per_row, false)) } else { buffer.new_texture(&descriptor, offset, bytes_per_row) }; - Ok(boxed(NmbTexture(texture.map_err(e("texture over buffer"))?))) + Ok(boxed(NmbTexture(Resource::tied(texture.map_err(e("texture over buffer"))?, tie)))) }) } @@ -1274,13 +1382,13 @@ pub extern "C" fn nmb_texture_new_on_surface(buffer: *mut NmbBuffer, descriptor: chroma_offset: u64, chroma_bytes_per_row: u64) -> *mut NmbTexture { guard(std::ptr::null_mut(), || { let descriptor = texture_descriptor(descriptor)?; - let buffer = &get(buffer)?.0; + let (tie, buffer) = (get(buffer)?.0.tie(), get(buffer)?.0.live()?); let mut surface = IoSurfacePlane::new(offset, bytes_per_row, true); if chroma_bytes_per_row != 0 { surface = surface.with_ycbcr(IoSurfaceYCbCr::new(pixel_format, plane, [width, height], chroma_offset, chroma_bytes_per_row)); } let texture = buffer.new_texture_with_iosurface_plane(&descriptor, surface); - Ok(boxed(NmbTexture(texture.map_err(e("texture over surface"))?))) + Ok(boxed(NmbTexture(Resource::tied(texture.map_err(e("texture over surface"))?, tie)))) }) } @@ -1290,8 +1398,9 @@ pub extern "C" fn nmb_texture_new_linear(buffer: *mut NmbBuffer, descriptor: *co bytes_per_image: u64) -> *mut NmbTexture { guard(std::ptr::null_mut(), || { let descriptor = texture_descriptor(descriptor)?; - let texture = get(buffer)?.0.new_linear_texture(&descriptor, offset, bytes_per_row, bytes_per_image); - Ok(boxed(NmbTexture(texture.map_err(e("linear texture"))?))) + let (tie, buffer) = (get(buffer)?.0.tie(), get(buffer)?.0.live()?); + let texture = buffer.new_linear_texture(&descriptor, offset, bytes_per_row, bytes_per_image); + Ok(boxed(NmbTexture(Resource::tied(texture.map_err(e("linear texture"))?, tie)))) }) } @@ -1300,7 +1409,8 @@ pub extern "C" fn nmb_texture_new_linear(buffer: *mut NmbBuffer, descriptor: *co pub extern "C" fn nmb_texture_new_view(texture: *mut NmbTexture, format: u64, kind: u64, levels: *const u64, slices: *const u64, whole: bool) -> *mut NmbTexture { guard(std::ptr::null_mut(), || { - let t = &get(texture)?.0; + // The tie comes first: a texture with a view of it is not discarded, since the view shares its memory. + let (tie, t) = (get(texture)?.0.tie(), get(texture)?.0.live()?); let format = raw::(format, "pixel format")?; let view = if whole { t.new_texture_view(format) @@ -1313,16 +1423,17 @@ pub extern "C" fn nmb_texture_new_view(texture: *mut NmbTexture, format: u64, ki let ranges = (l[0]..l[0].checked_add(l[1]).ok_or("levels overflow")?, s[0]..s[0].checked_add(s[1]).ok_or("slices overflow")?); t.new_texture_view_with_levels_slices(format, raw::(kind, "texture type")?, ranges.0, ranges.1) }; - Ok(boxed(NmbTexture(view.map_err(e("texture view"))?))) + Ok(boxed(NmbTexture(Resource::tied(view.map_err(e("texture view"))?, tie)))) }) } -/// `setPurgeableState:` on a texture: the runtime's answer as Metal's raw state, 0 where it fails. +/// `setPurgeableState:` on a texture: the answer as Metal's raw state, 0 where it fails. Bit 8 is set where the texture's contents +/// were discarded and placing it again failed: the answer's low byte is still Empty, and the reason is the last error. #[unsafe(no_mangle)] pub extern "C" fn nmb_texture_set_purgeable_state(texture: *const NmbTexture, state: u64) -> u64 { guard(0, || { let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; - Ok(get(texture)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw()) + get(texture)?.0.set_state(state) }) } @@ -1331,7 +1442,7 @@ pub extern "C" fn nmb_texture_set_purgeable_state(texture: *const NmbTexture, st pub extern "C" fn nmb_buffer_set_purgeable_state(buffer: *const NmbBuffer, state: u64) -> u64 { guard(0, || { let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; - Ok(get(buffer)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw()) + get(buffer)?.0.set_state(state) }) } @@ -1343,7 +1454,7 @@ pub extern "C" fn nmb_texture_release(texture: *mut NmbTexture) { #[unsafe(no_mangle)] pub extern "C" fn nmb_texture_describe(texture: *const NmbTexture, out: *mut NmbTextureDescriptor) { guard((), || { - let t = &get(texture)?.0; + let t = get(texture)?.0.live()?; let out = get_mut(out)?; *out = NmbTextureDescriptor { kind: t.texture_type().as_raw() as u64, @@ -1394,7 +1505,7 @@ pub unsafe extern "C" fn nmb_texture_replace(texture: *mut NmbTexture, region_wo } // SAFETY: the caller's contract. let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; - get(texture)?.0.replace_region(region(region_words)?, level, slice, bytes, bytes_per_row, bytes_per_image) + get(texture)?.0.live()?.replace_region(region(region_words)?, level, slice, bytes, bytes_per_row, bytes_per_image) .map_err(e("replace region"))?; Ok(true) }) @@ -1411,7 +1522,7 @@ pub unsafe extern "C" fn nmb_texture_get_bytes(texture: *const NmbTexture, bytes } // SAFETY: the caller's contract. let bytes = unsafe { std::slice::from_raw_parts_mut(bytes as *mut u8, length as usize) }; - get(texture)?.0.get_bytes(bytes, bytes_per_row, bytes_per_image, region(region_words)?, level, slice) + get(texture)?.0.live()?.get_bytes(bytes, bytes_per_row, bytes_per_image, region(region_words)?, level, slice) .map_err(e("get bytes"))?; Ok(true) }) @@ -2273,7 +2384,8 @@ pub extern "C" fn nmb_command_buffer_release(command_buffer: *mut NmbCommandBuff pub extern "C" fn nmb_command_buffer_uses_buffer(command_buffer: *const NmbCommandBuffer, buffer: *const NmbBuffer) -> i32 { guard(-1, || { let used = get(command_buffer)?.0.used_resources(); - Ok(i32::from(used.references_resource(get(buffer)?.0.as_resource()))) + // A discarded buffer is in no command buffer's work: one that named it held it until it completed. + Ok(get(buffer)?.0.peek().map_or(0, |buffer| i32::from(used.references_resource(buffer.as_resource())))) }) } @@ -2281,7 +2393,7 @@ pub extern "C" fn nmb_command_buffer_uses_buffer(command_buffer: *const NmbComma pub extern "C" fn nmb_command_buffer_uses_texture(command_buffer: *const NmbCommandBuffer, texture: *const NmbTexture) -> i32 { guard(-1, || { let used = get(command_buffer)?.0.used_resources(); - Ok(i32::from(used.references_resource(get(texture)?.0.as_resource()))) + Ok(get(texture)?.0.peek().map_or(0, |texture| i32::from(used.references_resource(texture.as_resource())))) }) } @@ -2584,7 +2696,7 @@ pub struct NmbRenderPassDescriptor { } fn texture_of(a: &NmbPassAttachment) -> Result, String> { - if a.texture.is_null() { Ok(None) } else { Ok(Some(get(a.texture)?.0.clone())) } + if a.texture.is_null() { Ok(None) } else { Ok(Some(get(a.texture)?.0.live()?.clone())) } } #[unsafe(no_mangle)] @@ -2595,7 +2707,7 @@ pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, let mut pass = RenderPassDescriptor::new(); for (slot, a) in pass.color_attachments.iter_mut().zip(d.colors.iter()) { slot.texture = texture_of(a)?; - slot.resolve_texture = optional(a.resolve_texture)?.map(|t| t.0.clone()); + slot.resolve_texture = optional(a.resolve_texture)?.map(|t| t.0.live().map(|t| t.clone())).transpose()?; slot.resolve_level = a.resolve_level; slot.resolve_slice = a.resolve_slice; slot.resolve_depth_plane = a.resolve_depth_plane; @@ -2608,7 +2720,7 @@ pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, } let depth = &mut pass.depth_attachment; depth.texture = texture_of(&d.depth)?; - depth.resolve_texture = optional(d.depth.resolve_texture)?.map(|t| t.0.clone()); + depth.resolve_texture = optional(d.depth.resolve_texture)?.map(|t| t.0.live().map(|t| t.clone())).transpose()?; depth.resolve_level = d.depth.resolve_level; depth.resolve_slice = d.depth.resolve_slice; depth.resolve_depth_plane = d.depth.resolve_depth_plane; @@ -2621,7 +2733,7 @@ pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, depth.clear_depth = d.depth.clear[0]; let stencil = &mut pass.stencil_attachment; stencil.texture = texture_of(&d.stencil)?; - stencil.resolve_texture = optional(d.stencil.resolve_texture)?.map(|t| t.0.clone()); + stencil.resolve_texture = optional(d.stencil.resolve_texture)?.map(|t| t.0.live().map(|t| t.clone())).transpose()?; stencil.resolve_level = d.stencil.resolve_level; stencil.resolve_slice = d.stencil.resolve_slice; stencil.resolve_depth_plane = d.stencil.resolve_depth_plane; @@ -2637,7 +2749,7 @@ pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, pass.render_target_array_length = d.array_length; pass.default_raster_sample_count = d.samples; if !d.visibility.is_null() { - pass.visibility_result_buffer = Some(get(d.visibility)?.0.clone()); + pass.visibility_result_buffer = Some(get(d.visibility)?.0.live()?.clone()); } let encoder = get(command_buffer)?.0.render_command_encoder(&pass).map_err(e("render encoder"))?; Ok(boxed(NmbEncoder::Render(encoder))) @@ -2703,7 +2815,8 @@ fn bad_stage(stage: u32) -> String { #[unsafe(no_mangle)] pub extern "C" fn nmb_encoder_set_buffer(encoder: *mut NmbEncoder, stage: u32, buffer: *mut NmbBuffer, offset: u64, index: u32) -> bool { guard(false, || { - let buffer = optional(buffer)?.map(|b| &b.0); + let buffer = optional(buffer)?.map(|b| b.0.live()).transpose()?; + let buffer = buffer.as_deref(); match (get_mut(encoder)?, stage) { (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer(buffer, offset, index), (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer(buffer, offset, index), @@ -2754,7 +2867,8 @@ pub unsafe extern "C" fn nmb_encoder_set_bytes(encoder: *mut NmbEncoder, stage: #[unsafe(no_mangle)] pub extern "C" fn nmb_encoder_set_texture(encoder: *mut NmbEncoder, stage: u32, texture: *mut NmbTexture, index: u32) -> bool { guard(false, || { - let texture = optional(texture)?.map(|t| &t.0); + let texture = optional(texture)?.map(|t| t.0.live()).transpose()?; + let texture = texture.as_deref(); match (get_mut(encoder)?, stage) { (NmbEncoder::Compute(c), COMPUTE) => c.set_texture(texture, index), (NmbEncoder::Render(r), VERTEX) => r.set_vertex_texture(texture, index), @@ -2812,7 +2926,7 @@ pub extern "C" fn nmb_compute_dispatch(encoder: *mut NmbEncoder, size: *const u6 pub extern "C" fn nmb_compute_dispatch_indirect(encoder: *mut NmbEncoder, buffer: *mut NmbBuffer, offset: u64, per_group: *const u64) -> bool { guard(false, || { let g = triple(per_group)?; - compute(encoder)?.dispatch_threadgroups_indirect(&get(buffer)?.0, offset, Size::new(g[0], g[1], g[2])) + compute(encoder)?.dispatch_threadgroups_indirect(&*get(buffer)?.0.live()?, offset, Size::new(g[0], g[1], g[2])) .map_err(e("indirect dispatch"))?; Ok(true) }) @@ -2934,7 +3048,7 @@ pub extern "C" fn nmb_render_draw_indexed(encoder: *mut NmbEncoder, primitive: u offset: u64, instances: u32, base_vertex: i32, base_instance: u32) -> bool { guard(false, || { render(encoder)?.draw_indexed_primitives(raw::(primitive, "primitive")?, count, - raw::(index_type, "index type")?, &get(indices)?.0, offset, instances, + raw::(index_type, "index type")?, &*get(indices)?.0.live()?, offset, instances, base_vertex, base_instance).map_err(e("indexed draw"))?; Ok(true) }) @@ -2945,8 +3059,8 @@ pub extern "C" fn nmb_render_draw_indexed(encoder: *mut NmbEncoder, primitive: u pub extern "C" fn nmb_render_set_tessellation_factor_buffer(encoder: *mut NmbEncoder, buffer: *mut NmbBuffer, offset: u64, instance_stride: u64) -> bool { guard(false, || { - let buffer = if buffer.is_null() { None } else { Some(&get(buffer)?.0) }; - render(encoder)?.set_tessellation_factor_buffer(buffer, offset, instance_stride).map_err(e("tessellation factor buffer"))?; + let buffer = optional_buffer(buffer)?; + render(encoder)?.set_tessellation_factor_buffer(buffer.as_deref(), offset, instance_stride).map_err(e("tessellation factor buffer"))?; Ok(true) }) } @@ -2959,8 +3073,8 @@ pub extern "C" fn nmb_render_set_tessellation_factor_scale(encoder: *mut NmbEnco }) } -fn optional_buffer<'a>(buffer: *mut NmbBuffer) -> Result, String> { - if buffer.is_null() { Ok(None) } else { Ok(Some(&get(buffer)?.0)) } +fn optional_buffer<'a>(buffer: *mut NmbBuffer) -> Result>, String> { + if buffer.is_null() { Ok(None) } else { get(buffer)?.0.live().map(Some) } } /// The four patch draws: `indirect` null takes `patch_start`, `patch_count`, `instances` and `base_instance`, otherwise reads them from @@ -2973,8 +3087,9 @@ pub extern "C" fn nmb_render_draw_patches(encoder: *mut NmbEncoder, control_poin indirect_offset: u64) -> bool { guard(false, || { let r = render(encoder)?; - let patch_index = optional_buffer(patch_index)?; - let result = match (optional_buffer(control_point_index)?, optional_buffer(indirect)?) { + let (patch_index, points, indirect) = (optional_buffer(patch_index)?, optional_buffer(control_point_index)?, optional_buffer(indirect)?); + let patch_index = patch_index.as_deref(); + let result = match (points.as_deref(), indirect.as_deref()) { (None, None) => r.draw_patches(control_points, patch_start, patch_count, patch_index, patch_index_offset, instances, base_instance), (None, Some(indirect)) => r.draw_patches_indirect(control_points, patch_index, patch_index_offset, indirect, indirect_offset), (Some(points), None) => r.draw_indexed_patches(control_points, patch_start, patch_count, patch_index, patch_index_offset, points, @@ -2991,7 +3106,7 @@ pub extern "C" fn nmb_render_draw_patches(encoder: *mut NmbEncoder, control_poin pub extern "C" fn nmb_blit_copy_buffer(encoder: *mut NmbEncoder, source: *mut NmbBuffer, source_offset: u64, destination: *mut NmbBuffer, destination_offset: u64, size: u64) -> bool { guard(false, || { - blit(encoder)?.copy_from_buffer_to_buffer(&get(source)?.0, source_offset, &get(destination)?.0, destination_offset, size) + blit(encoder)?.copy_from_buffer_to_buffer(&*get(source)?.0.live()?, source_offset, &*get(destination)?.0.live()?, destination_offset, size) .map_err(e("buffer copy"))?; Ok(true) }) @@ -3000,7 +3115,7 @@ pub extern "C" fn nmb_blit_copy_buffer(encoder: *mut NmbEncoder, source: *mut Nm #[unsafe(no_mangle)] pub extern "C" fn nmb_blit_fill_buffer(encoder: *mut NmbEncoder, buffer: *mut NmbBuffer, start: u64, end: u64, value: u8) -> bool { guard(false, || { - blit(encoder)?.fill_buffer(&get(buffer)?.0, start..end, value).map_err(e("fill"))?; + blit(encoder)?.fill_buffer(&*get(buffer)?.0.live()?, start..end, value).map_err(e("fill"))?; Ok(true) }) } @@ -3021,8 +3136,8 @@ pub extern "C" fn nmb_blit_copy_texture(encoder: *mut NmbEncoder, source: *mut N source_origin: *const u64, extent: *const u64, destination: *mut NmbTexture, destination_slice: u64, destination_level: u64, destination_origin: *const u64) -> bool { guard(false, || { - blit(encoder)?.copy_from_texture_to_texture(&get(source)?.0, source_slice, source_level, origin(source_origin)?, size(extent)?, - &get(destination)?.0, destination_slice, destination_level, + blit(encoder)?.copy_from_texture_to_texture(&*get(source)?.0.live()?, source_slice, source_level, origin(source_origin)?, size(extent)?, + &*get(destination)?.0.live()?, destination_slice, destination_level, origin(destination_origin)?).map_err(e("texture copy"))?; Ok(true) }) @@ -3034,8 +3149,8 @@ pub extern "C" fn nmb_blit_buffer_to_texture(encoder: *mut NmbEncoder, source: * bytes_per_image: u64, extent: *const u64, destination: *mut NmbTexture, slice: u64, level: u64, destination_origin: *const u64) -> bool { guard(false, || { - blit(encoder)?.copy_from_buffer_to_texture(&get(source)?.0, offset, bytes_per_row, bytes_per_image, size(extent)?, - &get(destination)?.0, slice, level, origin(destination_origin)?, BlitOption::None) + blit(encoder)?.copy_from_buffer_to_texture(&*get(source)?.0.live()?, offset, bytes_per_row, bytes_per_image, size(extent)?, + &*get(destination)?.0.live()?, slice, level, origin(destination_origin)?, BlitOption::None) .map_err(e("buffer to texture"))?; Ok(true) }) @@ -3047,8 +3162,8 @@ pub extern "C" fn nmb_blit_texture_to_buffer(encoder: *mut NmbEncoder, source: * source_origin: *const u64, extent: *const u64, destination: *mut NmbBuffer, offset: u64, bytes_per_row: u64, bytes_per_image: u64) -> bool { guard(false, || { - blit(encoder)?.copy_from_texture_to_buffer(&get(source)?.0, slice, level, origin(source_origin)?, size(extent)?, - &get(destination)?.0, offset, bytes_per_row, bytes_per_image, BlitOption::None) + blit(encoder)?.copy_from_texture_to_buffer(&*get(source)?.0.live()?, slice, level, origin(source_origin)?, size(extent)?, + &*get(destination)?.0.live()?, offset, bytes_per_row, bytes_per_image, BlitOption::None) .map_err(e("texture to buffer"))?; Ok(true) }) @@ -3057,7 +3172,7 @@ pub extern "C" fn nmb_blit_texture_to_buffer(encoder: *mut NmbEncoder, source: * #[unsafe(no_mangle)] pub extern "C" fn nmb_blit_generate_mipmaps(encoder: *mut NmbEncoder, texture: *mut NmbTexture) -> bool { guard(false, || { - blit(encoder)?.generate_mipmaps(&get(texture)?.0).map_err(e("mipmaps"))?; + blit(encoder)?.generate_mipmaps(&*get(texture)?.0.live()?).map_err(e("mipmaps"))?; Ok(true) }) } @@ -3066,7 +3181,7 @@ pub extern "C" fn nmb_blit_generate_mipmaps(encoder: *mut NmbEncoder, texture: * pub extern "C" fn nmb_texture_new_view_swizzle(texture: *mut NmbTexture, format: u64, kind: u64, levels: *const u64, slices: *const u64, swizzle: u64) -> *mut NmbTexture { guard(std::ptr::null_mut(), || { - let t = &get(texture)?.0; + let (tie, t) = (get(texture)?.0.tie(), get(texture)?.0.live()?); if levels.is_null() || slices.is_null() { return Err("null levels or slices".into()); } @@ -3076,34 +3191,34 @@ pub extern "C" fn nmb_texture_new_view_swizzle(texture: *mut NmbTexture, format: let channels = swizzle_of(swizzle)?.unwrap_or(TextureSwizzleChannels::DEFAULT); let view = t.new_texture_view_with_swizzle(raw::(format, "pixel format")?, raw::(kind, "texture type")?, ranges.0, ranges.1, channels); - Ok(boxed(NmbTexture(view.map_err(e("texture view"))?))) + Ok(boxed(NmbTexture(Resource::tied(view.map_err(e("texture view"))?, tie)))) }) } #[unsafe(no_mangle)] pub extern "C" fn nmb_texture_make_aliasable(texture: *const NmbTexture) -> bool { guard(false, || { - get(texture)?.0.make_aliasable().map_err(e("make aliasable"))?; + get(texture)?.0.live()?.make_aliasable().map_err(e("make aliasable"))?; Ok(true) }) } #[unsafe(no_mangle)] pub extern "C" fn nmb_texture_is_aliasable(texture: *const NmbTexture) -> bool { - guard(false, || Ok(get(texture)?.0.is_aliasable())) + guard(false, || Ok(get(texture)?.0.live()?.is_aliasable())) } #[unsafe(no_mangle)] pub extern "C" fn nmb_buffer_make_aliasable(buffer: *const NmbBuffer) -> bool { guard(false, || { - get(buffer)?.0.make_aliasable().map_err(e("make aliasable"))?; + get(buffer)?.0.live()?.make_aliasable().map_err(e("make aliasable"))?; Ok(true) }) } #[unsafe(no_mangle)] pub extern "C" fn nmb_buffer_is_aliasable(buffer: *const NmbBuffer) -> bool { - guard(false, || Ok(get(buffer)?.0.is_aliasable())) + guard(false, || Ok(get(buffer)?.0.live()?.is_aliasable())) } #[unsafe(no_mangle)] @@ -3164,7 +3279,7 @@ pub extern "C" fn nmb_render_set_scissors(encoder: *mut NmbEncoder, values: *con #[unsafe(no_mangle)] pub extern "C" fn nmb_texture_allocated_size(texture: *const NmbTexture) -> u64 { - guard(0, || Ok(get(texture)?.0.backing_allocation_size())) + guard(0, || Ok(get(texture)?.0.bytes())) } #[unsafe(no_mangle)] pub extern "C" fn nmb_device_current_allocated_size(device: *const NmbDevice) -> u64 { diff --git a/bridge/NativeMetalPurge.rs b/bridge/NativeMetalPurge.rs new file mode 100644 index 0000000..c223ea5 --- /dev/null +++ b/bridge/NativeMetalPurge.rs @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: MIT +//! Purgeable memory: what `setPurgeableState:` promises of a buffer, a texture or a heap, kept by the bridge above the runtime. +//! +//! Metal's contract. A resource whose state is Volatile may lose its contents. `setPurgeableState:` answers the state before the +//! call, and Empty where the contents were discarded; the caller sets NonVolatile before it uses the resource again and makes the +//! contents again when the answer is Empty. Apple's drivers do not discard at the call: the kernel purges volatile memory when it +//! needs pages. RenderBox, QuartzCore, SkyLight and Core Image keep their recycled buffers, textures and heaps Volatile on that +//! promise, so on a GPU whose memory is wired for life those caches cost their whole size. +//! +//! Where the discard is made. The runtime keeps a resource's state and never discards (its attributes.rs): a buffer's `VkBuffer` +//! and placement and an image's `VkImage` and placement are the resource's own for its life (buffer.rs `FixedPages`, texture.rs +//! `Owner::Runtime`; Vulkan binds memory once), and its views, copies and first moves rely on that. So the bridge discards with +//! the two operations the runtime performs for every resource: it releases the runtime object and later makes one again from the +//! same description. A release is safe by the runtime's own rules: a batch in flight and an encoder being recorded hold what they +//! use, so the placement is returned only once nothing of the runtime's reads it, and a block with no placement left follows the +//! pools' aging (kept empty blocks, spare blocks; bridge patches 0021, 0030, 0034). The handle the frontend holds stays the same +//! box for the resource's life, so a call that still names it never reaches freed memory: it finds the resource placed, or places +//! it again. +//! +//! What may be discarded ([`Resource::kept`]): a texture the device made of its own memory (`nmb_texture_new`, not a drawable), a +//! buffer the device made (`nmb_buffer_new`), a heap (`nmb_heap_new`), where the runtime keeps a purgeable state for it. Never: a +//! texture over a buffer or a surface, a no-copy buffer, a view, a resource placed in a heap, a drawable, and what the runtime +//! makes for itself (it has no handle here and no caller that could declare it Volatile). Those are [`Resource::Fixed`], with no +//! cell: their calls borrow the runtime object as before. +//! +//! When ([`Cell::sweep`]). A resource is discarded only while all of these hold, read under its lock: +//! - its state is Volatile or Empty, by the runtime's own answer (which keeps the state only of a resource with an allocation of +//! its own: at least 16 KiB, not memoryless); +//! - no command buffer that named it is being recorded or is in flight (`holds`: the frontend holds it from the call that names +//! it until the command buffer completed); +//! - nothing made of it lives (`ties`: a view of a texture, a texture over a buffer, a resource placed in a heap), since those +//! share its memory and would not follow a new placement; +//! - neither its CPU pointer nor its GPU address was ever handed out (`pinned`): Metal keeps both for a resource's life, and a +//! resource made again lies elsewhere; +//! - no call is inside it (its lock is free). +//! The triggers are the system's memory pressure signal ([`Cause::Pressure`]) and age ([`Cause::Age`]: Volatile or Empty and idle +//! for the runtime's spare age), because pressure events are rare on this system while memory is short. +//! +//! After a discard the state is Empty. NonVolatile places the resource again before it answers Empty; a query answers Empty and +//! places nothing (a query changes nothing in Metal, and placing for it would cost the memory again for a caller that only asks); +//! Volatile leaves it Empty, as Metal does. Any other use of a discarded resource is the caller's error under the contract (it did +//! not set NonVolatile first): the resource is placed again with new, zeroed memory, its state still Empty, as a GPU whose pages +//! were purged shows emptied pages, and it may be discarded again once idle. Placing again fails as an allocation fails: the call +//! that needed the resource fails with the allocation's error, `setPurgeableState:` still answers Empty, and the next use tries +//! again. +//! +//! `NATIVE_METAL_PURGE=0` (the boot argument `gpu-purge=0`) makes every resource `Fixed`: the state is kept by the runtime alone +//! and nothing is ever discarded, for a comparison on the same image. + +use metal2vulkan_runtime::{Buffer, Device, Heap, HeapDescriptor, HeapType, PurgeableState, ResourceOptions, Texture, TextureDescriptor}; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering::Relaxed, Ordering::SeqCst}; +use std::sync::{Arc, Mutex, MutexGuard, TryLockError, Weak}; +use std::time::{Duration, Instant}; + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// False where `NATIVE_METAL_PURGE=0` (the boot argument `gpu-purge=0`): read once, as the device opens. +fn purge_on() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("NATIVE_METAL_PURGE").is_none_or(|value| value != "0")) +} + +/// Why a sweep runs. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Cause { + /// The system reported memory pressure: every idle Volatile or Empty resource goes. + Pressure = 0, + /// The age timer: those Volatile or Empty and idle for the given age go. + Age = 1, +} + +/// Bit 8 of a `setPurgeableState:` answer: the contents were discarded and placing the resource again failed (the reason is the +/// call's error text). The low byte is still Metal's answer, Empty. +pub const AGAIN_FAILED: u64 = 1 << 8; + +/// What a device counts of its discards, for the frontend's PG rows. +#[derive(Default)] +struct Counts { + /// Resources discarded and their bytes, by [`Cause`]. + discarded: [AtomicU64; 2], + discarded_bytes: [AtomicU64; 2], + /// Resources placed again and their bytes: by a NonVolatile request, by another use. + again: [AtomicU64; 2], + again_bytes: [AtomicU64; 2], + /// `setPurgeableState:` answers of Empty for contents a sweep discarded. + empty_answers: AtomicU64, + /// Bytes discarded and not placed again, now. + out_bytes: AtomicU64, + /// Bytes that became Volatile or Empty so far, and of them those of resources that can never go: their CPU pointer or GPU + /// address was handed out. + declared_bytes: AtomicU64, + declared_pinned_bytes: AtomicU64, + /// Bytes the last sweep found Volatile or Empty and left: named by a command buffer not yet completed or inside a call, tied + /// to a view or a placed resource, not yet idle for the age. + left_bytes: [AtomicU64; 3], + /// Placements again that failed. + failed: AtomicU64, +} + +/// A device's purge state: whether it discards at all, the resources that may go, and the counts. +pub struct Purge { + device: Device, + on: bool, + /// The resources Volatile or Empty with memory, for the sweeps. + listed: Mutex>>, + counts: Counts, + /// The origin of the cells' idle clocks. + start: Instant, +} + +impl Purge { + /// The purge state of `device`, on unless the process was started with `NATIVE_METAL_PURGE=0`. + pub fn new(device: &Device) -> Arc { + Arc::new(Purge { device: device.clone(), on: purge_on(), listed: Mutex::new(Vec::new()), counts: Counts::default(), start: Instant::now() }) + } + + /// The age an idle Volatile or Empty resource goes at: the runtime's spare block age, so a resource and the block it emptied + /// go on one clock. + pub fn age() -> Duration { + Device::spare_age() + } + + fn now_ms(&self) -> u64 { + self.start.elapsed().as_millis() as u64 + } + + /// Discard what may go for `cause`, those idle for `idle` where the cause is age: the bytes discarded. + pub fn sweep(&self, cause: Cause, idle: Duration) -> u64 { + if !self.on { + return 0; + } + // Out of the list while they are looked at: a cell listed meanwhile is added to the new list. + let listed = std::mem::take(&mut *lock(&self.listed)); + let now = Instant::now(); + let (mut bytes, mut left, mut stay) = (0, [0u64; 3], Vec::with_capacity(listed.len())); + for weak in listed { + // Dropped outside every lock: this may be the last holder of a resource its caller released meanwhile. + let Some(cell) = weak.upgrade() else { + continue; + }; + match cell.sweep(cause, now, idle) { + Swept::Discarded(size) => { + bytes += size; + self.counts.discarded[cause as usize].fetch_add(1, Relaxed); + self.counts.discarded_bytes[cause as usize].fetch_add(size, Relaxed); + self.counts.out_bytes.fetch_add(size, Relaxed); + } + Swept::Left(why, size) => { + left[why as usize] += size; + // A tied resource is listed again when its last tie goes; the others may go at a later sweep. + if why != Left::Tied { + stay.push(weak); + } + } + Swept::Not => {} + } + } + for (count, bytes) in self.counts.left_bytes.iter().zip(left) { + count.store(bytes, Relaxed); + } + lock(&self.listed).extend(stay); + bytes + } + + /// How many resources a later sweep may still discard. + pub fn pending(&self) -> usize { + lock(&self.listed).len() + } + + /// The counts, into the caller's sixteen words: resources and bytes discarded by pressure, then by age; resources and bytes + /// placed again by a NonVolatile request, then by another use; Empty answers for discarded contents; bytes discarded now; + /// bytes that became Volatile or Empty so far and those of them pinned; the bytes the last sweep left because they were in + /// use, tied, or not idle long enough; placements again that failed. + pub fn counts(&self, out: &mut [u64]) { + let c = &self.counts; + let words = [ + c.discarded[0].load(Relaxed), c.discarded_bytes[0].load(Relaxed), c.discarded[1].load(Relaxed), c.discarded_bytes[1].load(Relaxed), + c.again[0].load(Relaxed), c.again_bytes[0].load(Relaxed), c.again[1].load(Relaxed), c.again_bytes[1].load(Relaxed), + c.empty_answers.load(Relaxed), c.out_bytes.load(Relaxed), c.declared_bytes.load(Relaxed), c.declared_pinned_bytes.load(Relaxed), + c.left_bytes[0].load(Relaxed), c.left_bytes[1].load(Relaxed), c.left_bytes[2].load(Relaxed), c.failed.load(Relaxed), + ]; + for (out, word) in out.iter_mut().zip(words) { + *out = word; + } + } +} + +/// Why a sweep left a Volatile or Empty resource. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Left { + /// A command buffer not yet completed names it, or a call is inside it. + Used = 0, + /// A view of it, a texture over it or a resource placed in it lives. + Tied = 1, + /// It was not Volatile or Empty and idle for the age yet. + Young = 2, +} + +enum Swept { + Discarded(u64), + Left(Left, u64), + /// No longer a candidate: placed NonVolatile again, discarded already, or pinned for good. + Not, +} + +trait Candidate: Send + Sync { + fn sweep(&self, cause: Cause, now: Instant, idle: Duration) -> Swept; +} + +/// A kind of resource the bridge can make again from its description. +pub trait Kind: Clone + Send + Sync + 'static { + /// What the resource was made of. + type Recipe: Send + Sync + 'static; + fn make(device: &Device, recipe: &Self::Recipe) -> Result; + /// The runtime's `setPurgeableState:`. + fn state(&self, state: PurgeableState) -> Result; + /// The bytes of device memory it holds. + fn bytes(&self) -> u64; +} + +impl Kind for Texture { + type Recipe = TextureDescriptor; + fn make(device: &Device, recipe: &TextureDescriptor) -> Result { + device.new_texture(recipe).map_err(|error| format!("texture: {error:?}")) + } + fn state(&self, state: PurgeableState) -> Result { + self.set_purgeable_state(state).map_err(|error| format!("purgeable state: {error:?}")) + } + fn bytes(&self) -> u64 { + self.backing_allocation_size() + } +} + +impl Kind for Buffer { + /// The length and the options. A buffer made with bytes is made again without them: its contents were discarded. + type Recipe = (u64, ResourceOptions); + fn make(device: &Device, recipe: &(u64, ResourceOptions)) -> Result { + device.new_buffer(recipe.0, recipe.1).map_err(|error| format!("buffer: {error:?}")) + } + fn state(&self, state: PurgeableState) -> Result { + self.set_purgeable_state(state).map_err(|error| format!("purgeable state: {error:?}")) + } + fn bytes(&self) -> u64 { + self.length() + } +} + +impl Kind for Heap { + /// The type, the size asked and the options. + type Recipe = (HeapType, u64, ResourceOptions); + fn make(device: &Device, recipe: &(HeapType, u64, ResourceOptions)) -> Result { + device.new_heap(&HeapDescriptor::new(recipe.0, recipe.1, recipe.2)).map_err(|error| format!("heap: {error:?}")) + } + fn state(&self, state: PurgeableState) -> Result { + self.set_purgeable_state(state).map_err(|error| format!("heap purgeable state: {error:?}")) + } + fn bytes(&self) -> u64 { + self.size() + } +} + +/// Whether the runtime keeps `object`'s purgeable state, asked of a new object: Volatile is read back, then NonVolatile put back. +/// Neither request changes the object's contents. +fn keeps_state(object: &T) -> bool { + let kept = object.state(PurgeableState::Volatile).is_ok() && matches!(object.state(PurgeableState::KeepCurrent), Ok(PurgeableState::Volatile)); + let _ = object.state(PurgeableState::NonVolatile); + kept +} + +fn discardable(state: PurgeableState) -> bool { + matches!(state, PurgeableState::Volatile | PurgeableState::Empty) +} + +struct Slot { + /// The runtime object; `None` while the resource is discarded. + object: Option, + /// The state Metal answers: the runtime's own, but Empty while `lost`. + state: PurgeableState, + /// When the state last changed, or the resource was last placed again by a use. + since: Instant, + /// Its contents were discarded since the caller last set NonVolatile. Always set while `object` is `None`; set with an object + /// once a use placed the resource again, whose runtime state is then a new object's and not asked. + lost: bool, +} + +/// A resource that may be discarded: its runtime object or none, how to make it again, and who uses it. +pub struct Cell { + purge: Arc, + recipe: T::Recipe, + /// The bytes of device memory it holds while placed. + bytes: u64, + slot: Mutex>, + /// The command buffers being recorded or in flight that name it. + holds: AtomicU32, + /// The views of it, textures over it and resources placed in it that live. + ties: AtomicU32, + /// Its CPU pointer or GPU address was handed out. + pinned: AtomicBool, + /// When its last hold went, in milliseconds of the device's purge clock. + idle_ms: AtomicU64, + /// It is in the device's list of candidates. Changed under `slot`'s lock. + listed: AtomicBool, + me: Weak>, +} + +impl Cell { + /// In the device's list from now on, once. Called with `slot`'s lock held. + fn list(&self) { + if self.pinned.load(SeqCst) || self.listed.swap(true, SeqCst) { + return; + } + let me: Weak = self.me.clone(); + lock(&self.purge.listed).push(me); + } + + /// The object, placed again where it was discarded: the use of a resource whose caller did not set NonVolatile first. + fn live(&self) -> Result { + if let Some(object) = &lock(&self.slot).object { + return Ok(object.clone()); + } + let slot = self.place(1)?; + Ok(slot.object.clone().expect("a placed resource has its object")) + } + + /// The object where the resource has one, placing nothing. + fn peek(&self) -> Option { + lock(&self.slot).object.clone() + } + + /// The slot with its object, made again where the resource was discarded and counted under `how` (0 a NonVolatile request, + /// 1 another use). The object is made with no lock held: making it is a creation call of the runtime's, which may wait, and + /// nothing of this resource is locked while the runtime works. Where another call placed the resource meanwhile, that object + /// stays and this call's goes. The contents are still the discarded ones (`lost`): the state stays Empty, and the resource may + /// go again once idle. + fn place(&self, how: usize) -> Result>, String> { + let counts = &self.purge.counts; + loop { + let slot = lock(&self.slot); + if slot.object.is_some() { + return Ok(slot); + } + drop(slot); + let made = T::make(&self.purge.device, &self.recipe).map_err(|why| { + counts.failed.fetch_add(1, Relaxed); + format!("placing a discarded resource again: {why}") + })?; + let mut slot = lock(&self.slot); + if slot.object.is_none() { + counts.again[how].fetch_add(1, Relaxed); + counts.again_bytes[how].fetch_add(self.bytes, Relaxed); + counts.out_bytes.fetch_sub(self.bytes, Relaxed); + slot.object = Some(made); + slot.since = Instant::now(); + self.list(); + return Ok(slot); + } + // Released with no lock held. + drop(slot); + drop(made); + } + } + + /// `setPurgeableState:`: Metal's raw answer, with [`AGAIN_FAILED`] where placing again failed. + fn set_state(&self, requested: PurgeableState) -> Result { + let mut slot = lock(&self.slot); + if slot.lost { + // Its contents were discarded: it is Empty, with memory or without, until NonVolatile brings it back. A query and + // Empty change nothing, and Volatile asked of an Empty object leaves it Empty, as Metal does. + if requested != PurgeableState::NonVolatile { + return Ok(PurgeableState::Empty.raw()); + } + if slot.object.is_none() { + drop(slot); + slot = match self.place(0) { + Ok(slot) => slot, + Err(why) => { + crate::objects::set_error(why); + return Ok(PurgeableState::Empty.raw() | AGAIN_FAILED); + } + }; + } + // Still lost unless another NonVolatile request brought it back while this one placed it. + if slot.lost { + slot.lost = false; + slot.state = PurgeableState::NonVolatile; + slot.since = Instant::now(); + self.purge.counts.empty_answers.fetch_add(1, Relaxed); + return Ok(PurgeableState::Empty.raw()); + } + } + // Not lost: the resource has its object, and its state is the runtime's. + let object = slot.object.clone().expect("a resource whose contents were not discarded has its object"); + let answer = object.state(requested)?; + // What the runtime keeps: Volatile asked of an Empty object stays Empty. + let now = match requested { + PurgeableState::KeepCurrent => answer, + _ => object.state(PurgeableState::KeepCurrent)?, + }; + if now != slot.state { + if discardable(now) && !discardable(slot.state) { + let counts = &self.purge.counts; + counts.declared_bytes.fetch_add(self.bytes, Relaxed); + if self.pinned.load(SeqCst) { + counts.declared_pinned_bytes.fetch_add(self.bytes, Relaxed); + } + } + slot.state = now; + slot.since = Instant::now(); + } + if discardable(now) { + self.list(); + } + Ok(answer.raw()) + } +} + +impl Candidate for Cell { + fn sweep(&self, cause: Cause, now: Instant, idle: Duration) -> Swept { + let mut slot = match self.slot.try_lock() { + Ok(slot) => slot, + Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(), + // A call is inside the resource. + Err(TryLockError::WouldBlock) => return Swept::Left(Left::Used, self.bytes), + }; + if slot.object.is_none() || !discardable(slot.state) || self.pinned.load(SeqCst) { + self.listed.store(false, SeqCst); + return Swept::Not; + } + // Read after the lock is taken: a use counts itself before it takes the lock (`Resource::hold`, `tie`, `pin`), so + // either it is seen here or it finds the resource discarded and places it again. + if self.holds.load(SeqCst) != 0 { + return Swept::Left(Left::Used, self.bytes); + } + if self.ties.load(SeqCst) != 0 { + self.listed.store(false, SeqCst); + return Swept::Left(Left::Tied, self.bytes); + } + if cause == Cause::Age { + let idle_since = self.purge.start + Duration::from_millis(self.idle_ms.load(SeqCst)); + if now.saturating_duration_since(slot.since.max(idle_since)) < idle { + return Swept::Left(Left::Young, self.bytes); + } + } + let object = slot.object.take(); + slot.state = PurgeableState::Empty; + slot.since = now; + slot.lost = true; + self.listed.store(false, SeqCst); + drop(slot); + // The runtime's release: the placement is returned once no batch in flight uses it. + drop(object); + Swept::Discarded(self.bytes) + } +} + +impl Drop for Cell { + fn drop(&mut self) { + let slot = self.slot.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()); + if slot.object.is_none() { + self.purge.counts.out_bytes.fetch_sub(self.bytes, Relaxed); + } + } +} + +/// What a tie is made on: a cell of any kind. +trait Tied: Send + Sync { + fn tie(&self); + fn untie(&self); +} + +impl Tied for Cell { + fn tie(&self) { + self.ties.fetch_add(1, SeqCst); + } + + fn untie(&self) { + if self.ties.fetch_sub(1, SeqCst) != 1 { + return; + } + // The last tie went: a candidate again where it is Volatile or Empty with memory. + let slot = lock(&self.slot); + if slot.object.is_some() && discardable(slot.state) { + self.list(); + } + } +} + +/// A view of a resource, a texture over a buffer, or a resource placed in a heap, held by what was made: the resource it was +/// made of is not discarded while it lives. +pub struct Tie(Arc); + +impl Tie { + /// A tie on `cell`, counted before the caller reads the cell's object ([`Cell::sweep`]). + fn on(cell: Arc) -> Tie { + cell.tie(); + Tie(cell) + } +} + +impl Drop for Tie { + fn drop(&mut self) { + self.0.untie(); + } +} + +/// A resource's runtime object as a call uses it: borrowed from a fixed resource, or a handle of its own to a kept one's (a +/// retain, so a discard made while the call runs leaves the call's object whole). +pub enum Live<'a, T> { + Fixed(&'a T), + Placed(T), +} + +impl std::ops::Deref for Live<'_, T> { + type Target = T; + fn deref(&self) -> &T { + match self { + Live::Fixed(object) => object, + Live::Placed(object) => object, + } + } +} + +/// A buffer, texture or heap behind a frontend handle. +pub enum Resource { + /// Never discarded: its object, and its tie on the resource it was made of where that one may be. + Fixed(T, Option), + /// May be discarded while Volatile or Empty and idle. + Kept(Arc>), +} + +impl Resource { + /// `object`, never discarded. + pub fn fixed(object: T) -> Resource { + Resource::Fixed(object, None) + } + + /// `object`, made of the resource `tie` is on, never discarded. + pub fn tied(object: T, tie: Option) -> Resource { + Resource::Fixed(object, tie) + } + + /// `object`, made of `recipe` by the device of `purge`, which may be discarded and made again of it; fixed where the device + /// does not discard, and where the runtime keeps no purgeable state for the object (one with no allocation of its own: under + /// 16 KiB, or memoryless), which can then never be Volatile and whose uses pay for no cell. + pub fn kept(purge: &Arc, object: T, recipe: T::Recipe) -> Resource { + if !purge.on || !keeps_state(&object) { + return Resource::Fixed(object, None); + } + let bytes = object.bytes(); + let slot = Slot { object: Some(object), state: PurgeableState::NonVolatile, since: Instant::now(), lost: false }; + Resource::Kept(Arc::new_cyclic(|me| Cell { + purge: Arc::clone(purge), + recipe, + bytes, + slot: Mutex::new(slot), + holds: AtomicU32::new(0), + ties: AtomicU32::new(0), + pinned: AtomicBool::new(false), + idle_ms: AtomicU64::new(0), + listed: AtomicBool::new(false), + me: me.clone(), + })) + } + + /// The object for a call that uses the resource, placed again where it was discarded. + pub fn live(&self) -> Result, String> { + match self { + Resource::Fixed(object, _) => Ok(Live::Fixed(object)), + Resource::Kept(cell) => cell.live().map(Live::Placed), + } + } + + /// The object where the resource has one, for a question that places nothing: `None` while it is discarded. + pub fn peek(&self) -> Option> { + match self { + Resource::Fixed(object, _) => Some(Live::Fixed(object)), + Resource::Kept(cell) => cell.peek().map(Live::Placed), + } + } + + /// What a kept resource was made of. + pub fn recipe(&self) -> Option<&T::Recipe> { + match self { + Resource::Fixed(..) => None, + Resource::Kept(cell) => Some(&cell.recipe), + } + } + + /// The bytes of device memory the resource holds while placed. + pub fn bytes(&self) -> u64 { + match self { + Resource::Fixed(object, _) => object.bytes(), + Resource::Kept(cell) => cell.bytes, + } + } + + /// `setPurgeableState:` with Metal's state: Metal's raw answer, with [`AGAIN_FAILED`] where the contents were discarded and + /// placing the resource again failed. + pub fn set_state(&self, requested: PurgeableState) -> Result { + match self { + Resource::Fixed(object, _) => Ok(object.state(requested)?.raw()), + Resource::Kept(cell) => cell.set_state(requested), + } + } + + /// A tie for what is about to be made of the resource (a view, a texture over it, a resource placed in it), taken before the + /// resource's object is read: what a fixed resource is itself tied to, since that is whose memory the new one shares. + pub fn tie(&self) -> Option { + match self { + Resource::Fixed(_, tie) => tie.as_ref().map(|tie| Tie::on(Arc::clone(&tie.0))), + Resource::Kept(cell) => { + let cell: Arc = cell.clone(); + Some(Tie::on(cell)) + } + } + } + + /// A command buffer names the resource from now until [`unhold`](Self::unhold): it is not discarded meanwhile. Counted before + /// the command's own use reads the resource's object. + pub fn hold(&self) { + if let Resource::Kept(cell) = self { + cell.holds.fetch_add(1, SeqCst); + } + } + + /// The command buffer that held the resource completed, or went without a commit. + pub fn unhold(&self) { + if let Resource::Kept(cell) = self { + cell.idle_ms.store(cell.purge.now_ms(), SeqCst); + cell.holds.fetch_sub(1, SeqCst); + } + } + + /// The resource's CPU pointer or GPU address is about to be handed out: it is never discarded from now on. Set before the + /// object is read, so a pointer handed out is the pointer of the placement the resource keeps. + pub fn pin(&self) { + if let Resource::Kept(cell) = self { + cell.pinned.store(true, SeqCst); + } + } +} diff --git a/bridge/purge_host.mm b/bridge/purge_host.mm new file mode 100644 index 0000000..021530f --- /dev/null +++ b/bridge/purge_host.mm @@ -0,0 +1,626 @@ +// SPDX-License-Identifier: MIT +// Purgeable memory through the public Metal API (bridge/NativeMetalPurge.rs): a resource declared Volatile is discarded only while +// nothing uses it, answers Empty afterwards, and is usable again; every case reads back what the GPU wrote. +// Arguments: the Vulkan loader, a cache directory, and the mode: "on" (the default behaviour), "off" (run with NATIVE_METAL_PURGE=0 in +// the environment, the boot argument gpu-purge=0: nothing is discarded), "timer" (the age timer discards by itself; waits two ages), +// "stress" (four threads that follow the contract and one that does not, against two threads that discard without pause). +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" + +static id gDeviceUnderTest; +// One queue a thread, kept for the thread's life: the helpers below take the calling thread's. +static thread_local void *tQueue; +static id queue() { + if (!tQueue) tQueue = (__bridge_retained void *)[gDeviceUnderTest newCommandQueue]; + return (__bridge id)tQueue; +} +static std::atomic gFailed; +#define CHECK(c, ...) do { if (!(c)) { std::printf("FAIL line %d: ", __LINE__); std::printf(__VA_ARGS__); std::printf("\n"); gFailed++; } } while (0) + +struct Counts { + uint64_t w[16]; + uint64_t discarded() const {return w[0] + w[2];} + uint64_t discardedBytes() const {return w[1] + w[3];} + uint64_t again() const {return w[4];} + uint64_t byUse() const {return w[6];} + uint64_t emptyAnswers() const {return w[8];} + uint64_t outBytes() const {return w[9];} + uint64_t pinnedBytes() const {return w[11];} + uint64_t leftUse() const {return w[12];} + uint64_t leftTie() const {return w[13];} + uint64_t leftYoung() const {return w[14];} + uint64_t failed() const {return w[15];} +}; +static Counts counts() { + Counts c = {}; + native_bridge_metal_purge_counts(c.w); + return c; +} +static uint64_t pressure() {return native_bridge_metal_purge(0, 0);} + +static bool finish(id commands, const char *what) { + [commands commit]; + [commands waitUntilCompleted]; + if (commands.status == MTLCommandBufferStatusCompleted) return true; + std::printf("FAIL %s: status %lu %s\n", what, (unsigned long)commands.status, commands.error.localizedDescription.UTF8String ?: ""); + gFailed++; + return false; +} +// A side x side RGBA8 texture's texels filled with `value` through a blit from a shared buffer. +static void fill(id texture, unsigned side, uint8_t value) { + id source = [gDeviceUnderTest newBufferWithLength:side * side * 4 options:MTLResourceStorageModeShared]; + memset(source.contents, value, side * side * 4); + id commands = [queue() commandBuffer]; + id blit = [commands blitCommandEncoder]; + [blit copyFromBuffer:source sourceOffset:0 sourceBytesPerRow:side * 4 sourceBytesPerImage:side * side * 4 sourceSize:MTLSizeMake(side, side, 1) + toTexture:texture destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0, 0, 0)]; + [blit endEncoding]; + finish(commands, "fill"); +} +// How many of the texture's bytes differ from `value`, read back through a blit into a shared buffer. +static unsigned wrong(id texture, unsigned side, uint8_t value) { + id out = [gDeviceUnderTest newBufferWithLength:side * side * 4 options:MTLResourceStorageModeShared]; + memset(out.contents, value ^ 0xff, side * side * 4); + id commands = [queue() commandBuffer]; + id blit = [commands blitCommandEncoder]; + [blit copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(side, side, 1) + toBuffer:out destinationOffset:0 destinationBytesPerRow:side * 4 destinationBytesPerImage:side * side * 4]; + [blit endEncoding]; + if (!finish(commands, "readback")) return side * side * 4; + unsigned bad = 0; + const uint8_t *bytes = (const uint8_t *)out.contents; + for (unsigned i = 0; i < side * side * 4; ++i) bad += bytes[i] != value; + return bad; +} +static unsigned wrongBuffer(id buffer, uint8_t value) { + const NSUInteger length = buffer.length; + id out = [gDeviceUnderTest newBufferWithLength:length options:MTLResourceStorageModeShared]; + memset(out.contents, value ^ 0xff, length); + id commands = [queue() commandBuffer]; + id blit = [commands blitCommandEncoder]; + [blit copyFromBuffer:buffer sourceOffset:0 toBuffer:out destinationOffset:0 size:length]; + [blit endEncoding]; + if (!finish(commands, "buffer readback")) return (unsigned)length; + unsigned bad = 0; + const uint8_t *bytes = (const uint8_t *)out.contents; + for (NSUInteger i = 0; i < length; ++i) bad += bytes[i] != value; + return bad; +} +static void fillBuffer(id buffer, uint8_t value) { + id commands = [queue() commandBuffer]; + id blit = [commands blitCommandEncoder]; + [blit fillBuffer:buffer range:NSMakeRange(0, buffer.length) value:value]; + [blit endEncoding]; + finish(commands, "buffer fill"); +} +static id texture(unsigned side, MTLStorageMode mode = MTLStorageModePrivate) { + auto d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:side height:side mipmapped:NO]; + d.storageMode = mode; + d.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView; + return [gDeviceUnderTest newTextureWithDescriptor:d]; +} +// The MiB placed in the runtime's pooled blocks (the telemetry's MEM row). +static unsigned long long placedMiB() { + static char text[16384]; + native_bridge_metal_telemetry(text, sizeof text); + unsigned long long pool = 0, placed = 0; + const char *row = strstr(text, "MEM pool "); + if (!row || sscanf(row, "MEM pool %llu(%llu)", &pool, &placed) != 2) {std::puts("FAIL no MEM row");gFailed++;} + return placed; +} + +// 1: volatile, discard, NonVolatile answers Empty, the texture is usable again. +static void textureCase() { + const unsigned side = 256; + id t = texture(side); + fill(t, side, 0x5a); + CHECK(wrong(t, side, 0x5a) == 0, "texture contents before the discard"); + CHECK([t setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateNonVolatile, "Volatile answers the state before"); + CHECK([t setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateVolatile, "the state is Volatile"); + const Counts before = counts(); + const uint64_t bytes = pressure(); + Counts after = counts(); + CHECK(bytes >= side * side * 4 && after.discarded() == before.discarded() + 1 && after.outBytes() == before.outBytes() + bytes, + "an idle Volatile texture is discarded (%llu bytes, %llu discards)", (unsigned long long)bytes, (unsigned long long)(after.discarded() - before.discarded())); + // A query answers Empty and places nothing. + CHECK([t setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "a discarded texture answers Empty to a query"); + CHECK(counts().outBytes() == after.outBytes(), "a query places nothing"); + // Volatile asked of an Empty object leaves it Empty. + CHECK([t setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateEmpty, "Volatile on a discarded texture answers Empty"); + CHECK([t setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "and leaves it Empty"); + CHECK([t setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "NonVolatile answers Empty for discarded contents"); + after = counts(); + CHECK(after.again() == before.again() + 1 && after.outBytes() == before.outBytes() && after.emptyAnswers() == before.emptyAnswers() + 1, + "NonVolatile places the texture again"); + CHECK([t setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateNonVolatile, "the state is NonVolatile again"); + CHECK(wrong(t, side, 0) == 0, "a texture placed again reads zero"); + fill(t, side, 0xa7); + CHECK(wrong(t, side, 0xa7) == 0, "a texture placed again is written and read"); + CHECK(pressure() == 0, "a NonVolatile texture is not discarded"); + CHECK(wrong(t, side, 0xa7) == 0, "and keeps its contents"); +} + +// 2: a texture a batch in flight uses, and one an encoder being recorded names, are not discarded. +static void inFlightCase() { + const unsigned side = 128; + id t = texture(side); + id source = [gDeviceUnderTest newBufferWithLength:side * side * 4 options:MTLResourceStorageModeShared]; + memset(source.contents, 0x3c, side * side * 4); + // In flight: committed behind an event that is not signaled yet. + id event = [gDeviceUnderTest newSharedEvent]; + id commands = [queue() commandBuffer]; + [commands encodeWaitForEvent:event value:1]; + id blit = [commands blitCommandEncoder]; + [blit copyFromBuffer:source sourceOffset:0 sourceBytesPerRow:side * 4 sourceBytesPerImage:side * side * 4 sourceSize:MTLSizeMake(side, side, 1) + toTexture:t destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0, 0, 0)]; + [blit endEncoding]; + [commands commit]; + [t setPurgeableState:MTLPurgeableStateVolatile]; + Counts before = counts(); + CHECK(pressure() == 0, "a texture a batch in flight writes is not discarded"); + Counts after = counts(); + CHECK(after.discarded() == before.discarded() && after.leftUse() >= side * side * 4, "it is left as in use (%llu bytes)", (unsigned long long)after.leftUse()); + event.signaledValue = 1; + [commands waitUntilCompleted]; + CHECK(commands.status == MTLCommandBufferStatusCompleted, "the gated command buffer completed"); + // The completion handler's turn releases the hold; the texture still has what the batch wrote. + CHECK([t setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateVolatile, "it was not discarded: NonVolatile answers Volatile"); + CHECK(wrong(t, side, 0x3c) == 0, "and has what the batch wrote"); + + // Being recorded: an encoder of a command buffer that is not committed names it. + id recording = [queue() commandBuffer]; + id open = [recording blitCommandEncoder]; + memset(source.contents, 0x6d, side * side * 4); + [open copyFromBuffer:source sourceOffset:0 sourceBytesPerRow:side * 4 sourceBytesPerImage:side * side * 4 sourceSize:MTLSizeMake(side, side, 1) + toTexture:t destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0, 0, 0)]; + [t setPurgeableState:MTLPurgeableStateVolatile]; + before = counts(); + CHECK(pressure() == 0 && counts().discarded() == before.discarded(), "a texture an encoder being recorded names is not discarded"); + [open endEncoding]; + finish(recording, "recorded copy"); + CHECK([t setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateVolatile, "it was not discarded while recorded"); + CHECK(wrong(t, side, 0x6d) == 0, "and has what the recorded copy wrote"); + // Idle now: it goes. + [t setPurgeableState:MTLPurgeableStateVolatile]; + for (unsigned i = 0; i < 50 && !pressure(); ++i) usleep(10000); // the completion's release of the hold may still be on its thread + CHECK([t setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "once nothing uses it, it is discarded"); + + // A command buffer released without a commit lets go of what it named. + id dropped = texture(side); + @autoreleasepool { + id never = [queue() commandBuffer]; + id unused = [never blitCommandEncoder]; + [unused generateMipmapsForTexture:dropped]; + [unused endEncoding]; + [dropped setPurgeableState:MTLPurgeableStateVolatile]; + CHECK(pressure() == 0, "named by a command buffer that is not committed"); + never = nil;unused = nil; + } + for (unsigned i = 0; i < 50 && !pressure(); ++i) usleep(10000); + CHECK([dropped setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "a command buffer released without a commit holds nothing"); +} + +// 3: views. A texture with a view is not discarded; a view made of a discarded texture shares the memory it is placed in again. +static void viewCase() { + const unsigned side = 128; + id parent = texture(side); + fill(parent, side, 0x11); + @autoreleasepool { + id view = [parent newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + // A view keeps no state of its own, as Metal's. + CHECK([view setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateNonVolatile && + [view setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateNonVolatile, "a view keeps no purgeable state"); + CHECK(pressure() == 0, "a parent that is not Volatile is not discarded for its view's request"); + [parent setPurgeableState:MTLPurgeableStateVolatile]; + const Counts before = counts(); + CHECK(pressure() == 0, "a Volatile texture with a live view is not discarded"); + CHECK(counts().leftTie() >= side * side * 4 && counts().discarded() == before.discarded(), "it is left as tied"); + CHECK(wrong(view, side, 0x11) == 0, "the view reads the parent's texels"); + view = nil; + } + CHECK(pressure() > 0, "once its view is gone the Volatile texture is discarded"); + // A view of a discarded parent: a use without NonVolatile first. The parent is placed again (zeroed) and the view shares it. + const Counts before = counts(); + id view = [parent newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + CHECK(view != nil, "a view of a discarded texture is made"); + CHECK(counts().byUse() == before.byUse() + 1, "making it placed the parent again"); + CHECK(wrong(view, side, 0) == 0, "the view reads zero"); + fill(view, side, 0x77); + CHECK(wrong(parent, side, 0x77) == 0, "what is written through the view is read through the parent"); + CHECK([parent setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "the parent's state is still Empty"); + CHECK([parent setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "NonVolatile answers Empty once"); + CHECK([parent setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateNonVolatile, "and NonVolatile after"); + CHECK(wrong(parent, side, 0x77) == 0, "the parent was not placed a second time"); +} + +// 4: heaps. A heap that holds a resource is not discarded; an empty one is, answers as an empty heap, and serves again. +static void heapCase() { + auto hd = [MTLHeapDescriptor new]; + hd.size = 1 << 20; + hd.storageMode = MTLStorageModePrivate; + id heap = [gDeviceUnderTest newHeapWithDescriptor:hd]; + CHECK(heap != nil, "heap"); + const NSUInteger size = heap.size, room = [heap maxAvailableSizeWithAlignment:256]; + @autoreleasepool { + id placed = [heap newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + CHECK(placed != nil && heap.usedSize == 65536, "a buffer in the heap"); + fillBuffer(placed, 0x42); + [heap setPurgeableState:MTLPurgeableStateVolatile]; + const Counts before = counts(); + CHECK(pressure() == 0 && counts().discarded() == before.discarded(), "a Volatile heap that holds a buffer is not discarded"); + CHECK(counts().leftTie() >= size, "it is left as tied"); + CHECK(wrongBuffer(placed, 0x42) == 0, "the buffer keeps its bytes"); + placed = nil; + } + uint64_t bytes = 0; + for (unsigned i = 0; i < 50 && !(bytes = pressure()); ++i) usleep(10000); + CHECK(bytes == size, "an empty Volatile heap is discarded whole (%llu of %lu)", (unsigned long long)bytes, (unsigned long)size); + // A discarded heap answers as the empty heap it is, and places nothing for it. + const uint64_t out = counts().outBytes(); + CHECK(heap.size == size && heap.usedSize == 0 && heap.currentAllocatedSize == size && [heap maxAvailableSizeWithAlignment:256] == room, + "a discarded heap answers as an empty heap"); + CHECK([heap setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty && counts().outBytes() == out, "and stays discarded"); + CHECK([heap setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "NonVolatile answers Empty"); + id again = [heap newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + CHECK(again != nil && heap.usedSize == 65536, "the heap serves a buffer again"); + CHECK(wrongBuffer(again, 0) == 0, "which reads zero"); + fillBuffer(again, 0x99); + CHECK(wrongBuffer(again, 0x99) == 0, "and is written and read"); + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:64 mipmapped:NO]; + td.storageMode = MTLStorageModePrivate; + td.usage = MTLTextureUsageShaderRead; + id heapTexture = [heap newTextureWithDescriptor:td]; + CHECK(heapTexture != nil, "the heap serves a texture again"); + fill(heapTexture, 64, 0x2e); + CHECK(wrong(heapTexture, 64, 0x2e) == 0, "which is written and read"); + again = nil;heapTexture = nil; + + // A resource made in a discarded heap without NonVolatile first: the heap is placed again by the use. + auto hd2 = [MTLHeapDescriptor new]; + hd2.size = 1 << 20; + hd2.storageMode = MTLStorageModeShared; + id shared = [gDeviceUnderTest newHeapWithDescriptor:hd2]; + [shared setPurgeableState:MTLPurgeableStateVolatile]; + CHECK(pressure() >= shared.size, "an empty Shared heap is discarded"); + const uint64_t uses = counts().byUse(); + id late = [shared newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + CHECK(late != nil && late.contents != nullptr && counts().byUse() == uses + 1, "a buffer made in a discarded heap places the heap again"); + memset(late.contents, 0x31, 4096); + CHECK(wrongBuffer(late, 0x31) == 0, "and is written by the CPU and read by the GPU"); + CHECK(pressure() == 0, "the heap that holds it stays"); +} + +// 5: buffers. One whose CPU pointer was handed out is never discarded; one without is. +static void bufferCase() { + id pinned = [gDeviceUnderTest newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + uint8_t *pointer = (uint8_t *)pinned.contents; + memset(pointer, 0x21, 65536); + Counts before = counts(); + CHECK([pinned setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateNonVolatile, "a Shared buffer keeps its state"); + CHECK(pressure() == 0 && counts().discarded() == before.discarded(), "a buffer whose pointer was handed out is not discarded"); + CHECK(counts().pinnedBytes() == before.pinnedBytes() + 65536, "it is counted as never discardable"); + CHECK(pinned.contents == pointer && pointer[0] == 0x21 && pointer[65535] == 0x21, "its pointer and bytes stay"); + CHECK(wrongBuffer(pinned, 0x21) == 0, "and the GPU reads them"); + CHECK([pinned setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateVolatile, "NonVolatile answers Volatile"); + + id priv = [gDeviceUnderTest newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + fillBuffer(priv, 0x64); + CHECK(priv.contents == nullptr, "a Private buffer has no pointer"); + [priv setPurgeableState:MTLPurgeableStateVolatile]; + uint64_t bytes = 0; + for (unsigned i = 0; i < 50 && !(bytes = pressure()); ++i) usleep(10000); + CHECK(bytes == 65536, "an idle Volatile Private buffer is discarded"); + CHECK(priv.length == 65536, "its length stays"); + CHECK([priv setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "NonVolatile answers Empty"); + CHECK(wrongBuffer(priv, 0) == 0, "a buffer placed again reads zero"); + fillBuffer(priv, 0x65); + CHECK(wrongBuffer(priv, 0x65) == 0, "and is written and read"); + + // A Shared buffer whose pointer was never asked for goes; the pointer asked for afterwards is the new placement's, and stays. + std::vector initial(65536, 0x17); + id unseen = [gDeviceUnderTest newBufferWithBytes:initial.data() length:initial.size() options:MTLResourceStorageModeShared]; + CHECK(wrongBuffer(unseen, 0x17) == 0, "a buffer made with bytes"); + [unseen setPurgeableState:MTLPurgeableStateVolatile]; + bytes = 0; + for (unsigned i = 0; i < 50 && !(bytes = pressure()); ++i) usleep(10000); + CHECK(bytes == 65536, "a Shared buffer whose pointer was never handed out is discarded"); + uint8_t *late = (uint8_t *)unseen.contents; + CHECK(late != nullptr && late[0] == 0 && late[65535] == 0, "its pointer, asked for after the discard, reads zero"); + CHECK([unseen setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "its state is Empty"); + memset(late, 0x18, 65536); + CHECK(pressure() == 0 && unseen.contents == late && late[0] == 0x18, "and it is never discarded again"); + CHECK([unseen setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "NonVolatile answers Empty once"); + CHECK(wrongBuffer(unseen, 0x18) == 0, "the GPU reads what the CPU wrote"); + + // A buffer whose GPU address was handed out is never discarded either. + id addressed = [gDeviceUnderTest newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + const uint64_t address = addressed.gpuAddress; + [addressed setPurgeableState:MTLPurgeableStateVolatile]; + CHECK(pressure() == 0 && addressed.gpuAddress == address, "a buffer whose GPU address was handed out is not discarded"); +} + +// 6: what never has a state: a texture too small for an allocation of its own, and a texture over a surface. +static void fixedCase() { + id small = texture(16); + fill(small, 16, 0x44); + CHECK([small setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateNonVolatile && + [small setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateNonVolatile, "a small texture keeps no state"); + NSDictionary *p = @{(id)kIOSurfaceWidth:@64, (id)kIOSurfaceHeight:@64, (id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}; + IOSurfaceRef surface = IOSurfaceCreate((__bridge CFDictionaryRef)p); + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:64 mipmapped:NO]; + td.storageMode = MTLStorageModeShared; + td.usage = MTLTextureUsageShaderRead | MTLTextureUsageRenderTarget; + id onSurface = surface ? [gDeviceUnderTest newTextureWithDescriptor:td iosurface:surface plane:0] : nil; + if (onSurface) { + fill(onSurface, 64, 0x58); + CHECK([onSurface setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateNonVolatile && + [onSurface setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateNonVolatile, "a texture over a surface keeps no state"); + } else std::puts("note: no IOSurface texture on this host; the surface case is not run"); + const Counts before = counts(); + CHECK(pressure() == 0 && counts().discarded() == before.discarded(), "neither is discarded"); + CHECK(wrong(small, 16, 0x44) == 0, "the small texture keeps its texels"); + if (onSurface) CHECK(wrong(onSurface, 64, 0x58) == 0, "the surface's texture keeps its texels"); + if (surface) CFRelease(surface); +} + +// 7: age, and a discarded texture used by a command without NonVolatile first. +static void ageAndUseCase() { + const unsigned side = 128; + id t = texture(side); + fill(t, side, 0x0f); + [t setPurgeableState:MTLPurgeableStateVolatile]; + Counts before = counts(); + CHECK(native_bridge_metal_purge(1, 3600000) == 0, "a texture Volatile for less than the age is not discarded by age"); + CHECK(counts().leftYoung() >= side * side * 4 && counts().discarded() == before.discarded(), "it is left as young"); + usleep(60000); + CHECK(native_bridge_metal_purge(1, 50) > 0, "idle for the age, it is discarded by age"); + CHECK(counts().w[2] == before.w[2] + 1, "counted under age"); + // Used by a command with no NonVolatile first (the caller's error): placed again, zeroed, and the command runs. + before = counts(); + CHECK(wrong(t, side, 0) == 0, "a discarded texture a command reads is placed again and reads zero"); + CHECK(counts().byUse() == before.byUse() + 1 && counts().failed() == 0, "counted as placed again by a use"); + CHECK([t setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, "its state is still Empty"); + // Empty with memory and idle: it may go again. + uint64_t again = 0; + for (unsigned i = 0; i < 50 && !(again = pressure()); ++i) usleep(10000); + CHECK(again > 0, "Empty with memory and idle, it is discarded again"); + CHECK([t setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "and answers Empty to NonVolatile"); + // replaceRegion and getBytes on a discarded Shared texture. + id shared = texture(64, MTLStorageModeShared); + [shared setPurgeableState:MTLPurgeableStateVolatile]; + CHECK(pressure() > 0, "a Shared texture is discarded"); + std::vector texels(64 * 64 * 4, 0x73), read(64 * 64 * 4, 0xff); + [shared replaceRegion:MTLRegionMake2D(0, 0, 64, 64) mipmapLevel:0 withBytes:texels.data() bytesPerRow:64 * 4]; + [shared getBytes:read.data() bytesPerRow:64 * 4 fromRegion:MTLRegionMake2D(0, 0, 64, 64) mipmapLevel:0]; + CHECK(read == texels, "replaceRegion and getBytes work on a texture that was discarded"); +} + +// 8: the memory comes back: placed bytes fall once the discarded textures' placements are returned. +static void memoryCase() { + const unsigned side = 1024, count = 6; // 4 MiB each + NSMutableArray> *textures = [NSMutableArray array]; + for (unsigned i = 0; i < count; ++i) { + id t = texture(side); + fill(t, side, (uint8_t)(0x80 + i)); + [textures addObject:t]; + } + const unsigned long long before = placedMiB(); + for (id t in textures) [t setPurgeableState:MTLPurgeableStateVolatile]; + uint64_t bytes = 0; + for (unsigned i = 0; i < 100 && bytes < (uint64_t)count * side * side * 4; ++i) {bytes += native_bridge_metal_relieve_memory();usleep(10000);} + CHECK(bytes >= (uint64_t)count * side * side * 4, "six 4 MiB textures are discarded (%llu bytes)", (unsigned long long)bytes); + native_bridge_metal_relieve_memory(); // the objects parked at the discard go, and the blocks they emptied + const unsigned long long after = placedMiB(); + std::printf("placed %lluM before, %lluM after the discard of %lluM\n", before, after, (unsigned long long)(bytes >> 20)); + CHECK(after + 20 <= before, "the placed bytes fall by the discarded textures' (%lluM to %lluM)", before, after); + for (unsigned i = 0; i < count; ++i) { + CHECK([textures[i] setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "texture %u answers Empty", i); + fill(textures[i], side, (uint8_t)(0x90 + i)); + } + for (unsigned i = 0; i < count; ++i) CHECK(wrong(textures[i], side, (uint8_t)(0x90 + i)) == 0, "texture %u is written and read again", i); + + // By time alone: after a discard with no pressure event, the runtime parks the released images for a second (its recycler) + // and its room thread lets them go when that time is up, though the process makes and releases nothing more (bridge patch + // 0034). Nothing is made or released here between the discard and the reading. + usleep(300000); // the readbacks' own buffers and holds are gone + const unsigned long long placed = placedMiB(); + for (id t in textures) [t setPurgeableState:MTLPurgeableStateVolatile]; + bytes = 0; + for (unsigned i = 0; i < 100 && bytes < (uint64_t)count * side * side * 4; ++i) {bytes += pressure();usleep(10000);} + CHECK(bytes >= (uint64_t)count * side * side * 4, "the six textures are discarded again (%llu bytes)", (unsigned long long)bytes); + const unsigned long long parked = placedMiB(); + usleep(2500000); + const unsigned long long aged = placedMiB(); + std::printf("placed %lluM before, %lluM right after the discard (parked), %lluM 2.5 s later\n", placed, parked, aged); + CHECK(aged + 20 <= placed, "the parked images go by time, with nothing made or released (%lluM to %lluM)", placed, aged); +} + +// gpu-purge=0: the state is kept, nothing is discarded, the answers are the runtime's. +static void offCase() { + const unsigned side = 256; + id t = texture(side); + fill(t, side, 0x5a); + CHECK([t setPurgeableState:MTLPurgeableStateVolatile] == MTLPurgeableStateNonVolatile, "Volatile answers NonVolatile"); + CHECK(pressure() == 0 && native_bridge_metal_purge(1, 0) == 0 && native_bridge_metal_relieve_memory() == 0, "nothing is discarded"); + CHECK([t setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateVolatile, "NonVolatile answers Volatile"); + CHECK(wrong(t, side, 0x5a) == 0, "the texture keeps its texels"); + auto hd = [MTLHeapDescriptor new]; + hd.size = 1 << 20; + id heap = [gDeviceUnderTest newHeapWithDescriptor:hd]; + [heap setPurgeableState:MTLPurgeableStateVolatile]; + id priv = [gDeviceUnderTest newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + fillBuffer(priv, 0x64); + [priv setPurgeableState:MTLPurgeableStateVolatile]; + CHECK(pressure() == 0, "nor a heap or a buffer"); + CHECK([heap setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateVolatile && + [priv setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateVolatile, "both answer Volatile"); + CHECK(wrongBuffer(priv, 0x64) == 0, "the buffer keeps its bytes"); + const Counts c = counts(); + for (uint64_t word : c.w) CHECK(word == 0, "every purge count is zero"); +} + +// The age timer by itself: a Volatile texture nothing uses goes within two ages, with no call from here. +static void timerCase() { + const unsigned side = 256; + id t = texture(side); + fill(t, side, 0x5a); + [t setPurgeableState:MTLPurgeableStateVolatile]; + const uint64_t age = nmb_purge_age_ms(); + const Counts before = counts(); + for (uint64_t waited = 0; waited < age * 2 + 2000 && counts().w[2] == before.w[2]; waited += 250) usleep(250000); + CHECK(counts().w[2] == before.w[2] + 1, "the age timer discarded the idle Volatile texture (age %llu ms)", (unsigned long long)age); + CHECK([t setPurgeableState:MTLPurgeableStateNonVolatile] == MTLPurgeableStateEmpty, "which answers Empty"); + fill(t, side, 0x66); + CHECK(wrong(t, side, 0x66) == 0, "and is usable again"); + + // A heap declared Volatile while it holds a buffer: the timer finds it tied and stops; the buffer's release starts it again. + auto hd = [MTLHeapDescriptor new]; + hd.size = 1 << 20; + id heap = [gDeviceUnderTest newHeapWithDescriptor:hd]; + const Counts held = counts(); + @autoreleasepool { + id placed = [heap newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + CHECK(placed != nil, "a buffer in the heap"); + [heap setPurgeableState:MTLPurgeableStateVolatile]; + usleep((useconds_t)(age / 2 + 1000) * 1000); // one pass of the timer + CHECK(counts().w[2] == held.w[2] && [heap setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateVolatile, "the heap that holds a buffer stays"); + placed = nil; + } + for (uint64_t waited = 0; waited < age * 2 + 2000 && counts().w[2] == held.w[2]; waited += 250) usleep(250000); + CHECK(counts().w[2] == held.w[2] + 1 && [heap setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty, + "the age timer discarded the heap after its buffer went"); +} + +// Threads that follow the contract (NonVolatile before a use, the contents made again on Empty, Volatile after) never read +// anything but what they last wrote, or zero right after an Empty answer, while two threads discard without pause; a thread that +// uses the same resources with no NonVolatile first (the caller's error) completes every command. +static void stressCase() { + const unsigned side = 64, workers = 4, each = 6; + struct Item {id texture;id buffer;std::atomic busy;uint8_t texels, bytes;}; + static Item items[workers * each]; + for (Item &item : items) { + item.texture = texture(side); + item.buffer = [gDeviceUnderTest newBufferWithLength:65536 options:MTLResourceStorageModePrivate]; + item.texels = item.bytes = 0; // new resources read zero + item.busy = false; + [item.texture setPurgeableState:MTLPurgeableStateVolatile]; + [item.buffer setPurgeableState:MTLPurgeableStateVolatile]; + } + static std::atomic stop; + static std::atomic rounds, empties, rogue; + std::vector threads; + for (unsigned sweeper = 0; sweeper < 2; ++sweeper) + threads.emplace_back([sweeper] { + while (!stop) { + if (sweeper) native_bridge_metal_relieve_memory(); else native_bridge_metal_purge(1, 0); + usleep(200); + } + }); + for (unsigned worker = 0; worker < workers; ++worker) + threads.emplace_back([worker] { + @autoreleasepool { + uint8_t next = (uint8_t)(1 + worker * 50); + for (unsigned round = 0; !stop; ++round) { + @autoreleasepool { + Item &item = items[worker * each + round % each]; + // The rogue thread may be inside this item: the contract is one thread's, so take turns. + if (item.busy.exchange(true)) continue; + const MTLPurgeableState texture = [item.texture setPurgeableState:MTLPurgeableStateNonVolatile]; + const MTLPurgeableState buffer = [item.buffer setPurgeableState:MTLPurgeableStateNonVolatile]; + if (texture == MTLPurgeableStateEmpty) {item.texels = 0;empties++;} + if (buffer == MTLPurgeableStateEmpty) {item.bytes = 0;empties++;} + CHECK(wrong(item.texture, side, item.texels) == 0, "worker %u round %u: texture differs from %#x (answer %lu)", worker, round, item.texels, (unsigned long)texture); + CHECK(wrongBuffer(item.buffer, item.bytes) == 0, "worker %u round %u: buffer differs from %#x (answer %lu)", worker, round, item.bytes, (unsigned long)buffer); + if (++next == 0) next = 1; + fill(item.texture, side, next);item.texels = next; + fillBuffer(item.buffer, next);item.bytes = next; + CHECK(wrong(item.texture, side, next) == 0 && wrongBuffer(item.buffer, next) == 0, "worker %u round %u: what was just written", worker, round); + [item.texture setPurgeableState:MTLPurgeableStateVolatile]; + [item.buffer setPurgeableState:MTLPurgeableStateVolatile]; + item.busy = false; + rounds++; + } + } + } + }); + // The rogue: reads and views resources with no NonVolatile first. Whatever it reads, its commands complete and nothing crashes. + threads.emplace_back([] { + @autoreleasepool { + for (unsigned round = 0; !stop; ++round) { + @autoreleasepool { + Item &item = items[round % (workers * each)]; + if (item.busy.exchange(true)) continue; + // Its uses may place a discarded resource again: what the owner then finds is zero, with its state Empty. + const bool emptyBefore = [item.texture setPurgeableState:MTLPurgeableStateKeepCurrent] == MTLPurgeableStateEmpty; + id view = [item.texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + CHECK(view != nil, "rogue view"); + const unsigned bad = wrong(view, side, item.texels), zero = wrong(view, side, 0); + CHECK(bad == 0 || zero == 0, "rogue round %u: a texture reads neither its last contents nor zero (was Empty %d)", round, emptyBefore); + (void)wrongBuffer(item.buffer, item.bytes); + view = nil; + item.busy = false; + rogue++; + } + } + } + }); + sleep(6); + stop = true; + for (std::thread &thread : threads) thread.join(); + const Counts c = counts(); + std::printf("stress: %llu contract rounds, %llu Empty answers handled, %llu rogue rounds; discarded %llu, placed again %llu by request and %llu by use\n", + (unsigned long long)rounds.load(), (unsigned long long)empties.load(), (unsigned long long)rogue.load(), + (unsigned long long)c.discarded(), (unsigned long long)c.again(), (unsigned long long)c.byUse()); + CHECK(rounds > 50 && empties > 10 && rogue > 10 && c.discarded() > 10 && c.failed() == 0, "the stress ran: rounds, discards and no failed placement"); + for (Item &item : items) {item.texture = nil;item.buffer = nil;} +} + +int main(int argc, char **argv) { + if (argc != 4) return 2; + const std::string mode = argv[3]; + @autoreleasepool { + gDeviceUnderTest = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!gDeviceUnderTest) return 2; + if (mode == "off") offCase(); + else if (mode == "timer") timerCase(); + else if (mode == "stress") stressCase(); + else { + @autoreleasepool {textureCase();} + @autoreleasepool {inFlightCase();} + @autoreleasepool {viewCase();} + @autoreleasepool {heapCase();} + @autoreleasepool {bufferCase();} + @autoreleasepool {fixedCase();} + @autoreleasepool {ageAndUseCase();} + @autoreleasepool {memoryCase();} + // The telemetry's rows of it, each within 90 characters. + static char text[16384]; + native_bridge_metal_telemetry(text, sizeof text); + for (const char *name : {"\nPG req ", "\nPGb out ", "\nPGc decl "}) { + const char *row = strstr(text, name); + CHECK(row != nullptr, "telemetry row%s", name + 1); + if (row) std::printf("%.*s\n", (int)strcspn(row + 1, "\n"), row + 1); + } + const Counts c = counts(); + CHECK(c.failed() == 0, "no placement again failed"); + std::printf("discarded %llu (%lluK by pressure, %lluK by age), placed again %llu by request and %llu by use, %llu Empty answers\n", + (unsigned long long)c.discarded(), (unsigned long long)(c.w[1] >> 10), (unsigned long long)(c.w[3] >> 10), + (unsigned long long)c.again(), (unsigned long long)c.byUse(), (unsigned long long)c.emptyAnswers()); + } + if (gFailed) {std::printf("FAIL: %u checks\n", gFailed.load());return 1;} + if (mode == "off") std::puts("PASS: with NATIVE_METAL_PURGE=0 the purgeable state is kept and nothing is discarded"); + else if (mode == "timer") std::puts("PASS: the age timer discards an idle Volatile texture, which answers Empty and is usable again"); + else if (mode == "stress") std::puts("PASS: under constant discards, callers that follow the contract read only what they wrote or zero after Empty; uses without it complete"); + else std::puts("PASS: Volatile resources are discarded only while idle, answer Empty, and are usable again; views, heaps, buffers, in-flight use, age and memory"); + } +} diff --git a/bridge/telemetry_host.mm b/bridge/telemetry_host.mm index de6ebc0..0794e83 100644 --- a/bridge/telemetry_host.mm +++ b/bridge/telemetry_host.mm @@ -56,7 +56,7 @@ int main(int argc, char **argv) { if (!line.compare(0, 3, "MK ") || !line.compare(0, 3, "ML ") || !line.compare(0, 3, "VP ") || !line.compare(0, 3, "TH ")) std::printf("%s\n", line.c_str()); begin = end + 1; } - for (const char *name : {"MEM", "MK", "MKb", "MKc", "ML", "PG", "PE", "PS", "T", "VP", "V", "SC", "PR", "TH", "P0", "S"}) { + for (const char *name : {"MEM", "MK", "MKb", "MKc", "ML", "PG", "PGb", "PGc", "PE", "PS", "T", "VP", "V", "SC", "PR", "TH", "P0", "S"}) { const auto found = at.find(name); if (found == at.end() || found->second >= 48) {std::printf("FAIL telemetry row %s missing from the first 48 rows\n", name);return 1;} }