From 5282bd96c73bd0e2e111c8380ce3c8b0c5c19ba9 Mon Sep 17 00:00:00 2001 From: rponeawa Date: Wed, 11 Nov 2026 00:00:00 +0800 Subject: [PATCH] fix(pool): map a pool run only while it is zero-filled Registering the pool mapped every range whole, write-combined, into the kernel map before the GPU's own windows were mapped, and on sheng the GPU then did not power up (GMU start stalled, no SMMU window). The kernel keeps no mapping of a range now: the value the allocator carries for a range is its physical address, and each scrub maps just the run it fills for the call. Host tests pass; not yet run on hardware. --- NativeAdrenoCpu.cpp | 46 +++++++++++++++++++++++---------------------- NativeAdrenoPool.h | 14 ++++++++------ 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/NativeAdrenoCpu.cpp b/NativeAdrenoCpu.cpp index 07d87c2..81e2c25 100644 --- a/NativeAdrenoCpu.cpp +++ b/NativeAdrenoCpu.cpp @@ -171,34 +171,36 @@ IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans __atomic_fetch_add(&native_adreno_reassert_ticks,mach_absolute_time()-began,__ATOMIC_RELAXED); return kIOReturnSuccess; } -/* A pool range's kernel mapping: an IOKit mapping of the physical range, write-combined (Normal non-cacheable), in the kernel map's - * ordinary address space. ml_io_map_wcomb is not used: it takes permanent addresses from the kernel's 1 GiB I/O range, which the - * registers of every device share and which panics when it runs out. The mapping is never removed; *owner keeps the object. */ +/* The kernel keeps no mapping of a pool range: a whole range mapped at registration (several hundred MiB, write-combined, in the + * kernel map, before the GPU's own windows are mapped) kept the GPU from powering up on sheng. The value the allocator takes for a + * range's "mapping" is the range's physical address, and each scrub maps just the run it fills for the time it takes. */ void *NativeAdrenoPoolMapRange(uint64_t physical,size_t bytes,void **owner) { if(!owner || !physical || !bytes || ((physical|bytes)&4095))return nullptr; - IOMemoryDescriptor *range=IOMemoryDescriptor::withPhysicalAddress((IOPhysicalAddress)physical,bytes,kIODirectionInOut); - if(!range)return nullptr; - IOMemoryMap *map=range->createMappingInTask(kernel_task,0,kIOMapAnywhere|kIOMapWriteCombineCache); - range->release(); - if(!map)return nullptr; - void *const address=reinterpret_cast(map->getVirtualAddress()); - if(!address || ((uintptr_t)address&4095) || map->getLength()!=bytes) {map->release();return nullptr;} - *owner=map; - return address; + *owner=nullptr; + return reinterpret_cast(physical); } /* Before a pool page goes to its next owner. A page a client could map may have lines in a CPU cache (the client chooses the cache - * mode of its own IOKit mapping): they are dropped first, by the range mapping's addresses, so none is written over the zeros or - * over the next owner's data later. The pages have no other mapping at this point, so nothing is lost with a dropped line. The zero - * fill goes through the write-combined mapping and the barrier completes it before the GPU or a client can read the pages. */ + * mode of its own IOKit mapping): they are dropped first, by the run mapping's addresses, so none is written over the zeros or over + * the next owner's data later. The pages have no other mapping at this point, so nothing is lost with a dropped line. The zero fill + * goes through a write-combined mapping of the run made for it, and the barrier completes it before the GPU or a client can read + * the pages; the mapping is gone again before the call returns. */ IOReturn NativeAdrenoPoolScrub(void *mapping,size_t bytes,bool drop_lines) { - if(!mapping || !bytes || (((uintptr_t)mapping|bytes)&4095))return kIOReturnBadArgument; - if(drop_lines) { - const IOReturn dropped=maintainOwnedPages(mapping,bytes,NativeAdrenoInvalidate); - if(dropped)return dropped; + const uint64_t physical=reinterpret_cast(mapping); + if(!physical || !bytes || ((physical|bytes)&4095))return kIOReturnBadArgument; + IOMemoryDescriptor *run=IOMemoryDescriptor::withPhysicalAddress((IOPhysicalAddress)physical,bytes,kIODirectionInOut); + if(!run)return kIOReturnNoMemory; + IOMemoryMap *map=run->createMappingInTask(kernel_task,0,kIOMapAnywhere|kIOMapWriteCombineCache); + run->release(); + if(!map)return kIOReturnNoMemory; + void *const address=reinterpret_cast(map->getVirtualAddress()); + IOReturn status=(!address || ((uintptr_t)address&4095) || map->getLength()!=bytes)?kIOReturnNoMemory:kIOReturnSuccess; + if(!status && drop_lines)status=maintainOwnedPages(address,bytes,NativeAdrenoInvalidate); + if(!status) { + bzero(address,bytes); + __asm__ volatile("dsb sy":::"memory"); } - bzero(mapping,bytes); - __asm__ volatile("dsb sy":::"memory"); - return kIOReturnSuccess; + map->release(); + return status; } diff --git a/NativeAdrenoPool.h b/NativeAdrenoPool.h index a6d37c1..5634ab3 100644 --- a/NativeAdrenoPool.h +++ b/NativeAdrenoPool.h @@ -9,10 +9,10 @@ * vm_page for such a page and phystokv panics on it, so nothing but this allocator, the SMMU tables and non-cacheable CPU mappings * ever names one: * - the allocator's own state (a bitmap a range) lives in kernel memory; - * - each range has one write-combined kernel mapping, made at registration, which is the only way the kernel writes the pages; - * - every page of a grant is zero-filled through that mapping before the grant leaves NativeAdrenoPoolAllocate, whatever held the + * - the kernel writes the pages only through a write-combined mapping of a run, made for its zero fill and gone after it; + * - every page of a grant is zero-filled through such a mapping before the grant leaves NativeAdrenoPoolAllocate, whatever held the * page before (boot firmware, another client), so a client never reads what it did not write. Freed pages keep their old bytes - * until then; no mapping but the range's own reaches a free page; + * until then; no kernel mapping reaches a free page; * - pages a client could map (a grant that had a client view) get their CPU cache lines dropped before that zero fill: a client can * ask IOKit for a cacheable view, and a line it left dirty would otherwise be written over the next owner's data later; * - a grant is reference counted: the BO holds one reference and every client view descriptor one, so its end comes only when no @@ -23,7 +23,7 @@ * gives such pages back instead (after the cache pass above), for a system whose GPU clients are trusted not to do that. * Lock order: the allocator's gate is a leaf. It is taken under the driver's lock, under IOKit's memory lock (a view descriptor's * free) or under no lock; nothing is called with it held. Registration has its own lock, taken before the gate, so the kernel - * mapping of a new range is made without the gate. Zero fill and cache maintenance run with no lock at all. */ + * bitmaps of a new range are made without the gate. Zero fill and cache maintenance run with no lock at all. */ /* mapping: the run in its range's kernel mapping. viewed: a client could have mapped a page of it since its last zero fill. */ struct NativeAdrenoPoolRun {uint64_t physical,bytes;void *mapping;uint64_t viewed;}; struct NativeAdrenoPoolGrant { @@ -76,8 +76,10 @@ void NativeAdrenoPoolRetire(NativeAdrenoPoolGrant *); /* Whether any page of [physical, physical+bytes) lies in a registered range. */ bool NativeAdrenoPoolContains(uint64_t physical,uint64_t bytes); /* The kernel services under the allocator (NativeAdrenoCpu.cpp; host doubles in tests/HostKernelHooks.cpp). - * NativeAdrenoPoolMapRange: one write-combined kernel mapping of a whole range, kept for good; *owner takes the object that holds it. - * NativeAdrenoPoolScrub: zero-fills [mapping, mapping+bytes) through that mapping and waits for the stores to reach DRAM; with + * NativeAdrenoPoolMapRange: the value that stands for a range's first page in the runs ("mapping"); the kernel's is the physical + * address and keeps no mapping, the host's is memory standing for the pages. *owner takes what holds it, if anything. + * NativeAdrenoPoolScrub: zero-fills the run at [mapping, mapping+bytes) (the kernel maps the run for the call) and waits for the + * stores to reach DRAM; with * drop_lines it first drops every CPU cache line over the pages, by the mapping's addresses (data cache maintenance by address acts * on the lines of the physical page whatever the attributes of the mapping used). A failure leaves the pages unfit to hand out. */ void *NativeAdrenoPoolMapRange(uint64_t physical,size_t bytes,void **owner);