diff --git a/NativeAdrenoCpu.cpp b/NativeAdrenoCpu.cpp index 07d87c2..81e2c25 100644 --- a/NativeAdrenoCpu.cpp +++ b/NativeAdrenoCpu.cpp @@ -171,34 +171,36 @@ IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans __atomic_fetch_add(&native_adreno_reassert_ticks,mach_absolute_time()-began,__ATOMIC_RELAXED); return kIOReturnSuccess; } -/* A pool range's kernel mapping: an IOKit mapping of the physical range, write-combined (Normal non-cacheable), in the kernel map's - * ordinary address space. ml_io_map_wcomb is not used: it takes permanent addresses from the kernel's 1 GiB I/O range, which the - * registers of every device share and which panics when it runs out. The mapping is never removed; *owner keeps the object. */ +/* The kernel keeps no mapping of a pool range: a whole range mapped at registration (several hundred MiB, write-combined, in the + * kernel map, before the GPU's own windows are mapped) kept the GPU from powering up on sheng. The value the allocator takes for a + * range's "mapping" is the range's physical address, and each scrub maps just the run it fills for the time it takes. */ void *NativeAdrenoPoolMapRange(uint64_t physical,size_t bytes,void **owner) { if(!owner || !physical || !bytes || ((physical|bytes)&4095))return nullptr; - IOMemoryDescriptor *range=IOMemoryDescriptor::withPhysicalAddress((IOPhysicalAddress)physical,bytes,kIODirectionInOut); - if(!range)return nullptr; - IOMemoryMap *map=range->createMappingInTask(kernel_task,0,kIOMapAnywhere|kIOMapWriteCombineCache); - range->release(); - if(!map)return nullptr; - void *const address=reinterpret_cast(map->getVirtualAddress()); - if(!address || ((uintptr_t)address&4095) || map->getLength()!=bytes) {map->release();return nullptr;} - *owner=map; - return address; + *owner=nullptr; + return reinterpret_cast(physical); } /* Before a pool page goes to its next owner. A page a client could map may have lines in a CPU cache (the client chooses the cache - * mode of its own IOKit mapping): they are dropped first, by the range mapping's addresses, so none is written over the zeros or - * over the next owner's data later. The pages have no other mapping at this point, so nothing is lost with a dropped line. The zero - * fill goes through the write-combined mapping and the barrier completes it before the GPU or a client can read the pages. */ + * mode of its own IOKit mapping): they are dropped first, by the run mapping's addresses, so none is written over the zeros or over + * the next owner's data later. The pages have no other mapping at this point, so nothing is lost with a dropped line. The zero fill + * goes through a write-combined mapping of the run made for it, and the barrier completes it before the GPU or a client can read + * the pages; the mapping is gone again before the call returns. */ IOReturn NativeAdrenoPoolScrub(void *mapping,size_t bytes,bool drop_lines) { - if(!mapping || !bytes || (((uintptr_t)mapping|bytes)&4095))return kIOReturnBadArgument; - if(drop_lines) { - const IOReturn dropped=maintainOwnedPages(mapping,bytes,NativeAdrenoInvalidate); - if(dropped)return dropped; + const uint64_t physical=reinterpret_cast(mapping); + if(!physical || !bytes || ((physical|bytes)&4095))return kIOReturnBadArgument; + IOMemoryDescriptor *run=IOMemoryDescriptor::withPhysicalAddress((IOPhysicalAddress)physical,bytes,kIODirectionInOut); + if(!run)return kIOReturnNoMemory; + IOMemoryMap *map=run->createMappingInTask(kernel_task,0,kIOMapAnywhere|kIOMapWriteCombineCache); + run->release(); + if(!map)return kIOReturnNoMemory; + void *const address=reinterpret_cast(map->getVirtualAddress()); + IOReturn status=(!address || ((uintptr_t)address&4095) || map->getLength()!=bytes)?kIOReturnNoMemory:kIOReturnSuccess; + if(!status && drop_lines)status=maintainOwnedPages(address,bytes,NativeAdrenoInvalidate); + if(!status) { + bzero(address,bytes); + __asm__ volatile("dsb sy":::"memory"); } - bzero(mapping,bytes); - __asm__ volatile("dsb sy":::"memory"); - return kIOReturnSuccess; + map->release(); + return status; } diff --git a/NativeAdrenoPool.h b/NativeAdrenoPool.h index a6d37c1..5634ab3 100644 --- a/NativeAdrenoPool.h +++ b/NativeAdrenoPool.h @@ -9,10 +9,10 @@ * vm_page for such a page and phystokv panics on it, so nothing but this allocator, the SMMU tables and non-cacheable CPU mappings * ever names one: * - the allocator's own state (a bitmap a range) lives in kernel memory; - * - each range has one write-combined kernel mapping, made at registration, which is the only way the kernel writes the pages; - * - every page of a grant is zero-filled through that mapping before the grant leaves NativeAdrenoPoolAllocate, whatever held the + * - the kernel writes the pages only through a write-combined mapping of a run, made for its zero fill and gone after it; + * - every page of a grant is zero-filled through such a mapping before the grant leaves NativeAdrenoPoolAllocate, whatever held the * page before (boot firmware, another client), so a client never reads what it did not write. Freed pages keep their old bytes - * until then; no mapping but the range's own reaches a free page; + * until then; no kernel mapping reaches a free page; * - pages a client could map (a grant that had a client view) get their CPU cache lines dropped before that zero fill: a client can * ask IOKit for a cacheable view, and a line it left dirty would otherwise be written over the next owner's data later; * - a grant is reference counted: the BO holds one reference and every client view descriptor one, so its end comes only when no @@ -23,7 +23,7 @@ * gives such pages back instead (after the cache pass above), for a system whose GPU clients are trusted not to do that. * Lock order: the allocator's gate is a leaf. It is taken under the driver's lock, under IOKit's memory lock (a view descriptor's * free) or under no lock; nothing is called with it held. Registration has its own lock, taken before the gate, so the kernel - * mapping of a new range is made without the gate. Zero fill and cache maintenance run with no lock at all. */ + * bitmaps of a new range are made without the gate. Zero fill and cache maintenance run with no lock at all. */ /* mapping: the run in its range's kernel mapping. viewed: a client could have mapped a page of it since its last zero fill. */ struct NativeAdrenoPoolRun {uint64_t physical,bytes;void *mapping;uint64_t viewed;}; struct NativeAdrenoPoolGrant { @@ -76,8 +76,10 @@ void NativeAdrenoPoolRetire(NativeAdrenoPoolGrant *); /* Whether any page of [physical, physical+bytes) lies in a registered range. */ bool NativeAdrenoPoolContains(uint64_t physical,uint64_t bytes); /* The kernel services under the allocator (NativeAdrenoCpu.cpp; host doubles in tests/HostKernelHooks.cpp). - * NativeAdrenoPoolMapRange: one write-combined kernel mapping of a whole range, kept for good; *owner takes the object that holds it. - * NativeAdrenoPoolScrub: zero-fills [mapping, mapping+bytes) through that mapping and waits for the stores to reach DRAM; with + * NativeAdrenoPoolMapRange: the value that stands for a range's first page in the runs ("mapping"); the kernel's is the physical + * address and keeps no mapping, the host's is memory standing for the pages. *owner takes what holds it, if anything. + * NativeAdrenoPoolScrub: zero-fills the run at [mapping, mapping+bytes) (the kernel maps the run for the call) and waits for the + * stores to reach DRAM; with * drop_lines it first drops every CPU cache line over the pages, by the mapping's addresses (data cache maintenance by address acts * on the lines of the physical page whatever the attributes of the mapping used). A failure leaves the pages unfit to hand out. */ void *NativeAdrenoPoolMapRange(uint64_t physical,size_t bytes,void **owner);