/* SPDX-License-Identifier: MIT */ #include "NativeAdrenoMemory.h" #include "NativeAdrenoPool.h" #include #include #include #include #include #include extern "C" vm_offset_t phystokv(uint64_t pa); /* Whether a physical page is memory the kernel manages (and phystokv covers): its own range, and on a kernel that adopts DRAM * outside that range (a second bank) the adopted pages too. Device memory and memory the firmware keeps are not. */ extern "C" boolean_t pmap_valid_address(uint64_t physical); static inline bool kernelPage(uint64_t physical) {return pmap_valid_address(physical)!=0;} extern "C" void pmap_set_cache_attributes(ppnum_t pn,unsigned int cacheattr); extern "C" unsigned int pmap_cache_attributes(ppnum_t pn); /* Time and bytes of every CPU cache maintenance pass (the board page shows them per submission), and the pages user BOs made * write-combined: [0] ticks, [1] bytes, [2] longest pass in ticks, [3] passes, [4] write-combined pages now. */ extern "C" {volatile uint64_t native_adreno_cache_stats[5];} /* Time spent checking and re-asserting write-combined page attributes (NativeAdrenoReassertWriteCombined), for the board page. */ extern "C" {volatile uint64_t native_adreno_reassert_ticks;} extern "C" {volatile uint64_t native_adreno_maint_phase[8][2];} void NativeAdrenoNotePhase(unsigned phase,size_t bytes) { __atomic_fetch_add(&native_adreno_maint_phase[phase&7][0],1,__ATOMIC_RELAXED); __atomic_fetch_add(&native_adreno_maint_phase[phase&7][1],(bytes+1023)>>10,__ATOMIC_RELAXED); } bool NativeAdrenoPagesUncached(const NativeAdrenoPhysicalSpan *spans,size_t count) { if(!spans || !count)return false; for(size_t i=0;i>12))&VM_MEM_NOT_CACHEABLE))return false; } return true; } static void noteMaintenance(uint64_t began,size_t bytes) { const uint64_t ticks=mach_absolute_time()-began; __atomic_fetch_add(&native_adreno_cache_stats[0],ticks,__ATOMIC_RELAXED); __atomic_fetch_add(&native_adreno_cache_stats[1],bytes,__ATOMIC_RELAXED); __atomic_fetch_add(&native_adreno_cache_stats[3],1,__ATOMIC_RELAXED); uint64_t longest=__atomic_load_n(&native_adreno_cache_stats[2],__ATOMIC_RELAXED); while(ticks>longest && !__atomic_compare_exchange_n(&native_adreno_cache_stats[2],&longest,ticks,true,__ATOMIC_RELAXED,__ATOMIC_RELAXED)) {} } IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) { if(!out)return kIOReturnBadArgument; *out={}; disable_preemption(); __asm__ volatile("mrs %0, tcr_el1" : "=r"(out->tcr)); __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(out->ttbr0)); __asm__ volatile("mrs %0, ttbr1_el1" : "=r"(out->ttbr1)); __asm__ volatile("mrs %0, sctlr_el1" : "=r"(out->sctlr)); out->kernel_page_bytes=page_size; out->kernel_map_page_bytes=vm_map_page_size(kernel_map); enable_preemption(); bool native=PAGE_SIZE==4096 && out->kernel_page_bytes==4096 && out->kernel_map_page_bytes==4096 && ((out->tcr>>14)&3)==0 && ((out->tcr>>30)&3)==2 && (out->sctlr&1) && (out->ttbr1&UINT64_C(0x0000fffffffff000)); return native?kIOReturnSuccess:kIOReturnUnsupported; } static IOReturn maintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) { /* Whole owned pages avoid IVAC corruption of a neighbour's dirty line. * No Apple coherent-I/O shortcut is used for these noncoherent DMA paths. * Pin this CPU while reading its CTR and issuing a run of cache operations: runs of up to 64 pages, each closed by a DSB on the * core that issued it, so preemption stays off for a bounded time (xnu-12377 panics past 6.25 ms; the 1.5 GiB pool took 41 ms). */ if(!mapping || !bytes || (((uintptr_t)mapping|bytes)&4095) || bytes>UINTPTR_MAX-(uintptr_t)mapping || op>NativeAdrenoInvalidate || op>16)&15); if(line>4096) {enable_preemption();return kIOReturnUnsupported;} const uintptr_t stop=end-address>64*4096?address+64*4096:end; __asm__ volatile("dsb sy":::"memory"); for(;addressUINT64_MAX-bytes || op>NativeAdrenoInvalidate || op>16)&15); if(line>4096) {enable_preemption();return kIOReturnUnsupported;} __asm__ volatile("dsb sy":::"memory"); for(unsigned run=0;run<64 && offsetUINT64_MAX-span.bytes || !kernelPage(span.physical) || !kernelPage(span.physical+span.bytes-4096))return kIOReturnBadArgument; pages+=span.bytes/4096; } for(size_t i=0;i>12),write_combined?VM_WIMG_WCOMB:VM_WIMG_USE_DEFAULT); if(write_combined)__atomic_fetch_add(&native_adreno_cache_stats[4],pages,__ATOMIC_RELAXED); else __atomic_fetch_sub(&native_adreno_cache_stats[4],pages,__ATOMIC_RELAXED); return kIOReturnSuccess; } IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans,size_t count,uint32_t *reset) { if(!spans || !count || !reset)return kIOReturnBadArgument; const uint64_t began=mach_absolute_time(); for(size_t i=0;iUINT64_MAX-span.bytes || !kernelPage(span.physical) || !kernelPage(span.physical+span.bytes-4096))return kIOReturnBadArgument; for(uint64_t offset=0;offset>12); /* From the default attribute the pmap also cleans and invalidates the page (pmap_sync_wimg), so lines a cacheable view * dirtied reach DRAM before the GPU reads it. */ if((pmap_cache_attributes(pn)&VM_WIMG_MASK)!=VM_WIMG_WCOMB) {pmap_set_cache_attributes(pn,VM_WIMG_WCOMB);++*reset;} } } __atomic_fetch_add(&native_adreno_reassert_ticks,mach_absolute_time()-began,__ATOMIC_RELAXED); return kIOReturnSuccess; } /* The kernel keeps no mapping of a pool range: a whole range mapped at registration (several hundred MiB, write-combined, in the * kernel map, before the GPU's own windows are mapped) kept the GPU from powering up on sheng. The value the allocator takes for a * range's "mapping" is the range's physical address, and each scrub maps just the run it fills for the time it takes. */ void *NativeAdrenoPoolMapRange(uint64_t physical,size_t bytes,void **owner) { if(!owner || !physical || !bytes || ((physical|bytes)&4095))return nullptr; *owner=nullptr; return reinterpret_cast(physical); } /* Before a pool page goes to its next owner. A page a client could map may have lines in a CPU cache (the client chooses the cache * mode of its own IOKit mapping): they are dropped first, by the run mapping's addresses, so none is written over the zeros or over * the next owner's data later. The pages have no other mapping at this point, so nothing is lost with a dropped line. The zero fill * goes through a write-combined mapping of the run made for it, and the barrier completes it before the GPU or a client can read * the pages; the mapping is gone again before the call returns. */ /* The run each CPU is zero-filling (physical address, bytes), 0 between fills: a store the system refuses panics in the fill, and * the board's panic record names the run. */ extern "C" {volatile uint64_t native_adreno_pool_scrubbing[16][2];} extern "C" int cpu_number(void); IOReturn NativeAdrenoPoolScrub(void *mapping,size_t bytes,bool drop_lines) { const uint64_t physical=reinterpret_cast(mapping); if(!physical || !bytes || ((physical|bytes)&4095))return kIOReturnBadArgument; IOMemoryDescriptor *run=IOMemoryDescriptor::withPhysicalAddress((IOPhysicalAddress)physical,bytes,kIODirectionInOut); if(!run)return kIOReturnNoMemory; IOMemoryMap *map=run->createMappingInTask(kernel_task,0,kIOMapAnywhere|kIOMapWriteCombineCache); run->release(); if(!map)return kIOReturnNoMemory; void *const address=reinterpret_cast(map->getVirtualAddress()); IOReturn status=(!address || ((uintptr_t)address&4095) || map->getLength()!=bytes)?kIOReturnNoMemory:kIOReturnSuccess; if(!status && drop_lines)status=maintainOwnedPages(address,bytes,NativeAdrenoInvalidate); if(!status) { volatile uint64_t *const mine=native_adreno_pool_scrubbing[(unsigned)cpu_number()&15]; mine[1]=bytes;mine[0]=physical; bzero(address,bytes); __asm__ volatile("dsb sy":::"memory"); mine[0]=0; } map->release(); return status; }