#define NATIVE_GPU_TRACE_OPTIONAL #include "NativeGpuTrace.h" /* SPDX-License-Identifier: MIT */ #include "NativeAdrenoUserClient.h" #include #include /* PRIO_DARWIN_GPU_*: the process's GPU role, set by the system */ extern "C" volatile uint32_t native_adreno_trace_stage; /* 0 keeps user BOs cacheable and maintained before every submission (int-070..097); the board sets it from the gpu-wc boot argument. */ /* gpu-owned-coherent=1: the GPU's own BOs use the IO-coherent path (CPU write-back view, GPU write-back inner-shareable mapping) instead of * write-combine: no write-combined audit and no cache maintenance per submission. Effective only once the board proved IO coherence. */ extern "C" {volatile uint32_t native_adreno_wc_enabled=1,native_adreno_owned_coherent=0;extern volatile uint32_t native_adreno_io_coherent;} static bool ownedCoherent() {return native_adreno_owned_coherent && native_adreno_io_coherent;} bool NativeAdrenoUserClient::systemWriteCombined() {return native_adreno_wc_enabled && !ownedCoherent();} /* BoImport record: [0] calls, [1] refused, [2..5] last refusal (status, stage, bytes, address), [6..9] first refusal. Stages 1..3 are this file's * checks, 10..19 NativeAdrenoMemory::importTask's. */ extern "C" {volatile uint64_t native_adreno_import_diag[10];} /* Imports of 16 MiB or more (the compositor's scanout surfaces), oldest overwritten: per entry the caller's address, the first physical page, the bytes. */ extern "C" volatile uint32_t native_adreno_import_stage; /* Imports and their release (int-233/234: Core Animation's render thread waited in vm_deallocate for pages a kernel wiring held): [0] imports * alive, [1] frees of an import refused, [2] last refusal status; then the 8 newest imports as {address, pid<<40|bytes, handle, state}, state * 1 alive, 2 released, 3 free refused. */ extern "C" {volatile uint64_t native_adreno_import_live[3+8*4];} extern "C" int proc_selfpid(void); static unsigned gImportNext; static void noteImportState(uint64_t handle,uint64_t state) { for(unsigned i=0;i<8;++i)if(native_adreno_import_live[3+i*4+2]==handle) {native_adreno_import_live[3+i*4+3]=state;return;} } extern "C" volatile uint32_t native_adreno_io_coherent; /* Per kernel method: [method][0] calls, [1] calls that returned non-zero, [2] last status, [3] total time (us, 32-bit). Method numbers are the ABI's (1..16). */ extern "C" {extern uint64_t (*native_adreno_vm_now_us)(void);volatile uint32_t native_adreno_method_stats[20][4];volatile uint32_t native_adreno_map_fail[7][2]; /* The last failed map: the bytes asked, and the mapping it met (address, bytes, handle, jobs, kind | imported 0x100 | upper 0x1000). */ volatile uint64_t native_adreno_map_met[7];} static void noteImport(IOReturn status,uint32_t stage,uint64_t bytes,uint64_t address) { if(!status)return; native_adreno_import_diag[1]=native_adreno_import_diag[1]+1; native_adreno_import_diag[2]=(uint32_t)status;native_adreno_import_diag[3]=stage;native_adreno_import_diag[4]=bytes;native_adreno_import_diag[5]=address; if(native_adreno_import_diag[1]==1) { native_adreno_import_diag[6]=(uint32_t)status;native_adreno_import_diag[7]=stage;native_adreno_import_diag[8]=bytes;native_adreno_import_diag[9]=address; } } #include #include extern "C" {volatile uint32_t native_adreno_window_overflow[4][3];} /* {count, last size KiB, last end MiB} per client window */ NativeAdrenoUserClient::Entry *NativeAdrenoUserClient::find(uint64_t handle) { const uint32_t index=entry_count_?index_.find(handle):UINT32_MAX; if(index>=entry_count_)return nullptr; Entry &entry=at(index); return entry.used && entry.handle==handle?&entry:nullptr; } /* A new live entry with a handle: counted and findable. */ void NativeAdrenoUserClient::publish(Entry &entry) { entry.used=true;entry.handle=next_handle_++;entry.token=entry.index+1;++live_; index_.insert(entry.handle,entry.index); } void NativeAdrenoUserClient::clear(Entry &entry) { if(entry.used && live_)--live_; index_.erase(entry.handle); const size_t index=entry.index;if(index=UINT32_MAX-EntryChunk)return false; /* tokens are entry indices plus one */ const size_t chunk_index=entry_count_/EntryChunk; if(chunk_index==chunk_slots_) { const size_t slots=chunk_slots_?chunk_slots_*2:16; Entry **made=IONewData(Entry *,slots); if(!made)return false; for(size_t i=0;i=UINT32_MAX/2)return false; const uint32_t size=queue_capacity_?queue_capacity_*2:8; Queue *made=IONewData(Queue,size); if(!made)return false; for(uint32_t i=0;imapped_state; } NativeAdrenoMemory *NativeAdrenoUserClient::memoryOf(uint64_t handle) { Entry *entry=find(handle);return entry && !entry->unreleasable?&entry->memory:nullptr; } IOReturn NativeAdrenoUserClient::open(uint32_t abi_version,NativeAdrenoVm *vm, NativeAdrenoSubmit *submit,NativeAdrenoRing *ring,uint32_t input_bits,uint32_t output_bits) { if(open_ || vm_ || submit_)return kIOReturnBusy; if(abi_version!=NativeAdrenoAbiVersion)return kIOReturnUnsupported; if(!vm || !submit || !ring || input_bits<31 || input_bits>48 || output_bits<32 || output_bits>48 || vm->poisoned() || submit->poisoned() || ring->poisoned())return kIOReturnBadArgument; if(!grow())return kIOReturnNoMemory; if(!index_.allocate()) {releaseEntries();return kIOReturnNoMemory;} if(!growQueues()) {releaseEntries();index_.release();return kIOReturnNoMemory;} free_hint_=0;live_=0; vm_=vm;submit_=submit;ring_=ring;input_bits_=input_bits;output_bits_=output_bits; next_handle_=1;open_=true; queues_[0]={0,true}; /* Each client is a new GPU context for the CP (useContext before each submission). */ static uint32_t contexts; do context_=__atomic_add_fetch(&contexts,1,__ATOMIC_RELAXED); while(!context_); return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::setScope(uint64_t queue_base,uint64_t window_start,uint64_t window_bytes) { if(!open_)return kIOReturnNotReady; if(queue_base%NativeAdrenoQueueRange || (window_start&4095) || (window_bytes&4095) || (window_bytes && (window_start+window_bytes(UINT64_C(1)<rangeOutstanding(queue_base,NativeAdrenoQueueRange))return kIOReturnBusy; queue_base_=queue_base;scoped_=true;window_start_=window_start;window_bytes_=window_bytes;window_peak_=0; return kIOReturnSuccess; } bool NativeAdrenoUserClient::ownRangeOutstanding() const { return (scoped_ || used_) && submit_->rangeOutstanding(queue_base_,NativeAdrenoQueueRange); } IOReturn NativeAdrenoUserClient::close() { if(!open_)return kIOReturnNotReady; if(liveObjects() || ownRangeOutstanding())return kIOReturnBusy; for(uint32_t i=1;iwaitRange(queue_base_,NativeAdrenoQueueRange,clock,timeout_us):kIOReturnSuccess; if(status)return status; for(size_t index=0;indexentry_count_)return nullptr; Entry &entry=at(token-1); return entry.used && !entry.unreleasable && entry.token==token && entry.mapping_leases?&entry.memory:nullptr; } uint32_t NativeAdrenoUserClient::mappingLeases(uint32_t token) const { if(!open_ || !token || token>entry_count_)return 0; const Entry &entry=at(token-1); return entry.used && entry.token==token?entry.mapping_leases:0; } IOReturn NativeAdrenoUserClient::query(NativeAdrenoAbiParams *params) { if(!params)return kIOReturnBadArgument; *params={}; if(!open_)return kIOReturnNotReady; params->abi_version=NativeAdrenoAbiVersion; params->input_bits=input_bits_;params->output_bits=output_bits_; params->flags=NativeAdrenoParamsHostImport|(ownedCoherent()?NativeAdrenoParamsOwnedCoherent:0u)| (NativeAdrenoMemory::memoryPressure()?NativeAdrenoParamsMemoryPressure:0u);params->max_bo_bytes=maxBoBytes(); return kIOReturnSuccess; } /* Per client slot (4, the kernel's own client last): [0] live objects now, [1] their peak, [2] refused BoAllocs, [3] last refusal status, * [4] bytes of the last refused request, [5] the window's highest mapped end in MiB above its start. Turnip turns any refusal into * VK_ERROR_OUT_OF_DEVICE_MEMORY, which its command streams do not check (int-237: backboardd wrote through a NULL stream pointer). */ extern "C" {volatile uint32_t native_adreno_client_stats[5][6];} /* Per client slot: MiB of its own BOs live now and at most (imports not counted). */ extern "C" {volatile uint32_t native_adreno_client_bytes[5][2];} static volatile uint32_t *clientStats(uint32_t slot) { return native_adreno_client_stats[slot<4?slot:4]; } static void noteLive(uint32_t slot,size_t live) { volatile uint32_t *stats=clientStats(slot); stats[0]=(uint32_t)live;if(live>stats[1])stats[1]=(uint32_t)live; } IOReturn NativeAdrenoUserClient::allocBo(const NativeAdrenoAbiBoAlloc *request, NativeAdrenoAbiBoAllocReply *reply) { const IOReturn status=allocBoInner(request,reply); if(status && request) { volatile uint32_t *stats=clientStats(stats_slot_); stats[2]=stats[2]+1;stats[3]=(uint32_t)status;stats[4]=(uint32_t)request->bytes; } if(entry_count_)noteLive(stats_slot_,liveObjects()); return status; } IOReturn NativeAdrenoUserClient::allocBoInner(const NativeAdrenoAbiBoAlloc *request, NativeAdrenoAbiBoAllocReply *reply) { if(!request || !reply)return kIOReturnBadArgument;reply->handle=0; if(!open_)return kIOReturnNotReady; const uint64_t limit=maxBoBytes(); if(!request->bytes || (request->bytes&4095) || request->bytes>limit || request->align_log2<12 || request->align_log2>24 || request->contiguous>1) return kIOReturnBadArgument; Entry *entry=free(); if(!entry)return kIOReturnNoResources; /* User BOs are write-combined for the CPU, as the GPU maps them non-cacheable: no per-submission cache maintenance. * int-147 kept them cacheable under IO coherence and mapped the GPU side write-back: the first Turnip IB executed zeros and the GPU hung * (the kernel's cacheable alias read zeros past the first line too). A coherent own BO needs a client that asks for a write-back view. */ const bool write_combined=systemWriteCombined(); IOReturn status; if(request->contiguous) { /* Only a caller that needs one physical run gets the kernel's contiguous allocation (XNU scans physical memory for it). */ status=entry->memory.allocate((size_t)request->bytes,output_bits_,true,(size_t)request->bytes); if(!status && write_combined)status=entry->memory.makeWriteCombined(); } else status=entry->memory.allocateSystem((size_t)request->bytes,output_bits_,(size_t)request->bytes,write_combined,offered_pages_); if(status) { /* A failed allocation can leave quarantine behind; never reuse it. */ clear(*entry); if(entry->memory.release()!=kIOReturnSuccess) { entry->used=true;entry->unreleasable=true;++live_; } return status==kIOReturnUnsupported?kIOReturnUnsupported:status; } publish(*entry); entry->mapped=0;entry->iova=0;entry->mapped_state=false; reply->handle=entry->handle; owned_bytes_+=entry->memory.bytes();if(owned_bytes_>owned_peak_)owned_peak_=owned_bytes_; {volatile uint32_t *b=native_adreno_client_bytes[stats_slot_<4?stats_slot_:4];b[0]=(uint32_t)(owned_bytes_>>20);b[1]=(uint32_t)(owned_peak_>>20);} return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::importBo(const NativeAdrenoAbiBoImport *request,void *task, NativeAdrenoAbiBoAllocReply *reply) { if(!request || !reply)return kIOReturnBadArgument;reply->handle=0; if(!open_)return kIOReturnNotReady; native_adreno_import_diag[0]=native_adreno_import_diag[0]+1; const uint64_t limit=maxBoBytes(); if(!task || !request->address || ((request->address|request->bytes)&4095) || !request->bytes || request->bytes>limit) { noteImport(kIOReturnBadArgument,1,request->bytes,request->address); return kIOReturnBadArgument; } Entry *entry=free(); if(!entry) {noteImport(kIOReturnNoResources,2,request->bytes,request->address);return kIOReturnNoResources;} const IOReturn status=entry->memory.importTask(task,request->address,(size_t)request->bytes,output_bits_,(size_t)request->bytes); if(status) { noteImport(status,native_adreno_import_stage,request->bytes,request->address); clear(*entry); if(entry->memory.release()!=kIOReturnSuccess) {entry->used=true;entry->unreleasable=true;++live_;} return status; } publish(*entry); entry->mapped=0;entry->iova=0;entry->mapped_state=false;entry->imported=true;entry->imported_pid=proc_selfpid(); reply->handle=entry->handle; native_adreno_import_live[0]=native_adreno_import_live[0]+1; { const unsigned slot=3+(gImportNext++%8)*4; native_adreno_import_live[slot]=request->address;native_adreno_import_live[slot+1]=(uint64_t)proc_selfpid()<<40|request->bytes; native_adreno_import_live[slot+2]=entry->handle;native_adreno_import_live[slot+3]=1; } return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::mapBo(const NativeAdrenoAbiBoMap *request, NativeAdrenoAbiBoMapReply *reply) { if(!request || !reply)return kIOReturnBadArgument;reply->iova=0; if(!open_)return kIOReturnNotReady; if(!(request->permissions&1) || (request->permissions&~3u) || request->writeback>1 || (request->shareability!=2 && request->shareability!=3) || request->reserved || (request->iova&4095))return kIOReturnBadArgument; Entry *entry=find(request->handle); if(!entry || entry->unreleasable)return kIOReturnBadArgument; if(entry->mapped_state)return kIOReturnBusy; const uint64_t end=request->iova+entry->memory.bytes(); if(window_bytes_ && (request->iovaiova || end>window_start_+window_bytes_)) { /* Per window: how often a mapping fell outside it, and the last one's size and end (MiB), for the boot log's [gpu windows] row. */ const uint32_t slot=stats_slot_; if(slot<4) { native_adreno_window_overflow[slot][0]=native_adreno_window_overflow[slot][0]+1; native_adreno_window_overflow[slot][1]=uint32_t(entry->memory.bytes()>>10); native_adreno_window_overflow[slot][2]=uint32_t(end>>20); } return kIOReturnBadArgument; } uint64_t handle=0; /* Memory the owner maps read-only could only be pinned for reading: the GPU gets the read half of what was asked. */ const unsigned permissions=entry->memory.readOnly()?(request->permissions&~2u):request->permissions; /* Imported memory is the owner's cacheable memory (IOSurfaces, no-copy buffers): once the board proved IO coherence, the GPU maps it * write-back inner-shareable (Linux's IOMMU_CACHE PTE) and the CPU caches over it need no maintenance. The GPU's own BOs stay * non-cacheable: their CPU view is write-combined, and a snoop could find a stale line of the kernel's cacheable alias. */ /* Only memory the GPU never writes is mapped coherent: sysmem passes written through write-back inner-shareable mappings stopped * the GPU inside its end-of-pass cache clean (BR and BV idle at the clean events), while non-coherent mappings never did. */ const bool coherent=(entry->memory.imported() && native_adreno_io_coherent && !(permissions&2u)) || (!entry->memory.imported() && ownedCoherent()); const IOReturn status=vm_->map(&entry->memory,0,request->iova,permissions, coherent?1u:request->writeback,coherent?3u:request->shareability,&handle); if(status)return status; entry->mapped=handle;entry->iova=request->iova;entry->mapped_state=true; if(end>window_peak_)window_peak_=end; if(window_bytes_ && window_peak_>window_start_)clientStats(stats_slot_)[5]=(uint32_t)((window_peak_-window_start_)>>20); reply->iova=request->iova; return kIOReturnSuccess; } static void noteFreeRefused(bool imported,uint64_t handle,IOReturn status) { if(!imported)return; native_adreno_import_live[1]=native_adreno_import_live[1]+1;native_adreno_import_live[2]=(uint32_t)status;noteImportState(handle,3); } IOReturn NativeAdrenoUserClient::freeBo(const NativeAdrenoAbiBoFree *request) { native_adreno_trace_stage=1; if(!request)return kIOReturnBadArgument; if(!open_)return kIOReturnNotReady; Entry *entry=find(request->handle); if(!entry)return kIOReturnBadArgument; if(entry->unreleasable) {noteFreeRefused(entry->imported,entry->handle,kIOReturnBusy);return kIOReturnBusy;} if(entry->mapping_leases) {noteFreeRefused(entry->imported,entry->handle,kIOReturnBusy);return kIOReturnBusy;} native_adreno_trace_stage=2; if(entry->mapped_state) { /* Completed jobs must not pin the buffer: retire what the CP has passed before the unmap decides. */ submit_->retireCompleted(); /* a poisoned execbuf refuses and the unmap below stays refused too */ native_adreno_trace_stage=3; const IOReturn unmap=vm_->unmap(entry->mapped); native_adreno_trace_stage=4; if(unmap) {noteFreeRefused(entry->imported,entry->handle,unmap);return unmap;} entry->mapped_state=false;entry->mapped=0;entry->iova=0; } const size_t bytes=entry->memory.bytes(); const IOReturn release=freed_pages_?entry->memory.releaseSystemInto(freed_pages_):entry->memory.release(); if(release) {noteFreeRefused(entry->imported,entry->handle,release);return release;} if(!entry->imported) { owned_bytes_-=bytes>20); } if(entry->imported) {native_adreno_import_live[0]=native_adreno_import_live[0]-1;noteImportState(entry->handle,2);} clear(*entry); return kIOReturnSuccess; } /* The GPU priority this process's next submission takes: its queues' foreground value while it renders visible UI (a GPU role above * utility, or an unmanaged task), their background value for the denied, background and utility roles. A process that never told us keeps the daemon's default (no ring choice). */ /* Which processes told the driver what (the newest 32): pid, foreground priority, background priority, the GPU role then, the time (ms of * the submission clock when the board gave one, else 0) and how many times this connection had told it before (0 the runtime's first call * at device open, later ones the process's own setGPUPriority:/setBackgroundGPUPriority: through the frontend). */ extern "C" {volatile uint32_t native_adreno_prio_log[32][6];volatile uint32_t native_adreno_prio_count;} extern "C" uint64_t native_adreno_now_us(void) __attribute__((weak)); int32_t NativeAdrenoUserClient::submitPriority() const { if(priority_foreground_<0 && priority_background_<0)return -1; const int32_t role=task_get_gpu_role(current_task()); /* A task RunningBoard denies the GPU renders nothing visible either: where it still submits it takes the background value. */ const bool background=role==PRIO_DARWIN_GPU_DENY || role==PRIO_DARWIN_GPU_BACKGROUND || role==PRIO_DARWIN_GPU_UTILITY; return background?(priority_background_>=0?priority_background_:priority_foreground_) :(priority_foreground_>=0?priority_foreground_:priority_background_); } IOReturn NativeAdrenoUserClient::setPriority(uint32_t foreground,uint32_t background) { if(!open_)return kIOReturnNotReady; if(foreground>5 || background>5 || foreground==3)return kIOReturnBadArgument; priority_foreground_=int32_t(foreground); priority_background_=int32_t(background); { const uint32_t n=native_adreno_prio_count; volatile uint32_t *row=native_adreno_prio_log[n&31]; row[0]=uint32_t(proc_selfpid());row[1]=foreground;row[2]=background;row[3]=uint32_t(task_get_gpu_role(current_task())); row[4]=&native_adreno_now_us?uint32_t(native_adreno_now_us()/1000):0;row[5]=priority_told_; native_adreno_prio_count=n+1; if(priority_told_<0xffffffffu)++priority_told_; } return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::submit(const NativeAdrenoAbiSubmit *request, NativeAdrenoAbiSubmitReply *reply,const NativeAdrenoRingClock *clock) { if(!request || !reply)return kIOReturnBadArgument;reply->fence=0; if(!open_)return kIOReturnNotReady; if(!request->buffer_count || request->buffer_count>NativeAdrenoAbiMaxBuffers || request->reserved)return kIOReturnBadArgument; if(request->queue>=queue_capacity_ || !queues_[request->queue].active) return kIOReturnBadArgument; if(request->wait_fence) { if(!clock)return kIOReturnBadArgument; const uint32_t wait_us=request->wait_timeout_ns==0?0: (request->wait_timeout_ns>UINT64_C(0xffffffff)*1000u?0xffffffffu: (uint32_t)(request->wait_timeout_ns/1000)); const IOReturn waited=submit_->observeFence((uint32_t)request->wait_fence,clock,wait_us); if(waited)return waited; } NativeAdrenoSubmitBuffer *buffers=submit_buffers_; for(uint32_t i=0;ibuffer_count;++i) { const NativeAdrenoAbiSubmitBuffer &wire=request->buffers[i]; if(!wire.bytes || (wire.bytes&3))return kIOReturnBadArgument; Entry *entry=find(wire.handle); if(!entry || !entry->mapped_state)return kIOReturnBadArgument; buffers[i]={entry->mapped,wire.iova,(size_t)wire.bytes}; } uint32_t fence=0; used_=true; ring_->useContext(context_); submit_->nextOwner(proc_selfpid()); submit_->nextPriority(submitPriority()); submit_->nextRole(task_get_gpu_role(current_task())); const IOReturn status=submit_->submit(vm_,buffers,request->buffer_count, queue_base_+request->queue,&fence,clock); if(status)return status; reply->fence=fence; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::attachTimer(const NativeAdrenoTimer *timer) { if(!open_)return kIOReturnNotReady; if(!timer || timer_ )return kIOReturnBadArgument; timer_=timer; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::timestamp(NativeAdrenoAbiTimestamp *reply) { if(!reply)return kIOReturnBadArgument;*reply={}; if(!open_)return kIOReturnNotReady; if(!timer_)return kIOReturnUnsupported; return timer_->read(&reply->ticks); } IOReturn NativeAdrenoUserClient::attachDeviceInfo(const NativeAdrenoAbiDeviceInfo *info) { if(!open_)return kIOReturnNotReady; if(!info || info_ || info->reserved || !info->chip_id || !info->va_size)return kIOReturnBadArgument; info_=info; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::deviceInfo(NativeAdrenoAbiDeviceInfo *reply) { if(!reply)return kIOReturnBadArgument;*reply={}; if(!open_)return kIOReturnNotReady; if(!info_)return kIOReturnUnsupported; *reply=*info_; if(window_bytes_){reply->va_start=window_start_;reply->va_size=window_bytes_;} return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::attachStatus(const NativeAdrenoStatus *status) { if(!open_)return kIOReturnNotReady; if(!status || status_)return kIOReturnBadArgument; status_=status; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::status(NativeAdrenoAbiStatus *reply) { if(!reply)return kIOReturnBadArgument;*reply={}; if(!open_)return kIOReturnNotReady; if(!status_)return kIOReturnUnsupported; /* msm MSM_PARAM_FAULTS: the global count plus this client's own VM's (a hang its submission caused). */ const NativeAdrenoStatusCounts counts=status_->counts(); reply->faults=counts.faults+(vm_?vm_->faults():0);reply->suspends=counts.suspends; return kIOReturnSuccess; } size_t NativeAdrenoUserClient::inputSize(uint32_t method) { switch(method) { case NativeAdrenoMethodQuery: case NativeAdrenoMethodGetTimestamp: case NativeAdrenoMethodGetStatus: case NativeAdrenoMethodGetDeviceInfo:return 0; case NativeAdrenoMethodBoAlloc:return sizeof(NativeAdrenoAbiBoAlloc); case NativeAdrenoMethodBoMap:return sizeof(NativeAdrenoAbiBoMap); case NativeAdrenoMethodBoImport:return sizeof(NativeAdrenoAbiBoImport); case NativeAdrenoMethodBoFree:return sizeof(NativeAdrenoAbiBoFree); case NativeAdrenoMethodSubmit:return sizeof(NativeAdrenoAbiSubmit); case NativeAdrenoMethodSetMetadata: case NativeAdrenoMethodGetMetadata:return sizeof(NativeAdrenoAbiMetadata); case NativeAdrenoMethodCreateQueue: case NativeAdrenoMethodCloseQueue:return sizeof(NativeAdrenoAbiQueue); case NativeAdrenoMethodBoCpuMap:return sizeof(NativeAdrenoAbiBoCpuMap); case NativeAdrenoMethodBoCpuUnmap:return sizeof(NativeAdrenoAbiBoCpuUnmap); case NativeAdrenoMethodWaitFence:return sizeof(NativeAdrenoAbiWaitFence); default:return SIZE_MAX; } } size_t NativeAdrenoUserClient::outputSize(uint32_t method) { switch(method) { case NativeAdrenoMethodQuery:return sizeof(NativeAdrenoAbiParams); case NativeAdrenoMethodBoAlloc:return sizeof(NativeAdrenoAbiBoAllocReply); case NativeAdrenoMethodBoMap:return sizeof(NativeAdrenoAbiBoMapReply); case NativeAdrenoMethodBoFree:return 0; case NativeAdrenoMethodSubmit:return sizeof(NativeAdrenoAbiSubmitReply); case NativeAdrenoMethodWaitFence:return 0; case NativeAdrenoMethodGetTimestamp:return sizeof(NativeAdrenoAbiTimestamp); case NativeAdrenoMethodGetStatus:return sizeof(NativeAdrenoAbiStatus); case NativeAdrenoMethodGetDeviceInfo:return sizeof(NativeAdrenoAbiDeviceInfo); case NativeAdrenoMethodGetMetadata:return sizeof(NativeAdrenoAbiMetadata); case NativeAdrenoMethodSetMetadata:return 0; case NativeAdrenoMethodCreateQueue:return sizeof(NativeAdrenoAbiQueueReply); case NativeAdrenoMethodCloseQueue:return 0; case NativeAdrenoMethodBoCpuMap:return sizeof(NativeAdrenoAbiBoCpuMapReply); case NativeAdrenoMethodBoCpuUnmap:return 0; case NativeAdrenoMethodBoImport:return sizeof(NativeAdrenoAbiBoAllocReply); default:return SIZE_MAX; } } IOReturn NativeAdrenoUserClient::dispatch(uint32_t method,const void *input, size_t input_size,void *output,size_t output_size,const NativeAdrenoRingClock *clock) { struct Tally { uint32_t method;uint64_t began;IOReturn *result;const NativeAdrenoRingClock *clock; Tally(uint32_t m,IOReturn *r,const NativeAdrenoRingClock *c):method(m),began(c && c->now_us?c->now_us(c->cookie):0),result(r),clock(c) {} ~Tally() { if(method>=20)return; const uint64_t now=clock && clock->now_us?clock->now_us(clock->cookie):0; native_adreno_method_stats[method][0]=native_adreno_method_stats[method][0]+1; if(*result)native_adreno_method_stats[method][1]=native_adreno_method_stats[method][1]+1; native_adreno_method_stats[method][2]=(uint32_t)*result; native_adreno_method_stats[method][3]=native_adreno_method_stats[method][3]+(uint32_t)(now-began); } }; /* The same clock times the VM's table transactions (the board service's probe never ran on this path, so nothing else installed it). */ static const NativeAdrenoRingClock *vm_clock=nullptr; vm_clock=clock; native_adreno_vm_now_us=[]()->uint64_t{return vm_clock && vm_clock->now_us?vm_clock->now_us(vm_clock->cookie):0;}; IOReturn tally_result=kIOReturnSuccess; Tally tally(method,&tally_result,clock); const size_t expected_in=inputSize(method),expected_out=outputSize(method); if(expected_in==SIZE_MAX || expected_out==SIZE_MAX)return kIOReturnUnsupported; if(input_size!=expected_in || output_size!=expected_out)return kIOReturnBadArgument; if((expected_in && !input) || (expected_out && !output))return kIOReturnBadArgument; IOReturn status=kIOReturnSuccess; switch(method) { case NativeAdrenoMethodQuery: { NativeAdrenoAbiParams reply{}; status=query(&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodBoAlloc: { NativeAdrenoAbiBoAlloc request{}; memcpy(&request,input,sizeof(request)); NativeAdrenoAbiBoAllocReply reply{}; status=allocBo(&request,&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodBoImport: status=kIOReturnUnsupported; break; case NativeAdrenoMethodBoMap: { NativeAdrenoAbiBoMap request{}; memcpy(&request,input,sizeof(request)); NativeAdrenoAbiBoMapReply reply{}; status=mapBo(&request,&reply); if(!status)memcpy(output,&reply,sizeof(reply)); else { /* Why maps fail (int-135: 55 of 191, and the translator answers each as a device out-of-memory): status, count, and what the table * held and the failed request asked for the last time. */ unsigned at=0;while(at<5 && native_adreno_map_fail[at][0] && native_adreno_map_fail[at][0]!=(uint32_t)status)++at; native_adreno_map_fail[at][0]=(uint32_t)status;native_adreno_map_fail[at][1]=native_adreno_map_fail[at][1]+1; native_adreno_map_fail[5][0]=(uint32_t)vm_->spanCount();native_adreno_map_fail[5][1]=(uint32_t)vm_->objectCount(); native_adreno_map_fail[6][0]=(uint32_t)(request.iova>>12);native_adreno_map_fail[6][1]=(uint32_t)(request.handle); { uint64_t met[5]={}; const bool found=vm_->occupant(request.iova,met); Entry *asked=find(request.handle); /* The bytes asked, with bit 63 where the handle itself was already mapped (the other source of this status). */ native_adreno_map_met[0]=request.iova; native_adreno_map_met[1]=(asked?asked->memory.bytes():0)|(asked && asked->mapped_state?UINT64_C(1)<<63:0); for(unsigned k=0;k<5;++k)native_adreno_map_met[2+k]=found?met[k]:~UINT64_C(0); } } break; } case NativeAdrenoMethodBoFree: { NativeAdrenoAbiBoFree request{}; memcpy(&request,input,sizeof(request)); status=freeBo(&request); break; } case NativeAdrenoMethodSubmit: { NativeAdrenoAbiSubmit &request=submit_request_; memcpy(&request,input,sizeof(request)); NativeAdrenoAbiSubmitReply reply{}; status=submit(&request,&reply,clock); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodWaitFence: { NativeAdrenoAbiWaitFence request{}; memcpy(&request,input,sizeof(request)); status=waitFence(&request,clock); break; } case NativeAdrenoMethodGetDeviceInfo: { NativeAdrenoAbiDeviceInfo reply{}; status=deviceInfo(&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodGetTimestamp: { NativeAdrenoAbiTimestamp reply{}; status=timestamp(&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodBoCpuMap: { NativeAdrenoAbiBoCpuMap request{}; memcpy(&request,input,sizeof(request)); NativeAdrenoAbiBoCpuMapReply reply{}; status=cpuMap(&request,&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodBoCpuUnmap: { NativeAdrenoAbiBoCpuUnmap request{}; memcpy(&request,input,sizeof(request)); status=cpuUnmap(&request); break; } case NativeAdrenoMethodCreateQueue: { NativeAdrenoAbiQueueReply reply{}; status=createQueue(&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodCloseQueue: { NativeAdrenoAbiQueue request{}; memcpy(&request,input,sizeof(request)); status=closeQueue(&request); break; } case NativeAdrenoMethodSetMetadata: { NativeAdrenoAbiMetadata request{}; memcpy(&request,input,sizeof(request)); status=setMetadata(&request); break; } case NativeAdrenoMethodGetMetadata: { NativeAdrenoAbiMetadata request{}; memcpy(&request,input,sizeof(request)); NativeAdrenoAbiMetadata reply{}; status=getMetadata(&request,&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } case NativeAdrenoMethodGetStatus: { NativeAdrenoAbiStatus reply{}; status=this->status(&reply); if(!status)memcpy(output,&reply,sizeof(reply)); break; } default:return kIOReturnUnsupported; } tally_result=status; return status; } IOReturn NativeAdrenoUserClient::cpuMap(const NativeAdrenoAbiBoCpuMap *request, NativeAdrenoAbiBoCpuMapReply *reply) { if(!request || !reply)return kIOReturnBadArgument;*reply={}; if(!open_)return kIOReturnNotReady; Entry *entry=find(request->handle); if(!entry || entry->unreleasable)return kIOReturnBadArgument; /* An imported BO is the caller's own memory, already mapped where it came from. */ if(entry->memory.imported())return kIOReturnUnsupported; if(entry->mapping_leases==UINT32_MAX)return kIOReturnNoResources; ++entry->mapping_leases; reply->token=entry->token;reply->bytes=entry->memory.bytes(); return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::cpuUnmap(const NativeAdrenoAbiBoCpuUnmap *request) { if(!request)return kIOReturnBadArgument; if(!open_)return kIOReturnNotReady; if(request->reserved)return kIOReturnBadArgument; Entry *entry=find(request->handle); if(!entry || entry->unreleasable || !entry->token)return kIOReturnBadArgument; if(request->token!=entry->token)return kIOReturnBadArgument; if(!entry->mapping_leases)return kIOReturnBusy; --entry->mapping_leases; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::createQueue(NativeAdrenoAbiQueueReply *reply) { if(!reply)return kIOReturnBadArgument; if(!open_)return kIOReturnNotReady; for(;;) { for(uint32_t i=1;iqueue=i; return kIOReturnSuccess; } if(!growQueues())return kIOReturnNoMemory; } } IOReturn NativeAdrenoUserClient::closeQueue(const NativeAdrenoAbiQueue *request) { if(!request)return kIOReturnBadArgument; if(!open_)return kIOReturnNotReady; if(!request->queue || request->queue>=queue_capacity_)return kIOReturnBadArgument; Queue &queue=queues_[request->queue]; if(!queue.active)return kIOReturnBadArgument; /* A queue must be drained by the caller before it disappears. */ if(submit_->queueOutstanding(queue_base_+request->queue)) return kIOReturnBusy; queue={}; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::setMetadata(const NativeAdrenoAbiMetadata *request) { if(!request)return kIOReturnBadArgument; if(!open_)return kIOReturnNotReady; if(request->reserved || request->bytes>NativeAdrenoAbiMetadataBytes)return kIOReturnBadArgument; Entry *entry=find(request->handle); if(!entry || entry->unreleasable)return kIOReturnBadArgument; if(request->bytes)memcpy(entry->metadata,request->text,request->bytes); entry->metadata_bytes=request->bytes; return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::getMetadata(const NativeAdrenoAbiMetadata *request, NativeAdrenoAbiMetadata *reply) { if(!request || !reply)return kIOReturnBadArgument; /* Snapshot the request so an in/out alias is still well defined. */ const uint64_t handle=request->handle; const uint32_t asked=request->bytes; const uint32_t reserved=request->reserved; *reply={}; if(!open_)return kIOReturnNotReady; if(reserved || asked>NativeAdrenoAbiMetadataBytes)return kIOReturnBadArgument; Entry *entry=find(handle); if(!entry || entry->unreleasable)return kIOReturnBadArgument; if(askedmetadata_bytes)return kIOReturnBadArgument; reply->handle=entry->handle;reply->bytes=entry->metadata_bytes; if(entry->metadata_bytes)memcpy(reply->text,entry->metadata,entry->metadata_bytes); return kIOReturnSuccess; } IOReturn NativeAdrenoUserClient::waitFence(const NativeAdrenoAbiWaitFence *request, const NativeAdrenoRingClock *clock) { if(!request || !clock)return kIOReturnBadArgument; if(!open_)return kIOReturnNotReady; if(!request->fence)return kIOReturnBadArgument; const uint32_t timeout_us=request->timeout_ns==0?0: (request->timeout_ns>UINT64_C(0xffffffff)*1000u?0xffffffffu: (uint32_t)(request->timeout_ns/1000)); return submit_->wait((uint32_t)request->fence,clock,timeout_us); }