mirror of
https://github.com/UniXNU/NativeAdreno.git
synced 2026-11-17 04:51:59 +03:00
4a3921a2b9
native_adreno_map_met keeps, for the last failed BoMap, the address and bytes asked, whether the handle was mapped already, and the mapping at or past that address: its address, bytes, handle, jobs and kind. Run on hardware: a refused map at the window's first address showed another mapping of the same size there.
785 lines
39 KiB
C++
785 lines
39 KiB
C++
#define NATIVE_GPU_TRACE_OPTIONAL
|
|
#include "NativeGpuTrace.h"
|
|
/* SPDX-License-Identifier: MIT */
|
|
#include "NativeAdrenoUserClient.h"
|
|
#include <kern/task.h>
|
|
#include <sys/resource_private.h> /* PRIO_DARWIN_GPU_*: the process's GPU role, set by the system */
|
|
extern "C" volatile uint32_t native_adreno_trace_stage;
|
|
/* 0 keeps user BOs cacheable and maintained before every submission (int-070..097); the board sets it from the gpu-wc boot argument. */
|
|
/* gpu-owned-coherent=1: the GPU's own BOs use the IO-coherent path (CPU write-back view, GPU write-back inner-shareable mapping) instead of
|
|
* write-combine: no write-combined audit and no cache maintenance per submission. Effective only once the board proved IO coherence. */
|
|
extern "C" {volatile uint32_t native_adreno_wc_enabled=1,native_adreno_owned_coherent=0;extern volatile uint32_t native_adreno_io_coherent;}
|
|
static bool ownedCoherent() {return native_adreno_owned_coherent && native_adreno_io_coherent;}
|
|
bool NativeAdrenoUserClient::systemWriteCombined() {return native_adreno_wc_enabled && !ownedCoherent();}
|
|
/* BoImport record: [0] calls, [1] refused, [2..5] last refusal (status, stage, bytes, address), [6..9] first refusal. Stages 1..3 are this file's
|
|
* checks, 10..19 NativeAdrenoMemory::importTask's. */
|
|
extern "C" {volatile uint64_t native_adreno_import_diag[10];}
|
|
/* Imports of 16 MiB or more (the compositor's scanout surfaces), oldest overwritten: per entry the caller's address, the first physical page, the bytes. */
|
|
extern "C" volatile uint32_t native_adreno_import_stage;
|
|
/* Imports and their release (int-233/234: Core Animation's render thread waited in vm_deallocate for pages a kernel wiring held): [0] imports
|
|
* alive, [1] frees of an import refused, [2] last refusal status; then the 8 newest imports as {address, pid<<40|bytes, handle, state}, state
|
|
* 1 alive, 2 released, 3 free refused. */
|
|
extern "C" {volatile uint64_t native_adreno_import_live[3+8*4];}
|
|
extern "C" int proc_selfpid(void);
|
|
static unsigned gImportNext;
|
|
static void noteImportState(uint64_t handle,uint64_t state)
|
|
{
|
|
for(unsigned i=0;i<8;++i)if(native_adreno_import_live[3+i*4+2]==handle) {native_adreno_import_live[3+i*4+3]=state;return;}
|
|
}
|
|
extern "C" volatile uint32_t native_adreno_io_coherent;
|
|
/* Per kernel method: [method][0] calls, [1] calls that returned non-zero, [2] last status, [3] total time (us, 32-bit). Method numbers are the ABI's (1..16). */
|
|
extern "C" {extern uint64_t (*native_adreno_vm_now_us)(void);volatile uint32_t native_adreno_method_stats[20][4];volatile uint32_t native_adreno_map_fail[7][2];
|
|
/* The last failed map: the bytes asked, and the mapping it met (address, bytes, handle, jobs, kind | imported 0x100 | upper 0x1000). */
|
|
volatile uint64_t native_adreno_map_met[7];}
|
|
static void noteImport(IOReturn status,uint32_t stage,uint64_t bytes,uint64_t address)
|
|
{
|
|
if(!status)return;
|
|
native_adreno_import_diag[1]=native_adreno_import_diag[1]+1;
|
|
native_adreno_import_diag[2]=(uint32_t)status;native_adreno_import_diag[3]=stage;native_adreno_import_diag[4]=bytes;native_adreno_import_diag[5]=address;
|
|
if(native_adreno_import_diag[1]==1) {
|
|
native_adreno_import_diag[6]=(uint32_t)status;native_adreno_import_diag[7]=stage;native_adreno_import_diag[8]=bytes;native_adreno_import_diag[9]=address;
|
|
}
|
|
}
|
|
#include <IOKit/IOLib.h>
|
|
#include <string.h>
|
|
extern "C" {volatile uint32_t native_adreno_window_overflow[4][3];} /* {count, last size KiB, last end MiB} per client window */
|
|
NativeAdrenoUserClient::Entry *NativeAdrenoUserClient::find(uint64_t handle)
|
|
{
|
|
const uint32_t index=entry_count_?index_.find(handle):UINT32_MAX;
|
|
if(index>=entry_count_)return nullptr;
|
|
Entry &entry=at(index);
|
|
return entry.used && entry.handle==handle?&entry:nullptr;
|
|
}
|
|
/* A new live entry with a handle: counted and findable. */
|
|
void NativeAdrenoUserClient::publish(Entry &entry)
|
|
{
|
|
entry.used=true;entry.handle=next_handle_++;entry.token=entry.index+1;++live_;
|
|
index_.insert(entry.handle,entry.index);
|
|
}
|
|
void NativeAdrenoUserClient::clear(Entry &entry)
|
|
{
|
|
if(entry.used && live_)--live_;
|
|
index_.erase(entry.handle);
|
|
const size_t index=entry.index;if(index<free_hint_)free_hint_=index;
|
|
entry.handle=0;entry.mapped=0;entry.iova=0;entry.used=false;
|
|
entry.mapped_state=false;entry.unreleasable=false;entry.imported=false;entry.metadata_bytes=0;
|
|
entry.token=0;entry.mapping_leases=0;
|
|
}
|
|
/* A free entry, with room in the handle index for it to be published; null without memory. */
|
|
NativeAdrenoUserClient::Entry *NativeAdrenoUserClient::free()
|
|
{
|
|
if(!index_.reserve(live_+1))return nullptr;
|
|
for(;;) {
|
|
for(size_t i=free_hint_;i<entry_count_;++i)if(!at(i).used) {free_hint_=i;return &at(i);}
|
|
free_hint_=entry_count_;
|
|
if(!grow())return nullptr;
|
|
}
|
|
}
|
|
bool NativeAdrenoUserClient::grow()
|
|
{
|
|
if(entry_count_>=UINT32_MAX-EntryChunk)return false; /* tokens are entry indices plus one */
|
|
const size_t chunk_index=entry_count_/EntryChunk;
|
|
if(chunk_index==chunk_slots_) {
|
|
const size_t slots=chunk_slots_?chunk_slots_*2:16;
|
|
Entry **made=IONewData(Entry *,slots);
|
|
if(!made)return false;
|
|
for(size_t i=0;i<slots;++i)made[i]=i<chunk_slots_?chunks_[i]:nullptr;
|
|
if(chunks_)IODeleteData(chunks_,Entry *,chunk_slots_);
|
|
chunks_=made;chunk_slots_=slots;
|
|
}
|
|
Entry *chunk=IONewData(Entry,EntryChunk);
|
|
if(!chunk)return false;
|
|
memset(chunk,0,sizeof(Entry)*EntryChunk);
|
|
for(size_t i=0;i<EntryChunk;++i)chunk[i].index=(uint32_t)(entry_count_+i);
|
|
chunks_[chunk_index]=chunk;entry_count_+=EntryChunk;
|
|
return true;
|
|
}
|
|
void NativeAdrenoUserClient::releaseEntries()
|
|
{
|
|
for(size_t i=0;i<chunk_slots_;++i)if(chunks_[i])IODeleteData(chunks_[i],Entry,EntryChunk);
|
|
if(chunks_)IODeleteData(chunks_,Entry *,chunk_slots_);
|
|
chunks_=nullptr;chunk_slots_=0;entry_count_=0;
|
|
}
|
|
bool NativeAdrenoUserClient::growQueues()
|
|
{
|
|
if(queue_capacity_>=UINT32_MAX/2)return false;
|
|
const uint32_t size=queue_capacity_?queue_capacity_*2:8;
|
|
Queue *made=IONewData(Queue,size);
|
|
if(!made)return false;
|
|
for(uint32_t i=0;i<size;++i)made[i]=i<queue_capacity_?queues_[i]:Queue{};
|
|
if(queues_)IODeleteData(queues_,Queue,queue_capacity_);
|
|
queues_=made;queue_capacity_=size;
|
|
return true;
|
|
}
|
|
size_t NativeAdrenoUserClient::liveObjects() const
|
|
{
|
|
return live_;
|
|
}
|
|
bool NativeAdrenoUserClient::mapped(uint64_t handle)
|
|
{
|
|
Entry *entry=find(handle);return entry && entry->mapped_state;
|
|
}
|
|
NativeAdrenoMemory *NativeAdrenoUserClient::memoryOf(uint64_t handle)
|
|
{
|
|
Entry *entry=find(handle);return entry && !entry->unreleasable?&entry->memory:nullptr;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::open(uint32_t abi_version,NativeAdrenoVm *vm,
|
|
NativeAdrenoSubmit *submit,NativeAdrenoRing *ring,uint32_t input_bits,uint32_t output_bits)
|
|
{
|
|
if(open_ || vm_ || submit_)return kIOReturnBusy;
|
|
if(abi_version!=NativeAdrenoAbiVersion)return kIOReturnUnsupported;
|
|
if(!vm || !submit || !ring || input_bits<31 || input_bits>48 ||
|
|
output_bits<32 || output_bits>48 || vm->poisoned() || submit->poisoned() ||
|
|
ring->poisoned())return kIOReturnBadArgument;
|
|
if(!grow())return kIOReturnNoMemory;
|
|
if(!index_.allocate()) {releaseEntries();return kIOReturnNoMemory;}
|
|
if(!growQueues()) {releaseEntries();index_.release();return kIOReturnNoMemory;}
|
|
free_hint_=0;live_=0;
|
|
vm_=vm;submit_=submit;ring_=ring;input_bits_=input_bits;output_bits_=output_bits;
|
|
next_handle_=1;open_=true;
|
|
queues_[0]={0,true};
|
|
/* Each client is a new GPU context for the CP (useContext before each submission). */
|
|
static uint32_t contexts;
|
|
do context_=__atomic_add_fetch(&contexts,1,__ATOMIC_RELAXED); while(!context_);
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::setScope(uint64_t queue_base,uint64_t window_start,uint64_t window_bytes)
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(queue_base%NativeAdrenoQueueRange || (window_start&4095) || (window_bytes&4095) ||
|
|
(window_bytes && (window_start+window_bytes<window_start || window_start+window_bytes>(UINT64_C(1)<<input_bits_))))
|
|
return kIOReturnBadArgument;
|
|
/* A client that owns no range yet has none to drain: its queue_base_ is the default 0, the render server's range, and the render server
|
|
* nearly always has a job in flight, so every other process was refused here with kIOReturnBusy (int-200: WidgetRenderer_Default). */
|
|
if(liveObjects() || ownRangeOutstanding() || submit_->rangeOutstanding(queue_base,NativeAdrenoQueueRange))return kIOReturnBusy;
|
|
queue_base_=queue_base;scoped_=true;window_start_=window_start;window_bytes_=window_bytes;window_peak_=0;
|
|
return kIOReturnSuccess;
|
|
}
|
|
bool NativeAdrenoUserClient::ownRangeOutstanding() const
|
|
{
|
|
return (scoped_ || used_) && submit_->rangeOutstanding(queue_base_,NativeAdrenoQueueRange);
|
|
}
|
|
IOReturn NativeAdrenoUserClient::close()
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(liveObjects() || ownRangeOutstanding())return kIOReturnBusy;
|
|
for(uint32_t i=1;i<queue_capacity_;++i)if(queues_[i].active)return kIOReturnBusy;
|
|
releaseEntries();index_.release();
|
|
if(queues_)IODeleteData(queues_,Queue,queue_capacity_);
|
|
queues_=nullptr;queue_capacity_=0;
|
|
queue_base_=0;scoped_=used_=false;window_start_=window_bytes_=window_peak_=0;
|
|
open_=false;vm_=nullptr;submit_=nullptr;ring_=nullptr;timer_=nullptr;status_=nullptr;info_=nullptr;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::releaseAll(const NativeAdrenoRingClock *clock,uint32_t timeout_us)
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!clock)return kIOReturnBadArgument;
|
|
IOReturn status=(scoped_ || used_)?submit_->waitRange(queue_base_,NativeAdrenoQueueRange,clock,timeout_us):kIOReturnSuccess;
|
|
if(status)return status;
|
|
for(size_t index=0;index<entry_count_;++index) {
|
|
Entry &entry=at(index);
|
|
if(!entry.used)continue;
|
|
entry.mapping_leases=0;
|
|
NativeAdrenoAbiBoFree request{entry.handle};
|
|
status=freeBo(&request);
|
|
if(status)return status;
|
|
}
|
|
for(uint32_t i=1;i<queue_capacity_;++i)queues_[i]=Queue{};
|
|
return close();
|
|
}
|
|
NativeAdrenoMemory *NativeAdrenoUserClient::memoryForToken(uint32_t token)
|
|
{
|
|
if(!open_ || !token || token>entry_count_)return nullptr;
|
|
Entry &entry=at(token-1);
|
|
return entry.used && !entry.unreleasable && entry.token==token && entry.mapping_leases?&entry.memory:nullptr;
|
|
}
|
|
uint32_t NativeAdrenoUserClient::mappingLeases(uint32_t token) const
|
|
{
|
|
if(!open_ || !token || token>entry_count_)return 0;
|
|
const Entry &entry=at(token-1);
|
|
return entry.used && entry.token==token?entry.mapping_leases:0;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::query(NativeAdrenoAbiParams *params)
|
|
{
|
|
if(!params)return kIOReturnBadArgument;
|
|
*params={};
|
|
if(!open_)return kIOReturnNotReady;
|
|
params->abi_version=NativeAdrenoAbiVersion;
|
|
params->input_bits=input_bits_;params->output_bits=output_bits_;
|
|
params->flags=NativeAdrenoParamsHostImport|(ownedCoherent()?NativeAdrenoParamsOwnedCoherent:0u)|
|
|
(NativeAdrenoMemory::memoryPressure()?NativeAdrenoParamsMemoryPressure:0u);params->max_bo_bytes=maxBoBytes();
|
|
return kIOReturnSuccess;
|
|
}
|
|
/* Per client slot (4, the kernel's own client last): [0] live objects now, [1] their peak, [2] refused BoAllocs, [3] last refusal status,
|
|
* [4] bytes of the last refused request, [5] the window's highest mapped end in MiB above its start. Turnip turns any refusal into
|
|
* VK_ERROR_OUT_OF_DEVICE_MEMORY, which its command streams do not check (int-237: backboardd wrote through a NULL stream pointer). */
|
|
extern "C" {volatile uint32_t native_adreno_client_stats[5][6];}
|
|
/* Per client slot: MiB of its own BOs live now and at most (imports not counted). */
|
|
extern "C" {volatile uint32_t native_adreno_client_bytes[5][2];}
|
|
static volatile uint32_t *clientStats(uint32_t slot) {
|
|
return native_adreno_client_stats[slot<4?slot:4];
|
|
}
|
|
static void noteLive(uint32_t slot,size_t live) {
|
|
volatile uint32_t *stats=clientStats(slot);
|
|
stats[0]=(uint32_t)live;if(live>stats[1])stats[1]=(uint32_t)live;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::allocBo(const NativeAdrenoAbiBoAlloc *request,
|
|
NativeAdrenoAbiBoAllocReply *reply)
|
|
{
|
|
const IOReturn status=allocBoInner(request,reply);
|
|
if(status && request) {
|
|
volatile uint32_t *stats=clientStats(stats_slot_);
|
|
stats[2]=stats[2]+1;stats[3]=(uint32_t)status;stats[4]=(uint32_t)request->bytes;
|
|
}
|
|
if(entry_count_)noteLive(stats_slot_,liveObjects());
|
|
return status;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::allocBoInner(const NativeAdrenoAbiBoAlloc *request,
|
|
NativeAdrenoAbiBoAllocReply *reply)
|
|
{
|
|
if(!request || !reply)return kIOReturnBadArgument;reply->handle=0;
|
|
if(!open_)return kIOReturnNotReady;
|
|
const uint64_t limit=maxBoBytes();
|
|
if(!request->bytes || (request->bytes&4095) || request->bytes>limit ||
|
|
request->align_log2<12 || request->align_log2>24 || request->contiguous>1)
|
|
return kIOReturnBadArgument;
|
|
Entry *entry=free();
|
|
if(!entry)return kIOReturnNoResources;
|
|
/* User BOs are write-combined for the CPU, as the GPU maps them non-cacheable: no per-submission cache maintenance.
|
|
* int-147 kept them cacheable under IO coherence and mapped the GPU side write-back: the first Turnip IB executed zeros and the GPU hung
|
|
* (the kernel's cacheable alias read zeros past the first line too). A coherent own BO needs a client that asks for a write-back view. */
|
|
const bool write_combined=systemWriteCombined();
|
|
IOReturn status;
|
|
if(request->contiguous) {
|
|
/* Only a caller that needs one physical run gets the kernel's contiguous allocation (XNU scans physical memory for it). */
|
|
status=entry->memory.allocate((size_t)request->bytes,output_bits_,true,(size_t)request->bytes);
|
|
if(!status && write_combined)status=entry->memory.makeWriteCombined();
|
|
}
|
|
else status=entry->memory.allocateSystem((size_t)request->bytes,output_bits_,(size_t)request->bytes,write_combined,offered_pages_);
|
|
if(status) {
|
|
/* A failed allocation can leave quarantine behind; never reuse it. */
|
|
clear(*entry);
|
|
if(entry->memory.release()!=kIOReturnSuccess) {
|
|
entry->used=true;entry->unreleasable=true;++live_;
|
|
}
|
|
return status==kIOReturnUnsupported?kIOReturnUnsupported:status;
|
|
}
|
|
publish(*entry);
|
|
entry->mapped=0;entry->iova=0;entry->mapped_state=false;
|
|
reply->handle=entry->handle;
|
|
owned_bytes_+=entry->memory.bytes();if(owned_bytes_>owned_peak_)owned_peak_=owned_bytes_;
|
|
{volatile uint32_t *b=native_adreno_client_bytes[stats_slot_<4?stats_slot_:4];b[0]=(uint32_t)(owned_bytes_>>20);b[1]=(uint32_t)(owned_peak_>>20);}
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::importBo(const NativeAdrenoAbiBoImport *request,void *task,
|
|
NativeAdrenoAbiBoAllocReply *reply)
|
|
{
|
|
if(!request || !reply)return kIOReturnBadArgument;reply->handle=0;
|
|
if(!open_)return kIOReturnNotReady;
|
|
native_adreno_import_diag[0]=native_adreno_import_diag[0]+1;
|
|
const uint64_t limit=maxBoBytes();
|
|
if(!task || !request->address || ((request->address|request->bytes)&4095) || !request->bytes || request->bytes>limit) {
|
|
noteImport(kIOReturnBadArgument,1,request->bytes,request->address);
|
|
return kIOReturnBadArgument;
|
|
}
|
|
Entry *entry=free();
|
|
if(!entry) {noteImport(kIOReturnNoResources,2,request->bytes,request->address);return kIOReturnNoResources;}
|
|
const IOReturn status=entry->memory.importTask(task,request->address,(size_t)request->bytes,output_bits_,(size_t)request->bytes);
|
|
if(status) {
|
|
noteImport(status,native_adreno_import_stage,request->bytes,request->address);
|
|
clear(*entry);
|
|
if(entry->memory.release()!=kIOReturnSuccess) {entry->used=true;entry->unreleasable=true;++live_;}
|
|
return status;
|
|
}
|
|
publish(*entry);
|
|
entry->mapped=0;entry->iova=0;entry->mapped_state=false;entry->imported=true;entry->imported_pid=proc_selfpid();
|
|
reply->handle=entry->handle;
|
|
native_adreno_import_live[0]=native_adreno_import_live[0]+1;
|
|
{
|
|
const unsigned slot=3+(gImportNext++%8)*4;
|
|
native_adreno_import_live[slot]=request->address;native_adreno_import_live[slot+1]=(uint64_t)proc_selfpid()<<40|request->bytes;
|
|
native_adreno_import_live[slot+2]=entry->handle;native_adreno_import_live[slot+3]=1;
|
|
}
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::mapBo(const NativeAdrenoAbiBoMap *request,
|
|
NativeAdrenoAbiBoMapReply *reply)
|
|
{
|
|
if(!request || !reply)return kIOReturnBadArgument;reply->iova=0;
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!(request->permissions&1) || (request->permissions&~3u) || request->writeback>1 ||
|
|
(request->shareability!=2 && request->shareability!=3) || request->reserved ||
|
|
(request->iova&4095))return kIOReturnBadArgument;
|
|
Entry *entry=find(request->handle);
|
|
if(!entry || entry->unreleasable)return kIOReturnBadArgument;
|
|
if(entry->mapped_state)return kIOReturnBusy;
|
|
const uint64_t end=request->iova+entry->memory.bytes();
|
|
if(window_bytes_ && (request->iova<window_start_ || end<request->iova || end>window_start_+window_bytes_)) {
|
|
/* Per window: how often a mapping fell outside it, and the last one's size and end (MiB), for the boot log's [gpu windows] row. */
|
|
const uint32_t slot=stats_slot_;
|
|
if(slot<4) {
|
|
native_adreno_window_overflow[slot][0]=native_adreno_window_overflow[slot][0]+1;
|
|
native_adreno_window_overflow[slot][1]=uint32_t(entry->memory.bytes()>>10);
|
|
native_adreno_window_overflow[slot][2]=uint32_t(end>>20);
|
|
}
|
|
return kIOReturnBadArgument;
|
|
}
|
|
uint64_t handle=0;
|
|
/* Memory the owner maps read-only could only be pinned for reading: the GPU gets the read half of what was asked. */
|
|
const unsigned permissions=entry->memory.readOnly()?(request->permissions&~2u):request->permissions;
|
|
/* Imported memory is the owner's cacheable memory (IOSurfaces, no-copy buffers): once the board proved IO coherence, the GPU maps it
|
|
* write-back inner-shareable (Linux's IOMMU_CACHE PTE) and the CPU caches over it need no maintenance. The GPU's own BOs stay
|
|
* non-cacheable: their CPU view is write-combined, and a snoop could find a stale line of the kernel's cacheable alias. */
|
|
/* Only memory the GPU never writes is mapped coherent: sysmem passes written through write-back inner-shareable mappings stopped
|
|
* the GPU inside its end-of-pass cache clean (BR and BV idle at the clean events), while non-coherent mappings never did. */
|
|
const bool coherent=(entry->memory.imported() && native_adreno_io_coherent && !(permissions&2u)) || (!entry->memory.imported() && ownedCoherent());
|
|
const IOReturn status=vm_->map(&entry->memory,0,request->iova,permissions,
|
|
coherent?1u:request->writeback,coherent?3u:request->shareability,&handle);
|
|
if(status)return status;
|
|
entry->mapped=handle;entry->iova=request->iova;entry->mapped_state=true;
|
|
if(end>window_peak_)window_peak_=end;
|
|
if(window_bytes_ && window_peak_>window_start_)clientStats(stats_slot_)[5]=(uint32_t)((window_peak_-window_start_)>>20);
|
|
reply->iova=request->iova;
|
|
return kIOReturnSuccess;
|
|
}
|
|
static void noteFreeRefused(bool imported,uint64_t handle,IOReturn status)
|
|
{
|
|
if(!imported)return;
|
|
native_adreno_import_live[1]=native_adreno_import_live[1]+1;native_adreno_import_live[2]=(uint32_t)status;noteImportState(handle,3);
|
|
}
|
|
IOReturn NativeAdrenoUserClient::freeBo(const NativeAdrenoAbiBoFree *request)
|
|
{
|
|
native_adreno_trace_stage=1;
|
|
if(!request)return kIOReturnBadArgument;
|
|
if(!open_)return kIOReturnNotReady;
|
|
Entry *entry=find(request->handle);
|
|
if(!entry)return kIOReturnBadArgument;
|
|
if(entry->unreleasable) {noteFreeRefused(entry->imported,entry->handle,kIOReturnBusy);return kIOReturnBusy;}
|
|
if(entry->mapping_leases) {noteFreeRefused(entry->imported,entry->handle,kIOReturnBusy);return kIOReturnBusy;}
|
|
native_adreno_trace_stage=2;
|
|
if(entry->mapped_state) {
|
|
/* Completed jobs must not pin the buffer: retire what the CP has passed before the unmap decides. */
|
|
submit_->retireCompleted(); /* a poisoned execbuf refuses and the unmap below stays refused too */
|
|
native_adreno_trace_stage=3;
|
|
const IOReturn unmap=vm_->unmap(entry->mapped);
|
|
native_adreno_trace_stage=4;
|
|
if(unmap) {noteFreeRefused(entry->imported,entry->handle,unmap);return unmap;}
|
|
entry->mapped_state=false;entry->mapped=0;entry->iova=0;
|
|
}
|
|
const size_t bytes=entry->memory.bytes();
|
|
const IOReturn release=freed_pages_?entry->memory.releaseSystemInto(freed_pages_):entry->memory.release();
|
|
if(release) {noteFreeRefused(entry->imported,entry->handle,release);return release;}
|
|
if(!entry->imported) {
|
|
owned_bytes_-=bytes<owned_bytes_?bytes:owned_bytes_;
|
|
native_adreno_client_bytes[stats_slot_<4?stats_slot_:4][0]=(uint32_t)(owned_bytes_>>20);
|
|
}
|
|
if(entry->imported) {native_adreno_import_live[0]=native_adreno_import_live[0]-1;noteImportState(entry->handle,2);}
|
|
clear(*entry);
|
|
return kIOReturnSuccess;
|
|
}
|
|
/* The GPU priority this process's next submission takes: its queues' foreground value while it renders visible UI (a GPU role above
|
|
* utility, or an unmanaged task), their background value for the denied, background and utility roles. A process that never told us keeps the daemon's default (no ring choice). */
|
|
/* Which processes told the driver what (the newest 32): pid, foreground priority, background priority, the GPU role then, the time (ms of
|
|
* the submission clock when the board gave one, else 0) and how many times this connection had told it before (0 the runtime's first call
|
|
* at device open, later ones the process's own setGPUPriority:/setBackgroundGPUPriority: through the frontend). */
|
|
extern "C" {volatile uint32_t native_adreno_prio_log[32][6];volatile uint32_t native_adreno_prio_count;}
|
|
extern "C" uint64_t native_adreno_now_us(void) __attribute__((weak));
|
|
int32_t NativeAdrenoUserClient::submitPriority() const
|
|
{
|
|
if(priority_foreground_<0 && priority_background_<0)return -1;
|
|
const int32_t role=task_get_gpu_role(current_task());
|
|
/* A task RunningBoard denies the GPU renders nothing visible either: where it still submits it takes the background value. */
|
|
const bool background=role==PRIO_DARWIN_GPU_DENY || role==PRIO_DARWIN_GPU_BACKGROUND || role==PRIO_DARWIN_GPU_UTILITY;
|
|
return background?(priority_background_>=0?priority_background_:priority_foreground_)
|
|
:(priority_foreground_>=0?priority_foreground_:priority_background_);
|
|
}
|
|
IOReturn NativeAdrenoUserClient::setPriority(uint32_t foreground,uint32_t background)
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(foreground>5 || background>5 || foreground==3)return kIOReturnBadArgument;
|
|
priority_foreground_=int32_t(foreground);
|
|
priority_background_=int32_t(background);
|
|
{
|
|
const uint32_t n=native_adreno_prio_count;
|
|
volatile uint32_t *row=native_adreno_prio_log[n&31];
|
|
row[0]=uint32_t(proc_selfpid());row[1]=foreground;row[2]=background;row[3]=uint32_t(task_get_gpu_role(current_task()));
|
|
row[4]=&native_adreno_now_us?uint32_t(native_adreno_now_us()/1000):0;row[5]=priority_told_;
|
|
native_adreno_prio_count=n+1;
|
|
if(priority_told_<0xffffffffu)++priority_told_;
|
|
}
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::submit(const NativeAdrenoAbiSubmit *request,
|
|
NativeAdrenoAbiSubmitReply *reply,const NativeAdrenoRingClock *clock)
|
|
{
|
|
if(!request || !reply)return kIOReturnBadArgument;reply->fence=0;
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!request->buffer_count || request->buffer_count>NativeAdrenoAbiMaxBuffers ||
|
|
request->reserved)return kIOReturnBadArgument;
|
|
if(request->queue>=queue_capacity_ || !queues_[request->queue].active)
|
|
return kIOReturnBadArgument;
|
|
if(request->wait_fence) {
|
|
if(!clock)return kIOReturnBadArgument;
|
|
const uint32_t wait_us=request->wait_timeout_ns==0?0:
|
|
(request->wait_timeout_ns>UINT64_C(0xffffffff)*1000u?0xffffffffu:
|
|
(uint32_t)(request->wait_timeout_ns/1000));
|
|
const IOReturn waited=submit_->observeFence((uint32_t)request->wait_fence,clock,wait_us);
|
|
if(waited)return waited;
|
|
}
|
|
NativeAdrenoSubmitBuffer *buffers=submit_buffers_;
|
|
for(uint32_t i=0;i<request->buffer_count;++i) {
|
|
const NativeAdrenoAbiSubmitBuffer &wire=request->buffers[i];
|
|
if(!wire.bytes || (wire.bytes&3))return kIOReturnBadArgument;
|
|
Entry *entry=find(wire.handle);
|
|
if(!entry || !entry->mapped_state)return kIOReturnBadArgument;
|
|
buffers[i]={entry->mapped,wire.iova,(size_t)wire.bytes};
|
|
}
|
|
uint32_t fence=0;
|
|
used_=true;
|
|
ring_->useContext(context_);
|
|
submit_->nextOwner(proc_selfpid());
|
|
submit_->nextPriority(submitPriority());
|
|
submit_->nextRole(task_get_gpu_role(current_task()));
|
|
const IOReturn status=submit_->submit(vm_,buffers,request->buffer_count,
|
|
queue_base_+request->queue,&fence,clock);
|
|
if(status)return status;
|
|
reply->fence=fence;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::attachTimer(const NativeAdrenoTimer *timer)
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!timer || timer_ )return kIOReturnBadArgument;
|
|
timer_=timer;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::timestamp(NativeAdrenoAbiTimestamp *reply)
|
|
{
|
|
if(!reply)return kIOReturnBadArgument;*reply={};
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!timer_)return kIOReturnUnsupported;
|
|
return timer_->read(&reply->ticks);
|
|
}
|
|
IOReturn NativeAdrenoUserClient::attachDeviceInfo(const NativeAdrenoAbiDeviceInfo *info)
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!info || info_ || info->reserved || !info->chip_id || !info->va_size)return kIOReturnBadArgument;
|
|
info_=info;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::deviceInfo(NativeAdrenoAbiDeviceInfo *reply)
|
|
{
|
|
if(!reply)return kIOReturnBadArgument;*reply={};
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!info_)return kIOReturnUnsupported;
|
|
*reply=*info_;
|
|
if(window_bytes_){reply->va_start=window_start_;reply->va_size=window_bytes_;}
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::attachStatus(const NativeAdrenoStatus *status)
|
|
{
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!status || status_)return kIOReturnBadArgument;
|
|
status_=status;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::status(NativeAdrenoAbiStatus *reply)
|
|
{
|
|
if(!reply)return kIOReturnBadArgument;*reply={};
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!status_)return kIOReturnUnsupported;
|
|
/* msm MSM_PARAM_FAULTS: the global count plus this client's own VM's (a hang its submission caused). */
|
|
const NativeAdrenoStatusCounts counts=status_->counts();
|
|
reply->faults=counts.faults+(vm_?vm_->faults():0);reply->suspends=counts.suspends;
|
|
return kIOReturnSuccess;
|
|
}
|
|
size_t NativeAdrenoUserClient::inputSize(uint32_t method)
|
|
{
|
|
switch(method) {
|
|
case NativeAdrenoMethodQuery:
|
|
case NativeAdrenoMethodGetTimestamp:
|
|
case NativeAdrenoMethodGetStatus:
|
|
case NativeAdrenoMethodGetDeviceInfo:return 0;
|
|
case NativeAdrenoMethodBoAlloc:return sizeof(NativeAdrenoAbiBoAlloc);
|
|
case NativeAdrenoMethodBoMap:return sizeof(NativeAdrenoAbiBoMap);
|
|
case NativeAdrenoMethodBoImport:return sizeof(NativeAdrenoAbiBoImport);
|
|
case NativeAdrenoMethodBoFree:return sizeof(NativeAdrenoAbiBoFree);
|
|
case NativeAdrenoMethodSubmit:return sizeof(NativeAdrenoAbiSubmit);
|
|
case NativeAdrenoMethodSetMetadata:
|
|
case NativeAdrenoMethodGetMetadata:return sizeof(NativeAdrenoAbiMetadata);
|
|
case NativeAdrenoMethodCreateQueue:
|
|
case NativeAdrenoMethodCloseQueue:return sizeof(NativeAdrenoAbiQueue);
|
|
case NativeAdrenoMethodBoCpuMap:return sizeof(NativeAdrenoAbiBoCpuMap);
|
|
case NativeAdrenoMethodBoCpuUnmap:return sizeof(NativeAdrenoAbiBoCpuUnmap);
|
|
case NativeAdrenoMethodWaitFence:return sizeof(NativeAdrenoAbiWaitFence);
|
|
default:return SIZE_MAX;
|
|
}
|
|
}
|
|
size_t NativeAdrenoUserClient::outputSize(uint32_t method)
|
|
{
|
|
switch(method) {
|
|
case NativeAdrenoMethodQuery:return sizeof(NativeAdrenoAbiParams);
|
|
case NativeAdrenoMethodBoAlloc:return sizeof(NativeAdrenoAbiBoAllocReply);
|
|
case NativeAdrenoMethodBoMap:return sizeof(NativeAdrenoAbiBoMapReply);
|
|
case NativeAdrenoMethodBoFree:return 0;
|
|
case NativeAdrenoMethodSubmit:return sizeof(NativeAdrenoAbiSubmitReply);
|
|
case NativeAdrenoMethodWaitFence:return 0;
|
|
case NativeAdrenoMethodGetTimestamp:return sizeof(NativeAdrenoAbiTimestamp);
|
|
case NativeAdrenoMethodGetStatus:return sizeof(NativeAdrenoAbiStatus);
|
|
case NativeAdrenoMethodGetDeviceInfo:return sizeof(NativeAdrenoAbiDeviceInfo);
|
|
case NativeAdrenoMethodGetMetadata:return sizeof(NativeAdrenoAbiMetadata);
|
|
case NativeAdrenoMethodSetMetadata:return 0;
|
|
case NativeAdrenoMethodCreateQueue:return sizeof(NativeAdrenoAbiQueueReply);
|
|
case NativeAdrenoMethodCloseQueue:return 0;
|
|
case NativeAdrenoMethodBoCpuMap:return sizeof(NativeAdrenoAbiBoCpuMapReply);
|
|
case NativeAdrenoMethodBoCpuUnmap:return 0;
|
|
case NativeAdrenoMethodBoImport:return sizeof(NativeAdrenoAbiBoAllocReply);
|
|
default:return SIZE_MAX;
|
|
}
|
|
}
|
|
IOReturn NativeAdrenoUserClient::dispatch(uint32_t method,const void *input,
|
|
size_t input_size,void *output,size_t output_size,const NativeAdrenoRingClock *clock)
|
|
{
|
|
struct Tally {
|
|
uint32_t method;uint64_t began;IOReturn *result;const NativeAdrenoRingClock *clock;
|
|
Tally(uint32_t m,IOReturn *r,const NativeAdrenoRingClock *c):method(m),began(c && c->now_us?c->now_us(c->cookie):0),result(r),clock(c) {}
|
|
~Tally() {
|
|
if(method>=20)return;
|
|
const uint64_t now=clock && clock->now_us?clock->now_us(clock->cookie):0;
|
|
native_adreno_method_stats[method][0]=native_adreno_method_stats[method][0]+1;
|
|
if(*result)native_adreno_method_stats[method][1]=native_adreno_method_stats[method][1]+1;
|
|
native_adreno_method_stats[method][2]=(uint32_t)*result;
|
|
native_adreno_method_stats[method][3]=native_adreno_method_stats[method][3]+(uint32_t)(now-began);
|
|
}
|
|
};
|
|
/* The same clock times the VM's table transactions (the board service's probe never ran on this path, so nothing else installed it). */
|
|
static const NativeAdrenoRingClock *vm_clock=nullptr;
|
|
vm_clock=clock;
|
|
native_adreno_vm_now_us=[]()->uint64_t{return vm_clock && vm_clock->now_us?vm_clock->now_us(vm_clock->cookie):0;};
|
|
IOReturn tally_result=kIOReturnSuccess;
|
|
Tally tally(method,&tally_result,clock);
|
|
const size_t expected_in=inputSize(method),expected_out=outputSize(method);
|
|
if(expected_in==SIZE_MAX || expected_out==SIZE_MAX)return kIOReturnUnsupported;
|
|
if(input_size!=expected_in || output_size!=expected_out)return kIOReturnBadArgument;
|
|
if((expected_in && !input) || (expected_out && !output))return kIOReturnBadArgument;
|
|
IOReturn status=kIOReturnSuccess;
|
|
switch(method) {
|
|
case NativeAdrenoMethodQuery: {
|
|
NativeAdrenoAbiParams reply{};
|
|
status=query(&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodBoAlloc: {
|
|
NativeAdrenoAbiBoAlloc request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
NativeAdrenoAbiBoAllocReply reply{};
|
|
status=allocBo(&request,&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodBoImport:
|
|
status=kIOReturnUnsupported;
|
|
break;
|
|
case NativeAdrenoMethodBoMap: {
|
|
NativeAdrenoAbiBoMap request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
NativeAdrenoAbiBoMapReply reply{};
|
|
status=mapBo(&request,&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
else {
|
|
/* Why maps fail (int-135: 55 of 191, and the translator answers each as a device out-of-memory): status, count, and what the table
|
|
* held and the failed request asked for the last time. */
|
|
unsigned at=0;while(at<5 && native_adreno_map_fail[at][0] && native_adreno_map_fail[at][0]!=(uint32_t)status)++at;
|
|
native_adreno_map_fail[at][0]=(uint32_t)status;native_adreno_map_fail[at][1]=native_adreno_map_fail[at][1]+1;
|
|
native_adreno_map_fail[5][0]=(uint32_t)vm_->spanCount();native_adreno_map_fail[5][1]=(uint32_t)vm_->objectCount();
|
|
native_adreno_map_fail[6][0]=(uint32_t)(request.iova>>12);native_adreno_map_fail[6][1]=(uint32_t)(request.handle);
|
|
{
|
|
uint64_t met[5]={};
|
|
const bool found=vm_->occupant(request.iova,met);
|
|
Entry *asked=find(request.handle);
|
|
/* The bytes asked, with bit 63 where the handle itself was already mapped (the other source of this status). */
|
|
native_adreno_map_met[0]=request.iova;
|
|
native_adreno_map_met[1]=(asked?asked->memory.bytes():0)|(asked && asked->mapped_state?UINT64_C(1)<<63:0);
|
|
for(unsigned k=0;k<5;++k)native_adreno_map_met[2+k]=found?met[k]:~UINT64_C(0);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodBoFree: {
|
|
NativeAdrenoAbiBoFree request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
status=freeBo(&request);
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodSubmit: {
|
|
NativeAdrenoAbiSubmit &request=submit_request_;
|
|
memcpy(&request,input,sizeof(request));
|
|
NativeAdrenoAbiSubmitReply reply{};
|
|
status=submit(&request,&reply,clock);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodWaitFence: {
|
|
NativeAdrenoAbiWaitFence request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
status=waitFence(&request,clock);
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodGetDeviceInfo: {
|
|
NativeAdrenoAbiDeviceInfo reply{};
|
|
status=deviceInfo(&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodGetTimestamp: {
|
|
NativeAdrenoAbiTimestamp reply{};
|
|
status=timestamp(&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodBoCpuMap: {
|
|
NativeAdrenoAbiBoCpuMap request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
NativeAdrenoAbiBoCpuMapReply reply{};
|
|
status=cpuMap(&request,&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodBoCpuUnmap: {
|
|
NativeAdrenoAbiBoCpuUnmap request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
status=cpuUnmap(&request);
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodCreateQueue: {
|
|
NativeAdrenoAbiQueueReply reply{};
|
|
status=createQueue(&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodCloseQueue: {
|
|
NativeAdrenoAbiQueue request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
status=closeQueue(&request);
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodSetMetadata: {
|
|
NativeAdrenoAbiMetadata request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
status=setMetadata(&request);
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodGetMetadata: {
|
|
NativeAdrenoAbiMetadata request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
NativeAdrenoAbiMetadata reply{};
|
|
status=getMetadata(&request,&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
case NativeAdrenoMethodGetStatus: {
|
|
NativeAdrenoAbiStatus reply{};
|
|
status=this->status(&reply);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
break;
|
|
}
|
|
default:return kIOReturnUnsupported;
|
|
}
|
|
tally_result=status;
|
|
return status;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::cpuMap(const NativeAdrenoAbiBoCpuMap *request,
|
|
NativeAdrenoAbiBoCpuMapReply *reply)
|
|
{
|
|
if(!request || !reply)return kIOReturnBadArgument;*reply={};
|
|
if(!open_)return kIOReturnNotReady;
|
|
Entry *entry=find(request->handle);
|
|
if(!entry || entry->unreleasable)return kIOReturnBadArgument;
|
|
/* An imported BO is the caller's own memory, already mapped where it came from. */
|
|
if(entry->memory.imported())return kIOReturnUnsupported;
|
|
if(entry->mapping_leases==UINT32_MAX)return kIOReturnNoResources;
|
|
++entry->mapping_leases;
|
|
reply->token=entry->token;reply->bytes=entry->memory.bytes();
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::cpuUnmap(const NativeAdrenoAbiBoCpuUnmap *request)
|
|
{
|
|
if(!request)return kIOReturnBadArgument;
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(request->reserved)return kIOReturnBadArgument;
|
|
Entry *entry=find(request->handle);
|
|
if(!entry || entry->unreleasable || !entry->token)return kIOReturnBadArgument;
|
|
if(request->token!=entry->token)return kIOReturnBadArgument;
|
|
if(!entry->mapping_leases)return kIOReturnBusy;
|
|
--entry->mapping_leases;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::createQueue(NativeAdrenoAbiQueueReply *reply)
|
|
{
|
|
if(!reply)return kIOReturnBadArgument;
|
|
if(!open_)return kIOReturnNotReady;
|
|
for(;;) {
|
|
for(uint32_t i=1;i<queue_capacity_;++i)if(!queues_[i].active) {
|
|
queues_[i]={i,true};
|
|
reply->queue=i;
|
|
return kIOReturnSuccess;
|
|
}
|
|
if(!growQueues())return kIOReturnNoMemory;
|
|
}
|
|
}
|
|
IOReturn NativeAdrenoUserClient::closeQueue(const NativeAdrenoAbiQueue *request)
|
|
{
|
|
if(!request)return kIOReturnBadArgument;
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!request->queue || request->queue>=queue_capacity_)return kIOReturnBadArgument;
|
|
Queue &queue=queues_[request->queue];
|
|
if(!queue.active)return kIOReturnBadArgument;
|
|
/* A queue must be drained by the caller before it disappears. */
|
|
if(submit_->queueOutstanding(queue_base_+request->queue))
|
|
return kIOReturnBusy;
|
|
queue={};
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::setMetadata(const NativeAdrenoAbiMetadata *request)
|
|
{
|
|
if(!request)return kIOReturnBadArgument;
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(request->reserved || request->bytes>NativeAdrenoAbiMetadataBytes)return kIOReturnBadArgument;
|
|
Entry *entry=find(request->handle);
|
|
if(!entry || entry->unreleasable)return kIOReturnBadArgument;
|
|
if(request->bytes)memcpy(entry->metadata,request->text,request->bytes);
|
|
entry->metadata_bytes=request->bytes;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::getMetadata(const NativeAdrenoAbiMetadata *request,
|
|
NativeAdrenoAbiMetadata *reply)
|
|
{
|
|
if(!request || !reply)return kIOReturnBadArgument;
|
|
/* Snapshot the request so an in/out alias is still well defined. */
|
|
const uint64_t handle=request->handle;
|
|
const uint32_t asked=request->bytes;
|
|
const uint32_t reserved=request->reserved;
|
|
*reply={};
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(reserved || asked>NativeAdrenoAbiMetadataBytes)return kIOReturnBadArgument;
|
|
Entry *entry=find(handle);
|
|
if(!entry || entry->unreleasable)return kIOReturnBadArgument;
|
|
if(asked<entry->metadata_bytes)return kIOReturnBadArgument;
|
|
reply->handle=entry->handle;reply->bytes=entry->metadata_bytes;
|
|
if(entry->metadata_bytes)memcpy(reply->text,entry->metadata,entry->metadata_bytes);
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoUserClient::waitFence(const NativeAdrenoAbiWaitFence *request,
|
|
const NativeAdrenoRingClock *clock)
|
|
{
|
|
if(!request || !clock)return kIOReturnBadArgument;
|
|
if(!open_)return kIOReturnNotReady;
|
|
if(!request->fence)return kIOReturnBadArgument;
|
|
const uint32_t timeout_us=request->timeout_ns==0?0:
|
|
(request->timeout_ns>UINT64_C(0xffffffff)*1000u?0xffffffffu:
|
|
(uint32_t)(request->timeout_ns/1000));
|
|
return submit_->wait((uint32_t)request->fence,clock,timeout_us);
|
|
}
|