mirror of
https://github.com/UniXNU/NativeAdreno.git
synced 2026-11-17 04:51:59 +03:00
1440 lines
89 KiB
C++
1440 lines
89 KiB
C++
/* SPDX-License-Identifier: MIT */
|
|
/* IOKit surface for an already-running GPU context. The board code (kits/sheng) brings the GPU up, then hands this file
|
|
* the live VM, submit object, ring, clock and optional timer; one IOService is published and each IOServiceOpen creates a
|
|
* connection that mounts its own NativeAdrenoUserClient core. Every connection of a user process gets its own GPU address space
|
|
* (NativeAdrenoProcessVm: its own page table and ASID, switched into the GPU's context bank for its submissions) and its own queue
|
|
* range on the one ring; there is no fixed number of connections. Closing a connection waits for its own jobs, unmaps and frees
|
|
* everything it created and frees its address space. The board's kernel connection uses the kernel table. A GPU hang is found by msm's
|
|
* hangcheck and restarted through the board's hook; only the client whose submission hung sees its device lost. This is not the Metal
|
|
* or Turnip ABI; it carries NativeAdrenoABI.h unchanged.
|
|
* BO CPU mappings are handed out as uncached physical-range mappings to match the GPU's non-cacheable PTEs; a system BO's view maps
|
|
* its own descriptor, and a pool BO's view its physical runs, write-combined (NativeAdrenoPoolView).
|
|
* The connection class is named AGXDeviceUserClient on purpose: the original sandbox profiles of the processes that use Metal
|
|
* allow IOServiceOpen for that user-client class name, and nothing else in this system can create one (the Apple AGX kext is
|
|
* never loaded without an Apple GPU), so the name only decides whether the Adreno Metal plugin may open its own driver. */
|
|
#include <IOKit/IOLib.h>
|
|
#include <IOKit/IOMemoryDescriptor.h>
|
|
#include <IOKit/IOBufferMemoryDescriptor.h>
|
|
#include <IOKit/IOService.h>
|
|
#include <IOKit/IOPlatformExpert.h>
|
|
#include <pexpert/pexpert.h>
|
|
#include <IOKit/IOUserClient.h>
|
|
#include <string.h>
|
|
#include <kern/thread.h>
|
|
#include <kern/thread_call.h>
|
|
#include <kern/clock.h>
|
|
#include "NativeAdrenoUserClient.h"
|
|
#include "NativeAdrenoGpuContext.h"
|
|
#include "NativeAdrenoPool.h"
|
|
#include "NativeGpuTrace.inc"
|
|
|
|
extern "C" void proc_selfname(char *, int);
|
|
/* The system's render server, the process that composes the display: backboardd on iPadOS, WindowServer on macOS. It gets the
|
|
* reserved row 0 and its mailbox page, the cache export and its own report line. */
|
|
#if defined(XNU_TARGET_OS_OSX) && XNU_TARGET_OS_OSX
|
|
static constexpr char kRenderServer[]="WindowServer";
|
|
#else
|
|
static constexpr char kRenderServer[]="backboardd";
|
|
#endif
|
|
static bool isRenderServer(const char *name,size_t size) {return !strncmp(name,kRenderServer,size);}
|
|
extern "C" volatile uint32_t native_adreno_table_switches[8];
|
|
/* The plugin's telemetry table (rows of text the Metal bridge keeps), sent as markers: 0xC000|row starts a row, 0x8000|(a<<7)|b appends two characters. */
|
|
extern "C" {char native_adreno_metal_rows[48][96];volatile uint32_t native_adreno_metal_rows_seq;}
|
|
|
|
extern "C" {
|
|
/* What the processes that reached this driver did, for the boot diagnostic page. Plain counters and short strings. */
|
|
struct NativeAdrenoMetalDiag {
|
|
uint32_t opens, busy, bad, closes, reports, property_reads, foreign_opens, foreign_type, name_reads, calls, last_selector, last_status, ext_entries, ext_selector, ext_status, ext_in, ext_out, tunnel_calls, probes, sel_bits[8], mb_maps, mb_calls, marks_total, marks[16], chk_macf, chk_res, chk_type, type_pass_lo, type_pass_hi, attach_status, inflight, inflight_since_calls, forced_unmaps;
|
|
char opener[3][17]; /* process names of the latest connections, newest first */
|
|
char reader[3][17]; /* process names that read a registry property of the accelerator, newest first */
|
|
char name_reader[17]; /* latest process that read MetalPluginName specifically */
|
|
char starter[3][17]; /* processes that sent plugin marker 1 (the plugin's initWithAcceleratorPort: started), newest first */
|
|
char report[64]; /* the plugin's own result line (SetMetadata text), latest */
|
|
char report2[64]; /* the plugin's Turnip-phase line (text starting "T " or "W "), latest */
|
|
uint32_t grants; /* opens granted to the render server by this driver in place of the sandbox verdict (the view in GpuPowerProbe.cpp ends before this) */
|
|
} native_adreno_metal_diag;
|
|
/* BO CPU views (clientMemoryForType for a BO token): [0] calls, [1] new descriptors, [2] served from the view cache, [3] refused (not a BO / out of
|
|
* resources), [4] last refusal's IOReturn, [5] threads inside now. int-115 showed a backboardd thread in IOConnectMapMemory64 ~650 times a second. */
|
|
extern "C" {volatile uint32_t native_adreno_view_diag[6],native_adreno_view_kept;} /* kept: BoCpuUnmap calls that left the view for another lease */
|
|
/* The BO views each process holds, as native_adreno_bo_owners holds the BOs: {pid, live KiB, live views, peak KiB}. A view is an IOKit
|
|
* mapping of the BO's memory in the client, and the VM bills such a mapping to the task that maps it by its whole size (vm_map_enter's
|
|
* iokit_acct entries: the iokit_mapped ledger, which is part of the footprint the memory limits judge). A BO no process views is billed
|
|
* to none. A view mapped twice is billed twice and counted once here. */
|
|
volatile uint32_t native_adreno_view_owners[24][4];
|
|
}
|
|
|
|
/* The open type the Adreno plugin passes to IOServiceOpen ('AG'); it lets the driver tell the plugin from the original IOGPU library
|
|
* (type 1 plus an API value) and from any other client of the accelerator. */
|
|
enum : UInt32 {NativeAdrenoPluginOpenType=0x4147,NativeAdrenoTunnelMagic=0x4e544e4c /* 'NTNL' */};
|
|
/* The display hand-off (boot-arg fb-gpu=1, drivers/GopFramebuffer) reaches the plugin through this already admitted connection: the
|
|
* original sandbox profiles admit this class but not the presenter's own service. Methods 0x7000 + the presenter's NGP_METHOD numbers (mode,
|
|
* wait for a frame, complete it) are forwarded unchanged, without the GPU lock (a wait sleeps); memory type NativeAdrenoScanoutType is the
|
|
* writable scanout. Nothing here touches the GPU. */
|
|
enum : UInt32 {NativeAdrenoHandoffBase=0x7000,NativeAdrenoHandoffLast=0x7003,NativeAdrenoScanoutType=0x5343,
|
|
NativeAdrenoHandoffOpenType=0x4148};
|
|
/* Open type 0x4148 is a connection that serves only the hand-off: it does not mount a GPU core, so it does not need (or take) a
|
|
* GPU connection slot (the Vulkan driver holds one for as long as the Metal device lives). Same class, so the same sandbox admission. */
|
|
extern "C" IOReturn native_gop_handoff_call(uint32_t op,task_t task,const void *in,size_t inSize,void *out,size_t outSize);
|
|
/* Export (int-147): method NativeAdrenoExportMethod hands one in-order piece of a blob to the board's UFS log (kits/sheng/BootLog.c), which writes
|
|
* the newest complete blob to its export area. The render server sends its pipeline caches this way, so a later image can be built with them.
|
|
* Input: {generation, total, offset, bytes} then the bytes. No GPU state is touched; any connection of the render server may send it. */
|
|
enum : UInt32 {NativeAdrenoExportMethod=0x7100,NativeAdrenoExportMaxPiece=3968};
|
|
extern "C" int boot_log_export_put(uint32_t generation,uint32_t total,uint32_t offset,const void *data,uint32_t size) __attribute__((weak_import));
|
|
extern "C" {volatile uint32_t native_adreno_export_stats[4];} /* [0] pieces taken, [1] refused, [2] last generation, [3] its total bytes */
|
|
extern "C" IOMemoryDescriptor *native_gop_handoff_memory(void);
|
|
|
|
/* Mailbox: a last-resort command channel for a sandbox that admits no external-method selector at all. It is one shared page (memory
|
|
* type NativeAdrenoMailboxType through IOConnectMapMemory64, which the selector filter does not cover) and a kernel poller that runs
|
|
* whatever the live connection writes into it. Layout (little endian dwords): +0 magic, +4 request seq, +8 response seq, +12 method,
|
|
* +16 status, +20 input bytes, +24 output bytes, input at +0x40 (max 0x700), output at +0x800 (max 0x700). */
|
|
enum : UInt32 {NativeAdrenoMailboxType=0xC0DE,NativeAdrenoMailboxMagic=0x4d41494c /* 'MAIL' */};
|
|
namespace {
|
|
IOBufferMemoryDescriptor *gMailbox=nullptr;
|
|
volatile uint32_t *gMailboxWords=nullptr;
|
|
volatile bool gPollerStarted=false;
|
|
/* True while the owner's task holds a mapping of the mailbox page: only then can a request appear. Written under g.lock; the poller
|
|
* reads it once without the lock as a hint and again under it. Its address is the poller's sleep event. */
|
|
bool gMailboxMapped=false;
|
|
}
|
|
class AGXDeviceUserClient;
|
|
/* The render server's connection serves the mailbox; it is retained while set (under g.lock), so the poller never runs on a freed object. */
|
|
static AGXDeviceUserClient *gMailboxOwner=nullptr;
|
|
void nativeAdrenoMailboxPoller(void *,wait_result_t);
|
|
static IOReturn runCore(AGXDeviceUserClient *,uint32_t method,const void *input,size_t input_size,void *output,size_t output_size,uint64_t *lockWait);
|
|
/* Every call into a connection, for the boot export (kernel.gpucalls.bin): when it came in (mach units), microseconds in the driver and
|
|
* of those waiting for the device lock, the ABI method, how long it held the lock (16 us units), the caller's pid, the status, and for an
|
|
* allocation its KiB. The newest 262144; the board's export code allocates the ring once the home screen draws, and no call is recorded
|
|
* before. */
|
|
struct NativeAdrenoCallRec {uint64_t entered;uint32_t us,lock_us;uint16_t method,held16;int32_t pid;uint32_t status,kib;};
|
|
static_assert(sizeof(NativeAdrenoCallRec)==32,"call record");
|
|
extern "C" {NativeAdrenoCallRec *native_adreno_call_ring;volatile uint32_t native_adreno_call_count;int proc_selfpid(void);}
|
|
/* A call's lock clock: [0] waited for the lock, [1] held it (mach units), [2] when it was last taken. */
|
|
enum {LockWaited,LockHeld,LockSince,LockClockWords};
|
|
static void lockTimed(uint64_t *clock);
|
|
static void unlockTimed(uint64_t *clock);
|
|
/* A large BoAlloc's phases for the boot export (kernel.boalloc.bin, the board fills it with the caller's user stack): microseconds waiting
|
|
* for the lock before the client's limit was read, making the descriptor, wiring its pages, cleaning them, waiting for the lock again,
|
|
* the allocation under the lock, and giving back pages it did not take. flags: 1 write-combined, 2 the prepared pages were taken, 4 they
|
|
* were cleaned outside the lock, 8 prepared at all, 16 the pages are pool pages (their taking and zero fill is the time of making the
|
|
* descriptor; nothing is wired or cleaned). spans: the BO's physical runs. Only BOs of 256 KiB and more, once the board sets the
|
|
* hook. */
|
|
struct NativeAdrenoBoTrace {uint64_t entered;uint32_t kib,flags,limit_us,create_us,wire_us,clean_us,lock_us,dispatch_us,discard_us,spans;
|
|
int32_t status;uint32_t reserved;};
|
|
static_assert(sizeof(NativeAdrenoBoTrace)==56,"bo trace");
|
|
extern "C" {void (*native_adreno_bo_hook)(const NativeAdrenoBoTrace *);}
|
|
/* BoFrees whose pages were unwired and freed after the lock (the BoFree row of the boot log). */
|
|
extern "C" {volatile uint32_t native_adreno_freed_outside;}
|
|
extern "C" volatile uint32_t native_adreno_bo_memory[8]; /* NativeAdrenoMemory.cpp: [1] counts the spans of system BOs */
|
|
|
|
/* The two sandbox gates IOServiceOpen applies after newUserClient (XNU IOUserClient.cpp is_io_service_open_extended): the MACF open check on
|
|
* the client object and the sandbox's filter resolver. They are run here for the caller too, only to record which one refuses it. */
|
|
extern "C" int mac_iokit_check_open(void *cred,void *client,unsigned type);
|
|
extern "C" void *kauth_cred_get(void);
|
|
extern const struct io_filter_callbacks *gIOUCFilterCallbacks;
|
|
|
|
extern "C" volatile uint32_t native_adreno_trace_stage;
|
|
extern "C" {
|
|
/* One entry per IOServiceOpen of the accelerator (not markers): who, which type, what the two sandbox gates said. */
|
|
struct NativeAdrenoOpenRecord {char name[20]; uint32_t type, macf, res, attach, foreign;};
|
|
struct NativeAdrenoOpenRecord native_adreno_open_log[8];
|
|
volatile uint32_t native_adreno_open_log_count;
|
|
/* One row per process name that opened the plugin type (first eight names): opens granted, refused (sandbox, or no address space),
|
|
* closes, the board log row of its live connection (4 the shared row, 0xff none), the highest GPU address its mappings reached (MiB) and its last refusal. */
|
|
struct NativeAdrenoClientRecord {char name[20]; uint32_t opens, refused, closes, slot, peak_mib, last_refusal;};
|
|
struct NativeAdrenoClientRecord native_adreno_client_log[8];
|
|
/* The last text line (the plugin's failure reason) each process other than the render server streamed: process name and a 64-byte line. */
|
|
struct NativeAdrenoClientText {char name[20]; char text[64];};
|
|
struct NativeAdrenoClientText native_adreno_client_text[8];
|
|
/* Why the latest mount of a connection failed: [0] 1 no address space, 2 the core's open, 3 the window scope; [1] its IOReturn; [2] the queue base; [3] failures so far. */
|
|
volatile uint32_t native_adreno_mount_fail[4];
|
|
/* Processes that read MetalPluginName (the Metal discovery), first twelve names, with counts. */
|
|
struct NativeAdrenoAskerRecord {char name[20]; uint32_t reads;};
|
|
struct NativeAdrenoAskerRecord native_adreno_metal_askers[12];
|
|
}
|
|
/* Every user connection's GPU addresses: its own address space from GpuClientVaStart (msm ADRENO_VM_START) to the aperture's end, above
|
|
* the kernel buffers' fixed addresses and the GMEM window that each address space also maps, as msm's per-process VM
|
|
* (adreno_private_vm_size: BIT(ias) - ADRENO_VM_START). A kernel connection (the board's self-test) maps where it likes. */
|
|
static uint64_t clientWindowBytes(uint32_t input_bits)
|
|
{return input_bits>32 && input_bits<=48?(UINT64_C(1)<<input_bits)-GpuClientVaStart:0;}
|
|
/* The board's qcom_scm_set_gpu_smmu_aperture result ([0] availability, [1] the call's x0: 0 is success); [2] the CP switch check's
|
|
* status, [3] the table mode in use. */
|
|
extern "C" {volatile uint32_t native_adreno_cp_aperture[4];
|
|
/* The CP switch check: the bank's image (NativeAdrenoGpuContext::observeContext) before the switch [0..7] and after it [8..15], and the
|
|
* kit status of each [16], [17]; a fault the switch left latched: FAR lo, hi, FSYNR0 [18..20] and the status after clearing it [21]. */
|
|
volatile uint32_t native_adreno_cp_image[22];
|
|
/* The newest 8 GPU restarts: [0] sequence, [1..2] time (us), [3] the fence it hung on, [4] status, [5] jobs replayed, [6] stranded << 0 | guilty << 1 | SMMU fault << 2,
|
|
* [7] why the second ring was given up (0 not). */
|
|
volatile uint32_t native_adreno_recover_log[8][12];volatile uint32_t native_adreno_recover_log_count;
|
|
/* checkCpSwitch's step results: createVm, probeBuffer, submit, waitAll, readTtbr0 (each 0 when it passed, ffffffff when not reached). */
|
|
volatile uint32_t native_adreno_cp_check[6]={0xffffffffu,0xffffffffu,0xffffffffu,0xffffffffu,0xffffffffu,0xffffffffu};}
|
|
/* Table switching (NativeAdrenoSubmit): 2 the CP switches in the ring (the default once the board opened the CP's SMMU aperture), 1 the
|
|
* CPU writes TTBR0 with the GPU idle. gpu-ttbr=1 or 2 forces one. */
|
|
static unsigned tableMode()
|
|
{
|
|
unsigned mode=native_adreno_cp_aperture[0] && !(native_adreno_cp_aperture[0]>>31) && !native_adreno_cp_aperture[1]?2:1;
|
|
PE_parse_boot_argn("gpu-ttbr",&mode,sizeof mode);
|
|
return mode==2?2:1;
|
|
}
|
|
|
|
/* The fence waiters' wake path (runCore's WaitFence): [0] GPU interrupts the board delivered (native_adreno_gpu_interrupt), [1] those
|
|
* with the CP's fence bit, [2] completion passes run, [3] waiters a pass woke, [4] waits that slept, [5] sleeps that ended at the caller's
|
|
* deadline, [6] waiters the periodic pass woke (their interrupt never came or came before they slept), [7] ring switches the passes
|
|
* triggered or completed. */
|
|
extern "C" {volatile uint64_t native_adreno_fence_wake[8];}
|
|
namespace {
|
|
/* A thread waiting for a fence, on its own stack and in g.waiters while it waits. armed: it has not seen its fence complete and may be
|
|
* asleep on its own address; the completion pass disarms and wakes it. */
|
|
struct FenceWaiter {uint32_t fence;bool armed;FenceWaiter *next,*prev;};
|
|
void shiftName(char names[3][17]) {
|
|
memcpy(names[2], names[1], 17);
|
|
memcpy(names[1], names[0], 17);
|
|
proc_selfname(names[0], 17);
|
|
}
|
|
struct Published {
|
|
bool published=false;
|
|
NativeAdrenoVm *vm=nullptr;
|
|
NativeAdrenoSubmit *submit=nullptr;
|
|
NativeAdrenoRing *ring=nullptr;
|
|
NativeAdrenoRingClock clock{};
|
|
const NativeAdrenoTimer *timer=nullptr;
|
|
const NativeAdrenoAbiDeviceInfo *info=nullptr;
|
|
IOLock *lock=nullptr;
|
|
IOService *service=nullptr;
|
|
NativeAdrenoGpuContext *context=nullptr;
|
|
bool rows[4]{}; /* the board log's client rows (NativeAdrenoUserClient::setStatsSlot): 0 the render server, 1..3 the next processes */
|
|
uint32_t next_client=1; /* queue range n (NativeAdrenoQueueRange) per user connection; range 0 is the kernel connection's */
|
|
bool kernel_client=false;
|
|
uint32_t input_bits=0,output_bits=0;
|
|
uint32_t opened=0,closed=0;
|
|
/* Global fault and restart counts every connection reports (NativeAdrenoUserClient::status adds its own VM's faults). */
|
|
NativeAdrenoStatus status;
|
|
NativeAdrenoRestart restart=nullptr;
|
|
void *restart_cookie=nullptr;
|
|
/* The fence waiters and the completion pass that wakes them. wake_lock guards the list and every waiter's armed flag. It is a
|
|
* leaf: the pass takes it while it holds `lock`, a waiter takes it alone, and nothing takes `lock` while holding it. */
|
|
IOLock *wake_lock=nullptr;
|
|
FenceWaiter *waiters=nullptr;
|
|
thread_call_t fence_call=nullptr;
|
|
/* The board hands its GPU interrupts to native_adreno_gpu_interrupt (seen once one asked for a pass); until then, or with
|
|
* gpu-fence-wake=0, a waiter pauses on the board's clock instead. */
|
|
volatile bool interrupts=false;
|
|
bool wake_by_interrupt=true;
|
|
} g;
|
|
/* The completion pass: what msm does on every CP_CACHE_FLUSH_TS interrupt (a6xx_irq: msm_gpu_retire, then a6xx_preempt_trigger) and
|
|
* once per hang check period for an interrupt that never came (hangcheck_handler). Finished jobs are retired, which lets go of their
|
|
* buffers and looks at the switch between the two rings again: a ring that went idle leaves the CP to the other ring's queued work, and
|
|
* a switch the CP finished is completed here even when its own handler found the lock busy. Then every waiter whose fence the CP has
|
|
* written is woken. Runs on a thread (it takes the driver's lock), never in the interrupt. */
|
|
void fencePass(bool periodic)
|
|
{
|
|
if(!g.published || !g.submit || !g.wake_lock)return;
|
|
uint32_t woken=0;
|
|
IOLockLock(g.lock);
|
|
const NativeAdrenoPreemptStats before=g.submit->preemptStats();
|
|
(void)g.submit->retireCompleted();
|
|
const NativeAdrenoPreemptStats after=g.submit->preemptStats();
|
|
IOLockLock(g.wake_lock);
|
|
for(FenceWaiter *w=g.waiters;w;w=w->next)
|
|
if(w->armed && !g.submit->fencePending(w->fence)) {w->armed=false;++woken;IOLockWakeup(g.wake_lock,w,false);}
|
|
native_adreno_fence_wake[2]=native_adreno_fence_wake[2]+1;
|
|
native_adreno_fence_wake[periodic?6:3]=native_adreno_fence_wake[periodic?6:3]+woken;
|
|
native_adreno_fence_wake[7]=native_adreno_fence_wake[7]+(after.triggers-before.triggers)+(after.completed-before.completed);
|
|
IOLockUnlock(g.wake_lock);
|
|
IOLockUnlock(g.lock);
|
|
}
|
|
void fenceCall(thread_call_param_t,thread_call_param_t) {fencePass(false);}
|
|
}
|
|
/* The board's GPU interrupt handler calls this with the RBBM_INT_0_STATUS word it read and cleared, in interrupt context. A fence the
|
|
* CP wrote (CP_CACHE_FLUSH_TS, bit 20) or a finished ring switch (CP_SW, bit 8) asks for the completion pass; nothing else is done
|
|
* here, and nothing here blocks. */
|
|
extern "C" void native_adreno_gpu_interrupt(uint32_t status)
|
|
{
|
|
native_adreno_fence_wake[0]=native_adreno_fence_wake[0]+1;
|
|
if(status&(1u<<20))native_adreno_fence_wake[1]=native_adreno_fence_wake[1]+1;
|
|
if(!(status&((1u<<20)|(1u<<8))) || !g.fence_call)return;
|
|
g.interrupts=true;
|
|
(void)thread_call_enter(g.fence_call);
|
|
}
|
|
/* GPU hangs (msm hangcheck, NativeAdrenoSubmit::hangCheck) and restarts: [0] hangs, [1] restarts, [2] failed restarts, [3] the last hung
|
|
* fence, [4] the last restart's IOReturn, [5] submissions replayed so far, [6] hangs with a guilty job, [7] hangs of the kernel's own VM. */
|
|
extern "C" {volatile uint32_t native_adreno_recovery[8];}
|
|
/* SMMU context faults ended before a restart (NativeAdrenoGpuContext::endFault): [0] how many, then the last one's FSR, FAR lo, FAR hi,
|
|
* FSYNR0. A latched FSR fails every later table operation's proof of the bank, so a restart that left it would poison the tables. */
|
|
extern "C" {volatile uint32_t native_adreno_recovery_fault[5];}
|
|
/* The board's GPU restart (a6xx_recover's reset and hw_init), installed before or after publication. */
|
|
extern "C" void native_adreno_set_restart(NativeAdrenoRestart restart,void *cookie)
|
|
{
|
|
if(g.lock)IOLockLock(g.lock);
|
|
g.restart=restart;g.restart_cookie=cookie;
|
|
if(g.lock)IOLockUnlock(g.lock);
|
|
}
|
|
/* msm's hangcheck timer: every DRM_MSM_HANGCHECK_DEFAULT_PERIOD (500 ms, msm_gpu.h) the ring is checked for progress; a hang restarts the
|
|
* GPU through the board's hook and replays the other submissions (recover_worker). Without a hook a hang is only counted. */
|
|
static void nativeAdrenoHangcheck(void *,wait_result_t)
|
|
{
|
|
for(;;) {
|
|
IOSleep(500);
|
|
IOLockLock(g.lock);
|
|
bool hung=false;
|
|
if(g.submit && !g.submit->hangCheck(&hung) && hung) {
|
|
native_adreno_recovery[0]=native_adreno_recovery[0]+1;
|
|
if(g.restart) {
|
|
uint32_t fault[4]={};uint64_t fault_table=0;
|
|
if(g.context)(void)g.context->readTtbr0(&fault_table);
|
|
if(g.context && !g.context->endFault(fault) && fault[0]) {
|
|
native_adreno_recovery_fault[0]=native_adreno_recovery_fault[0]+1;
|
|
for(unsigned i=0;i<4;++i)native_adreno_recovery_fault[1+i]=fault[i];
|
|
}
|
|
NativeAdrenoRecovery result{};
|
|
const IOReturn status=g.submit->recover(g.restart,g.restart_cookie,&g.clock,&result,fault[0]);
|
|
native_adreno_recovery[3]=result.fence;native_adreno_recovery[4]=uint32_t(status?status:result.status);
|
|
native_adreno_recovery[5]=native_adreno_recovery[5]+result.replayed;
|
|
if(result.guilty)native_adreno_recovery[6]=native_adreno_recovery[6]+1;
|
|
if(result.guilty==g.vm)native_adreno_recovery[7]=native_adreno_recovery[7]+1;
|
|
if(status)native_adreno_recovery[2]=native_adreno_recovery[2]+1;
|
|
else native_adreno_recovery[1]=native_adreno_recovery[1]+1;
|
|
/* Only a GPU that did not come back is everyone's loss; after a restart every client goes on, as Metal's do. */
|
|
{
|
|
const uint32_t n=native_adreno_recover_log_count;volatile uint32_t *row=native_adreno_recover_log[n&7];
|
|
const uint64_t now=g.clock.now_us?g.clock.now_us(g.clock.cookie):0;
|
|
row[0]=n+1;row[1]=uint32_t(now);row[2]=uint32_t(now>>32);row[3]=result.fence;row[4]=uint32_t(status?status:result.status);
|
|
row[5]=result.replayed;row[6]=(g.submit->lastRecoveryStranded()?1u:0u)|(result.guilty?2u:0u)|(result.fault_status?4u:0u);row[7]=g.submit->deadReason();
|
|
for(unsigned i=0;i<4;++i)row[8+i]=g.submit->recoveryInfo(i);
|
|
native_adreno_recover_log_count=n+1;
|
|
gpuTraceEvent(GpuTraceRecovery,0,result.fence,result.completed,result.replayed,row[6],row[7],
|
|
row[8],row[11],uint32_t(status?status:result.status),result.fault_status,
|
|
uint64_t(fault[2])<<32|fault[1],fault[3],fault_table);
|
|
}
|
|
if(status)g.status.noteFault();
|
|
if(!status)g.status.noteSuspend();
|
|
}
|
|
}
|
|
IOLockUnlock(g.lock);
|
|
/* msm's hangcheck_handler retires here too ("workaround for missing irq"): a fence whose interrupt was lost still wakes its
|
|
* waiters and moves the rings within one period. */
|
|
fencePass(true);
|
|
}
|
|
}
|
|
|
|
class NativeAdrenoGpuPublished : public IOService {
|
|
OSDeclareDefaultStructors(NativeAdrenoGpuPublished)
|
|
public:
|
|
IOReturn newUserClient(task_t owningTask,void *securityID,UInt32 type,
|
|
OSDictionary *properties,IOUserClient **handler) APPLE_KEXT_OVERRIDE;
|
|
};
|
|
OSDefineMetaClassAndStructors(NativeAdrenoGpuPublished,IOService)
|
|
|
|
/* A client's view of a pool BO (NativeAdrenoPool.h): a descriptor of the grant's physical runs whose mappings are write-combined
|
|
* unless the client asks IOKit for another mode. Pool pages have no VM object that keeps them alive for a mapping, so the view
|
|
* does: it holds a reference to the grant from its making until its own free, and IOKit frees a descriptor only after every
|
|
* IOMemoryMap made from it, in whatever task, is gone. A BO freed while a client still maps it therefore leaves its pages out of
|
|
* the pool, as a view of a descriptor BO keeps that BO's VM object.
|
|
* A mapping IOKit does not know of is the other way to keep the pages mapped: the client can copy its mapping in the VM
|
|
* (mach_vm_remap, a memory entry), which holds the VM object of this descriptor's device pager, not the descriptor. Where the kernel
|
|
* reports the pager's close (iokit_physical_descriptor_close_notify), the view's reference goes only then, when no map entry, memory
|
|
* entry or copy refers to that VM object any more, and the pages return to the pool. Without that report the pool does not reuse
|
|
* the pages of a grant that had a view unless the board set gpu-pool=2 (NativeAdrenoPool.h). */
|
|
extern "C" kern_return_t iokit_physical_descriptor_close_notify(IOMemoryDescriptor *,void (*)(void *),void *) __attribute__((weak_import));
|
|
class NativeAdrenoPoolView : public IOGeneralMemoryDescriptor {
|
|
OSDeclareDefaultStructors(NativeAdrenoPoolView)
|
|
NativeAdrenoPoolGrant *grant_=nullptr;
|
|
bool tracked_=false; /* the kernel drops the grant's reference at the pager's close (NativeAdrenoPoolViewClosed), not free() */
|
|
public:
|
|
static NativeAdrenoPoolView *make(NativeAdrenoPoolGrant *grant);
|
|
void free() APPLE_KEXT_OVERRIDE;
|
|
};
|
|
OSDefineMetaClassAndStructors(NativeAdrenoPoolView,IOGeneralMemoryDescriptor)
|
|
NativeAdrenoPoolView *NativeAdrenoPoolView::make(NativeAdrenoPoolGrant *grant)
|
|
{
|
|
if(!grant || !grant->count)return nullptr;
|
|
IOAddressRange *ranges=IONewData(IOAddressRange,grant->count);
|
|
if(!ranges)return nullptr;
|
|
for(uint32_t i=0;i<grant->count;++i)ranges[i]={grant->runs[i].physical,grant->runs[i].bytes};
|
|
NativeAdrenoPoolView *view=new NativeAdrenoPoolView;
|
|
/* The descriptor copies the ranges. Its own cache mode is what a mapping that names none gets. */
|
|
if(view && !view->initWithOptions(ranges,grant->count,0,nullptr,static_cast<IOOptionBits>(kIODirectionInOut)|
|
|
static_cast<IOOptionBits>(kIOMemoryTypePhysical64)|(static_cast<IOOptionBits>(kIOWriteCombineCache)<<kIOMemoryBufferCacheShift),nullptr)) {
|
|
view->release();view=nullptr;
|
|
}
|
|
IODeleteData(ranges,IOAddressRange,grant->count);
|
|
if(view) {
|
|
NativeAdrenoPoolRetain(grant);
|
|
/* Before anything can map the view: from here on the reference belongs to the kernel's report, which comes exactly once. */
|
|
const bool hook=&iokit_physical_descriptor_close_notify!=nullptr;
|
|
native_adreno_pool_views[4]=hook?1:0;
|
|
view->tracked_=hook && iokit_physical_descriptor_close_notify(view,NativeAdrenoPoolViewClosed,grant)==KERN_SUCCESS;
|
|
NativeAdrenoPoolNoteView(grant,view->tracked_); /* a client can map the pages from here on */
|
|
view->grant_=grant;
|
|
}
|
|
return view;
|
|
}
|
|
void NativeAdrenoPoolView::free()
|
|
{
|
|
/* Every IOMemoryMap of this descriptor is gone: each held a reference to it. */
|
|
NativeAdrenoPoolGrant *const grant=grant_;
|
|
const bool tracked=tracked_;
|
|
grant_=nullptr;
|
|
/* A tracked view's reference goes with the kernel's report, from this free (never mapped) or from the VM's last unmap later. */
|
|
IOGeneralMemoryDescriptor::free();
|
|
if(!tracked)NativeAdrenoPoolRelease(grant);
|
|
}
|
|
|
|
class AGXDeviceUserClient : public IOUserClient {
|
|
OSDeclareDefaultStructors(AGXDeviceUserClient)
|
|
bool live_=false;
|
|
bool handoffOnly_=false;
|
|
bool kernel_=false;
|
|
NativeAdrenoUserClient *core_=nullptr; /* own core; deleted in free(), so a call that raced the close sees live_ false, never freed memory */
|
|
NativeAdrenoProcessVm *space_=nullptr; /* a user connection's own address space */
|
|
unsigned row_=4;
|
|
int32_t pid_=0; /* the process that opened it, for the views' table */
|
|
char name_[20]{};
|
|
/* One user-view descriptor per BO token, reused by every clientMemoryForType call. IOConnectUnmapMemory64 asks clientMemoryForType for
|
|
* the descriptor again and unmaps only a mapping that belongs to THAT descriptor (kIOMapReference); a fresh descriptor per call made the
|
|
* unmap a silent no-op, the user PTEs outlived BoCpuUnmap, and BoFree handed pages with a live user mapping to the VM:
|
|
* pmap_assert_free panicked the freeing core and the panic screen parked it (the "BoFree hang", dead touch). */
|
|
struct View {uint32_t token;IOMemoryDescriptor *descriptor;uint64_t bytes;};
|
|
/* One view per live BO token. The cache holds every BO the client has (it grows with the tokens): a BO whose view is not cached makes
|
|
* IOConnectUnmapMemory64 (which asks clientMemoryForType again) a no-op, the user mapping outlives BoFree and the freed pages panic
|
|
* pmap_assert_free. A cache of 64 held until the render server's hundreds of mapped BOs (int-087). */
|
|
/* A token is its BO's entry index plus one, so it names its own slot (a walk over every slot ran on each CPU map and unmap). */
|
|
View *viewSlot(uint32_t token) {return views_ && token && token<=view_capacity_?&views_[token-1]:nullptr;}
|
|
View *views_=nullptr; /* on the heap, grown to the highest token (growViews) */
|
|
size_t view_capacity_=0;
|
|
bool growViews(uint32_t token);
|
|
void dropViews(uint32_t token);
|
|
static IOReturn action(OSObject *target,void *reference,IOExternalMethodArguments *arguments);
|
|
IOReturn run(uint32_t method,const void *input,size_t input_size,void *output,size_t output_size);
|
|
IOReturn runTimed(uint32_t method,const void *input,size_t input_size,void *output,size_t output_size,uint64_t *lockWait);
|
|
public:
|
|
void setHandoffOnly() {handoffOnly_=true;}
|
|
bool mount(bool kernel);
|
|
friend IOReturn runCore(AGXDeviceUserClient *,uint32_t,const void *,size_t,void *,size_t,uint64_t *);
|
|
friend void nativeAdrenoMailboxPoller(void *,wait_result_t);
|
|
IOReturn clientClose() APPLE_KEXT_OVERRIDE;
|
|
void free() APPLE_KEXT_OVERRIDE;
|
|
IOReturn clientMemoryForType(UInt32 type,IOOptionBits *options,IOMemoryDescriptor **memory) APPLE_KEXT_OVERRIDE;
|
|
IOReturn externalMethod(uint32_t selector,IOExternalMethodArguments *arguments,
|
|
IOExternalMethodDispatch *dispatch,OSObject *target,void *reference) APPLE_KEXT_OVERRIDE;
|
|
};
|
|
OSDefineMetaClassAndStructors(AGXDeviceUserClient,IOUserClient)
|
|
|
|
void nativeAdrenoMailboxPoller(void *,wait_result_t)
|
|
{
|
|
thread_set_thread_name(current_thread(),"gpu-mailbox");
|
|
uint32_t last=0;
|
|
for(;;) {
|
|
/* A request is a store into the shared page, so nothing can signal it; the page being mapped can. */
|
|
if(!gMailboxMapped) {
|
|
IOLockLock(g.lock);
|
|
while(!gMailboxMapped)IOLockSleep(g.lock,&gMailboxMapped,THREAD_UNINT);
|
|
IOLockUnlock(g.lock);
|
|
}
|
|
IOSleep(1);
|
|
if(!gMailboxOwner || !gMailboxWords)continue;
|
|
const uint32_t request=gMailboxWords[1];
|
|
if(request==last || request==gMailboxWords[2])continue;
|
|
last=request;
|
|
const uint32_t method=gMailboxWords[3],in_size=gMailboxWords[5],out_size=gMailboxWords[6];
|
|
IOReturn status=kIOReturnBadArgument;
|
|
if(in_size<=0x700 && out_size<=0x700) {
|
|
uint8_t input[0x700],output[0x700];
|
|
memcpy(input,const_cast<uint32_t *>(gMailboxWords)+0x10,in_size);
|
|
IOLockLock(g.lock);
|
|
AGXDeviceUserClient *owner=gMailboxOwner;
|
|
if(owner)owner->retain();
|
|
IOLockUnlock(g.lock);
|
|
status=owner?runCore(owner,method,input,in_size,output,out_size,nullptr):kIOReturnNotReady;
|
|
if(owner)owner->release();
|
|
if(!status)memcpy(const_cast<uint32_t *>(gMailboxWords)+0x200,output,out_size);
|
|
}
|
|
native_adreno_metal_diag.mb_calls=native_adreno_metal_diag.mb_calls+1;
|
|
gMailboxWords[4]=uint32_t(status);
|
|
__asm__ volatile("dmb sy" ::: "memory");
|
|
gMailboxWords[2]=request;
|
|
__asm__ volatile("dmb sy" ::: "memory");
|
|
}
|
|
}
|
|
|
|
/* The client log row of name (created for the first eight names); null when the table is full. Under g.lock. */
|
|
static NativeAdrenoClientRecord *clientRecord(const char *name)
|
|
{
|
|
for(auto &record:native_adreno_client_log) {
|
|
if(!record.name[0]) {strlcpy(record.name,name,sizeof record.name);record.slot=0xff;return &record;}
|
|
if(!strncmp(record.name,name,sizeof record.name))return &record;
|
|
}
|
|
return nullptr;
|
|
}
|
|
bool AGXDeviceUserClient::mount(bool kernel)
|
|
{
|
|
proc_selfname(name_,sizeof name_);
|
|
pid_=proc_selfpid();
|
|
NativeAdrenoUserClient *core=new NativeAdrenoUserClient;
|
|
if(!core)return false;
|
|
IOLockLock(g.lock);
|
|
bool ok=false;
|
|
const bool server=!kernel && isRenderServer(name_,sizeof name_);
|
|
NativeAdrenoProcessVm *space=nullptr;
|
|
IOReturn opened=kIOReturnSuccess;
|
|
if(kernel)opened=g.kernel_client?kIOReturnBusy:kIOReturnSuccess;
|
|
else opened=g.context?g.context->createVm(&space):kIOReturnNotReady;
|
|
NativeAdrenoVm *vm=kernel?g.vm:space?space->vm():nullptr;
|
|
if(!opened && !vm)opened=kIOReturnNotReady;
|
|
if(!opened)opened=core->open(NativeAdrenoAbiVersion,vm,g.submit,g.ring,g.input_bits,g.output_bits);
|
|
if(opened!=kIOReturnSuccess) {native_adreno_mount_fail[0]=space?2:1;native_adreno_mount_fail[1]=uint32_t(opened);native_adreno_mount_fail[2]=0;native_adreno_mount_fail[3]=native_adreno_mount_fail[3]+1;}
|
|
if(opened==kIOReturnSuccess) {
|
|
if(g.timer)core->attachTimer(g.timer);
|
|
if(g.info)core->attachDeviceInfo(g.info);
|
|
core->attachStatus(&g.status);
|
|
/* Queue ranges: the kernel connection keeps range 0; each user connection a range of its own, never reused while the system runs. */
|
|
uint64_t base=0;
|
|
IOReturn scoped=kIOReturnSuccess;
|
|
if(!kernel) {
|
|
base=(uint64_t)g.next_client*NativeAdrenoQueueRange;
|
|
const uint64_t window=clientWindowBytes(g.input_bits);
|
|
scoped=!g.next_client || !window?kIOReturnNoResources:core->setScope(base,GpuClientVaStart,window);
|
|
} else scoped=core->setScope(0,0,0);
|
|
if(scoped==kIOReturnSuccess) {
|
|
if(!kernel)++g.next_client;
|
|
unsigned row=4;
|
|
if(server && !g.rows[0])row=0;
|
|
for(unsigned i=1;!server && row==4 && i<4;++i)if(!g.rows[i])row=i;
|
|
if(row<4)g.rows[row]=true;
|
|
core->setStatsSlot(row);
|
|
core_=core;core=nullptr;space_=space;space=nullptr;row_=row;kernel_=kernel;live_=true;ok=true;
|
|
if(kernel)g.kernel_client=true;
|
|
++g.opened;
|
|
if(NativeAdrenoClientRecord *record=clientRecord(name_))record->slot=row;
|
|
} else {
|
|
native_adreno_mount_fail[0]=3;native_adreno_mount_fail[1]=uint32_t(scoped);native_adreno_mount_fail[2]=uint32_t(base>>32);native_adreno_mount_fail[3]=native_adreno_mount_fail[3]+1;
|
|
core->releaseAll(&g.clock,0);
|
|
}
|
|
}
|
|
/* Not used by any core (a failed open or scope): nothing of it reached the GPU. */
|
|
if(space && g.context->destroyVm(space)) {}
|
|
/* The mailbox page and its poller serve the render server's connection only. */
|
|
if(ok && server && row_==0) {
|
|
if(!gMailbox) {
|
|
gMailbox=IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task,kIODirectionInOut,4096,4096);
|
|
if(gMailbox) {gMailboxWords=static_cast<volatile uint32_t *>(gMailbox->getBytesNoCopy());memset(const_cast<uint32_t *>(gMailboxWords),0,4096);}
|
|
}
|
|
if(gMailboxWords) {
|
|
memset(const_cast<uint32_t *>(gMailboxWords),0,4096);
|
|
gMailboxWords[0]=NativeAdrenoMailboxMagic;
|
|
if(!gPollerStarted) {
|
|
thread_t thread=nullptr;
|
|
if(kernel_thread_start(nativeAdrenoMailboxPoller,nullptr,&thread)==KERN_SUCCESS) {thread_deallocate(thread);gPollerStarted=true;}
|
|
}
|
|
}
|
|
if(gMailboxOwner)gMailboxOwner->release();
|
|
retain();gMailboxOwner=this;gMailboxMapped=false;
|
|
}
|
|
IOLockUnlock(g.lock);
|
|
if(core)delete core;
|
|
return ok;
|
|
}
|
|
/* Removes the cached view(s) of token (0 = all) and tears down every user mapping of them, so the BO's pages are never freed while a user
|
|
* PTE still points at them, whether or not the client unmapped. */
|
|
void AGXDeviceUserClient::dropViews(uint32_t token)
|
|
{
|
|
for(;;) {
|
|
IOMemoryDescriptor *view=nullptr;
|
|
IOLockLock(g.lock);
|
|
uint64_t bytes=0;
|
|
if(token) {View *v=viewSlot(token);if(v && v->descriptor && v->token==token) {view=v->descriptor;bytes=v->bytes;v->descriptor=nullptr;v->token=0;v->bytes=0;}}
|
|
else if(views_)for(size_t i=0;i<view_capacity_;++i) {View &v=views_[i];if(v.descriptor) {view=v.descriptor;bytes=v.bytes;v.descriptor=nullptr;v.token=0;v.bytes=0;break;}}
|
|
if(view)NativeAdrenoNoteHolder(native_adreno_view_owners,pid_,-int64_t(bytes));
|
|
IOLockUnlock(g.lock);
|
|
if(!view)return;
|
|
/* The returned reference is the last one outside the descriptor's own list: dropping it frees the map, which unmaps the user range. */
|
|
/* A pool view's mappings are dropped here for certain: a mapping left behind would keep the view, and with it the grant's
|
|
* pages, out of the pool for as long as the connection lives. */
|
|
const bool pool=OSDynamicCast(NativeAdrenoPoolView,view)!=nullptr;
|
|
while(IOMemoryMap *map=removeMappingForDescriptor(view)) {
|
|
native_adreno_metal_diag.forced_unmaps=native_adreno_metal_diag.forced_unmaps+1;
|
|
if(pool)map->release();
|
|
}
|
|
view->release();
|
|
}
|
|
}
|
|
/* Room for token's view slot (tokens are entry indices plus one, so they stay below the client's object count). */
|
|
bool AGXDeviceUserClient::growViews(uint32_t token)
|
|
{
|
|
if(!token)return true;
|
|
if(token<=view_capacity_)return true;
|
|
size_t size=view_capacity_?view_capacity_:64;
|
|
while(size<token)size*=2;
|
|
View *made=IONewZero(View,size);
|
|
if(!made)return false;
|
|
for(size_t i=0;i<view_capacity_;++i)made[i]=views_[i];
|
|
if(views_)IODelete(views_,View,view_capacity_);
|
|
views_=made;view_capacity_=size;
|
|
return true;
|
|
}
|
|
IOReturn AGXDeviceUserClient::clientClose()
|
|
{
|
|
dropViews(0);
|
|
IOLockLock(g.lock);
|
|
IOReturn status=kIOReturnSuccess;
|
|
AGXDeviceUserClient *mailbox=nullptr;
|
|
if(gMailboxOwner==this) {mailbox=gMailboxOwner;gMailboxOwner=nullptr;gMailboxMapped=false;}
|
|
if(live_) {
|
|
const uint64_t peak=core_->windowPeak();
|
|
status=core_->releaseAll(&g.clock,2000000);
|
|
/* The address space goes once the GPU no longer runs it: idle, and the kernel table back in the bank if it was installed. */
|
|
if(!status && space_)status=g.submit->forgetVm(space_->vm(),&g.clock,2000000);
|
|
if(!status && space_) {status=g.context->destroyVm(space_);if(!status)space_=nullptr;}
|
|
/* A failed teardown leaves the objects (and the address space) live: nothing is forced or reused. */
|
|
if(!status) {
|
|
live_=false;
|
|
if(kernel_)g.kernel_client=false;
|
|
if(row_<4)g.rows[row_]=false;
|
|
row_=4;
|
|
++g.closed;native_adreno_metal_diag.closes=native_adreno_metal_diag.closes+1;
|
|
if(NativeAdrenoClientRecord *record=clientRecord(name_)) {
|
|
record->closes=record->closes+1;record->slot=0xff;
|
|
if(peak>>20>record->peak_mib)record->peak_mib=uint32_t(peak>>20);
|
|
}
|
|
}
|
|
}
|
|
IOLockUnlock(g.lock);
|
|
if(mailbox)mailbox->release();
|
|
if(!status)terminate();
|
|
return status;
|
|
}
|
|
void AGXDeviceUserClient::free()
|
|
{
|
|
dropViews(0);
|
|
if(views_) {IODelete(views_,View,view_capacity_);views_=nullptr;view_capacity_=0;}
|
|
/* Still live only when a teardown failed: its objects stay referenced by the VM and the ring, so the core is kept, not freed. */
|
|
if(core_ && !live_) {delete core_;core_=nullptr;}
|
|
IOUserClient::free();
|
|
}
|
|
IOReturn AGXDeviceUserClient::clientMemoryForType(UInt32 type,IOOptionBits *options,IOMemoryDescriptor **memory)
|
|
{
|
|
if(memory)*memory=nullptr;
|
|
if(!memory || !options)return kIOReturnBadArgument;
|
|
IOReturn status=kIOReturnNotFound;
|
|
if(type==NativeAdrenoScanoutType) {
|
|
if(!live_ && !handoffOnly_)return kIOReturnNotReady;
|
|
IOMemoryDescriptor *scanout=native_gop_handoff_memory();
|
|
if(!scanout)return kIOReturnNotReady;
|
|
*memory=scanout; /* already retained for the caller */
|
|
*options=kIOMapAnywhere|kIOMapInhibitCache; /* firmware framebuffer: device memory, never cached by the CPU */
|
|
return kIOReturnSuccess;
|
|
}
|
|
if(type==NativeAdrenoMailboxType) {
|
|
if(live_ && gMailbox && gMailboxOwner==this) {
|
|
gMailbox->retain();
|
|
*memory=gMailbox;
|
|
*options=kIOMapAnywhere; /* ordinary cached memory: both sides are CPUs, so it is coherent */
|
|
native_adreno_metal_diag.mb_maps=native_adreno_metal_diag.mb_maps+1;
|
|
IOLockLock(g.lock);
|
|
if(gMailboxOwner==this) {gMailboxMapped=true;IOLockWakeup(g.lock,&gMailboxMapped,false);}
|
|
IOLockUnlock(g.lock);
|
|
return kIOReturnSuccess;
|
|
}
|
|
return kIOReturnNotReady;
|
|
}
|
|
native_adreno_view_diag[0]=native_adreno_view_diag[0]+1;
|
|
/* [5]: threads inside this function right now (a thread stuck here, on the gate or in a mapping, shows as a non-zero count that stays). */
|
|
struct Inside {Inside() {__atomic_fetch_add(&native_adreno_view_diag[5],1,__ATOMIC_RELAXED);} ~Inside() {__atomic_fetch_sub(&native_adreno_view_diag[5],1,__ATOMIC_RELAXED);}} inside;
|
|
IOLockLock(g.lock);
|
|
if(!growViews(type)) {IOLockUnlock(g.lock);return kIOReturnNoMemory;}
|
|
View *v=viewSlot(type);
|
|
if(v && v->descriptor && v->token==type) {
|
|
v->descriptor->retain();*memory=v->descriptor;
|
|
*options=kIOMapAnywhere|(OSDynamicCast(NativeAdrenoPoolView,v->descriptor)?kIOMapWriteCombineCache:kIOMapInhibitCache);
|
|
native_adreno_view_diag[2]=native_adreno_view_diag[2]+1;
|
|
IOLockUnlock(g.lock);
|
|
return kIOReturnSuccess;
|
|
}
|
|
NativeAdrenoMemory *backing=live_?core_->memoryForToken(type):nullptr;
|
|
if(backing && backing->system() && backing->descriptor() && v && !v->descriptor) {
|
|
/* System memory: the view maps the BO's own descriptor, its VM object, as Metal's resources are mapped into their process. The
|
|
* pages keep the object's attribute (no device-pager view sets them back, int-160) and live while the view does. */
|
|
IOMemoryDescriptor *descriptor=backing->descriptor();
|
|
descriptor->retain();v->token=type;v->descriptor=descriptor;v->bytes=backing->bytes();
|
|
NativeAdrenoNoteHolder(native_adreno_view_owners,pid_,int64_t(v->bytes));
|
|
descriptor->retain();*memory=descriptor;
|
|
*options=kIOMapAnywhere;
|
|
native_adreno_view_diag[1]=native_adreno_view_diag[1]+1;
|
|
status=kIOReturnSuccess;
|
|
}
|
|
else if(backing && backing->poolGrant()) {
|
|
/* Pool pages: the view describes the grant's physical runs, write-combined as the client's view of any system BO is (the
|
|
* GPU maps them non-cacheable), and holds the grant until IOKit frees it (NativeAdrenoPoolView). */
|
|
NativeAdrenoPoolView *descriptor=v && !v->descriptor?NativeAdrenoPoolView::make(backing->poolGrant()):nullptr;
|
|
if(!descriptor)status=v && !v->descriptor?kIOReturnNoMemory:kIOReturnNoResources;
|
|
else {
|
|
v->token=type;v->descriptor=descriptor;v->bytes=backing->bytes(); /* the cache keeps make()'s reference */
|
|
NativeAdrenoNoteHolder(native_adreno_view_owners,pid_,int64_t(v->bytes));
|
|
descriptor->retain();*memory=descriptor;
|
|
*options=kIOMapAnywhere|kIOMapWriteCombineCache;
|
|
native_adreno_view_diag[1]=native_adreno_view_diag[1]+1;
|
|
status=kIOReturnSuccess;
|
|
}
|
|
}
|
|
else if(backing) {
|
|
/* The BO's pages are cached in the kernel and the GPU's PTEs are non-cacheable, so the client needs an uncached
|
|
* view. A mapping of the buffer descriptor with a different cache mode is not reliable, so describe the same raw
|
|
* physical spans (the ones the SMMU maps) and ask for kIOMapInhibitCache, exactly how the board code maps the
|
|
* ring and the GMU windows. The descriptor does not retain the pages: the client must unmap before BoCpuUnmap/BoFree. */
|
|
/* A BO's pages can come scattered, one span per page (a 64 MiB BO is 16384 of them): the descriptor takes them all. A fixed 64-entry
|
|
* array made every scattered BO above 256 KiB fail to map (VK_ERROR_MEMORY_MAP_FAILED, int-084), only when memory happened to be
|
|
* fragmented. The descriptor copies the ranges. */
|
|
const size_t count=backing->spanCount();
|
|
if(!count)status=kIOReturnNoResources;
|
|
else {
|
|
IOAddressRange *ranges=IONewData(IOAddressRange,count);
|
|
if(!ranges)status=kIOReturnNoMemory;
|
|
else {
|
|
for(size_t i=0;i<count;++i)ranges[i]={backing->spans()[i].physical,backing->spans()[i].bytes};
|
|
IOMemoryDescriptor *descriptor=IOMemoryDescriptor::withAddressRanges(ranges,(UInt32)count,
|
|
static_cast<IOOptionBits>(kIODirectionInOut)|static_cast<IOOptionBits>(kIOMemoryTypePhysical64),nullptr);
|
|
IODeleteData(ranges,IOAddressRange,count);
|
|
if(!descriptor)status=kIOReturnNoMemory;
|
|
else {
|
|
bool cached=false;
|
|
if(v && !v->descriptor) {
|
|
v->token=type;v->descriptor=descriptor;v->bytes=backing->bytes();descriptor->retain();cached=true;
|
|
NativeAdrenoNoteHolder(native_adreno_view_owners,pid_,int64_t(v->bytes));
|
|
}
|
|
if(!cached) {
|
|
/* Never hand out a view that cannot be torn down again. */
|
|
descriptor->release();
|
|
status=kIOReturnNoResources;
|
|
} else {
|
|
*memory=descriptor;
|
|
*options=kIOMapAnywhere|kIOMapInhibitCache;
|
|
native_adreno_view_diag[1]=native_adreno_view_diag[1]+1;
|
|
status=kIOReturnSuccess;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
IOLockUnlock(g.lock);
|
|
if(status){native_adreno_view_diag[3]=native_adreno_view_diag[3]+1;native_adreno_view_diag[4]=(uint32_t)status;}
|
|
return status;
|
|
}
|
|
/* Fence waits as the callers see them (int-147; method 6's own statistics count every poll): [0] waits, [1] us waited in total, [2] pauses
|
|
* taken outside the connection lock, [3] the longest wait in us, [4] polls answered without the lock. */
|
|
extern "C" {volatile uint64_t native_adreno_wait_outer[5];}
|
|
static void lockTimed(uint64_t *clock)
|
|
{
|
|
if(!clock) {IOLockLock(g.lock);return;}
|
|
if(!IOLockTryLock(g.lock)) {
|
|
const uint64_t began=mach_absolute_time();
|
|
IOLockLock(g.lock);
|
|
clock[LockWaited]+=mach_absolute_time()-began;
|
|
}
|
|
clock[LockSince]=mach_absolute_time();
|
|
}
|
|
static void unlockTimed(uint64_t *clock)
|
|
{
|
|
const uint64_t now=mach_absolute_time();
|
|
IOLockUnlock(g.lock);
|
|
if(clock)clock[LockHeld]+=now-clock[LockSince];
|
|
}
|
|
static uint32_t microseconds(uint64_t from,uint64_t to)
|
|
{
|
|
uint64_t ns=0;
|
|
if(to>from)absolutetime_to_nanoseconds(to-from,&ns);
|
|
return uint32_t(ns/1000>UINT32_MAX?UINT32_MAX:ns/1000);
|
|
}
|
|
/* Whether the CP has not reached fence yet on the ring that issued it (NativeAdrenoSubmit::fencePending), read when the lock is free.
|
|
* False when the answer needs the locked path: the lock is busy, the fence is not one a job holds, or the submit object is poisoned.
|
|
* The display's prefix (native_adreno_fences) is not the measure here: it stays below a finished fence of the high ring for as long
|
|
* as an older fence of the low ring is pending, and a waiter that trusted it slept until its deadline with its own fence written. */
|
|
static bool fencePending(uint64_t fence)
|
|
{
|
|
if(!fence || fence>UINT32_MAX || !g.submit || !IOLockTryLock(g.lock))return false;
|
|
const bool pending=g.submit->fencePending(uint32_t(fence));
|
|
IOLockUnlock(g.lock);
|
|
return pending;
|
|
}
|
|
static IOReturn runCore(AGXDeviceUserClient *connection,uint32_t method,const void *input,size_t input_size,void *output,size_t output_size,uint64_t *lockWait)
|
|
{
|
|
if(method==NativeAdrenoMethodSetMetadata && input && input_size==sizeof(NativeAdrenoAbiMetadata)) {
|
|
const NativeAdrenoAbiMetadata *meta=static_cast<const NativeAdrenoAbiMetadata *>(input);
|
|
/* The raw test's line and the Turnip phase's line ("T ..." result or "W ..." watchdog sample) are kept apart, so neither hides the other. */
|
|
const bool turnip=meta->bytes>=2 && (meta->text[0]=='T' || meta->text[0]=='W') && meta->text[1]==' ';
|
|
char *line=turnip?native_adreno_metal_diag.report2:native_adreno_metal_diag.report;
|
|
uint32_t n=meta->bytes<sizeof(native_adreno_metal_diag.report)-1?meta->bytes:uint32_t(sizeof(native_adreno_metal_diag.report)-1);
|
|
memset(line,0,sizeof(native_adreno_metal_diag.report));
|
|
memcpy(line,meta->text,n);
|
|
native_adreno_metal_diag.reports=native_adreno_metal_diag.reports+1;
|
|
}
|
|
/* A call that is still running when the boot page is photographed shows as inflight = its method number + 1 (0 = none), so a hang in the
|
|
* kernel is attributable to one ABI method. */
|
|
native_adreno_metal_diag.inflight=method+1;
|
|
native_adreno_metal_diag.inflight_since_calls=native_adreno_metal_diag.calls;
|
|
IOReturn status;
|
|
if(method==NativeAdrenoMethodWaitFence && connection->live_ && input && input_size==sizeof(NativeAdrenoAbiWaitFence) && g.clock.now_us) {
|
|
/* A fence wait sleeps outside the lock: under it every other thread's submit, map or allocation would wait for the GPU to
|
|
* finish unrelated work. While the CP has not reached the fence the check reads the ring's fence word when the lock is free;
|
|
* the core's own zero-timeout wait (retirement, statistics, the stall handler) runs under the lock once the fence has passed
|
|
* and when the time is up.
|
|
* With the board's interrupts (native_adreno_gpu_interrupt) the waiter sleeps on its own record until the completion pass
|
|
* finds its fence written, and the only timeout is the caller's deadline. It arms itself before it looks at the fence, so a
|
|
* completion between the look and the sleep is not lost: the pass that follows that completion disarms it, and a disarmed
|
|
* waiter does not sleep. Without them the pause is the board's (until the next GPU interrupt, at most its pause time). */
|
|
NativeAdrenoAbiWaitFence poll{};
|
|
memcpy(&poll,input,sizeof(poll));
|
|
const uint64_t timeout_us=poll.timeout_ns/1000,began=g.clock.now_us(g.clock.cookie);
|
|
poll.timeout_ns=0;
|
|
uint64_t waited=0;
|
|
status=kIOReturnTimeout;
|
|
FenceWaiter self{uint32_t(poll.fence),false,nullptr,nullptr};
|
|
const bool wake=timeout_us && g.interrupts && g.wake_by_interrupt && g.wake_lock;
|
|
bool slept=false;
|
|
if(wake) {
|
|
IOLockLock(g.wake_lock);
|
|
self.next=g.waiters;if(g.waiters)g.waiters->prev=&self;g.waiters=&self;
|
|
IOLockUnlock(g.wake_lock);
|
|
}
|
|
for(;;) {
|
|
waited=g.clock.now_us(g.clock.cookie)-began;
|
|
bool polled=false;
|
|
if(wake) {IOLockLock(g.wake_lock);self.armed=true;IOLockUnlock(g.wake_lock);}
|
|
if(waited>=timeout_us || !fencePending(poll.fence)) {
|
|
/* The connection's own core while it is live; a close that ran between two polls ends the wait. */
|
|
lockTimed(lockWait);
|
|
status=connection->live_?connection->core_->dispatch(method,&poll,sizeof(poll),output,output_size,&g.clock):kIOReturnNotReady;
|
|
unlockTimed(lockWait);
|
|
polled=true;
|
|
waited=g.clock.now_us(g.clock.cookie)-began;
|
|
if(status!=kIOReturnTimeout || waited>=timeout_us)break;
|
|
} else native_adreno_wait_outer[4]=native_adreno_wait_outer[4]+1;
|
|
native_adreno_wait_outer[2]=native_adreno_wait_outer[2]+1;
|
|
if(wake) {
|
|
const uint64_t left=timeout_us-waited;
|
|
uint64_t deadline=0;
|
|
clock_interval_to_deadline(left>UINT32_MAX?UINT32_MAX:uint32_t(left),NSEC_PER_USEC,&deadline);
|
|
IOLockLock(g.wake_lock);
|
|
if(self.armed) {
|
|
slept=true;
|
|
if(IOLockSleepDeadline(g.wake_lock,&self,deadline,THREAD_UNINT)==THREAD_TIMED_OUT)
|
|
native_adreno_fence_wake[5]=native_adreno_fence_wake[5]+1;
|
|
}
|
|
IOLockUnlock(g.wake_lock);
|
|
continue;
|
|
}
|
|
if(!g.clock.idle || g.clock.idle(g.clock.cookie)) {
|
|
if(!polled) {
|
|
lockTimed(lockWait);
|
|
status=connection->live_?connection->core_->dispatch(method,&poll,sizeof(poll),output,output_size,&g.clock):kIOReturnNotReady;
|
|
unlockTimed(lockWait);
|
|
waited=g.clock.now_us(g.clock.cookie)-began;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
if(wake) {
|
|
IOLockLock(g.wake_lock);
|
|
if(self.prev)self.prev->next=self.next;else g.waiters=self.next;
|
|
if(self.next)self.next->prev=self.prev;
|
|
if(slept)native_adreno_fence_wake[4]=native_adreno_fence_wake[4]+1;
|
|
IOLockUnlock(g.wake_lock);
|
|
}
|
|
native_adreno_wait_outer[0]=native_adreno_wait_outer[0]+1;native_adreno_wait_outer[1]=native_adreno_wait_outer[1]+waited;
|
|
if(waited>native_adreno_wait_outer[3])native_adreno_wait_outer[3]=waited;
|
|
} else {
|
|
/* A BoAlloc's system pages are made and wired before the lock is taken (NativeAdrenoMemory::prepareSystem): i14aef633e spent 4.0 s
|
|
* in 5454 allocations under it, while every other process's submit, map and wait poll queued behind. The lock is held briefly
|
|
* first for the client's limits; pages the call does not take are given back after it. */
|
|
NativeAdrenoSystemPages pages{};
|
|
const bool alloc=method==NativeAdrenoMethodBoAlloc && input && input_size==sizeof(NativeAdrenoAbiBoAlloc);
|
|
NativeAdrenoAbiBoAlloc request{};
|
|
uint64_t began=0,limited=0,made=0,wired=0,prepared=0,locked=0,dispatched=0;
|
|
uint32_t spans=0;
|
|
bool offered=false,cleaned=false,pooled=false;
|
|
if(alloc) {
|
|
memcpy(&request,input,sizeof(request));
|
|
began=mach_absolute_time();
|
|
lockTimed(lockWait);
|
|
const uint64_t limit=connection->live_?connection->core_->boLimit():0;
|
|
const unsigned bits=connection->live_?connection->core_->outputBits():0;
|
|
unlockTimed(lockWait);
|
|
limited=mach_absolute_time();
|
|
/* Pool pages when the board registered some (zero-filled here), else the VM's. */
|
|
if(!request.contiguous && request.bytes && !(request.bytes&4095) && request.bytes<=limit)
|
|
(void)NativeAdrenoMemory::prepareSystem((size_t)request.bytes,NativeAdrenoUserClient::systemWriteCombined(),&pages,bits);
|
|
offered=pages.held();pooled=pages.grant!=nullptr;made=pages.made;wired=pages.wired;cleaned=pages.cleaned;
|
|
prepared=mach_absolute_time();
|
|
}
|
|
/* i e38a0b1eb: a BoFree held the lock up to 38 ms, unwiring and freeing a large BO's pages; they now go after the lock. */
|
|
NativeAdrenoSystemPages freed{};
|
|
const bool freeing=method==NativeAdrenoMethodBoFree;
|
|
lockTimed(lockWait);
|
|
locked=mach_absolute_time();
|
|
spans=native_adreno_bo_memory[1];
|
|
/* Offered for every BoAlloc, held or not: allocateSystem then knows it runs under the lock and leaves the pool alone. */
|
|
if(connection->live_)connection->core_->offerSystemPages(alloc?&pages:nullptr);
|
|
if(connection->live_ && freeing)connection->core_->collectFreedPages(&freed);
|
|
status=connection->live_?connection->core_->dispatch(method,input,input_size,output,output_size,&g.clock):kIOReturnNotReady;
|
|
if(connection->live_)connection->core_->offerSystemPages(nullptr);
|
|
if(connection->live_ && freeing)connection->core_->collectFreedPages(nullptr);
|
|
if(method==NativeAdrenoMethodBoMap && !status && connection->live_ && !connection->kernel_)
|
|
if(NativeAdrenoClientRecord *record=clientRecord(connection->name_))
|
|
if(connection->core_->windowPeak()>>20>record->peak_mib)record->peak_mib=uint32_t(connection->core_->windowPeak()>>20);
|
|
spans=native_adreno_bo_memory[1]-spans;
|
|
dispatched=mach_absolute_time();
|
|
unlockTimed(lockWait);
|
|
const bool taken=offered && !pages.held();
|
|
NativeAdrenoMemory::discardSystem(&pages);
|
|
if(freed.held()) {native_adreno_freed_outside=native_adreno_freed_outside+1;NativeAdrenoMemory::discardSystem(&freed);}
|
|
if(alloc && request.bytes>=(UINT64_C(256)<<10))
|
|
if(void (*hook)(const NativeAdrenoBoTrace *)=native_adreno_bo_hook) {
|
|
NativeAdrenoBoTrace trace{};
|
|
trace.entered=began;trace.kib=uint32_t(request.bytes>>10);
|
|
trace.flags=(NativeAdrenoUserClient::systemWriteCombined()?1u:0u)|(taken?2u:0u)|(cleaned?4u:0u)|(offered?8u:0u)|(pooled?16u:0u);
|
|
trace.limit_us=microseconds(began,limited);
|
|
trace.create_us=microseconds(limited,offered?made:prepared);
|
|
trace.wire_us=offered?microseconds(made,wired):0;
|
|
trace.clean_us=offered?microseconds(wired,prepared):0;
|
|
trace.lock_us=microseconds(prepared,locked);trace.dispatch_us=microseconds(locked,dispatched);
|
|
trace.discard_us=microseconds(dispatched,mach_absolute_time());trace.spans=spans;trace.status=int32_t(status);
|
|
hook(&trace);
|
|
}
|
|
}
|
|
native_adreno_metal_diag.inflight=0;
|
|
native_adreno_metal_diag.calls=native_adreno_metal_diag.calls+1;
|
|
native_adreno_metal_diag.last_selector=method;
|
|
native_adreno_metal_diag.last_status=uint32_t(status);
|
|
return status;
|
|
}
|
|
IOReturn AGXDeviceUserClient::run(uint32_t method,const void *input,size_t input_size,void *output,size_t output_size)
|
|
{
|
|
uint64_t lockClock[LockClockWords]={};
|
|
const uint64_t entered=mach_absolute_time();
|
|
const IOReturn status=runTimed(method,input,input_size,output,output_size,lockClock);
|
|
if(NativeAdrenoCallRec *ring=native_adreno_call_ring) {
|
|
const uint32_t held=microseconds(0,lockClock[LockHeld])/16;
|
|
const uint32_t n=__atomic_fetch_add(&native_adreno_call_count,1,__ATOMIC_RELAXED);
|
|
NativeAdrenoCallRec &r=ring[n&262143];
|
|
r.entered=entered;r.us=microseconds(entered,mach_absolute_time());r.lock_us=microseconds(0,lockClock[LockWaited]);
|
|
r.method=uint16_t(method);r.held16=uint16_t(held>0xffff?0xffff:held);r.pid=proc_selfpid();
|
|
r.status=uint32_t(status);
|
|
r.kib=method==NativeAdrenoMethodBoAlloc && input && input_size==sizeof(NativeAdrenoAbiBoAlloc)?
|
|
uint32_t(static_cast<const NativeAdrenoAbiBoAlloc *>(input)->bytes>>10):0;
|
|
}
|
|
return status;
|
|
}
|
|
/* The processes' notes (NativeAdrenoMethodNote): rows of [words used, pid, the record's words], 512 of each kind, allocated at the first
|
|
* note (the kernel's data segment has no room for them as arrays). The board's export reads the two pointers. */
|
|
enum { NativeAdrenoNoteRows = 512, NativeAdrenoNoteRowWords = 2+NativeAdrenoNoteWords };
|
|
extern "C" {volatile uint32_t *native_adreno_pass_log=nullptr;volatile uint32_t native_adreno_pass_count;
|
|
volatile uint32_t *native_adreno_pipe_log=nullptr;volatile uint32_t native_adreno_pipe_count;}
|
|
static volatile uint32_t *noteRows(volatile uint32_t **slot)
|
|
{
|
|
volatile uint32_t *rows=*slot;
|
|
if(rows)return rows;
|
|
uint32_t *fresh=static_cast<uint32_t *>(IOMallocZero(NativeAdrenoNoteRows*NativeAdrenoNoteRowWords*sizeof(uint32_t)));
|
|
if(!fresh)return nullptr;
|
|
volatile uint32_t *expected=nullptr;
|
|
if(!__atomic_compare_exchange_n(slot,&expected,fresh,false,__ATOMIC_ACQ_REL,__ATOMIC_ACQUIRE)) {
|
|
IOFree(fresh,NativeAdrenoNoteRows*NativeAdrenoNoteRowWords*sizeof(uint32_t));
|
|
return expected;
|
|
}
|
|
return fresh;
|
|
}
|
|
IOReturn AGXDeviceUserClient::runTimed(uint32_t method,const void *input,size_t input_size,void *output,size_t output_size,uint64_t *lockWait)
|
|
{
|
|
if(method==NativeGpuTraceMethod) {
|
|
if(!input || input_size!=sizeof(NativeGpuTraceRecord) || output_size)return kIOReturnBadArgument;
|
|
const auto *record=static_cast<const NativeGpuTraceRecord *>(input);
|
|
if(record->event!=GpuTraceThread && record->event!=GpuTraceProblem && record->event!=GpuTraceTurnipPass &&
|
|
record->event!=GpuTraceTurnipGmemSide && record->event!=GpuTraceTurnipAttachment)return kIOReturnBadArgument;
|
|
native_gpu_trace_record(record->event,proc_selfpid(),record->value,12);
|
|
return kIOReturnSuccess;
|
|
}
|
|
|
|
if(method>=NativeAdrenoHandoffBase && method<=NativeAdrenoHandoffLast)
|
|
return native_gop_handoff_call(method-NativeAdrenoHandoffBase,current_task(),input,input_size,output,output_size);
|
|
if(method==NativeAdrenoExportMethod) {
|
|
struct Piece {uint32_t generation,total,offset,bytes;};
|
|
char caller[20]={};proc_selfname(caller,sizeof caller);
|
|
Piece piece{};
|
|
if(input && input_size>=sizeof(piece))memcpy(&piece,input,sizeof(piece));
|
|
const bool valid=!output_size && input && input_size>=sizeof(piece) && piece.bytes && piece.bytes<=NativeAdrenoExportMaxPiece &&
|
|
input_size==sizeof(piece)+piece.bytes && isRenderServer(caller,sizeof caller);
|
|
const int stored=valid && boot_log_export_put?boot_log_export_put(piece.generation,piece.total,piece.offset,
|
|
static_cast<const uint8_t *>(input)+sizeof(piece),piece.bytes):-1;
|
|
if(stored) {native_adreno_export_stats[1]=native_adreno_export_stats[1]+1;return kIOReturnBadArgument;}
|
|
native_adreno_export_stats[0]=native_adreno_export_stats[0]+1;
|
|
native_adreno_export_stats[2]=piece.generation;native_adreno_export_stats[3]=piece.total;
|
|
return kIOReturnSuccess;
|
|
}
|
|
if(method==NativeAdrenoMethodBoCpuUnmap && input && input_size==sizeof(NativeAdrenoAbiBoCpuUnmap)) {
|
|
/* The view is shared by every CPU mapping of the BO, and dropViews removes all of them. A BO mapped twice (the view cache served 207
|
|
* second requests in int-177) lost both mappings at the first unmap, and the other holder's next store hit an unmapped page: the
|
|
* render server's crash in the command stream writes of vkCmdCopyBuffer. Only the last lease takes the view down; the earlier
|
|
* ones leave their mapping until it, or the client unmaps it itself (IOConnectUnmapMemory64 works on the cached view). */
|
|
const uint32_t token=static_cast<const NativeAdrenoAbiBoCpuUnmap *>(input)->token;
|
|
IOLockLock(g.lock);
|
|
const uint32_t leases=live_?core_->mappingLeases(token):0;
|
|
IOLockUnlock(g.lock);
|
|
if(leases>1)native_adreno_view_kept=native_adreno_view_kept+1;
|
|
else dropViews(token);
|
|
}
|
|
if(method==NativeAdrenoMethodNote) {
|
|
/* Rows for the board's export (BootDiagExport.c, rows G and L): kind, the caller's pid and the record's words. A slot is claimed with
|
|
* one atomic add, so processes noting at once never share a row; a reader that laps a writer sees one torn row at worst. */
|
|
if(!input || input_size!=sizeof(NativeAdrenoAbiNote) || output_size)return kIOReturnBadArgument;
|
|
NativeAdrenoAbiNote note{};
|
|
memcpy(¬e,input,sizeof(note));
|
|
if((note.kind!=NativeAdrenoNotePass && note.kind!=NativeAdrenoNotePipeline) || note.count>NativeAdrenoNoteWords)return kIOReturnBadArgument;
|
|
const bool pass=note.kind==NativeAdrenoNotePass;
|
|
volatile uint32_t *count=pass?&native_adreno_pass_count:&native_adreno_pipe_count;
|
|
volatile uint32_t *rows=noteRows(pass?&native_adreno_pass_log:&native_adreno_pipe_log);
|
|
if(!rows)return kIOReturnNoMemory;
|
|
const uint32_t n=__atomic_fetch_add(count,1,__ATOMIC_RELAXED);
|
|
volatile uint32_t *row=rows+(n%NativeAdrenoNoteRows)*NativeAdrenoNoteRowWords;
|
|
row[0]=note.count;row[1]=uint32_t(proc_selfpid());
|
|
for(uint32_t i=0;i<NativeAdrenoNoteWords;++i)row[2+i]=i<note.count?note.words[i]:0;
|
|
return kIOReturnSuccess;
|
|
}
|
|
if(method==NativeAdrenoMethodSetPriority) {
|
|
/* The GPU priorities this process's Metal queues asked for (MTLGPUPriority numbering): the frontend reports its queues'
|
|
* strongest to its device, which forwards it here once. The switch between the two rings takes the foreground value while
|
|
* the process renders visible UI and the background one otherwise (the process's GPU role, task_get_gpu_role). */
|
|
if(!input || input_size!=sizeof(NativeAdrenoAbiPriority) || output_size)return kIOReturnBadArgument;
|
|
NativeAdrenoAbiPriority request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
if(request.foreground>5 || request.background>5)return kIOReturnBadArgument;
|
|
lockTimed(lockWait);
|
|
const IOReturn status=live_?core_->setPriority(request.foreground,request.background):kIOReturnNotReady;
|
|
unlockTimed(lockWait);
|
|
native_adreno_metal_diag.inflight=0;
|
|
native_adreno_metal_diag.calls=native_adreno_metal_diag.calls+1;
|
|
native_adreno_metal_diag.last_selector=method;
|
|
native_adreno_metal_diag.last_status=uint32_t(status);
|
|
return status;
|
|
}
|
|
if(method==NativeAdrenoMethodBoImport) {
|
|
/* The pages belong to the task making this call (an external method runs in the caller's context). */
|
|
if(!input || input_size!=sizeof(NativeAdrenoAbiBoImport) || !output || output_size!=sizeof(NativeAdrenoAbiBoAllocReply))
|
|
return kIOReturnBadArgument;
|
|
NativeAdrenoAbiBoImport request{};
|
|
memcpy(&request,input,sizeof(request));
|
|
NativeAdrenoAbiBoAllocReply reply{};
|
|
native_adreno_metal_diag.inflight=method+1;
|
|
lockTimed(lockWait);
|
|
const IOReturn status=live_?core_->importBo(&request,current_task(),&reply):kIOReturnNotReady;
|
|
unlockTimed(lockWait);
|
|
native_adreno_metal_diag.inflight=0;
|
|
native_adreno_metal_diag.calls=native_adreno_metal_diag.calls+1;
|
|
native_adreno_metal_diag.last_selector=method;
|
|
native_adreno_metal_diag.last_status=uint32_t(status);
|
|
if(!status)memcpy(output,&reply,sizeof(reply));
|
|
return status;
|
|
}
|
|
return runCore(this,method,input,input_size,output,output_size,lockWait);
|
|
}
|
|
IOReturn AGXDeviceUserClient::action(OSObject *target,void *,IOExternalMethodArguments *arguments)
|
|
{
|
|
AGXDeviceUserClient *connection=OSDynamicCast(AGXDeviceUserClient,target);
|
|
if(!connection || !arguments)return kIOReturnBadArgument;
|
|
return connection->run(arguments->selector,arguments->structureInput,arguments->structureInputSize,
|
|
arguments->structureOutput,arguments->structureOutputSize);
|
|
}
|
|
IOReturn AGXDeviceUserClient::externalMethod(uint32_t selector,IOExternalMethodArguments *arguments,
|
|
IOExternalMethodDispatch *,OSObject *target,void *reference)
|
|
{
|
|
if(selector==NativeGpuTraceMethod && arguments)
|
|
return run(selector,arguments->structureInput,arguments->structureInputSize,
|
|
arguments->structureOutput,arguments->structureOutputSize);
|
|
/* One entry per ABI method; sizes are the exact wire records. */
|
|
static const IOExternalMethodDispatch table[]={
|
|
{&action,0,0,0,sizeof(NativeAdrenoAbiParams)},
|
|
{&action,0,sizeof(NativeAdrenoAbiBoAlloc),0,sizeof(NativeAdrenoAbiBoAllocReply)},
|
|
{&action,0,sizeof(NativeAdrenoAbiBoMap),0,sizeof(NativeAdrenoAbiBoMapReply)},
|
|
{&action,0,sizeof(NativeAdrenoAbiBoFree),0,0},
|
|
{&action,0,sizeof(NativeAdrenoAbiSubmit),0,sizeof(NativeAdrenoAbiSubmitReply)},
|
|
{&action,0,sizeof(NativeAdrenoAbiWaitFence),0,0},
|
|
{&action,0,0,0,sizeof(NativeAdrenoAbiTimestamp)},
|
|
{&action,0,0,0,sizeof(NativeAdrenoAbiStatus)},
|
|
{&action,0,sizeof(NativeAdrenoAbiMetadata),0,0},
|
|
{&action,0,sizeof(NativeAdrenoAbiMetadata),0,sizeof(NativeAdrenoAbiMetadata)},
|
|
{&action,0,sizeof(NativeAdrenoAbiQueue),0,sizeof(NativeAdrenoAbiQueueReply)},
|
|
{&action,0,sizeof(NativeAdrenoAbiQueue),0,0},
|
|
{&action,0,sizeof(NativeAdrenoAbiBoCpuMap),0,sizeof(NativeAdrenoAbiBoCpuMapReply)},
|
|
{&action,0,sizeof(NativeAdrenoAbiBoCpuUnmap),0,0},
|
|
{&action,0,0,0,sizeof(NativeAdrenoAbiDeviceInfo)},
|
|
{&action,0,sizeof(NativeAdrenoAbiBoImport),0,sizeof(NativeAdrenoAbiBoAllocReply)},
|
|
{&action,0,sizeof(NativeAdrenoAbiPriority),0,0},
|
|
{&action,0,sizeof(NativeAdrenoAbiNote),0,0}};
|
|
native_adreno_metal_diag.ext_entries=native_adreno_metal_diag.ext_entries+1;
|
|
if(selector<256)native_adreno_metal_diag.sel_bits[selector>>5]|=1u<<(selector&31);
|
|
/* Tunnel. The sandbox resolves a filter policy for this class at open time and then admits only the selectors its profile lists,
|
|
* so which selector numbers reach this object is decided outside the driver. A tunnel call carries the real ABI method in its
|
|
* first eight bytes and may use ANY admitted selector: {magic, method, payload...} in, the method's exact output out. Method
|
|
* 0xffffffff is a probe that only proves the selector was admitted. */
|
|
if(arguments && arguments->structureInput && arguments->structureInputSize>=8 &&
|
|
static_cast<const uint32_t *>(arguments->structureInput)[0]==NativeAdrenoTunnelMagic) {
|
|
const uint32_t *header=static_cast<const uint32_t *>(arguments->structureInput);
|
|
if(header[1]==0xffffffffu) {
|
|
native_adreno_metal_diag.probes=native_adreno_metal_diag.probes+1;
|
|
return kIOReturnSuccess;
|
|
}
|
|
native_adreno_metal_diag.tunnel_calls=native_adreno_metal_diag.tunnel_calls+1;
|
|
const IOReturn tunneled=run(header[1],reinterpret_cast<const uint8_t *>(arguments->structureInput)+8,
|
|
arguments->structureInputSize-8,arguments->structureOutput,arguments->structureOutputSize);
|
|
native_adreno_metal_diag.ext_status=uint32_t(tunneled);
|
|
return tunneled;
|
|
}
|
|
native_adreno_metal_diag.ext_selector=selector;
|
|
native_adreno_metal_diag.ext_in=arguments?arguments->structureInputSize:0xffff;
|
|
native_adreno_metal_diag.ext_out=arguments?arguments->structureOutputSize:0xffff;
|
|
if(selector<1 || selector>sizeof(table)/sizeof(table[0])) {
|
|
native_adreno_metal_diag.ext_status=uint32_t(kIOReturnUnsupported);
|
|
return kIOReturnUnsupported;
|
|
}
|
|
const IOReturn result=IOUserClient::externalMethod(selector,arguments,const_cast<IOExternalMethodDispatch *>(&table[selector-1]),
|
|
target?target:this,reference);
|
|
native_adreno_metal_diag.ext_status=uint32_t(result);
|
|
return result;
|
|
}
|
|
/* limit > 0 caps the connections ever granted: a plugin that crashes its host process on every launch (the process respawns and
|
|
* loads it again) must not be able to keep a system process in a crash loop, so after the cap further opens are refused. */
|
|
extern "C" IOReturn native_adreno_new_user_client(IOService *provider,task_t owningTask,void *securityID,UInt32 type,
|
|
OSDictionary *properties,IOUserClient **handler,unsigned limit)
|
|
{
|
|
if(handler)*handler=nullptr;
|
|
if(!handler || !g.published)return kIOReturnNotReady;
|
|
/* Progress markers: open types 0x4147xxxx (xxxx nonzero) are not connections. The plugin sends one before and after each step so
|
|
* that a sandbox that admits neither selectors nor mapping, or a crash, still leaves a trace of how far it got. */
|
|
if((type>>16)==NativeAdrenoPluginOpenType && (type&0xffff)) {
|
|
const uint32_t value=type&0xffff,slot=native_adreno_metal_diag.marks_total&15;
|
|
/* Text stream into report2, one character per marker: 0x7fff starts a line, 0x70cc appends character cc. It needs no connection,
|
|
* so the plugin's Turnip watchdog can report while Turnip itself holds its connection slot. Not counted as markers. */
|
|
if(value>=0x8000) {
|
|
static unsigned row=48,length;
|
|
if(value>=0xC000) {
|
|
row=value&0x3f;length=0;
|
|
if(row<48)memset(native_adreno_metal_rows[row],0,sizeof(native_adreno_metal_rows[row]));
|
|
native_adreno_metal_rows_seq=native_adreno_metal_rows_seq+1;
|
|
} else if(row<48) {
|
|
const char pair[2]={char((value>>7)&0x7f),char(value&0x7f)};
|
|
for(unsigned i=0;i<2;++i)if(pair[i] && length<sizeof(native_adreno_metal_rows[row])-1)native_adreno_metal_rows[row][length++]=pair[i];
|
|
}
|
|
return kIOReturnUnsupported;
|
|
}
|
|
if(value>=0x7000 && value<0x8000) {
|
|
/* 0x7ffe starts a line in report (GPU7) instead: the Metal bridge's result, after the raw test's line has been read. */
|
|
static uint32_t length;
|
|
static char *line=native_adreno_metal_diag.report2;
|
|
static_assert(sizeof(native_adreno_metal_diag.report)==sizeof(native_adreno_metal_diag.report2),"stream lines");
|
|
if(value==0x7fff || value==0x7ffe) {
|
|
line=value==0x7fff?native_adreno_metal_diag.report2:native_adreno_metal_diag.report;
|
|
/* A line from a process other than the render server goes to that process's own row (native_adreno_client_text): the render server
|
|
* rewrites report2 every ten seconds, so the reason another process's device failed would be gone from it. */
|
|
char who[20]={};
|
|
proc_selfname(who,sizeof who);
|
|
if(value==0x7fff && !isRenderServer(who,sizeof who)) {
|
|
for(auto &entry:native_adreno_client_text) {
|
|
if(!entry.name[0]) strlcpy(entry.name,who,sizeof entry.name);
|
|
if(!strncmp(entry.name,who,sizeof entry.name)) {line=entry.text;break;}
|
|
}
|
|
}
|
|
memset(line,0,sizeof(native_adreno_metal_diag.report2));length=0;
|
|
}
|
|
else if(value<0x7100 && length<sizeof(native_adreno_metal_diag.report2)-1) {line[length++]=char(value&0xff);line[length]=0;}
|
|
return kIOReturnUnsupported;
|
|
}
|
|
native_adreno_metal_diag.marks[slot]=value;
|
|
native_adreno_metal_diag.marks_total=native_adreno_metal_diag.marks_total+1;
|
|
if(value==1) {
|
|
memcpy(native_adreno_metal_diag.starter[2],native_adreno_metal_diag.starter[1],17);
|
|
memcpy(native_adreno_metal_diag.starter[1],native_adreno_metal_diag.starter[0],17);
|
|
proc_selfname(native_adreno_metal_diag.starter[0],17);
|
|
}
|
|
return kIOReturnUnsupported;
|
|
}
|
|
shiftName(native_adreno_metal_diag.opener);
|
|
AGXDeviceUserClient *connection=new AGXDeviceUserClient;
|
|
if(!connection)return kIOReturnNoMemory;
|
|
if(!connection->initWithTask(owningTask,securityID,type,properties)) {
|
|
connection->release();
|
|
native_adreno_metal_diag.bad=native_adreno_metal_diag.bad+1;
|
|
return kIOReturnNoResources;
|
|
}
|
|
/* IOService::newUserClient attaches and starts what it creates (a client without a provider chain is also a reason for the sandbox to
|
|
* refuse). Then the two sandbox gates IOServiceOpen would apply afterwards are run here for every opener and logged with the
|
|
* process name and open type, so the boot page shows which processes the sandbox lets reach this class and with which type. */
|
|
unsigned attach_state=0;
|
|
if(provider) {
|
|
const bool attached=connection->attach(provider);
|
|
attach_state=attached?1:2;
|
|
if(attached && !connection->start(provider)) {
|
|
connection->detach(provider);
|
|
attach_state=3;
|
|
}
|
|
}
|
|
native_adreno_metal_diag.attach_status=attach_state;
|
|
const uint32_t macf=uint32_t(mac_iokit_check_open(kauth_cred_get(),connection,type));
|
|
uint32_t resolver=0xfeedfeed;
|
|
if(gIOUCFilterCallbacks && gIOUCFilterCallbacks->io_filter_resolver) {
|
|
io_filter_policy_t policy=0;
|
|
const IOReturn resolved=gIOUCFilterCallbacks->io_filter_resolver(owningTask,connection,type,&policy);
|
|
resolver=uint32_t(resolved);
|
|
if(resolved==kIOReturnSuccess && policy && gIOUCFilterCallbacks->io_filter_release)gIOUCFilterCallbacks->io_filter_release(policy);
|
|
}
|
|
native_adreno_metal_diag.chk_type=type;
|
|
native_adreno_metal_diag.chk_macf=macf;
|
|
native_adreno_metal_diag.chk_res=resolver;
|
|
{
|
|
NativeAdrenoOpenRecord &record=native_adreno_open_log[native_adreno_open_log_count&7];
|
|
proc_selfname(record.name,sizeof(record.name));
|
|
record.type=type;record.macf=macf;record.res=resolver;record.attach=attach_state;record.foreign=type!=NativeAdrenoPluginOpenType;
|
|
native_adreno_open_log_count=native_adreno_open_log_count+1;
|
|
}
|
|
if(type==NativeAdrenoHandoffOpenType) {
|
|
connection->setHandoffOnly();
|
|
*handler=connection;
|
|
return kIOReturnSuccess;
|
|
}
|
|
/* Only the Adreno plugin's open (type NativeAdrenoPluginOpenType) is served. The original IOGPU library opens the accelerator with type 1
|
|
* (and an API value in the upper bits) for an IOGPUDeviceUserClient, which this driver is not; refuse it cleanly. */
|
|
if(type!=NativeAdrenoPluginOpenType) {
|
|
connection->release();
|
|
native_adreno_metal_diag.foreign_opens=native_adreno_metal_diag.foreign_opens+1;
|
|
native_adreno_metal_diag.foreign_type=type;
|
|
return kIOReturnUnsupported;
|
|
}
|
|
/* Every process that runs the Adreno plugin opens with its type: on Apple hardware every Metal process opens the AGX user client, and the
|
|
* original profiles here name no class of this driver for the render server (MACF 1 for the plugin's types at every boot since int-038).
|
|
* The plugin's two open types are granted here, by this driver: the grant is a property of the client object that the kernel's open path
|
|
* (patches/xnu/native-adreno-sandbox-grant.patch) honours in place of the sandbox verdict. The verdict is still taken above and logged.
|
|
* Which processes run the plugin at all is the plugin's allow-list (NativeAdrenoMetal.clients). */
|
|
const bool kernel=owningTask==kernel_task;
|
|
char opener[20]={};
|
|
proc_selfname(opener,sizeof opener);
|
|
if(!kernel && (macf || resolver!=uint32_t(kIOReturnSuccess))) {
|
|
connection->setProperty("NativeSandboxGrant",kOSBooleanTrue);
|
|
native_adreno_metal_diag.grants=native_adreno_metal_diag.grants+1;
|
|
}
|
|
/* limit (boot-arg gpu-metal-opens, default 0 = none) caps the connections granted to each process name. No cap by default: a
|
|
* client that crashed and restarts (iconservicesagent, i-39b429634) must get the GPU again, or it is left without one for the boot. */
|
|
const unsigned cap=limit;
|
|
IOLockLock(g.lock);
|
|
NativeAdrenoClientRecord *record=kernel?nullptr:clientRecord(opener);
|
|
const bool capped=cap && record && record->opens>=cap;
|
|
IOLockUnlock(g.lock);
|
|
IOReturn refusal=kIOReturnSuccess;
|
|
if(capped)refusal=kIOReturnBusy;
|
|
else if(!connection->mount(kernel))refusal=kIOReturnBusy; /* no address space or queue range */
|
|
if(refusal) {
|
|
connection->release();
|
|
native_adreno_metal_diag.busy=native_adreno_metal_diag.busy+1;
|
|
IOLockLock(g.lock);
|
|
if(record) {record->refused=record->refused+1;record->last_refusal=uint32_t(refusal);}
|
|
IOLockUnlock(g.lock);
|
|
return refusal;
|
|
}
|
|
IOLockLock(g.lock);
|
|
if(record)record->opens=record->opens+1;
|
|
IOLockUnlock(g.lock);
|
|
native_adreno_metal_diag.opens=native_adreno_metal_diag.opens+1;
|
|
*handler=connection;
|
|
return kIOReturnSuccess;
|
|
}
|
|
extern "C" void native_adreno_note_property_read(const char *key)
|
|
{
|
|
if(key && !strcmp(key,"MetalPluginName")) {
|
|
proc_selfname(native_adreno_metal_diag.name_reader,17);
|
|
native_adreno_metal_diag.name_reads=native_adreno_metal_diag.name_reads+1;
|
|
char name[20]={};
|
|
proc_selfname(name,sizeof name);
|
|
/* Unlocked: the discovery runs at process start, a lost count is harmless and a row is claimed by its first byte. */
|
|
for(auto &asker:native_adreno_metal_askers) {
|
|
if(!asker.name[0]) {strlcpy(asker.name,name,sizeof asker.name);asker.reads=1;break;}
|
|
if(!strncmp(asker.name,name,sizeof asker.name)) {asker.reads=asker.reads+1;break;}
|
|
}
|
|
}
|
|
shiftName(native_adreno_metal_diag.reader);
|
|
native_adreno_metal_diag.property_reads=native_adreno_metal_diag.property_reads+1;
|
|
}
|
|
IOReturn NativeAdrenoGpuPublished::newUserClient(task_t owningTask,void *securityID,UInt32 type,
|
|
OSDictionary *properties,IOUserClient **handler)
|
|
{
|
|
return native_adreno_new_user_client(this,owningTask,securityID,type,properties,handler,0);
|
|
}
|
|
|
|
static IOReturn checkCpSwitch()
|
|
{
|
|
uint32_t kit=0,image[8]={};
|
|
(void)g.context->observeContext(&kit,image);
|
|
native_adreno_cp_image[16]=kit;for(unsigned i=0;i<8;++i)native_adreno_cp_image[i]=image[i];
|
|
NativeAdrenoProcessVm *space=nullptr;
|
|
IOReturn status=g.context->createVm(&space);
|
|
native_adreno_cp_check[0]=uint32_t(status);
|
|
if(status)return status;
|
|
NativeAdrenoSubmitBuffer ib{};
|
|
status=g.context->probeBuffer(space,&ib.handle,&ib.iova,&ib.bytes);
|
|
native_adreno_cp_check[1]=uint32_t(status);
|
|
uint32_t fence=0;
|
|
/* The proof is of the first ring's CP switch, as before there were two rings. */
|
|
g.submit->nextPriority(0);
|
|
if(!status) {
|
|
status=g.submit->submit(space->vm(),&ib,1,0,&fence,&g.clock);
|
|
native_adreno_cp_check[2]=uint32_t(status);
|
|
native_adreno_cp_check[5]=fence;
|
|
}
|
|
g.submit->nextPriority(-1);
|
|
if(!status) {
|
|
status=g.submit->waitAll(&g.clock,2000000);
|
|
native_adreno_cp_check[3]=uint32_t(status);
|
|
}
|
|
uint64_t seen=0;
|
|
if(!status) {
|
|
status=g.context->readTtbr0(&seen);
|
|
native_adreno_cp_check[4]=uint32_t(status);
|
|
}
|
|
if(!status && seen!=space->vm()->ttbr0())status=kIOReturnUnsupported;
|
|
IOReturn freed=g.submit->forgetVm(space->vm(),&g.clock,2000000);
|
|
if(!freed)freed=g.context->destroyVm(space);
|
|
/* Every table operation proves the bank's image (TTBR0 aside): a CP switch that changed another register would fail the next one
|
|
* and poison the kernel table. */
|
|
const IOReturn observed=g.context->observeContext(&kit,image);
|
|
native_adreno_cp_image[17]=kit;for(unsigned i=0;i<8;++i)native_adreno_cp_image[8+i]=image[i];
|
|
if(observed && kit==uint32_t(ArmSmmu4KFault)) {
|
|
uint32_t far[3]={};
|
|
if(!g.context->clearFault(image[7],far)) {
|
|
native_adreno_cp_image[18]=far[0];native_adreno_cp_image[19]=far[1];native_adreno_cp_image[20]=far[2];
|
|
const IOReturn again=g.context->observeContext(&kit,image);
|
|
native_adreno_cp_image[21]=kit|0x100;
|
|
/* The GPU touched an address the probe's table does not map: the switch itself worked. */
|
|
if(!again && !status && !freed)return kIOReturnSuccess;
|
|
}
|
|
}
|
|
if(!status && !freed && observed)status=kIOReturnNotPermitted;
|
|
return status?status:freed;
|
|
}
|
|
/* The GPU interrupt's side of a preemption switch (GpuPowerProbe.cpp's handler calls it): the CP moved to the other ring, whose
|
|
* write pointer is given again. False when no switch was in flight. */
|
|
/* For the boot log: out[0..6] the switch counters (triggers, completed, skipped, faulted, disjoint skips, wptr restores, last us),
|
|
* out[7] the ring the CP runs, out[8] a switch in flight, out[9] CP_CONTEXT_SWITCH_CNTL read now, out[10..15] checkCpSwitch's steps, out[16] why the second ring was given up (0 not; 1 the trigger refused, 2 a switch
|
|
* did not finish, 3 a job there waited too long), out[17] restarts that replayed its jobs, out[18..25] ring 0's then ring 1's record words (rptr, wptr, errno, info). False
|
|
* when preemption is off or the lock is busy. */
|
|
extern "C" bool native_adreno_preempt_stats(uint32_t *out)
|
|
{
|
|
if(!out)return false;
|
|
for(unsigned i=0;i<6;++i)out[10+i]=native_adreno_cp_check[i];
|
|
if(!g.published || !g.submit || !IOLockTryLock(g.lock))return false;
|
|
NativeAdrenoPreemptStats st{};uint32_t loaded=0,cntl=0,dead=0,rescues=0;bool flight=false;
|
|
const bool on=g.submit->preemptStats(&st,&loaded,&flight,&cntl,&dead,&rescues);
|
|
IOLockUnlock(g.lock);
|
|
if(!on)return false;
|
|
out[0]=st.triggers;out[1]=st.completed;out[2]=st.skipped;out[3]=st.faulted;out[4]=st.disjoint_skips;out[5]=st.restored_wptr;
|
|
out[6]=st.last_us;out[7]=loaded;out[8]=flight;out[9]=cntl;out[16]=dead;out[17]=rescues;
|
|
/* each record as the CPU sees it: read pointer, write pointer, errno, info (the CP's words are only as fresh as the mapping) */
|
|
for(unsigned r=0;r<2;++r) {
|
|
out[18+r*4]=g.submit->recordWord(r,20);out[19+r*4]=g.submit->recordWord(r,24);
|
|
out[20+r*4]=g.submit->recordWord(r,8);out[21+r*4]=g.submit->recordWord(r,4);
|
|
}
|
|
return true;
|
|
}
|
|
/* The board's submission clock in us, for the logs written outside the submit path (0 before the GPU is published). */
|
|
extern "C" uint64_t native_adreno_now_us(void)
|
|
{
|
|
return g.published && g.clock.now_us?g.clock.now_us(g.clock.cookie):0;
|
|
}
|
|
extern "C" bool native_adreno_preempt_irq()
|
|
{
|
|
if(!g.published || !g.submit)return false;
|
|
/* The interrupt's handler may run where a thread holds the lock: the switch's completion is taken here when the lock is free,
|
|
* and otherwise by the next submission's or retirement's pass (NativeAdrenoSubmit::driveSwitch completes a switch in flight
|
|
* before it starts another), so nothing waits on the lock from this path. */
|
|
if(!IOLockTryLock(g.lock))return false;
|
|
const bool done=g.submit->preemptIrq();
|
|
IOLockUnlock(g.lock);
|
|
return done;
|
|
}
|
|
/* For the display's direct scanout (NativeLegacyDisplay): the newest fence handed out and the newest one the CP has written. The compositor
|
|
* submits a frame's swap when its command buffer is scheduled, not when the GPU finished it; a display that reads the surface itself waits
|
|
* for the fences handed out before the swap arrived. False before the GPU is published. */
|
|
extern "C" bool native_adreno_fences_detail(uint32_t *submitted,uint32_t *retired,uint32_t *detail)
|
|
{
|
|
if(detail){detail[0]=detail[1]=detail[2]=0;}
|
|
if(!g.published || !g.submit || !submitted || !retired) {
|
|
return false;
|
|
}
|
|
if(!IOLockTryLock(g.lock)){
|
|
if(detail)detail[2]=1;
|
|
return false;
|
|
}
|
|
const bool known=g.submit->fenceProgress(submitted,retired,detail);
|
|
if(detail)detail[2]=known?2:3;
|
|
IOLockUnlock(g.lock);return known;
|
|
}
|
|
extern "C" bool native_adreno_fences(uint32_t *submitted,uint32_t *retired)
|
|
{
|
|
return native_adreno_fences_detail(submitted,retired,nullptr);
|
|
}
|
|
/* Publishes the service once. Everything passed in must outlive the system (the board probe keeps them in statics) and
|
|
* the VM, submit object and ring must already be attached to the live GPU context. The core must not be open. */
|
|
extern "C" IOReturn native_adreno_publish(NativeAdrenoVm *vm,NativeAdrenoSubmit *submit,NativeAdrenoRing *ring,
|
|
const NativeAdrenoRingClock *clock,const NativeAdrenoTimer *timer,const NativeAdrenoAbiDeviceInfo *info,
|
|
uint32_t input_bits,uint32_t output_bits)
|
|
{
|
|
if(g.published || !vm || !submit || !ring || !clock || !clock->now_us)return kIOReturnBadArgument;
|
|
gpuTraceInit();
|
|
g.lock=IOLockAlloc();
|
|
NativeAdrenoGpuPublished *service=new NativeAdrenoGpuPublished;
|
|
if(!g.lock || !service || !service->init() || !service->attach(IOService::getPlatform())) {
|
|
if(service)service->release();
|
|
return kIOReturnNoMemory;
|
|
}
|
|
/* The fence waiters' wake path. Without its lock or its thread call the waiters keep the board's pause. gpu-fence-wake=0 keeps
|
|
* that pause too, for a comparison on the same image. */
|
|
{
|
|
unsigned by_interrupt=1;
|
|
PE_parse_boot_argn("gpu-fence-wake",&by_interrupt,sizeof by_interrupt);
|
|
g.wake_by_interrupt=by_interrupt!=0;
|
|
g.wake_lock=IOLockAlloc();
|
|
g.fence_call=g.wake_lock?thread_call_allocate(fenceCall,nullptr):nullptr;
|
|
}
|
|
g.vm=vm;g.submit=submit;g.ring=ring;g.clock=*clock;g.timer=timer;g.info=info;g.input_bits=input_bits;g.output_bits=output_bits;
|
|
/* Per-process address spaces: without table switching every user connection would be refused (createVm needs it). */
|
|
g.context=NativeAdrenoGpuContextObject();
|
|
IOReturn switching=g.context?g.context->enableSwitching():kIOReturnNotReady;
|
|
if(!switching)switching=submit->setTableSwitching(tableMode(),
|
|
[](void *cookie,uint64_t ttbr0)->IOReturn {return static_cast<NativeAdrenoGpuContext *>(cookie)->writeTtbr0(ttbr0);},g.context);
|
|
native_adreno_table_switches[6]=uint32_t(switching);
|
|
if(switching)g.context=nullptr;
|
|
else if(submit->tableMode()==2) {
|
|
/* The CP's switch must reach the bank: one NOP submission on a fresh address space, then the bank's TTBR0 is read back. A
|
|
* switch that did not land falls back to the CPU's. */
|
|
const IOReturn checked=checkCpSwitch();
|
|
native_adreno_cp_aperture[2]=uint32_t(checked);
|
|
if(checked && !submit->outstanding())(void)submit->setTableSwitching(1,
|
|
[](void *cookie,uint64_t ttbr0)->IOReturn {return static_cast<NativeAdrenoGpuContext *>(cookie)->writeTtbr0(ttbr0);},g.context);
|
|
}
|
|
native_adreno_cp_aperture[3]=submit->tableMode();
|
|
g.service=service;g.published=true;
|
|
gpuTraceEvent(GpuTraceConfig,0,1,submit->tableMode(),submit->preemption(),input_bits,output_bits,submit->preemptionMode());
|
|
#ifdef NATIVE_GPU_BUILD_ID
|
|
const uint64_t build[]={NATIVE_GPU_BUILD_ID};
|
|
native_gpu_trace_record(GpuTraceBuild,0,build,1);
|
|
#endif
|
|
{
|
|
thread_t thread=nullptr;
|
|
if(kernel_thread_start(nativeAdrenoHangcheck,nullptr,&thread)==KERN_SUCCESS)thread_deallocate(thread);
|
|
}
|
|
service->setName("NativeAdrenoGpu");
|
|
service->registerService();
|
|
return kIOReturnSuccess;
|
|
}
|
|
extern "C" uint32_t native_adreno_publish_stats(uint32_t *closed)
|
|
{
|
|
if(closed)*closed=g.closed;
|
|
return g.opened;
|
|
}
|
|
extern "C" IOService *native_adreno_published_service()
|
|
{
|
|
return g.service;
|
|
}
|