#define NATIVE_GPU_TRACE_OPTIONAL #include "NativeGpuTrace.h" /* SPDX-License-Identifier: MIT */ #include "NativeAdrenoSubmit.h" #include "AdrenoPM4.h" #include /* CP_INDIRECT_BUFFER is opcode 0x3f with a 3-word payload: 64-bit GPU address * and IB_SIZE in dwords (bits 0..19), exactly as the pinned adreno_pm4.xml * domain states and Linux msm_gem_submit.c converts the userspace byte size. */ enum : uint32_t {SubmitIndirectBuffer=0x3f,SubmitIbWords=3, SubmitMaxBuffers=(uint32_t)NativeAdrenoSubmitMaxBuffers,SubmitMaxIbBytes=0xfffffu*4u, SubmitAuditBytes=256u*1024u}; /* IB bytes the write-combine audit compares per submission */ extern "C" {volatile uint32_t native_adreno_coherence[8];} /* IBs per submission: [0] submissions, [1] IBs, [2] the most in one, [3] submissions with more than 16 (one frame was several before). */ extern "C" {volatile uint32_t native_adreno_submit_ibs[4];} /* The submission (1-based count) at which the write-combine audit found a dirty IB line or failed; 0 = never. */ extern "C" {volatile uint32_t native_adreno_wc_distrusted;} /* The write-combine audit before each submission while trusted: [0] submissions audited, [1] IB lines compared, [2] dirty lines, * [3] stale clean lines, [4] the last audit failure (any failure also ends the trust), [5] pages found back at the default attribute * and write-combined again. */ extern "C" {volatile uint32_t native_adreno_wc_audit[6];} extern "C" boolean_t PE_parse_boot_argn(const char *,void *,int); /* The line audits before a submission (the write-combined IBs of the first 16, the synchronized driver buffers of the first 8 and of * every 1024th) are diagnostics: they detect a CPU cache that holds a line the GPU should have seen, and on one they switch to * maintaining every such buffer. They found none in int-141, i566, run67 or run68 (dirty=0 every time); the pool BOs have no * cacheable alias for them to compare at all, and the write-combine re-assertion, which corrects a page that went back to the default * attribute, still runs at every submission. In run68 the 8 audits of the synchronized buffers took 11.9 of the 12.1 ms the phase * spent over the session (1.5 ms each, under the driver's lock), and every 1024th submission paid one more. `gpu-audit=1` runs them * again as before; this word is 0 (off), 1 (on) or ~0 before the boot argument is read. */ extern "C" {volatile uint32_t native_adreno_audit_mode=~0u;} static bool auditsWanted() { if(native_adreno_audit_mode==~0u) { uint32_t on=0;PE_parse_boot_argn("gpu-audit",&on,sizeof on); native_adreno_audit_mode=on?1:0; } return native_adreno_audit_mode!=0; } /* Fence waits: [0] calls, [1] timeouts, [2] the fence of the last timed-out wait, [3] the fence value the GPU had written then, [4] the newest fence handed out, * [5] the newest completed, [6] waits for a fence no job holds (observeFence path), [7] the last fence refused as never submitted. */ extern "C" {volatile uint32_t native_adreno_fence_wait[8];} /* The CP's state once a fence stopped advancing (NativeAdrenoRing::stallState): [0] the fence, [1] the timed-out waits of it so far, then * the ring's words. Taken at the 10th and again at the 1000th timed-out wait of the fence that is the next to complete. native_adreno_stall_mem * holds, from the same moment, the SMMU context's FSR, FAR (lo, hi) and FSYNR0, then for the BR IB1, the BV IB1 and the BV IB2 17 words: the * dword index in the IB where the 16 words that follow start (the IB1s are found from the ring packet before each read pointer, so the index * counts from the IB's start; for the BV IB2 only the fetch address is known and the index is 0xffff0000 + words before it), then the words. * The last 4: the map handle that was next when the stalled job was submitted, then the handles of the mappings under the BR IB1, the BV IB1 * and the BV IB2 (0: unmapped). A handle at or above the first was mapped after the submission: its memory was freed and reused under the GPU. */ extern "C" {volatile uint32_t native_adreno_stall[2+NativeAdrenoRingStallWords];volatile uint32_t native_adreno_stall_mem[4+3*17+4];} /* Which ring that snapshot is about, taken with it. The registers of native_adreno_stall are those of the ring the CP runs, and the * fence it names is the one after the waited ring's last written fence, which with two rings can be a fence of the other ring or one * already retired: [0] the fence the snapshot names, [1] the fence the timed-out wait was for, [2] that fence's ring (0 high, 1 low), * [3] the ring the CP runs, [4] a ring switch in flight, [5] jobs the CP has not finished on ring 0 and [6] on ring 1, [7] whether [0] * has a job (bit 0), that job's ring << 8 and its priority + 1 << 16, [8] us since the waited fence's job was submitted (0: no job or * no clock), [9] us the snapshot's fence had been pending. A waited fence on a ring the CP does not run, with the other ring busy or * idle, is a job waiting for the switch, not a CP that stopped. */ extern "C" {volatile uint32_t native_adreno_stall_ring[10];} extern "C" {volatile uint32_t native_adreno_switch_timeout[2][30];volatile uint32_t native_adreno_switch_timeouts;} /* 16 words of the IB at start, from 12 dwords before the dword the thread has reached (size minus what remains). */ static void peekIb(const NativeAdrenoVm *vm,uint64_t start,uint32_t size,uint32_t remaining,volatile uint32_t *slot) { uint32_t words[16]={}; const uint32_t reached=remaining<=size?size-remaining:0,first=reached>12?reached-12:0; if(vm->peekWordsUncached(start+4ull*first,words,16))for(unsigned i=0;i<16;++i)words[i]=0xdeadbeef; slot[0]=first; for(unsigned i=0;i<16;++i)slot[1+i]=words[i]; } /* 16 words around a fetch address whose IB start is unknown: from 12 words before it, or nearer when that crosses into no buffer. */ static void peekFetch(const NativeAdrenoVm *vm,uint64_t fetch,volatile uint32_t *slot) { uint32_t words[16]={},before=12; while(before && vm->peekWordsUncached(fetch-4*before,words,16))before-=4; if(!before && vm->peekWordsUncached(fetch,words,16))for(unsigned i=0;i<16;++i)words[i]=0xdeadbeef; slot[0]=0xffff0000u+before; for(unsigned i=0;i<16;++i)slot[1+i]=words[i]; } /* Where the stalled job's command buffers name the IB2 the thread stopped in: every IB1 of the job (the CP_INDIRECT_BUFFER packets * of its ring words) is read for a CP_INDIRECT_BUFFER whose low address word is the IB2 base register's. [0] how many were found, * [1] the IB1's place in the job, [2..3] its address, [4] the packet's dword offset in it, [5..10] the packet's four words and the * two before it, [11] IB1s read, [12] dwords read, [13] IB1s that could not be read. Nothing found with every IB1 read means the * address did not come from the job's commands. */ extern "C" {volatile uint32_t native_adreno_stall_find[14];} static bool indirectBuffer(uint32_t header) {return (header>>28)==7u && ((header>>16)&0x7fu)==0x3fu && (header&0x3fffu)==3;} static void findIb2(const NativeAdrenoVm *vm,const uint32_t *ring_words,size_t ring_count,uint32_t ib2_low) { uint32_t out[14]={}; static uint32_t chunk[1024]; uint32_t index=0; for(size_t i=0;i+3peekWordsUncached(ib1+4ull*at,chunk,take)) {++out[13];break;} out[12]+=take; for(uint32_t k=0;k+3>32);out[4]=at+k; out[5]=k>=2?chunk[k-2]:carry[0];out[6]=k>=1?chunk[k-1]:carry[1]; for(unsigned w=0;w<4;++w)out[7+w]=chunk[k+w]; } ++out[0]; } carry[0]=chunk[take>=2?take-2:0];carry[1]=chunk[take-1]; } ++index; } for(unsigned i=0;i<14;++i)native_adreno_stall_find[i]=out[i]; } static uint32_t stall_fence,stall_timeouts,stall_stage; /* stage: 0 none yet, 1 write pointer given again, 2 translations dropped, 3 bus drained */ static uint64_t stall_began_us,stall_last_us; /* the fence's first and latest timed-out wait */ /* How long fences stayed pending from their first timed-out wait to their last: [i] under 1 << i ms for i < 7, [7] 64 ms or more. */ extern "C" {volatile uint32_t native_adreno_fence_pending[8];} /* What the stall handler tried: [0] attempts, [1] the fence of the last attempt, [2] attempts after which that fence completed, [3] the * timed-out waits that fence had seen when it did. */ /* [4] bus drains, [5] client halt acknowledge, [6] arbiter halt acknowledge, [7] the fence of the last drain, [8] drains after which that fence completed */ extern "C" {volatile uint32_t native_adreno_unstick[15];} /* The IBs of the last submissions: [slot] fence, iova low/high, bytes, then the first 28 dwords of the first IB. The words are a * diagnostic (diag=1, zero otherwise) and are read through NativeAdrenoVm::headWords: the board's uncached reader makes and removes * a kernel mapping for every word, 28 pairs a submission under the driver's lock. */ extern "C" {volatile uint32_t native_adreno_ib_log[4][4+28];volatile uint32_t native_adreno_ib_log_count;} /* Where an accepted submission's time under the driver's lock goes (us, summed; the clock is the caller's): [0] submissions timed, * [1] validation, retiring finished jobs and holding the buffers, [2] the write-combine re-assertion and the audits, [3] the pass * over the mappings with its cache maintenance (and every 64th time the coherence sample), [4] the submission log, [5] the table * switch, the ring and the job record, [6] all of it. A wait_fence before the submission is not in it. */ extern "C" {volatile uint64_t native_adreno_submit_us[8];} extern "C" volatile uint32_t native_adreno_table_switches[8]; /* Where the last recover() stopped: [0] the step (1 fence read, 2 restart hook, 3 table write, 4 nop encode, 5 replay vm, 6 replay submit), [1] the IOReturn or value there, [2] fence, [3] last submitted. */ extern "C" {volatile uint32_t native_adreno_recover_trace[4];} /* The jobs' timeline (the newest native_adreno_tl_cap retired, allocated at attach): [0] fence | indirect buffers << 32 | ring status << 58, [1] submit * time (us), [2] the CP's always-on counter before the job's buffers, [3] after them (19.2 MHz ticks; 0 when not stamped), [4] GPU level at submit | * the submitting pid << 32, [5] when the driver saw the job retired (us, the submission clock), [6] ring | (priority + 1) << 4 | GPU role << 8 | * ring the CP ran at submit << 16 | switch in flight << 20 | restarts that replayed it << 24, [7] when it was last replayed (us, 0 never). * native_adreno_tl_count is the total. */ /* diag=1, set by the service before any client attaches: the driver's diagnostics (the jobs' timeline, the CP stamps) run only then. */ extern "C" {volatile uint32_t native_adreno_diag;} extern "C" {volatile uint64_t (*native_adreno_tl)[8]=nullptr;volatile uint32_t native_adreno_tl_cap=0;volatile uint32_t native_adreno_tl_count;} /* The stall handler's actions (the newest 32): [0] time (us), [1] fence, [2] event (1 write pointer given again and translations dropped, 3 bus drain, * 4 snapshot, 0 the stall ended) | timed-out waits << 8, [3] time the fence had been pending (us), [4] read and write pointer before | after << 32 for * events 1 and 3, [5] client queue, [6] the job's first buffer handle | its buffer count << 48, [7] the sequence number. */ extern "C" {volatile uint64_t native_adreno_stall_log[32][8];volatile uint32_t native_adreno_stall_log_count;} extern "C" volatile uint32_t native_gpu_dcvs[15] __attribute__((weak_import)); extern "C" {volatile uint32_t native_adreno_job_time[8];volatile uint64_t native_adreno_long_jobs[8][8],native_adreno_slow_jobs[16][8]; volatile uint32_t native_adreno_stall_kick[26];} static uint64_t stall_kick_us; /* when the last first stage acted (us, the submission clock) */ /* Every retired job, whatever the diagnostics: the duration of a job from the CP's stamps when its ring wrote them, else from the * driver's clock (see native_adreno_job_time). The 8 longest since boot and the newest 16 over a 120 Hz frame name the job: its * process, queue, ring, priority, IBs and their size. */ void NativeAdrenoSubmit::noteJobTime(const Job &job,uint32_t stamped,uint64_t start,uint64_t end) { const uint64_t now=clock_now_?clock_now_(clock_cookie_):0; const unsigned r=job.ring<2?job.ring:0; const bool newest=!ring_retired_fence_[r] || (int32_t)(job.fence-ring_retired_fence_[r])>0; uint64_t us=0;unsigned source=0; /* status 31: attached, stamps on, page big enough, acquired, slot read; the stamps count 19.2 MHz ticks. A slot whose end is not * after its start, or that spans more than a minute, is another fence's (64 slots) and is not used. */ if(stamped==31 && end>start && end-start=job.submit_us) { uint64_t began=job.submit_us; if(newest && ring_retired_us_[r]>began)began=ring_retired_us_[r]; us=now-began;source=2; } if(now && newest) {ring_retired_fence_[r]=job.fence;ring_retired_us_[r]=now;} if(native_adreno_stall_kick[0]==job.fence && stall_kick_us && now>=stall_kick_us && !native_adreno_stall_kick[21]) native_adreno_stall_kick[21]=(uint32_t)(now-stall_kick_us>UINT32_MAX?UINT32_MAX:now-stall_kick_us); if(!source)return; const uint32_t n=native_adreno_job_time[0]; native_adreno_job_time[0]=n+1; if(source==1)native_adreno_job_time[1]=native_adreno_job_time[1]+1; if(us>8000)native_adreno_job_time[2]=native_adreno_job_time[2]+1; if(us>16000)native_adreno_job_time[3]=native_adreno_job_time[3]+1; if(us>33000)native_adreno_job_time[4]=native_adreno_job_time[4]+1; if(us>100000)native_adreno_job_time[5]=native_adreno_job_time[5]+1; if(us>native_adreno_job_time[6])native_adreno_job_time[6]=(uint32_t)(us>UINT32_MAX?UINT32_MAX:us); const uint64_t row[8]={(uint64_t)job.fence|(uint64_t)(job.word_count/4)<<32|(uint64_t)r<<40|(uint64_t)(uint8_t(job.priority+1)&0xff)<<44| (uint64_t)source<<52|(uint64_t)(job.replays<15?job.replays:15)<<56,us,job.ib_bytes, (uint64_t)(uint32_t)job.owner|(uint64_t)(job.role&0xff)<<32|(uint64_t)(job.level&0xffff)<<40,job.queue,job.submit_us, now>job.submit_us && job.submit_us?now-job.submit_us:0,n}; unsigned slot=0; for(unsigned i=1;i<8;++i)if(native_adreno_long_jobs[i][1]native_adreno_long_jobs[slot][1])for(unsigned k=0;k<8;++k)native_adreno_long_jobs[slot][k]=row[k]; if(us>8000) { const uint32_t m=native_adreno_job_time[7]; for(unsigned k=0;k<8;++k)native_adreno_slow_jobs[m&15][k]=row[k]; native_adreno_job_time[7]=m+1; } } IOReturn NativeAdrenoSubmit::poison() { poisoned_=true;return kIOReturnIOError; } NativeAdrenoSubmit::Job *NativeAdrenoSubmit::find(uint32_t fence) { if(fence)for(auto &job:jobs_)if(job.active && job.fence==fence)return &job; return nullptr; } NativeAdrenoSubmit::Job *NativeAdrenoSubmit::free() { for(auto &job:jobs_)if(!job.active)return &job; Job **items=IONewData(Job *,jobs_.count+1); Job *job=items?new Job:nullptr; if(!job) {if(items)IODeleteData(items,Job *,jobs_.count+1);return nullptr;} for(size_t i=0;ipoisoned() || ring->poisoned())return kIOReturnBadArgument; vm_=vm;ring_=ring;rings_[0]=ring;ring_count_=1;attached_=true;stats_={};table_vm_=vm; if(!native_adreno_tl && native_adreno_diag) { /* the jobs' timeline is a diagnostic (diag=1) */ const uint32_t cap=16384; volatile uint64_t (*memory)[8]=(volatile uint64_t (*)[8])IOMallocZero(sizeof(uint64_t)*8*cap); if(memory) {native_adreno_tl=memory;native_adreno_tl_cap=cap;} } return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::setTableSwitching(unsigned mode,IOReturn (*write)(void *,uint64_t),void *cookie) { if(!attached_ || poisoned_)return kIOReturnNotReady; if(mode>2 || (mode && !write) || outstanding())return kIOReturnBadArgument; table_mode_=mode;write_table_=write;table_cookie_=cookie;table_vm_=vm_; for(unsigned i=0;itableInstalled(vm_->ttbr0()); if(preemption() && ring(i)->resetExecutedTable(vm_->ttbr0()))return poison(); } return kIOReturnSuccess; } /* Makes vm the table the GPU runs for the next submission. Mode 1: once every earlier job has finished, the CPU writes TTBR0 (the bank * caches translations per ASID, so the switch needs no TLB operation). Mode 2: the ring's next prologue carries the switch, which the CP * runs after every earlier submission. */ IOReturn NativeAdrenoSubmit::useVm(NativeAdrenoVm *vm,const NativeAdrenoRingClock *clock,unsigned index) { if(!table_mode_)return kIOReturnSuccess; if(table_mode_==2) {ring(index)->useTable(vm->ttbr0());table_vm_=vm;return kIOReturnSuccess;} if(vm==table_vm_)return kIOReturnSuccess; if(!clock)return kIOReturnBadArgument; IOReturn status=waitAll(clock,2000000); if(status)return status; status=write_table_(table_cookie_,vm->ttbr0()); if(status)return poison(); native_adreno_table_switches[1]=native_adreno_table_switches[1]+1; table_vm_=vm;return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::forgetVm(NativeAdrenoVm *vm,const NativeAdrenoRingClock *clock,uint32_t timeout_us) { if(!attached_ || poisoned_)return kIOReturnNotReady; if(!vm || vm==vm_)return kIOReturnBadArgument; IOReturn status=waitAll(clock,timeout_us); if(status)return status; bool used=table_vm_==vm; for(unsigned i=0;itable()==vm->ttbr0(); if(!used)return kIOReturnSuccess; /* The GPU is idle and every queued switch has run: the CPU puts the default table back. */ status=write_table_?write_table_(table_cookie_,vm_->ttbr0()):kIOReturnNotReady; if(status)return poison(); native_adreno_table_switches[1]=native_adreno_table_switches[1]+1; for(unsigned i=0;itableInstalled(vm_->ttbr0()); if(preemption() && ring(i)->resetExecutedTable(vm_->ttbr0()))return poison(); } table_vm_=vm_;return kIOReturnSuccess; } /* The two rings share one fence numbering: the fence's low bits name its ring in a table the switch fills as each ring issues * (FenceMap entries; a fence is 32 bits, so the table is indexed by its low bits and checked against the fence it holds). Fences of * the first ring alone (no preemption) go to it without a lookup. */ void NativeAdrenoSubmit::noteFence(uint32_t fence,uint8_t index) { const uint32_t slot=fence&(FenceMap-1); fence_tag_[slot]=fence;fence_ring_[slot]=index; } NativeAdrenoRing *NativeAdrenoSubmit::ringOfFence(uint32_t fence) const { if(!preemption())return rings_[0]; for(const auto &job:jobs_)if(job.active && job.fence==fence)return rings_[job.ring]; if(!fence_tag_)return nullptr; const uint32_t slot=fence&(FenceMap-1); if(fence_tag_[slot]==fence) { const uint8_t index=fence_ring_[slot]; return indexlastSubmitted())<=0)return rings_[0]; return ring_count_>1?rings_[1]:rings_[0]; } /* The ring a submission of this priority goes to: high (0), normal-high (4) and real-time (5) to the first, the rest to the second. * A ring alone takes everything. */ unsigned NativeAdrenoSubmit::ringForPriority() const { if(!preemption() || preempt_dead_ || table_mode_!=2)return 0; /* Mode 2 separates normal work from requested high priorities. */ const int32_t p=next_priority_; return p==2 || p==3 || (preempt_setup_.mode>=2 && (p==1 || p<0))?1:0; } const NativeAdrenoSubmit::Job *NativeAdrenoSubmit::pendingOn(unsigned index) { NativeAdrenoRing *const r=ring(index); uint32_t done=0; if(!r || r->fenceValue(&done))return nullptr; const Job *first=nullptr; for(const auto &job:jobs_) if(job.active && job.ring==index && (int32_t)(job.fence-done)>0 && (!first || (int32_t)(job.fence-first->fence)<0))first=&job; return first; } /* Switch by requested priority. Queue FIFO is preserved at submission; explicit cross-queue waits are resolved by the client. * IB handles describe command storage, not the resources it accesses, so they cannot prove dependencies. */ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock) { (void)clock; if(!preemption())return; /* A switch the interrupt saw finish (or that finished while its handler could not take the lock) is taken first. */ if(preempt_.inFlight()) { if(!completeSwitch()) { /* The CP never finished a switch: no more are tried (the restart in recover() brings the CP back). */ if(preempt_.timedOut(PreemptSwitchTimeoutUs)) { uint32_t progress[6]={}; (void)ring(0)->progressState(progress); { /* The CP's state at a switch it never finished, for the first two: the stall words (as native_adreno_stall), the * switch control word and rings, then the ring's progress words. */ const uint32_t n=native_adreno_switch_timeouts; if(n<2) { uint32_t words[NativeAdrenoRingStallWords]={}; (void)ring(0)->stallState(words); volatile uint32_t *row=native_adreno_switch_timeout[n]; for(uint32_t i=0;isubmit_us>low_service_us_?first->submit_us:low_service_us_; if(!(now && since && now-since>PreemptAgingUs))continue; } /* The incoming ring's preemption record keeps what it was given at setup and what the CP saved into it when it switched the * ring out (read pointer, info, errno); only the write pointer the CP resumes at is given again before every switch. */ NativeAdrenoRing *const incoming=ring(i); if(!incoming || !preempt_setup_.pages.record_cpu[i] || !preempt_setup_.pages.smmu_cpu[i])continue; (void)incoming->writePreemptWptr(preempt_setup_.pages.record_cpu[i]); /* Restore the table at the saved CP position. Queued work may name a later VM. */ uint64_t ttbr0=0; if(incoming->executedTable(&ttbr0)) {markPreemptDead(5);return;} (void)NativeAdrenoPreempt::writeSmmuInfo(preempt_setup_.pages.smmu_cpu[i],ttbr0,uint32_t(ttbr0>>48),0); if(preempt_.trigger(i,true)) {aging_grant_=i==1 && high;ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;} /* The trigger itself refused (a register write failed): the second ring cannot be reached, so it is given up at once and the * jobs already queued on it are replayed by the restart the hang check asks for. */ preempt_.noteSkipped(); markPreemptDead(1); return; } } /* The interrupt's side: a completed switch makes its ring the loaded one and gives that ring's write pointer. */ bool NativeAdrenoSubmit::completeSwitch() { if(!preemption() || !preempt_.inFlight())return false; if(!preempt_.complete())return false; NativeAdrenoRing *const now=ring(preempt_.loaded()); for(unsigned i=0;isetCurrent(false); if(!now) {markPreemptDead(6);return false;} now->setCurrent(true); if(now->restoreWptr()) {markPreemptDead(6);return false;} preempt_.noteRestoredWptr(); if(!preempt_.releaseKeepalive()) {markPreemptDead(7);return false;} if(preempt_.loaded()==1) { low_service_us_=clock_now_?clock_now_(clock_cookie_):0; low_slice_until_=aging_grant_ && low_service_us_?low_service_us_+PreemptAgingSliceUs:0; } aging_grant_=false; return true; } bool NativeAdrenoSubmit::preemptIrq() { if(!completeSwitch())return false; // Work arriving during the switch is reconsidered as soon as it completes. driveSwitch(nullptr);return true; } bool NativeAdrenoSubmit::fenceProgress(uint32_t *submitted,uint32_t *retired,uint32_t *rings) { if(!attached_ || poisoned_ || !submitted || !retired)return false; uint32_t done[2]={},issued=fence_issued_; for(unsigned i=0;ifenceValue(&done[i]))return false; if((int32_t)(ring(i)->lastSubmitted()-issued)>0)issued=ring(i)->lastSubmitted(); } uint32_t prefix=issued; for(const auto &job:jobs_)if(job.active && (int32_t)(job.fence-done[job.ring])>0 && (int32_t)((job.fence-1)-prefix)<0)prefix=job.fence-1; if(rings){rings[0]=done[0];rings[1]=done[1];} *submitted=issued;*retired=prefix;return true; } bool NativeAdrenoSubmit::fencePending(uint32_t fence) { if(!attached_ || poisoned_ || !fence)return false; const Job *const job=find(fence); /* Two rings: a fence without a job was retired (wait() succeeds at once) or never handed out (wait() refuses it). */ if(preemption() && !job)return false; NativeAdrenoRing *const r=job?ring(job->ring):ringOfFence(fence); uint32_t done=0; if(!r || r->poisoned() || fence>r->lastSubmitted() || r->fenceValue(&done))return false; return donepreemption() || !low->preemption())return kIOReturnBadArgument; for(unsigned i=0;i(IOMallocZero(sizeof(uint32_t)*FenceMap)); if(!fence_ring_)fence_ring_=static_cast(IOMallocZero(FenceMap)); if(!fence_tag_ || !fence_ring_)return kIOReturnNoMemory; /* Each ring's SMMU info and record, filled once here (the switch rewrites them before every move). */ for(unsigned i=0;ittbr0(); if(!NativeAdrenoPreempt::writeSmmuInfo(setup.pages.smmu_cpu[i],ttbr0,uint32_t(ttbr0>>48),0))return kIOReturnBadArgument; if(r->resetExecutedTable(ttbr0))return kIOReturnIOError; (void)r->writePreemptRecord(setup.pages.record_cpu[i]); } rings_[0]=high;rings_[1]=low;ring_count_=2; gpuTraceEvent(GpuTraceConfig,0,2,setup.mode,table_mode_,PreemptSwitchTimeoutUs,PreemptAgingUs,PreemptAgingSliceUs); high->setCurrent(true);low->setCurrent(false); preempt_setup_=setup;low_service_us_=low_slice_until_=0;aging_grant_=false; const IOReturn attached_preempt=preempt_.attach(&setup.io,setup.pages,0,setup.now,setup.clock); if(attached_preempt) {ring_count_=1;rings_[1]=nullptr;return attached_preempt;} preempt_on_=true; return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::release(Job &job) { IOReturn status=kIOReturnSuccess; NativeAdrenoVm *vm=job.vm?job.vm:vm_; /* A process's address space that was taken out of use keeps every buffer it mapped (NativeAdrenoVm::failAt), so its job has * nothing to give back; the job ends here and the other processes go on. The kernel's own address space failing stops all. */ if(vm!=vm_ && vm->poisoned()) {++dropped_jobs_;native_adreno_recover_trace[0]=20;native_adreno_recover_trace[1]=dropped_jobs_;job={};return kIOReturnSuccess;} for(size_t i=0;iretireJobAfterCompletion(job.handles[i]); if(retired)status=retired;else ++stats_.references_retired; } if(status)return poison(); if(vm->acquireImported())return poison(); { uint64_t start=0,end=0; NativeAdrenoRing *r=ringOfFence(job.fence); const uint32_t stamped=r?r->timeline(job.fence,&start,&end):0; gpuTraceEvent(GpuTraceRetire,job.owner,job.fence,job.queue,job.ring,job.submit_us,start,end,stamped,job.replays); noteJobTime(job,stamped,start,end); if(native_adreno_tl_cap) { const uint32_t n=native_adreno_tl_count;volatile uint64_t *slot=native_adreno_tl[n&(native_adreno_tl_cap-1)]; slot[0]=(uint64_t)job.fence|(uint64_t)job.count<<32|(uint64_t)stamped<<58;slot[1]=job.submit_us; slot[2]=start;slot[3]=end;slot[4]=(uint64_t)job.level|(uint64_t)(uint32_t)job.owner<<32; slot[5]=clock_now_?clock_now_(clock_cookie_):0; slot[6]=(uint64_t)job.ring|(uint64_t)uint8_t(job.priority+1)<<4|(uint64_t)job.role<<8|(uint64_t)job.loaded<<16| (uint64_t)job.flight<<20|(uint64_t)job.replays<<24; slot[7]=job.replay_us; native_adreno_tl_count=n+1; } } job={};return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::detach() { if(poisoned_ || outstanding())return kIOReturnBusy; if(!attached_)return kIOReturnNotReady; attached_=false;vm_=nullptr;ring_=nullptr;return kIOReturnSuccess; } bool NativeAdrenoSubmit::queueOutstanding(uint64_t queue) const { return rangeOutstanding(queue,1); } bool NativeAdrenoSubmit::rangeOutstanding(uint64_t first,uint64_t count) const { for(const auto &job:jobs_)if(job.active && job.queue-firstunusable())return kIOReturnNotPermitted; if(!buffers || !count || count>SubmitMaxBuffers)return kIOReturnBadArgument; /* The phases' clock: each lap takes the time since the one before; an accepted submission adds them to native_adreno_submit_us. */ uint64_t lap_us=clock && clock->now_us?clock->now_us(clock->cookie):0,laps[6]={}; const uint64_t began_us=lap_us; auto lap=[&](unsigned slot) { if(!began_us)return; const uint64_t now=clock->now_us(clock->cookie); laps[slot]=now-lap_us;lap_us=now; }; native_adreno_submit_ibs[0]=native_adreno_submit_ibs[0]+1;native_adreno_submit_ibs[1]=native_adreno_submit_ibs[1]+(uint32_t)count; if(count>native_adreno_submit_ibs[2])native_adreno_submit_ibs[2]=(uint32_t)count; if(count>16)native_adreno_submit_ibs[3]=native_adreno_submit_ibs[3]+1; /* Every buffer is validated against the live VM before any reference. */ uint64_t *handles=scratch_handles_;size_t handle_count=0; uint32_t *words=scratch_words_;size_t word_count=0; uint64_t ib_bytes=0; static_assert(sizeof(scratch_words_)/sizeof(scratch_words_[0])>=SubmitMaxBuffers*(1+SubmitIbWords),"ring words"); for(size_t i=0;iSubmitMaxIbBytes) return kIOReturnBadArgument; uint64_t iova=0;size_t bytes=0;unsigned permissions=0; IOReturn status=vm_->describe(buffer.handle,&iova,&bytes,&permissions); if(status)return status; if(!(permissions&ArmSmmu4KRead) || buffer.iovabytes || buffer.bytes>bytes-(buffer.iova-iova))return kIOReturnBadArgument; uint32_t header=0; if(AdrenoPM4Type7(SubmitIndirectBuffer,SubmitIbWords,&header)!=AdrenoPM4Success) return poison(); words[word_count++]=header; words[word_count++]=(uint32_t)buffer.iova; words[word_count++]=(uint32_t)(buffer.iova>>32); words[word_count++]=(uint32_t)(buffer.bytes/4); ib_bytes+=buffer.bytes; bool known=false; for(size_t j=0;jholdJob(handles[i]); if(status) { while(i) {vm_->retireJobAfterCompletion(handles[--i]);++stats_.references_retired;} return status; } ++stats_.references_held; } lap(1); uint32_t value=0; /* int-070: the first IB word as the kernel's cacheable alias and DRAM see it before any maintenance, then DRAM after the CPU * caches over every mapped BO are cleaned and invalidated. The board shows the last submission's record on a stall. */ uint32_t cached=~0u,before=~0u,ignored=0,after=~0u; /* Write-combined stores this CPU made before the call (the client's command words) complete before anything below. */ __asm__ volatile("dsb sy":::"memory"); /* int-098/099: with write-combined user BOs and no maintenance the CP stopped at the end of the first IB, as in int-065..069 when an * IB sat in a CPU cache. While write-combined BOs are trusted, every line of this submission's IBs is compared as a CPU cache and * DRAM hold it (bounded). A dirty line means a client view still caches its BOs: maintain every BO from now on, this submission * included (the audit already wrote the IB lines back; the rest of the BOs may hold more). */ if(vm_->writeCombinedTrusted()) { /* int-100: the dirty IB page was back at XNU's default (cacheable) attribute. Write-combine every such page again first; the * pmap writes its lines back on the way, so the audit below should then find none. */ uint32_t reset=0; IOReturn audited=vm_->reassertWriteCombined(&reset); native_adreno_wc_audit[5]=native_adreno_wc_audit[5]+reset; uint32_t lines=0,dirty=0,stale=0;size_t budget=SubmitAuditBytes; /* The re-assertion above is the correction and runs every time; the audit only detects. It found no dirty line in int-141's * 133798 nor in i566's 2765132 (1358 audits of every eighth submission, about 2 ms each under the publish lock, so every eighth * frame of every client waited), so only the first 16 submissions are compared, and only with gpu-audit=1. */ ++audit_tick_; const bool audit_now=audit_tick_<=16 && auditsWanted(); if(!audit_now)budget=0; for(size_t i=0;iauditRange(buffers[i].iova,bytes,&lines,&dirty,&stale);budget-=bytes; } if(audit_now && audited!=kIOReturnUnsupported) { /* no board hook (host): nothing to compare */ native_adreno_wc_audit[0]=native_adreno_wc_audit[0]+1; native_adreno_wc_audit[1]=native_adreno_wc_audit[1]+lines; native_adreno_wc_audit[2]=native_adreno_wc_audit[2]+dirty; native_adreno_wc_audit[3]=native_adreno_wc_audit[3]+stale; if(audited)native_adreno_wc_audit[4]=audited; } if(dirty || (audited && audited!=kIOReturnUnsupported)) { vm_->distrustWriteCombined(); native_adreno_wc_distrusted=native_adreno_coherence[0]+1; } } /* The driver's own buffers (ring, memptrs, SQE, preemption records) are not cleaned at every submission: their writers keep the * GPU's view current (NativeAdrenoMemory::declareSynchronized). The first 8 submissions and every 1024th after them compare the * head of each one as a CPU cache and DRAM hold it (the ring lines the submission before stored are among them); a dirty line (a * writer that stored through a cacheable mapping and did not clean) ends that trust for good, and from this submission on they * are cleaned every time. */ if((native_adreno_submit_ibs[0]<=8 || !(native_adreno_submit_ibs[0]&1023)) && !native_adreno_sync_audit[4] && auditsWanted()) { uint32_t lines=0,dirty=0,stale=0; const IOReturn audited=vm_->auditSynchronized(&lines,&dirty,&stale); if(audited!=kIOReturnUnsupported) { /* no board hook (host): nothing to compare */ native_adreno_sync_audit[0]=native_adreno_sync_audit[0]+1; native_adreno_sync_audit[1]=native_adreno_sync_audit[1]+lines; native_adreno_sync_audit[2]=native_adreno_sync_audit[2]+dirty; native_adreno_sync_audit[3]=native_adreno_sync_audit[3]+stale; if(audited)native_adreno_sync_audit[5]=(uint32_t)audited; if(dirty || audited)native_adreno_sync_audit[4]=native_adreno_sync_audit[0]; } } lap(2); /* The coherence sample reads the first IB's word through a fresh uncached mapping before and after the publication: every 64th * submission only (two such mappings per submission were CPU time on every frame). */ static uint32_t coherence_turn; const bool sample=!(coherence_turn++&63); if(sample)(void)vm_->sampleWord(buffers[0].iova,&cached,&before); IOReturn status=vm_->publishMapped(); if(!status && sample) { (void)vm_->sampleWord(buffers[0].iova,&ignored,&after); native_adreno_coherence[0]=native_adreno_coherence[0]+1; native_adreno_coherence[1]=cached;native_adreno_coherence[2]=before;native_adreno_coherence[3]=after; if(cached!=before || before!=after) { if(!native_adreno_coherence[4]) { native_adreno_coherence[5]=cached;native_adreno_coherence[6]=before;native_adreno_coherence[7]=after; } native_adreno_coherence[4]=native_adreno_coherence[4]+1; } } /* No TLB invalidation before a submission: a map fills only invalid leaves (no TLB holds an invalid descriptor), an unmap drops * its own pages before it returns, and the tables above the leaves never change. The TLBIALL here emptied the GPU's TLB on every * submission (10944 in i568), so each frame began with page walks. A stalled fence still invalidates (below). */ if(status) { for(size_t i=0;iretireJobAfterCompletion(handles[i]);++stats_.references_retired;} return status; } lap(3); /* The ring this job goes to (the first ring alone unless the board turned the switch on). */ unsigned index=preemption()?ringForPriority():0; /* A priority change cannot overtake this queue's earlier submissions. It moves * rings once its outstanding prefix has completed. Queue IDs are globally scoped. */ for(const auto &earlier:jobs_) if(earlier.active && earlier.queue==queue) {index=earlier.ring;break;} NativeAdrenoRing *const target=ring(index); { const uint32_t slot=native_adreno_ib_log_count&3;uint32_t head[28]={}; if(native_adreno_diag)(void)vm_->headWords(buffers[0].handle,buffers[0].iova,head,28); native_adreno_ib_log[slot][0]=target->lastSubmitted()+1;native_adreno_ib_log[slot][1]=(uint32_t)buffers[0].iova; native_adreno_ib_log[slot][2]=(uint32_t)(buffers[0].iova>>32);native_adreno_ib_log[slot][3]=(uint32_t)buffers[0].bytes; for(unsigned i=0;i<28;++i)native_adreno_ib_log[slot][4+i]=head[i]; native_adreno_ib_log_count=native_adreno_ib_log_count+1; } lap(4); target->useContext(ring(0)->context()); status=useVm(vm,clock,index); if(status) { for(size_t i=0;iretireJobAfterCompletion(handles[i]);++stats_.references_retired;} return status; } /* One numbering for both rings: a fence tells its own ring, and a switch's arrival retires the jobs queued behind it. */ /* The board's earlier submissions (the CP init and the probes) numbered themselves on the first ring: the next number is above * every ring's newest, and above the last this object handed out. */ uint32_t base=fence_issued_; for(unsigned i=0;ilastSubmitted()-base)>0)base=ring(i)->lastSubmitted(); const uint32_t number=(value=base)+1; status=preemption()?target->submitFramed(words,word_count,number,&value):target->submitFramed(words,word_count,&value); if(status) { if(!value)fence_issued_=base; /* A poisoned ring may already have consumed the WPTR write, so the * referenced buffers cannot be retired here. */ if(target->poisoned()) { *job=Job{};++stats_.poisoned_jobs; for(size_t i=0;ihandles[i]=handles[i]; job->count=handle_count;job->queue=queue;job->active=true;job->vm_next=vm_->nextHandle();job->vm=vm; return poison(); } while(handle_count) {vm_->retireJobAfterCompletion(handles[--handle_count]);++stats_.references_retired;} /* No room in the ring yet: the caller waits for an earlier job and submits again. */ return status==kIOReturnNoResources?kIOReturnBusy:status; } for(size_t i=0;ihandles[i]=handles[i]; job->count=handle_count;job->fence=value;job->queue=queue;job->active=true;job->vm_next=vm_->nextHandle();job->vm=vm; job->context=target->context();job->word_count=word_count;job->ring=uint8_t(index); job->priority=int8_t(next_priority_);job->role=uint8_t(next_role_); job->ib_bytes=(uint32_t)(ib_bytes>UINT32_MAX?UINT32_MAX:ib_bytes); job->loaded=uint8_t(preemption()?preempt_.loaded():0);job->flight=preemption() && preempt_.inFlight()?1:0; if(preemption()) {fence_issued_=value;noteFence(value,uint8_t(index));} job->submit_us=clock && clock->now_us?clock->now_us(clock->cookie):0; if(clock && clock->now_us) {clock_now_=clock->now_us;clock_cookie_=clock->cookie;} job->level=&native_gpu_dcvs?native_gpu_dcvs[1]:0; job->owner=next_owner_;next_owner_=0; gpuTraceEvent(GpuTraceSubmit,job->owner,value,queue,index,job->priority,job->role,job->loaded,job->flight,job->context,job->submit_us,handle_count); for(size_t i=0;iwords[i]=words[i]; ++stats_.submissions; if(outstanding()>stats_.peak_jobs)stats_.peak_jobs=outstanding(); *fence=value; driveSwitch(clock); lap(5); if(began_us) { native_adreno_submit_us[0]=native_adreno_submit_us[0]+1; for(unsigned i=1;i<6;++i)native_adreno_submit_us[i]=native_adreno_submit_us[i]+laps[i]; native_adreno_submit_us[6]=native_adreno_submit_us[6]+(lap_us-began_us); } return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::observeFence(uint32_t fence,const NativeAdrenoRingClock *clock, uint32_t timeout_us) { if(!attached_ || poisoned_)return kIOReturnNotReady; if(!fence)return kIOReturnBadArgument; if(preemption() && fence_issued_ && (int32_t)(fence-fence_issued_)<=0 && !find(fence))return kIOReturnSuccess; driveSwitch(clock); NativeAdrenoRing *const r=ringOfFence(fence); if(!r)return kIOReturnBadArgument; const NativeAdrenoRingFence result=waitOnRing(r,false,fence,clock,timeout_us,nullptr); if(result==NativeAdrenoRingFencePoisoned)return poison(); if(result==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument; if(result==NativeAdrenoRingFenceTimeout)return kIOReturnTimeout; return kIOReturnSuccess; } /* A stage runs once per stalled fence: when the stage before it has run and the fence has been pending long enough. */ static bool stage_due(uint32_t stage,uint32_t before,uint64_t pending,uint64_t after_us) { return stage==before && pending>=after_us; } /* The CP snapshot is taken at 50 ms and at 1 s of one stall. */ static uint32_t stall_snapshots; static bool snapshot_due(uint32_t *taken,uint64_t pending) { const uint32_t want=pending>=1000000?2:pending>=50000?1:0; if(want<=*taken)return false; *taken=want;return true; } void NativeAdrenoSubmit::noteStall(uint32_t observed,const NativeAdrenoRingClock *clock,uint32_t waited) { const uint32_t next=observed+1; /* the fence the CP is working on */ const uint64_t now=clock && clock->now_us?clock->now_us(clock->cookie):0; /* The ring that fence's job ran on; the first ring when the fence is not one of this submit object's. */ NativeAdrenoRing *const stalled=ringOfFence(next)?ringOfFence(next):ring(0); auto log=[&](uint32_t fence,uint32_t event,uint64_t pending,uint32_t before_r,uint32_t before_w) { const uint32_t n=native_adreno_stall_log_count;volatile uint64_t *slot=native_adreno_stall_log[n&31]; const Job *owner=find(fence); slot[0]=now;slot[1]=fence;slot[2]=(uint64_t)event|(uint64_t)stall_timeouts<<8;slot[3]=pending; slot[4]=(uint64_t)before_r|(uint64_t)before_w<<16|(uint64_t)stalled->readPointer()<<32|(uint64_t)(stalled->writePointer()&0xffff)<<48; slot[5]=owner?owner->queue:0;slot[6]=owner?(owner->handles[0]&0xffffffffffffull)|(uint64_t)owner->count<<48:0;slot[7]=n; native_adreno_stall_log_count=n+1; }; if(next!=stall_fence) { if(stall_timeouts && now) { const uint64_t ms=(stall_last_us-stall_began_us)/1000; uint32_t bucket=0;while(bucket<7 && ms>=(1ull<=1000)log(stall_fence,0,stall_last_us-stall_began_us,0,0); /* shorter ends are ordinary jobs */ } stall_began_us=now;stall_snapshots=0; /* The CP moved on: what the stall that just ended took to clear. [9] first-stage recoveries, [10] second, [11] third, [12] timeouts of the longest. */ if(stall_stage && stall_timeouts) { native_adreno_unstick[8+stall_stage]=native_adreno_unstick[8+stall_stage]+1; if(stall_timeouts>native_adreno_unstick[12])native_adreno_unstick[12]=stall_timeouts; } stall_fence=next;stall_timeouts=0;stall_stage=0; /* Where the CP was at the fence's first timed-out wait, to tell at the first stage whether it moved since. */ if(stalled->progressState(stall_begin_cp_))for(unsigned i=0;i<6;++i)stall_begin_cp_[i]=~0u; if(stalled->bvProgressState(stall_begin_bv_))for(unsigned i=0;i<5;++i)stall_begin_bv_[i]=~0u; } stall_last_us=now; const uint64_t pending=now-stall_began_us; const bool kick_due=now && stage_due(stall_stage,0,pending,50000); /* Before the first stage acts: where the CP is now and 500 us later. Unchanged since the first wait and over the 500 us, the CP * stopped; changed, it is still working through the job (a long job, not a lost write pointer). The snapshot below is taken * before the stage too, so its registers are the stall's and not the stage's result. */ bool working=false; if(kick_due) { uint32_t at[6]={},later[6]={},bv[5]={},bvLater[5]={}; if(stalled->progressState(at))for(unsigned k=0;k<6;++k)at[k]=~0u; if(stalled->bvProgressState(bv))for(unsigned k=0;k<5;++k)bv[k]=~0u; IODelay(500); if(stalled->progressState(later))for(unsigned k=0;k<6;++k)later[k]=~0u; if(stalled->bvProgressState(bvLater))for(unsigned k=0;k<5;++k)bvLater[k]=~0u; uint32_t moved=0; for(unsigned k=0;k<6;++k) { if(at[k]!=stall_begin_cp_[k])moved|=1; if(later[k]!=at[k])moved|=2; native_adreno_stall_kick[2+k]=stall_begin_cp_[k];native_adreno_stall_kick[8+k]=at[k];native_adreno_stall_kick[14+k]=later[k]; } /* The BV thread: with concurrent binning BR stands at a wait for BV (CP_BV_BR_COUNT_OPS, a BV timestamp) while the visibility * pass runs, so BR's words alone would call a working GPU stopped and the stages below would act on it. */ for(unsigned k=0;k<5;++k) { if(bv[k]!=stall_begin_bv_[k])moved|=4; if(bvLater[k]!=bv[k])moved|=8; } const Job *const owner=find(next); native_adreno_stall_kick[20]=moved;native_adreno_stall_kick[21]=0; native_adreno_stall_kick[22]=owner?(uint32_t)owner->owner:0; native_adreno_stall_kick[23]=owner?(uint32_t)(owner->word_count/4)|(uint32_t)owner->ring<<16|(uint32_t)uint8_t(owner->priority+1)<<24:0; native_adreno_stall_kick[24]=owner?owner->ib_bytes:0; native_adreno_stall_kick[1]=pending>UINT32_MAX?UINT32_MAX:(uint32_t)pending; native_adreno_stall_kick[0]=stall_fence; native_adreno_stall_kick[25]=native_adreno_stall_kick[25]+1; /* The CP moved: a long job, not a stop (every kick of four runs found it moving, tiled passes of 100 to 160 ms). Nothing is * done to a working GPU: the stages below count their time from this sample, so they act only after that long without * progress. */ if(moved) { working=true;stall_began_us=now; for(unsigned k=0;k<6;++k)stall_begin_cp_[k]=later[k]; for(unsigned k=0;k<5;++k)stall_begin_bv_[k]=bvLater[k]; } } ++stall_timeouts; if(now && snapshot_due(&stall_snapshots,pending))snapshot(next,waited,now,pending,stalled); /* A fence that stops while the CP sits idle with work in the ring (int-317 hung with RBBM idle, RPTR short of WPTR, both threads at the same * cache clean) is nearly always a write pointer the CP never took: giving it again after a few waits ends it in a couple of milliseconds * (i343: 206 of 213 stalls ended at the second stage, 60 to 85 ms each, most of it the 100 waits before the write). After 10 waits now: * i469 stalled 143 times in about 200 s, each a visible hitch. The same value written again changes nothing when the CP is busy. */ /* After 10 waits the translations are dropped and the write pointer is given again. i494/i495 measured what ends these: a job lasts about * 5 ms against 1 ms waits, so most "stalls" are ordinary jobs still running; the write pointer is never dropped (fenced writes saw no * drop) and invalidating at map time changed nothing. i553: the handler ran on 6712 of 6911 jobs. It now waits for 50 ms without * progress, measured in time (a timed-out wait is one poll, 100 us or less), and so does the snapshot below. */ if(kick_due && !working) { native_adreno_unstick[0]=native_adreno_unstick[0]+1;native_adreno_unstick[1]=stall_fence;stall_stage=1; const uint32_t r0=stalled->readPointer(),w0=stalled->writePointer()&0xffff; (void)vm_->invalidateTlb();(void)stalled->kick(); stall_kick_us=now; log(stall_fence,1,pending,r0&0xffff,w0); } /* Still stopped well after that: the way a6xx drains the bus before a reset, which lets a write that was never acknowledged end. */ if(now && stage_due(stall_stage,1,pending,200000)) { uint32_t ack[2]={}; native_adreno_unstick[4]=native_adreno_unstick[4]+1;native_adreno_unstick[7]=stall_fence;stall_stage=3; const uint32_t r0=stalled->readPointer(),w0=stalled->writePointer()&0xffff; (void)stalled->drainBus(ack); native_adreno_unstick[5]=ack[0];native_adreno_unstick[6]=ack[1]; log(stall_fence,3,pending,r0&0xffff,w0); } } /* The CP snapshot of a stalled fence (native_adreno_stall, _stall_mem, _stall_ring, and a stall log entry). */ void NativeAdrenoSubmit::snapshot(uint32_t next,uint32_t waited,uint64_t now,uint64_t pending,NativeAdrenoRing *stalled) { uint32_t words[NativeAdrenoRingStallWords]={},fault[4]={}; if(stalled->stallState(words))return; { const uint32_t n=native_adreno_stall_log_count;volatile uint64_t *slot=native_adreno_stall_log[n&31]; const Job *owner=find(next); slot[0]=now;slot[1]=next;slot[2]=4u|(uint64_t)stall_timeouts<<8;slot[3]=pending; slot[4]=(uint64_t)(words[8]&0xffff)|(uint64_t)(words[9]&0xffff)<<16|(uint64_t)stalled->readPointer()<<32|(uint64_t)(stalled->writePointer()&0xffff)<<48; slot[5]=owner?owner->queue:0;slot[6]=owner?(owner->handles[0]&0xffffffffffffull)|(uint64_t)owner->count<<48:0;slot[7]=n; native_adreno_stall_log_count=n+1; } (void)vm_->faultState(fault); for(uint32_t i=0;i<4;++i)native_adreno_stall_mem[i]=fault[i]; uint64_t ib=0;uint32_t size=0; /* The IBs are the stalled job's: read them through its address space (the kernel's maps none of a client's BOs). */ const Job *job=find(next); const NativeAdrenoVm *space=job && job->vm?job->vm:vm_; if(!stalled->ibBefore(words[8],&ib,&size))peekIb(space,ib,size,words[12],&native_adreno_stall_mem[4]); else peekFetch(space,(uint64_t)words[11]<<32|words[10],&native_adreno_stall_mem[4]); if(!stalled->ibBefore(words[16],&ib,&size))peekIb(space,ib,size,words[18],&native_adreno_stall_mem[4+17]); else peekFetch(space,(uint64_t)words[11]<<32|words[17],&native_adreno_stall_mem[4+17]); /* BV IB1 base: the low word only */ peekFetch(space,(uint64_t)words[21]<<32|words[20],&native_adreno_stall_mem[4+34]); if(job && job->word_count)findIb2(space,job->words,job->word_count,words[13]); { uint64_t brIb=(uint64_t)words[11]<<32|words[10],bvIb=(uint64_t)words[11]<<32|words[17]; if(!stalled->ibBefore(words[8],&ib,&size))brIb=ib; if(!stalled->ibBefore(words[16],&ib,&size))bvIb=ib; native_adreno_stall_mem[55]=job?(uint32_t)job->vm_next:0; native_adreno_stall_mem[56]=(uint32_t)space->handleAt(brIb); native_adreno_stall_mem[57]=(uint32_t)space->handleAt(bvIb); native_adreno_stall_mem[58]=(uint32_t)space->handleAt((uint64_t)words[21]<<32|words[20]); } { const Job *const asked=find(waited); NativeAdrenoRing *const asked_ring=asked?ring(asked->ring):ringOfFence(waited); uint32_t done[2]={},unfinished[2]={}; for(unsigned i=0;ifenceValue(&done[i]); for(const auto &other:jobs_)if(other.active && other.ring<2 && (int32_t)(other.fence-done[other.ring])>0)++unfinished[other.ring]; native_adreno_stall_ring[1]=waited; native_adreno_stall_ring[2]=ring_count_>1 && asked_ring==rings_[1]?1:0; native_adreno_stall_ring[3]=preemption()?preempt_.loaded():0; native_adreno_stall_ring[4]=preemption() && preempt_.inFlight()?1:0; native_adreno_stall_ring[5]=unfinished[0];native_adreno_stall_ring[6]=unfinished[1]; native_adreno_stall_ring[7]=job?1u|(uint32_t)job->ring<<8|(uint32_t)uint8_t(job->priority+1)<<16:0; native_adreno_stall_ring[8]=asked && asked->submit_us && now>asked->submit_us? (now-asked->submit_us>UINT32_MAX?UINT32_MAX:(uint32_t)(now-asked->submit_us)):0; native_adreno_stall_ring[9]=pending>UINT32_MAX?UINT32_MAX:(uint32_t)pending; native_adreno_stall_ring[0]=next; } for(uint32_t i=0;inow_us) return held?r->wait(fence,clock,timeout_us,observed):r->observeFence(fence,clock,timeout_us,observed); const uint64_t start=clock->now_us(clock->cookie); for(;;) { const NativeAdrenoRingFence state=held?r->wait(fence,clock,0,observed):r->observeFence(fence,clock,0,observed); if(state!=NativeAdrenoRingFenceTimeout)return state; if(clock->now_us(clock->cookie)-start>=timeout_us)return NativeAdrenoRingFenceTimeout; if(clock->idle && clock->idle(clock->cookie))return NativeAdrenoRingFenceTimeout; driveSwitch(clock); } } IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *clock, uint32_t timeout_us) { if(!attached_ || poisoned_)return kIOReturnNotReady; Job *job=find(fence); if(preemption() && fence && fence_issued_ && (int32_t)(fence-fence_issued_)<=0 && !job)return kIOReturnSuccess; driveSwitch(clock); uint32_t observed=0; native_adreno_fence_wait[0]=native_adreno_fence_wait[0]+1;native_adreno_unstick[13]=timeout_us; NativeAdrenoRing *const r=job?ring(job->ring):ringOfFence(fence); if(!r)return kIOReturnBadArgument; if(!job) { native_adreno_fence_wait[6]=native_adreno_fence_wait[6]+1; const NativeAdrenoRingFence passed=waitOnRing(r,false,fence,clock,timeout_us,&observed); if(passed==NativeAdrenoRingFencePoisoned)return poison(); if(passed==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;} if(passed==NativeAdrenoRingFenceTimeout) { ++stats_.timeouts; native_adreno_fence_wait[1]=native_adreno_fence_wait[1]+1;native_adreno_fence_wait[2]=fence;native_adreno_fence_wait[3]=observed; native_adreno_fence_wait[4]=r->lastSubmitted();native_adreno_fence_wait[5]=r->completedFence(); noteStall(observed,clock,fence); return kIOReturnTimeout; } return kIOReturnSuccess; } const NativeAdrenoRingFence result=waitOnRing(r,true,fence,clock,timeout_us,&observed); if(result==NativeAdrenoRingFencePoisoned)return poison(); if(result==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;} if(result==NativeAdrenoRingFenceTimeout) { ++stats_.timeouts; native_adreno_fence_wait[1]=native_adreno_fence_wait[1]+1;native_adreno_fence_wait[2]=fence;native_adreno_fence_wait[3]=observed; native_adreno_fence_wait[4]=r->lastSubmitted();native_adreno_fence_wait[5]=r->completedFence(); noteStall(observed,clock,fence); return kIOReturnTimeout; } IOReturn status=release(*job); if(status)return status; ++stats_.completed; return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::waitAll(const NativeAdrenoRingClock *clock,uint32_t timeout_us) { if(!attached_ || poisoned_)return kIOReturnNotReady; while(outstanding()) { Job *oldest=nullptr; for(auto &job:jobs_)if(job.active && (!oldest || job.fence-oldest->fence>0x80000000u))oldest=&job; const IOReturn status=wait(oldest->fence,clock,timeout_us); if(status)return status; } return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::waitRange(uint64_t first,uint64_t count,const NativeAdrenoRingClock *clock,uint32_t timeout_us) { if(!attached_ || poisoned_)return kIOReturnNotReady; while(rangeOutstanding(first,count)) { Job *oldest=nullptr; for(auto &job:jobs_)if(job.active && job.queue-firstfence>0x80000000u))oldest=&job; const IOReturn status=wait(oldest->fence,clock,timeout_us); if(status)return status; } return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::retireCompleted() { if(!attached_ || poisoned_)return kIOReturnNotReady; for(auto &job:jobs_) { if(!job.active)continue; NativeAdrenoRing *const r=ringOfFence(job.fence); const NativeAdrenoRingFence state=r?r->observeFence(job.fence,nullptr,0,nullptr):NativeAdrenoRingFenceArgument; if(state==NativeAdrenoRingFencePoisoned)return poison(); if(state!=NativeAdrenoRingFenceComplete)continue; const IOReturn status=release(job); if(status)return status; ++stats_.completed; } /* A ring that went idle leaves the CP to the other ring's queued work (driveSwitch decides by priority). */ driveSwitch(nullptr); return kIOReturnSuccess; } bool NativeAdrenoSubmit::madeProgress() { if(progress_retries_>=3)return false; uint32_t state[6]={},bv[5]={}; if(ring(0)->progressState(state) || ring(0)->bvProgressState(bv))return false; bool progress=false; for(unsigned i=0;i<6;++i)if(state[i]!=last_cp_[i])progress=true; for(unsigned i=0;i<5;++i)if(bv[i]!=last_bv_[i])progress=true; /* BR may be waiting for a visibility pass BV still runs */ for(unsigned i=0;i<6;++i)last_cp_[i]=state[i]; for(unsigned i=0;i<5;++i)last_bv_[i]=bv[i]; if(!progress)return false; ++progress_retries_; return true; } IOReturn NativeAdrenoSubmit::hangCheck(bool *hung) { if(!hung)return kIOReturnBadArgument;*hung=false; if(!attached_ || poisoned_)return kIOReturnNotReady; /* The newest submission of either ring: each ring's fence numbers rise, so the later of the two is the device's newest. */ const uint32_t last0=ring(0)->lastSubmitted(),last1=preemption()?ring(1)->lastSubmitted():0; const uint32_t last=(int32_t)(last1-last0)>0?last1:last0; if(!last)return kIOReturnSuccess; /* A job on a ring that the CP cannot reach: switching was given up (dead), or a second-ring job has waited so long that the * switch is not coming. The first ring's progress hides this from the check below, so it is judged on its own. */ if(preemption()) { const uint64_t now=clock_now_?clock_now_(clock_cookie_):0; /* A busy high ring starves the low one by design (driveSwitch lets it through after PreemptAgingUs); only a low-ring job that waits * with the high ring idle and no switch in flight is a CP that cannot be reached. */ const bool reachable=!preempt_.inFlight() && !pendingOn(0); /* The same holds for a first-ring job once switching is given up with the CP left on the second ring (a switch back that * the CP never finished): the job waits for a CP that will not come, and is not to blame for it. */ for(auto &job:jobs_)if(job.active && job.ring!=preempt_.loaded() && (job.ring==1 || preempt_dead_)) { const bool aged=job.ring==1 && reachable && now && job.submit_us && now-job.submit_us>PreemptStrandedUs; if(preempt_dead_ || aged) { markPreemptDead(preempt_dead_?dead_reason_:3); stranded_=true;*hung=true;++stats_.hangs; return kIOReturnSuccess; } } } NativeAdrenoRing *const r=ringOfFence(last)?ringOfFence(last):ring(0); uint32_t fence=0; if(r->observeFence(last,nullptr,0,&fence)==NativeAdrenoRingFencePoisoned)return poison(); if(fence!=hang_fence_) {hang_fence_=fence;progress_retries_=0;return kIOReturnSuccess;} if(fence>=last || madeProgress())return kIOReturnSuccess; progress_retries_=0;++stats_.hangs;*hung=true; return kIOReturnSuccess; } IOReturn NativeAdrenoSubmit::recover(NativeAdrenoRestart restart,void *cookie,const NativeAdrenoRingClock *clock, NativeAdrenoRecovery *out) { if(!out)return kIOReturnBadArgument;*out={}; if(!attached_)return kIOReturnNotReady; if(!restart)return kIOReturnUnsupported; /* The newest fence of either ring and what the CP last wrote on the ring the restart left it on (the first ring). */ const uint32_t last0=ring(0)->lastSubmitted(),last1=preemption()?ring(1)->lastSubmitted():0; const uint32_t last=(int32_t)(last1-last0)>0?last1:last0; uint32_t fence=0; const bool stranded=stranded_; stranded_=false;last_stranded_=stranded; if(stranded) { /* No job is to blame: the second ring's work never got a CP. What each ring finished retires by that ring's own fence, and the * replay starts below the oldest job left, whichever ring it sat on. */ for(auto &job:jobs_)if(job.active) { uint32_t done=0; NativeAdrenoRing *const r=ringOfFence(job.fence)?ringOfFence(job.fence):ring(0); if(!r->fenceValue(&done) && (int32_t)(done-job.fence)>=0) { IOReturn status=release(job);if(status) {native_adreno_recover_trace[0]=7;native_adreno_recover_trace[1]=uint32_t(status);return status;} ++stats_.completed; } } bool any=false;uint32_t oldest=last; for(auto &job:jobs_)if(job.active && (!any || (int32_t)(job.fence-oldest)<0)) {oldest=job.fence;any=true;} fence=any?oldest-1:last; ++rescues_; native_adreno_recover_trace[2]=fence;native_adreno_recover_trace[3]=last; } else if(preemption()) { /* Two rings: a job is done when its own ring's fence says so (ring 0's fence alone knows nothing of what ring 1 finished, and * would name a finished job as the hung one). The oldest job left is the one the CP was running, the others wait behind it. */ bool any=false;uint32_t oldest=last; for(auto &job:jobs_)if(job.active) { uint32_t done=0; NativeAdrenoRing *const r=ringOfFence(job.fence)?ringOfFence(job.fence):ring(0); if(!r->fenceValue(&done) && (int32_t)(done-job.fence)>=0) { IOReturn status=release(job);if(status) {native_adreno_recover_trace[0]=8;native_adreno_recover_trace[1]=uint32_t(status);return status;} ++stats_.completed; } else if(!any || (int32_t)(job.fence-oldest)<0) {oldest=job.fence;any=true;} } fence=any?oldest-1:last; native_adreno_recover_trace[2]=fence;native_adreno_recover_trace[3]=last; } else { const IOReturn fr=ring(0)->fenceValue(&fence); native_adreno_recover_trace[2]=fence;native_adreno_recover_trace[3]=last; if(fr || fence>last) {native_adreno_recover_trace[0]=1;native_adreno_recover_trace[1]=uint32_t(fr);return poison();} /* What already completed retires as usual. */ for(auto &job:jobs_)if(job.active && job.fence<=fence) { IOReturn status=release(job);if(status)return status; ++stats_.completed; } } const uint32_t hung=fencefence=hung; if(guilty) { NativeAdrenoVm *vm=guilty->vm?guilty->vm:vm_; vm->noteHang(vm!=vm_);out->guilty=vm; } /* A hang inside a second-ring job is blamed on the second ring: it is given up (reason 4), the replay and everything after run on the first. */ if(guilty && guilty->ring==1)markPreemptDead(4); recovery_info_[0]=guilty?uint32_t(guilty->ring):9;recovery_info_[1]=last0;recovery_info_[2]=last1;recovery_info_[3]=preemption()?preempt_.loaded():0; IOReturn status=restart(cookie,ring(0),hung); out->status=status; if(status) {native_adreno_recover_trace[0]=2;native_adreno_recover_trace[1]=uint32_t(status);return poison();} poisoned_=false;hang_fence_=hung;progress_retries_=0; if(guilty) {status=release(*guilty);if(status) {native_adreno_recover_trace[0]=9;native_adreno_recover_trace[1]=uint32_t(status);return status;}} /* The GPU is idle after the restart's CP init: the CPU installs the kernel table, and each replayed job switches to its own. */ if(table_mode_) { if(!write_table_) {native_adreno_recover_trace[0]=3;native_adreno_recover_trace[1]=0;return poison();} status=write_table_(table_cookie_,vm_->ttbr0()); if(status) {native_adreno_recover_trace[0]=3;native_adreno_recover_trace[1]=uint32_t(status);return poison();} if(table_mode_==2)ring(0)->tableInstalled(vm_->ttbr0()); table_vm_=vm_; } /* The restart's CP init ran on the first ring: it is the one the CP runs and every replayed job goes there, whatever ring it * came from (the words are ring-agnostic and their fence numbers are kept, so each waiter still finds its value). */ if(preemption()) { preempt_.restart(0);low_service_us_=low_slice_until_=0;aging_grant_=false; ring(0)->setCurrent(true); (void)ring(1)->restartSecondary(hung); ring(1)->setCurrent(false); for(unsigned i=0;itableInstalled(vm_->ttbr0()); if(ring(i)->resetExecutedTable(vm_->ttbr0())) {native_adreno_recover_trace[0]=10;native_adreno_recover_trace[1]=i;return poison();} } } uint32_t nop=0; if(AdrenoPM4Type7(0x10,0,&nop)!=AdrenoPM4Success) {native_adreno_recover_trace[0]=4;return poison();} /* CP_NOP */ for(uint32_t next=hung+1;next && next<=last;++next) { Job *job=find(next); NativeAdrenoVm *vm=job && job->vm?job->vm:vm_; const bool run=job && !vm->unusable() && job->word_count; status=useVm(vm,clock); if(status) {native_adreno_recover_trace[0]=5;native_adreno_recover_trace[1]=uint32_t(status);return poison();} ring(0)->useContext(job?job->context:0); uint32_t value=0; status=preemption()?ring(0)->submitFramed(run?job->words:&nop,run?job->word_count:1,next,&value): ring(0)->submitFramed(run?job->words:&nop,run?job->word_count:1,&value); if(status || value!=next) {native_adreno_recover_trace[0]=6;native_adreno_recover_trace[1]=status?uint32_t(status):value;return poison();} if(job) { if(job->replays<255)++job->replays; job->replay_us=clock && clock->now_us?clock->now_us(clock->cookie):0; } if(job && preemption()) {job->ring=0;noteFence(next,0);} ++out->replayed; } ++stats_.restarts; return kIOReturnSuccess; } void NativeAdrenoSubmit::markPreemptDead(uint32_t reason) { if(!preempt_dead_) {preempt_dead_=true;dead_reason_=reason;} }