diff --git a/NativeAdrenoPublish.cpp b/NativeAdrenoPublish.cpp index f3e781f..099b632 100644 --- a/NativeAdrenoPublish.cpp +++ b/NativeAdrenoPublish.cpp @@ -1137,11 +1137,10 @@ extern "C" bool native_adreno_preempt_irq() * for the fences handed out before the swap arrived. False before the GPU is published. */ extern "C" bool native_adreno_fences(uint32_t *submitted,uint32_t *retired) { - if(!g.published || !g.ring || !submitted || !retired)return false; - uint32_t value=0; - if(g.ring->fenceValue(&value))return false; - *submitted=g.ring->lastSubmitted();*retired=value; - return true; + if(!g.published || !g.submit || !submitted || !retired)return false; + if(!IOLockTryLock(g.lock))return false; + const bool known=g.submit->fenceProgress(submitted,retired); + IOLockUnlock(g.lock);return known; } /* Publishes the service once. Everything passed in must outlive the system (the board probe keeps them in statics) and * the VM, submit object and ring must already be attached to the live GPU context. The core must not be open. */ diff --git a/NativeAdrenoSubmit.cpp b/NativeAdrenoSubmit.cpp index 3f30508..842411d 100644 --- a/NativeAdrenoSubmit.cpp +++ b/NativeAdrenoSubmit.cpp @@ -201,7 +201,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock) if(!preemption())return; /* A switch the interrupt saw finish (or that finished while its handler could not take the lock) is taken first. */ if(preempt_.inFlight()) { - if(!preemptIrq()) { + if(!completeSwitch()) { /* The CP never finished a switch: no more are tried (the restart in recover() brings the CP back). */ if(preempt_.timedOut(PreemptSwitchTimeoutUs)) { uint32_t progress[6]={}; @@ -214,6 +214,8 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock) } if(preempt_dead_)return; const uint32_t loaded=preempt_.loaded(); + const uint64_t now=clock_now_?clock_now_(clock_cookie_):0; + if(loaded==1 && now && nowsubmit_us && now-first->submit_us>PreemptAgingUs))continue; + const uint64_t since=first->submit_us>low_service_us_?first->submit_us:low_service_us_; + if(!(now && since && now-since>PreemptAgingUs))continue; } /* The incoming ring's preemption record keeps what it was given at setup and what the CP saved into it when it switched the * ring out (read pointer, info, errno); only the write pointer the CP resumes at is given again before every switch. */ @@ -235,7 +237,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock) uint64_t ttbr0=0; if(incoming->executedTable(&ttbr0)) {markPreemptDead(5);return;} (void)NativeAdrenoPreempt::writeSmmuInfo(preempt_setup_.pages.smmu_cpu[i],ttbr0,uint32_t(ttbr0>>48),0); - if(preempt_.trigger(i,true)) {ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;} + if(preempt_.trigger(i,true)) {aging_grant_=i==1 && high;ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;} /* The trigger itself refused (a register write failed): the second ring cannot be reached, so it is given up at once and the * jobs already queued on it are replayed by the restart the hang check asks for. */ preempt_.noteSkipped(); @@ -244,7 +246,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock) } } /* The interrupt's side: a completed switch makes its ring the loaded one and gives that ring's write pointer. */ -bool NativeAdrenoSubmit::preemptIrq() +bool NativeAdrenoSubmit::completeSwitch() { if(!preemption() || !preempt_.inFlight())return false; if(!preempt_.complete())return false; @@ -255,8 +257,32 @@ bool NativeAdrenoSubmit::preemptIrq() if(now->restoreWptr()) {markPreemptDead(6);return false;} preempt_.noteRestoredWptr(); if(!preempt_.releaseKeepalive()) {markPreemptDead(7);return false;} + if(preempt_.loaded()==1) { + low_service_us_=clock_now_?clock_now_(clock_cookie_):0; + low_slice_until_=aging_grant_ && low_service_us_?low_service_us_+PreemptAgingSliceUs:0; + } + aging_grant_=false; return true; } +bool NativeAdrenoSubmit::preemptIrq() +{ + if(!completeSwitch())return false; + // Work arriving during the switch is reconsidered as soon as it completes. + driveSwitch(nullptr);return true; +} +bool NativeAdrenoSubmit::fenceProgress(uint32_t *submitted,uint32_t *retired) +{ + if(!attached_ || poisoned_ || !submitted || !retired)return false; + uint32_t done[2]={},issued=fence_issued_; + for(unsigned i=0;ifenceValue(&done[i]))return false; + if((int32_t)(ring(i)->lastSubmitted()-issued)>0)issued=ring(i)->lastSubmitted(); + } + uint32_t prefix=issued; + for(const auto &job:jobs_)if(job.active && (int32_t)(job.fence-done[job.ring])>0 && + (int32_t)((job.fence-1)-prefix)<0)prefix=job.fence-1; + *submitted=issued;*retired=prefix;return true; +} IOReturn NativeAdrenoSubmit::setPreemption(NativeAdrenoRing *high,NativeAdrenoRing *low,const PreemptSetup &setup,bool on) { if(!attached_ || poisoned_)return kIOReturnNotReady; @@ -280,7 +306,7 @@ IOReturn NativeAdrenoSubmit::setPreemption(NativeAdrenoRing *high,NativeAdrenoRi } rings_[0]=high;rings_[1]=low;ring_count_=2; high->setCurrent(true);low->setCurrent(false); - preempt_setup_=setup; + preempt_setup_=setup;low_service_us_=low_slice_until_=0;aging_grant_=false; const IOReturn attached_preempt=preempt_.attach(&setup.io,setup.pages,0,setup.now,setup.clock); if(attached_preempt) {ring_count_=1;rings_[1]=nullptr;return attached_preempt;} preempt_on_=true; @@ -820,7 +846,7 @@ IOReturn NativeAdrenoSubmit::recover(NativeAdrenoRestart restart,void *cookie,co /* The restart's CP init ran on the first ring: it is the one the CP runs and every replayed job goes there, whatever ring it * came from (the words are ring-agnostic and their fence numbers are kept, so each waiter still finds its value). */ if(preemption()) { - preempt_.restart(0); + preempt_.restart(0);low_service_us_=low_slice_until_=0;aging_grant_=false; ring(0)->setCurrent(true); (void)ring(1)->restartSecondary(hung); ring(1)->setCurrent(false); diff --git a/NativeAdrenoSubmit.h b/NativeAdrenoSubmit.h index bee3d15..8c44bd1 100644 --- a/NativeAdrenoSubmit.h +++ b/NativeAdrenoSubmit.h @@ -116,12 +116,14 @@ private: /* A switch that cannot be made (the trigger refused, or the CP did not finish one in time) leaves the second ring's jobs without a * CP to run them. The ring is then given up for good (preempt_dead_, with the reason) and the GPU restart replays what it holds on * the first ring (stranded_: recover() starts from the oldest job that never ran). */ - enum : uint32_t {PreemptSwitchTimeoutUs=1000000,PreemptStrandedUs=2000000,PreemptAgingUs=250000}; /* switch watchdog 1 s; a ring-1 job unreached for 2 s */ - bool preempt_dead_=false,stranded_=false; + enum : uint32_t {PreemptSwitchTimeoutUs=1000000,PreemptStrandedUs=2000000,PreemptAgingUs=250000,PreemptAgingSliceUs=2000}; /* switch watchdog 1 s; a ring-1 job unreached for 2 s */ + bool preempt_dead_=false,stranded_=false,aging_grant_=false; + uint64_t low_service_us_=0,low_slice_until_=0; uint32_t dead_reason_=0,rescues_=0; void markPreemptDead(uint32_t reason) {if(!preempt_dead_) {preempt_dead_=true;dead_reason_=reason;}} NativeAdrenoRing *ringOfFence(uint32_t fence) const; unsigned ringForPriority() const; + bool completeSwitch(); /* The oldest job on `index` that the ring's CP has not finished (its fence beyond what that ring last wrote); null when none. */ const Job *pendingOn(unsigned index); void noteFence(uint32_t fence,uint8_t ring); @@ -162,6 +164,8 @@ public: /* The interrupt's side of a switch: the ring the CP moved to is the one whose write pointer is given again. Returns whether a * switch completed. Called from the GPU interrupt (native_adreno_preempt_irq). */ bool preemptIrq(); + /* Global completed prefix for scanout: later completions on another ring cannot skip pending work. */ + bool fenceProgress(uint32_t *submitted,uint32_t *retired); /* The switch counters, the ring the CP runs, whether a switch is in flight and the control register as the hardware holds it. */ /* A word of ring `index`'s preemption record as the CPU sees it now (the CP saves the read pointer, info and errno there). */ uint32_t recordWord(unsigned index,uint32_t byte_offset) const diff --git a/tests/test_submit.cpp b/tests/test_submit.cpp index 66ba2a5..40d1832 100644 --- a/tests/test_submit.cpp +++ b/tests/test_submit.cpp @@ -297,12 +297,22 @@ int main() CHECK(p.submit.setTableSwitching(2,write,nullptr)==0); p.ring.tableInstalled(0xaaaa000);low.tableInstalled(0xbbbb000);p.ring.useContext(42); NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};uint32_t first=0,second=0; - p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&first,&ringclock)==0); + clock_us=1;p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&first,&ringclock)==0); CHECK(low.lastSubmitted()==first && p.ring.lastSubmitted()==0 && low.table()==p.vm.ttbr0() && p.ring.table()==0xaaaa000); CHECK(low.context()==42 && cp.regs[0x08a0]==0x341); + uint32_t overtaking=0; + p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,8,&overtaking,&ringclock)==0); + cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq()); + CHECK(cp.regs[0x08a0]==0x341 && p.submit.deadReason()==0); // immediately switch back for newly queued high work + cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq()); + clock_us=300001;CHECK(p.submit.retireCompleted()==0 && cp.regs[0x08a0]==0x341); + cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq() && cp.regs[0x08a0]==0x340); // aged low ring gets a time slice + clock_us+=2001;CHECK(p.submit.retireCompleted()==0 && cp.regs[0x08a0]==0x341); + cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq() && cp.regs[0x08a0]==0x340); // no immediate aging ping-pong + fencemem[1]=overtaking;CHECK(p.submit.retireCompleted()==0); cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq()); p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&second,&ringclock)==0); - CHECK(low.lastSubmitted()==second && p.ring.lastSubmitted()==0); // own queue cannot overtake + CHECK(low.lastSubmitted()==second && p.ring.lastSubmitted()==overtaking); // own queue cannot overtake lowfence[1]=second;CHECK(p.submit.retireCompleted()==0); CHECK(p.submit.submit(&p.vm,&buffer,1,7,&second,&ringclock)==0 && p.ring.lastSubmitted()==second); CHECK(p.ring.table()==p.vm.ttbr0()); @@ -317,8 +327,10 @@ int main() CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==1); cp.regs[0x08a0]=0x340;(void)p.submit.preemptIrq(); } + uint32_t issued=0,retired=0;CHECK(p.submit.fenceProgress(&issued,&retired) && issued==second && retired==first-1); CHECK(p.submit.observeFence(first,nullptr,0)==kIOReturnTimeout); lowfence[1]=first;CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==0); + CHECK(p.submit.fenceProgress(&issued,&retired) && issued==second && retired==second); CHECK(p.submit.wait(first,&ringclock,0)==0 && p.submit.observeFence(first,nullptr,0)==0); CHECK(p.submit.wait(second+1,&ringclock,0)==kIOReturnBadArgument); CHECK(p.submit.setTableSwitching(1,write,nullptr)==0);p.submit.nextPriority(2); @@ -326,7 +338,7 @@ int main() fencemem[1]=second;CHECK(p.submit.retireCompleted()==0); std::puts("PASS two-ring priorities, per-ring table/context, queue FIFO, fence-history rollover and CPU-table fallback"); } - std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 unusable-VM refusal and 1 failed restart\n"); + std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 usable VM after recovery and 1 failed restart\n"); } /* Physical-page maintenance (imported BOs): record it; no host page is touched. */ /* Host double: the page attribute change of user BOs has no host equivalent (no host page is touched). */