fix(gpu): reschedule priority work and track global fence progress

This commit is contained in:
rponeawa
2026-11-02 11:41:19 +08:00
Unverified
parent b9cc81e870
commit ccabc7e446
4 changed files with 58 additions and 17 deletions
+4 -5
View File
@@ -1137,11 +1137,10 @@ extern "C" bool native_adreno_preempt_irq()
* for the fences handed out before the swap arrived. False before the GPU is published. */
extern "C" bool native_adreno_fences(uint32_t *submitted,uint32_t *retired)
{
if(!g.published || !g.ring || !submitted || !retired)return false;
uint32_t value=0;
if(g.ring->fenceValue(&value))return false;
*submitted=g.ring->lastSubmitted();*retired=value;
return true;
if(!g.published || !g.submit || !submitted || !retired)return false;
if(!IOLockTryLock(g.lock))return false;
const bool known=g.submit->fenceProgress(submitted,retired);
IOLockUnlock(g.lock);return known;
}
/* Publishes the service once. Everything passed in must outlive the system (the board probe keeps them in statics) and
* the VM, submit object and ring must already be attached to the live GPU context. The core must not be open. */
+33 -7
View File
@@ -201,7 +201,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
if(!preemption())return;
/* A switch the interrupt saw finish (or that finished while its handler could not take the lock) is taken first. */
if(preempt_.inFlight()) {
if(!preemptIrq()) {
if(!completeSwitch()) {
/* The CP never finished a switch: no more are tried (the restart in recover() brings the CP back). */
if(preempt_.timedOut(PreemptSwitchTimeoutUs)) {
uint32_t progress[6]={};
@@ -214,6 +214,8 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
}
if(preempt_dead_)return;
const uint32_t loaded=preempt_.loaded();
const uint64_t now=clock_now_?clock_now_(clock_cookie_):0;
if(loaded==1 && now && now<low_slice_until_ && pendingOn(1))return;
/* Ring 0 is the high ring: while it has work the CP does not leave it for the low ring, and it takes the CP back from the low ring as
* soon as it has some. The low ring's work still gets the GPU when the high ring is idle, and after it has waited PreemptAgingUs. */
const Job *const high=pendingOn(0);
@@ -223,8 +225,8 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
const Job *const first=pendingOn(i);
if(!first)continue;
if(i!=0 && high) {
const uint64_t now=clock_now_?clock_now_(clock_cookie_):0;
if(!(now && first->submit_us && now-first->submit_us>PreemptAgingUs))continue;
const uint64_t since=first->submit_us>low_service_us_?first->submit_us:low_service_us_;
if(!(now && since && now-since>PreemptAgingUs))continue;
}
/* The incoming ring's preemption record keeps what it was given at setup and what the CP saved into it when it switched the
* ring out (read pointer, info, errno); only the write pointer the CP resumes at is given again before every switch. */
@@ -235,7 +237,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
uint64_t ttbr0=0;
if(incoming->executedTable(&ttbr0)) {markPreemptDead(5);return;}
(void)NativeAdrenoPreempt::writeSmmuInfo(preempt_setup_.pages.smmu_cpu[i],ttbr0,uint32_t(ttbr0>>48),0);
if(preempt_.trigger(i,true)) {ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;}
if(preempt_.trigger(i,true)) {aging_grant_=i==1 && high;ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;}
/* The trigger itself refused (a register write failed): the second ring cannot be reached, so it is given up at once and the
* jobs already queued on it are replayed by the restart the hang check asks for. */
preempt_.noteSkipped();
@@ -244,7 +246,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
}
}
/* The interrupt's side: a completed switch makes its ring the loaded one and gives that ring's write pointer. */
bool NativeAdrenoSubmit::preemptIrq()
bool NativeAdrenoSubmit::completeSwitch()
{
if(!preemption() || !preempt_.inFlight())return false;
if(!preempt_.complete())return false;
@@ -255,8 +257,32 @@ bool NativeAdrenoSubmit::preemptIrq()
if(now->restoreWptr()) {markPreemptDead(6);return false;}
preempt_.noteRestoredWptr();
if(!preempt_.releaseKeepalive()) {markPreemptDead(7);return false;}
if(preempt_.loaded()==1) {
low_service_us_=clock_now_?clock_now_(clock_cookie_):0;
low_slice_until_=aging_grant_ && low_service_us_?low_service_us_+PreemptAgingSliceUs:0;
}
aging_grant_=false;
return true;
}
bool NativeAdrenoSubmit::preemptIrq()
{
if(!completeSwitch())return false;
// Work arriving during the switch is reconsidered as soon as it completes.
driveSwitch(nullptr);return true;
}
bool NativeAdrenoSubmit::fenceProgress(uint32_t *submitted,uint32_t *retired)
{
if(!attached_ || poisoned_ || !submitted || !retired)return false;
uint32_t done[2]={},issued=fence_issued_;
for(unsigned i=0;i<ring_count_;++i) {
if(ring(i)->fenceValue(&done[i]))return false;
if((int32_t)(ring(i)->lastSubmitted()-issued)>0)issued=ring(i)->lastSubmitted();
}
uint32_t prefix=issued;
for(const auto &job:jobs_)if(job.active && (int32_t)(job.fence-done[job.ring])>0 &&
(int32_t)((job.fence-1)-prefix)<0)prefix=job.fence-1;
*submitted=issued;*retired=prefix;return true;
}
IOReturn NativeAdrenoSubmit::setPreemption(NativeAdrenoRing *high,NativeAdrenoRing *low,const PreemptSetup &setup,bool on)
{
if(!attached_ || poisoned_)return kIOReturnNotReady;
@@ -280,7 +306,7 @@ IOReturn NativeAdrenoSubmit::setPreemption(NativeAdrenoRing *high,NativeAdrenoRi
}
rings_[0]=high;rings_[1]=low;ring_count_=2;
high->setCurrent(true);low->setCurrent(false);
preempt_setup_=setup;
preempt_setup_=setup;low_service_us_=low_slice_until_=0;aging_grant_=false;
const IOReturn attached_preempt=preempt_.attach(&setup.io,setup.pages,0,setup.now,setup.clock);
if(attached_preempt) {ring_count_=1;rings_[1]=nullptr;return attached_preempt;}
preempt_on_=true;
@@ -820,7 +846,7 @@ IOReturn NativeAdrenoSubmit::recover(NativeAdrenoRestart restart,void *cookie,co
/* The restart's CP init ran on the first ring: it is the one the CP runs and every replayed job goes there, whatever ring it
* came from (the words are ring-agnostic and their fence numbers are kept, so each waiter still finds its value). */
if(preemption()) {
preempt_.restart(0);
preempt_.restart(0);low_service_us_=low_slice_until_=0;aging_grant_=false;
ring(0)->setCurrent(true);
(void)ring(1)->restartSecondary(hung);
ring(1)->setCurrent(false);
+6 -2
View File
@@ -116,12 +116,14 @@ private:
/* A switch that cannot be made (the trigger refused, or the CP did not finish one in time) leaves the second ring's jobs without a
* CP to run them. The ring is then given up for good (preempt_dead_, with the reason) and the GPU restart replays what it holds on
* the first ring (stranded_: recover() starts from the oldest job that never ran). */
enum : uint32_t {PreemptSwitchTimeoutUs=1000000,PreemptStrandedUs=2000000,PreemptAgingUs=250000}; /* switch watchdog 1 s; a ring-1 job unreached for 2 s */
bool preempt_dead_=false,stranded_=false;
enum : uint32_t {PreemptSwitchTimeoutUs=1000000,PreemptStrandedUs=2000000,PreemptAgingUs=250000,PreemptAgingSliceUs=2000}; /* switch watchdog 1 s; a ring-1 job unreached for 2 s */
bool preempt_dead_=false,stranded_=false,aging_grant_=false;
uint64_t low_service_us_=0,low_slice_until_=0;
uint32_t dead_reason_=0,rescues_=0;
void markPreemptDead(uint32_t reason) {if(!preempt_dead_) {preempt_dead_=true;dead_reason_=reason;}}
NativeAdrenoRing *ringOfFence(uint32_t fence) const;
unsigned ringForPriority() const;
bool completeSwitch();
/* The oldest job on `index` that the ring's CP has not finished (its fence beyond what that ring last wrote); null when none. */
const Job *pendingOn(unsigned index);
void noteFence(uint32_t fence,uint8_t ring);
@@ -162,6 +164,8 @@ public:
/* The interrupt's side of a switch: the ring the CP moved to is the one whose write pointer is given again. Returns whether a
* switch completed. Called from the GPU interrupt (native_adreno_preempt_irq). */
bool preemptIrq();
/* Global completed prefix for scanout: later completions on another ring cannot skip pending work. */
bool fenceProgress(uint32_t *submitted,uint32_t *retired);
/* The switch counters, the ring the CP runs, whether a switch is in flight and the control register as the hardware holds it. */
/* A word of ring `index`'s preemption record as the CPU sees it now (the CP saves the read pointer, info and errno there). */
uint32_t recordWord(unsigned index,uint32_t byte_offset) const
+15 -3
View File
@@ -297,12 +297,22 @@ int main()
CHECK(p.submit.setTableSwitching(2,write,nullptr)==0);
p.ring.tableInstalled(0xaaaa000);low.tableInstalled(0xbbbb000);p.ring.useContext(42);
NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};uint32_t first=0,second=0;
p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&first,&ringclock)==0);
clock_us=1;p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&first,&ringclock)==0);
CHECK(low.lastSubmitted()==first && p.ring.lastSubmitted()==0 && low.table()==p.vm.ttbr0() && p.ring.table()==0xaaaa000);
CHECK(low.context()==42 && cp.regs[0x08a0]==0x341);
uint32_t overtaking=0;
p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,8,&overtaking,&ringclock)==0);
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq());
CHECK(cp.regs[0x08a0]==0x341 && p.submit.deadReason()==0); // immediately switch back for newly queued high work
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq());
clock_us=300001;CHECK(p.submit.retireCompleted()==0 && cp.regs[0x08a0]==0x341);
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq() && cp.regs[0x08a0]==0x340); // aged low ring gets a time slice
clock_us+=2001;CHECK(p.submit.retireCompleted()==0 && cp.regs[0x08a0]==0x341);
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq() && cp.regs[0x08a0]==0x340); // no immediate aging ping-pong
fencemem[1]=overtaking;CHECK(p.submit.retireCompleted()==0);
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq());
p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&second,&ringclock)==0);
CHECK(low.lastSubmitted()==second && p.ring.lastSubmitted()==0); // own queue cannot overtake
CHECK(low.lastSubmitted()==second && p.ring.lastSubmitted()==overtaking); // own queue cannot overtake
lowfence[1]=second;CHECK(p.submit.retireCompleted()==0);
CHECK(p.submit.submit(&p.vm,&buffer,1,7,&second,&ringclock)==0 && p.ring.lastSubmitted()==second);
CHECK(p.ring.table()==p.vm.ttbr0());
@@ -317,8 +327,10 @@ int main()
CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==1);
cp.regs[0x08a0]=0x340;(void)p.submit.preemptIrq();
}
uint32_t issued=0,retired=0;CHECK(p.submit.fenceProgress(&issued,&retired) && issued==second && retired==first-1);
CHECK(p.submit.observeFence(first,nullptr,0)==kIOReturnTimeout);
lowfence[1]=first;CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==0);
CHECK(p.submit.fenceProgress(&issued,&retired) && issued==second && retired==second);
CHECK(p.submit.wait(first,&ringclock,0)==0 && p.submit.observeFence(first,nullptr,0)==0);
CHECK(p.submit.wait(second+1,&ringclock,0)==kIOReturnBadArgument);
CHECK(p.submit.setTableSwitching(1,write,nullptr)==0);p.submit.nextPriority(2);
@@ -326,7 +338,7 @@ int main()
fencemem[1]=second;CHECK(p.submit.retireCompleted()==0);
std::puts("PASS two-ring priorities, per-ring table/context, queue FIFO, fence-history rollover and CPU-table fallback");
}
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 unusable-VM refusal and 1 failed restart\n");
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 usable VM after recovery and 1 failed restart\n");
}
/* Physical-page maintenance (imported BOs): record it; no host page is touched. */
/* Host double: the page attribute change of user BOs has no host equivalent (no host page is touched). */