fix(submit): look at the ring switch after every pause of a wait

A wait with a timeout runs with the driver's lock held for all of it (a
connection's teardown and the table hand-back wait 2 s, a submission's
wait_fence, the board's calls). It decided the switch between the two rings
once, at its start, and then only read the fence word between pauses, while
the lock kept every other caller of the switch out. A low-ring job behind a
busy high ring therefore stayed without the CP after the high ring went idle,
for the whole timeout; a switch the CP finished meanwhile was not completed
either. The hang check then found a low-ring job older than 2 s with the
high ring idle, took the ring for unreachable, gave it up and restarted the
GPU (one boot log shows a 2 s wait followed by "hangs 1 restarts 1", "dead 3
rescues 1" and a 2.17 s wait of another caller).

wait() and observeFence() go through waitOnRing(), which with two rings
runs driveSwitch again after every pause. A host test plays the CP from the
pause: without the change the wait times out, with it the switch is
triggered, completed and the low-ring job finishes. Not run on hardware.
This commit is contained in:
rponeawa
2026-11-11 00:00:00 +08:00
Unverified
parent 798710e43d
commit fa4f4aa98b
3 changed files with 58 additions and 3 deletions
+22 -3
View File
@@ -549,7 +549,7 @@ IOReturn NativeAdrenoSubmit::observeFence(uint32_t fence,const NativeAdrenoRingC
driveSwitch(clock);
NativeAdrenoRing *const r=ringOfFence(fence);
if(!r)return kIOReturnBadArgument;
const NativeAdrenoRingFence result=r->observeFence(fence,clock,timeout_us,nullptr);
const NativeAdrenoRingFence result=waitOnRing(r,false,fence,clock,timeout_us,nullptr);
if(result==NativeAdrenoRingFencePoisoned)return poison();
if(result==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument;
if(result==NativeAdrenoRingFenceTimeout)return kIOReturnTimeout;
@@ -651,6 +651,25 @@ void NativeAdrenoSubmit::noteStall(uint32_t observed,const NativeAdrenoRingClock
native_adreno_stall[1]=stall_timeouts;
native_adreno_stall[0]=next;
}
/* A wait that pauses is made with the caller's lock held for all of it (a connection's teardown, a CPU table switch, the board's own
* calls), so nothing else can look at the switch between the two rings meanwhile. It looks itself after every pause: the fence may be
* of a ring the CP is not running, which gets the CP once the other ring goes idle or its job has aged, and a switch the CP finished
* has to be completed for the incoming ring to get its write pointer. Without that a low-ring job behind a busy high ring sat out the
* whole timeout, and after 2 s the hang check took it for a ring the CP cannot reach and restarted the GPU. */
NativeAdrenoRingFence NativeAdrenoSubmit::waitOnRing(NativeAdrenoRing *r,bool held,uint32_t fence,const NativeAdrenoRingClock *clock,
uint32_t timeout_us,uint32_t *observed)
{
if(!preemption() || !timeout_us || !clock || !clock->now_us)
return held?r->wait(fence,clock,timeout_us,observed):r->observeFence(fence,clock,timeout_us,observed);
const uint64_t start=clock->now_us(clock->cookie);
for(;;) {
const NativeAdrenoRingFence state=held?r->wait(fence,clock,0,observed):r->observeFence(fence,clock,0,observed);
if(state!=NativeAdrenoRingFenceTimeout)return state;
if(clock->now_us(clock->cookie)-start>=timeout_us)return NativeAdrenoRingFenceTimeout;
if(clock->idle && clock->idle(clock->cookie))return NativeAdrenoRingFenceTimeout;
driveSwitch(clock);
}
}
IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *clock,
uint32_t timeout_us)
{
@@ -664,7 +683,7 @@ IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *cl
if(!r)return kIOReturnBadArgument;
if(!job) {
native_adreno_fence_wait[6]=native_adreno_fence_wait[6]+1;
const NativeAdrenoRingFence passed=r->observeFence(fence,clock,timeout_us,&observed);
const NativeAdrenoRingFence passed=waitOnRing(r,false,fence,clock,timeout_us,&observed);
if(passed==NativeAdrenoRingFencePoisoned)return poison();
if(passed==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;}
if(passed==NativeAdrenoRingFenceTimeout) {
@@ -676,7 +695,7 @@ IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *cl
}
return kIOReturnSuccess;
}
const NativeAdrenoRingFence result=r->wait(fence,clock,timeout_us,&observed);
const NativeAdrenoRingFence result=waitOnRing(r,true,fence,clock,timeout_us,&observed);
if(result==NativeAdrenoRingFencePoisoned)return poison();
if(result==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;}
if(result==NativeAdrenoRingFenceTimeout) {
+4
View File
@@ -131,6 +131,10 @@ private:
* through it. */
NativeAdrenoRing *ring(unsigned index=0) {return index<ring_count_?rings_[index]:ring_;}
void driveSwitch(const NativeAdrenoRingClock *clock);
/* The ring's wait for wait(): held, a fence a job holds (NativeAdrenoRing::wait), otherwise an earlier one (observeFence). With
* two rings a wait that pauses drives the switch between them after every pause. */
NativeAdrenoRingFence waitOnRing(NativeAdrenoRing *r,bool held,uint32_t fence,const NativeAdrenoRingClock *clock,
uint32_t timeout_us,uint32_t *observed);
NativeAdrenoSubmitStats stats_{};
void noteStall(uint32_t observed,const NativeAdrenoRingClock *clock); /* the CP's state for a fence that stopped advancing (native_adreno_stall) */
bool attached_=false,poisoned_=false;
+32
View File
@@ -338,6 +338,38 @@ int main()
fencemem[1]=second;CHECK(p.submit.retireCompleted()==0);
std::puts("PASS two-ring priorities, per-ring table/context, queue FIFO, fence-history rollover and CPU-table fallback");
}
{
/* A wait that pauses under the caller's lock (a connection's teardown) for a low-ring job behind a busy high ring. The pause
* plays the CP: the high job ends, a triggered switch finishes, and the low ring's job runs only once the CP was switched
* to it. Nothing but the wait looks at the rings, so the wait itself has to trigger and complete the switch. */
Path p;CHECK(exposed(p)==kIOReturnSuccess);NativeAdrenoRing low;
static uint32_t lowfence[1024];std::memset(lowfence,0,sizeof(lowfence));
uint32_t lowmem[8192]{};uint8_t rec[2][64]{},info[2][32]{};
CHECK(low.attachSecondary(&ringio,&ringcache,lowmem,sizeof(lowmem),0x200000,lowfence,sizeof(lowfence),0x208000)==0);
CHECK(p.ring.setPreempt(0x300000,0x400000,0x108008)==0 && low.setPreempt(0x500000,0x600000,0x208008)==0);
NativeAdrenoSubmit::PreemptSetup setup{};setup.io={&cp,cwrite,cread,cbarrier,nullptr,nullptr};
setup.pages={{0x300000,0x500000},{0x400000,0x600000},{rec[0],rec[1]},{info[0],info[1]}};
setup.now=cnow;setup.clock=&cp;setup.mode=2;
CHECK(p.submit.setPreemption(&p.ring,&low,setup,true)==0);
auto write=[](void *,uint64_t)->IOReturn{return 0;};
CHECK(p.submit.setTableSwitching(2,write,nullptr)==0);
p.ring.tableInstalled(0xaaaa000);low.tableInstalled(0xbbbb000);
NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};
static uint32_t high_fence,low_fence,pauses,switched;high_fence=low_fence=pauses=switched=0;
clock_us=1;p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,8,&high_fence,&ringclock)==0);
p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&low_fence,&ringclock)==0);
CHECK(p.ring.lastSubmitted()==high_fence && low.lastSubmitted()==low_fence && !(cp.regs[0x08a0]&1)); /* the high ring keeps the CP */
const NativeAdrenoRingClock playing{&cp,cnow,[](void *)->int {
clock_us+=1000;++pauses;
if(pauses==1)fencemem[1]=high_fence; /* the high ring goes idle */
else if(cp.regs[0x08a0]&1) {cp.regs[0x08a0]&=~1u;switched=pauses;} /* the CP finishes a triggered switch */
else if(switched && pauses>switched)lowfence[1]=low_fence; /* and only then runs the low ring's job */
return 0;}};
CHECK(p.submit.wait(low_fence,&playing,2000000)==kIOReturnSuccess);
CHECK(switched && pauses<=8 && p.submit.preemptStats().triggers==1 && p.submit.preemptStats().completed==1);
CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==0);
std::puts("PASS a pausing wait drives the ring switch for a low-ring job");
}
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 usable VM after recovery and 1 failed restart\n");
}
/* Physical-page maintenance (imported BOs): record it; no host page is touched. */