diff --git a/NativeAdrenoSubmit.cpp b/NativeAdrenoSubmit.cpp index e411ef8..b1b3fd2 100644 --- a/NativeAdrenoSubmit.cpp +++ b/NativeAdrenoSubmit.cpp @@ -549,7 +549,7 @@ IOReturn NativeAdrenoSubmit::observeFence(uint32_t fence,const NativeAdrenoRingC driveSwitch(clock); NativeAdrenoRing *const r=ringOfFence(fence); if(!r)return kIOReturnBadArgument; - const NativeAdrenoRingFence result=r->observeFence(fence,clock,timeout_us,nullptr); + const NativeAdrenoRingFence result=waitOnRing(r,false,fence,clock,timeout_us,nullptr); if(result==NativeAdrenoRingFencePoisoned)return poison(); if(result==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument; if(result==NativeAdrenoRingFenceTimeout)return kIOReturnTimeout; @@ -651,6 +651,25 @@ void NativeAdrenoSubmit::noteStall(uint32_t observed,const NativeAdrenoRingClock native_adreno_stall[1]=stall_timeouts; native_adreno_stall[0]=next; } +/* A wait that pauses is made with the caller's lock held for all of it (a connection's teardown, a CPU table switch, the board's own + * calls), so nothing else can look at the switch between the two rings meanwhile. It looks itself after every pause: the fence may be + * of a ring the CP is not running, which gets the CP once the other ring goes idle or its job has aged, and a switch the CP finished + * has to be completed for the incoming ring to get its write pointer. Without that a low-ring job behind a busy high ring sat out the + * whole timeout, and after 2 s the hang check took it for a ring the CP cannot reach and restarted the GPU. */ +NativeAdrenoRingFence NativeAdrenoSubmit::waitOnRing(NativeAdrenoRing *r,bool held,uint32_t fence,const NativeAdrenoRingClock *clock, + uint32_t timeout_us,uint32_t *observed) +{ + if(!preemption() || !timeout_us || !clock || !clock->now_us) + return held?r->wait(fence,clock,timeout_us,observed):r->observeFence(fence,clock,timeout_us,observed); + const uint64_t start=clock->now_us(clock->cookie); + for(;;) { + const NativeAdrenoRingFence state=held?r->wait(fence,clock,0,observed):r->observeFence(fence,clock,0,observed); + if(state!=NativeAdrenoRingFenceTimeout)return state; + if(clock->now_us(clock->cookie)-start>=timeout_us)return NativeAdrenoRingFenceTimeout; + if(clock->idle && clock->idle(clock->cookie))return NativeAdrenoRingFenceTimeout; + driveSwitch(clock); + } +} IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *clock, uint32_t timeout_us) { @@ -664,7 +683,7 @@ IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *cl if(!r)return kIOReturnBadArgument; if(!job) { native_adreno_fence_wait[6]=native_adreno_fence_wait[6]+1; - const NativeAdrenoRingFence passed=r->observeFence(fence,clock,timeout_us,&observed); + const NativeAdrenoRingFence passed=waitOnRing(r,false,fence,clock,timeout_us,&observed); if(passed==NativeAdrenoRingFencePoisoned)return poison(); if(passed==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;} if(passed==NativeAdrenoRingFenceTimeout) { @@ -676,7 +695,7 @@ IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *cl } return kIOReturnSuccess; } - const NativeAdrenoRingFence result=r->wait(fence,clock,timeout_us,&observed); + const NativeAdrenoRingFence result=waitOnRing(r,true,fence,clock,timeout_us,&observed); if(result==NativeAdrenoRingFencePoisoned)return poison(); if(result==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;} if(result==NativeAdrenoRingFenceTimeout) { diff --git a/NativeAdrenoSubmit.h b/NativeAdrenoSubmit.h index e60a945..eed860e 100644 --- a/NativeAdrenoSubmit.h +++ b/NativeAdrenoSubmit.h @@ -131,6 +131,10 @@ private: * through it. */ NativeAdrenoRing *ring(unsigned index=0) {return indexIOReturn{return 0;}; + CHECK(p.submit.setTableSwitching(2,write,nullptr)==0); + p.ring.tableInstalled(0xaaaa000);low.tableInstalled(0xbbbb000); + NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096}; + static uint32_t high_fence,low_fence,pauses,switched;high_fence=low_fence=pauses=switched=0; + clock_us=1;p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,8,&high_fence,&ringclock)==0); + p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&low_fence,&ringclock)==0); + CHECK(p.ring.lastSubmitted()==high_fence && low.lastSubmitted()==low_fence && !(cp.regs[0x08a0]&1)); /* the high ring keeps the CP */ + const NativeAdrenoRingClock playing{&cp,cnow,[](void *)->int { + clock_us+=1000;++pauses; + if(pauses==1)fencemem[1]=high_fence; /* the high ring goes idle */ + else if(cp.regs[0x08a0]&1) {cp.regs[0x08a0]&=~1u;switched=pauses;} /* the CP finishes a triggered switch */ + else if(switched && pauses>switched)lowfence[1]=low_fence; /* and only then runs the low ring's job */ + return 0;}}; + CHECK(p.submit.wait(low_fence,&playing,2000000)==kIOReturnSuccess); + CHECK(switched && pauses<=8 && p.submit.preemptStats().triggers==1 && p.submit.preemptStats().completed==1); + CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==0); + std::puts("PASS a pausing wait drives the ring switch for a low-ring job"); + } std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 usable VM after recovery and 1 failed restart\n"); } /* Physical-page maintenance (imported BOs): record it; no host page is touched. */