mirror of
https://github.com/UniXNU/NativeAdreno.git
synced 2026-11-17 04:51:59 +03:00
fix(gpu): reschedule priority work and track global fence progress
This commit is contained in:
@@ -1137,11 +1137,10 @@ extern "C" bool native_adreno_preempt_irq()
|
||||
* for the fences handed out before the swap arrived. False before the GPU is published. */
|
||||
extern "C" bool native_adreno_fences(uint32_t *submitted,uint32_t *retired)
|
||||
{
|
||||
if(!g.published || !g.ring || !submitted || !retired)return false;
|
||||
uint32_t value=0;
|
||||
if(g.ring->fenceValue(&value))return false;
|
||||
*submitted=g.ring->lastSubmitted();*retired=value;
|
||||
return true;
|
||||
if(!g.published || !g.submit || !submitted || !retired)return false;
|
||||
if(!IOLockTryLock(g.lock))return false;
|
||||
const bool known=g.submit->fenceProgress(submitted,retired);
|
||||
IOLockUnlock(g.lock);return known;
|
||||
}
|
||||
/* Publishes the service once. Everything passed in must outlive the system (the board probe keeps them in statics) and
|
||||
* the VM, submit object and ring must already be attached to the live GPU context. The core must not be open. */
|
||||
|
||||
+33
-7
@@ -201,7 +201,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
|
||||
if(!preemption())return;
|
||||
/* A switch the interrupt saw finish (or that finished while its handler could not take the lock) is taken first. */
|
||||
if(preempt_.inFlight()) {
|
||||
if(!preemptIrq()) {
|
||||
if(!completeSwitch()) {
|
||||
/* The CP never finished a switch: no more are tried (the restart in recover() brings the CP back). */
|
||||
if(preempt_.timedOut(PreemptSwitchTimeoutUs)) {
|
||||
uint32_t progress[6]={};
|
||||
@@ -214,6 +214,8 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
|
||||
}
|
||||
if(preempt_dead_)return;
|
||||
const uint32_t loaded=preempt_.loaded();
|
||||
const uint64_t now=clock_now_?clock_now_(clock_cookie_):0;
|
||||
if(loaded==1 && now && now<low_slice_until_ && pendingOn(1))return;
|
||||
/* Ring 0 is the high ring: while it has work the CP does not leave it for the low ring, and it takes the CP back from the low ring as
|
||||
* soon as it has some. The low ring's work still gets the GPU when the high ring is idle, and after it has waited PreemptAgingUs. */
|
||||
const Job *const high=pendingOn(0);
|
||||
@@ -223,8 +225,8 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
|
||||
const Job *const first=pendingOn(i);
|
||||
if(!first)continue;
|
||||
if(i!=0 && high) {
|
||||
const uint64_t now=clock_now_?clock_now_(clock_cookie_):0;
|
||||
if(!(now && first->submit_us && now-first->submit_us>PreemptAgingUs))continue;
|
||||
const uint64_t since=first->submit_us>low_service_us_?first->submit_us:low_service_us_;
|
||||
if(!(now && since && now-since>PreemptAgingUs))continue;
|
||||
}
|
||||
/* The incoming ring's preemption record keeps what it was given at setup and what the CP saved into it when it switched the
|
||||
* ring out (read pointer, info, errno); only the write pointer the CP resumes at is given again before every switch. */
|
||||
@@ -235,7 +237,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
|
||||
uint64_t ttbr0=0;
|
||||
if(incoming->executedTable(&ttbr0)) {markPreemptDead(5);return;}
|
||||
(void)NativeAdrenoPreempt::writeSmmuInfo(preempt_setup_.pages.smmu_cpu[i],ttbr0,uint32_t(ttbr0>>48),0);
|
||||
if(preempt_.trigger(i,true)) {ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;}
|
||||
if(preempt_.trigger(i,true)) {aging_grant_=i==1 && high;ring(0)->setCurrent(false);ring(1)->setCurrent(false);return;}
|
||||
/* The trigger itself refused (a register write failed): the second ring cannot be reached, so it is given up at once and the
|
||||
* jobs already queued on it are replayed by the restart the hang check asks for. */
|
||||
preempt_.noteSkipped();
|
||||
@@ -244,7 +246,7 @@ void NativeAdrenoSubmit::driveSwitch(const NativeAdrenoRingClock *clock)
|
||||
}
|
||||
}
|
||||
/* The interrupt's side: a completed switch makes its ring the loaded one and gives that ring's write pointer. */
|
||||
bool NativeAdrenoSubmit::preemptIrq()
|
||||
bool NativeAdrenoSubmit::completeSwitch()
|
||||
{
|
||||
if(!preemption() || !preempt_.inFlight())return false;
|
||||
if(!preempt_.complete())return false;
|
||||
@@ -255,8 +257,32 @@ bool NativeAdrenoSubmit::preemptIrq()
|
||||
if(now->restoreWptr()) {markPreemptDead(6);return false;}
|
||||
preempt_.noteRestoredWptr();
|
||||
if(!preempt_.releaseKeepalive()) {markPreemptDead(7);return false;}
|
||||
if(preempt_.loaded()==1) {
|
||||
low_service_us_=clock_now_?clock_now_(clock_cookie_):0;
|
||||
low_slice_until_=aging_grant_ && low_service_us_?low_service_us_+PreemptAgingSliceUs:0;
|
||||
}
|
||||
aging_grant_=false;
|
||||
return true;
|
||||
}
|
||||
bool NativeAdrenoSubmit::preemptIrq()
|
||||
{
|
||||
if(!completeSwitch())return false;
|
||||
// Work arriving during the switch is reconsidered as soon as it completes.
|
||||
driveSwitch(nullptr);return true;
|
||||
}
|
||||
bool NativeAdrenoSubmit::fenceProgress(uint32_t *submitted,uint32_t *retired)
|
||||
{
|
||||
if(!attached_ || poisoned_ || !submitted || !retired)return false;
|
||||
uint32_t done[2]={},issued=fence_issued_;
|
||||
for(unsigned i=0;i<ring_count_;++i) {
|
||||
if(ring(i)->fenceValue(&done[i]))return false;
|
||||
if((int32_t)(ring(i)->lastSubmitted()-issued)>0)issued=ring(i)->lastSubmitted();
|
||||
}
|
||||
uint32_t prefix=issued;
|
||||
for(const auto &job:jobs_)if(job.active && (int32_t)(job.fence-done[job.ring])>0 &&
|
||||
(int32_t)((job.fence-1)-prefix)<0)prefix=job.fence-1;
|
||||
*submitted=issued;*retired=prefix;return true;
|
||||
}
|
||||
IOReturn NativeAdrenoSubmit::setPreemption(NativeAdrenoRing *high,NativeAdrenoRing *low,const PreemptSetup &setup,bool on)
|
||||
{
|
||||
if(!attached_ || poisoned_)return kIOReturnNotReady;
|
||||
@@ -280,7 +306,7 @@ IOReturn NativeAdrenoSubmit::setPreemption(NativeAdrenoRing *high,NativeAdrenoRi
|
||||
}
|
||||
rings_[0]=high;rings_[1]=low;ring_count_=2;
|
||||
high->setCurrent(true);low->setCurrent(false);
|
||||
preempt_setup_=setup;
|
||||
preempt_setup_=setup;low_service_us_=low_slice_until_=0;aging_grant_=false;
|
||||
const IOReturn attached_preempt=preempt_.attach(&setup.io,setup.pages,0,setup.now,setup.clock);
|
||||
if(attached_preempt) {ring_count_=1;rings_[1]=nullptr;return attached_preempt;}
|
||||
preempt_on_=true;
|
||||
@@ -820,7 +846,7 @@ IOReturn NativeAdrenoSubmit::recover(NativeAdrenoRestart restart,void *cookie,co
|
||||
/* The restart's CP init ran on the first ring: it is the one the CP runs and every replayed job goes there, whatever ring it
|
||||
* came from (the words are ring-agnostic and their fence numbers are kept, so each waiter still finds its value). */
|
||||
if(preemption()) {
|
||||
preempt_.restart(0);
|
||||
preempt_.restart(0);low_service_us_=low_slice_until_=0;aging_grant_=false;
|
||||
ring(0)->setCurrent(true);
|
||||
(void)ring(1)->restartSecondary(hung);
|
||||
ring(1)->setCurrent(false);
|
||||
|
||||
@@ -116,12 +116,14 @@ private:
|
||||
/* A switch that cannot be made (the trigger refused, or the CP did not finish one in time) leaves the second ring's jobs without a
|
||||
* CP to run them. The ring is then given up for good (preempt_dead_, with the reason) and the GPU restart replays what it holds on
|
||||
* the first ring (stranded_: recover() starts from the oldest job that never ran). */
|
||||
enum : uint32_t {PreemptSwitchTimeoutUs=1000000,PreemptStrandedUs=2000000,PreemptAgingUs=250000}; /* switch watchdog 1 s; a ring-1 job unreached for 2 s */
|
||||
bool preempt_dead_=false,stranded_=false;
|
||||
enum : uint32_t {PreemptSwitchTimeoutUs=1000000,PreemptStrandedUs=2000000,PreemptAgingUs=250000,PreemptAgingSliceUs=2000}; /* switch watchdog 1 s; a ring-1 job unreached for 2 s */
|
||||
bool preempt_dead_=false,stranded_=false,aging_grant_=false;
|
||||
uint64_t low_service_us_=0,low_slice_until_=0;
|
||||
uint32_t dead_reason_=0,rescues_=0;
|
||||
void markPreemptDead(uint32_t reason) {if(!preempt_dead_) {preempt_dead_=true;dead_reason_=reason;}}
|
||||
NativeAdrenoRing *ringOfFence(uint32_t fence) const;
|
||||
unsigned ringForPriority() const;
|
||||
bool completeSwitch();
|
||||
/* The oldest job on `index` that the ring's CP has not finished (its fence beyond what that ring last wrote); null when none. */
|
||||
const Job *pendingOn(unsigned index);
|
||||
void noteFence(uint32_t fence,uint8_t ring);
|
||||
@@ -162,6 +164,8 @@ public:
|
||||
/* The interrupt's side of a switch: the ring the CP moved to is the one whose write pointer is given again. Returns whether a
|
||||
* switch completed. Called from the GPU interrupt (native_adreno_preempt_irq). */
|
||||
bool preemptIrq();
|
||||
/* Global completed prefix for scanout: later completions on another ring cannot skip pending work. */
|
||||
bool fenceProgress(uint32_t *submitted,uint32_t *retired);
|
||||
/* The switch counters, the ring the CP runs, whether a switch is in flight and the control register as the hardware holds it. */
|
||||
/* A word of ring `index`'s preemption record as the CPU sees it now (the CP saves the read pointer, info and errno there). */
|
||||
uint32_t recordWord(unsigned index,uint32_t byte_offset) const
|
||||
|
||||
+15
-3
@@ -297,12 +297,22 @@ int main()
|
||||
CHECK(p.submit.setTableSwitching(2,write,nullptr)==0);
|
||||
p.ring.tableInstalled(0xaaaa000);low.tableInstalled(0xbbbb000);p.ring.useContext(42);
|
||||
NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};uint32_t first=0,second=0;
|
||||
p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&first,&ringclock)==0);
|
||||
clock_us=1;p.submit.nextPriority(1);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&first,&ringclock)==0);
|
||||
CHECK(low.lastSubmitted()==first && p.ring.lastSubmitted()==0 && low.table()==p.vm.ttbr0() && p.ring.table()==0xaaaa000);
|
||||
CHECK(low.context()==42 && cp.regs[0x08a0]==0x341);
|
||||
uint32_t overtaking=0;
|
||||
p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,8,&overtaking,&ringclock)==0);
|
||||
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq());
|
||||
CHECK(cp.regs[0x08a0]==0x341 && p.submit.deadReason()==0); // immediately switch back for newly queued high work
|
||||
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq());
|
||||
clock_us=300001;CHECK(p.submit.retireCompleted()==0 && cp.regs[0x08a0]==0x341);
|
||||
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq() && cp.regs[0x08a0]==0x340); // aged low ring gets a time slice
|
||||
clock_us+=2001;CHECK(p.submit.retireCompleted()==0 && cp.regs[0x08a0]==0x341);
|
||||
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq() && cp.regs[0x08a0]==0x340); // no immediate aging ping-pong
|
||||
fencemem[1]=overtaking;CHECK(p.submit.retireCompleted()==0);
|
||||
cp.regs[0x08a0]=0x340;CHECK(p.submit.preemptIrq());
|
||||
p.submit.nextPriority(0);CHECK(p.submit.submit(&p.vm,&buffer,1,7,&second,&ringclock)==0);
|
||||
CHECK(low.lastSubmitted()==second && p.ring.lastSubmitted()==0); // own queue cannot overtake
|
||||
CHECK(low.lastSubmitted()==second && p.ring.lastSubmitted()==overtaking); // own queue cannot overtake
|
||||
lowfence[1]=second;CHECK(p.submit.retireCompleted()==0);
|
||||
CHECK(p.submit.submit(&p.vm,&buffer,1,7,&second,&ringclock)==0 && p.ring.lastSubmitted()==second);
|
||||
CHECK(p.ring.table()==p.vm.ttbr0());
|
||||
@@ -317,8 +327,10 @@ int main()
|
||||
CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==1);
|
||||
cp.regs[0x08a0]=0x340;(void)p.submit.preemptIrq();
|
||||
}
|
||||
uint32_t issued=0,retired=0;CHECK(p.submit.fenceProgress(&issued,&retired) && issued==second && retired==first-1);
|
||||
CHECK(p.submit.observeFence(first,nullptr,0)==kIOReturnTimeout);
|
||||
lowfence[1]=first;CHECK(p.submit.retireCompleted()==0 && p.submit.outstanding()==0);
|
||||
CHECK(p.submit.fenceProgress(&issued,&retired) && issued==second && retired==second);
|
||||
CHECK(p.submit.wait(first,&ringclock,0)==0 && p.submit.observeFence(first,nullptr,0)==0);
|
||||
CHECK(p.submit.wait(second+1,&ringclock,0)==kIOReturnBadArgument);
|
||||
CHECK(p.submit.setTableSwitching(1,write,nullptr)==0);p.submit.nextPriority(2);
|
||||
@@ -326,7 +338,7 @@ int main()
|
||||
fencemem[1]=second;CHECK(p.submit.retireCompleted()==0);
|
||||
std::puts("PASS two-ring priorities, per-ring table/context, queue FIFO, fence-history rollover and CPU-table fallback");
|
||||
}
|
||||
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 unusable-VM refusal and 1 failed restart\n");
|
||||
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed); 1 hangcheck and 1 recovery with replay, 1 usable VM after recovery and 1 failed restart\n");
|
||||
}
|
||||
/* Physical-page maintenance (imported BOs): record it; no host page is touched. */
|
||||
/* Host double: the page attribute change of user BOs has no host equivalent (no host page is touched). */
|
||||
|
||||
Reference in New Issue
Block a user