/* SPDX-License-Identifier: MIT */ #include "NativeAdrenoPreempt.h" #include /* Pinned definitions, all from registers/adreno/a6xx.xml (dword register indices) and a6xx_preempt.c: * CP_CONTEXT_SWITCH_CNTL 0x08a0, CP_CONTEXT_SWITCH_SMMU_INFO 0x08a1, CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR 0x08a3, * RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE 0x8e50. * The control word: STOP at bit 0, LEVEL at bits 6..7, USES_GMEM at bit 8, SKIP_SAVE_RESTORE at bit 9 (a6xx_cp_context_switch_cntl). * a6xx_preempt_trigger writes LEVEL(1) | USES_GMEM | SKIP_SAVE_RESTORE | STOP with the incoming ring's SMMU info and record * already programmed, and keeps the GPU awake with GMU_PWR_COL_PREEMPT_KEEPALIVE until a6xx_preempt_irq finds STOP cleared. * A record's magic is A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6E (a6xx_gpu.h) and the SMMU info's GEN7_CP_SMMU_INFO_MAGIC 0x241350d5. */ enum : uint32_t { PreemptCntl = 0x08a0, PreemptSmmuInfo = 0x08a1, PreemptRestoreAddr = 0x08a3, PreemptGmemSaveRestore = 0x8e50, CntlStop = 1u << 0, CntlLevel1 = 1u << 6, CntlUsesGmem = 1u << 8, CntlSkipSaveRestore = 1u << 9, RecordMagic = 0xAE399D6Eu, SmmuInfoMagic = 0x241350d5u, GmuPreemptKeepalive = 0x1f8c4, /* a6xx_gmu.xml GMU_PWR_COL_PREEMPT_KEEPALIVE (dword offset in the GMU's window) */ RecordCntlOffset = 16, /* magic, info, errno, data, cntl */ RecordRptrOffset = 20, RecordWptrOffset = 24, RecordRptrAddrOffset = 32, RecordRbaseOffset = 40, RecordCounterOffset = 48, RecordBvRptrAddrOffset = 56 }; extern "C" {volatile uint32_t native_adreno_switch_log[NativeAdrenoSwitchLogRows][NativeAdrenoSwitchLogWords];volatile uint32_t native_adreno_switch_count;} enum : uint32_t {RegRbbmStatus=0x0210,RegRbbmInt0Status=0x0201,RegCpHwFault=0x0821,RegCpRbRptr=0x0806,RegCpRbWptr=0x0807}; static void put32(uint8_t *at,uint32_t value) { at[0]=(uint8_t)value;at[1]=(uint8_t)(value>>8);at[2]=(uint8_t)(value>>16);at[3]=(uint8_t)(value>>24); } static void put64(uint8_t *at,uint64_t value) { put32(at,(uint32_t)value);put32(at+4,(uint32_t)(value>>32)); } static uint32_t get32(const uint8_t *at) { return uint32_t(at[0])|uint32_t(at[1])<<8|uint32_t(at[2])<<16|uint32_t(at[3])<<24; } bool NativeAdrenoPreempt::writeRecord(uint8_t *record,uint64_t rbase,uint64_t rptr_addr,uint64_t bv_rptr_addr,uint32_t cntl,uint32_t wptr) { if(!record || !rbase || !rptr_addr)return false; put32(record+0,RecordMagic); put32(record+4,0); /* info: written by the CP */ put32(record+8,0); /* errno: written by the CP */ put32(record+12,0); /* data: the YIELD and SET_MARKER payload, written by the CP */ put32(record+RecordCntlOffset,cntl); put32(record+RecordRptrOffset,0); put32(record+RecordWptrOffset,wptr); put32(record+28,0); put64(record+RecordRptrAddrOffset,rptr_addr); put64(record+RecordRbaseOffset,rbase); put64(record+RecordCounterOffset,0); /* the preemption counters: 0 disables the block, as msm's A7XX rings do */ put64(record+RecordBvRptrAddrOffset,bv_rptr_addr); return true; } bool NativeAdrenoPreempt::setRecordWptr(uint8_t *record,uint32_t wptr) { if(!record)return false; put32(record+RecordWptrOffset,wptr); return true; } uint32_t NativeAdrenoPreempt::recordWord(const uint8_t *record,uint32_t byte_offset) { return record&&byte_offset<64?get32(record+byte_offset):0; } uint32_t NativeAdrenoPreempt::recordWptr(const uint8_t *record) { return record?get32(record+RecordWptrOffset):0; } bool NativeAdrenoPreempt::writeSmmuInfo(uint8_t *page,uint64_t ttbr0,uint32_t asid,uint32_t context_idr) { if(!page || !ttbr0)return false; put32(page+0,SmmuInfoMagic); put32(page+4,0); put64(page+8,ttbr0); put32(page+16,asid); put32(page+20,context_idr); put32(page+24,0); /* context_bank: the GPU's bank is 0 */ return true; } IOReturn NativeAdrenoPreempt::attach(const NativeAdrenoPreemptIo *io,const NativeAdrenoPreemptRings &rings, uint32_t loaded,uint64_t (*now)(void *),void *clock) { if(attached_)return kIOReturnBusy; if(!io || !io->write32 || !io->read32 || !io->barrier)return kIOReturnBadArgument; if(loaded>=NativeAdrenoPreemptRingCount)return kIOReturnBadArgument; for(uint32_t i=0;i>32); row[5]=loaded_|to<<8; snapshot(row,12); native_adreno_switch_count=n+1; return row; } void NativeAdrenoPreempt::closeEvent(uint32_t outcome) { volatile uint32_t *row=event_; if(!row)return; const uint64_t now=now_us_?now_us_(clock_cookie_):0; row[3]=uint32_t(now);row[4]=uint32_t(now>>32); row[5]=(row[5]&0xffffu)|outcome<<16; uint32_t v=0xffffffffu; row[7]=readCntl(); row[8]=io_.read32(io_.cookie,RegRbbmStatus,&v)?0xffffffffu:v;v=0xffffffffu; row[9]=io_.read32(io_.cookie,RegRbbmInt0Status,&v)?0xffffffffu:v;v=0xffffffffu; row[10]=io_.read32(io_.cookie,RegCpHwFault,&v)?0xffffffffu:v; uint32_t rptr=0xffff,wptr=0xffff; if(!io_.read32(io_.cookie,RegCpRbRptr,&v))rptr=v&0xffff; if(!io_.read32(io_.cookie,RegCpRbWptr,&v))wptr=v&0xffff; row[11]=rptr|wptr<<16; snapshot(row,20); event_=nullptr; } void NativeAdrenoPreempt::noteTimeout(const uint32_t progress[6]) { if(!armed_ || !event_)return; if(progress)for(unsigned i=0;i<6;++i)event_[28+i]=progress[i]; stats_.faulted++; closeEvent(2); } bool NativeAdrenoPreempt::trigger(uint32_t ring,bool uses_gmem) { if(!attached_ || armed_ || ring>=NativeAdrenoPreemptRingCount || ring==loaded_)return false; /* Hold the GPU awake before programming any restore register. */ if(io_.gmu_write32 && io_.gmu_write32(io_.gmu_cookie,GmuPreemptKeepalive,1))return false; keepalive_=true; if(io_.barrier(io_.cookie)) {releaseKeepalive();return false;} const uint64_t info=rings_.smmu_info[ring],record=rings_.record[ring]; event_=openEvent(ring); if(io_.write32(io_.cookie,PreemptSmmuInfo,(uint32_t)info) || io_.write32(io_.cookie,PreemptSmmuInfo+1,(uint32_t)(info>>32)) || io_.write32(io_.cookie,PreemptRestoreAddr,(uint32_t)record) || io_.write32(io_.cookie,PreemptRestoreAddr+1,(uint32_t)(record>>32))) {stats_.faulted++;closeEvent(3);releaseKeepalive();return false;} if(io_.barrier(io_.cookie)) {stats_.faulted++;closeEvent(3);releaseKeepalive();return false;} const uint32_t cntl=CntlStop|CntlLevel1|CntlSkipSaveRestore|(uses_gmem?CntlUsesGmem:0); if(io_.write32(io_.cookie,PreemptCntl,cntl)) {stats_.faulted++;closeEvent(3);releaseKeepalive();return false;} event_[6]=readCntl(); next_=ring;armed_=true; trigger_us_=now_us_?now_us_(clock_cookie_):0; stats_.triggers=stats_.triggers+1; return true; } bool NativeAdrenoPreempt::timedOut(uint32_t limit_us) const { if(!armed_ || !now_us_)return false; return now_us_(clock_cookie_)-trigger_us_>limit_us; } uint32_t NativeAdrenoPreempt::readCntl() const { uint32_t value=0xffffffffu; if(attached_ && io_.read32)(void)io_.read32(io_.cookie,PreemptCntl,&value); return value; } bool NativeAdrenoPreempt::complete() { if(!attached_ || !armed_)return false; uint32_t cntl=0; if(io_.read32(io_.cookie,PreemptCntl,&cntl)) {stats_.faulted=stats_.faulted+1;return false;} if(cntl&CntlStop)return false; closeEvent(1); loaded_=next_;armed_=false; if(trigger_us_ && now_us_) { const uint64_t us=now_us_(clock_cookie_)-trigger_us_; stats_.last_us=uint32_t(us>UINT32_MAX?UINT32_MAX:us); } stats_.completed=stats_.completed+1; return true; } bool NativeAdrenoPreempt::releaseKeepalive() { if(!keepalive_)return true; if(io_.gmu_write32 && io_.gmu_write32(io_.gmu_cookie,GmuPreemptKeepalive,0))return false; keepalive_=false;return true; }