mirror of
https://github.com/UniXNU/NativeAdreno.git
synced 2026-11-17 13:01:46 +03:00
216 lines
9.3 KiB
C++
216 lines
9.3 KiB
C++
/* SPDX-License-Identifier: MIT */
|
|
#include "NativeAdrenoPreempt.h"
|
|
#include <IOKit/IOLib.h>
|
|
|
|
/* Pinned definitions, all from registers/adreno/a6xx.xml (dword register indices) and a6xx_preempt.c:
|
|
* CP_CONTEXT_SWITCH_CNTL 0x08a0, CP_CONTEXT_SWITCH_SMMU_INFO 0x08a1, CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR 0x08a3,
|
|
* RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE 0x8e50.
|
|
* The control word: STOP at bit 0, LEVEL at bits 6..7, USES_GMEM at bit 8, SKIP_SAVE_RESTORE at bit 9 (a6xx_cp_context_switch_cntl).
|
|
* a6xx_preempt_trigger writes LEVEL(1) | USES_GMEM | SKIP_SAVE_RESTORE | STOP with the incoming ring's SMMU info and record
|
|
* already programmed, and keeps the GPU awake with GMU_PWR_COL_PREEMPT_KEEPALIVE until a6xx_preempt_irq finds STOP cleared.
|
|
* A record's magic is A6XX_PREEMPT_RECORD_MAGIC 0xAE399D6E (a6xx_gpu.h) and the SMMU info's GEN7_CP_SMMU_INFO_MAGIC 0x241350d5. */
|
|
enum : uint32_t {
|
|
PreemptCntl = 0x08a0,
|
|
PreemptSmmuInfo = 0x08a1,
|
|
PreemptRestoreAddr = 0x08a3,
|
|
PreemptGmemSaveRestore = 0x8e50,
|
|
CntlStop = 1u << 0,
|
|
CntlLevel1 = 1u << 6,
|
|
CntlUsesGmem = 1u << 8,
|
|
CntlSkipSaveRestore = 1u << 9,
|
|
RecordMagic = 0xAE399D6Eu,
|
|
SmmuInfoMagic = 0x241350d5u,
|
|
GmuPreemptKeepalive = 0x1f8c4, /* a6xx_gmu.xml GMU_PWR_COL_PREEMPT_KEEPALIVE (dword offset in the GMU's window) */
|
|
RecordCntlOffset = 16, /* magic, info, errno, data, cntl */
|
|
RecordRptrOffset = 20,
|
|
RecordWptrOffset = 24,
|
|
RecordRptrAddrOffset = 32,
|
|
RecordRbaseOffset = 40,
|
|
RecordCounterOffset = 48,
|
|
RecordBvRptrAddrOffset = 56
|
|
};
|
|
extern "C" {volatile uint32_t native_adreno_switch_log[NativeAdrenoSwitchLogRows][NativeAdrenoSwitchLogWords];volatile uint32_t native_adreno_switch_count;}
|
|
enum : uint32_t {RegRbbmStatus=0x0210,RegRbbmInt0Status=0x0201,RegCpHwFault=0x0821,RegCpRbRptr=0x0806,RegCpRbWptr=0x0807};
|
|
static void put32(uint8_t *at,uint32_t value)
|
|
{
|
|
at[0]=(uint8_t)value;at[1]=(uint8_t)(value>>8);at[2]=(uint8_t)(value>>16);at[3]=(uint8_t)(value>>24);
|
|
}
|
|
static void put64(uint8_t *at,uint64_t value)
|
|
{
|
|
put32(at,(uint32_t)value);put32(at+4,(uint32_t)(value>>32));
|
|
}
|
|
static uint32_t get32(const uint8_t *at)
|
|
{
|
|
return uint32_t(at[0])|uint32_t(at[1])<<8|uint32_t(at[2])<<16|uint32_t(at[3])<<24;
|
|
}
|
|
bool NativeAdrenoPreempt::writeRecord(uint8_t *record,uint64_t rbase,uint64_t rptr_addr,uint64_t bv_rptr_addr,uint32_t cntl,uint32_t wptr)
|
|
{
|
|
if(!record || !rbase || !rptr_addr)return false;
|
|
put32(record+0,RecordMagic);
|
|
put32(record+4,0); /* info: written by the CP */
|
|
put32(record+8,0); /* errno: written by the CP */
|
|
put32(record+12,0); /* data: the YIELD and SET_MARKER payload, written by the CP */
|
|
put32(record+RecordCntlOffset,cntl);
|
|
put32(record+RecordRptrOffset,0);
|
|
put32(record+RecordWptrOffset,wptr);
|
|
put32(record+28,0);
|
|
put64(record+RecordRptrAddrOffset,rptr_addr);
|
|
put64(record+RecordRbaseOffset,rbase);
|
|
put64(record+RecordCounterOffset,0); /* the preemption counters: 0 disables the block, as msm's A7XX rings do */
|
|
put64(record+RecordBvRptrAddrOffset,bv_rptr_addr);
|
|
return true;
|
|
}
|
|
bool NativeAdrenoPreempt::setRecordWptr(uint8_t *record,uint32_t wptr)
|
|
{
|
|
if(!record)return false;
|
|
put32(record+RecordWptrOffset,wptr);
|
|
return true;
|
|
}
|
|
uint32_t NativeAdrenoPreempt::recordWord(const uint8_t *record,uint32_t byte_offset)
|
|
{
|
|
return record&&byte_offset<64?get32(record+byte_offset):0;
|
|
}
|
|
uint32_t NativeAdrenoPreempt::recordWptr(const uint8_t *record)
|
|
{
|
|
return record?get32(record+RecordWptrOffset):0;
|
|
}
|
|
bool NativeAdrenoPreempt::writeSmmuInfo(uint8_t *page,uint64_t ttbr0,uint32_t asid,uint32_t context_idr)
|
|
{
|
|
if(!page || !ttbr0)return false;
|
|
put32(page+0,SmmuInfoMagic);
|
|
put32(page+4,0);
|
|
put64(page+8,ttbr0);
|
|
put32(page+16,asid);
|
|
put32(page+20,context_idr);
|
|
put32(page+24,0); /* context_bank: the GPU's bank is 0 */
|
|
return true;
|
|
}
|
|
IOReturn NativeAdrenoPreempt::attach(const NativeAdrenoPreemptIo *io,const NativeAdrenoPreemptRings &rings,
|
|
uint32_t loaded,uint64_t (*now)(void *),void *clock)
|
|
{
|
|
if(attached_)return kIOReturnBusy;
|
|
if(!io || !io->write32 || !io->read32 || !io->barrier)return kIOReturnBadArgument;
|
|
if(loaded>=NativeAdrenoPreemptRingCount)return kIOReturnBadArgument;
|
|
for(uint32_t i=0;i<NativeAdrenoPreemptRingCount;++i)
|
|
if(!rings.record[i] || !rings.smmu_info[i])return kIOReturnBadArgument;
|
|
io_=*io;rings_=rings;loaded_=loaded;next_=loaded;armed_=false;
|
|
now_us_=now;clock_cookie_=clock;
|
|
/* a6xx_preempt_hw_init: the GMEM save and restore on, the SMMU info zeroed (no switch in flight). */
|
|
if(io_.write32(io_.cookie,PreemptGmemSaveRestore,1) ||
|
|
io_.write32(io_.cookie,PreemptSmmuInfo,0) || io_.write32(io_.cookie,PreemptSmmuInfo+1,0) ||
|
|
io_.barrier(io_.cookie))return kIOReturnIOError;
|
|
attached_=true;
|
|
return kIOReturnSuccess;
|
|
}
|
|
IOReturn NativeAdrenoPreempt::detach()
|
|
{
|
|
if(!attached_)return kIOReturnNotReady;
|
|
if(armed_)return kIOReturnBusy;
|
|
if(!releaseKeepalive())return kIOReturnIOError;
|
|
attached_=false;io_={};return kIOReturnSuccess;
|
|
}
|
|
/* Both rings' records as the CPU sees them: read pointer, write pointer, errno, info. */
|
|
void NativeAdrenoPreempt::snapshot(volatile uint32_t *row,unsigned first) const
|
|
{
|
|
__asm__ volatile("dsb sy":::"memory");
|
|
for(unsigned r=0;r<NativeAdrenoPreemptRingCount;++r) {
|
|
const uint8_t *rec=rings_.record_cpu[r];
|
|
row[first+r*4+0]=recordWord(rec,20);row[first+r*4+1]=recordWord(rec,24);
|
|
row[first+r*4+2]=recordWord(rec,8);row[first+r*4+3]=recordWord(rec,4);
|
|
}
|
|
}
|
|
volatile uint32_t *NativeAdrenoPreempt::openEvent(uint32_t to)
|
|
{
|
|
const uint32_t n=native_adreno_switch_count;
|
|
volatile uint32_t *row=native_adreno_switch_log[n&(NativeAdrenoSwitchLogRows-1)];
|
|
for(unsigned i=0;i<NativeAdrenoSwitchLogWords;++i)row[i]=0;
|
|
const uint64_t now=now_us_?now_us_(clock_cookie_):0;
|
|
row[0]=n+1;row[1]=uint32_t(now);row[2]=uint32_t(now>>32);
|
|
row[5]=loaded_|to<<8;
|
|
snapshot(row,12);
|
|
native_adreno_switch_count=n+1;
|
|
return row;
|
|
}
|
|
void NativeAdrenoPreempt::closeEvent(uint32_t outcome)
|
|
{
|
|
volatile uint32_t *row=event_;
|
|
if(!row)return;
|
|
const uint64_t now=now_us_?now_us_(clock_cookie_):0;
|
|
row[3]=uint32_t(now);row[4]=uint32_t(now>>32);
|
|
row[5]=(row[5]&0xffffu)|outcome<<16;
|
|
uint32_t v=0xffffffffu;
|
|
row[7]=readCntl();
|
|
row[8]=io_.read32(io_.cookie,RegRbbmStatus,&v)?0xffffffffu:v;v=0xffffffffu;
|
|
row[9]=io_.read32(io_.cookie,RegRbbmInt0Status,&v)?0xffffffffu:v;v=0xffffffffu;
|
|
row[10]=io_.read32(io_.cookie,RegCpHwFault,&v)?0xffffffffu:v;
|
|
uint32_t rptr=0xffff,wptr=0xffff;
|
|
if(!io_.read32(io_.cookie,RegCpRbRptr,&v))rptr=v&0xffff;
|
|
if(!io_.read32(io_.cookie,RegCpRbWptr,&v))wptr=v&0xffff;
|
|
row[11]=rptr|wptr<<16;
|
|
snapshot(row,20);
|
|
event_=nullptr;
|
|
}
|
|
void NativeAdrenoPreempt::noteTimeout(const uint32_t progress[6])
|
|
{
|
|
if(!armed_ || !event_)return;
|
|
if(progress)for(unsigned i=0;i<6;++i)event_[28+i]=progress[i];
|
|
stats_.faulted++;
|
|
closeEvent(2);
|
|
}
|
|
bool NativeAdrenoPreempt::trigger(uint32_t ring,bool uses_gmem)
|
|
{
|
|
if(!attached_ || armed_ || ring>=NativeAdrenoPreemptRingCount || ring==loaded_)return false;
|
|
/* Hold the GPU awake before programming any restore register. */
|
|
if(io_.gmu_write32 && io_.gmu_write32(io_.gmu_cookie,GmuPreemptKeepalive,1))return false;
|
|
keepalive_=true;
|
|
if(io_.barrier(io_.cookie)) {releaseKeepalive();return false;}
|
|
const uint64_t info=rings_.smmu_info[ring],record=rings_.record[ring];
|
|
event_=openEvent(ring);
|
|
if(io_.write32(io_.cookie,PreemptSmmuInfo,(uint32_t)info) ||
|
|
io_.write32(io_.cookie,PreemptSmmuInfo+1,(uint32_t)(info>>32)) ||
|
|
io_.write32(io_.cookie,PreemptRestoreAddr,(uint32_t)record) ||
|
|
io_.write32(io_.cookie,PreemptRestoreAddr+1,(uint32_t)(record>>32)))
|
|
{stats_.faulted++;closeEvent(3);releaseKeepalive();return false;}
|
|
if(io_.barrier(io_.cookie)) {stats_.faulted++;closeEvent(3);releaseKeepalive();return false;}
|
|
const uint32_t cntl=CntlStop|CntlLevel1|CntlSkipSaveRestore|(uses_gmem?CntlUsesGmem:0);
|
|
if(io_.write32(io_.cookie,PreemptCntl,cntl)) {stats_.faulted++;closeEvent(3);releaseKeepalive();return false;}
|
|
event_[6]=readCntl();
|
|
next_=ring;armed_=true;
|
|
trigger_us_=now_us_?now_us_(clock_cookie_):0;
|
|
stats_.triggers=stats_.triggers+1;
|
|
return true;
|
|
}
|
|
bool NativeAdrenoPreempt::timedOut(uint32_t limit_us) const
|
|
{
|
|
if(!armed_ || !now_us_)return false;
|
|
return now_us_(clock_cookie_)-trigger_us_>limit_us;
|
|
}
|
|
uint32_t NativeAdrenoPreempt::readCntl() const
|
|
{
|
|
uint32_t value=0xffffffffu;
|
|
if(attached_ && io_.read32)(void)io_.read32(io_.cookie,PreemptCntl,&value);
|
|
return value;
|
|
}
|
|
bool NativeAdrenoPreempt::complete()
|
|
{
|
|
if(!attached_ || !armed_)return false;
|
|
uint32_t cntl=0;
|
|
if(io_.read32(io_.cookie,PreemptCntl,&cntl)) {stats_.faulted=stats_.faulted+1;return false;}
|
|
if(cntl&CntlStop)return false;
|
|
closeEvent(1);
|
|
loaded_=next_;armed_=false;
|
|
if(trigger_us_ && now_us_) {
|
|
const uint64_t us=now_us_(clock_cookie_)-trigger_us_;
|
|
stats_.last_us=uint32_t(us>UINT32_MAX?UINT32_MAX:us);
|
|
}
|
|
stats_.completed=stats_.completed+1;
|
|
return true;
|
|
}
|
|
|
|
bool NativeAdrenoPreempt::releaseKeepalive()
|
|
{
|
|
if(!keepalive_)return true;
|
|
if(io_.gmu_write32 && io_.gmu_write32(io_.gmu_cookie,GmuPreemptKeepalive,0))return false;
|
|
keepalive_=false;return true;
|
|
}
|