mirror of
https://github.com/UniXNU/NativeAdreno.git
synced 2026-11-17 04:51:59 +03:00
db7751c3c3
Every process table maps the GPU context's eleven kernel buffers (8548 KiB, two preemption records of 4192 KiB among them), and each submission cleaned and invalidated all of them: 11 passes and 8548 KiB a submission in macOS runs 57 to 61, about 351 of a submission's 579 us in run61. Their users keep the GPU's view current themselves (a non-cacheable view, or a clean after a store through the kernel's mapping), so the buffers are declared synchronized and a map, an unmap and a submission leave them alone. The probe's store through the kernel's mapping is cleaned where it is made. The first 8 submissions and every 1024th audit the head of each buffer against DRAM; a dirty line restores the maintenance for good. A submission, a completion and the write-combine re-assertion visit only the mappings they have work for instead of every mapping of the client. The submission log reads the IB words only with diag=1 and no longer through one kernel mapping per word. gpu-maint-all=1 restores the passes over everything for a comparison. Counters: native_adreno_maint_kind, native_adreno_maint_walk, native_adreno_sync_audit, native_adreno_sync_skips, native_adreno_submit_us. Host tests cover it (test_maintenance, test_gpuctx, test_submit). Not run on hardware.
105 lines
6.8 KiB
C++
105 lines
6.8 KiB
C++
/* SPDX-License-Identifier: MIT */
|
|
/* The GPU context's kernel buffers over a software SMMU register peer: the bring-up declares them synchronized, a process address
|
|
* space maps and unmaps all of them without CPU cache maintenance, and the one store the driver makes through its own mapping
|
|
* (the table-switch probe) is cleaned where it is made. gpu-maint-all=1 gives the maintenance at every map and unmap back. */
|
|
#include "XnuMemoryFixture.h"
|
|
#include "NativeAdrenoGpuContext.h"
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <vector>
|
|
#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0)
|
|
IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state)
|
|
{CHECK(state);*state={};return kIOReturnSuccess;}
|
|
/* Maintenance through a kernel mapping: every call, with its operation and size. */
|
|
struct Call {void *mapping;size_t bytes;unsigned op;};
|
|
static std::vector<Call> calls;
|
|
IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op)
|
|
{CHECK(mapping && bytes && op<=NativeAdrenoInvalidate);calls.push_back({mapping,bytes,(unsigned)op});return kIOReturnSuccess;}
|
|
#define SMMU_BYTES 0x40000
|
|
alignas(8) static uint32_t mmio[SMMU_BYTES/4];
|
|
static void reset()
|
|
{
|
|
std::memset(mmio,0,sizeof(mmio));
|
|
mmio[0]=0x002f0406;mmio[0x20/4]=0x4c017e18;mmio[0x24/4]=0x40000016;mmio[0x28/4]=0x1433;
|
|
for(unsigned i=0;i<24;++i)mmio[(0xc00+4*i)/4]=0x20000; /* every S2CR FAULT, every SMR invalid */
|
|
}
|
|
static int read32(void *,size_t at,uint32_t *v){CHECK(at%4==0 && at<=SMMU_BYTES-4);*v=mmio[at/4];return 0;}
|
|
static int read64(void *,size_t at,uint64_t *v){CHECK(at%8==0 && at<=SMMU_BYTES-8);std::memcpy(v,mmio+at/4,8);return 0;}
|
|
static int write32(void *,size_t at,uint32_t v)
|
|
{
|
|
CHECK(at%4==0 && at<=SMMU_BYTES-4);
|
|
if((at&0xfff)==0x58 && at>=0x20000) {mmio[at/4]&=~v;return 0;} /* context FSR is write-one-to-clear */
|
|
mmio[at/4]=v;return 0;
|
|
}
|
|
static int write64(void *,size_t at,uint64_t v){CHECK(at%8==0 && at<=SMMU_BYTES-8);std::memcpy(mmio+at/4,&v,8);return 0;}
|
|
static int barrier(void *){return 0;}
|
|
/* How many of the calls from `first` on were made on one of the context's buffers, and with which operation. */
|
|
static unsigned onBuffers(const NativeAdrenoGpuContext &context,size_t first,unsigned op)
|
|
{
|
|
unsigned found=0;
|
|
for(size_t c=first;c<calls.size();++c)for(unsigned i=0;i<GpuBufferCount;++i)
|
|
if(calls[c].mapping==context.buffer(i).mapping() && calls[c].op==op)++found;
|
|
return found;
|
|
}
|
|
int main()
|
|
{
|
|
const ArmSmmu4KIo io{mmio,SMMU_BYTES,read32,read64,write32,write64,barrier};
|
|
const unsigned slots[2]={0,1};const uint16_t sids[2]={0,1};
|
|
native_adreno_maint_all=0;
|
|
reset();
|
|
static NativeAdrenoGpuContext context;
|
|
NativeAdrenoGmuContextReport report;
|
|
CHECK(context.bringUp(&io,0,2,slots,sids,nullptr,&report)==kIOReturnSuccess && context.ready());
|
|
size_t kib=0;
|
|
for(unsigned i=0;i<GpuBufferCount;++i) {CHECK(context.buffer(i).synchronized());kib+=context.buffer(i).bytes()>>10;}
|
|
CHECK(kib==8548); /* what every submission used to clean and invalidate in each process table */
|
|
/* Each buffer: the zero fill cleaned, then cleaned and dropped as it is declared; the map into the kernel table adds nothing. */
|
|
CHECK(onBuffers(context,0,NativeAdrenoClean)==GpuBufferCount && onBuffers(context,0,NativeAdrenoCleanInvalidate)==GpuBufferCount &&
|
|
onBuffers(context,0,NativeAdrenoInvalidate)==0);
|
|
CHECK(native_adreno_sync_skips[0]==GpuBufferCount);
|
|
CHECK(context.enableSwitching()==kIOReturnSuccess);
|
|
/* A process address space maps every buffer at its fixed address and no CPU cache line is touched for it. */
|
|
NativeAdrenoProcessVm *space=nullptr;
|
|
size_t mark=calls.size();
|
|
CHECK(context.createVm(&space)==kIOReturnSuccess && space && space->vm() && space->vm()->objectCount()==GpuBufferCount);
|
|
for(unsigned op=NativeAdrenoClean;op<=NativeAdrenoInvalidate;++op)CHECK(onBuffers(context,mark,op)==0);
|
|
CHECK(native_adreno_sync_skips[0]==2*GpuBufferCount);
|
|
/* A submission's pass over that table: nothing to visit, eleven mappings left alone. */
|
|
const uint64_t left=native_adreno_maint_kind[NativeAdrenoKindDriver][2],visited=native_adreno_maint_walk[1];
|
|
mark=calls.size();
|
|
CHECK(space->vm()->publishMapped()==kIOReturnSuccess && calls.size()==mark);
|
|
CHECK(native_adreno_maint_kind[NativeAdrenoKindDriver][2]==left+GpuBufferCount && native_adreno_maint_walk[1]==visited);
|
|
/* The probe's store goes through the kernel's mapping of the scratch page: that page is cleaned there. */
|
|
uint64_t handle=0,iova=0;size_t bytes=0;
|
|
mark=calls.size();
|
|
CHECK(context.probeBuffer(space,&handle,&iova,&bytes)==kIOReturnSuccess && handle && iova==GpuScratchIova+0xF00 && bytes==4);
|
|
CHECK(calls.size()==mark+1 && calls[mark].mapping==context.buffer(4).mapping() && calls[mark].bytes==GpuPageBytes &&
|
|
calls[mark].op==NativeAdrenoClean);
|
|
CHECK(static_cast<const uint32_t *>(context.buffer(4).mapping())[0xF00/4]!=0);
|
|
/* Its teardown unmaps them without invalidating: ten are writable, the SQE is not. */
|
|
mark=calls.size();
|
|
CHECK(context.destroyVm(space)==kIOReturnSuccess && onBuffers(context,mark,NativeAdrenoInvalidate)==0);
|
|
CHECK(native_adreno_sync_skips[1]==GpuBufferCount-1 && context.liveVms()==0);
|
|
/* The comparison: every map cleans and invalidates what the GPU may write (and cleans the read-only SQE), every unmap
|
|
* invalidates the writable ones. */
|
|
native_adreno_maint_all=1;
|
|
mark=calls.size();
|
|
CHECK(context.createVm(&space)==kIOReturnSuccess);
|
|
CHECK(onBuffers(context,mark,NativeAdrenoCleanInvalidate)==GpuBufferCount-1 && onBuffers(context,mark,NativeAdrenoClean)==1);
|
|
mark=calls.size();
|
|
CHECK(space->vm()->publishMapped()==kIOReturnSuccess && onBuffers(context,mark,NativeAdrenoCleanInvalidate)==GpuBufferCount);
|
|
mark=calls.size();
|
|
CHECK(context.destroyVm(space)==kIOReturnSuccess && onBuffers(context,mark,NativeAdrenoInvalidate)==GpuBufferCount-1);
|
|
native_adreno_maint_all=0;
|
|
std::printf("PASS: GPU context bring-up declares its 11 buffers (8548 KiB) synchronized; a process address space maps, submits over "
|
|
"and unmaps them with no CPU cache maintenance; the probe cleans the page it stored into; gpu-maint-all restores the passes\n");
|
|
}
|
|
IOReturn NativeAdrenoSetPagesCacheMode(const NativeAdrenoPhysicalSpan *spans,size_t count,bool){return spans && count?kIOReturnSuccess:kIOReturnBadArgument;}
|
|
IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans,size_t count,uint32_t *reset)
|
|
{return spans && count && reset?kIOReturnSuccess:kIOReturnBadArgument;}
|
|
IOReturn NativeAdrenoMaintainPhysicalPages(uint64_t physical,size_t bytes,NativeAdrenoCacheOperation op)
|
|
{
|
|
if(!physical || !bytes || ((physical|bytes)&4095) || op>NativeAdrenoInvalidate)std::abort();
|
|
++fixture.physical_calls;fixture.physical_bytes+=bytes;return kIOReturnSuccess;
|
|
}
|