/* SPDX-License-Identifier: MIT */ /* The passes over a VM's mappings (map, unmap, submission, completion, re-assertion) over the real VM and memory code and a software * SMMUv2 register peer: which mappings each pass maintains, which it leaves alone and how many it visits. Host cache doubles only * count their calls; no CPU cache, DRAM or GPU is involved. */ #include "XnuMemoryFixture.h" #include "NativeAdrenoVm.h" #include #include #define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) #define BYTES 0x40000 #define CB 0x22000 extern "C" volatile uint32_t native_adreno_io_coherent,native_adreno_coherent_skips[4]; IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) {CHECK(out);*out={};return kIOReturnSuccess;} /* Maintenance through a kernel mapping, counted for the buffers under test only (a map also cleans its new table pages this way): * their number and the operations in order. */ static std::vector tracked; static std::vector operations; static unsigned tracked_calls; IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) { CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); ++fixture.cache_calls; for(void *base:tracked)if(base==mapping) {++tracked_calls;operations.push_back(op);} return kIOReturnSuccess; } /* A buffer with a kernel mapping, allocated and tracked. */ static IOReturn buffer(NativeAdrenoMemory &memory,size_t bytes) { const IOReturn status=memory.allocate(bytes,40,true,bytes); if(!status)tracked.push_back(memory.mapping()); return status; } static uint32_t memory[BYTES/4]; static ArmSmmu4KIo io; static ArmSmmu4KStreamState streams; static ArmSmmu4KStream routes[2]={{2,0},{4,1}}; static ArmSmmu4KStreamResult route_result; static ArmSmmu4KContextConfig context; static ArmSmmu4KContextImage image; static int read32(void *,size_t at,uint32_t *v){CHECK(at%4==0 && at<=BYTES-4);*v=memory[at/4];return 0;} static int read64(void *,size_t at,uint64_t *v){CHECK(at%8==0 && at<=BYTES-8);std::memcpy(v,memory+at/4,8);return 0;} static int write32(void *,size_t at,uint32_t v){CHECK(at%4==0 && at<=BYTES-4);memory[at/4]=v;return 0;} static int write64(void *,size_t at,uint64_t v){CHECK(at%8==0 && at<=BYTES-8);std::memcpy(memory+at/4,&v,8);return 0;} static int barrier(void *){return 0;} struct Space { NativeAdrenoTablePages tables; ArmSmmu4KLiveArena arena; ArmSmmu4KLiveState state{}; ArmSmmu4KLiveVisibility visibility; NativeAdrenoVm vm; }; /* A prepared arena of 4096 pages at 0x800000, an enabled context image and exact stream routes. */ static IOReturn exposed(Space &s) { std::memset(memory,0,sizeof(memory));streams=(ArmSmmu4KStreamState){}; io=(ArmSmmu4KIo){&memory,BYTES,read32,read64,write32,write64,barrier}; memory[0x20/4]=(1u<<30)|(1u<<27)|(4u<<9)|8; memory[0x24/4]=0x40000010;memory[0x28/4]=0x1433;memory[0]=0x436; for(unsigned i=0;i<8;++i)memory[(0xc00+4*i)/4]=0x20000; memory[0x800/4]=1u<<31|9;memory[0xc00/4]=3; IOReturn smmu=NativeAdrenoPrepareSmmuMemory(&s.tables,32,40,0x800000,4096*4096,0,4096,&s.arena,&s.state,&s.visibility); if(smmu)return smmu; context=(ArmSmmu4KContextConfig){.lower=&s.arena.config,.lower_tables=&s.state.tables, .id0=memory[0x20/4],.id2=memory[0x28/4],.asid=1,.coherent_walk=1, .stall_on_fault=0,.continue_after_fault=1}; ArmSmmu4KContextConfig actual=context; CHECK(ArmSmmu4KEncodeContext(&actual,&image)==ArmSmmu4KSuccess); memory[0x1008/4]=image.cbar;memory[0x1808/4]=image.cba2r; memory[CB/4]=image.sctlr;memory[(CB+0x10)/4]=image.tcr2;memory[(CB+0x30)/4]=image.tcr; std::memcpy(memory+(CB+0x20)/4,&image.ttbr0,8); std::memcpy(memory+(CB+0x28)/4,&image.ttbr1,8); memory[(CB+0x38)/4]=image.mair0; if(ArmSmmu4KAttachStreams(&io,&streams,routes,2,2,&context,&route_result)!=ArmSmmu4KSuccess) return kIOReturnBadArgument; NativeAdrenoVmRegion lower={&s.tables,&s.arena,&s.state}; return s.vm.attach(&io,&context,&streams,&lower,nullptr,32,1000); } static void teardown(Space &s) { CHECK(s.vm.detach()==kIOReturnSuccess && s.tables.release()==kIOReturnSuccess); tracked.clear(); CHECK(fixture.live.empty() && fixture.metadata.empty() && !fixture.imports_live); } struct Counters { uint64_t kind[NativeAdrenoKindCount][3],walk[8]; uint32_t skips[2],coherent[4]; unsigned owned,physical; Counters() { for(unsigned k=0;k(0x77); CHECK(read.importTask(task,0x10000000,2*4096,40,2*4096)==kIOReturnSuccess); CHECK(written.importTask(task,0x20000000,2*4096,40,2*4096)==kIOReturnSuccess); uint64_t r=0,w=0; CHECK(s.vm.map(&read,0,0x800000,1,0,2,&r)==kIOReturnSuccess && s.vm.map(&written,0,0x810000,3,0,2,&w)==kIOReturnSuccess); Counters a; CHECK(s.vm.publishMapped()==kIOReturnSuccess && fixture.physical_calls==a.physical+4); /* two scattered spans each */ CHECK(native_adreno_maint_kind[NativeAdrenoKindImportRead][0]==a.kind[NativeAdrenoKindImportRead][0]+1 && native_adreno_maint_kind[NativeAdrenoKindImportWritten][0]==a.kind[NativeAdrenoKindImportWritten][0]+1); CHECK(s.vm.acquireImported()==kIOReturnSuccess && fixture.physical_calls==a.physical+6); CHECK(native_adreno_maint_walk[3]==a.walk[3]+1 && native_adreno_maint_walk[4]==a.walk[4]+2); native_adreno_io_coherent=1; Counters b; CHECK(s.vm.publishMapped()==kIOReturnSuccess && s.vm.acquireImported()==kIOReturnSuccess); CHECK(fixture.physical_calls==b.physical+6 && native_adreno_maint_walk[5]==b.walk[5]+1); CHECK(native_adreno_maint_walk[1]==b.walk[1]+2 && native_adreno_maint_walk[4]==b.walk[4]+2); CHECK(native_adreno_coherent_skips[2]==b.coherent[2] && native_adreno_coherent_skips[3]==b.coherent[3]); CHECK(native_adreno_maint_kind[NativeAdrenoKindImportRead][0]==b.kind[NativeAdrenoKindImportRead][0]+1 && native_adreno_maint_kind[NativeAdrenoKindImportWritten][0]==b.kind[NativeAdrenoKindImportWritten][0]+1); /* The comparison's passes count the same. */ native_adreno_maint_all=1; Counters c; CHECK(s.vm.publishMapped()==kIOReturnSuccess && s.vm.acquireImported()==kIOReturnSuccess); CHECK(fixture.physical_calls==c.physical+6); CHECK(native_adreno_coherent_skips[2]==c.coherent[2] && native_adreno_coherent_skips[3]==c.coherent[3]); CHECK(native_adreno_maint_walk[1]==c.walk[1]+2 && native_adreno_maint_walk[4]==c.walk[4]+2); native_adreno_maint_all=0;native_adreno_io_coherent=0; CHECK(s.vm.unmap(r)==kIOReturnSuccess && s.vm.unmap(w)==kIOReturnSuccess); CHECK(read.release()==kIOReturnSuccess && written.release()==kIOReturnSuccess); teardown(s); } /* The audit of the synchronized buffers: the first 32 KiB of each, a page a call, and every page's lines dropped again; other * mappings are not looked at. */ { Space s;CHECK(exposed(s)==kIOReturnSuccess); NativeAdrenoMemory small,large,plain; CHECK(buffer(small,8192)==kIOReturnSuccess && small.declareSynchronized()==kIOReturnSuccess); CHECK(buffer(large,65536)==kIOReturnSuccess && large.declareSynchronized()==kIOReturnSuccess); CHECK(buffer(plain,4096)==kIOReturnSuccess); uint64_t a=0,b=0,c=0; CHECK(s.vm.map(&small,0,0x800000,3,0,2,&a)==kIOReturnSuccess && s.vm.map(&large,0,0x810000,3,0,2,&b)==kIOReturnSuccess && s.vm.map(&plain,0,0x830000,3,0,2,&c)==kIOReturnSuccess); uint32_t lines=0,dirty=0,stale=0; CHECK(s.vm.auditSynchronized(&lines,&dirty,&stale)==kIOReturnUnsupported); /* no board hook */ static std::vector pages;pages.clear(); native_adreno_line_audit=[](uint64_t physical,size_t bytes,uint32_t *found,uint32_t *) { CHECK(bytes==4096 && !(physical&4095));pages.push_back(physical);*found+=1;return 0;}; const unsigned physical=fixture.physical_calls; CHECK(s.vm.auditSynchronized(&lines,&dirty,&stale)==kIOReturnSuccess); CHECK(pages.size()==10 && lines==640 && dirty==10 && fixture.physical_calls==physical+10); CHECK(pages[0]==small.spans()[0].physical && pages[1]==pages[0]+4096 && pages[2]==large.spans()[0].physical && pages[9]==pages[2]+7*4096); native_adreno_line_audit=[](uint64_t,size_t,uint32_t *,uint32_t *) {return 1;}; CHECK(s.vm.auditSynchronized(&lines,&dirty,&stale)==kIOReturnIOError); native_adreno_line_audit=nullptr; CHECK(s.vm.unmap(a)==kIOReturnSuccess && s.vm.unmap(b)==kIOReturnSuccess && s.vm.unmap(c)==kIOReturnSuccess); CHECK(small.release()==kIOReturnSuccess && large.release()==kIOReturnSuccess && plain.release()==kIOReturnSuccess); teardown(s); } /* The submission log's words: through the kernel's mapping where the memory has one; for system memory through the board's * cacheable reader, and through the uncached one only when that is all the board gave. */ { Space s;CHECK(exposed(s)==kIOReturnSuccess); NativeAdrenoMemory mapped,system; fixture.mode=ScatterUnlessContiguous; /* system memory arrives a page at a time */ CHECK(buffer(mapped,8192)==kIOReturnSuccess && system.allocateSystem(8192,40,8192,true)==kIOReturnSuccess); fixture.mode=Contiguous; CHECK(system.spanCount()==2); uint64_t m=0,y=0; CHECK(s.vm.map(&mapped,0,0x800000,3,0,2,&m)==kIOReturnSuccess && s.vm.map(&system,0,0x810000,3,0,2,&y)==kIOReturnSuccess); static_cast(mapped.mapping())[1024]=0x70970001u;static_cast(mapped.mapping())[1025]=3; static_cast(mapped.mapping())[1027]=0x1234; uint32_t words[4]={9,9,9,9}; CHECK(s.vm.headWords(m,0x801000,words,4)==kIOReturnSuccess && words[0]==0x70970001u && words[1]==3 && words[2]==0 && words[3]==0x1234); CHECK(s.vm.headWords(m,0x801ff8,words,4)==kIOReturnSuccess && words[2]==0 && words[3]==0); /* the object's end */ CHECK(s.vm.headWords(m,0x810000,words,4)==kIOReturnBadArgument && s.vm.headWords(m+7,0x800000,words,4)==kIOReturnBadArgument); CHECK(s.vm.headWords(y,0x810000,words,4)==kIOReturnUnsupported); /* no reader */ static std::vector cached,dram;cached.clear();dram.clear(); native_adreno_dram_read32=[](uint64_t physical,uint32_t *value) {dram.push_back(physical);*value=0xd0000000u;return 0;}; CHECK(s.vm.headWords(y,0x810ff8,words,4)==kIOReturnSuccess && dram.size()==4 && words[3]==0xd0000000u); /* The words cross from the first page into the second, which is elsewhere in DRAM. */ CHECK(dram[0]==system.spans()[0].physical+0xff8 && dram[1]==dram[0]+4 && dram[2]==system.spans()[1].physical && dram[3]==dram[2]+4); native_adreno_cached_read32=[](uint64_t physical,uint32_t *value) {cached.push_back(physical);*value=0xc0000000u;return 0;}; dram.clear(); CHECK(s.vm.headWords(y,0x810000,words,4)==kIOReturnSuccess && cached.size()==4 && dram.empty() && words[0]==0xc0000000u); native_adreno_dram_read32=nullptr;native_adreno_cached_read32=nullptr; CHECK(s.vm.unmap(m)==kIOReturnSuccess && s.vm.unmap(y)==kIOReturnSuccess); CHECK(mapped.release()==kIOReturnSuccess && system.release()==kIOReturnSuccess); teardown(s); } std::printf("PASS: synchronized driver buffers left out of map, submission and unmap; gpu-maint-all and the audit's distrust restore " "their maintenance; passes visit only mappings with work (304 objects, 3 visited); system memory by attribute; imports with and " "without IO coherence; the synchronized audit's pages; the submission log's readers\n"); } /* Physical-page maintenance (imports, system memory): record it; no host page is touched. */ IOReturn NativeAdrenoSetPagesCacheMode(const NativeAdrenoPhysicalSpan *spans,size_t count,bool){return spans && count?kIOReturnSuccess:kIOReturnBadArgument;} IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans,size_t count,uint32_t *reset) {return spans && count && reset?kIOReturnSuccess:kIOReturnBadArgument;} IOReturn NativeAdrenoMaintainPhysicalPages(uint64_t physical,size_t bytes,NativeAdrenoCacheOperation op) { if(!physical || !bytes || ((physical|bytes)&4095) || op>NativeAdrenoInvalidate)std::abort(); ++fixture.physical_calls;fixture.physical_bytes+=bytes;return kIOReturnSuccess; } void NativeAdrenoNotePhase(unsigned,size_t) {} bool NativeAdrenoPagesUncached(const NativeAdrenoPhysicalSpan *,size_t) {return false;} extern "C" unsigned int native_pmap_user_mappings(ppnum_t,void **,uint64_t *,char *) {return 0;}