commit 5a34e353ed67bf12ba8bf3a310c85fcde896bb87 Author: bluseliu50 <265648257+bluseliu50@users.noreply.github.com> Date: Tue Oct 20 18:12:06 2026 +0800 Import NativeMetal Co-authored-by: rpone diff --git a/CacheExport.h b/CacheExport.h new file mode 100644 index 0000000..9fb72ff --- /dev/null +++ b/CacheExport.h @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: MIT +// Pipeline-cache export (int-147). The render server's runtime keeps its translated shaders and the Vulkan driver's pipeline records in its +// cache directory, but that directory lives on the RAM Data disk: every boot compiled the whole UI again (int-146: 98 pipelines, 63 s, +// up to 1.6 s each, also while the user opened App Library). This thread sends the directory to the kernel's UFS log (method 0x7100 on a +// hand-off connection, kits/sheng/BootLog.c export region) whenever it changed and then stayed unchanged for one period; the host reads it +// back with the log and builds the next image's Data with it, so those pipelines start as cache hits. +// A boot whose image carried a pack exports only the files the pack did not have (see baked()). +// Stream: "M2VCACH1", u32 directory length, directory, u32 file count, then per file u32 path length, u32 mode, u32 uid, u32 gid, u64 size, +// path (relative to the directory), contents. Export blob: "SHEXR001", u32 stream bytes, u32 0, the stream as it is: it goes out in pieces +// while the files are read, so the render server never holds more than a piece (int-149: one boot's new variants were about 100 MB). +// Image pack: "SHEXP001", u32 stream bytes, u32 compressed bytes, the stream zlib-compressed (scripts/add-gpu-plugin-data.py merges exports +// into one). The export only reads. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef CACHEEXPORT_FORMAT_ONLY // the format alone (host check, bridge/cache_export_host.cpp); the plugin also gets the sender +#include "NativeAdrenoUserTest.h" +#endif + +namespace cacheexport { + +// kLimit: the largest export blob, one half of the kernel's 256 MiB export region. +constexpr uint32_t kMethod = 0x7100, kPiece = 3968, kLimit = 128u << 20; +struct File {std::string path; struct stat st;}; + +// A pack in the image is the union of several boots' exports (scripts/add-gpu-plugin-data.py); it is read as a stream, so only its own +// file and the stream bounds limit it. +constexpr uint64_t kPackLimit = 1ull << 30; + +// Files the image's pack carried (path and size). The export leaves them out: the host has them already and merges each boot's export +// with the earlier ones, so a boot sends only what it compiled itself (int-148: the whole directory outgrew the export area once the UI's +// own variants were added, and the export refused the build, marker 0x69f2). +static std::set &baked() {static std::set files; return files;} +static std::string bakedKey(const std::string &path, uint64_t size) {return path + '\n' + std::to_string(size);} +static void dropBaked(std::vector &files) { + if (baked().empty()) return; + std::vector fresh; + for (File &f : files) if (!baked().count(bakedKey(f.path, (uint64_t)f.st.st_size))) fresh.push_back(std::move(f)); + files.swap(fresh); +} + +static void scan(const std::string &root, const std::string &relative, std::vector &out, unsigned depth) { + if (depth > 8) return; + DIR *dir = opendir((root + "/" + relative).c_str()); + if (!dir) return; + while (dirent *entry = readdir(dir)) { + if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; + const std::string path = relative.empty() ? std::string(entry->d_name) : relative + "/" + entry->d_name; + struct stat st = {}; + if (lstat((root + "/" + path).c_str(), &st) != 0) continue; + if (S_ISDIR(st.st_mode)) scan(root, path, out, depth + 1); + else if (S_ISREG(st.st_mode)) out.push_back({path, st}); + } + closedir(dir); +} + +// Count, bytes and newest modification of the directory: equal on two scans = nothing is being written. +static uint64_t signature(const std::vector &files) { + uint64_t s = files.size() * 0x9e3779b97f4a7c15ull; + for (const File &f : files) s = s * 31 + (uint64_t)f.st.st_size * 1000003 + (uint64_t)f.st.st_mtimespec.tv_sec * 7 + (uint64_t)f.st.st_mtimespec.tv_nsec; + return s; +} + +template static void put(std::vector &b, T v) {const uint8_t *p = (const uint8_t *)&v; b.insert(b.end(), p, p + sizeof v);} + +// The stream's length for these files, from their sizes as scanned. +static uint64_t streamBytes(const std::string &root, const std::vector &files) { + uint64_t n = 8 + 4 + root.size() + 4; + for (const File &f : files) n += 24 + f.path.size() + (uint64_t)f.st.st_size; + return n; +} + +// The stream, handed to emit(bytes, count) in order (emit returns false to stop); false when a file vanished or no longer has the size +// it was scanned with, or emit stopped. +template static bool stream(const std::string &root, const std::vector &files, Emit &&emit) { + std::vector head; + {static const char magic[] = "M2VCACH1"; head.insert(head.end(), magic, magic + 8);} + put(head, (uint32_t)root.size()); + head.insert(head.end(), root.begin(), root.end()); + put(head, (uint32_t)files.size()); + if (!emit(head.data(), head.size())) return false; + std::vector chunk(65536); + for (const File &f : files) { + FILE *in = fopen((root + "/" + f.path).c_str(), "rb"); + if (!in) return false; + head.clear(); + put(head, (uint32_t)f.path.size()); + put(head, (uint32_t)f.st.st_mode); + put(head, (uint32_t)f.st.st_uid); + put(head, (uint32_t)f.st.st_gid); + put(head, (uint64_t)f.st.st_size); + head.insert(head.end(), f.path.begin(), f.path.end()); + bool ok = emit(head.data(), head.size()); + uint64_t left = (uint64_t)f.st.st_size; + while (ok && left) { + const size_t want = left < chunk.size() ? (size_t)left : chunk.size(); + ok = fread(chunk.data(), 1, want, in) == want && emit(chunk.data(), want); + left -= want; + } + ok = ok && fgetc(in) == EOF; // grown since the scan: refuse rather than send a short copy + fclose(in); + if (!ok) return false; + } + return true; +} + +#ifdef CACHEEXPORT_FORMAT_ONLY +// The image pack format (SHEXP001, compressed), for the host check; scripts/add-gpu-plugin-data.py builds the same. False when a file +// vanished or zlib is missing. +static bool build(const std::string &root, const std::vector &files, std::vector &out) { + using BoundFn = unsigned long (*)(unsigned long); + using CompressFn = int (*)(uint8_t *, unsigned long *, const uint8_t *, unsigned long, int); + static void *z = dlopen("/usr/lib/libz.1.dylib", RTLD_NOW | RTLD_LOCAL); + if (!z) return false; + auto bound = (BoundFn)dlsym(z, "compressBound"); + auto compress = (CompressFn)dlsym(z, "compress2"); + if (!bound || !compress) return false; + std::vector raw; + if (!stream(root, files, [&](const uint8_t *p, size_t n) {raw.insert(raw.end(), p, p + n); return true;})) return false; + unsigned long zbytes = bound(raw.size()); + std::vector zdata(zbytes); + if (compress(zdata.data(), &zbytes, raw.data(), raw.size(), 6) != 0) return false; + out.clear(); + {static const char magic[] = "SHEXP001"; out.insert(out.end(), magic, magic + 8);} + put(out, (uint32_t)raw.size()); + put(out, (uint32_t)zbytes); + out.insert(out.end(), zdata.begin(), zdata.begin() + (long)zbytes); + return true; +} +#endif + +// Reads a pack's stream: inflated as it is consumed, so neither the pack nor the stream is held in memory. +struct PackReader { + using InitFn = int (*)(z_streamp, const char *, int); + using InflateFn = int (*)(z_streamp, int); + using EndFn = int (*)(z_streamp); + FILE *in = nullptr; + z_stream z = {}; + InflateFn inflateFn = nullptr; + EndFn endFn = nullptr; + bool open = false, ended = false; + uint64_t left = 0, produced = 0, expect = 0; // compressed bytes not yet read from the file, stream bytes produced, stream bytes announced + uint8_t input[65536]; + ~PackReader() {if (open) endFn(&z); if (in) fclose(in);} + bool start(const char *path) { + static void *lib = dlopen("/usr/lib/libz.1.dylib", RTLD_NOW | RTLD_LOCAL); + auto init = lib ? (InitFn)dlsym(lib, "inflateInit_") : nullptr; + inflateFn = lib ? (InflateFn)dlsym(lib, "inflate") : nullptr; + endFn = lib ? (EndFn)dlsym(lib, "inflateEnd") : nullptr; + if (!init || !inflateFn || !endFn || !(in = fopen(path, "rb"))) return false; + struct stat st = {}; + uint8_t head[16]; + uint32_t raw = 0, zbytes = 0; + if (fstat(fileno(in), &st) != 0 || (uint64_t)st.st_size > kPackLimit || fread(head, 1, 16, in) != 16 || memcmp(head, "SHEXP001", 8)) return false; + memcpy(&raw, head + 8, 4); + memcpy(&zbytes, head + 12, 4); + if ((uint64_t)zbytes != (uint64_t)st.st_size - 16) return false; + left = zbytes; expect = raw; + open = init(&z, ZLIB_VERSION, (int)sizeof z) == Z_OK; + return open; + } + bool read(void *out, size_t n) { + z.next_out = static_cast(out); + z.avail_out = (uInt)n; + while (z.avail_out) { + if (ended) return false; + if (!z.avail_in && left) { + const size_t want = left < sizeof input ? (size_t)left : sizeof input; + if (fread(input, 1, want, in) != want) return false; + left -= want; + z.next_in = input; + z.avail_in = (uInt)want; + } + const uInt before = z.avail_out; + const int r = inflateFn(&z, Z_NO_FLUSH); + produced += before - z.avail_out; + if (r == Z_STREAM_END) ended = true; + else if (r != Z_OK) return false; + if (produced > expect) return false; + } + return true; + } + bool skip(uint64_t n) { + uint8_t scratch[16384]; + while (n) {const size_t k = n < sizeof scratch ? (size_t)n : sizeof scratch; if (!read(scratch, k)) return false; n -= k;} + return true; + } + // The whole stream was consumed: zlib's check passed at its end and nothing follows it. + bool finished() { + if (!ended) {uint8_t b; if (read(&b, 1)) return false;} + return ended && produced == expect && !left && !z.avail_in; + } +}; + +// Import, before the device opens: the image may carry a pack beside the bundle (NativeAdrenoMetal.m2vpack, put there by +// scripts/add-gpu-plugin-data.py --pipeline-pack). Its files go into this process's own cache directory, so they carry this process's owner +// and mode; a file that is already there with the same size is kept. Files are written beside their place and renamed only once the +// whole pack has been read and checked: a damaged pack imports nothing. Returns the number of files written, or -1. +static int importPack(const char *pack, const std::string &directory) { + if (access(pack, R_OK) != 0) return 0; + PackReader r; + if (!r.start(pack)) return -1; + std::vector written; + auto fail = [&]() {for (const std::string &f : written) unlink((f + ".import").c_str()); return -1;}; + char magic[8]; + uint32_t rootLength = 0, count = 0; + if (!r.read(magic, 8) || memcmp(magic, "M2VCACH1", 8) || !r.read(&rootLength, 4) || rootLength > 4096 || !r.skip(rootLength) || + !r.read(&count, 4) || count > 1u << 20) return fail(); + std::set seen; + std::vector chunk(65536); + for (uint32_t i = 0; i < count; ++i) { + uint32_t pathLength = 0, mode = 0, uid = 0, gid = 0; + uint64_t size = 0; + if (!r.read(&pathLength, 4) || !r.read(&mode, 4) || !r.read(&uid, 4) || !r.read(&gid, 4) || !r.read(&size, 8)) return fail(); + if (pathLength == 0 || pathLength > 1024 || size > r.expect) return fail(); + std::string path(pathLength, '\0'); + if (!r.read(&path[0], pathLength)) return fail(); + if (path[0] == '/' || path.find("..") != std::string::npos || path.find('\0') != std::string::npos) return fail(); + seen.insert(bakedKey(path, size)); + const std::string full = directory + "/" + path; + struct stat st = {}; + if (stat(full.c_str(), &st) == 0 && (uint64_t)st.st_size == size) {if (!r.skip(size)) return fail(); continue;} + for (size_t slash = full.find('/', directory.size() + 1); slash != std::string::npos; slash = full.find('/', slash + 1)) + mkdir(full.substr(0, slash).c_str(), 0700); + written.push_back(full); + FILE *out = fopen((full + ".import").c_str(), "wb"); + if (!out) return fail(); + bool ok = true; + for (uint64_t left = size; ok && left;) { + const size_t k = left < chunk.size() ? (size_t)left : chunk.size(); + ok = r.read(chunk.data(), k) && fwrite(chunk.data(), 1, k, out) == k; + left -= k; + } + ok = fclose(out) == 0 && ok; + if (!ok) return fail(); + } + if (!r.finished()) return fail(); + int renamed = 0; + for (const std::string &f : written) if (rename((f + ".import").c_str(), f.c_str()) == 0) ++renamed; else unlink((f + ".import").c_str()); + baked().insert(seen.begin(), seen.end()); // only a pack read to its end counts + return renamed; +} + +#ifndef CACHEEXPORT_FORMAT_ONLY +// Markers 0x69xx: 00 started, 01 connection, 02 an export went out (then 0x6Bnn: its size in MiB, saturated), f0 no connection, +// f1 no selector, f2 a file changed or vanished while it was read, f3 a piece refused, f4 the new files exceed one export. +static void run(Gpu &marker, std::string root) { + while (root.size() > 1 && root.back() == '/') root.pop_back(); + marker.mark(0x6900); + Gpu link; + link.call = marker.call; link.openFn = marker.openFn; link.closeFn = marker.closeFn; link.port = marker.port; + if (!link.openFn || link.openFn(link.port, mach_task_self(), 0x4148, &link.conn) != 0 || !link.conn) {marker.mark(0x69f0); return;} + if (!link.probe()) {marker.mark(0x69f1); return;} + marker.mark(0x6901); + uint64_t previous = 0, exported = 0; + uint32_t generation = 0; + std::vector buffer(24 + kPiece); + for (;;) { + sleep(15); + std::vector files; + scan(root, "", files, 0); + dropBaked(files); + const uint64_t now = signature(files); + const bool stable = now == previous; + previous = now; + if (!stable || now == exported || files.empty()) continue; + const uint64_t total = 16 + streamBytes(root, files); + if (total > kLimit) {marker.mark(0x69f4); exported = now; continue;} + ++generation; + uint32_t offset = 0, filled = 0; + bool sent = true; + auto flush = [&]() { + const uint32_t header[6] = {kTunnelMagic, kMethod, generation, (uint32_t)total, offset, filled}; + memcpy(buffer.data(), header, sizeof header); + sent = link.call(link.conn, link.selector, buffer.data(), sizeof header + filled, nullptr, nullptr) == 0; + offset += filled; + filled = 0; + return sent; + }; + auto emit = [&](const uint8_t *p, size_t n) { + while (n) { + const size_t k = n < kPiece - filled ? n : kPiece - filled; + memcpy(buffer.data() + 24 + filled, p, k); + filled += (uint32_t)k; p += k; n -= k; + if (filled == kPiece && !flush()) return false; + } + return true; + }; + uint8_t head[16] = {'S', 'H', 'E', 'X', 'R', '0', '0', '1'}; + const uint32_t streamed = (uint32_t)(total - 16); + memcpy(head + 8, &streamed, 4); + const bool read = emit(head, sizeof head) && stream(root, files, emit); + if (read && filled) flush(); + if (!sent) {marker.mark(0x69f3); continue;} // retried after the next change or period + if (!read || offset != total) {marker.mark(0x69f2); continue;} + exported = now; + marker.mark(0x6902); + marker.mark(0x6B00 | (uint32_t)(total >> 20 > 0xff ? 0xff : total >> 20)); + } +} + +#endif + +} // namespace cacheexport diff --git a/GpuScanoutClient.h b/GpuScanoutClient.h new file mode 100644 index 0000000..6ffcb6d --- /dev/null +++ b/GpuScanoutClient.h @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: MIT +// Userspace side of the GPU hand-off (boot-arg sheng-fb-gpu=1): serves the display driver's frames with the GPU. For every offered frame the +// GPU copies the compositor's surface pages into the scanout mapping (GpuScanoutPresenter.h), and the swap is completed with the GPU's +// verdict. A failed copy is reported as failed; the display driver then falls back to its own path and counts it. +// The calls reach the kernel through a Transport. In the plugin that is the Adreno accelerator's connection (the original sandbox profiles +// admit that class, not the presenter's own service): methods 0x7000 + NGP_METHOD_* through the tunnel selector, scanout as memory type 0x5343. +#pragma once +#import +#include +#include +#include +#include "GpuScanoutPresenter.h" +#include "../NativeMobileDisplay/NativeGOPWire.h" + +namespace gpuscanout { + +struct Transport { + void *context; + // NGP_METHOD_MODE / WAIT_SWAP / COMPLETE_SWAP with the wire records; returns the IOReturn (0 = success). + int (*call)(void *context, uint32_t method, const void *in, size_t inSize, void *out, size_t outSize); + // The scanout, writable, mapped into this task. + int (*mapScanout)(void *context, uint64_t *address, uint64_t *length); + // Progress marker (the plugin shows the latest ones on the boot page, which survives a panic that parks the calling core). + void (*mark)(void *context, uint32_t value); +}; + +struct Stats { + uint64_t frames, failures, timeouts; + int lastResult; // gpupresent::Result of the last frame + int step; // 0 idle, 2 mode refused, 3 scanout not mappable/importable, 4 serving + int error; // the failing call's return, when there is one + uint64_t verified; // frames whose scanout was read back and compared with the surface + uint64_t mismatchRows; // rows of those frames that differ (CPU reads only: nothing is rendered or copied by the CPU) + uint32_t lastMs; // the GPU copy's time for the last frame + uint64_t totalMs; // and for all frames (the mean is totalMs / (frames + failures)) +}; + +class Client { +public: + // Reads the mode and maps the scanout; false with stats.step telling where it stopped. + bool open(id device, const Transport &transport, Stats *stats) { + transport_ = transport; + stats_ = stats; + NGPMode mode = {}; + note(0x6710); + int status = transport_.call(transport_.context, NGP_METHOD_MODE, nullptr, 0, &mode, sizeof mode); + note(0x6711); + if (status || mode.version != NGP_VERSION || mode.format != NGP_BGRX8888 || !mode.width || !mode.height || + mode.stride < mode.width * 4 || mode.bytes < mode.stride * mode.height) return fail(2, status); + const size_t page = (size_t)getpagesize(); + uint64_t address = 0, length = 0; + note(0x6712); + status = transport_.mapScanout(transport_.context, &address, &length); + note(0x6713); + if (status || !address || length < mode.bytes) return fail(3, status); + // The mapping covers whole pages; the framebuffer allocation ends on a page boundary, the copy never writes past the last row. + gpupresent::Scanout scanout = {(void *)(uintptr_t)address, (size_t)((mode.bytes + page - 1) & ~(page - 1)), (size_t)mode.stride, + (uint32_t)mode.width, (uint32_t)mode.height}; + note(0x6714); + scanout_ = scanout; + presenter_ = new gpupresent::Presenter(device, scanout); + note(0x6715); + if (!presenter_->ready()) return fail(3, 0); + stats_->step = 4; + return true; + } + // One wait (at most timeoutMs) and, when the display driver offered a frame, one GPU copy and its completion. Returns false on a + // protocol error that makes further serving pointless. + bool serveOne(uint64_t timeoutMs) { + NGPWait wait = {timeoutMs}; + NGPSwap swap = {}; + if (!stats_->timeouts && !stats_->frames) note(0x6720); // first wait + const int status = transport_.call(transport_.context, NGP_METHOD_WAIT_SWAP, &wait, sizeof wait, &swap, sizeof swap); + if (status) { + if ((unsigned)status == 0xe00002d6u /* kIOReturnTimeout */) {++stats_->timeouts; return true;} + ++stats_->failures; + stats_->error = status; + return false; + } + const size_t page = (size_t)getpagesize(); + note(0x6730); // a frame was offered + const uint64_t began = mach_absolute_time(); + const gpupresent::Result r = presenter_->present({(const void *)(uintptr_t)swap.address, (size_t)((swap.bytes + page - 1) & ~(page - 1)), + (size_t)swap.stride, swap.width, swap.height}); + { + static mach_timebase_info_data_t base; + if (!base.denom) mach_timebase_info(&base); + stats_->lastMs = (uint32_t)(((mach_absolute_time() - began) * base.numer / base.denom) / 1000000); + stats_->totalMs += stats_->lastMs; + } + stats_->lastResult = (int)r; + if (r == gpupresent::kOk && stats_->verified < 4) verify(swap); + note(0x6731); // the GPU copy returned + NGPComplete done = {swap.sequence, r == gpupresent::kOk ? 0u : 0xe00002bcu /* kIOReturnError */, 0}; + const int completed = transport_.call(transport_.context, NGP_METHOD_COMPLETE_SWAP, &done, sizeof done, nullptr, 0); + if (completed) {++stats_->failures; stats_->error = completed; return false;} + if (r == gpupresent::kOk) ++stats_->frames; else ++stats_->failures; + return true; + } +private: + // Reads the scanout back with aligned 32-bit loads (the framebuffer is device memory; wider loads can fault) and compares the top rows with the + // surface the GPU copied. The lower part of the panel is skipped: the diagnostic renderer paints there. Evidence only; nothing is written. + void verify(const NGPSwap &swap) { + const uint32_t rows = swap.height < 400 ? swap.height : 400, words = swap.width; + const volatile uint32_t *shown = (const volatile uint32_t *)scanout_.base; + const volatile uint32_t *surface = (const volatile uint32_t *)(uintptr_t)swap.address; + for (uint32_t y = 0; y < rows; ++y) { + const volatile uint32_t *a = shown + (size_t)y * (scanout_.stride / 4), *b = surface + (size_t)y * (swap.stride / 4); + for (uint32_t x = 0; x < words; ++x) + if (a[x] != b[x]) {++stats_->mismatchRows; break;} + } + ++stats_->verified; + } + void note(uint32_t value) {if (transport_.mark) transport_.mark(transport_.context, value);} + bool fail(int step, int error) {stats_->step = step; stats_->error = error; return false;} + Transport transport_ = {}; + gpupresent::Presenter *presenter_ = nullptr; + gpupresent::Scanout scanout_ = {}; + Stats *stats_ = nullptr; +}; + +} // namespace gpuscanout diff --git a/GpuScanoutPresenter.h b/GpuScanoutPresenter.h new file mode 100644 index 0000000..9a9e938 --- /dev/null +++ b/GpuScanoutPresenter.h @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT +// GPU presentation: the GPU copies a finished BGRA frame (the compositor's IOSurface pages) into the firmware scanout memory. Both are +// imported without a copy (newBufferWithBytesNoCopy: pinned by the kernel and mapped into the GPU's 4 KiB context), the copy is a blit on +// the GPU's queue, and the call returns after the GPU retired it. No CPU touches a pixel; there is no staging copy and no software fallback +// here. Used through the public Metal API, on any device (the native frontend on the tablet, the Mac's device for the host check). +#pragma once +#import +#include +#include +#include + +namespace gpupresent { + +struct Scanout { + void *base; // writable mapping of the firmware framebuffer (page aligned) + size_t bytes; // whole extent, a multiple of the page size + size_t stride; // bytes per row + uint32_t width, height; +}; + +struct Frame { + const void *base; // page-aligned mapping of the surface pages + size_t bytes; // allocation size, a multiple of the page size + size_t stride; + uint32_t width, height; +}; + +enum Result { kOk = 0, kBadFrame, kBadScanout, kNoResource, kGpuFailed }; + +class Presenter { +public: + Presenter(id device, const Scanout &scanout) : device_(device), scanout_(scanout) { + queue_ = [device newCommandQueue]; + if (!validScanout() || !queue_) return; + target_ = [device newBufferWithBytesNoCopy:scanout.base length:scanout.bytes options:MTLResourceStorageModeShared deallocator:nil]; + } + bool ready() const { return target_ != nil; } + // One frame: encodes the GPU copy, commits, waits for the GPU. Rows are copied as whole rows of width*4 bytes. + Result present(const Frame &frame) { + if (!ready()) return kBadScanout; + const size_t page = (size_t)getpagesize(), row = (size_t)frame.width * 4; + if (!frame.base || ((uintptr_t)frame.base & (page - 1)) || !frame.bytes || (frame.bytes & (page - 1)) || + frame.width != scanout_.width || frame.height != scanout_.height || frame.stride < row || + (size_t)frame.height > frame.bytes / frame.stride || frame.stride * (frame.height - 1) + row > frame.bytes) return kBadFrame; + id source = [device_ newBufferWithBytesNoCopy:(void *)frame.base length:frame.bytes options:MTLResourceStorageModeShared deallocator:nil]; + if (!source) return kNoResource; + id commands = [queue_ commandBuffer]; + id blit = commands ? [commands blitCommandEncoder] : nil; + if (!blit) return kNoResource; + if (frame.stride == scanout_.stride) { + // Equal pitch: the rows are one contiguous run. + [blit copyFromBuffer:source sourceOffset:0 toBuffer:target_ destinationOffset:0 size:frame.stride * (frame.height - 1) + row]; + } else { + // A linear texture over the surface pages reads them with the surface's pitch and writes the scanout's pitch. + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:frame.width + height:frame.height mipmapped:NO]; + d.storageMode = MTLStorageModeShared; + d.usage = MTLTextureUsageShaderRead; + if (frame.stride % [device_ minimumLinearTextureAlignmentForPixelFormat:MTLPixelFormatBGRA8Unorm]) { + for (uint32_t y = 0; y < frame.height; ++y) + [blit copyFromBuffer:source sourceOffset:y * frame.stride toBuffer:target_ destinationOffset:y * scanout_.stride size:row]; + } else { + id texture = [source newTextureWithDescriptor:d offset:0 bytesPerRow:frame.stride]; + if (!texture) {[blit endEncoding]; return kNoResource;} + [blit copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) + sourceSize:MTLSizeMake(frame.width, frame.height, 1) toBuffer:target_ destinationOffset:0 + destinationBytesPerRow:scanout_.stride destinationBytesPerImage:scanout_.stride * frame.height]; + } + } + [blit endEncoding]; + [commands commit]; + [commands waitUntilCompleted]; + return commands.status == MTLCommandBufferStatusCompleted ? kOk : kGpuFailed; + } +private: + bool validScanout() const { + const size_t page = (size_t)getpagesize(); + return scanout_.base && !((uintptr_t)scanout_.base & (page - 1)) && scanout_.bytes && !(scanout_.bytes & (page - 1)) && + scanout_.width && scanout_.height && scanout_.stride >= (size_t)scanout_.width * 4 && + scanout_.stride * scanout_.height <= scanout_.bytes; + } + id device_; + Scanout scanout_; + id queue_; + id target_; +}; + +} // namespace gpupresent diff --git a/HostMetalCommandServer.h b/HostMetalCommandServer.h new file mode 100644 index 0000000..6c5502d --- /dev/null +++ b/HostMetalCommandServer.h @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: MIT +#import +#import +NSDictionary *ServeOfflineMetalCommands(const char *,id,NSArray> *,NSArray *,unsigned expectedSessions=0); diff --git a/HostMetalCommandServer.mm b/HostMetalCommandServer.mm new file mode 100644 index 0000000..bd7d548 --- /dev/null +++ b/HostMetalCommandServer.mm @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: MIT +// Actual host Metal objects and commands. Guest requests select operations; +// no pre-scripted draw, completion or texture content is returned. +#import "HostMetalCommandServer.h" +#import +#import +#import +#include "OfflineMetalWire.h" +#include "OfflineMetalFunctionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalVertexWire.h" +#include "OfflineMetalFenceWire.h" +#include "OfflineMetalCommandWire.h" +#include "OfflineMetalEventWire.h" +#include "OfflineMetalSharedEventWire.h" +#include "OfflineMetalSessionWire.h" +#include +#include +#include +#include +#include +#include +#include +#include +static bool exact(int fd,void *bytes,size_t size,bool sending) { + unsigned char *p=(unsigned char *)bytes; + while(size){ssize_t n=sending?write(fd,p,size):read(fd,p,size);if(n<0 && errno==EINTR)continue;if(n<=0)return false;p+=n;size-=n;}return true; +} +static NSString *hashBytes(const void *bytes,size_t size) { + if(size>UINT32_MAX)return nil;unsigned char hash[32];CC_SHA256(bytes,(CC_LONG)size,hash); + NSMutableString *s=[NSMutableString string];for(unsigned i=0;i<32;i++)[s appendFormat:@"%02x",hash[i]];return s; +} +static NSData *jsonBytes(id value) {return [NSJSONSerialization dataWithJSONObject:value options:0 error:nil];} +NSDictionary *ServeOfflineMetalCommands(const char *path,id device,NSArray> *libraries,NSArray *libraryRows,unsigned expectedSessions) { + if(expectedSessions>128)return nil; + if(strlen(path)>=sizeof(((sockaddr_un *)nullptr)->sun_path))return nil; + NSMutableDictionary *fenceProducers=[NSMutableDictionary dictionary],*eventProducers=[NSMutableDictionary dictionary]; + NSMutableArray *eventsCreated=[NSMutableArray array],*eventOperations=[NSMutableArray array],*sharedValues=[NSMutableArray array],*sharedLabels=[NSMutableArray array],*notificationJournal=[NSMutableArray array],*exportJournal=[NSMutableArray array]; + NSMutableDictionary *sharedExports=[NSMutableDictionary dictionary]; + NSMutableDictionary *sharedStates=[NSMutableDictionary dictionary]; + NSMutableDictionary *notifications=[NSMutableDictionary dictionary]; + NSLock *notificationLock=[NSLock new];MTLSharedEventListener *sharedListener=[[MTLSharedEventListener alloc] initWithDispatchQueue:dispatch_get_global_queue(QOS_CLASS_DEFAULT,0)]; + uint64_t nextExport=1,nextNotification=1;uint32_t controlRequest=1; + NSMutableArray *fencesCreated=[NSMutableArray array],*fenceOperations=[NSMutableArray array],*textureHazards=[NSMutableArray array]; + int listener=socket(AF_UNIX,SOCK_STREAM,0);if(listener<0)return nil; + sockaddr_un address={};address.sun_family=AF_UNIX;strcpy(address.sun_path,path); + if(bind(listener,(sockaddr *)&address,sizeof(address)) || listen(listener,1)){close(listener);return nil;} + fprintf(stderr,"OFFLINE_METAL_READY\n");fflush(stderr); + int fd=accept(listener,nullptr,nullptr);close(listener);if(fd<0)return nil; + timeval timeout={90,0};setsockopt(fd,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout));setsockopt(fd,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout)); + int yes=1;setsockopt(fd,SOL_SOCKET,SO_NOSIGPIPE,&yes,sizeof(yes)); + NSMutableDictionary *objects=[NSMutableDictionary dictionary];uint64_t next=1; + NSMutableDictionary *sessions=[NSMutableDictionary dictionary]; + NSMutableDictionary *sessionRequests=[NSMutableDictionary dictionary]; + uint64_t nextSession=1;unsigned openedSessions=0,closedSessions=0; + NSMutableArray *completed=[NSMutableArray array],*readbacks=[NSMutableArray array],*operations=[NSMutableArray array],*uploads=[NSMutableArray array]; + NSMutableArray *bufferWrites=[NSMutableArray array],*bufferReads=[NSMutableArray array],*bufferCopies=[NSMutableArray array]; + NSMutableArray *queuePriorities=[NSMutableArray array]; + NSMutableArray *enqueued=[NSMutableArray array]; + NSMutableDictionary *pendingEnqueues=[NSMutableDictionary dictionary]; + NSMutableArray *textureCopies=[NSMutableArray array]; + NSMutableArray *depthStates=[NSMutableArray array],*stencilReferences=[NSMutableArray array],*attachmentPasses=[NSMutableArray array]; + NSMutableDictionary *openEncoders=[NSMutableDictionary dictionary];NSMutableArray *abortedEncoders=[NSMutableArray array]; + NSMutableArray *specializations=[NSMutableArray array]; + NSMutableDictionary *asyncCommands=[NSMutableDictionary dictionary]; + NSMutableArray *submitted=[NSMutableArray array],*commandObservations=[NSMutableArray array],*failedCommands=[NSMutableArray array]; + NSMutableSet *recordedCompletions=[NSMutableSet set],*recordedFailures=[NSMutableSet set]; + auto observeCommand=[&](uint64_t handle,id command,NSString *reason)->OMCommandResult { + OMCommandResult value={(uint64_t)command.status,0,0}; + if(value.status==MTLCommandBufferStatusCompleted) { + value.start_ns=(uint64_t)(command.GPUStartTime*1e9);value.end_ns=(uint64_t)(command.GPUEndTime*1e9); + if(value.start_ns && value.end_ns>value.start_ns && ![recordedCompletions containsObject:@(handle)]) { + [recordedCompletions addObject:@(handle)];[completed addObject:@{@"command":@(handle),@"status":@(value.status),@"gpu_start_ns":@(value.start_ns),@"gpu_end_ns":@(value.end_ns),@"observed_by":reason}]; + } + } else if(value.status==MTLCommandBufferStatusError && ![recordedFailures containsObject:@(handle)]) { + [recordedFailures addObject:@(handle)];[failedCommands addObject:@{@"command":@(handle),@"status":@(value.status),@"actual_error":command.error.localizedDescription?:@""}]; + } + return value; + }; + auto finishSubmitted=[&](uint64_t selected,bool all,NSString *reason) { + for(NSNumber *handle in asyncCommands.allKeys) { + NSDictionary *row=asyncCommands[handle];if(!all && [row[@"session"] unsignedLongLongValue]!=selected)continue; + id command=row[@"command"];[command waitUntilCompleted]; + observeCommand(handle.unsignedLongLongValue,command,reason);[asyncCommands removeObjectForKey:handle]; + } + }; + auto abortEncoders=[&](uint64_t selected,bool all,NSString *reason) { + for(NSNumber *handle in openEncoders.allKeys) { + NSDictionary *row=openEncoders[handle];if(!all && [row[@"session"] unsignedLongLongValue]!=selected)continue; + id encoder=row[@"encoder"];id command=row[@"command_object"]; + uint64_t status=command.status;[encoder endEncoding]; + [abortedEncoders addObject:@{@"encoder":handle,@"command":row[@"command_handle"],@"session":row[@"session"], + @"actual_command_status":@(status),@"reason":reason,@"encoding_ended":@YES,@"gpu_submission_created_by_cleanup":@NO}]; + [openEncoders removeObjectForKey:handle]; + } + }; + auto hasOpenEncoder=[&](uint64_t command)->bool { + for(NSDictionary *row in openEncoders.allValues)if([row[@"command_handle"] unsignedLongLongValue]==command)return true;return false; + }; + auto cancelNotifications=[&](uint64_t selected,bool all) { + [notificationLock lock]; + for(NSNumber *key in notifications.allKeys) { + NSMutableDictionary *record=notifications[key];if(!all && [record[@"session"] unsignedLongLongValue]!=selected)continue; + record[@"cancelled"]=@YES;[notificationJournal addObject:@{@"notification":key,@"session":record[@"session"],@"event":record[@"event"],@"requested":record[@"requested"],@"actually_fired":record[@"ready"],@"actual_value":record[@"value"]}];[notifications removeObjectForKey:key]; + } + [notificationLock unlock]; + }; + NSMutableDictionary *renderExtents=[NSMutableDictionary dictionary];NSMutableArray *blendPipelines=[NSMutableArray array],*textureBindings=[NSMutableArray array],*vertexPipelines=[NSMutableArray array]; + uint32_t bootBytes=0;bool finished=false,healthy=true; + auto save=[&](id object)->uint64_t {if(!object)return 0;uint64_t key=next++;objects[@(key)]=object;return key;}; + for(uint32_t request=1;request<=8192 && healthy && !finished;request++) { + @autoreleasepool { + OMWireHeader header={}; + if(request==1) { + uint32_t prefix=0,count=0; + do {unsigned char byte=0;if(!exact(fd,&byte,1,false)){healthy=false;break;} + prefix=(prefix>>8)|((uint32_t)byte<<24);count++;if(count>65536){healthy=false;break;} + } while(count<4 || prefix!=OM_WIRE_MAGIC); + if(!healthy)break;bootBytes=count-4;header.magic=prefix; + if(!exact(fd,(unsigned char *)&header+4,sizeof(header)-4,false)){healthy=false;break;} + } else if(!exact(fd,&header,sizeof(header),false)){healthy=false;break;} + if(header.magic!=OM_WIRE_MAGIC || header.version!=(expectedSessions?OFFLINE_METAL_SESSION_VERSION:OFFLINE_METAL_VERSION) + || (!expectedSessions && header.request!=request) || header.bytes>4080){healthy=false;break;} + uint64_t session=0;bool validSession=true; + if(expectedSessions) { + if(!exact(fd,&session,sizeof(session),false)){healthy=false;break;} + if(header.operation==OM_SESSION_OPEN)validSession=!session && header.request==1 && !header.bytes; + else if(!session && header.operation==OM_SHARED_EXPORT_DROP){validSession=header.request==controlRequest;if(validSession)controlRequest++;} + else { + objects=sessions[@(session)];NSNumber *expected=sessionRequests[@(session)]; + validSession=objects && expected && header.request==expected.unsignedIntValue; + if(validSession)sessionRequests[@(session)]=@(header.request+1); + } + } + std::vector input(header.bytes); + if(!input.empty() && !exact(fd,input.data(),input.size(),false)){healthy=false;break;} + OMResult result={};NSData *body=nil;NSError *error=nil; + auto get=[&](uint64_t handle,Protocol *protocol)->id {id object=objects[@(handle)];return object && [object conformsToProtocol:protocol]?object:nil;}; + auto getEncoder=[&](uint64_t handle,Protocol *protocol)->id {return openEncoders[@(handle)]?get(handle,protocol):nil;}; +#define REQUEST(Type) if(input.size()!=sizeof(Type)){result.error=-2;break;} Type q={};memcpy(&q,input.data(),sizeof(q)) + if(!validSession)result.error=-2; + else @try { + switch(header.operation) { + case OM_SESSION_OPEN: + if(!expectedSessions || openedSessions>=expectedSessions){result.error=-2;break;} + result.handle=nextSession++;sessions[@(result.handle)]=[NSMutableDictionary dictionary]; + sessionRequests[@(result.handle)]=@1;openedSessions++;break; + case OM_SESSION_CLOSE: + if(!expectedSessions || !input.empty()){result.error=-2;break;} + // Remove only this real namespace's independent references. Command + // buffers retain their actual GPU dependencies according to Metal. + abortEncoders(session,false,@"session-close"); + finishSubmitted(session,false,@"session-close"); + cancelNotifications(session,false); + for(NSNumber *handle in objects.allKeys){[renderExtents removeObjectForKey:handle];[fenceProducers removeObjectForKey:handle];[eventProducers removeObjectForKey:handle];NSMutableDictionary *shared=sharedStates[handle];if([shared[@"producer"][@"session"] unsignedLongLongValue]==session)[shared removeObjectForKey:@"producer"];[sharedStates removeObjectForKey:handle];} + for(NSNumber *handle in pendingEnqueues.allKeys)if([pendingEnqueues[handle][@"session"] unsignedLongLongValue]==session)[pendingEnqueues removeObjectForKey:handle]; + [objects removeAllObjects];[sessions removeObjectForKey:@(session)];[sessionRequests removeObjectForKey:@(session)]; + closedSessions++;finished=openedSessions==expectedSessions && !sessions.count && !sharedExports.count;break; + case OM_DEVICE: + if(!input.empty()){result.error=-2;break;} + body=jsonBytes(@{@"name":device.name,@"registryID":@(device.registryID),@"maxBufferLength":@(MIN(device.maxBufferLength,(NSUInteger)OFFLINE_METAL_MAX_BUFFER)), + @"lowPower":@(device.lowPower),@"removable":@(device.removable),@"headless":@(device.headless), + @"maxThreads":@[@(device.maxThreadsPerThreadgroup.width),@(device.maxThreadsPerThreadgroup.height),@(device.maxThreadsPerThreadgroup.depth)]});break; + case OM_QUEUE: + if(!input.empty()){result.error=-2;break;}result.handle=save([device newCommandQueue]);if(!result.handle)result.error=-3;break; + case OM_QUEUE_PRIORITY_GET:case OM_QUEUE_PRIORITY_SET: { + REQUEST(OMQueuePriority);id queue=get(q.queue,@protocol(MTLCommandQueue)); + if(!queue || q.background>1 || (header.operation==OM_QUEUE_PRIORITY_GET && q.priority)){result.error=-2;break;} + bool writing=header.operation==OM_QUEUE_PRIORITY_SET; + if(writing && (q.priority>5 || (!q.background && q.priority==3))){result.error=-4;break;} + SEL getter=sel_registerName(q.background?"getBackgroundGPUPriority":"getGPUPriority"); + SEL setter=sel_registerName(q.background?"setBackgroundGPUPriority:":"setGPUPriority:"); + Method readMethod=class_getInstanceMethod(object_getClass(queue),getter),writeMethod=class_getInstanceMethod(object_getClass(queue),setter); + if(!readMethod || strcmp(method_getTypeEncoding(readMethod),"Q16@0:8") || + (writing && (!writeMethod || strcmp(method_getTypeEncoding(writeMethod),"B24@0:8Q16")))){result.error=-4;break;} + uint64_t before=((NSUInteger(*)(id,SEL))objc_msgSend)(queue,getter),value=before; + if(writing)value=((BOOL(*)(id,SEL,NSUInteger))objc_msgSend)(queue,setter,q.priority)?1:0; + uint64_t after=((NSUInteger(*)(id,SEL))objc_msgSend)(queue,getter); + body=[NSData dataWithBytes:&value length:sizeof(value)]; + [queuePriorities addObject:@{@"queue":@(q.queue),@"session":@(session),@"background":@(q.background),@"writing":@(writing), + @"requested":@(q.priority),@"before":@(before),@"after":@(after),@"result":@(value)}];break; + } + case OM_LIBRARY: { + REQUEST(OMLibrary);NSMutableString *hash=[NSMutableString string];for(unsigned i=0;i<32;i++)[hash appendFormat:@"%02x",q.sha256[i]]; + NSUInteger i=0;for(;i library=libraries[i];result.handle=save(library);body=jsonBytes(library.functionNames);break; + } + case OM_FUNCTION: { + OMFunction q={};OMSpecializedFunction extended={};bool specialized=input.size()!=sizeof(q); + if(!specialized)memcpy(&q,input.data(),sizeof(q)); + else { + if(input.size()OFFLINE_METAL_MAX_CONSTANTS || input.size()!=sizeof(extended)+extended.count*sizeof(OMFunctionConstant)){result.error=-2;break;} + } + id library=get(q.library,@protocol(MTLLibrary)); + if(!library || !memchr(q.name,0,sizeof(q.name))){result.error=-2;break;} + NSString *name=[NSString stringWithUTF8String:q.name];if(!name.length){result.error=-2;break;} + id function=nil; + if(specialized) { + MTLFunctionConstantValues *values=[MTLFunctionConstantValues new];NSMutableArray *rows=[NSMutableArray array];NSMutableSet *keys=[NSMutableSet set]; + for(uint64_t i=0;i1 || row.index>65535 || + (row.named && (!memchr(row.name,0,sizeof(row.name)) || row.index)) || (!row.named && row.name[0])){result.error=-2;break;} + NSString *constantName=row.named?[NSString stringWithUTF8String:row.name]:nil; + NSString *key=row.named?[NSString stringWithFormat:@"name:%@",constantName]:[NSString stringWithFormat:@"index:%llu",row.index]; + if((row.named && !constantName.length) || [keys containsObject:key]){result.error=-2;break;}[keys addObject:key]; + if(row.named)[values setConstantValue:row.data type:(MTLDataType)row.type withName:constantName]; + else [values setConstantValue:row.data type:(MTLDataType)row.type atIndex:row.index]; + NSMutableString *hex=[NSMutableString string];for(uint64_t j=0;jMTLHazardTrackingModeTracked){result.error=-2;break;} + bool attachment=q.format==MTLPixelFormatStencil8 || q.format==MTLPixelFormatDepth32Float || q.format==MTLPixelFormatDepth32Float_Stencil8; + if(q.type!=MTLTextureType2D || (!attachment && q.format!=MTLPixelFormatR32Float && q.format!=MTLPixelFormatRG16Float && q.format!=MTLPixelFormatBGRA8Unorm && q.format!=MTLPixelFormatRGBA8Unorm) || !q.width || !q.height || q.width>2048 || q.height>2048 + || q.depth!=1 || q.levels!=1 || q.samples!=1 || q.array!=1 || (q.storage!=MTLStorageModeShared && q.storage!=MTLStorageModePrivate) || q.cache!=MTLCPUCacheModeDefaultCache){result.error=-4;break;} + if(attachment && (q.storage!=MTLStorageModePrivate || q.usage!=MTLTextureUsageRenderTarget)){result.error=-4;break;} + MTLTextureDescriptor *d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:(MTLPixelFormat)q.format width:q.width height:q.height mipmapped:NO]; + d.usage=q.usage;d.storageMode=(MTLStorageMode)q.storage;d.hazardTrackingMode=(MTLHazardTrackingMode)hazard; + id texture=[device newTextureWithDescriptor:d];if(!texture || (hazard && texture.hazardTrackingMode!=hazard)){result.error=-4;break;} + result.handle=save(texture); + if(header.operation==OM_TEXTURE_HAZARD){uint64_t actual=texture.hazardTrackingMode;body=[NSData dataWithBytes:&actual length:sizeof(actual)];[textureHazards addObject:@{@"texture":@(result.handle),@"session":@(session),@"requested":@(hazard),@"actual":@(actual)}];} + break; + } + case OM_PIPELINE:case OM_PIPELINE_BLEND:case OM_PIPELINE_ATTACHMENTS: { + OMPipeline q={};OMBlend blends[8]={};uint64_t depth=0,stencil=0;OMVertexPipeline vertexInput={};bool hasVertexInput=false; + if(header.operation==OM_PIPELINE) {if(input.size()!=sizeof(q)){result.error=-2;break;}memcpy(&q,input.data(),sizeof(q));} + else if(header.operation==OM_PIPELINE_BLEND){if(input.size()!=sizeof(OMBlendPipeline)){result.error=-2;break;}OMBlendPipeline ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.pipeline;memcpy(blends,ext.colors,sizeof(blends));} + else { + if(input.size()==sizeof(vertexInput)) {memcpy(&vertexInput,input.data(),sizeof(vertexInput));if(!OMVertexInputsValid(vertexInput)){result.error=-2;break;}hasVertexInput=true;} + else if(input.size()!=sizeof(OMAttachmentPipeline)){result.error=-2;break;} + OMAttachmentPipeline ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.colors.pipeline;memcpy(blends,ext.colors.colors,sizeof(blends));depth=ext.depthFormat;stencil=ext.stencilFormat; + } + id vertex=get(q.vertex,@protocol(MTLFunction)),fragment=get(q.fragment,@protocol(MTLFunction)); + if(!vertex || !fragment || q.samples!=1 || (depth && depth!=MTLPixelFormatDepth32Float && depth!=MTLPixelFormatDepth32Float_Stencil8) || + (stencil && stencil!=MTLPixelFormatStencil8 && stencil!=MTLPixelFormatDepth32Float_Stencil8)){result.error=-2;break;} + MTLRenderPipelineDescriptor *d=[MTLRenderPipelineDescriptor new];d.vertexFunction=vertex;d.fragmentFunction=fragment;d.rasterSampleCount=q.samples; + d.depthAttachmentPixelFormat=(MTLPixelFormat)depth;d.stencilAttachmentPixelFormat=(MTLPixelFormat)stencil; + NSMutableArray *actualAttributes=[NSMutableArray array],*actualLayouts=[NSMutableArray array]; + if(hasVertexInput) { + MTLVertexDescriptor *v=[MTLVertexDescriptor vertexDescriptor];bool used[31]={}; + for(unsigned i=0;i<31;i++) { + const auto &a=vertexInput.attributes[i];if(!a.format)continue; + v.attributes[i].format=(MTLVertexFormat)a.format;v.attributes[i].offset=a.offset;v.attributes[i].bufferIndex=a.buffer;used[a.buffer]=true; + [actualAttributes addObject:@{@"index":@(i),@"format":@(v.attributes[i].format),@"offset":@(v.attributes[i].offset),@"buffer":@(v.attributes[i].bufferIndex)}]; + } + for(unsigned i=0;i<31;i++)if(used[i]) { + const auto &l=vertexInput.layouts[i];v.layouts[i].stride=l.stride;v.layouts[i].stepFunction=(MTLVertexStepFunction)l.step;v.layouts[i].stepRate=l.rate; + [actualLayouts addObject:@{@"index":@(i),@"stride":@(v.layouts[i].stride),@"step":@(v.layouts[i].stepFunction),@"rate":@(v.layouts[i].stepRate)}]; + } + d.vertexDescriptor=v; + } + for(unsigned i=0;i<8;i++) { + d.colorAttachments[i].pixelFormat=(MTLPixelFormat)q.formats[i];d.colorAttachments[i].writeMask=(MTLColorWriteMask)q.masks[i]; + if(header.operation!=OM_PIPELINE) { + const OMBlend &b=blends[i];if(b.enabled>1 || b.sourceRGB>MTLBlendFactorOneMinusSource1Alpha || b.destinationRGB>MTLBlendFactorOneMinusSource1Alpha || + b.sourceAlpha>MTLBlendFactorOneMinusSource1Alpha || b.destinationAlpha>MTLBlendFactorOneMinusSource1Alpha || b.rgbOperation>MTLBlendOperationMax || b.alphaOperation>MTLBlendOperationMax){result.error=-2;break;} + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];a.blendingEnabled=b.enabled; + a.sourceRGBBlendFactor=(MTLBlendFactor)b.sourceRGB;a.destinationRGBBlendFactor=(MTLBlendFactor)b.destinationRGB;a.rgbBlendOperation=(MTLBlendOperation)b.rgbOperation; + a.sourceAlphaBlendFactor=(MTLBlendFactor)b.sourceAlpha;a.destinationAlphaBlendFactor=(MTLBlendFactor)b.destinationAlpha;a.alphaBlendOperation=(MTLBlendOperation)b.alphaOperation; + } + } + if(result.error)break;result.handle=save([device newRenderPipelineStateWithDescriptor:d error:&error]);if(!result.handle)result.error=-3; + if(result.handle && hasVertexInput)[vertexPipelines addObject:@{@"pipeline":@(result.handle),@"session":@(session),@"vertex":vertex.name,@"fragment":fragment.name,@"attributes":actualAttributes,@"layouts":actualLayouts}]; + if(result.handle && header.operation==OM_PIPELINE_BLEND)[blendPipelines addObject:@{@"pipeline":@(result.handle),@"vertex":vertex.name,@"fragment":fragment.name}];break; + } + case OM_FENCE: + if(!input.empty()){result.error=-2;break;} + result.handle=save([device newFence]);if(!result.handle){result.error=-4;break;} + [fencesCreated addObject:@{@"fence":@(result.handle),@"session":@(session)}];break; + case OM_RENDER_FENCE_UPDATE:case OM_RENDER_FENCE_WAIT:case OM_BLIT_FENCE_UPDATE:case OM_BLIT_FENCE_WAIT: { + REQUEST(OMFenceDependency); + bool render=header.operation==OM_RENDER_FENCE_UPDATE || header.operation==OM_RENDER_FENCE_WAIT; + bool update=header.operation==OM_RENDER_FENCE_UPDATE || header.operation==OM_BLIT_FENCE_UPDATE; + id encoder=getEncoder(q.encoder,render?@protocol(MTLRenderCommandEncoder):@protocol(MTLBlitCommandEncoder)); + id fence=get(q.fence,@protocol(MTLFence));NSDictionary *active=openEncoders[@(q.encoder)]; + if(!encoder || !fence || !active || (render?(!q.stages || (q.stages&~3ULL)):q.stages!=0)){result.error=-2;break;} + id command=active[@"command_object"];NSDictionary *producer=fenceProducers[@(q.fence)]; + if(!update) { + // Same-queue producers must be actually committed. Cross-queue fences + // stay limited to completed producers after a retained real pending- + // producer readback failure. Use actual GPU events for overlap across + // queues; CPU encoding metadata never represents a signaled fence. + id source=producer[@"command"];MTLCommandBufferStatus actual=source.status; + if(!producer || [producer[@"encoder"] unsignedLongLongValue]==q.encoder || openEncoders[producer[@"encoder"]] || + (source!=command && (actualMTLCommandBufferStatusCompleted || + (source.commandQueue!=command.commandQueue && actual!=MTLCommandBufferStatusCompleted)))){result.error=-4;break;} + } + if(render) { + if(update)[(id)encoder updateFence:fence afterStages:(MTLRenderStages)q.stages]; + else [(id)encoder waitForFence:fence beforeStages:(MTLRenderStages)q.stages]; + } else { + if(update)[(id)encoder updateFence:fence];else [(id)encoder waitForFence:fence]; + } + [fenceOperations addObject:@{@"fence":@(q.fence),@"encoder":@(q.encoder),@"session":@(session),@"command":active[@"command_handle"],@"stages":@(q.stages),@"render":@(render),@"update":@(update)}]; + if(update)fenceProducers[@(q.fence)]=@{@"encoder":@(q.encoder),@"command":command};break; + } + case OM_SHARED_EVENT: { + if(!input.empty()){result.error=-2;break;}id event=[device newSharedEvent];if(!event){result.error=-4;break;} + if(event.device && event.device.registryID!=device.registryID){result.error=-4;break;} + result.handle=save(event);sharedStates[@(result.handle)]=[NSMutableDictionary dictionary];[eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session),@"shared":@YES,@"device_is_nil":@(event.device==nil)}]; + body=jsonBytes(@{@"device_is_nil":@(event.device==nil),@"device_is_owner":@(event.device && event.device.registryID==device.registryID),@"signaled_value":@(event.signaledValue),@"label":event.label?:[NSNull null]});break; + } + case OM_SHARED_VALUE_GET: { + REQUEST(OMHandle);id event=get(q.handle,@protocol(MTLSharedEvent));if(!event){result.error=-2;break;} + uint64_t value=event.signaledValue;body=[NSData dataWithBytes:&value length:sizeof(value)]; + [sharedValues addObject:@{@"event":@(q.handle),@"session":@(session),@"value":@(value),@"cpu_set":@NO}];break; + } + case OM_SHARED_VALUE_SET: { + REQUEST(OMSharedEventValue);id event=get(q.event,@protocol(MTLSharedEvent));if(!event || q.value event=get(q.event,@protocol(MTLSharedEvent)); + if(!event || q.present>1 || q.bytes>512 || input.size()!=sizeof(q)+q.bytes || (!q.present && q.bytes)){result.error=-2;break;} + NSString *label=q.present?[[NSString alloc] initWithBytes:input.data()+sizeof(q) length:q.bytes encoding:NSUTF8StringEncoding]:nil; + if(q.present && !label){result.error=-2;break;}event.label=label;body=jsonBytes(@{@"label":event.label?:[NSNull null]}); + [sharedLabels addObject:@{@"event":@(q.event),@"session":@(session),@"actual_label":event.label?:[NSNull null]}];break; + } + case OM_SHARED_NOTIFY: { + REQUEST(OMSharedEventValue);id event=get(q.event,@protocol(MTLSharedEvent));if(!event || !sharedListener){result.error=-2;break;} + uint64_t identifier=nextNotification++;NSMutableDictionary *record=[@{@"session":@(session),@"event":@(q.event),@"requested":@(q.value),@"ready":@NO,@"value":@0,@"cancelled":@NO,@"identity_match":@NO} mutableCopy]; + [notificationLock lock];notifications[@(identifier)]=record;[notificationLock unlock];__weak id expected=event; + [event notifyListener:sharedListener atValue:q.value block:^(id actual,uint64_t value) { + [notificationLock lock];if(![record[@"cancelled"] boolValue]){record[@"ready"]=@YES;record[@"value"]=@(value);record[@"identity_match"]=@(actual==expected);}[notificationLock unlock]; + }];result.handle=identifier;break; + } + case OM_SHARED_NOTIFICATION_POLL: { + REQUEST(OMHandle);[notificationLock lock];NSMutableDictionary *record=notifications[@(q.handle)]; + if(!record || [record[@"session"] unsignedLongLongValue]!=session){result.error=-2;[notificationLock unlock];break;} + OMSharedNotification value={};value.ready=[record[@"ready"] boolValue]; + if(value.ready){if(![record[@"identity_match"] boolValue] || [record[@"value"] unsignedLongLongValue]<[record[@"requested"] unsignedLongLongValue])result.error=-3;else {value.event=[record[@"event"] unsignedLongLongValue];value.value=[record[@"value"] unsignedLongLongValue];}} + [notificationLock unlock];if(!result.error)body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_SHARED_EXPORT: { + REQUEST(OMHandle);id event=get(q.handle,@protocol(MTLSharedEvent));if(!expectedSessions || !event || !sharedStates[@(q.handle)]){result.error=-2;break;} + MTLSharedEventHandle *handle=[event newSharedEventHandle];if(!handle){result.error=-4;break;}result.handle=nextExport++;sharedExports[@(result.handle)]=@{@"handle":handle,@"state":sharedStates[@(q.handle)]}; + Method trace=class_getInstanceMethod(object_getClass(handle),sel_registerName("labelTraceID"));BOOL supported=trace && !strcmp(method_getTypeEncoding(trace),"Q16@0:8"); + uint64_t traceID=supported?((uint64_t (*)(id,SEL))objc_msgSend)(handle,sel_registerName("labelTraceID")):0; + body=jsonBytes(@{@"label":handle.label?:[NSNull null],@"trace_id_supported":@(supported),@"trace_id":supported?@(traceID):[NSNull null]}); + [exportJournal addObject:@{@"operation":@"export",@"export":@(result.handle),@"event":@(q.handle),@"session":@(session)}];break; + } + case OM_SHARED_IMPORT: { + REQUEST(OMHandle);NSDictionary *exported=sharedExports[@(q.handle)];MTLSharedEventHandle *handle=exported[@"handle"];if(!session || !handle){result.error=-2;break;} + id event=[device newSharedEventWithHandle:handle];if(!event || (event.device && event.device.registryID!=device.registryID)){result.error=-4;break;}result.handle=save(event);sharedStates[@(result.handle)]=exported[@"state"]; + body=jsonBytes(@{@"device_is_nil":@(event.device==nil),@"device_is_owner":@(event.device && event.device.registryID==device.registryID),@"signaled_value":@(event.signaledValue),@"label":event.label?:[NSNull null]}); + [eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session),@"shared":@YES,@"imported":@YES,@"device_is_nil":@(event.device==nil)}]; + [exportJournal addObject:@{@"operation":@"import",@"export":@(q.handle),@"event":@(result.handle),@"session":@(session)}];break; + } + case OM_SHARED_EXPORT_DROP: { + REQUEST(OMHandle);if(!expectedSessions || session || !sharedExports[@(q.handle)]){result.error=-2;break;}[sharedExports removeObjectForKey:@(q.handle)]; + [exportJournal addObject:@{@"operation":@"drop",@"export":@(q.handle),@"session":@0}];finished=openedSessions==expectedSessions && !sessions.count && !sharedExports.count;break; + } + case OM_EVENT: + if(!input.empty()){result.error=-2;break;} + result.handle=save([device newEvent]);if(!result.handle){result.error=-4;break;} + [eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session)}];break; + case OM_EVENT_SIGNAL:case OM_EVENT_WAIT: { + REQUEST(OMEventDependency);id command=get(q.command,@protocol(MTLCommandBuffer));id event=get(q.event,@protocol(MTLEvent)); + if(!command || !event || !q.value || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.command)){result.error=-2;break;} + NSMutableDictionary *shared=sharedStates[@(q.event)];NSDictionary *producer=shared?shared[@"producer"]:eventProducers[@(q.event)];bool signal=header.operation==OM_EVENT_SIGNAL; + uint64_t previous=[producer[@"value"] unsignedLongLongValue],actualValue=shared?[(id)event signaledValue]:0; + uint64_t actualProducer=0; + if(signal) { + if(q.value<=previous || (shared && q.value<=actualValue)){result.error=-2;break;} + [command encodeSignalEvent:event value:q.value];NSDictionary *record=@{@"value":@(q.value),@"command":command,@"session":@(session)};if(shared)shared[@"producer"]=record;else eventProducers[@(q.event)]=record; + } else { + id source=producer[@"command"];actualProducer=source.status; + if(!(shared && actualValue>=q.value) && (!producer || q.value>previous || source==command || actualProducerMTLCommandBufferStatusCompleted)){result.error=-4;break;} + [command encodeWaitForEvent:event value:q.value]; + } + [eventOperations addObject:@{@"event":@(q.event),@"command":@(q.command),@"session":@(session),@"value":@(q.value),@"signal":@(signal),@"shared":@(shared!=nil),@"actual_signaled_value":@(actualValue),@"actual_producer_status":@(actualProducer)}];break; + } + case OM_COMMAND: { + REQUEST(OMHandle);id queue=get(q.handle,@protocol(MTLCommandQueue)); + if(!queue){result.error=-2;break;}result.handle=save([queue commandBuffer]);if(!result.handle)result.error=-3;break; + } + case OM_COMMAND_ENQUEUE: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer)); + if(!command || command.status!=MTLCommandBufferStatusNotEnqueued){result.error=-2;break;} + // This synchronous transport currently supports one pending reservation + // per real queue. Refuse unsupported ordering before blocking the server + // behind a command whose predecessor cannot yet receive its commit. + for(NSDictionary *row in pendingEnqueues.allValues)if([row[@"command"] commandQueue]==command.commandQueue){result.error=-4;break;} + if(result.error)break;[command enqueue];uint64_t status=command.status; + if(status!=MTLCommandBufferStatusEnqueued){result.error=-3;break;} + pendingEnqueues[@(q.handle)]=@{@"command":command,@"session":@(session)}; + body=[NSData dataWithBytes:&status length:sizeof(status)]; + [enqueued addObject:@{@"command":@(q.handle),@"session":@(session),@"status":@(status),@"operation_request":@(request)}];break; + } + case OM_RENDER:case OM_RENDER_ATTACHMENTS: { + OMRender q={};OMDepthAttachment depth={};OMStencilAttachment stencil={}; + if(header.operation==OM_RENDER){if(input.size()!=sizeof(q)){result.error=-2;break;}memcpy(&q,input.data(),sizeof(q));} + else {if(input.size()!=sizeof(OMAttachmentRender)){result.error=-2;break;}OMAttachmentRender ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.colors;depth=ext.depth;stencil=ext.stencil;} + id command=get(q.command,@protocol(MTLCommandBuffer));if(!command || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.command)){result.error=-2;break;} + MTLRenderPassDescriptor *d=[MTLRenderPassDescriptor renderPassDescriptor];NSUInteger width=0,height=0; + for(unsigned i=0;i<8;i++)if(q.colors[i].texture) { + id texture=get(q.colors[i].texture,@protocol(MTLTexture));if(!texture || q.colors[i].load>MTLLoadActionClear || q.colors[i].store>MTLStoreActionStore || (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height; + d.colorAttachments[i].texture=texture;d.colorAttachments[i].loadAction=(MTLLoadAction)q.colors[i].load;d.colorAttachments[i].storeAction=(MTLStoreAction)q.colors[i].store; + const double *v=q.colors[i].clear;d.colorAttachments[i].clearColor=MTLClearColorMake(v[0],v[1],v[2],v[3]); + } + if(depth.texture) { + id texture=get(depth.texture,@protocol(MTLTexture)); + if(!texture || (texture.pixelFormat!=MTLPixelFormatDepth32Float && texture.pixelFormat!=MTLPixelFormatDepth32Float_Stencil8) || + depth.load>MTLLoadActionClear || depth.store>MTLStoreActionStore || !std::isfinite(depth.clear) || depth.clear<0 || depth.clear>1 || + (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height;d.depthAttachment.texture=texture;d.depthAttachment.loadAction=(MTLLoadAction)depth.load; + d.depthAttachment.storeAction=(MTLStoreAction)depth.store;d.depthAttachment.clearDepth=depth.clear; + } + if(stencil.texture) { + id texture=get(stencil.texture,@protocol(MTLTexture)); + if(!texture || (texture.pixelFormat!=MTLPixelFormatStencil8 && texture.pixelFormat!=MTLPixelFormatDepth32Float_Stencil8) || + stencil.load>MTLLoadActionClear || stencil.store>MTLStoreActionStore || stencil.clear>255 || + (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height;d.stencilAttachment.texture=texture;d.stencilAttachment.loadAction=(MTLLoadAction)stencil.load; + d.stencilAttachment.storeAction=(MTLStoreAction)stencil.store;d.stencilAttachment.clearStencil=(uint32_t)stencil.clear; + } + if(!width || !height){result.error=-2;break;} + if(result.error)break;result.handle=save([command renderCommandEncoderWithDescriptor:d]);if(!result.handle)result.error=-3; + else { + renderExtents[@(result.handle)]=@[@(width),@(height)]; + openEncoders[@(result.handle)]=@{@"encoder":objects[@(result.handle)],@"command_object":command,@"command_handle":@(q.command),@"session":@(session)}; + if(depth.texture || stencil.texture)[attachmentPasses addObject:@{@"command":@(q.command),@"encoder":@(result.handle),@"session":@(session), + @"depth":@(depth.texture),@"stencil":@(stencil.texture),@"depth_clear":@(depth.clear),@"stencil_clear":@(stencil.clear), + @"depth_load":@(depth.load),@"stencil_load":@(stencil.load),@"width":@(width),@"height":@(height)}]; + }break; + } + case OM_DEPTH_STENCIL_STATE: { + REQUEST(OMDepthStencil); + auto validFace=[](const OMStencilFace &f)->bool { + if(f.enabled>1)return false; + if(!f.enabled)return !(f.compare|f.failure|f.depthFailure|f.pass|f.readMask|f.writeMask); + return f.compare<=MTLCompareFunctionAlways && f.failure<=MTLStencilOperationDecrementWrap && + f.depthFailure<=MTLStencilOperationDecrementWrap && f.pass<=MTLStencilOperationDecrementWrap && f.readMask<=UINT32_MAX && f.writeMask<=UINT32_MAX; + }; + if(q.compare>MTLCompareFunctionAlways || q.write>1 || !validFace(q.front) || !validFace(q.back)){result.error=-2;break;} + auto face=[](const OMStencilFace &f)->MTLStencilDescriptor * { + if(!f.enabled)return nil;MTLStencilDescriptor *s=[MTLStencilDescriptor new];s.stencilCompareFunction=(MTLCompareFunction)f.compare; + s.stencilFailureOperation=(MTLStencilOperation)f.failure;s.depthFailureOperation=(MTLStencilOperation)f.depthFailure;s.depthStencilPassOperation=(MTLStencilOperation)f.pass; + s.readMask=(uint32_t)f.readMask;s.writeMask=(uint32_t)f.writeMask;return s; + }; + MTLDepthStencilDescriptor *d=[MTLDepthStencilDescriptor new];d.depthCompareFunction=(MTLCompareFunction)q.compare;d.depthWriteEnabled=q.write; + d.frontFaceStencil=face(q.front);d.backFaceStencil=face(q.back);result.handle=save([device newDepthStencilStateWithDescriptor:d]); + if(!result.handle){result.error=-3;break;} + [depthStates addObject:@{@"state":@(result.handle),@"session":@(session),@"depth_compare":@(q.compare),@"depth_write":@(q.write), + @"front_compare":@(q.front.compare),@"back_compare":@(q.back.compare),@"front_write_mask":@(q.front.writeMask),@"back_write_mask":@(q.back.writeMask)}];break; + } + case OM_SET_DEPTH_STENCIL: { + REQUEST(OMSetDepthStencil);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id state=q.state?get(q.state,@protocol(MTLDepthStencilState)):nil; + if(!encoder || (q.state && !state)){result.error=-2;break;}[encoder setDepthStencilState:state];break; + } + case OM_STENCIL_REFERENCE: { + REQUEST(OMStencilReference);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || q.front>UINT32_MAX || q.back>UINT32_MAX){result.error=-2;break;} + [encoder setStencilFrontReferenceValue:(uint32_t)q.front backReferenceValue:(uint32_t)q.back]; + [stencilReferences addObject:@{@"encoder":@(q.encoder),@"session":@(session),@"front":@(q.front),@"back":@(q.back)}];break; + } + case OM_SET_PIPELINE: { + REQUEST(OMSetPipeline);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id pipeline=get(q.pipeline,@protocol(MTLRenderPipelineState)); + if(!encoder || !pipeline){result.error=-2;break;}[encoder setRenderPipelineState:pipeline];break; + } + case OM_SET_CULL: { + REQUEST(OMSetCull);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));if(!encoder || q.mode>MTLCullModeBack){result.error=-2;break;}[encoder setCullMode:(MTLCullMode)q.mode];break; + } + case OM_DRAW: { + REQUEST(OMDraw);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || q.primitive>MTLPrimitiveTypeTriangleStrip || !q.count || q.count>65536 || q.start>65536 || !q.instances || q.instances>1024){result.error=-2;break;} + [encoder drawPrimitives:(MTLPrimitiveType)q.primitive vertexStart:q.start vertexCount:q.count instanceCount:q.instances];break; + } + case OM_END: { + REQUEST(OMHandle);id encoder=get(q.handle,@protocol(MTLCommandEncoder));if(!encoder || !openEncoders[@(q.handle)]){result.error=-2;break;}[encoder endEncoding];[openEncoders removeObjectForKey:@(q.handle)];break; + } + case OM_COMMIT:case OM_COMMAND_SUBMIT: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer));if(!command || hasOpenEncoder(q.handle)){result.error=-2;break;} + if(command.status>MTLCommandBufferStatusEnqueued){result.error=-2;break;} + for(NSNumber *handle in pendingEnqueues)if(handle.unsignedLongLongValue!=q.handle && [pendingEnqueues[handle][@"command"] commandQueue]==command.commandQueue){result.error=-4;break;} + if(result.error)break;[command commit];[pendingEnqueues removeObjectForKey:@(q.handle)]; + if(header.operation==OM_COMMIT)[command waitUntilCompleted]; + else asyncCommands[@(q.handle)]=@{@"session":@(session),@"command":command}; + OMCommandResult value=observeCommand(q.handle,command,header.operation==OM_COMMIT?@"blocking-commit":@"nonblocking-submit");error=command.error; + if(value.statusMTLCommandBufferStatusCompleted || error || + (value.status==MTLCommandBufferStatusCompleted && (!value.start_ns || value.end_ns<=value.start_ns))){result.error=-3;break;} + if(header.operation==OM_COMMAND_SUBMIT)[submitted addObject:@{@"command":@(q.handle),@"session":@(session),@"actual_status":@(value.status),@"wait_until_completed_called":@NO}]; + body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_COMMAND_STATUS: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer)); + if(!command || command.statusMTLCommandBufferStatusCompleted || error || (value.status==MTLCommandBufferStatusCompleted && (!value.start_ns || value.end_ns<=value.start_ns))){result.error=-3;break;} + body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_READ_TEXTURE: { + REQUEST(OMReadTexture);id texture=get(q.texture,@protocol(MTLTexture)); + if(!texture || texture.storageMode!=MTLStorageModeShared || q.level || !q.width || !q.height || q.x>texture.width || q.width>texture.width-q.x + || q.y>texture.height || q.height>texture.height-q.y || (texture.pixelFormat!=MTLPixelFormatR32Float && texture.pixelFormat!=MTLPixelFormatRG16Float && texture.pixelFormat!=MTLPixelFormatBGRA8Unorm && texture.pixelFormat!=MTLPixelFormatRGBA8Unorm) + || q.width>OFFLINE_METAL_MAX_REPLY/4/q.height){result.error=-2;break;} + NSMutableData *pixels=[NSMutableData dataWithLength:q.width*q.height*4]; + [texture getBytes:pixels.mutableBytes bytesPerRow:q.width*4 fromRegion:MTLRegionMake2D(q.x,q.y,q.width,q.height) mipmapLevel:0];body=pixels; + [readbacks addObject:@{@"texture":@(q.texture),@"bytes":@(pixels.length),@"sha256":hashBytes(pixels.bytes,pixels.length)}];break; + } + case OM_WRITE_TEXTURE: { + if(input.size() texture=get(q.texture,@protocol(MTLTexture)); + if(!texture || texture.storageMode!=MTLStorageModeShared || q.level || !q.width || !q.height || q.x>texture.width || q.width>texture.width-q.x + || q.y>texture.height || q.height>texture.height-q.y || (texture.pixelFormat!=MTLPixelFormatR32Float && texture.pixelFormat!=MTLPixelFormatRG16Float && texture.pixelFormat!=MTLPixelFormatBGRA8Unorm && texture.pixelFormat!=MTLPixelFormatRGBA8Unorm) + || q.width>SIZE_MAX/4 || q.stride!=q.width*4 || q.height>(input.size()-sizeof(q))/q.stride + || q.height*q.stride!=input.size()-sizeof(q)){result.error=-2;break;} + const void *pixels=input.data()+sizeof(q); + [texture replaceRegion:MTLRegionMake2D(q.x,q.y,q.width,q.height) mipmapLevel:0 withBytes:pixels bytesPerRow:q.stride]; + [uploads addObject:@{@"texture":@(q.texture),@"x":@(q.x),@"y":@(q.y),@"width":@(q.width),@"height":@(q.height), + @"bytes":@(q.stride*q.height),@"sha256":hashBytes(pixels,q.stride*q.height)}];break; + } + case OM_BUFFER: { + REQUEST(OMBuffer); + if(!q.bytes || q.bytes>OFFLINE_METAL_MAX_BUFFER || q.bytes>device.maxBufferLength || q.options){result.error=-4;break;} + result.handle=save([device newBufferWithLength:q.bytes options:MTLResourceStorageModeShared]);if(!result.handle)result.error=-3;break; + } + case OM_WRITE_BUFFER: { + if(input.size() buffer=get(q.buffer,@protocol(MTLBuffer)); + if(!buffer || !q.bytes || q.offset>buffer.length || q.bytes>buffer.length-q.offset || q.bytes!=input.size()-sizeof(q)){result.error=-2;break;} + memcpy((unsigned char *)buffer.contents+q.offset,input.data()+sizeof(q),q.bytes); + [bufferWrites addObject:@{@"buffer":@(q.buffer),@"offset":@(q.offset),@"bytes":@(q.bytes),@"sha256":hashBytes(input.data()+sizeof(q),q.bytes)}];break; + } + case OM_READ_BUFFER: { + REQUEST(OMBufferRegion);id buffer=get(q.buffer,@protocol(MTLBuffer)); + if(!buffer || !q.bytes || q.bytes>OFFLINE_METAL_MAX_REPLY || q.offset>buffer.length || q.bytes>buffer.length-q.offset){result.error=-2;break;} + body=[NSData dataWithBytes:(unsigned char *)buffer.contents+q.offset length:q.bytes]; + [bufferReads addObject:@{@"buffer":@(q.buffer),@"offset":@(q.offset),@"bytes":@(q.bytes),@"sha256":hashBytes(body.bytes,body.length)}];break; + } + case OM_VERTEX_BUFFER:case OM_FRAGMENT_BUFFER: { + REQUEST(OMSetBuffer);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + id buffer=q.buffer?get(q.buffer,@protocol(MTLBuffer)):nil; + if(!encoder || q.index>=31 || (q.buffer && !buffer) || (!buffer && q.offset) || (buffer && q.offset>=buffer.length)){result.error=-2;break;} + if(header.operation==OM_VERTEX_BUFFER)[encoder setVertexBuffer:buffer offset:q.offset atIndex:q.index]; + else [encoder setFragmentBuffer:buffer offset:q.offset atIndex:q.index];break; + } + case OM_DRAW_INDEXED: { + REQUEST(OMDrawIndexed);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id buffer=get(q.buffer,@protocol(MTLBuffer)); + uint64_t stride=q.type==MTLIndexTypeUInt16?2:4; + if(!encoder || !buffer || q.primitive>MTLPrimitiveTypeTriangleStrip || q.type>MTLIndexTypeUInt32 || !q.count || + q.count>65536 || !q.instances || q.instances>1024 || q.offset%stride || q.offset>buffer.length || q.count>(buffer.length-q.offset)/stride){result.error=-2;break;} + [encoder drawIndexedPrimitives:(MTLPrimitiveType)q.primitive indexCount:q.count indexType:(MTLIndexType)q.type + indexBuffer:buffer indexBufferOffset:q.offset instanceCount:q.instances baseVertex:q.baseVertex baseInstance:q.baseInstance];break; + } + case OM_BLIT: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer));if(!command || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.handle)){result.error=-2;break;} + result.handle=save([command blitCommandEncoder]);if(!result.handle)result.error=-3; + else openEncoders[@(result.handle)]=@{@"encoder":objects[@(result.handle)],@"command_object":command,@"command_handle":@(q.handle),@"session":@(session)};break; + } + case OM_COPY_BUFFER: { + REQUEST(OMCopyBuffer);id encoder=getEncoder(q.encoder,@protocol(MTLBlitCommandEncoder)); + id source=get(q.source,@protocol(MTLBuffer)),destination=get(q.destination,@protocol(MTLBuffer)); + if(!encoder || !source || !destination || !q.bytes || ((q.sourceOffset|q.destinationOffset|q.bytes)&3) || q.sourceOffset>source.length || q.bytes>source.length-q.sourceOffset || + q.destinationOffset>destination.length || q.bytes>destination.length-q.destinationOffset || + (source==destination && q.sourceOffset encoder=getEncoder(q.encoder,@protocol(MTLBlitCommandEncoder)); + id source=get(q.source,@protocol(MTLTexture)),destination=get(q.destination,@protocol(MTLTexture)); + if(!encoder || !source || !destination || source==destination || source.pixelFormat!=destination.pixelFormat || + source.textureType!=MTLTextureType2D || destination.textureType!=MTLTextureType2D || + q.sourceSlice || q.destinationSlice || q.sourceLevel || q.destinationLevel || q.sourceOrigin[2] || q.destinationOrigin[2] || q.sourceSize[2]!=1 || + !q.sourceSize[0] || !q.sourceSize[1] || q.sourceOrigin[0]>source.width || q.sourceSize[0]>source.width-q.sourceOrigin[0] || + q.sourceOrigin[1]>source.height || q.sourceSize[1]>source.height-q.sourceOrigin[1] || + q.destinationOrigin[0]>destination.width || q.sourceSize[0]>destination.width-q.destinationOrigin[0] || + q.destinationOrigin[1]>destination.height || q.sourceSize[1]>destination.height-q.destinationOrigin[1]){result.error=-2;break;} + if(source.pixelFormat!=MTLPixelFormatR32Float && source.pixelFormat!=MTLPixelFormatRG16Float && source.pixelFormat!=MTLPixelFormatRGBA8Unorm && source.pixelFormat!=MTLPixelFormatBGRA8Unorm){result.error=-4;break;} + [encoder copyFromTexture:source sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(q.sourceOrigin[0],q.sourceOrigin[1],0) + sourceSize:MTLSizeMake(q.sourceSize[0],q.sourceSize[1],1) toTexture:destination destinationSlice:0 destinationLevel:0 + destinationOrigin:MTLOriginMake(q.destinationOrigin[0],q.destinationOrigin[1],0)]; + [textureCopies addObject:@{@"source":@(q.source),@"destination":@(q.destination),@"session":@(session),@"format":@(source.pixelFormat), + @"source_storage":@(source.storageMode),@"destination_storage":@(destination.storageMode),@"width":@(q.sourceSize[0]),@"height":@(q.sourceSize[1])}];break; + } + case OM_VERTEX_TEXTURE:case OM_FRAGMENT_TEXTURE: { + REQUEST(OMSetTexture);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id texture=q.texture?get(q.texture,@protocol(MTLTexture)):nil; + if(!encoder || q.index>=128 || (q.texture && !texture)){result.error=-2;break;} + if(header.operation==OM_VERTEX_TEXTURE)[encoder setVertexTexture:texture atIndex:q.index];else [encoder setFragmentTexture:texture atIndex:q.index]; + [textureBindings addObject:@{@"encoder":@(q.encoder),@"texture":@(q.texture),@"index":@(q.index),@"vertex":@(header.operation==OM_VERTEX_TEXTURE)}];break; + } + case OM_SCISSOR: { + REQUEST(OMScissor);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));NSArray *extent=renderExtents[@(q.encoder)]; + if(!encoder || !extent || !q.width || !q.height || q.x>[extent[0] unsignedLongLongValue] || q.y>[extent[1] unsignedLongLongValue] || + q.width>[extent[0] unsignedLongLongValue]-q.x || q.height>[extent[1] unsignedLongLongValue]-q.y){result.error=-2;break;} + [encoder setScissorRect:(MTLScissorRect){(NSUInteger)q.x,(NSUInteger)q.y,(NSUInteger)q.width,(NSUInteger)q.height}];break; + } + case OM_VIEWPORT: { + REQUEST(OMViewport);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || !std::isfinite(q.x) || !std::isfinite(q.y) || !std::isfinite(q.width) || !std::isfinite(q.height) || + !std::isfinite(q.nearDepth) || !std::isfinite(q.farDepth) || q.width<=0 || q.height<=0 || q.nearDepth<0 || q.farDepth>1 || q.nearDepth>q.farDepth){result.error=-2;break;} + [encoder setViewport:(MTLViewport){q.x,q.y,q.width,q.height,q.nearDepth,q.farDepth}];break; + } + case OM_BLEND_COLOR: { + REQUEST(OMBlendColor);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || !std::isfinite(q.red) || !std::isfinite(q.green) || !std::isfinite(q.blue) || !std::isfinite(q.alpha)){result.error=-2;break;} + [encoder setBlendColorRed:q.red green:q.green blue:q.blue alpha:q.alpha];break; + } + case OM_RELEASE: {REQUEST(OMHandle);if(!objects[@(q.handle)]){result.error=-2;break;}[objects removeObjectForKey:@(q.handle)];[renderExtents removeObjectForKey:@(q.handle)];[fenceProducers removeObjectForKey:@(q.handle)];[eventProducers removeObjectForKey:@(q.handle)];[sharedStates removeObjectForKey:@(q.handle)];break;} + case OM_FINISH:if(expectedSessions || !input.empty()){result.error=-2;break;}finishSubmitted(0,true,@"finish");finished=true;break; + default:result.error=-4;break; + } + } @catch(NSException *exception) {result.error=-5;body=[exception.reason dataUsingEncoding:NSUTF8StringEncoding];} +#undef REQUEST + if(error)body=[error.localizedDescription dataUsingEncoding:NSUTF8StringEncoding]; + if(body.length>OFFLINE_METAL_MAX_REPLY){result.error=-2;body=nil;} + if(result.error) { + NSString *reason=[[NSString alloc] initWithData:body?:[NSData data] encoding:NSUTF8StringEncoding]; + fprintf(stderr,"OFFLINE_METAL_ERROR request=%u operation=%u error=%d %s\n",request,header.operation,result.error, + reason?[[reason substringToIndex:MIN(reason.length,(NSUInteger)512)] UTF8String]:"");fflush(stderr); + } + [operations addObject:@{@"request":@(request),@"client_request":@(header.request),@"session":@(session),@"operation":@(header.operation),@"input_bytes":@(input.size()),@"error":@(result.error),@"handle":@(result.handle),@"result_bytes":@(body.length)}]; + header.operation|=OM_WIRE_RESPONSE;header.bytes=(uint32_t)(sizeof(result)+body.length); + healthy=exact(fd,&header,sizeof(header),true) && (!expectedSessions || exact(fd,&session,sizeof(session),true)) + && exact(fd,&result,sizeof(result),true) && (!body.length || exact(fd,(void *)body.bytes,body.length,true)); + } + } + close(fd);unlink(path); + // End abandoned encoding before releasing its real objects. Never commit + // interrupted buffers or invent a GPU completion to make teardown succeed. + abortEncoders(0,true,healthy?@"unclosed-at-finish":@"transport-lost"); + finishSubmitted(0,true,healthy?@"finish":@"transport-lost"); + cancelNotifications(0,true); + return @{@"host_device":device.name,@"operations":operations,@"completed_commands":completed,@"texture_readbacks":readbacks,@"texture_uploads":uploads, + @"transport_healthy":@(healthy),@"session_finished":@(finished),@"enqueued_commands":enqueued,@"pending_enqueues":@(pendingEnqueues.count), + @"aborted_encoders":abortedEncoders,@"open_encoders":@(openEncoders.count), + @"shared_event_values":sharedValues,@"shared_event_labels":sharedLabels,@"shared_event_notifications":notificationJournal,@"shared_event_exports":exportJournal,@"live_shared_exports":@(sharedExports.count), + @"events_created":eventsCreated,@"event_operations":eventOperations, + @"submitted_commands":submitted,@"command_observations":commandObservations,@"failed_commands":failedCommands,@"pending_submissions":@(asyncCommands.count), + @"function_specializations":specializations,@"fences_created":fencesCreated,@"fence_operations":fenceOperations,@"texture_hazard_modes":textureHazards, + @"buffer_uploads":bufferWrites,@"buffer_readbacks":bufferReads,@"buffer_copies":bufferCopies,@"queue_priorities":queuePriorities,@"texture_copies":textureCopies, + @"depth_stencil_states":depthStates,@"stencil_references":stencilReferences,@"attachment_passes":attachmentPasses, + @"blend_pipelines":blendPipelines,@"texture_bindings":textureBindings,@"vertex_pipelines":vertexPipelines, + @"boot_console_prefix_bytes":@(bootBytes),@"isolated_sessions":@(expectedSessions!=0),@"opened_sessions":@(openedSessions),@"closed_sessions":@(closedSessions), + @"cpu_pixel_renderer":@NO,@"guest_system_device_discovery":@NO, + @"adreno_execution":@NO,@"original_desktop":@NO}; +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..424da21 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 NativeMetal contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NativeAdrenoMetalPlugin.mm b/NativeAdrenoMetalPlugin.mm new file mode 100644 index 0000000..00b8b9b --- /dev/null +++ b/NativeAdrenoMetalPlugin.mm @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: MIT +// Adreno Metal plugin, reached through the original discovery (IOAcceleratorES -> MetalPluginName -> NSBundle -> principal class -> +// initWithAcceleratorPort:). Default (stage 1): it runs the shared userspace GPU test (NativeAdrenoUserTest.h) from inside a real system +// process, under that process's sandbox, and then returns nil. No device is registered, so the original software rendering keeps running unchanged. +// Published (the image carries the file NativeAdrenoMetal.publish beside the bundle): backboardd, the one process that owns the GPU, returns the native +// Metal frontend's device (the metal2vulkan runtime on Turnip, on the Adreno 740), which the original discovery registers; the CARenderServer then +// renders the system UI through it, and the display driver's frames are written to the scanout by the GPU (GpuScanoutClient.h). Every other process +// returns nil and keeps the software renderer. +#import +#include +#include +#include +#include +#include +#include "NativeAdrenoUserTest.h" +#include "NativeMetalBridgePhase.h" +#include "CacheExport.h" + +@interface _MTLDevice : NSObject +@end +@interface NativeAdrenoMetalDeviceBase : NSObject // only a lock object for the published mode's one-time setup +@end +@implementation NativeAdrenoMetalDeviceBase +@end +@interface NativeAdrenoMetalDevice : _MTLDevice +- (id)initWithAcceleratorPort:(unsigned)port; +@end + +// ---- published mode ---- +namespace publish { +// The image's marker: a file beside the bundle in the union view of /System/Library/Extensions (the bundle's own seal stays untouched). +// A file beside the bundle: its path in path, true when it is readable. +static bool requestedPath(const char *name, char *path, size_t size) { + Dl_info info = {}; + if (!dladdr(reinterpret_cast(&requestedPath), &info) || !info.dli_fname) return false; + snprintf(path, size, "%s", info.dli_fname); + char *slash = strrchr(path, '/'); // the bundle's executable: /NativeAdrenoMetal.bundle/NativeAdrenoMetal + if (!slash) return false; + *slash = 0; + slash = strrchr(path, '/'); + if (!slash) return false; + snprintf(slash + 1, size - size_t(slash + 1 - path), "%s", name); + return access(path, R_OK) == 0; +} +static bool requested() { + char path[1024]; + return requestedPath("NativeAdrenoMetal.publish", path, sizeof path); +} +static Gpu *makeGpu(unsigned port) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return nullptr; + Gpu *gpu = new Gpu; + gpu->call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod"); + gpu->map = (MapFn)dlsym(iokit, "IOConnectMapMemory64"); + gpu->unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64"); + gpu->openFn = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen"); + gpu->closeFn = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose"); + gpu->port = port; + return gpu; +} +static std::string gCacheDirectory; // the runtime's cache directory (the one that opened the device), exported by CacheExport.h +// Progress markers 0x68xx (GPU6): 6800 entered, 6801 Turnip found, 6802 device made, 6803 returned to the discovery, 68f1 no Turnip, 68f2 no device. +static unsigned gDiagBits; +// Reads of the render server's own state for the framebuffer-format question (diagnostic only, for the exact QuartzCore of the target image; a read +// that fails or a build that differs reports zeros). Each value is one marker, 0x7Kvv: K picks the item, vv the value. +static bool readTask(uintptr_t address, void *out, size_t size) { + vm_size_t got = 0; + return vm_read_overwrite(mach_task_self(), address, size, (vm_address_t) out, &got) == KERN_SUCCESS && got == size; +} +static bool writeTask(uintptr_t address, const void *data, size_t size) { + return vm_write(mach_task_self(), address, (vm_offset_t) data, (mach_msg_type_number_t) size) == KERN_SUCCESS; +} +// EXPERIMENT (second report onward): this display path takes plain 8-bit BGRA only, so tell the render server's display that it has neither the deep +// (10-bit) framebuffer nor universal buffer compression. Whether the state it reads those from is the whole story is what the markers show. +static unsigned gReports; +struct DisplaySample {unsigned flags = 0, mode = 0, behaviour = 0, globals = 0; uint16_t display = 0; uintptr_t object = 0;}; +static DisplaySample sampleDisplay(id first, intptr_t slide) { + DisplaySample out; + uintptr_t a = first ? (uintptr_t) *(void **) ((char *) (__bridge void *) first + 8) : 0, b = 0, d = 0; + if (a && readTask(a + 0x40, &b, 8) && b && readTask(b + 0x60, &d, 8) && d) { + uint32_t f = 0; uint64_t m = 0; uint8_t byte[4] = {}; + readTask(d + 0x238, &f, 4); + readTask(d + 0x258, &out.display, 2); + readTask(d + 0x248, &m, 8); + readTask(d + 0x6c17, &byte[0], 1); + readTask(d + 0x6658 + 0x57d, &byte[1], 1); + readTask(d + 0x6658 + 0x58a, &byte[2], 1); + readTask(d + 0x6658 + 0x59b, &byte[3], 1); + out.object = d; + out.flags = f & 0xff; + out.mode = (unsigned) ((m >> 55) & 31) | (m ? 0x20 : 0); + out.behaviour = (byte[0] ? 1 : 0) | (byte[1] ? 2 : 0) | (byte[2] ? 4 : 0) | (byte[3] ? 8 : 0); + } + uint8_t v[4] = {}; + readTask(0x1f491ae10 + slide, &v[0], 1); // the render server's deep-framebuffer flag + readTask(0x1ed7abf10 + slide, &v[1], 1); // framebuffer compression in effect + readTask(0x1f4919dc0 + slide, &v[2], 1); // its default + readTask(0x1f4919c20 + slide, &v[3], 1); // universal buffer compression + out.globals = (v[0] ? 1 : 0) | (v[1] ? 2 : 0) | (v[2] ? 4 : 0) | (v[3] ? 8 : 0); + return out; +} +static void markSample(Gpu &gpu, const DisplaySample &x, unsigned base) { + gpu.mark(base | (x.flags << 4) | x.globals); // display flags (bit 3 deep), globals: deep/compression/default/universal + gpu.mark((base + 0x1000) | (x.display >> 4)); // display limits word: 0x200 compression support, 0x1c00 type + gpu.mark((base + 0x2000) | (x.mode << 4) | x.behaviour); // mode colour field, ICC/other behaviour bytes +} +static void reportDisplayState(Gpu &gpu) { + ++gReports; + static DisplaySample before; + static bool changed; + Class serverClass = NSClassFromString(@"CAWindowServer"); + void *start = dlsym(RTLD_DEFAULT, "CARenderServerStart"); + if (!serverClass || !start) return; + const intptr_t slide = (intptr_t) start - 0x189651efc; + @try { + id server = ((id (*)(id, SEL)) objc_msgSend)(serverClass, NSSelectorFromString(@"serverIfRunning")); + NSArray *displays = server ? ((id (*)(id, SEL)) objc_msgSend)(server, NSSelectorFromString(@"displays")) : nil; + id first = displays.count ? displays[0] : nil; + DisplaySample now = sampleDisplay(first, slide); + if (!changed) { + before = now; + if (gReports >= 2 && now.object) { + uint32_t f = 0; + if (readTask(now.object + 0x238, &f, 4) && (f & 8)) {f &= ~8u; writeTask(now.object + 0x238, &f, 4);} + uint8_t zero = 0; + writeTask(0x1f4919c20 + slide, &zero, 1); + changed = true; + now = sampleDisplay(first, slide); + } + } + markSample(gpu, before, 0x7000); + markSample(gpu, now, 0xA000); + } @catch (NSException *) {} +} +static id deviceForPort(unsigned port) { + static id device; + static bool tried; + @synchronized([NativeAdrenoMetalDeviceBase class]) { + if (tried) return device; + tried = true; + // One process owns the GPU (the driver grants one connection): the render server. The others keep the software renderer. + if (strcmp(getprogname(), "backboardd") != 0) return nil; + // The discovery releases the accelerator port when it is done; the monitor and the scanout client keep using it. + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); + } + Gpu *gpu = makeGpu(port); + if (!gpu) return nil; + gpu->mark(0x6800); + char turnip[1024]; + if (!turnip::libraryPath(turnip, sizeof turnip) || access(turnip, R_OK) != 0) {gpu->mark(0x68f1); return nil;} + gpu->mark(0x6801); + // The runtime refuses to open without a writable shader-cache directory, and which directories backboardd's sandbox lets it write is not + // known from here: try the user cache and temp directories, then /tmp and /var/tmp, and use the first that opens a device. + uint64_t registryID = 0; + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto entryID = (int (*)(unsigned, uint64_t *))dlsym(iokit, "IORegistryEntryGetRegistryEntryID")) entryID(port, ®istryID); + } + // What the render server sees for its framebuffer choice (marker 0x6Axx): bit0 the deep-framebuffer preference is set, bit1 it is false, + // bit2 MobileGestalt's answer behind the render server's deep flag is true, bit3 it exists, bit4 the compression capability dictionary exists, + // bit5 the process is not root. + { + unsigned bits = 0; + if (CFTypeRef value = CFPreferencesCopyAppValue(CFSTR("CAEnableDeepFramebuffer"), kCFPreferencesCurrentApplication)) { + bits |= 1; + if (CFGetTypeID(value) == CFBooleanGetTypeID() && !CFBooleanGetValue((CFBooleanRef) value)) bits |= 2; + CFRelease(value); + } + if (auto copyAnswer = (CFTypeRef (*)(CFStringRef)) dlsym(RTLD_DEFAULT, "MGCopyAnswer")) { + if (CFTypeRef deep = copyAnswer(CFSTR("Aixt/MEN2O2B7f+8m4TxUA"))) { + bits |= 8; + if (CFGetTypeID(deep) == CFBooleanGetTypeID() && CFBooleanGetValue((CFBooleanRef) deep)) bits |= 4; + CFRelease(deep); + } + if (CFTypeRef caps = copyAnswer(CFSTR("AoKnINTLPoKML3ctoP0AZg"))) {bits |= 16; CFRelease(caps);} + } + if (getuid() != 0) bits |= 32; + gDiagBits = bits; + gpu->mark(0x6A00 | bits); + } + // The render server compresses its framebuffer (an Apple/AGX layout this GPU does not write) unless told not to; the display driver takes plain rows. + setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1); + // The runtime's creation log reports each VkPipeline making's time to the device's sink, which the telemetry's V row sums. + setenv("METAL2VULKAN_LOG_PIPELINES", "1", 0); + CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); + CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); + char candidates[4][512] = {}; + if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0; + if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0; + strlcpy(candidates[2], "/tmp/", sizeof candidates[2]); + strlcpy(candidates[3], "/private/var/tmp/", sizeof candidates[3]); + for (unsigned i = 0; i < 4 && !device; ++i) { + if (!candidates[i][0]) continue; + strlcat(candidates[i], "m2v", sizeof candidates[i]); + mkdir(candidates[i], 0700); + if (access(candidates[i], W_OK) != 0) continue; + // The image's pipeline pack (an earlier boot's export), into this directory before the runtime reads it. Marker 0x6Cnn: files written + // (saturated at 0xfe), 0x6Cff the pack was refused. + if (char pack[1024]; requestedPath("NativeAdrenoMetal.m2vpack", pack, sizeof pack)) { + const int imported = cacheexport::importPack(pack, candidates[i]); + if (imported) gpu->mark(0x6C00 | (imported < 0 ? 0xff : imported > 0xfe ? 0xfe : imported)); + } + @try { + device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr); + } @catch (NSException *exception) { + device = nil; + } + if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i];} // which directory served + } + if (!device) { + gpu->mark(0x68f2); + char text[64]; + snprintf(text, sizeof text, "PUB no device: %.40s", nmb_last_error()); + streamText(*gpu, text); + return nil; + } + gpu->mark(0x6802); + // The status line (GPU8) every ten seconds, and the display hand-off client on its own thread: both only read or serve, neither is the discovery's. + pthread_t monitor, scanout; + pthread_attr_t attributes; + pthread_attr_init(&attributes); + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + pthread_attr_setstacksize(&attributes, 1u << 20); + pthread_create(&monitor, &attributes, [](void *arg) -> void * { + Gpu *g = static_cast(arg); + for (;;) { + sleep(10); + char status[64], text[64]; + static unsigned phase; + // status (counters, then the last two refusals' texts), formats; the stream carries 63 characters per line, so a long status goes + // out in 57-character pieces on consecutive lines (the UFS log keeps every line). + char full[420]; + native_bridge_metal_status(full, sizeof full); + const size_t length = strlen(full); + const unsigned pieces = (unsigned)((length + 56) / 57); + const unsigned step = phase++ % (1 + (pieces ? (pieces > 7 ? 7 : pieces) : 1)); + if (step == 0) { + native_bridge_metal_formats(status, sizeof status); + snprintf(text, sizeof text, "PUB %.58s", status); + } else { + snprintf(text, sizeof text, "ST%u %.57s", step - 1, full + (size_t)(step - 1) * 57); + } + streamText(*g, text); + reportDisplayState(*g); + g->mark(0x6A00 | gDiagBits); // the diagnostics page lists the last few markers only + } + return nullptr; + }, gpu); + // The fixed CPU/memory/pipeline workload of native_bridge_metal_benchmark, once while the render server starts and once on an idle + // desktop, against the host's run of the same (pipeline_timing_host draw=2): how much slower the tablet is, and where. + pthread_t benchmark; + pthread_create(&benchmark, &attributes, [](void *arg) -> void * { + id dev = (__bridge id)arg; + sleep(3); + native_bridge_metal_benchmark(dev, "/System/Library/Frameworks/QuartzCore.framework/default.metallib", 8, nullptr, 0); + sleep(150); + native_bridge_metal_benchmark(dev, "/System/Library/Frameworks/QuartzCore.framework/default.metallib", 8, nullptr, 0); + return nullptr; + }, (__bridge void *)device); + pthread_t telemetry; + pthread_create(&telemetry, &attributes, [](void *arg) -> void * { + Gpu *g = static_cast(arg); + sleep(5); + // The bridge's telemetry table, resent whole every four seconds as markers (two characters each): 0xC000|row starts a row of the + // kernel's table, 0x8000|(a<<7)|b appends two characters. Rows are at most 90 characters; 48 rows. + for (;;) { + static char table[6144]; + native_bridge_metal_telemetry(table, sizeof table); + unsigned row = 0; + for (char *line = table; *line && row < 48;) { + char *end = strchr(line, '\n'); + if (end) *end = 0; + g->mark(0xC000u | row); + const size_t length = strlen(line) < 90 ? strlen(line) : 90; + for (size_t i = 0; i < length; i += 2) { + const unsigned a = (unsigned char)line[i] & 0x7f, b = i + 1 < length ? (unsigned char)line[i + 1] & 0x7f : 0; + g->mark(0x8000u | (a << 7) | b); + } + ++row; + if (!end) break; + line = end + 1; + } + sleep(6); + } + return nullptr; + }, gpu); + pthread_create(&scanout, &attributes, [](void *arg) -> void * { + Gpu *g = static_cast(arg); + sleep(3); + @try { + bridgephase::serveScanout(*g, device); + } @catch (NSException *exception) { + g->mark(0x68f3); + } + return nullptr; + }, gpu); + pthread_t exporter; + pthread_create(&exporter, &attributes, [](void *arg) -> void * { + cacheexport::run(*static_cast(arg), gCacheDirectory); + return nullptr; + }, gpu); + pthread_attr_destroy(&attributes); + gpu->mark(0x6803); + return device; + } +} +} // namespace publish + +@implementation NativeAdrenoMetalDevice +- (id)initWithAcceleratorPort:(unsigned)port { + self = [super init]; + if (!self || !port) return nil; + if (publish::requested()) return publish::deviceForPort(port); + static bool attempted; + if (attempted) return nil; + attempted = true; + // The test runs on its own thread: a kernel call that never returns (int-042: BoFree) then stalls that thread only, not SpringBoard's + // main thread (the UI froze with the test running inline). + // The discovery code releases the accelerator port as soon as this returns nil, which is before the test thread has started: the first + // marker (and every later open) then went to a dead name and never reached the kernel (K0 in int-043..052, with the plugin file opened by + // SpringBoard). Keep one extra user reference on the port, and let the thread look the service up itself as well. + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); + } + struct Job {unsigned port; char name[32];}; + Job *job = new Job{port, {}}; + strlcpy(job->name, getprogname(), sizeof job->name); + pthread_t thread; + pthread_attr_t attributes; + pthread_attr_init(&attributes); + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + if (pthread_create(&thread, &attributes, [](void *arg) -> void * { + Job *j = static_cast(arg); + unsigned port = j->port; + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + auto matching = (void *(*)(const char *))dlsym(iokit, "IOServiceMatching"); + auto getService = (unsigned (*)(unsigned, void *))dlsym(iokit, "IOServiceGetMatchingService"); + if (matching && getService) { + const unsigned own = getService(0, matching("IOAcceleratorES")); + if (own) port = own; + } + } + gAfterTurnip = bridgephase::run; + adrenoUserTest(port, j->name); + delete j; + return nullptr; + }, job) != 0) delete job; + pthread_attr_destroy(&attributes); + return nil; +} +@end diff --git a/NativeAdrenoProbeDaemon.cpp b/NativeAdrenoProbeDaemon.cpp new file mode 100644 index 0000000..a5f3b29 --- /dev/null +++ b/NativeAdrenoProbeDaemon.cpp @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT +// Adreno probe daemon: started by launchd from an owner job (a union view over /System/Library/LaunchDaemons, see the boot-time addon +// mount), so it runs outside every sandbox profile. It waits for the Adreno accelerator service to appear, runs the shared userspace GPU +// test once and exits. The result reaches the kernel through the driver's progress markers and SetMetadata. +#include +#include "NativeAdrenoUserTest.h" + +using MatchingFn = void *(*)(const char *); +using GetServiceFn = unsigned (*)(unsigned, void *); + +int main() { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return 1; + auto matching = (MatchingFn)dlsym(iokit, "IOServiceMatching"); + auto getService = (GetServiceFn)dlsym(iokit, "IOServiceGetMatchingService"); + if (!matching || !getService) return 2; + // The service is published a few seconds into boot (after the GPU is powered); poll for up to two minutes. + unsigned port = 0; + for (unsigned i = 0; i < 240 && !port; ++i) { + port = getService(0, matching("IOAcceleratorES")); + if (!port) usleep(500000); + } + if (!port) return 3; + return adrenoUserTest(port, "adrenoprobed") >= 0 ? 0 : 4; +} diff --git a/NativeAdrenoUserTest.h b/NativeAdrenoUserTest.h new file mode 100644 index 0000000..2ed55dd --- /dev/null +++ b/NativeAdrenoUserTest.h @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: MIT +// The userspace GPU test shared by the Adreno Metal plugin (runs inside whichever system process the original Metal discovery loads it +// into, under that process's sandbox) and the launchd probe daemon (no sandbox profile). It proves the whole userspace path to the GPU: +// open the driver, move every ABI method through an admitted selector (or the mailbox page), map buffer objects write-combined or +// uncached, write an indirect buffer from userspace, submit it, wait for the fence, read the result back, and report one line to the +// kernel through SetMetadata. Progress markers are IOServiceOpen calls with type 0x4147, which the driver counts and refuses. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "NativeAdrenoABI.h" +#include "NativeTurnipSmoke.h" +#if __has_include("TurnipReplay.h") +#include "TurnipReplay.h" +#define NATIVE_TURNIP_REPLAY 1 +#endif +extern "C" { +#include "AdrenoPM4.h" +} + +namespace { +using IOServiceOpenFn = int (*)(unsigned, unsigned, uint32_t, unsigned *); +using IOServiceCloseFn = int (*)(unsigned); +using CallFn = int (*)(unsigned, uint32_t, const void *, size_t, void *, size_t *); +using MapFn = int (*)(unsigned, uint32_t, unsigned, uint64_t *, uint64_t *, uint32_t); +using UnmapFn = int (*)(unsigned, uint32_t, unsigned, uint64_t); + +// The sandbox admits only some external-method selector numbers for this user-client class, and which ones is decided by the +// profile, not by us. Every ABI method therefore travels through one admitted selector as {magic, method, payload}; the driver +// recognises the magic before its own selector table. Probing 0..255 finds an admitted selector without knowing the profile. +constexpr uint32_t kTunnelMagic = 0x4e544e4c, kProbe = 0xffffffffu; +struct Gpu { + unsigned conn = 0; + CallFn call = nullptr; + MapFn map = nullptr; + UnmapFn unmap = nullptr; + uint32_t selector = 0; + unsigned admitted = 0; + uint64_t lastTicks = 0; + unsigned port = 0; + IOServiceOpenFn openFn = nullptr; + IOServiceCloseFn closeFn = nullptr; + // Progress marker: an IOServiceOpen with type 0x4147. The driver counts it and refuses it, so it needs no selector, no + // mapping and no connection, and survives a sandbox that admits nothing else or a crash after it. + void mark(uint32_t value) { + unsigned c = 0; + if (openFn && port && openFn(port, mach_task_self(), 0x41470000u | (value & 0xffff), &c) == 0 && c && closeFn) closeFn(c); + } + volatile uint32_t *mailbox = nullptr; + uint32_t mailboxSeq = 0; + int mailboxMethod(uint32_t m, const void *in, size_t is, void *out, size_t os) { + if (!mailbox || is > 0x700 || os > 0x700) return -1; + if (is) memcpy(const_cast(mailbox) + 0x10, in, is); + mailbox[3] = m; + mailbox[5] = (uint32_t)is; + mailbox[6] = (uint32_t)os; + __sync_synchronize(); + const uint32_t seq = ++mailboxSeq; + mailbox[1] = seq; + __sync_synchronize(); + for (unsigned i = 0; i < 6000 && mailbox[2] != seq; ++i) usleep(500); // the kernel poller runs at about 1 kHz + if (mailbox[2] != seq) return -2; + __sync_synchronize(); + const int status = (int)mailbox[4]; + if (!status && os) memcpy(out, const_cast(mailbox) + 0x200, os); + return status; + } + int method(uint32_t m, const void *in, size_t is, void *out, size_t os) { + if (mailbox) return mailboxMethod(m, in, is, out, os); + uint8_t buffer[8 + sizeof(NativeAdrenoAbiSubmit)]; + if (is > sizeof(NativeAdrenoAbiSubmit)) return -1; + const uint32_t header[2] = {kTunnelMagic, m}; + memcpy(buffer, header, 8); + if (is) memcpy(buffer + 8, in, is); + size_t o = os; + return call(conn, selector, buffer, 8 + is, os ? out : nullptr, os ? &o : nullptr); + } + // No selector is admitted: use the mailbox page (IOConnectMapMemory64 is not covered by the selector filter). + bool openMailbox() { + uint64_t address = 0, length = 0; + if (map(conn, 0xC0DE, mach_task_self(), &address, &length, 1) != 0 || length < 4096) return false; + mailbox = reinterpret_cast(address); + return mailbox[0] == 0x4d41494c; + } + // Returns true when some selector was admitted; the first one becomes the tunnel selector. + bool probe() { + const uint32_t header[2] = {kTunnelMagic, kProbe}; + bool found = false; + for (uint32_t s = 0; s < 256; ++s) { + if (call(conn, s, header, sizeof header, nullptr, nullptr) == 0) { + if (!found) selector = s; + found = true; + ++admitted; + } + } + return found; + } +}; + +void report(Gpu &gpu, const char *process, int step, int error, int verified, int verifiedWc, uint64_t ticks) { + // SetMetadata with a bogus handle: the driver records the text before it validates the handle. + NativeAdrenoAbiMetadata meta = {}; + meta.handle = 1; + char text[NativeAdrenoAbiMetadataBytes] = {}; + snprintf(text, sizeof text, "%.9s s%d e%x v%d/%d t%llx a%u/%u", process, step, error, verified, verifiedWc, + (unsigned long long)ticks, gpu.mailbox ? 0xffffu : gpu.admitted, gpu.selector); + meta.bytes = (uint32_t)strlen(text); + memcpy(meta.text, text, meta.bytes); + gpu.method(NativeAdrenoMethodSetMetadata, &meta, sizeof meta, nullptr, 0); +} + +// Everything the Turnip backend does for a first submission, in the same order; returns the verified dword count or -1. +int runTest(Gpu &gpu, uint32_t mapFlags, int &step, int &error) { + auto check = [&](int status) { + if (status && !error) error = status; + if (!status) ++step; + gpu.mark(status ? 0x4000 | (status & 0xfff) : 0x3000 | (mapFlags == 0x401 ? 0x100 : 0) | step); + return status == 0; + }; + NativeAdrenoAbiParams params = {}; + if (!check(gpu.method(NativeAdrenoMethodQuery, nullptr, 0, ¶ms, sizeof params))) return -1; + NativeAdrenoAbiDeviceInfo info = {}; + if (!check(gpu.method(NativeAdrenoMethodGetDeviceInfo, nullptr, 0, &info, sizeof info))) return -1; + NativeAdrenoAbiBoAlloc alloc = {4096, 12, 0}; + NativeAdrenoAbiBoAllocReply ib = {}, target = {}; + if (!check(gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &ib, sizeof ib)) || + !check(gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &target, sizeof target))) return -1; + const uint64_t ibIova = info.va_start, tgtIova = info.va_start + 0x1000; + NativeAdrenoAbiBoMap mapIb = {ib.handle, ibIova, 1, 0, 2, 0}, mapTgt = {target.handle, tgtIova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + if (!check(gpu.method(NativeAdrenoMethodBoMap, &mapIb, sizeof mapIb, &mr, sizeof mr)) || + !check(gpu.method(NativeAdrenoMethodBoMap, &mapTgt, sizeof mapTgt, &mr, sizeof mr))) return -1; + NativeAdrenoAbiBoCpuMap cmIb = {ib.handle}, cmTgt = {target.handle}; + NativeAdrenoAbiBoCpuMapReply crIb = {}, crTgt = {}; + if (!check(gpu.method(NativeAdrenoMethodBoCpuMap, &cmIb, sizeof cmIb, &crIb, sizeof crIb)) || + !check(gpu.method(NativeAdrenoMethodBoCpuMap, &cmTgt, sizeof cmTgt, &crTgt, sizeof crTgt))) return -1; + uint64_t ibAddr = 0, ibLen = 0, tgtAddr = 0, tgtLen = 0; + if (!check(gpu.map(gpu.conn, crIb.token, mach_task_self(), &ibAddr, &ibLen, mapFlags)) || + !check(gpu.map(gpu.conn, crTgt.token, mach_task_self(), &tgtAddr, &tgtLen, mapFlags))) return -1; + if (ibLen < 4096 || tgtLen < 4096) {error = 0xbad1; return -1;} + volatile uint32_t *iw = reinterpret_cast(ibAddr); + volatile uint32_t *tw = reinterpret_cast(tgtAddr); + for (unsigned i = 0; i < 1024; ++i) tw[i] = 0xdeadbeef; + const uint32_t values[3] = {0x0adc0de1u, 0xfeed0002u, 0x600d0003u}; + uint32_t words[12]; + for (unsigned i = 0; i < 3; ++i) { + if (AdrenoPM4Type7(0x3d, 3, &words[i * 4]) != AdrenoPM4Success) {error = 0xbad2; return -1;} + words[i * 4 + 1] = uint32_t(tgtIova + i * 4); + words[i * 4 + 2] = uint32_t(tgtIova >> 32); + words[i * 4 + 3] = values[i]; + } + for (unsigned i = 0; i < 12; ++i) iw[i] = words[i]; + __asm__ volatile("dsb sy" ::: "memory"); // the mapping is non-cacheable and the GPU is an outer-shareable observer + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {ib.handle, ibIova, sizeof words}; + NativeAdrenoAbiSubmitReply sr = {}; + if (!check(gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr))) return -1; + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + if (!check(gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0))) return -1; + __asm__ volatile("dsb sy" ::: "memory"); + const int verified = (tw[0] == values[0]) + (tw[1] == values[1]) + (tw[2] == values[2]) + (tw[3] == 0xdeadbeef); + NativeAdrenoAbiTimestamp ts = {}; + uint64_t ticks = 0; + if (check(gpu.method(NativeAdrenoMethodGetTimestamp, nullptr, 0, &ts, sizeof ts))) ticks = ts.ticks; + gpu.unmap(gpu.conn, crIb.token, mach_task_self(), ibAddr); + gpu.unmap(gpu.conn, crTgt.token, mach_task_self(), tgtAddr); + NativeAdrenoAbiBoCpuUnmap ui = {ib.handle, crIb.token, 0}, ut = {target.handle, crTgt.token, 0}; + check(gpu.method(NativeAdrenoMethodBoCpuUnmap, &ui, sizeof ui, nullptr, 0)); + check(gpu.method(NativeAdrenoMethodBoCpuUnmap, &ut, sizeof ut, nullptr, 0)); + NativeAdrenoAbiBoFree fi = {ib.handle}, ft = {target.handle}; + check(gpu.method(NativeAdrenoMethodBoFree, &fi, sizeof fi, nullptr, 0)); + check(gpu.method(NativeAdrenoMethodBoFree, &ft, sizeof ft, nullptr, 0)); + gpu.lastTicks = ticks; + return verified; +} +} + +// A report line as a marker stream (0x7fff, then 0x70cc per character): works while another connection holds the driver's single slot. +// Two threads of one process may report (the published mode's status monitor and the scanout client): the driver keeps one text cursor, so a line +// goes out whole. +static pthread_mutex_t gStreamLock = PTHREAD_MUTEX_INITIALIZER; +static void streamText(Gpu &gpu, const char *text) { + pthread_mutex_lock(&gStreamLock); + gpu.mark(0x7fff); + for (unsigned i = 0; i < 63 && text[i]; ++i) gpu.mark(0x7000 | (uint8_t)text[i]); + pthread_mutex_unlock(&gStreamLock); +} + +// Line into report (GPU7) instead of report2 (GPU8). +__attribute__((unused)) static void streamText7(Gpu &gpu, const char *text) { + pthread_mutex_lock(&gStreamLock); + gpu.mark(0x7ffe); + for (unsigned i = 0; i < 63 && text[i]; ++i) gpu.mark(0x7000 | (uint8_t)text[i]); + pthread_mutex_unlock(&gStreamLock); +} + +// Remap probe (int-064): does the CP read the new pages after a GPU address is moved to other physical pages? int-063's W1 run (the +// replay IB on fresh buffers at the same addresses) stopped both CP threads in the SQE's unknown-opcode trap, i.e. on wrong data. A +// CP_MEM_TO_MEM from a remapped page into a result page reports what the CP sees instead of executing it, so no variant can stop the CP: +// v0 first mapping, v1 remap to a page allocated before the old one is freed (so the old page keeps its old pattern), +// v2 as v1 with a UCHE invalidate (CP_EVENT_WRITE7 0x31) and CP_WAIT_FOR_IDLE before the copy, +// v3 as v1 but remapped while the CP is busy spinning on a flag (CP_WAIT_REG_MEM), then released: GX stays up across the TLB invalidate. +// Each result: '1' the new pattern, 'o' the previous mapping's pattern, 'z' zero, 'd' result not written, 'g' other, 'x' setup failed, +// 't' fence timeout. Reported as "P" in front of the replay line. The addresses (0x5f000000..) are never used elsewhere. +struct ProbeBo { + uint64_t handle = 0, address = 0; + uint32_t token = 0; + volatile uint32_t *words = nullptr; +}; +static int probeAlloc(Gpu &gpu, ProbeBo &b, uint32_t fill) { + NativeAdrenoAbiBoAlloc alloc = {4096, 12, 0}; + NativeAdrenoAbiBoAllocReply reply = {}; + if (int e = gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &reply, sizeof reply)) return e; + b.handle = reply.handle; + NativeAdrenoAbiBoCpuMap cm = {b.handle}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + b.token = cr.token; + uint64_t length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &b.address, &length, 0x101)) return e; + if (length < 4096) return 0xbad5; + b.words = reinterpret_cast(b.address); + for (unsigned i = 0; i < 1024; ++i) b.words[i] = fill; + __asm__ volatile("dsb sy" ::: "memory"); + return 0; +} +static int probeMap(Gpu &gpu, const ProbeBo &b, uint64_t iova) { + NativeAdrenoAbiBoMap map = {b.handle, iova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + return gpu.method(NativeAdrenoMethodBoMap, &map, sizeof map, &mr, sizeof mr); +} +static void probeFree(Gpu &gpu, ProbeBo &b) { + if (!b.handle) return; + if (b.address) { + gpu.unmap(gpu.conn, b.token, mach_task_self(), b.address); + NativeAdrenoAbiBoCpuUnmap cu = {b.handle, b.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + } + NativeAdrenoAbiBoFree f = {b.handle}; + gpu.method(NativeAdrenoMethodBoFree, &f, sizeof f, nullptr, 0); + b = ProbeBo(); +} +static char gProbe[6] = "P----"; +__attribute__((unused)) static void remapProbe(Gpu &gpu) { + constexpr uint64_t kSource = 0x5f000000ull, kResult = 0x5f010000ull, kIb = 0x5f020000ull, kFlag = 0x5f030000ull; + const uint32_t patterns[4] = {0x0a1a1a10u, 0x0b2b2b20u, 0x0c3c3c30u, 0x0d4d4d40u}; + ProbeBo ib, result, flag, source; + bool stuck = false; + if (probeAlloc(gpu, ib, 0) || probeAlloc(gpu, result, 0xdeadbeef) || probeAlloc(gpu, flag, 0) || + probeAlloc(gpu, source, patterns[0]) || probeMap(gpu, ib, kIb) || probeMap(gpu, result, kResult) || + probeMap(gpu, flag, kFlag) || probeMap(gpu, source, kSource)) { + for (unsigned v = 1; v < 5; ++v) gProbe[v] = 'x'; + probeFree(gpu, source); probeFree(gpu, flag); probeFree(gpu, result); probeFree(gpu, ib); + return; + } + for (unsigned v = 0; v < 4 && !stuck; ++v) { + gpu.mark(0x6d00 | v); + ProbeBo next; + if (v) { + // The new page exists before the old one is freed, so the two differ and the old one still holds the old pattern. + if (probeAlloc(gpu, next, patterns[v])) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + } + uint32_t words[32], n = 0; + auto packet = [&](uint32_t opcode, std::initializer_list payload) { + if (AdrenoPM4Type7(opcode, uint32_t(payload.size()), &words[n]) != AdrenoPM4Success) return false; + ++n; + for (uint32_t w : payload) words[n++] = w; + return true; + }; + bool ok = true; + if (v == 3) ok = packet(0x3c, {3u | (1u << 4), uint32_t(kFlag), uint32_t(kFlag >> 32), 1u, 0xffffffffu, 16u}); // wait flag == 1 + if (v == 2) ok = packet(0x46, {0x31u}) && packet(0x26, {}); + ok = ok && packet(0x73, {0u, uint32_t(kResult), uint32_t(kResult >> 32), uint32_t(kSource), uint32_t(kSource >> 32)}); + if (!ok) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + for (uint32_t i = 0; i < n; ++i) ib.words[i] = words[i]; + result.words[0] = 0xdeadbeef; + flag.words[0] = 0; + __asm__ volatile("dsb sy" ::: "memory"); + if (v && v != 3) { + probeFree(gpu, source); + if (probeMap(gpu, next, kSource)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); break;} + source = next; + } + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {ib.handle, kIb, uint64_t(n) * 4}; + NativeAdrenoAbiSubmitReply sr = {}; + if (gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + if (v == 3) { + usleep(20000); // the CP is spinning on the flag + probeFree(gpu, source); + if (probeMap(gpu, next, kSource)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); stuck = true; break;} + source = next; + flag.words[0] = 1; + __asm__ volatile("dsb sy" ::: "memory"); + } + int waited = -1; + for (unsigned t = 0; t < 10 && waited; ++t) { + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + waited = gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0); + } + if (waited) {gProbe[1 + v] = 't'; stuck = true; break;} + const uint32_t got = result.words[0], want = patterns[v]; + gProbe[1 + v] = got == want ? '1' : (v && got == patterns[v - 1]) ? 'o' : got == 0 ? 'z' : got == 0xdeadbeef ? 'd' : 'g'; + } + // A stuck CP may still reference the buffers. + if (!stuck) {probeFree(gpu, source); probeFree(gpu, flag); probeFree(gpu, result); probeFree(gpu, ib);} +} + +#ifdef NATIVE_TURNIP_REPLAY +// Prefix replay of Turnip's first submission (recorded on the host by scripts/gen-turnip-replay.py with the tablet's 4 KiB page size, so +// the GPU addresses are the ones Turnip itself uses on the tablet): its buffers are mapped at the recorded GPU addresses and growing +// prefixes of its IB, cut after each TYPE7 packet, are submitted one by one, each waited for up to 2 s. Reported on GPU7 as +// "B / t o c e", markers +// 0x6b00 | thread variant, then 0x6a00 | prefix. +struct Replay { + static constexpr unsigned kCount = sizeof turnipreplay::kBuffers / sizeof turnipreplay::kBuffers[0]; + uint64_t handles[kCount] = {}; + unsigned ibIndex = kCount; + bool hung = false; +}; +// Allocates and maps every recorded buffer and fills it through a CPU mapping with the given flags: 0x101 (Device, aligned volatile +// stores, as the prefix replay always did) or 0x401 (write combine filled with memcpy/memset, as Turnip fills its BOs). +static int replayLoad(Gpu &gpu, Replay &r, uint32_t mapFlags) { + using namespace turnipreplay; + for (unsigned i = 0; i < Replay::kCount; ++i) { + const Buffer &b = kBuffers[i]; + NativeAdrenoAbiBoAlloc alloc = {b.bytes, 12, 0}; + NativeAdrenoAbiBoAllocReply reply = {}; + if (int e = gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &reply, sizeof reply)) return e; + r.handles[i] = reply.handle; + NativeAdrenoAbiBoMap map = {reply.handle, b.iova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoMap, &map, sizeof map, &mr, sizeof mr)) return e; + NativeAdrenoAbiBoCpuMap cm = {reply.handle}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + uint64_t address = 0, length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, mapFlags)) return e; + if (length < b.bytes) return 0xbad3; + if (mapFlags == 0x401) { + memcpy(reinterpret_cast(address), b.data, b.words * 4); + memset(reinterpret_cast(address) + b.words * 4, 0, b.bytes - b.words * 4); + } else { + volatile uint32_t *w = reinterpret_cast(address); + for (uint64_t k = 0; k < b.bytes / 4; ++k) w[k] = k < b.words ? b.data[k] : 0; + } + __asm__ volatile("dsb sy" ::: "memory"); + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + NativeAdrenoAbiBoCpuUnmap cu = {reply.handle, cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + if (b.iova <= kIbIova && kIbIova < b.iova + b.bytes) r.ibIndex = i; + } + return r.ibIndex == Replay::kCount ? 0xbad4 : 0; +} +// Submits the first dwords of the IB and waits up to 2 s; a timeout leaves the buffers referenced by the stuck CP. +static int replayRun(Gpu &gpu, Replay &r, uint32_t dwords) { + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {r.handles[r.ibIndex], turnipreplay::kIbIova, uint64_t(dwords) * 4}; + NativeAdrenoAbiSubmitReply sr = {}; + if (int e = gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr)) return e; + int waited = -1; + for (unsigned t = 0; t < 10 && waited; ++t) { + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + waited = gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0); + } + if (waited) r.hung = true; + return waited; +} +// A stuck CP still references the buffers; they are only released when everything retired. +static void replayFree(Gpu &gpu, Replay &r) { + if (r.hung) return; + for (uint64_t &h : r.handles) + if (h) {NativeAdrenoAbiBoFree f = {h}; gpu.method(NativeAdrenoMethodBoFree, &f, sizeof f, nullptr, 0); h = 0;} +} +// One whole-IB control run on fresh buffers at the same GPU addresses: '1' retired, '0' stuck, 'x' setup failed. Marker 0x6c00 | tag. +static char replayControl(Gpu &gpu, uint32_t mapFlags, uint32_t tag, bool &hung) { + Replay r; + gpu.mark(0x6c00 | tag); + char result = 'x'; + if (!replayLoad(gpu, r, mapFlags)) result = replayRun(gpu, r, turnipreplay::kCuts[sizeof turnipreplay::kCuts / sizeof turnipreplay::kCuts[0] - 1]) ? (r.hung ? '0' : 'x') : '1'; + hung = r.hung; + replayFree(gpu, r); + return result; +} +static char gReplayControls[4] = "---"; // W1 W2 N, appended to the B line +static char gReplayLine[64]; +// Whole-IB runs before any prefix (int-065). int-064's remap probe read the new pages in every variant (P1111), yet the whole IB on fresh +// buffers after the prefix series stopped the CP again (W1). These runs separate "a fresh first run stops" (prefix runs had left GPU-written +// memory behind) from "a second whole run stops": E, r1 the whole IB on fresh buffers before anything else, r2 the same +// buffers again, r3 fresh buffers at the same addresses (W1's case); '1' retired, '0' stuck, 'x' setup failed, '-' not run. After r1 +// retires, "d" lists how many words of each buffer differ from the recording and "f" the first changed word outside the fill target, as +// @=, so the memory the GPU wrote is known. +static char gEarly[48] = "E---"; +__attribute__((unused)) static bool earlyRuns(Gpu &gpu) { + using namespace turnipreplay; + const uint32_t whole = kCuts[sizeof kCuts / sizeof kCuts[0] - 1]; + char diff[40] = ""; + Replay r; + gpu.mark(0x6e01); + if (replayLoad(gpu, r, 0x101)) {gEarly[1] = 'x'; replayFree(gpu, r); return true;} + gEarly[1] = replayRun(gpu, r, whole) ? (r.hung ? '0' : 'x') : '1'; + if (gEarly[1] == '1') { + unsigned counts[Replay::kCount] = {}, firstBuffer = 0, firstWord = 0; + uint32_t firstValue = 0; + bool found = false; + for (unsigned i = 0; i < Replay::kCount; ++i) { + const Buffer &b = kBuffers[i]; + NativeAdrenoAbiBoCpuMap cm = {r.handles[i]}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) continue; + uint64_t address = 0, length = 0; + if (!gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, 0x101)) { + const volatile uint32_t *w = reinterpret_cast(address); + for (uint64_t k = 0; k < b.bytes / 4 && k < length / 4; ++k) { + const uint32_t want = k < b.words ? b.data[k] : 0; + if (w[k] == want) continue; + ++counts[i]; + // The fill target is the recording's empty buffer; its change is expected. + if (!found && b.words) {found = true; firstBuffer = i; firstWord = unsigned(k); firstValue = w[k];} + } + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + } + NativeAdrenoAbiBoCpuUnmap cu = {r.handles[i], cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + } + int at = snprintf(diff, sizeof diff, " d"); + for (unsigned i = 0; i < Replay::kCount && at < int(sizeof diff); ++i) + at += snprintf(diff + at, sizeof diff - at, "%s%x", i ? "," : "", counts[i]); + if (found && at < int(sizeof diff)) snprintf(diff + at, sizeof diff - at, " f%u@%x=%x", firstBuffer, firstWord, firstValue); + } + if (gEarly[1] == '1') {gpu.mark(0x6e02); gEarly[2] = replayRun(gpu, r, whole) ? (r.hung ? '0' : 'x') : '1';} + const bool stuck = r.hung; + replayFree(gpu, r); + if (!stuck && gEarly[2] == '1') { + Replay fresh; + gpu.mark(0x6e03); + gEarly[3] = replayLoad(gpu, fresh, 0x101) ? 'x' : replayRun(gpu, fresh, whole) ? (fresh.hung ? '0' : 'x') : '1'; + replayFree(gpu, fresh); + if (fresh.hung) gEarly[3] = '0'; + } + snprintf(gEarly + 4, sizeof gEarly - 4, "%s", diff); + streamText7(gpu, gEarly); + return gEarly[1] == '1' && gEarly[2] == '1' && gEarly[3] == '1'; +} +// Fresh-buffer prefix scan (int-066). int-065: the whole IB on fresh buffers, before any other replay, stopped the CP (E0--), while the +// prefix series on one set of buffers retires every prefix. Here every prefix runs on freshly loaded buffers, so memory written by +// earlier runs cannot help it; the first prefix that stops names the packet that needs that memory. When every prefix retires, CP state +// left by the earlier runs is what helped, and the whole IB then runs once more on fresh buffers (e). Reported on GPU7 as +// "F / o c e". +static bool freshScan(Gpu &gpu) { + using namespace turnipreplay; + constexpr unsigned kPrefixes = sizeof kCuts / sizeof kCuts[0]; + unsigned retired = 0; + bool stuck = false; + char whole = '-'; + for (unsigned p = 0; p < kPrefixes && !stuck; ++p) { + Replay r; + gpu.mark(0x6f80 | p); + int e = replayLoad(gpu, r, 0x101); + if (!e) e = replayRun(gpu, r, kCuts[p]); + stuck = r.hung; + replayFree(gpu, r); + if (e) break; + ++retired; + } + if (retired == kPrefixes) { + Replay r; + gpu.mark(0x6fa0); + whole = replayLoad(gpu, r, 0x101) ? 'x' : replayRun(gpu, r, kCuts[kPrefixes - 1]) ? (r.hung ? '0' : 'x') : '1'; + stuck = r.hung; + replayFree(gpu, r); + } + char text[64]; + snprintf(text, sizeof text, "F %u/%u o%x c%u e%c", retired, kPrefixes, retired < kPrefixes ? kCutOps[retired] : 0, + retired < kPrefixes ? kCuts[retired] : 0, whole); + streamText7(gpu, text); + return !stuck && whole == '1'; +} +static bool replayPhase(Gpu &gpu) { + using namespace turnipreplay; + constexpr unsigned kPrefixes = sizeof kCuts / sizeof kCuts[0]; + Replay r; + unsigned retired = 0, variants = 0; + int error = replayLoad(gpu, r, 0x101); + // The first packet is CP_THREAD_CONTROL(BOTH) and int-060 stopped on it alone. Before the prefixes, the same two dwords run with + // BR only (1) and BV only (2), so a hang names the thread: t. + auto patch = [&](uint32_t value) { + const Buffer &b = kBuffers[r.ibIndex]; + NativeAdrenoAbiBoCpuMap cm = {r.handles[r.ibIndex]}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + uint64_t address = 0, length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, 0x101)) return e; + reinterpret_cast(address)[(kIbIova - b.iova) / 4 + 1] = value; + __asm__ volatile("dsb sy" ::: "memory"); + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + NativeAdrenoAbiBoCpuUnmap cu = {r.handles[r.ibIndex], cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + return 0; + }; + const uint32_t original = r.ibIndex < Replay::kCount ? kBuffers[r.ibIndex].data[(kIbIova - kBuffers[r.ibIndex].iova) / 4 + 1] : 0; + for (uint32_t thread = 1; thread <= 2 && !error; ++thread) { + gpu.mark(0x6b00 | thread); + if (!(error = patch(thread)) && !(error = replayRun(gpu, r, 2))) ++variants; + } + if (!error) error = patch(original); + for (unsigned p = 0; p < kPrefixes && !error; ++p) { + gpu.mark(0x6a00 | p); + if ((error = replayRun(gpu, r, kCuts[p]))) break; + ++retired; + } + replayFree(gpu, r); + // Turnip's own fill of this IB stopped in int-062 while this replay retired. The control runs separate the remaining differences, each + // on fresh buffers at the same addresses: W1 Device-mapped stores (address reuse after a free), W2 write-combine memcpy (how Turnip + // fills its BOs); the caller adds N, a second connection (a second context entry on a CP that already ran work). + bool stuck = r.hung; + if (retired == kPrefixes && !stuck) gReplayControls[0] = replayControl(gpu, 0x101, 1, stuck); + if (retired == kPrefixes && !stuck) gReplayControls[1] = replayControl(gpu, 0x401, 2, stuck); + snprintf(gReplayLine, sizeof gReplayLine, "%.4s B %u/%u t%u o%x c%u e%x ", gEarly, retired, kPrefixes, variants, + retired < kPrefixes ? kCutOps[retired] : 0, retired < kPrefixes ? kCuts[retired] : 0, unsigned(error)); + char text[64]; + snprintf(text, sizeof text, "%s%s", gReplayLine, gReplayControls); + streamText7(gpu, text); + return retired == kPrefixes && !stuck; +} +// N: the whole IB on a second connection, whose first submission carries the A7XX context entry again (Turnip's connection does that +// after the replay). Returns false when the CP stopped. +static bool replaySecondClient(Gpu &gpu) { + unsigned conn = 0; + if (gpu.openFn(gpu.port, mach_task_self(), 0x4147, &conn) != 0 || !conn) {gReplayControls[2] = 'o'; return true;} + const unsigned first = gpu.conn; + volatile uint32_t *mailbox = gpu.mailbox; + gpu.conn = conn; + gpu.mailbox = nullptr; + bool ok = !mailbox || gpu.openMailbox(); + bool stuck = false; + gReplayControls[2] = ok ? replayControl(gpu, 0x101, 3, stuck) : 'm'; + gpu.closeFn(conn); + gpu.conn = first; + gpu.mailbox = mailbox; + char text[64]; + snprintf(text, sizeof text, "%s%s", gReplayLine, gReplayControls); + streamText7(gpu, text); + return !stuck; +} +#endif + +// Run after a passing Turnip phase by the plugin (the Metal bridge self-test); null in the probe daemon. +static void (*gAfterTurnip)(Gpu &gpu, const char *turnipPath); + +// Turnip phase: only where libvulkan_freedreno.dylib sits next to this code (the plugin bundle), never in the probe daemon. +static void turnipPhase(Gpu &gpu) { + char path[1024]; + if (!turnip::libraryPath(path, sizeof path) || access(path, R_OK) != 0) return; + gpu.mark(0x6000); + auto marker = [&](uint32_t value) { gpu.mark(value); }; + const turnip::Result result = turnip::runWatched(marker, [&](const char *text) { streamText(gpu, text); }, path); + char text[NativeAdrenoAbiMetadataBytes] = {}; + snprintf(text, sizeof text, "T s%d r%d v%d c%d%s g%d/%d%s %.30s", result.stage, result.vkResult, result.verified, result.computeWords, + result.computeStage ? "!" : "", result.renderTriangle, result.renderOther, result.renderStage ? "!" : "", turnip::gFirstError); + streamText(gpu, text); + gpu.mark(result.ok ? 0x6fff : 0x6ffe); + // Also when results are wrong but every Vulkan step ran: one boot then shows both layers. + if (result.stage == 0 && result.vkResult == 0 && gAfterTurnip) gAfterTurnip(gpu, path); +} + +// Returns the GPU-verified dword count of the uncached run, or a negative number; every outcome is also reported to the kernel. +static int adrenoUserTest(unsigned port, const char *process) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return -1; + auto open = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen"); + auto close = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose"); + Gpu gpu; + gpu.call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod"); + gpu.map = (MapFn)dlsym(iokit, "IOConnectMapMemory64"); + gpu.unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64"); + gpu.port = port; + gpu.openFn = open; + gpu.closeFn = close; + gpu.mark(1); + if (!open || !close || !gpu.call || !gpu.map || !gpu.unmap) return -1; + gpu.mark(2); + // 0x4147 ('AG') is the open type the driver serves; the original IOGPU library's type-1 opens are refused. + const int openResult = open(port, mach_task_self(), 0x4147, &gpu.conn); + if (openResult != 0 || !gpu.conn) { + // The exact kern_return_t, as two markers (high and low half), after the failure marker. + gpu.mark(0x4f01); + gpu.mark(((uint32_t)openResult >> 16) & 0xffff ? ((uint32_t)openResult >> 16) & 0xffff : 0xffff); + gpu.mark((uint32_t)openResult & 0xffff ? (uint32_t)openResult & 0xffff : 0xffff); + return -1; + } + gpu.mark(3); + const bool probed = gpu.probe(); + gpu.mark(0x1000 | (gpu.admitted & 0xfff)); + bool mailboxOk = false; + if (!probed) {mailboxOk = gpu.openMailbox(); gpu.mark(mailboxOk ? 0x2001 : 0x2000);} + if (!probed && !mailboxOk) {close(gpu.conn); return -2;} // neither channel works: the driver's E row shows what reached it + // mapFlags are what the CLIENT asks for: IOUserClient::mapClientMemory64 lets them override the cache mode the driver returns, so + // the cache bits must be set here. 0x101 = anywhere + inhibit cache (aligned dword accesses only), 0x401 = anywhere + write + // combine (Normal non-cacheable; what a Vulkan driver needs, since it uses memcpy and unaligned stores). + int step = 0, error = 0; + const int verified = runTest(gpu, 0x101, step, error); + int stepWc = 0, errorWc = 0; + const int verifiedWc = error ? -2 : runTest(gpu, 0x401, stepWc, errorWc); + report(gpu, process, step + stepWc, error ? error : errorWc, verified, verifiedWc, gpu.lastTicks); + gpu.mark(0x5000 | (verified & 0xff)); + bool replayed = true; +#ifdef NATIVE_TURNIP_REPLAY + // Plugin only (the Turnip library sits next to it): a stuck replay leaves the CP stopped, so Turnip is not started after it. + char turnipPath[1024]; + if (verified == 4 && verifiedWc == 4 && turnip::libraryPath(turnipPath, sizeof turnipPath) && access(turnipPath, R_OK) == 0) { + replayed = freshScan(gpu) && replayPhase(gpu); + } +#endif + close(gpu.conn); +#ifdef NATIVE_TURNIP_REPLAY + if (replayed && verified == 4 && verifiedWc == 4) replayed = replaySecondClient(gpu); +#endif + if (verified == 4 && verifiedWc == 4 && replayed) turnipPhase(gpu); + return verified; +} diff --git a/NativeBridgeMetalFrontend.h b/NativeBridgeMetalFrontend.h new file mode 100644 index 0000000..5c0c7ca --- /dev/null +++ b/NativeBridgeMetalFrontend.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// Entry points of the native Metal frontend over the bridge (NativeBridgeMetalFrontend.mm). +#pragma once +#include +#include +#ifdef __OBJC__ +#import +extern "C" { +// Opens the process's one device: loader exports vkGetInstanceProcAddr (Turnip), cache is a writable shader-cache directory, registryID is the +// accelerator service's registry entry. missing (may be null) is told the class and selector of every unimplemented call. +id native_bridge_metal_open(const char *loader, const char *cache, uint64_t registryID, void (*missing)(const char *, const char *)); +// Bare AIR bitcode as a library (the bridge's fixtures; Metal itself accepts only metallibs). +id native_bridge_metal_library_with_air(const void *bytes, size_t length); +// One line for the plugin's status report: command buffers committed/failed locally, pipelines made, refusals, the last refusal's text. +size_t native_bridge_metal_status(char *out, size_t size); +// The pixel formats of the textures asked for (first eight distinct) and how many texture requests the runtime refused. +size_t native_bridge_metal_formats(char *out, size_t size); +size_t native_bridge_metal_telemetry(char *out, size_t size); +// Structured cumulative artifact/driver-cache counters for offline qualification. +struct NmbShaderCacheStats; +bool native_bridge_metal_shader_cache_stats(struct NmbShaderCacheStats *out); +bool native_bridge_metal_work_stats(uint64_t out[10]); +// The same fixed CPU, memory and pipeline workload on the tablet and the host, so their times compare directly (int-145): compute, first-touch +// page faults, memset bandwidth, CommonCrypto SHA-256, then the first `pairs` vertex/fragment pairs of the given QuartzCore metallib made into +// BGRA8 render states and drawn once each (pipeline_timing_host's draw=1 order). The results become the telemetry's Y rows; also returned. +size_t native_bridge_metal_benchmark(id device, const char *metallib, unsigned pairs, char *out, size_t size); +} +#endif diff --git a/NativeBridgeMetalFrontend.mm b/NativeBridgeMetalFrontend.mm new file mode 100644 index 0000000..569750c --- /dev/null +++ b/NativeBridgeMetalFrontend.mm @@ -0,0 +1,1521 @@ +// SPDX-License-Identifier: MIT +// Native Metal API frontend over the bridge (bridge/NativeMetalObjects.h): the metal2vulkan runtime on Turnip, on the Adreno 740 through +// NativeAdreno. Same shape as the offline frontend: the device is a dynamically allocated subclass of the target's own _MTLDevice that +// only adds this file's implementations; every other object is an own class answering the Metal protocol methods it implements. +// Every result comes from the runtime and the GPU: a refused call returns nil/NO with the runtime's reason, an encoder call the runtime +// refuses fails its command buffer (status Error, handlers run) instead of being dropped. A selector this frontend does not implement is +// reported through the missing-selector hook before the usual exception, so a hardware run names the gap. +// Shared events work within this process (no cross-process handle yet). Not yet implemented here: heaps, argument buffers, indirect commands, drawables/presentation. +#import +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include "bridge/NativeMetalObjects.h" +#include "NativeBridgeMetalFrontend.h" + +extern "C" void sys_dcache_flush(void *start, size_t length); + +static NmbDevice *gDevice; +static id gDeviceObject; +static uint64_t gRegistryID; +static NSString *gName; +static void (*gMissing)(const char *, const char *); + +// What this device has been asked and refused, for the plugin's status line (native_bridge_metal_status): commits, render/compute pipelines made, +// refusals, and the last two refusals' texts. Counters only; nothing here changes a result. +static std::atomic gCommits,gCommitFailures,gPipelines,gProblems,gCompleted; +// What the render server actually asks of the GPU (int-141: 59 commits in 200 s, and a desktop that behaves like CPU compositing): render passes, +// draws, the pixels of their first colour attachment, compute dispatches, blits and the largest pass (pixels). +static std::atomic gPasses,gDraws,gPassPixels,gDispatches,gBlits,gBiggestPass,gFullScreenPasses; +// Where the time goes on the CPU side of the bridge: pipeline, library and function creation (count, total and slowest, in microseconds), and the +// last commits with what each held and how long the GPU took (commit to completion). +static uint64_t nowUs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW)/1000;} +struct Timed {std::atomic count{0},total{0},slowest{0}; + void add(uint64_t began){const uint64_t took=nowUs()-began;count++;total+=took;uint64_t seen=slowest.load();while(took>seen && !slowest.compare_exchange_weak(seen,took)){}}}; +static Timed gTimePipeline,gTimeLibrary,gTimeFunction,gTimeEncodeToCommit; +static Timed gTimeWaitScheduled,gTimeWaitCompleted; +static Timed gTimeResourceWait; +static std::atomic gResourceChecks,gResourceSkipped; +static Timed gTimeSurfaceFlush; +static std::atomic gSurfaceFlushBytes,gSurfaceFlushDuplicates; +struct EncodedWork {uint32_t passes,draws,dispatches,blits;uint64_t pixels;}; +struct CommitRecord {uint64_t committedUs=0,callbackUs=0,ageUs=0;int64_t scheduledUs=-1;uint32_t index=0;EncodedWork work={};}; +static CommitRecord gCommitRing[32]; +static std::mutex gCommitLock; +static std::mutex gProblemLock; +static char gLastProblems[2][200]; +static void noteProblem(NSString *what) { + gProblems++; + std::lock_guard lock(gProblemLock); + memcpy(gLastProblems[1],gLastProblems[0],sizeof gLastProblems[0]); + strlcpy(gLastProblems[0],what.UTF8String?:"?",sizeof gLastProblems[0]); +} +#define NBLOG(...) do{NSString *nbText=[NSString stringWithFormat:__VA_ARGS__];NSLog(@"%@",nbText);noteProblem(nbText);}while(0) +// The pixel formats textures were asked for (first eight distinct), with how many were refused: native_bridge_metal_formats. +static uint32_t gFormats[8];static unsigned gFormatCount;static std::atomic gTextureRefusals; +static void noteFormat(uint32_t format,bool made) { + std::lock_guard lock(gProblemLock); + if(!made)gTextureRefusals++; + for(unsigned i=0;i)device {return gDeviceObject;} +- (void)doesNotRecognizeSelector:(SEL)selector { + if(gMissing)gMissing(class_getName(object_getClass(self)),sel_getName(selector)); + noteProblem([NSString stringWithFormat:@"missing %s %s",class_getName(object_getClass(self)),sel_getName(selector)]); + [super doesNotRecognizeSelector:selector]; +} +@end + +// ---- resources ---- +// The wait takes a strong snapshot of already-committed buffers; new commits require the +// caller's usual resource synchronization. Storage dependency checks are in the runtime. +static NSMutableSet *gInFlight; +static void waitForResource(id resource); +// Resource SPI the runtime has no state for: no protected content (Metal's default options), and no owner transfer +// (memory attribution is the kernel driver's, by the importing task), refused as not supported. +#define NB_RESOURCE_SPI \ +- (uint64_t)protectionOptions {return 0;} \ +- (id)heap {return _parentHeap;} \ +- (BOOL)isAliasable {return NO;} \ +- (int)setOwnerWithIdentity:(unsigned)identity {(void)identity;return 46; /* KERN_NOT_SUPPORTED */} \ +- (void)waitUntilComplete {waitForResource(self);} +@interface NBBuffer : NBObject +@property(nonatomic) NmbBuffer *handle; +@property(nonatomic) MTLResourceOptions options; +@property(nonatomic,copy) void (^deallocator)(void *,NSUInteger); +@property(nonatomic) void *wrapped; +@property(nonatomic) IOSurfaceRef surfaceOwner; // keeps a shared import's CPU mapping alive +@property(nonatomic,strong) id parentHeap; // the heap that made the buffer, nil otherwise +@property(nonatomic) NSUInteger heapOffset; +@end +@interface NBTexture : NBObject +@property(nonatomic) NmbTexture *handle; +@property(nonatomic) NmbTextureDescriptor descriptor; +@property(nonatomic,strong) NBBuffer *buffer; +@property(nonatomic) IOSurfaceRef surface; +@property(nonatomic) NSUInteger offset,rowBytes; +@property(nonatomic,strong) id parent; +@property(nonatomic,strong) id parentHeap; +@property(nonatomic) NSUInteger heapOffset; +@end +@implementation NBBuffer +- (void)dealloc { + const NSUInteger length=nmb_buffer_length(_handle);nmb_buffer_release(_handle); + if(_deallocator)_deallocator(_wrapped,length); + if(_surfaceOwner)CFRelease(_surfaceOwner); +} +- (NSUInteger)length {return nmb_buffer_length(_handle);} +- (void *)contents {return nmb_buffer_contents(_handle);} +- (uint64_t)gpuAddress {return nmb_buffer_gpu_address(_handle);} +- (MTLResourceOptions)resourceOptions {return _options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (MTLHazardTrackingMode)hazardTrackingMode {return (MTLHazardTrackingMode)((_options&MTLResourceHazardTrackingModeMask)>>MTLResourceHazardTrackingModeShift);} +- (void)didModifyRange:(NSRange)range {(void)range;} +- (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { + const uint64_t answer=nmb_buffer_set_purgeable_state(_handle,state); + if(!answer)NBLOG(@"native-metal: buffer setPurgeableState refused: %@",lastError()); + return (MTLPurgeableState)answer; +} +- (NSUInteger)allocatedSize {return nmb_buffer_length(_handle);} +- (IOSurfaceRef)iosurface {return nullptr;} // a buffer is never the backing of a surface here (surfaces back textures) +NB_RESOURCE_SPI +@end +// Reuse only an import still held by a texture/view or pending command. Weak +// values never extend a GPU allocation's lifetime. The strong buffer snapshot +// and its retained surface prevent object/address reuse during a hit. A new +// mapping, allocation identity or size needs a separate import; ordinary +// caller-owned no-copy buffers do not enter this cache. +static NSMapTable *gSurfaceImports; +static std::mutex gSurfaceImportLock; +static NBBuffer *surfaceBuffer(IOSurfaceRef surface,void *base,size_t bytes) { + struct Key {uint64_t identity,address,bytes;}; + const Key value={IOSurfaceGetID(surface),(uintptr_t)base,bytes}; + NSValue *key=value.identity?[NSValue valueWithBytes:&value objCType:@encode(Key)]:nil; + std::lock_guard lock(gSurfaceImportLock); + if(!gSurfaceImports)gSurfaceImports=[NSMapTable strongToWeakObjectsMapTable]; + NBBuffer *buffer=key?[gSurfaceImports objectForKey:key]:nil; + if(buffer)return buffer; + NmbBuffer *handle=nmb_buffer_new_no_copy(gDevice,base,bytes,MTLResourceStorageModeShared); + if(!handle){NBLOG(@"native-metal: surface import refused: %@",lastError());return nil;} + buffer=[NBBuffer new];buffer.handle=handle;buffer.options=MTLResourceStorageModeShared; + buffer.surfaceOwner=(IOSurfaceRef)CFRetain(surface); + if(key) { + // Bound metadata without retaining old frames. Evicting an association is + // harmless: its owners still retain the actual import until their own end. + if(gSurfaceImports.count>=256) { + for(NSValue *old in gSurfaceImports.keyEnumerator.allObjects) + if(![gSurfaceImports objectForKey:old])[gSurfaceImports removeObjectForKey:old]; + if(gSurfaceImports.count>=256)[gSurfaceImports removeObjectForKey:gSurfaceImports.keyEnumerator.nextObject]; + } + [gSurfaceImports setObject:buffer forKey:key]; + } + return buffer; +} +static bool textureDescriptor(MTLTextureDescriptor *d,NmbTextureDescriptor *out) { + if(!d)return false; + *out={(uint64_t)d.textureType,(uint64_t)d.pixelFormat,d.width,d.height,d.depth,d.mipmapLevelCount,d.sampleCount,d.arrayLength,(uint64_t)d.usage,(uint64_t)d.resourceOptions}; + return true; +} +@implementation NBTexture +- (void)dealloc {nmb_texture_release(_handle);if(_surface)CFRelease(_surface);} +- (MTLTextureType)textureType {return (MTLTextureType)_descriptor.type;} +- (MTLPixelFormat)pixelFormat {return (MTLPixelFormat)_descriptor.format;} +- (NSUInteger)width {return _descriptor.width;} +- (NSUInteger)height {return _descriptor.height;} +- (NSUInteger)depth {return _descriptor.depth;} +- (NSUInteger)mipmapLevelCount {return _descriptor.levels;} +- (NSUInteger)sampleCount {return _descriptor.samples;} +- (NSUInteger)arrayLength {return _descriptor.array_length;} +- (MTLTextureUsage)usage {return (MTLTextureUsage)_descriptor.usage;} +- (MTLResourceOptions)resourceOptions {return (MTLResourceOptions)_descriptor.options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_descriptor.options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_descriptor.options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (BOOL)isFramebufferOnly {return NO;} +- (IOSurfaceRef)iosurface {return _surface;} +- (NSUInteger)iosurfacePlane {return 0;} +- (id)buffer {return _surface?nil:(id)_buffer;} +- (NSUInteger)bufferOffset {return _offset;} +- (NSUInteger)bufferBytesPerRow {return _rowBytes;} +- (id)parentTexture {return _parent;} +- (NSUInteger)rotation {return 0;} +- (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { + const uint64_t answer=nmb_texture_set_purgeable_state(_handle,state); + if(!answer)NBLOG(@"native-metal: texture setPurgeableState refused: %@",lastError()); + return (MTLPurgeableState)answer; +} +NB_RESOURCE_SPI +- (id)view:(NmbTexture *)handle { + if(!handle){NBLOG(@"native-metal: texture view refused: %@",lastError());return nil;} + NBTexture *view=[NBTexture new];view.handle=handle;NmbTextureDescriptor descriptor;nmb_texture_describe(handle,&descriptor);view.descriptor=descriptor; + view.parent=(id)self;view.buffer=_buffer;view.offset=_offset;view.rowBytes=_rowBytes; + if(_surface)view.surface=(IOSurfaceRef)CFRetain(_surface); + return (id)view; +} +- (id)newTextureViewWithPixelFormat:(MTLPixelFormat)format { + return [self view:nmb_texture_new_view(_handle,format,0,nullptr,nullptr,true)]; +} +- (id)newTextureViewWithPixelFormat:(MTLPixelFormat)format textureType:(MTLTextureType)type levels:(NSRange)levels slices:(NSRange)slices { + const uint64_t l[2]={levels.location,levels.length},s[2]={slices.location,slices.length}; + return [self view:nmb_texture_new_view(_handle,format,type,l,s,false)]; +} +- (void)replaceRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level slice:(NSUInteger)slice withBytes:(const void *)bytes bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image { + const uint64_t region[6]={r.origin.x,r.origin.y,r.origin.z,r.size.width,r.size.height,r.size.depth}; + const uint64_t length=(r.size.depth>1?image*(r.size.depth-1):0)+row*r.size.height; + if(!nmb_texture_replace(_handle,region,level,slice,bytes,row,image,length))NBLOG(@"native-metal: replaceRegion refused: %@",lastError()); +} +- (void)replaceRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level withBytes:(const void *)bytes bytesPerRow:(NSUInteger)row { + [self replaceRegion:r mipmapLevel:level slice:0 withBytes:bytes bytesPerRow:row bytesPerImage:row*r.size.height]; +} +- (void)getBytes:(void *)bytes bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image fromRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level slice:(NSUInteger)slice { + const uint64_t region[6]={r.origin.x,r.origin.y,r.origin.z,r.size.width,r.size.height,r.size.depth}; + const uint64_t length=(r.size.depth>1?image*(r.size.depth-1):0)+row*r.size.height; + if(!nmb_texture_get_bytes(_handle,bytes,row,image,length,region,level,slice))NBLOG(@"native-metal: getBytes refused: %@",lastError()); +} +- (void)getBytes:(void *)bytes bytesPerRow:(NSUInteger)row fromRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level { + [self getBytes:bytes bytesPerRow:row bytesPerImage:row*r.size.height fromRegion:r mipmapLevel:level slice:0]; +} +@end +// A command can use many independent imports/views of one IOSurface. Their +// owners keep the mapping live through this phase. Maintain each exact CPU +// range once; no address identity or decision is carried into another phase +// or command, and distinct/partially overlapping ranges retain their calls. +static void flushSurfaces(NSMutableSet *surfaces) { + if(!surfaces.count)return; + std::vector> ranges; + ranges.reserve(surfaces.count); + for(NBTexture *texture in surfaces)if(texture.surface) + ranges.emplace_back(reinterpret_cast(IOSurfaceGetBaseAddress(texture.surface)),IOSurfaceGetAllocSize(texture.surface)); + std::sort(ranges.begin(),ranges.end()); + const auto last=std::unique(ranges.begin(),ranges.end()); + gSurfaceFlushDuplicates+=(uint64_t)(ranges.end()-last); + for(auto it=ranges.begin();it!=last;++it) { + const uint64_t began=nowUs(); + sys_dcache_flush(reinterpret_cast(it->first),it->second); + gSurfaceFlushBytes+=it->second;gTimeSurfaceFlush.add(began); + } +} +static NBTexture *textureOnBuffer(NBBuffer *buffer,MTLTextureDescriptor *d,NSUInteger offset,NSUInteger row,bool surface) { + NmbTextureDescriptor descriptor;if(!buffer || !textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new_on_buffer(buffer.handle,&descriptor,offset,row,surface); + if(!handle){NBLOG(@"native-metal: texture over buffer refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.buffer=buffer;texture.offset=offset;texture.rowBytes=row;return texture; +} +@implementation NBBuffer (Textures) +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset bytesPerRow:(NSUInteger)row { + return (id)textureOnBuffer(self,d,offset,row,false); +} +- (id)newLinearTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new_linear(self.handle,&descriptor,offset,row,image); + if(!handle){NBLOG(@"native-metal: linear texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.buffer=self;texture.offset=offset;texture.rowBytes=row;return (id)texture; +} +@end + +// ---- libraries, functions, states ---- +@interface NBLibrary : NBObject +@property(nonatomic) NmbLibrary *handle; +@end +@interface NBFunction : NBObject +@property(nonatomic) NmbFunction *handle; +@property(nonatomic,copy) NSString *name; +@property(nonatomic) MTLFunctionType functionType; +@end +@implementation NBFunction +- (void)dealloc {nmb_function_release(_handle);} +- (MTLFunctionOptions)options {return MTLFunctionOptionNone;} +@end +// The target's own accessors of MTLFunctionConstantValues (as the offline frontend reads them); their ABI is checked before any call. +@interface MTLFunctionConstantValues (NBOriginalConstantAccess) +- (NSArray *)newIndexedConstantArray; +- (NSArray *)newNamedConstantArray; +@end +@protocol NBOriginalConstantValue +- (NSUInteger)dataType; +- (void *)data; +- (NSUInteger)index; +- (NSString *)name; +@end +static bool methodABI(id object,SEL selector,const char *encoding) { + Method method=class_getInstanceMethod(object_getClass(object),selector); + return method && !strcmp(method_getTypeEncoding(method),encoding); +} +// Bytes of a function constant of MTLDataType `type` (scalars and 2/4-wide vectors; 3-wide vectors are not qualified). +static uint64_t constantBytes(uint64_t type) { + switch(type) { + case 3:case 29:case 33:return 4; + case 4:case 30:case 34:return 8; + case 6:case 32:case 36:return 16; + case 16:case 37:case 41:return 2; + case 17:case 38:case 42:return 4; + case 19:case 40:case 44:return 8; + case 45:case 49:case 53:return 1; + case 46:case 50:case 54:return 2; + case 48:case 52:case 56:return 4; + case 81:case 85:return 8; + case 82:case 86:return 16; + case 84:case 88:return 32; + default:return 0; + } +} +@implementation NBLibrary +- (void)dealloc {nmb_library_release(_handle);} +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error { + if(!name || !values || !methodABI(values,@selector(newIndexedConstantArray),"@16@0:8") || !methodABI(values,@selector(newNamedConstantArray),"@16@0:8")) { + if(error)*error=nbError(@"unqualified function constant values");return nil; + } + MTLFunctionConstantValues *snapshot=[values copy]; + NSArray *indexed=[snapshot newIndexedConstantArray],*named=[snapshot newNamedConstantArray]; + std::vector constants; + for(unsigned group=0;group<2;group++)for(id value in (group?named:indexed)) { + if(!methodABI(value,@selector(dataType),"Q16@0:8") || !methodABI(value,@selector(data),"^v16@0:8") || + !methodABI(value,group?@selector(name):@selector(index),group?"@16@0:8":"Q16@0:8")) { + if(error)*error=nbError(@"unqualified constant value ABI");return nil; + } + id v=value; + const uint64_t bytes=constantBytes(v.dataType); + if(!bytes || !v.data){if(error)*error=nbError([NSString stringWithFormat:@"constant type %lu is not qualified",(unsigned long)v.dataType]);return nil;} + constants.push_back({v.dataType,group?0:v.index,group?v.name.UTF8String:nullptr,v.data,bytes}); + } + const uint64_t functionBegan=nowUs(); + NmbFunction *handle=nmb_function_new_specialized(_handle,name.UTF8String,constants.data(),(uint32_t)constants.size());gTimeFunction.add(functionBegan); + if(!handle){if(error)*error=nbError([NSString stringWithFormat:@"specialized function: %@",lastError()]);return nil;} + NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + if(error)*error=nil;return (id)function; +} +- (void)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values completionHandler:(void (^)(id,NSError *))handler { + NSError *error=nil;id function=[self newFunctionWithName:name constantValues:values error:&error]; + handler(function,error); +} +- (MTLLibraryType)type {return MTLLibraryTypeExecutable;} +// The descriptor's name and constants; its specialized name only labels Metal's function and its options ask for compile-time work the +// runtime does when the function is made. +- (id)newFunctionWithDescriptor:(MTLFunctionDescriptor *)d error:(NSError **)error { + if(!d.name){if(error)*error=nbError(@"a function name is required");return nil;} + if(d.constantValues)return [self newFunctionWithName:d.name constantValues:d.constantValues error:error]; + id function=[self newFunctionWithName:d.name]; + if(!function){if(error)*error=nbError([NSString stringWithFormat:@"function %@: %@",d.name,lastError()]);return nil;} + if(error)*error=nil;return function; +} +- (NSArray *)functionNames { + NSMutableArray *names=[NSMutableArray array]; + for(uint32_t i=0,n=nmb_library_function_count(_handle);i)newFunctionWithName:(NSString *)name { + const uint64_t functionBegan=nowUs(); + NmbFunction *handle=name?nmb_function_new(_handle,name.UTF8String):nullptr;gTimeFunction.add(functionBegan);if(!handle)return nil; + NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + return (id)function; +} +@end +@interface NBComputePipeline : NBObject +@property(nonatomic) NmbComputePipeline *handle; +@end +@implementation NBComputePipeline +- (void)dealloc {nmb_compute_pipeline_release(_handle);} +- (NSUInteger)maxTotalThreadsPerThreadgroup {return nmb_compute_pipeline_max_threads(_handle);} +- (NSUInteger)threadExecutionWidth {return nmb_compute_pipeline_execution_width(_handle);} +@end +@interface NBRenderPipeline : NBObject +@property(nonatomic) NmbRenderPipeline *handle; +@end +@implementation NBRenderPipeline +- (void)dealloc {nmb_render_pipeline_release(_handle);} +// The threadgroup size a tile dispatch of this pipeline could use: the device's limit (no tile shaders are offered). +- (NSUInteger)maxTotalThreadsPerThreadgroup { + uint64_t s[3]={};if(!nmb_device_max_threads_per_threadgroup(gDevice,s))return 0; + return s[0]; +} +@end +@interface NBSampler : NBObject +@property(nonatomic) NmbSampler *handle; +@end +@implementation NBSampler +- (void)dealloc {nmb_sampler_release(_handle);} +- (NSUInteger)pixelFormat {return MTLPixelFormatInvalid;} // SPI: a sampler built from a plain descriptor names no format +@end +@interface NBDepthStencil : NBObject +@property(nonatomic) NmbDepthStencil *handle; +@end +@implementation NBDepthStencil +- (void)dealloc {nmb_depth_stencil_release(_handle);} +@end + +// ---- events and fences ---- +@interface NBEvent : NBObject +@property(nonatomic) NmbEvent *handle; +@end +@implementation NBEvent +- (void)dealloc {nmb_event_release(_handle);} +@end +@interface NBSharedEvent : NBEvent +@end +static void notified(void *context,uint64_t value); +@implementation NBSharedEvent +- (uint64_t)signaledValue {return nmb_shared_event_value(self.handle);} +- (void)setSignaledValue:(uint64_t)value {nmb_shared_event_set_value(self.handle,value);} +- (BOOL)waitUntilSignaledValue:(uint64_t)value timeoutMS:(uint64_t)milliseconds {return nmb_shared_event_wait(self.handle,value,milliseconds);} +// The block runs on the listener's dispatch queue (Metal's contract); the bridge's listener thread only forwards it there. +- (void)notifyListener:(MTLSharedEventListener *)listener atValue:(uint64_t)value block:(MTLSharedEventNotificationBlock)block { + if(!block)return; + dispatch_queue_t queue=listener.dispatchQueue?:dispatch_get_global_queue(QOS_CLASS_DEFAULT,0); + NSArray *job=@[self,queue,[block copy]]; + if(!nmb_shared_event_notify(self.handle,value,notified,(__bridge_retained void *)job)) { + (void)(__bridge_transfer NSArray *)(__bridge void *)job; + NBLOG(@"native-metal: notifyListener refused: %@",lastError()); + } +} +// A handle for another process needs a kernel-backed event this bridge does not have yet: refused, never a handle that could not work. +- (MTLSharedEventHandle *)newSharedEventHandle { + if(gMissing)gMissing("NBSharedEvent","newSharedEventHandle (cross-process)"); + return nil; +} +@end +static void notified(void *context,uint64_t value) { + NSArray *job=(__bridge_transfer NSArray *)context; + id event=job[0];MTLSharedEventNotificationBlock block=job[2]; + dispatch_async(job[1],^{block(event,value);}); +} +@interface NBFence : NBObject +@property(nonatomic) NmbFence *handle; +@end +@implementation NBFence +- (void)dealloc {nmb_fence_release(_handle);} +@end + +// ---- command buffers and encoders ---- +@interface NBQueue : NBObject +@property(nonatomic) NmbQueue *handle; +@property(atomic,strong) dispatch_queue_t completionQueue; // SPI: where command buffers' handlers run (nil: the runtime's thread) +@end +@interface NBCommandBuffer : NBObject { +@public + EncodedWork encodedWork; // the command buffer's single encoder thread, not global counter deltas + uint64_t createdUs; +} +@property(nonatomic) NmbCommandBuffer *handle; +@property(nonatomic,strong) NBQueue *queue; +@property(atomic,copy) NSString *encodeFailure; +@property(atomic) BOOL committed,failedLocally; +@property(nonatomic) uint64_t committedUs; +@property(nonatomic) uint32_t commitIndex; +@property(nonatomic,strong) NSMutableArray *scheduledHandlers,*completedHandlers; +@property(nonatomic,strong) NSMutableSet *surfaces; +// Runtime handles retain GPU storage, but only the Objective-C owner retains +// a no-copy deallocator, its memory owner and texture/heap ancestry. +@property(nonatomic,strong) NSMutableSet *resourceOwners; +@property(nonatomic,strong) dispatch_group_t handlers; +@property(nonatomic,strong) dispatch_group_t schedulingHandlers; +@property(nonatomic) BOOL schedulingDelivered; +- (void)refuse:(NSString *)what; +@end +@interface NBEncoder : NBObject +@property(nonatomic) NmbEncoder *handle; +@property(nonatomic,strong) NBCommandBuffer *commandBuffer; +@end +static void waitForResource(id resource) { + const uint64_t began=nowUs(); + NSArray *pending;@synchronized(gInFlight){pending=gInFlight.allObjects;} + for(NBCommandBuffer *commandBuffer in pending) { + int dependency=-1; + if([resource isKindOfClass:[NBBuffer class]])dependency=nmb_command_buffer_uses_buffer(commandBuffer.handle,[(NBBuffer *)resource handle]); + else if([resource isKindOfClass:[NBTexture class]])dependency=nmb_command_buffer_uses_texture(commandBuffer.handle,[(NBTexture *)resource handle]); + gResourceChecks++; + // An unknown result retains the old conservative wait. False means no matching storage + // (or GPU work already completed), not a fabricated completion for a pending dependency. + if(dependency!=0)[(id)commandBuffer waitUntilCompleted];else gResourceSkipped++; + } + gTimeResourceWait.add(began); +} +@implementation NBEncoder +- (void)dealloc {if(_handle)nmb_encoder_discard(_handle);} +- (bool)check:(bool)ok what:(const char *)what { + if(!ok)[_commandBuffer refuse:[NSString stringWithFormat:@"%s: %@",what,lastError()]]; + return ok; +} +- (void)endEncoding { + NmbEncoder *handle=_handle;_handle=nullptr; + if(!handle){[_commandBuffer refuse:@"endEncoding twice"];return;} + [self check:nmb_encoder_end(handle) what:"endEncoding"]; +} +- (void)insertDebugSignpost:(NSString *)s {(void)s;} +- (void)pushDebugGroup:(NSString *)s {(void)s;} +- (void)popDebugGroup {} +// Stage 0 compute, 1 vertex, 2 fragment (the bridge's numbering). +- (void)buffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { + if(buffer && ![(id)buffer isKindOfClass:[NBBuffer class]]){[_commandBuffer refuse:@"foreign buffer"];return;} + if(buffer)[_commandBuffer.resourceOwners addObject:buffer]; + [self check:nmb_encoder_set_buffer(_handle,stage,buffer?[(NBBuffer *)buffer handle]:nullptr,offset,(uint32_t)index) what:"setBuffer"]; +} +- (void)texture:(id)texture index:(NSUInteger)index stage:(uint32_t)stage { + if(texture && ![(id)texture isKindOfClass:[NBTexture class]]){[_commandBuffer refuse:@"foreign texture"];return;} + if(texture)[_commandBuffer.resourceOwners addObject:texture]; + if(texture && [(NBTexture *)texture surface])[_commandBuffer.surfaces addObject:texture]; + [self check:nmb_encoder_set_texture(_handle,stage,texture?[(NBTexture *)texture handle]:nullptr,(uint32_t)index) what:"setTexture"]; +} +- (void)sampler:(id)sampler index:(NSUInteger)index stage:(uint32_t)stage { + if(sampler && ![(id)sampler isKindOfClass:[NBSampler class]]){[_commandBuffer refuse:@"foreign sampler"];return;} + [self check:nmb_encoder_set_sampler(_handle,stage,sampler?[(NBSampler *)sampler handle]:nullptr,(uint32_t)index) what:"setSamplerState"]; +} +- (void)bytes:(const void *)bytes length:(NSUInteger)length index:(NSUInteger)index stage:(uint32_t)stage { + [self check:nmb_encoder_set_bytes(_handle,stage,bytes,length,(uint32_t)index) what:"setBytes"]; +} +- (void)fence:(id)fence stages:(uint64_t)stages update:(bool)update { + if(![(id)fence isKindOfClass:[NBFence class]]){[_commandBuffer refuse:@"foreign fence"];return;} + NmbFence *handle=[(NBFence *)fence handle]; + [self check:update?nmb_encoder_update_fence(_handle,handle,stages):nmb_encoder_wait_fence(_handle,handle,stages) what:update?"updateFence":"waitForFence"]; +} +- (void)updateFence:(id)fence {[self fence:fence stages:0 update:true];} +- (void)waitForFence:(id)fence {[self fence:fence stages:0 update:false];} +- (void)offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { + [self check:nmb_encoder_set_buffer_offset(_handle,stage,offset,(uint32_t)index) what:"setBufferOffset"]; +} +@end + +@interface NBComputeEncoder : NBEncoder +@end +@implementation NBComputeEncoder +- (MTLDispatchType)dispatchType {return MTLDispatchTypeSerial;} +- (void)setComputePipelineState:(id)state { + if(![(id)state isKindOfClass:[NBComputePipeline class]]){[self.commandBuffer refuse:@"foreign compute pipeline"];return;} + [self check:nmb_compute_set_pipeline(self.handle,[(NBComputePipeline *)state handle]) what:"setComputePipelineState"]; +} +- (void)setBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:0];} +- (void)setBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:0];} +- (void)setBuffers:(const id __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k)t atIndex:(NSUInteger)i {[self texture:t index:i stage:0];} +- (void)setTextures:(const id __unsafe_unretained [])t withRange:(NSRange)r { + for(NSUInteger k=0;k)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:0];} +- (void)setThreadgroupMemoryLength:(NSUInteger)n atIndex:(NSUInteger)i { + [self check:nmb_compute_set_threadgroup_memory(self.handle,(uint32_t)n,(uint32_t)i) what:"setThreadgroupMemoryLength"]; +} +- (void)dispatch:(MTLSize)size group:(MTLSize)group threads:(bool)threads { + gDispatches++;self.commandBuffer->encodedWork.dispatches++; + const uint64_t s[3]={size.width,size.height,size.depth},g[3]={group.width,group.height,group.depth}; + [self check:nmb_compute_dispatch(self.handle,s,g,threads) what:"dispatch"]; +} +- (void)dispatchThreads:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:true];} +- (void)dispatchThreadgroups:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:false];} +- (void)useResource:(id)r usage:(MTLResourceUsage)u {(void)r;(void)u;} +- (void)memoryBarrierWithScope:(MTLBarrierScope)s {(void)s;} +@end + +@interface NBRenderEncoder : NBEncoder +@end +@implementation NBRenderEncoder +- (void)setRenderPipelineState:(id)state { + if(![(id)state isKindOfClass:[NBRenderPipeline class]]){[self.commandBuffer refuse:@"foreign render pipeline"];return;} + [self check:nmb_render_set_pipeline(self.handle,[(NBRenderPipeline *)state handle]) what:"setRenderPipelineState"]; +} +- (void)setVertexBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:1];} +- (void)setFragmentBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:2];} +- (void)setVertexBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:1];} +- (void)setFragmentBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:2];} +- (void)setVertexBuffers:(const id __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k)t atIndex:(NSUInteger)i {[self texture:t index:i stage:1];} +- (void)setFragmentTexture:(id)t atIndex:(NSUInteger)i {[self texture:t index:i stage:2];} +- (void)setFragmentTextures:(const id __unsafe_unretained [])t withRange:(NSRange)r { + for(NSUInteger k=0;k)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:1];} +- (void)setFragmentSamplerState:(id)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:2];} +- (void)setDepthStencilState:(id)state { + if(state && ![(id)state isKindOfClass:[NBDepthStencil class]]){[self.commandBuffer refuse:@"foreign depth/stencil state"];return;} + [self check:nmb_render_set_depth_stencil(self.handle,state?[(NBDepthStencil *)state handle]:nullptr) what:"setDepthStencilState"]; +} +- (void)setViewport:(MTLViewport)v { + const double values[6]={v.originX,v.originY,v.width,v.height,v.znear,v.zfar}; + [self check:nmb_render_set_viewport(self.handle,values) what:"setViewport"]; +} +- (void)setScissorRect:(MTLScissorRect)r { + const uint64_t values[4]={r.x,r.y,r.width,r.height}; + [self check:nmb_render_set_scissor(self.handle,values) what:"setScissorRect"]; +} +// The runtime takes cull mode, winding and fill mode together; this encoder keeps Metal's defaults until each is set. +- (void)raster { + NSNumber *cull=objc_getAssociatedObject(self,"cull"),*winding=objc_getAssociatedObject(self,"winding"),*fill=objc_getAssociatedObject(self,"fill"); + [self check:nmb_render_set_raster(self.handle,cull.unsignedLongLongValue,winding.unsignedLongLongValue,fill.unsignedLongLongValue) what:"raster state"]; +} +- (void)setCullMode:(MTLCullMode)m {objc_setAssociatedObject(self,"cull",@(m),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setFrontFacingWinding:(MTLWinding)w {objc_setAssociatedObject(self,"winding",@(w),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setTriangleFillMode:(MTLTriangleFillMode)f {objc_setAssociatedObject(self,"fill",@(f),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setDepthBias:(float)bias slopeScale:(float)slope clamp:(float)clamp { + [self check:nmb_render_set_depth_bias(self.handle,bias,slope,clamp) what:"setDepthBias"]; +} +- (void)setLineWidth:(float)width {[self check:nmb_render_set_line_width(self.handle,width) what:"setLineWidth"];} +- (void)setVisibilityResultMode:(MTLVisibilityResultMode)mode offset:(NSUInteger)offset { + [self check:nmb_render_set_visibility_mode(self.handle,mode,offset) what:"setVisibilityResultMode"]; +} +- (void)setFragmentTexture:(id)t atTextureIndex:(NSUInteger)ti samplerState:(id)s atSamplerIndex:(NSUInteger)si { + [self texture:t index:ti stage:2];[self sampler:s index:si stage:2]; +} +// Tile shaders and imageblock tile memory (Apple GPU families 4 and up) are not offered: supportsTileShaders answers no, and a call that +// reaches one anyway fails its command buffer. +- (NSUInteger)tileWidth {return 0;} +- (NSUInteger)tileHeight {return 0;} +- (void)setTileTexture:(id)t atIndex:(NSUInteger)i {(void)t;(void)i;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)setTileBytes:(const void *)p length:(NSUInteger)n atIndex:(NSUInteger)i {(void)p;(void)n;(void)i;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)dispatchThreadsPerTile:(MTLSize)size {(void)size;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)dispatchThreadsPerTile:(MTLSize)size inRegion:(MTLRegion)region {(void)size;(void)region;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)setStencilReferenceValue:(uint32_t)v {[self check:nmb_render_set_stencil_reference(self.handle,v,v) what:"setStencilReferenceValue"];} +- (void)setStencilFrontReferenceValue:(uint32_t)f backReferenceValue:(uint32_t)b { + [self check:nmb_render_set_stencil_reference(self.handle,f,b) what:"setStencilReferenceValues"]; +} +- (void)setBlendColorRed:(float)r green:(float)g blue:(float)b alpha:(float)a { + const float c[4]={r,g,b,a};[self check:nmb_render_set_blend_color(self.handle,c) what:"setBlendColor"]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n instanceCount:(NSUInteger)i baseInstance:(NSUInteger)b { + gDraws++;self.commandBuffer->encodedWork.draws++; + [self check:nmb_render_draw(self.handle,t,(uint32_t)s,(uint32_t)n,(uint32_t)i,(uint32_t)b) what:"drawPrimitives"]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n instanceCount:(NSUInteger)i { + [self drawPrimitives:t vertexStart:s vertexCount:n instanceCount:i baseInstance:0]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n { + [self drawPrimitives:t vertexStart:s vertexCount:n instanceCount:1 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o + instanceCount:(NSUInteger)i baseVertex:(NSInteger)v baseInstance:(NSUInteger)bi { + if(![(id)b isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:@"foreign index buffer"];return;} + [self.commandBuffer.resourceOwners addObject:b]; + gDraws++;self.commandBuffer->encodedWork.draws++; + [self check:nmb_render_draw_indexed(self.handle,t,(uint32_t)n,k,[(NBBuffer *)b handle],o,(uint32_t)i,(int32_t)v,(uint32_t)bi) what:"drawIndexedPrimitives"]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o + instanceCount:(NSUInteger)i { + [self drawIndexedPrimitives:t indexCount:n indexType:k indexBuffer:b indexBufferOffset:o instanceCount:i baseVertex:0 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o { + [self drawIndexedPrimitives:t indexCount:n indexType:k indexBuffer:b indexBufferOffset:o instanceCount:1 baseVertex:0 baseInstance:0]; +} +- (void)useResource:(id)r usage:(MTLResourceUsage)u stages:(MTLRenderStages)s {(void)r;(void)u;(void)s;} +- (void)updateFence:(id)fence afterStages:(MTLRenderStages)stages {[self fence:fence stages:stages update:true];} +- (void)waitForFence:(id)fence beforeStages:(MTLRenderStages)stages {[self fence:fence stages:stages update:false];} +@end + +@interface NBBlitEncoder : NBEncoder +@end +@implementation NBBlitEncoder +// The arguments are passed as a plain array: an NSArray literal raises on a nil element, and a nil resource (a refused allocation) must fail the +// command buffer like any invalid argument, not abort the host process. +#define NB_OBJECTS(...) ((__unsafe_unretained id[]){__VA_ARGS__}) +- (bool)buffers:(__unsafe_unretained id *)objects count:(size_t)count { + for(size_t i=0;i)s sourceOffset:(NSUInteger)so toBuffer:(id)d destinationOffset:(NSUInteger)dO size:(NSUInteger)n { + if(![self buffers:NB_OBJECTS(s,d) count:2])return; + [self check:nmb_blit_copy_buffer(self.handle,[(NBBuffer *)s handle],so,[(NBBuffer *)d handle],dO,n) what:"copyFromBuffer:toBuffer:"]; +} +- (void)fillBuffer:(id)b range:(NSRange)r value:(uint8_t)v { + if(![self buffers:NB_OBJECTS(b) count:1])return; + [self check:nmb_blit_fill_buffer(self.handle,[(NBBuffer *)b handle],r.location,r.location+r.length,v) what:"fillBuffer"]; +} +- (void)copyFromTexture:(id)s sourceSlice:(NSUInteger)ss sourceLevel:(NSUInteger)sl sourceOrigin:(MTLOrigin)so sourceSize:(MTLSize)z + toTexture:(id)d destinationSlice:(NSUInteger)ds destinationLevel:(NSUInteger)dl destinationOrigin:(MTLOrigin)dO { + if(![self textures:NB_OBJECTS(s,d) count:2])return; + const uint64_t a[3]={so.x,so.y,so.z},size[3]={z.width,z.height,z.depth},b[3]={dO.x,dO.y,dO.z}; + [self check:nmb_blit_copy_texture(self.handle,[(NBTexture *)s handle],ss,sl,a,size,[(NBTexture *)d handle],ds,dl,b) what:"copyFromTexture:toTexture:"]; +} +- (void)copyFromTexture:(id)s toTexture:(id)d { + if(![self textures:NB_OBJECTS(s,d) count:2])return; + const NSUInteger levels=MIN(s.mipmapLevelCount,d.mipmapLevelCount),slices=MIN(s.arrayLength,d.arrayLength); + for(NSUInteger slice=0;slice>level,(NSUInteger)1),MAX(s.height>>level,(NSUInteger)1),MAX(s.depth>>level,(NSUInteger)1)); + [self copyFromTexture:s sourceSlice:slice sourceLevel:level sourceOrigin:MTLOriginMake(0,0,0) sourceSize:size toTexture:d destinationSlice:slice + destinationLevel:level destinationOrigin:MTLOriginMake(0,0,0)]; + } +} +- (void)copyFromBuffer:(id)s sourceOffset:(NSUInteger)o sourceBytesPerRow:(NSUInteger)row sourceBytesPerImage:(NSUInteger)image + sourceSize:(MTLSize)z toTexture:(id)d destinationSlice:(NSUInteger)slice destinationLevel:(NSUInteger)level + destinationOrigin:(MTLOrigin)dO { + if(![self buffers:NB_OBJECTS(s) count:1] || ![self textures:NB_OBJECTS(d) count:1])return; + const uint64_t size[3]={z.width,z.height,z.depth},b[3]={dO.x,dO.y,dO.z}; + [self check:nmb_blit_buffer_to_texture(self.handle,[(NBBuffer *)s handle],o,row,image,size,[(NBTexture *)d handle],slice,level,b) what:"copyFromBuffer:toTexture:"]; +} +- (void)copyFromTexture:(id)s sourceSlice:(NSUInteger)slice sourceLevel:(NSUInteger)level sourceOrigin:(MTLOrigin)so sourceSize:(MTLSize)z + toBuffer:(id)d destinationOffset:(NSUInteger)o destinationBytesPerRow:(NSUInteger)row destinationBytesPerImage:(NSUInteger)image { + if(![self textures:NB_OBJECTS(s) count:1] || ![self buffers:NB_OBJECTS(d) count:1])return; + const uint64_t a[3]={so.x,so.y,so.z},size[3]={z.width,z.height,z.depth}; + [self check:nmb_blit_texture_to_buffer(self.handle,[(NBTexture *)s handle],slice,level,a,size,[(NBBuffer *)d handle],o,row,image) what:"copyFromTexture:toBuffer:"]; +} +- (void)generateMipmapsForTexture:(id)t { + if(![self textures:NB_OBJECTS(t) count:1])return; + [self check:nmb_blit_generate_mipmaps(self.handle,[(NBTexture *)t handle]) what:"generateMipmapsForTexture"]; +} +- (void)synchronizeResource:(id)r {(void)r;} +@end + +static void completed(void *context); +static void scheduled(void *context); +@implementation NBCommandBuffer +- (instancetype)init { + if((self=[super init])){_scheduledHandlers=[NSMutableArray array];_completedHandlers=[NSMutableArray array];_surfaces=[NSMutableSet set];_resourceOwners=[NSMutableSet set];_handlers=dispatch_group_create();_schedulingHandlers=dispatch_group_create();} + if(self)createdUs=nowUs(); + return self; +} +- (void)dealloc {nmb_command_buffer_release(_handle);} +- (id)commandQueue {return (id)_queue;} +- (BOOL)retainedReferences {return YES;} +// Protected content is not supported (the runtime's AppleOnlyCall::SetProtectionOptions takes Metal's default only): any other option fails +// the command buffer. +- (uint64_t)protectionOptions {return 0;} +- (void)setProtectionOptions:(uint64_t)options {if(options)[self refuse:[NSString stringWithFormat:@"protection options 0x%llx",options]];} +- (void)refuse:(NSString *)what { + if(!self.encodeFailure){self.encodeFailure=what;NBLOG(@"native-metal: command buffer refused: %@",what);} +} +- (MTLCommandBufferStatus)status { + if(self.failedLocally)return MTLCommandBufferStatusError; + return (MTLCommandBufferStatus)nmb_command_buffer_status(_handle); +} +- (NSError *)error { + if(self.failedLocally)return nbError(self.encodeFailure); + if(self.status!=MTLCommandBufferStatusError)return nil; + char text[512]={};nmb_command_buffer_error(_handle,text,sizeof text); + return nbError([NSString stringWithUTF8String:text]); +} +// Debug groups and cache hints name work for tools and the driver's caches; they change nothing the GPU computes here. +- (void)pushDebugGroup:(NSString *)name {(void)name;} +- (void)popDebugGroup {} +- (void)encodeCacheHintTag:(uint64_t)tag resourceGroups:(const id __unsafe_unretained [])groups count:(NSUInteger)count {(void)tag;(void)groups;(void)count;} +- (void)encodeCacheHintFinalize:(uint64_t)tag resourceGroups:(const id __unsafe_unretained [])groups count:(NSUInteger)count {(void)tag;(void)groups;(void)count;} +// The kernel driver's own submission times are not recorded: 0 is Metal's answer where they are unavailable. +- (CFTimeInterval)kernelStartTime {return 0;} +- (CFTimeInterval)kernelEndTime {return 0;} +- (BOOL)commitAndWaitUntilSubmitted {[(id)self commit];[self waitUntilScheduled];return !self.failedLocally;} +- (CFTimeInterval)GPUStartTime {return nmb_command_buffer_gpu_time(_handle,false);} +- (CFTimeInterval)GPUEndTime {return nmb_command_buffer_gpu_time(_handle,true);} +- (id)encoder:(NBEncoder *)encoder handle:(NmbEncoder *)handle what:(NSString *)what { + if(self.committed){[self refuse:[what stringByAppendingString:@" after commit"]];return nil;} + if(!handle){[self refuse:[NSString stringWithFormat:@"%@: %@",what,lastError()]];return nil;} + encoder.handle=handle;encoder.commandBuffer=self;return encoder; +} +- (id)computeCommandEncoder { + return [self encoder:[NBComputeEncoder new] handle:self.committed?nullptr:nmb_compute_encoder_new(_handle) what:@"computeCommandEncoder"]; +} +- (id)computeCommandEncoderWithDispatchType:(MTLDispatchType)type { + if(type!=MTLDispatchTypeSerial){[self refuse:@"concurrent compute dispatch"];return nil;} + return [self computeCommandEncoder]; +} +- (id)blitCommandEncoder { + gBlits++;encodedWork.blits++; + return [self encoder:[NBBlitEncoder new] handle:self.committed?nullptr:nmb_blit_encoder_new(_handle) what:@"blitCommandEncoder"]; +} +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)d { + NmbRenderPassDescriptor pass={}; + { + id first=d.colorAttachments[0].texture; + if(first) { + const uint64_t pixels=(uint64_t)first.width*first.height; + gPasses++;gPassPixels+=pixels;encodedWork.passes++;encodedWork.pixels+=pixels; + uint64_t seen=gBiggestPass.load();while(pixels>seen && !gBiggestPass.compare_exchange_weak(seen,pixels)){} + if(pixels>=2000000)gFullScreenPasses++; + } + } + auto attachment=[&](MTLRenderPassAttachmentDescriptor *a,NmbPassAttachment &out)->bool { + if(!a.texture)return true; + if(![(id)a.texture isKindOfClass:[NBTexture class]])return false; + NBTexture *texture=(NBTexture *)a.texture;if(texture.surface)[_surfaces addObject:texture]; + [_resourceOwners addObject:texture]; + out.texture=texture.handle;out.level=a.level;out.slice=a.slice;out.depth_plane=a.depthPlane;out.load=a.loadAction;out.store=a.storeAction; + return true; + }; + for(unsigned i=0;i<8;i++) { + MTLRenderPassColorAttachmentDescriptor *c=d.colorAttachments[i]; + if(!attachment(c,pass.colors[i])){[self refuse:@"foreign color attachment"];return nil;} + pass.colors[i].clear[0]=c.clearColor.red;pass.colors[i].clear[1]=c.clearColor.green;pass.colors[i].clear[2]=c.clearColor.blue;pass.colors[i].clear[3]=c.clearColor.alpha; + } + if(!attachment(d.depthAttachment,pass.depth) || !attachment(d.stencilAttachment,pass.stencil)){[self refuse:@"foreign depth/stencil attachment"];return nil;} + pass.depth.clear[0]=d.depthAttachment.clearDepth;pass.stencil.clear[0]=d.stencilAttachment.clearStencil; + pass.width=d.renderTargetWidth;pass.height=d.renderTargetHeight;pass.array_length=d.renderTargetArrayLength;pass.samples=d.defaultRasterSampleCount; + if(d.visibilityResultBuffer) { + if(![(id)d.visibilityResultBuffer isKindOfClass:[NBBuffer class]]){[self refuse:@"foreign visibility result buffer"];return nil;} + [_resourceOwners addObject:d.visibilityResultBuffer]; + pass.visibility=[(NBBuffer *)d.visibilityResultBuffer handle]; + } + return [self encoder:[NBRenderEncoder new] handle:self.committed?nullptr:nmb_render_encoder_new(_handle,&pass) what:@"renderCommandEncoder"]; +} +- (void)addScheduledHandler:(MTLCommandBufferHandler)block {if(block)@synchronized(self){[_scheduledHandlers addObject:[block copy]];}} +- (void)addCompletedHandler:(MTLCommandBufferHandler)block {if(block)@synchronized(self){[_completedHandlers addObject:[block copy]];}} +- (void)enqueue {} +- (void)event:(id)event value:(uint64_t)value signal:(bool)signal { + if(![(id)event isKindOfClass:[NBEvent class]]){[self refuse:@"foreign event"];return;} + if(self.committed){[self refuse:@"event encoded after commit"];return;} + NmbEvent *handle=[(NBEvent *)event handle]; + if(!(signal?nmb_command_buffer_signal_event(_handle,handle,value):nmb_command_buffer_wait_event(_handle,handle,value))) + [self refuse:[NSString stringWithFormat:@"%s: %@",signal?"encodeSignalEvent":"encodeWaitForEvent",lastError()]]; +} +- (void)encodeSignalEvent:(id)event value:(uint64_t)value {[self event:event value:value signal:true];} +- (void)encodeWaitForEvent:(id)event value:(uint64_t)value {[self event:event value:value signal:false];} +// The runtime has submitted the work. Deliver scheduling without waiting for GPU completion. +- (void)didSchedule { + { + const uint64_t took=nowUs()-self.committedUs; + std::lock_guard lock(gCommitLock); + CommitRecord &r=gCommitRing[self.commitIndex%32]; + if(r.index==self.commitIndex)r.scheduledUs=took; + } + NSArray *handlers; + @synchronized(self){ + _schedulingDelivered=YES;handlers=[_scheduledHandlers copy];[_scheduledHandlers removeAllObjects]; + } + void (^run)(void)=^{ + for(MTLCommandBufferHandler block in handlers)block((id)self); + dispatch_group_leave(self->_schedulingHandlers); + }; + // Run on the runtime's scheduled-handler thread, preserving its ordering and own-wait + // deadlock checks. The queue's completionQueue routes completed handlers only. + run(); +} +// After completion (or a refusal): invalidate the CPU's view of imported surfaces, then completed handlers, once. +- (void)finish { + flushSurfaces(_surfaces); + // Actual GPU completion (or a local refusal that never submitted) is the + // release point. A retained completed command must not pin every old frame. + [_surfaces removeAllObjects];[_resourceOwners removeAllObjects]; + @synchronized(gInFlight){[gInFlight removeObject:self];} + NSArray *completedHandlers; + @synchronized(self){ + // A frontend refusal or a failed submission never scheduled. Unblock its waiter without inventing a scheduled event. + if(!_schedulingDelivered){_schedulingDelivered=YES;[_scheduledHandlers removeAllObjects];dispatch_group_leave(_schedulingHandlers);} + completedHandlers=[_completedHandlers copy];[_completedHandlers removeAllObjects]; + } + void (^run)(void)=^{ + for(MTLCommandBufferHandler block in completedHandlers)block((id)self); + dispatch_group_leave(self->_handlers); + }; + dispatch_queue_t queue=_queue.completionQueue; + if(queue)dispatch_async(queue,run);else run(); +} +- (void)commit { + if(self.committed){NBLOG(@"native-metal: commit twice");return;} + self.committed=YES; + self.committedUs=nowUs(); + { + std::lock_guard lock(gCommitLock); + // Assign once under the same lock as publication. A concurrent commit cannot reuse this index. + const uint32_t index=(uint32_t)(gCommits.fetch_add(1)+1);self.commitIndex=index; + CommitRecord &r=gCommitRing[index%32]; + r={};r.index=index;r.committedUs=self.committedUs;r.ageUs=self.committedUs-createdUs;r.work=encodedWork; + } + dispatch_group_enter(_handlers); + dispatch_group_enter(_schedulingHandlers); + @synchronized(gInFlight){[gInFlight addObject:self];} + if(!self.encodeFailure) { + flushSurfaces(_surfaces); + // Completion owns this reference; the runtime orders the non-owning scheduled callback before it. + void *context=(__bridge_retained void *)self; + if(nmb_command_buffer_on_completed(_handle,completed,context) && + nmb_command_buffer_on_scheduled(_handle,scheduled,context) && nmb_command_buffer_commit(_handle))return; + (void)CFBridgingRelease(context); // refused registration/commit cannot call the registered handlers + [self refuse:[@"commit: " stringByAppendingString:lastError()]]; + } + self.failedLocally=YES;gCommitFailures++; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{[self finish];}); +} +- (void)waitUntilCompleted { + if(!self.committed)return; + const uint64_t began=nowUs(); + if(!self.failedLocally)nmb_command_buffer_wait(_handle,0); + dispatch_group_wait(_handlers,DISPATCH_TIME_FOREVER); + gTimeWaitCompleted.add(began); +} +- (void)waitUntilScheduled { + if(!self.committed)return; + const uint64_t began=nowUs(); + if(!self.failedLocally && !nmb_command_buffer_wait_scheduled(_handle)){ + NBLOG(@"native-metal: waitUntilScheduled refused: %@",lastError());return; + } + dispatch_group_wait(_schedulingHandlers,DISPATCH_TIME_FOREVER); + gTimeWaitScheduled.add(began); +} +@end +static void scheduled(void *context) {[(__bridge NBCommandBuffer *)context didSchedule];} +// The runtime's completion: the command buffer's own reference, taken at commit, is given back here. +static void completed(void *context) { + gCompleted++; + NBCommandBuffer *commandBuffer=(__bridge_transfer NBCommandBuffer *)context; + { + const uint64_t took=nowUs()-commandBuffer.committedUs; + gTimeEncodeToCommit.count++;gTimeEncodeToCommit.total+=took; + uint64_t seen=gTimeEncodeToCommit.slowest.load();while(took>seen && !gTimeEncodeToCommit.slowest.compare_exchange_weak(seen,took)){} + std::lock_guard lock(gCommitLock); + CommitRecord &r=gCommitRing[commandBuffer.commitIndex%32]; + if(r.index==commandBuffer.commitIndex)r.callbackUs=took; + } + [commandBuffer finish]; +} + +@implementation NBQueue +- (void)dealloc {nmb_queue_release(_handle);} +- (id)commandBuffer { + NmbCommandBuffer *handle=nmb_command_buffer_new(_handle); + if(!handle){NBLOG(@"native-metal: commandBuffer refused: %@",lastError());return nil;} + NBCommandBuffer *commandBuffer=[NBCommandBuffer new];commandBuffer.handle=handle;commandBuffer.queue=self;return (id)commandBuffer; +} +- (id)commandBufferWithUnretainedReferences {return [self commandBuffer];} +- (void)setSubmissionQueue:(dispatch_queue_t)queue {(void)queue;} // submission happens on the caller's thread here +// setCompletionQueue: is the property's setter. +// GPU priorities are not applied (one hardware queue): reported as not applied. +- (BOOL)setGPUPriority:(NSUInteger)priority {(void)priority;return NO;} +- (BOOL)setBackgroundGPUPriority:(NSUInteger)priority {(void)priority;return NO;} +- (id)commandBufferWithDescriptor:(MTLCommandBufferDescriptor *)d {(void)d;return [self commandBuffer];} +@end + +// ---- the device ---- +template static T failWith(NSError **error,NSString *what,T value) { + noteProblem([NSString stringWithFormat:@"%@: %@",what,lastError()]); + if(error)*error=nbError([NSString stringWithFormat:@"%@: %@",what,lastError()]); + return value; +} +// A render pipeline descriptor in the bridge's form; false, with the error, for a function of another device. +static bool renderDescriptor(MTLRenderPipelineDescriptor *d,NmbRenderPipelineDescriptor *p,NSError **error) { + if(![(id)d.vertexFunction isKindOfClass:[NBFunction class]] || (d.fragmentFunction && ![(id)d.fragmentFunction isKindOfClass:[NBFunction class]])) { + if(error)*error=nbError(@"foreign function");return false; + } + p->vertex=[(NBFunction *)d.vertexFunction handle];p->fragment=d.fragmentFunction?[(NBFunction *)d.fragmentFunction handle]:nullptr; + for(unsigned i=0;i<8;i++) { + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i]; + p->colors[i]={(uint64_t)a.pixelFormat,(uint64_t)a.writeMask,(uint64_t)a.blendingEnabled,(uint64_t)a.sourceRGBBlendFactor,(uint64_t)a.destinationRGBBlendFactor, + (uint64_t)a.rgbBlendOperation,(uint64_t)a.sourceAlphaBlendFactor,(uint64_t)a.destinationAlphaBlendFactor,(uint64_t)a.alphaBlendOperation}; + } + p->depth_format=d.depthAttachmentPixelFormat;p->stencil_format=d.stencilAttachmentPixelFormat;p->samples=d.rasterSampleCount; + p->topology=d.inputPrimitiveTopology;p->rasterization=d.rasterizationEnabled; + if(MTLVertexDescriptor *v=d.vertexDescriptor) { + for(unsigned i=0;i<31;i++) { + MTLVertexAttributeDescriptor *a=v.attributes[i];if(a.format==MTLVertexFormatInvalid)continue; + p->vertex_inputs=1;p->attributes[i]={(uint64_t)a.format,a.offset,a.bufferIndex}; + if(a.bufferIndex<31){MTLVertexBufferLayoutDescriptor *l=v.layouts[a.bufferIndex];p->layouts[a.bufferIndex]={l.stride,(uint64_t)l.stepFunction,l.stepRate};} + } + } + return true; +} + +// ---- heaps ---- +// Metal heaps over the runtime's: an automatic heap places each resource, a placement heap takes an offset. Resources made from one keep it +// alive (their `heap`); a heap's resources share its storage mode, CPU cache mode and (here) hazard tracking mode. +@interface NBHeap : NBObject +@property(nonatomic) NmbHeap *handle; +@property(nonatomic) MTLResourceOptions options; +@property(nonatomic) MTLHeapType heapType; +@end +@implementation NBHeap +- (void)dealloc {nmb_heap_release(_handle);} +- (NSUInteger)size {return nmb_heap_size(_handle);} +- (NSUInteger)usedSize {return nmb_heap_used_size(_handle);} +- (NSUInteger)currentAllocatedSize {return nmb_heap_allocated_size(_handle);} +- (NSUInteger)maxAvailableSizeWithAlignment:(NSUInteger)alignment {return nmb_heap_max_available_size(_handle,alignment);} +- (MTLResourceOptions)resourceOptions {return _options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (MTLHazardTrackingMode)hazardTrackingMode {return (MTLHazardTrackingMode)((_options&MTLResourceHazardTrackingModeMask)>>MTLResourceHazardTrackingModeShift);} +- (MTLHeapType)type {return _heapType;} +- (uint64_t)protectionOptions {return 0;} +- (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { + const uint64_t answer=nmb_heap_set_purgeable_state(_handle,state); + if(!answer)NBLOG(@"native-metal: heap setPurgeableState refused: %@",lastError()); + return (MTLPurgeableState)answer; +} +- (id)bufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options offset:(uint64_t)offset { + NmbBuffer *handle=nmb_heap_new_buffer(_handle,length,options,offset); + if(!handle){NBLOG(@"native-metal: heap buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;buffer.parentHeap=(id)self; + buffer.heapOffset=offset==UINT64_MAX?0:offset;return (id)buffer; +} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options {return [self bufferWithLength:length options:options offset:UINT64_MAX];} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options offset:(NSUInteger)offset { + return [self bufferWithLength:length options:options offset:offset]; +} +- (id)textureWithDescriptor:(MTLTextureDescriptor *)d offset:(uint64_t)offset { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_heap_new_texture(_handle,&descriptor,offset); + if(!handle){NBLOG(@"native-metal: heap texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.parentHeap=(id)self;texture.heapOffset=offset==UINT64_MAX?0:offset;return (id)texture; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d {return [self textureWithDescriptor:d offset:UINT64_MAX];} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset {return [self textureWithDescriptor:d offset:offset];} +@end +// Apple's SPI resource groups name resources for cache hints (encodeCacheHintTag:/Finalize:); a group only keeps its resources alive. +@interface NBResourceGroup : NBObject +@property(nonatomic,strong) NSArray *resources; +@end +@implementation NBResourceGroup +@end + +// Creation and archive population must use the same complete supported descriptor. +static bool computeDescriptor(MTLComputePipelineDescriptor *d,NmbComputePipelineDescriptor *p,NSError **error) { + if(!d || ![(id)d.computeFunction isKindOfClass:[NBFunction class]]){if(error)*error=nbError(@"foreign compute function");return false;} + // Metal may materialize empty stage/linked descriptors on access. Preserve the + // default meaning while refusing state this frontend cannot yet represent. + MTLStageInputOutputDescriptor *stage=d.stageInputDescriptor; + bool empty=!stage || (stage.indexType==MTLIndexTypeUInt16 && stage.indexBufferIndex==0); + for(NSUInteger index=0;stage && index<31;++index) { + MTLAttributeDescriptor *a=stage.attributes[index];MTLBufferLayoutDescriptor *b=stage.layouts[index]; + empty=empty && a.format==MTLAttributeFormatInvalid && !a.offset && !a.bufferIndex && !b.stride && + b.stepFunction==MTLStepFunctionPerVertex && b.stepRate==1; + } + if(!empty){if(error)*error=nbError(@"compute stage inputs are not implemented");return false;} + MTLLinkedFunctions *linked=d.linkedFunctions; + if(linked.functions.count || linked.binaryFunctions.count || linked.privateFunctions.count || linked.groups.count || + d.preloadedLibraries.count || d.insertLibraries.count) { + if(error)*error=nbError(@"compute linked/dynamic libraries are not implemented");return false; + } + p->function=[(NBFunction *)d.computeFunction handle];p->max_threads=d.maxTotalThreadsPerThreadgroup; + p->multiple_of_execution_width=d.threadGroupSizeIsMultipleOfThreadExecutionWidth;p->indirect=d.supportIndirectCommandBuffers; + p->max_call_stack_depth=d.maxCallStackDepth;p->adding_binary_functions=d.supportAddingBinaryFunctions; + for(NSUInteger index=0;index<31;++index)p->mutability[index]=d.buffers[index].mutability; + return true; +} + +// ---- binary archives ---- +@interface NBBinaryArchive : NBObject +@property(nonatomic) NmbBinaryArchive *handle; +@end +@implementation NBBinaryArchive +- (void)dealloc {nmb_binary_archive_release(_handle);} +- (NSUInteger)options {return 0;} +- (BOOL)addRenderPipelineFunctionsWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + auto *p=new NmbRenderPipelineDescriptor(); + const bool ok=renderDescriptor(d,p,error) && nmb_binary_archive_add_render(_handle,p);delete p; + if(!ok)return failWith(error,@"archive render pipeline",NO); + if(error)*error=nil;return YES; +} +- (BOOL)addComputePipelineFunctionsWithDescriptor:(MTLComputePipelineDescriptor *)d error:(NSError **)error { + NmbComputePipelineDescriptor p={};if(!computeDescriptor(d,&p,error))return NO; + if(!nmb_binary_archive_add_compute_descriptor(_handle,&p))return failWith(error,@"archive compute pipeline",NO); + if(error)*error=nil;return YES; +} +- (BOOL)serializeToURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=nbError(@"a file URL is required");return NO;} + if(!nmb_binary_archive_serialize(_handle,url.fileSystemRepresentation))return failWith(error,@"archive serialize",NO); + if(error)*error=nil;return YES; +} +@end + +static bool pipelineArchives(NSArray *list,std::vector &archives,NSError **error) { + for(id archive in list) { + if(![archive isKindOfClass:[NBBinaryArchive class]]){if(error)*error=nbError(@"foreign binary archive");return false;} + archives.push_back([(NBBinaryArchive *)archive handle]); + } + return true; +} + +@interface NBDeviceMethods : NSObject +@end +@implementation NBDeviceMethods +- (NSString *)name {return gName;} +- (uint64_t)registryID {return gRegistryID;} +- (NSUInteger)maxBufferLength {return nmb_device_max_buffer_length(gDevice);} +- (BOOL)hasUnifiedMemory {return YES;} +- (BOOL)isLowPower {return NO;} +- (BOOL)isRemovable {return NO;} +- (BOOL)isHeadless {return NO;} +// supportsFamily: and the other queries the runtime answers are installed from it (installAnswers). No legacy feature set is claimed: the +// base implementation reads Apple tables this device does not have. +- (BOOL)supportsFeatureSet:(MTLFeatureSet)set {(void)set;return NO;} +// The original discovery (_MTLAddDevice) runs these four on every device it registers. The target's own versions fill the base class's limits +// and feature tables from a hardware profile (`featureProfile`, `indirectArgumentBufferCapabilities`, a table of Apple GPUs) that this device +// has not got: running them would claim an Apple GPU's capabilities. They are empty here, so a limit or feature the runtime measured is answered +// by this file's own answers (installAnswers) and any other stays the base class's neutral 0/NO. Nothing is claimed that nothing measured. +- (void)initLimits {} +- (void)initFeatureQueries {} +- (void)initGPUFamilySupport {} +- (void)initWorkarounds {} +- (NSString *)productName {return gName;} +- (NSString *)vendorName {char name[64]={};nmb_device_vendor_name(gDevice,name,sizeof name);return [NSString stringWithUTF8String:name];} +- (MTLSize)maxThreadsPerThreadgroup { + uint64_t s[3]={};if(!nmb_device_max_threads_per_threadgroup(gDevice,s))NBLOG(@"native-metal: maxThreadsPerThreadgroup: %@",lastError()); + return MTLSizeMake(s[0],s[1],s[2]); +} +- (void)doesNotRecognizeSelector:(SEL)selector { + if(gMissing)gMissing("NativeBridgeMetalDevice",sel_getName(selector)); + noteProblem([NSString stringWithFormat:@"missing device %s",sel_getName(selector)]); + [super doesNotRecognizeSelector:selector]; +} +- (id)newEvent { + NmbEvent *handle=nmb_event_new(gDevice,false);if(!handle)return nil; + NBEvent *event=[NBEvent new];event.handle=handle;return (id)event; +} +- (id)newSharedEvent { + NmbEvent *handle=nmb_event_new(gDevice,true);if(!handle)return nil; + NBSharedEvent *event=[NBSharedEvent new];event.handle=handle;return (id)event; +} +- (id)newFence { + NmbFence *handle=nmb_fence_new(gDevice);if(!handle)return nil; + NBFence *fence=[NBFence new];fence.handle=handle;return (id)fence; +} +- (id)newCommandQueue { + NmbQueue *handle=nmb_queue_new(gDevice);if(!handle)return nil; + NBQueue *queue=[NBQueue new];queue.handle=handle;return (id)queue; +} +- (id)newHeapWithDescriptor:(MTLHeapDescriptor *)d { + if(!d)return nil; + NmbHeap *handle=nmb_heap_new(gDevice,d.type,d.size,(uint64_t)d.resourceOptions); + if(!handle){NBLOG(@"native-metal: heap refused: %@",lastError());return nil;} + NBHeap *heap=[NBHeap new];heap.handle=handle;heap.options=d.resourceOptions;heap.heapType=d.type;return (id)heap; +} +- (id)newResourceGroupFromResources:(const id __unsafe_unretained [])resources count:(NSUInteger)count { + NBResourceGroup *group=[NBResourceGroup new]; + NSMutableArray *held=[NSMutableArray arrayWithCapacity:count]; + for(NSUInteger i=0;i)newTextureWithBytesNoCopy:(void *)bytes length:(NSUInteger)length descriptor:(MTLTextureDescriptor *)d deallocator:(void (^)(void *,NSUInteger))deallocator { + (void)bytes;(void)length;(void)d;(void)deallocator; + NBLOG(@"native-metal: newTextureWithBytesNoCopy:length:descriptor:deallocator: is not implemented (Apple's texture layout of raw memory is not known)"); + return nil; +} +- (id)newTextureLayoutWithDescriptor:(MTLTextureDescriptor *)d isHeapOrBufferBacked:(BOOL)backed { + (void)d;(void)backed; + NBLOG(@"native-metal: newTextureLayoutWithDescriptor:isHeapOrBufferBacked: is not implemented (Apple's layout object is not known)"); + return nil; +} +- (id)newCommandQueueWithMaxCommandBufferCount:(NSUInteger)count {(void)count;return [(id)self newCommandQueue];} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options { + NmbBuffer *handle=nmb_buffer_new(gDevice,length,options,nullptr); + if(!handle){NBLOG(@"native-metal: buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;return (id)buffer; +} +- (id)newBufferWithBytes:(const void *)bytes length:(NSUInteger)length options:(MTLResourceOptions)options { + if(!bytes)return nil; + NmbBuffer *handle=nmb_buffer_new(gDevice,length,options,bytes); + if(!handle){NBLOG(@"native-metal: buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;return (id)buffer; +} +- (id)newBufferWithBytesNoCopy:(void *)pointer length:(NSUInteger)length options:(MTLResourceOptions)options deallocator:(void (^)(void *,NSUInteger))deallocator { + NmbBuffer *handle=nmb_buffer_new_no_copy(gDevice,pointer,length,options); + if(!handle){NBLOG(@"native-metal: no-copy buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options; + buffer.wrapped=pointer;buffer.deallocator=deallocator; + return (id)buffer; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new(gDevice,&descriptor);noteFormat((uint32_t)descriptor.format,handle!=nullptr); + if(!handle){NBLOG(@"native-metal: texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor;return (id)texture; +} +// The surface's pages are the texels: its mapping in this process is imported without a copy (pinned by the kernel, mapped into the GPU's +// 4 KiB context by Turnip), so every process that imports the surface renders and samples the same memory. +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d iosurface:(IOSurfaceRef)surface plane:(NSUInteger)plane { + if(!surface || !d || plane || IOSurfaceGetPlaneCount(surface))return nil; + void *base=IOSurfaceGetBaseAddress(surface);const size_t page=(size_t)getpagesize(),allocation=IOSurfaceGetAllocSize(surface); + if(!page || (page&(page-1)) || !base || ((uintptr_t)base&(page-1)) || !allocation || allocation>SIZE_MAX-(page-1))return nil; + const size_t bytes=(allocation+page-1)&~(page-1); + NBBuffer *buffer=surfaceBuffer(surface,base,bytes);if(!buffer)return nil; + NBTexture *texture=textureOnBuffer(buffer,d,0,IOSurfaceGetBytesPerRow(surface),true);if(!texture)return nil; + texture.surface=(IOSurfaceRef)CFRetain(surface);return (id)texture; +} +- (id)newLibraryWithData:(dispatch_data_t)data error:(NSError **)error { + const void *bytes=nullptr;size_t size=0;dispatch_data_t mapped=dispatch_data_create_map(data,&bytes,&size); + const uint64_t libraryBegan=nowUs(); + NmbLibrary *handle=nmb_library_new(gDevice,bytes,size,false);(void)mapped;gTimeLibrary.add(libraryBegan); + if(!handle)return failWith>(error,@"library",nil); + NBLibrary *library=[NBLibrary new];library.handle=handle;if(error)*error=nil;return (id)library; +} +- (id)newLibraryWithFile:(NSString *)path error:(NSError **)error { + NSData *bytes=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:error]:nil;if(!bytes)return nil; + const uint64_t libraryBegan=nowUs(); + NmbLibrary *handle=nmb_library_new(gDevice,bytes.bytes,bytes.length,false);gTimeLibrary.add(libraryBegan); + if(!handle)return failWith>(error,@"library",nil); + NBLibrary *library=[NBLibrary new];library.handle=handle;if(error)*error=nil;return (id)library; +} +- (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=nbError(@"a file URL is required");return nil;} + return [(id)self newLibraryWithFile:url.path error:error]; +} +- (id)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(NSError **)error { + NSURL *url=[bundle URLForResource:@"default" withExtension:@"metallib"]; + if(!url){if(error)*error=nbError(@"the bundle has no default.metallib");return nil;} + return [(id)self newLibraryWithURL:url error:error]; +} +- (id)newDefaultLibrary {return [(id)self newDefaultLibraryWithBundle:[NSBundle mainBundle] error:nil];} +- (id)newComputePipelineStateWithFunction:(id)function error:(NSError **)error { + return [(id)self newComputePipelineStateWithFunction:function options:MTLPipelineOptionNone reflection:nil error:error]; +} +- (id)newComputePipelineStateWithFunction:(id)function options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedComputePipelineReflection *)reflection error:(NSError **)error { + if(reflection)*reflection=nil; + if(![(id)function isKindOfClass:[NBFunction class]]){if(error)*error=nbError(@"foreign function");return nil;} + const NmbPipelineOptions creation={(uint64_t)options,nullptr,0}; + const uint64_t pipelineBegan=nowUs(); + NmbComputePipeline *handle=nmb_compute_pipeline_new_with_options(gDevice,[(NBFunction *)function handle],&creation);gTimePipeline.add(pipelineBegan); + if(!handle)return failWith>(error,@"compute pipeline",nil); + NBComputePipeline *state=[NBComputePipeline new];state.handle=handle;gPipelines++;if(error)*error=nil;return (id)state; +} +- (id)newComputePipelineStateWithDescriptor:(MTLComputePipelineDescriptor *)d options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedComputePipelineReflection *)reflection error:(NSError **)error { + if(reflection)*reflection=nil; + NmbComputePipelineDescriptor p={};if(!computeDescriptor(d,&p,error))return nil; + std::vector archives; + if(!pipelineArchives(d.binaryArchives,archives,error))return nil; + const NmbPipelineOptions creation={(uint64_t)options,archives.data(),archives.size()}; + const uint64_t pipelineBegan=nowUs(); + NmbComputePipeline *handle=nmb_compute_pipeline_new_with_descriptor(gDevice,&p,&creation);gTimePipeline.add(pipelineBegan); + if(!handle)return failWith>(error,@"compute pipeline",nil); + NBComputePipeline *state=[NBComputePipeline new];state.handle=handle;state.label=d.label;gPipelines++; + if(error)*error=nil;return (id)state; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + return [(id)self newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionNone reflection:nil error:error]; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedRenderPipelineReflection *)reflection error:(NSError **)error { + if(reflection)*reflection=nil; + auto *p=new NmbRenderPipelineDescriptor(); + if(!renderDescriptor(d,p,error)){delete p;return nil;} + std::vector archives; + if(!pipelineArchives(d.binaryArchives,archives,error)){delete p;return nil;} + const NmbPipelineOptions creation={(uint64_t)options,archives.data(),archives.size()}; + const uint64_t pipelineBegan=nowUs(); + NmbRenderPipeline *handle=nmb_render_pipeline_new_with_options(gDevice,p,&creation);delete p;gTimePipeline.add(pipelineBegan); + if(!handle)return failWith>(error,@"render pipeline",nil); + NBRenderPipeline *state=[NBRenderPipeline new];state.handle=handle;gPipelines++;if(error)*error=nil;return (id)state; +} +- (id)newBinaryArchiveWithDescriptor:(MTLBinaryArchiveDescriptor *)d error:(NSError **)error { + if(d.url && !d.url.isFileURL){if(error)*error=nbError(@"a file URL is required");return nil;} + NmbBinaryArchive *handle=nmb_binary_archive_new(gDevice,d.url?d.url.fileSystemRepresentation:nullptr); + if(!handle)return failWith>(error,@"binary archive",nil); + NBBinaryArchive *archive=[NBBinaryArchive new];archive.handle=handle;if(error)*error=nil;return (id)archive; +} +- (NSUInteger)requiredLinearTextureBytesPerRowForDescriptor:(MTLTextureDescriptor *)d { + const uint64_t row=nmb_device_linear_bytes_per_row(gDevice,d.pixelFormat,d.width); + if(!row)NBLOG(@"native-metal: requiredLinearTextureBytesPerRow: %@",lastError()); + return row; +} +- (id)newSamplerStateWithDescriptor:(MTLSamplerDescriptor *)d { + NmbSamplerDescriptor s={(uint64_t)d.minFilter,(uint64_t)d.magFilter,(uint64_t)d.mipFilter,(uint64_t)d.sAddressMode,(uint64_t)d.tAddressMode,(uint64_t)d.rAddressMode, + (uint64_t)d.borderColor,(uint64_t)d.normalizedCoordinates,d.maxAnisotropy,(uint64_t)d.compareFunction,(uint64_t)d.lodAverage,d.lodMinClamp,d.lodMaxClamp}; + NmbSampler *handle=nmb_sampler_new(gDevice,&s); + if(!handle){NBLOG(@"native-metal: sampler refused: %@",lastError());return nil;} + NBSampler *sampler=[NBSampler new];sampler.handle=handle;return (id)sampler; +} +- (id)newDepthStencilStateWithDescriptor:(MTLDepthStencilDescriptor *)d { + auto face=[](MTLStencilDescriptor *s)->NmbStencil { + return {(uint64_t)s.stencilCompareFunction,(uint64_t)s.stencilFailureOperation,(uint64_t)s.depthFailureOperation,(uint64_t)s.depthStencilPassOperation,s.readMask,s.writeMask}; + }; + NmbDepthStencilDescriptor s={(uint64_t)d.depthCompareFunction,(uint64_t)d.depthWriteEnabled,face(d.frontFaceStencil),face(d.backFaceStencil)}; + NmbDepthStencil *handle=nmb_depth_stencil_new(gDevice,&s); + if(!handle){NBLOG(@"native-metal: depth/stencil state refused: %@",lastError());return nil;} + NBDepthStencil *state=[NBDepthStencil new];state.handle=handle;return (id)state; +} +@end + +// The device queries the runtime answers (nmb_device_answer_selector), as methods of the device class. One without an argument is read once +// and installed only where the device has an answer of its own; one with an argument asks the runtime each call, and a refusal answers 0/NO +// with the reason logged. +static std::unordered_map gAnswers; +static NSUInteger countAnswer(id,SEL cmd) {return (NSUInteger)gAnswers.at(cmd);} +static BOOL flagAnswer(id,SEL cmd) {return gAnswers.at(cmd)!=0;} +static uint64_t askAnswer(SEL cmd,NSUInteger arg) { + uint64_t value=0; + if(!nmb_device_answer(gDevice,sel_getName(cmd),arg,&value)){NBLOG(@"native-metal: %s(%lu): %@",sel_getName(cmd),(unsigned long)arg,lastError());return 0;} + return value; +} +static NSUInteger countArgAnswer(id,SEL cmd,NSUInteger arg) {return (NSUInteger)askAnswer(cmd,arg);} +static BOOL flagArgAnswer(id,SEL cmd,NSUInteger arg) {return askAnswer(cmd,arg)!=0;} +static void installAnswers(Class cls) { + uint32_t kind=0; + for(size_t i=0;const char *name=nmb_device_answer_selector(i,&kind);i++) { + SEL sel=sel_registerName(name);uint64_t value=0; + switch(kind) { + case NMB_ANSWER_COUNT: case NMB_ANSWER_FLAG: + if(!nmb_device_answer(gDevice,name,0,&value)){NBLOG(@"native-metal: no device answer: %@",lastError());continue;} + // Extended-range 10-bit (XR10) framebuffers are withheld: this display chain presents 8-bit BGRX scanout only. The render server picks its + // wide-gamut 64-bit (and Apple-compressed) framebuffer when a device offers them, and the display driver could not show it (int-089: the + // swap's surface was '&w4a', 8 bytes per pixel, and every swap was refused). + if(!strcmp(name,"supportsExtendedXR10Formats"))value=0; + gAnswers[sel]=value; + class_addMethod(cls,sel,kind==NMB_ANSWER_COUNT?(IMP)countAnswer:(IMP)flagAnswer,kind==NMB_ANSWER_COUNT?"Q16@0:8":"B16@0:8"); + break; + case NMB_ANSWER_COUNT_ARG: class_addMethod(cls,sel,(IMP)countArgAnswer,"Q24@0:8Q16");break; + case NMB_ANSWER_FLAG_ARG: class_addMethod(cls,sel,(IMP)flagArgAnswer,"B24@0:8Q16");break; + } + } +} + +extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint64_t registryID,void (*missing)(const char *,const char *)) { + if(gDeviceObject)return gDeviceObject; + gMissing=missing;gInFlight=[NSMutableSet set]; + gDevice=nmb_device_open(loader,cache); + if(!gDevice){NBLOG(@"native-metal: device refused: %s",nmb_last_error());return nil;} + char name[128]={};nmb_device_name(gDevice,name,sizeof name);gName=[NSString stringWithUTF8String:name];gRegistryID=registryID; + Class base=objc_getClass("_MTLDevice"); +#if !TARGET_OS_IPHONE + // Host instrument: on NSObject every selector a client sends that this file lacks reaches the missing hook, instead of the host's + // own _MTLDevice implementation. + if(getenv("NMB_DEVICE_BASE_NSOBJECT"))base=[NSObject class]; +#endif +#if TARGET_OS_IPHONE + // The target's base class ABI (iPadOS 17.7.11): the dynamic subclass adds no instance variables. + if(!base || class_getInstanceSize(base)!=536){NBLOG(@"native-metal: unexpected _MTLDevice ABI");return nil;} +#endif + if(!base)base=[NSObject class]; + Class cls=objc_allocateClassPair(base,"NativeBridgeMetalDevice",0);if(!cls)return nil; + unsigned count=0;Method *methods=class_copyMethodList([NBDeviceMethods class],&count); + for(unsigned i=0;i lock(gProblemLock); + return (size_t)snprintf(out,size,"c%llu/%llu d%llu p%llu x%llu R%llu D%llu F%llu M%llu C%llu B%llu %s | %s",(unsigned long long)gCommits.load(),(unsigned long long)gCommitFailures.load(),(unsigned long long)gCompleted.load(), + (unsigned long long)gPipelines.load(),(unsigned long long)gProblems.load(),(unsigned long long)gPasses.load(),(unsigned long long)gDraws.load(),(unsigned long long)gFullScreenPasses.load(),(unsigned long long)(gBiggestPass.load()>>10),(unsigned long long)gDispatches.load(),(unsigned long long)gBlits.load(),gLastProblems[0],gLastProblems[1]); +} + +static size_t native_bridge_metal_formats_locked(char *out,size_t size) { + int n=snprintf(out,size,"tex x%llu f",(unsigned long long)gTextureRefusals.load()); + for(unsigned i=0;i0 && (size_t)n0?(size_t)n:0; +} +// The benchmark's rows (native_bridge_metal_benchmark), kept for the telemetry table. +static std::mutex gBenchLock; +static char gBench[4][96]; +static std::atomic gBenchRuns; +extern "C" size_t native_bridge_metal_benchmark(id device,const char *metallib,unsigned pairs,char *out,size_t size) { + auto now=[]{return clock_gettime_nsec_np(CLOCK_UPTIME_RAW);}; + // the first run fills rows 0-1 (Y, Y1), a later one rows 2-3 (Z, Z1), so a busy and an idle run are both kept + const unsigned base=gBenchRuns++?2u:0u; + auto put=[&](unsigned row,const char *format,auto... args){std::lock_guard lock(gBenchLock);snprintf(gBench[base+row],sizeof gBench[base+row],format,args...); + if(base)gBench[base+row][0]='Z';}; + // compute: a dependent integer chain (no memory), 100M steps + uint64_t t=now(),x=0x9e3779b97f4a7c15ull; + for(uint32_t i=0;i<100000000u;i++){x^=x<<13;x^=x>>7;x^=x<<17;} + const uint64_t computeUs=(now()-t)/1000; + // first touch: 32 MiB of fresh anonymous memory (the tablet's free memory is small), one store per 4 KiB page (the kernel's fault and zero-fill), then four memsets of it + const size_t bytes=32u<<20; + uint64_t faultUs=0,fillUs=0,shaUs=0; + void *m=mmap(nullptr,bytes,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(m!=MAP_FAILED) { + volatile uint8_t *b=(volatile uint8_t *)m; + t=now();for(size_t at=0;at dev=(id)device;NSError *error=nil; + t=now(); + id qc=metallib?[dev newLibraryWithURL:[NSURL fileURLWithPath:@(metallib)] error:&error]:nil; + libraryUs=(now()-t)/1000; + if(qc) { + NSArray *names=qc.functionNames; + id queue=[dev newCommandQueue]; + MTLTextureDescriptor *td=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:64 height:64 mipmapped:NO]; + td.usage=MTLTextureUsageRenderTarget; + id target=[dev newTextureWithDescriptor:td]; + id zeros=[dev newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + unsigned n=0; + for(NSString *name in names) { + if(n>=pairs)break; + id f=[qc newFunctionWithName:name]; + if(f.functionType!=MTLFunctionTypeVertex)continue; + NSString *fragmentName=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if([fragmentName isEqualToString:name] || ![names containsObject:fragmentName])continue; + ++n; + MTLRenderPipelineDescriptor *d=[MTLRenderPipelineDescriptor new]; + d.vertexFunction=f;d.fragmentFunction=[qc newFunctionWithName:fragmentName]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm; + t=now(); + id state=[dev newRenderPipelineStateWithDescriptor:d error:&error]; + const uint64_t took=(now()-t)/1000; + if(!state){++refused;continue;} + ++made;createUs+=took;createMax=std::max(createMax,took); + t=now(); + MTLRenderPassDescriptor *pass=[MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture=target;pass.colorAttachments[0].loadAction=MTLLoadActionClear;pass.colorAttachments[0].storeAction=MTLStoreActionStore; + id commands=[queue commandBuffer]; + id encoder=[commands renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:state]; + for(NSUInteger i=0;i<8;++i){[encoder setVertexBuffer:zeros offset:0 atIndex:i];[encoder setFragmentBuffer:zeros offset:0 atIndex:i];} + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; + [encoder endEncoding];[commands commit];[commands waitUntilCompleted]; + const uint64_t drew=(now()-t)/1000; + drawUs+=drew;drawMax=std::max(drawMax,drew); + } + } + } + put(1,"Y1 qc lib=%llums made=%llu refused=%llu create=%llums max=%llums draw=%llums max=%llums",(unsigned long long)(libraryUs/1000), + (unsigned long long)made,(unsigned long long)refused,(unsigned long long)(createUs/1000),(unsigned long long)(createMax/1000), + (unsigned long long)(drawUs/1000),(unsigned long long)(drawMax/1000)); + size_t at=0; + if(out && size) { + std::lock_guard lock(gBenchLock); + out[0]=0;for(unsigned r=base;rat?size-at:0,"%s\n",gBench[r]); + } + return at; +} +// The telemetry table the plugin streams to the kernel's log: newline-separated rows of at most 90 characters. +extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) { + size_t at=0;out[0]=0; + auto row=[&](const char *format,auto... args){ + char text[200];int n=snprintf(text,sizeof text,format,args...);if(n<=0)return;if(n>90)n=90; + if(at+(size_t)n+2>=size)return;memcpy(out+at,text,(size_t)n);at+=(size_t)n;out[at++]='\n';out[at]=0;}; + auto ms=[](uint64_t us){return (unsigned long long)(us/1000);}; + {std::lock_guard lock(gBenchLock);for(unsigned r=0;r<4;r++)if(gBench[r][0])row("%.90s",gBench[r]);} + row("S c%llu/%llu d%llu p%llu x%llu R%llu D%llu F%llu M%lluk C%llu B%llu",(unsigned long long)gCommits.load(),(unsigned long long)gCommitFailures.load(), + (unsigned long long)gCompleted.load(),(unsigned long long)gPipelines.load(),(unsigned long long)gProblems.load(),(unsigned long long)gPasses.load(), + (unsigned long long)gDraws.load(),(unsigned long long)gFullScreenPasses.load(),(unsigned long long)(gBiggestPass.load()>>10), + (unsigned long long)gDispatches.load(),(unsigned long long)gBlits.load()); + row("T pipeline n=%llu total=%llums max=%llums",(unsigned long long)gTimePipeline.count.load(),ms(gTimePipeline.total.load()),ms(gTimePipeline.slowest.load())); + row("T library n=%llu total=%llums max=%llums",(unsigned long long)gTimeLibrary.count.load(),ms(gTimeLibrary.total.load()),ms(gTimeLibrary.slowest.load())); + row("T function n=%llu total=%llums max=%llums", + (unsigned long long)gTimeFunction.count.load(),ms(gTimeFunction.total.load()),ms(gTimeFunction.slowest.load())); + row("G commit-to-complete n=%llu total=%llums max=%llums mean=%lluus",(unsigned long long)gTimeEncodeToCommit.count.load(),ms(gTimeEncodeToCommit.total.load()), + ms(gTimeEncodeToCommit.slowest.load()),(unsigned long long)(gTimeEncodeToCommit.count.load()?gTimeEncodeToCommit.total.load()/gTimeEncodeToCommit.count.load():0)); + row("J scheduled wait n=%llu total=%llums max=%llums",(unsigned long long)gTimeWaitScheduled.count.load(),ms(gTimeWaitScheduled.total.load()),ms(gTimeWaitScheduled.slowest.load())); + row("J completed wait n=%llu total=%llums max=%llums",(unsigned long long)gTimeWaitCompleted.count.load(),ms(gTimeWaitCompleted.total.load()),ms(gTimeWaitCompleted.slowest.load())); + row("J resource wait n=%llu total=%llums max=%llums",(unsigned long long)gTimeResourceWait.count.load(),ms(gTimeResourceWait.total.load()),ms(gTimeResourceWait.slowest.load())); + row("J resource checks=%llu skipped=%llu",(unsigned long long)gResourceChecks.load(),(unsigned long long)gResourceSkipped.load()); + row("J surface cache n=%llu bytes=%llu dup=%llu us=%llu max=%llu",(unsigned long long)gTimeSurfaceFlush.count.load(), + (unsigned long long)gSurfaceFlushBytes.load(),(unsigned long long)gSurfaceFlushDuplicates.load(), + (unsigned long long)gTimeSurfaceFlush.total.load(),(unsigned long long)gTimeSurfaceFlush.slowest.load()); + // The runtime's VkPipeline makings (the creation log, METAL2VULKAN_LOG_PIPELINES; they happen at draws, not at the state's creation) and + // its shader cache: where the driver's compile time goes. + uint64_t work[10]={}; + if(gDevice && nmb_device_work_stats(gDevice,work)) { + row("V vkpipe n=%llu total=%llums max=%llums waited=%llums failed=%llu", + (unsigned long long)work[0],ms(work[1]),ms(work[2]),ms(work[3]),(unsigned long long)work[4]); + row("V cache hit%llu miss%llu wr%llu | msg i%llu w%llu",(unsigned long long)work[7],(unsigned long long)work[8],(unsigned long long)work[9],(unsigned long long)work[5], + (unsigned long long)work[6]); + char warning[200];size_t n=nmb_last_warning(warning,sizeof warning); + if(n){row("W %.86s",warning);if(n>86)row("Wb %.86s",warning+86);} + } + { + std::lock_guard lock(gProblemLock); + char formats[160];native_bridge_metal_formats_locked(formats,sizeof formats);row("F %.86s",formats); + row("P0 %.86s",gLastProblems[0]);row("P0b %.86s",strlen(gLastProblems[0])>86?gLastProblems[0]+86:""); + row("P1 %.86s",gLastProblems[1]);row("P1b %.86s",strlen(gLastProblems[1])>86?gLastProblems[1]+86:""); + } + { + std::lock_guard lock(gCommitLock); + const uint32_t newest=(uint32_t)gCommits.load(); + row("%s","Q timing v2: per-buffer API counts; ctc=commit-to-callback; age includes caller idle"); + for(uint32_t k=0;k<20 && k>10),r.work.dispatches,r.work.blits,(unsigned long long)r.callbackUs); + if(k<4)row("E%u age=%lluus scheduled=%lldus",r.index,(unsigned long long)r.ageUs,(long long)r.scheduledUs); + } + } + return at; +} + +extern "C" size_t native_bridge_metal_formats(char *out,size_t size) { + std::lock_guard lock(gProblemLock); + return native_bridge_metal_formats_locked(out,size); +} diff --git a/NativeIOGPUProviderProbe.cpp b/NativeIOGPUProviderProbe.cpp new file mode 100644 index 0000000..debc868 --- /dev/null +++ b/NativeIOGPUProviderProbe.cpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Unpublished original-provider ABI instrument. Required methods never return +// fabricated success. An explicit missing-resource control returns NULL from +// descriptor slot170; all other required calls remain non-returning captures. +// Original C2/D2 and every inherited operational address +// remain intact; registration is intercepted only on this own probe class. +#include "OriginalGPUProviderABI.h" +extern "C" __attribute__((noreturn)) void native_original_provider_required(unsigned,const unsigned long long *); +extern "C" IOMemoryDescriptor *native_original_provider_descriptor(const unsigned long long *); +class NativeIOGPUProviderProbe : public IOGPU { + OSDeclareDefaultStructors(NativeIOGPUProviderProbe) +public: + __attribute__((noreturn)) void registerService(IOOptionBits = 0) override; + IOMemoryDescriptor *abiSlot170(UInt32 *) override; + __attribute__((noreturn)) void abiSlot174() override; + __attribute__((noreturn)) void abiSlot189() override; + __attribute__((noreturn)) void abiSlot192(void *) override; + __attribute__((noreturn)) bool abiSlot195(IOService *) override; + __attribute__((noreturn)) void abiSlot196() override; + __attribute__((noreturn)) void abiSlot197() override; + __attribute__((noreturn)) void abiSlot200() override; + __attribute__((noreturn)) void abiSlot202() override; + __attribute__((noreturn)) void abiSlot205() override; +}; +OSDefineMetaClassAndStructors(NativeIOGPUProviderProbe,IOGPU) +// Save actual GPR arguments, caller LR, live CPU translation registers and SP. +// No FP instruction is executed in an unqualified EL1 FP context. These +// non-returning hooks do not reinterpret or mutate unknown pointer arguments. +#define SAVE_REQUIRED_FRAME \ + "sub sp, sp, #128\n" \ + "stp x0, x1, [sp, #0]\n" "stp x2, x3, [sp, #16]\n" \ + "stp x4, x5, [sp, #32]\n" "stp x6, x7, [sp, #48]\n" \ + "stp x8, x30, [sp, #64]\n" \ + "mrs x9, tcr_el1\n" "mrs x10, sctlr_el1\n" "stp x9, x10, [sp, #80]\n" \ + "mrs x9, ttbr0_el1\n" "mrs x10, ttbr1_el1\n" "stp x9, x10, [sp, #96]\n" \ + "add x9, sp, #128\n" "mrs x10, daif\n" "stp x9, x10, [sp, #112]\n" +#define CAPTURE_REQUIRED(number) __asm__ volatile(SAVE_REQUIRED_FRAME \ + "mov w0, #" #number "\n" "mov x1, sp\n" \ + "bl _native_original_provider_required\n" "brk #1\n") +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::registerService(IOOptionBits) {CAPTURE_REQUIRED(84);} +__attribute__((naked)) IOMemoryDescriptor *NativeIOGPUProviderProbe::abiSlot170(UInt32 *) { + __asm__ volatile(SAVE_REQUIRED_FRAME + "mov x0, sp\n" "bl _native_original_provider_descriptor\n" + "ldr x30, [sp, #72]\n" "add sp, sp, #128\n" "ret\n"); +} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot174() {CAPTURE_REQUIRED(174);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot189() {CAPTURE_REQUIRED(189);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot192(void *) {CAPTURE_REQUIRED(192);} +__attribute__((naked,noreturn)) bool NativeIOGPUProviderProbe::abiSlot195(IOService *) {CAPTURE_REQUIRED(195);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot196() {CAPTURE_REQUIRED(196);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot197() {CAPTURE_REQUIRED(197);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot200() {CAPTURE_REQUIRED(200);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot202() {CAPTURE_REQUIRED(202);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot205() {CAPTURE_REQUIRED(205);} +extern "C" IOService *native_original_provider_allocate(void) {return new NativeIOGPUProviderProbe;} +extern "C" unsigned native_original_provider_instances(void) {return NativeIOGPUProviderProbe::gMetaClass.getInstanceCount();} diff --git a/NativeMetalBridgePhase.h b/NativeMetalBridgePhase.h new file mode 100644 index 0000000..86c9439 --- /dev/null +++ b/NativeMetalBridgePhase.h @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: MIT +// Plugin-only phase after Turnip: the Metal bridge self-test (bridge/NativeMetalBridge.rs) on the same Turnip, on an 8 MiB worker under a +// watchdog. Markers 0x6501..0x650f come from the bridge itself; the result line goes to GPU7 ("M ..."), a stuck worker is reported there +// as "MW ". +// After a passing self-test the worker runs the frontend checks (bridge/FrontendChecks.h) through the public Metal API on the native frontend +// (NativeBridgeMetalFrontend.mm) over the same Turnip: markers 0x6600 (device), 0x6601..0x660f, then 0x66f0 pass / 0x66f1 fail. Its verdict +// ends the same GPU7 line: "F0" (with the first unimplemented selector, if any), "F ", "Fo"/"Fx" + reason +// when the device or the fixtures were refused. +#pragma once +#include "bridge/NativeMetalBridge.h" +#include "bridge/NativeMetalObjects.h" +#include "bridge/FrontendChecks.h" +#include "NativeBridgeMetalFrontend.h" +#include "GpuScanoutClient.h" +#include +#include +#include +#include +#include + +namespace bridgephase { +struct Job { + const char *turnip; + char cache[512]; + NmbSelftest result; + uint32_t code; + frontendchecks::Report frontend; + bool frontendRan, frontendOpened; + char missing[64], error[48]; + volatile uint32_t last; + volatile bool done; +}; +static Gpu *gGpu; +static Job *gJob; +static id gDevice; // the frontend device of the checks, kept for the scanout stage +static void mark(uint32_t value) { + if (gJob) gJob->last = value; + if (gGpu) gGpu->mark(value); +} +static void missingSelector(const char *cls, const char *selector) { + if (gJob && !gJob->missing[0]) snprintf(gJob->missing, sizeof gJob->missing, "FM %.20s %.36s", cls, selector); +} +// The frontend checks on the native frontend over Turnip (QuartzCore is already translated by the self-test, so it is not repeated). +static void frontend(Job *job) { + mark(0x6600); + id device = native_bridge_metal_open(job->turnip, job->cache, 0, missingSelector); + // The runtime's reasons are per thread: keep this worker's for the report. + if (!device) {strlcpy(job->error, nmb_last_error(), sizeof job->error); return;} + job->frontendOpened = true; + gDevice = device; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + air = nmb_function_constant_air(&bytes); + id fcLibrary = native_bridge_metal_library_with_air(air, bytes); + if (!library || !fcLibrary) {strlcpy(job->error, nmb_last_error(), sizeof job->error); return;} + job->frontend.mark = mark; + job->frontendRan = true; + // An exception escaping a worker thread aborts the whole host process (SpringBoard died that way, int-075/083): report it as a failed check. + @try { + frontendchecks::run(device, library, fcLibrary, nil, &job->frontend); + } @catch (NSException *exception) { + ++job->frontend.failed; + snprintf(job->frontend.first, sizeof job->frontend.first, "exception %.40s", exception.name.UTF8String ?: "?"); + } +} +static void *worker(void *arg) { + Job *job = static_cast(arg); + // The target's own compositor library, read in place (never copied anywhere). + const uint8_t *qc = nullptr; + size_t qcBytes = 0; + const int fd = open("/System/Library/Frameworks/QuartzCore.framework/default.metallib", O_RDONLY); + struct stat st = {}; + if (fd >= 0 && fstat(fd, &st) == 0 && st.st_size > 0) { + void *m = mmap(nullptr, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (m != MAP_FAILED) {qc = static_cast(m); qcBytes = (size_t)st.st_size;} + } + if (fd >= 0) close(fd); + job->code = nmb_selftest(job->turnip, job->cache, qc, qcBytes, mark, &job->result); + if (qc) munmap(const_cast(qc), qcBytes); + if (!job->code) frontend(job); + __atomic_store_n(&job->done, true, __ATOMIC_RELEASE); + return nullptr; +} +// Serves the display driver's frames with the GPU on `device` (the kernel offers them only with sheng-fb-gpu=1). The calls go through a hand-off-only +// connection to the accelerator (open type 0x4148: no GPU slot is taken): the original sandbox profiles admit that class and not the presenter's own +// service. Reports on GPU7 only when the state changes: "P e" (0 no connection, 1 no selector, 2 mode refused, 3 scanout not +// mappable/importable, 4 serving), then "PF / r v/ t" at the first frame, every fifth after +// and the first failure, "PX ..." when serving stops. Returns when serving stops. +struct ScanoutContext { + Gpu link; + Gpu *marker; +}; +static void serveScanout(Gpu &gpu, id device) { + ScanoutContext context; + context.marker = &gpu; + Gpu &link = context.link; + link.call = gpu.call; link.map = gpu.map; link.unmap = gpu.unmap; link.openFn = gpu.openFn; link.closeFn = gpu.closeFn; link.port = gpu.port; + gpuscanout::Stats st = {}; + char line[64]; + gpu.mark(0x6700); + const int openResult = link.openFn ? link.openFn(link.port, mach_task_self(), 0x4148, &link.conn) : -1; + if (openResult != 0 || !link.conn) { + snprintf(line, sizeof line, "P0 e%x", (unsigned)openResult); + } else if (gpu.mark(0x6701), !link.probe()) { + snprintf(line, sizeof line, "P1"); + } else { + gpu.mark(0x6702); + gpuscanout::Transport transport = {&context, + [](void *c, uint32_t method, const void *in, size_t is, void *out, size_t os) -> int { + return static_cast(c)->link.method(0x7000 + method, in, is, out, os); + }, + [](void *c, uint64_t *address, uint64_t *length) -> int { + Gpu &g = static_cast(c)->link; + return g.map(g.conn, 0x5343, mach_task_self(), address, length, 0x101); // anywhere + inhibit cache + }, + [](void *c, uint32_t value) {static_cast(c)->marker->mark(value);}}; + gpuscanout::Client client; + const bool opened = client.open((id)device, transport, &st); + snprintf(line, sizeof line, "P%d e%x", st.step, (unsigned)st.error); + streamText7(gpu, line); + if (opened) { + uint64_t reportedFrames = 0, reportedFailures = 0; + while (client.serveOne(500)) { + const bool frame = st.frames && (st.frames == 1 || st.frames - reportedFrames >= 5); + if (frame || (st.failures && !reportedFailures)) { + reportedFrames = st.frames ? st.frames : reportedFrames; + reportedFailures = st.failures ? st.failures : reportedFailures; + const uint64_t served = st.frames + st.failures; + snprintf(line, sizeof line, "PF %llu/%llu r%d v%llu/%llu t%u a%llu", (unsigned long long)st.frames, (unsigned long long)st.failures, + st.lastResult, (unsigned long long)st.verified, (unsigned long long)st.mismatchRows, st.lastMs, + (unsigned long long)(served ? st.totalMs / served : 0)); + streamText7(gpu, line); + } + } + snprintf(line, sizeof line, "PX %llu/%llu r%d e%x", (unsigned long long)st.frames, (unsigned long long)st.failures, st.lastResult, + (unsigned)st.error); + } + } + streamText7(gpu, line); +} +static void run(Gpu &gpu, const char *turnipPath) { + gpu.mark(0x6500); + Job *job = new Job{}; + job->turnip = turnipPath; + // A per-user cache directory of the host process (its sandbox allows it); the translated-shader cache goes in a subdirectory. + if (!confstr(_CS_DARWIN_USER_CACHE_DIR, job->cache, sizeof job->cache - 16)) strlcpy(job->cache, "/tmp/", sizeof job->cache); + strlcat(job->cache, "m2v", sizeof job->cache); + mkdir(job->cache, 0700); + gGpu = &gpu; + gJob = job; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8u << 20); + pthread_t thread; + const bool started = pthread_create(&thread, &attr, worker, job) == 0; + pthread_attr_destroy(&attr); + if (!started) {gpu.mark(0x65fd); return;} + static const unsigned checkpoints[] = {30, 90, 180}; + unsigned next = 0; + for (unsigned tenths = 0; !__atomic_load_n(&job->done, __ATOMIC_ACQUIRE); ++tenths) { + usleep(100000); + if (next < 3 && tenths == checkpoints[next] * 10) { + char text[64]; + snprintf(text, sizeof text, "MW %x %us", job->last, checkpoints[next]); + streamText7(gpu, text); + ++next; + } + } + pthread_join(thread, nullptr); + const NmbSelftest &r = job->result; + // One GPU7 line (the page keeps only the latest): the self-test's counts, then after a passing self-test the frontend's verdict. + char text[64]; + const int at = snprintf(text, sizeof text, "M s%x c%u r%u i%u q%u k%u/%u p%u/%u", r.stage, r.compute_bad, r.render_bad, r.import_bad, + r.qc_functions, r.qc_kernels_ok, r.qc_kernels_failed, r.qc_pairs_ok, r.qc_pairs_failed); + char *rest = text + (at > 0 && at < (int)sizeof text ? at : 0); + const size_t room = sizeof text - (size_t)(rest - text); + if (job->code) snprintf(rest, room, " %s", r.message); + else if (!job->frontendOpened) snprintf(rest, room, " Fo %s", job->error); + else if (!job->frontendRan) snprintf(rest, room, " Fx %s", job->error); + else if (job->frontend.failed) snprintf(rest, room, " F%u %s", job->frontend.failed, job->frontend.first); + else snprintf(rest, room, " F0%s%s", job->missing[0] ? " " : "", job->missing[0] ? job->missing + 3 : ""); + streamText7(gpu, text); + const bool ok = !job->code && !r.compute_bad && !r.render_bad && !r.import_bad; + gpu.mark(ok ? 0x65f0 : 0x65f1); + if (!job->code) gpu.mark(job->frontendRan && !job->frontend.failed ? 0x66f0 : 0x66f1); + const bool scanout = ok && job->frontendRan && !job->frontend.failed && gDevice; + gJob = nullptr; + delete job; + // After the checks' verdict has been on the page for a while: serve the display driver's frames with the GPU. + if (scanout) { + sleep(8); + serveScanout(gpu, gDevice); + } + gGpu = nullptr; +} +} // namespace bridgephase diff --git a/NativeOfflineAccelerator.cpp b/NativeOfflineAccelerator.cpp new file mode 100644 index 0000000..11f1b9d --- /dev/null +++ b/NativeOfflineAccelerator.cpp @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Unpublished original-base lifecycle qualification. No GPU service or device +// is attached/started/published, and no original client contract is claimed. +#include "OriginalAcceleratorABI.h" +class NativeOfflineAccelerator : public IOAcceleratorES { + OSDeclareDefaultStructors(NativeOfflineAccelerator) +}; +OSDefineMetaClassAndStructors(NativeOfflineAccelerator,IOAcceleratorES) +extern "C" IOService *native_offline_accelerator_allocate(void) {return new NativeOfflineAccelerator;} +extern "C" unsigned native_offline_accelerator_instance_size(void) {return sizeof(NativeOfflineAccelerator);} +extern "C" unsigned native_offline_accelerator_instances(void) {return NativeOfflineAccelerator::gMetaClass.getInstanceCount();} diff --git a/NativeOfflineGPU.cpp b/NativeOfflineGPU.cpp new file mode 100644 index 0000000..ae33bae --- /dev/null +++ b/NativeOfflineGPU.cpp @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: MIT +// Explicit offline GPU broker. No normal Metal/IOAccelerator publication. +// The kernel, not request bytes, selects each userclient's host namespace. +#include +#include +#include +#include +#include "NativeVirtioConsoleTransport.h" +#include "OfflineMetalSessionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalSharedEventWire.h" +#include +extern "C" { +volatile uint32_t native_gpu_broker_stage,native_gpu_broker_clients,native_gpu_broker_active,native_gpu_broker_closed; +volatile uint32_t native_gpu_broker_requests,native_gpu_broker_failures,native_gpu_broker_refusals; +volatile int native_gpu_broker_status=-1; +volatile uint64_t native_gpu_broker_reply_bytes; +volatile uint32_t native_gpu_broker_cpu_context_count; +volatile uint32_t native_gpu_shared_exports,native_gpu_shared_imports,native_gpu_shared_drops,native_gpu_shared_refusals,native_gpu_shared_tokens_live,native_gpu_shared_ports; +volatile uint64_t native_gpu_broker_cpu_contexts[8][8]; +} +class NativeOfflineGPUClient; +class NativeGPUSharedEventToken; +class NativeOfflineGPU : public IOService { + OSDeclareDefaultStructors(NativeOfflineGPU) + friend class NativeOfflineGPUClient; + friend class NativeGPUSharedEventToken; + IOService *transport=nullptr; + IOLock *gate=nullptr; + bool online=false; + uint32_t nextControlRequest=1; + IOReturn readExact(void *bytes,size_t size) { + auto p=static_cast(bytes); + while(size) { + size_t count=size>4096?4096:size; + IOReturn status=native_console_receive(transport,this,p,&count); + if(status!=kIOReturnSuccess)return status; + if(!count || count>size)return kIOReturnIOError; + p+=count;size-=count; + } + return kIOReturnSuccess; + } + // gate covers the complete actual request/reply, including every byte + // beyond a VirtIO descriptor. Userclients retain separate reply buffers. + IOReturn exchange(uint64_t session,const OMWireHeader &request,const void *input, + unsigned char **reply,size_t *length) { + *reply=nullptr;*length=0;if(!online)return kIOReturnNotReady; + OMSessionWireHeader header={request,session};header.wire.version=OFFLINE_METAL_SESSION_VERSION; + IOReturn status=native_console_send(transport,this,&header,sizeof(header)); + if(status==kIOReturnSuccess && request.bytes)status=native_console_send(transport,this,input,request.bytes); + OMSessionWireHeader response={};OMResult result={}; + if(status==kIOReturnSuccess)status=readExact(&response,sizeof(response)); + if(status==kIOReturnSuccess && (response.wire.magic!=OM_WIRE_MAGIC || response.wire.version!=OFFLINE_METAL_SESSION_VERSION || + response.wire.operation!=(request.operation|OM_WIRE_RESPONSE) || response.wire.request!=request.request || + response.session!=session || response.wire.bytesOFFLINE_METAL_MAX_REPLY))status=kIOReturnIOError; + if(status==kIOReturnSuccess)status=readExact(&result,sizeof(result)); + if(status==kIOReturnSuccess && result.reserved)status=kIOReturnIOError; + unsigned char *buffer=nullptr;size_t size=sizeof(OMWireHeader)+response.wire.bytes; + if(status==kIOReturnSuccess) { + buffer=static_cast(IOMallocData(size)); + if(!buffer)status=kIOReturnNoMemory; + } + if(status==kIOReturnSuccess) { + response.wire.version=OFFLINE_METAL_VERSION; + memcpy(buffer,&response.wire,sizeof(OMWireHeader));memcpy(buffer+sizeof(OMWireHeader),&result,sizeof(result)); + status=readExact(buffer+sizeof(OMWireHeader)+sizeof(result),response.wire.bytes-sizeof(result)); + } + native_gpu_broker_status=status; + if(status!=kIOReturnSuccess) { + if(buffer)IOFreeData(buffer,size); + // A partial stream cannot be reused for another client's reply. + online=false;native_gpu_broker_failures=native_gpu_broker_failures+1;return status; + } + native_gpu_broker_requests=native_gpu_broker_requests+1; + native_gpu_broker_reply_bytes=native_gpu_broker_reply_bytes+size; + if(result.error)native_gpu_broker_refusals=native_gpu_broker_refusals+1; + *reply=buffer;*length=size;return kIOReturnSuccess; + } + IOReturn createSession(task_t owner,uint64_t *session) { + *session=0;IOLockLock(gate); + uint64_t tcr=0,ttbr0=0,ttbr1=0,sctlr=0; + __asm__ volatile("mrs %0, tcr_el1" : "=r"(tcr)); + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(ttbr0)); + __asm__ volatile("mrs %0, ttbr1_el1" : "=r"(ttbr1)); + __asm__ volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + bool native4k=((tcr>>14)&3)==0 && ((tcr>>30)&3)==2 && (sctlr&1) && ttbr0 && ttbr1; + unsigned index=native_gpu_broker_cpu_context_count; + native_gpu_broker_cpu_context_count=index+1; + if(index<8) { + uint64_t values[8]={(uint64_t)proc_selfpid(),0,tcr,ttbr0,ttbr1,sctlr,(uintptr_t)owner,(uint64_t)native4k}; + for(unsigned n=0;n<8;n++)native_gpu_broker_cpu_contexts[index][n]=values[n]; + } + if(!native4k) { + native_gpu_broker_status=kIOReturnUnsupported;native_gpu_broker_failures=native_gpu_broker_failures+1; + IOLockUnlock(gate);return kIOReturnUnsupported; + } + OMWireHeader request={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SESSION_OPEN,0,1}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(0,request,nullptr,&reply,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result)); + if(result.error || !result.handle || length!=sizeof(OMWireHeader)+sizeof(result))status=kIOReturnNotOpen; + else { + *session=result.handle;native_gpu_broker_clients=native_gpu_broker_clients+1;native_gpu_broker_active=native_gpu_broker_active+1; + if(index<8)native_gpu_broker_cpu_contexts[index][1]=result.handle; + } + } + if(reply)IOFreeData(reply,length);IOLockUnlock(gate);return status; + } + void closeSession(uint64_t session,uint32_t number) { + IOLockLock(gate);OMWireHeader request={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SESSION_CLOSE,0,number}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(session,request,nullptr,&reply,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result)); + if(!result.error){native_gpu_broker_closed=native_gpu_broker_closed+1;native_gpu_broker_active=native_gpu_broker_active-1;} + } + if(reply)IOFreeData(reply,length);IOLockUnlock(gate); + } + void dropSharedExport(uint64_t exported) { + if(!exported)return;IOLockLock(gate); + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SHARED_EXPORT_DROP,sizeof(OMHandle),nextControlRequest};OMHandle input={exported}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(0,header,&input,&reply,&length); + if(status==kIOReturnSuccess){nextControlRequest++;OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result));if(!result.error)__atomic_fetch_add(&native_gpu_shared_drops,1,__ATOMIC_RELAXED);} + if(reply)IOFreeData(reply,length);IOLockUnlock(gate); + } +public: + bool start(IOService *provider) override { + if(!IOService::start(provider))return false; + gate=IOLockAlloc();if(!gate){IOService::stop(provider);return false;} + if(!provider->open(this)){IOLockFree(gate);gate=nullptr;IOService::stop(provider);return false;} + transport=provider;online=true;native_gpu_broker_stage=2;return true; + } + void stop(IOService *provider) override { + if(gate){IOLockLock(gate);online=false;if(transport)transport->close(this);transport=nullptr;IOLockUnlock(gate);} + IOService::stop(provider); + } + void free() override { + if(transport){transport->close(this);transport=nullptr;} + if(gate){IOLockFree(gate);gate=nullptr;}IOService::free(); + } + IOReturn newUserClient(task_t,void *,UInt32,OSDictionary *,IOUserClient **) override; +}; +// A real kernel OSObject carried exclusively by an IKOT_IOKIT_IDENT Mach +// right. No user-supplied integer can stand in for this object's authority. +class NativeGPUSharedEventToken : public OSObject { + OSDeclareDefaultStructors(NativeGPUSharedEventToken) + friend class NativeOfflineGPUClient; + NativeOfflineGPU *controller=nullptr; + uint64_t exported=0; + bool initWithExport(NativeOfflineGPU *owner,uint64_t identifier) { + if(!owner || !identifier || !OSObject::init())return false; + controller=owner;controller->retain();exported=identifier; + __atomic_fetch_add(&native_gpu_shared_tokens_live,1,__ATOMIC_RELAXED);return true; + } + void free() override { + if(controller){uint64_t identifier=exported;exported=0;controller->dropSharedExport(identifier);controller->release();controller=nullptr;__atomic_fetch_sub(&native_gpu_shared_tokens_live,1,__ATOMIC_RELAXED);} + OSObject::free(); + } +}; +class NativeOfflineGPUClient : public IOUserClient { + OSDeclareDefaultStructors(NativeOfflineGPUClient) + NativeOfflineGPU *controller=nullptr; + task_t owner=nullptr; // The native IOUserClient base owns this task reference. + IOLock *gate=nullptr; + uint64_t session=0; + uint32_t nextRequest=1,nextHostRequest=1; + unsigned char *reply=nullptr;size_t replyLength=0,replyOffset=0; + void clearReply() {if(reply)IOFreeData(reply,replyLength);reply=nullptr;replyLength=replyOffset=0;} + void closeContext() { + if(gate)IOLockLock(gate); + clearReply();if(session && controller)controller->closeSession(session,nextHostRequest); + session=0;controller=nullptr;if(gate)IOLockUnlock(gate); + } + static IOReturn sendAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(args->structureInputDescriptor)return kIOReturnUnsupported; + if(!args->structureInput || args->structureInputSizestructureInputSize>4096)return kIOReturnBadArgument; + OMWireHeader header={};memcpy(&header,args->structureInput,sizeof(header)); + IOLockLock(self->gate);IOReturn status=kIOReturnNotOpen; + if(self->session && self->controller) { + if(self->replyOffset!=self->replyLength)status=kIOReturnBusy; + else if(header.magic!=OM_WIRE_MAGIC || header.version!=OFFLINE_METAL_VERSION || header.request!=self->nextRequest || + header.bytes+sizeof(header)!=args->structureInputSize || header.bytes>4080 || + header.operationOM_SHARED_LABEL_SET || (header.operation==OM_FINISH && header.bytes))status=kIOReturnBadArgument; + else { + self->clearReply();uint16_t operation=header.operation;if(operation==OM_FINISH)header.operation=OM_SESSION_CLOSE; + IOLockLock(self->controller->gate); + OMWireHeader hostHeader=header;hostHeader.request=self->nextHostRequest; + status=self->controller->exchange(self->session,hostHeader,static_cast(args->structureInput)+sizeof(header),&self->reply,&self->replyLength); + if(status==kIOReturnSuccess) { + self->nextRequest++;self->nextHostRequest++; + // Preserve the real namespace-close result in the client's + // own framing; no host-wide Finish is forwarded. + header.operation=operation|OM_WIRE_RESPONSE;header.bytes=(uint32_t)(self->replyLength-sizeof(header)); + memcpy(self->reply,&header,sizeof(header));OMResult result={};memcpy(&result,self->reply+sizeof(header),sizeof(result)); + if(operation==OM_FINISH && !result.error) { + self->session=0;native_gpu_broker_closed=native_gpu_broker_closed+1;native_gpu_broker_active=native_gpu_broker_active-1; + } + } + IOLockUnlock(self->controller->gate); + } + } + IOLockUnlock(self->gate);return status; + } + static IOReturn receiveAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(args->structureOutputDescriptor)return kIOReturnUnsupported; + if(!args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + IOLockLock(self->gate);size_t available=self->replyLength-self->replyOffset; + size_t count=args->structureOutputSize;if(count>available)count=available; + if(count){memcpy(args->structureOutput,self->reply+self->replyOffset,count);self->replyOffset+=count;} + args->structureOutputSize=(uint32_t)count;IOLockUnlock(self->gate);return count?kIOReturnSuccess:kIOReturnNotReady; + } + IOReturn sharedExchange(uint16_t operation,uint64_t identifier,unsigned char **output,size_t *length) { + *output=nullptr;*length=0; + if(!session || !controller)return kIOReturnNotOpen; + if(replyOffset!=replyLength)return kIOReturnBusy; + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,operation,sizeof(OMHandle),nextHostRequest};OMHandle input={identifier}; + IOLockLock(controller->gate);IOReturn status=controller->exchange(session,header,&input,output,length); + if(status==kIOReturnSuccess)nextHostRequest++;IOLockUnlock(controller->gate);return status; + } + static IOReturn exportSharedAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(current_task()!=self->owner || args->structureOutputDescriptor || !args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + args->scalarOutput[0]=0;IOLockLock(self->gate);unsigned char *bytes=nullptr;size_t length=0; + IOReturn status=self->sharedExchange(OM_SHARED_EXPORT,args->scalarInput[0],&bytes,&length);OMResult result={}; + if(status==kIOReturnSuccess) { + memcpy(&result,bytes+sizeof(OMWireHeader),sizeof(result));size_t prefix=sizeof(OMWireHeader)+sizeof(result); + if(result.error || !result.handle)status=kIOReturnBadArgument; + else { + __atomic_fetch_add(&native_gpu_shared_exports,1,__ATOMIC_RELAXED); + size_t count=length-prefix; + if(!count || count>args->structureOutputSize){self->controller->dropSharedExport(result.handle);status=kIOReturnNoSpace;} + else { + auto token=new NativeGPUSharedEventToken; + if(!token || !token->initWithExport(self->controller,result.handle)) { + OSSafeReleaseNULL(token);self->controller->dropSharedExport(result.handle);status=kIOReturnNoMemory; + } else { + // Kernel MACH_PORT_NULL is an ipc_port pointer, while + // this API returns a name in the owner's namespace. + mach_port_name_t name=0;status=IOUserClient::copyPortNameForObjectInTask(self->owner,token,&name); + if(status==kIOReturnSuccess && name && name!=MACH_PORT_DEAD) { + args->scalarOutput[0]=name;memcpy(args->structureOutput,bytes+prefix,count);args->structureOutputSize=(uint32_t)count; + __atomic_fetch_add(&native_gpu_shared_ports,1,__ATOMIC_RELAXED); + } else if(status==kIOReturnSuccess)status=kIOReturnNoResources; + token->release(); + } + } + } + } + if(bytes)IOFreeData(bytes,length);IOLockUnlock(self->gate); + if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gpu_shared_refusals,1,__ATOMIC_RELAXED);return status; + } + static IOReturn importSharedAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(current_task()!=self->owner || args->scalarInput[0]>UINT32_MAX || args->structureOutputDescriptor || !args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + args->scalarOutput[0]=0;OSObject *object=nullptr;IOReturn status=IOUserClient::copyObjectForPortNameInTask(self->owner,(mach_port_name_t)args->scalarInput[0],&object); + auto token=OSDynamicCast(NativeGPUSharedEventToken,object);unsigned char *bytes=nullptr;size_t length=0;IOLockLock(self->gate); + if(status!=kIOReturnSuccess || !token || token->controller!=self->controller || !token->exported)status=kIOReturnBadArgument; + else { + status=self->sharedExchange(OM_SHARED_IMPORT,token->exported,&bytes,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,bytes+sizeof(OMWireHeader),sizeof(result));size_t prefix=sizeof(OMWireHeader)+sizeof(result),count=length-prefix; + if(result.error || !result.handle)status=kIOReturnBadArgument; + else if(!count || count>args->structureOutputSize) { + // Retire the just-created independent event handle through + // the same real session without consuming a user sequence. + unsigned char *released=nullptr;size_t releasedLength=0; + (void)self->sharedExchange(OM_RELEASE,result.handle,&released,&releasedLength);if(released)IOFreeData(released,releasedLength);status=kIOReturnNoSpace; + } else {args->scalarOutput[0]=result.handle;memcpy(args->structureOutput,bytes+prefix,count);args->structureOutputSize=(uint32_t)count;__atomic_fetch_add(&native_gpu_shared_imports,1,__ATOMIC_RELAXED);} + } + } + if(bytes)IOFreeData(bytes,length);OSSafeReleaseNULL(object);IOLockUnlock(self->gate); + if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gpu_shared_refusals,1,__ATOMIC_RELAXED);return status; + } +public: + bool initWithTask(task_t task,void *security,UInt32 type,OSDictionary *properties) override { + if(type || clientHasPrivilege(security,kIOClientPrivilegeAdministrator)!=kIOReturnSuccess || + !IOUserClient::initWithTask(task,security,type,properties))return false; + owner=task;gate=IOLockAlloc();return gate!=nullptr; + } + bool start(IOService *provider) override { + controller=OSDynamicCast(NativeOfflineGPU,provider); + if(!controller || !IOUserClient::start(provider))return false; + if(controller->createSession(owner,&session)!=kIOReturnSuccess){controller=nullptr;IOUserClient::stop(provider);return false;} + return true; + } + void stop(IOService *provider) override {closeContext();IOUserClient::stop(provider);} + void free() override {closeContext();if(gate){IOLockFree(gate);gate=nullptr;}IOUserClient::free();} + IOReturn clientClose() override {terminate();return kIOReturnSuccess;} + IOReturn externalMethod(uint32_t selector,IOExternalMethodArguments *args,IOExternalMethodDispatch *,OSObject *,void *) override { + static const IOExternalMethodDispatch methods[]={{sendAction,0,kIOUCVariableStructureSize,0,0},{receiveAction,0,0,0,kIOUCVariableStructureSize}, + {exportSharedAction,1,0,1,kIOUCVariableStructureSize},{importSharedAction,1,0,1,kIOUCVariableStructureSize}}; + if(selector>=4)return kIOReturnUnsupported; + return IOUserClient::externalMethod(selector,args,const_cast(methods+selector),this,nullptr); + } +}; +OSDefineMetaClassAndStructors(NativeOfflineGPU,IOService) +OSDefineMetaClassAndStructors(NativeOfflineGPUClient,IOUserClient) +OSDefineMetaClassAndStructors(NativeGPUSharedEventToken,OSObject) +IOReturn NativeOfflineGPU::newUserClient(task_t task,void *security,UInt32 type,OSDictionary *properties,IOUserClient **out) { + if(!out)return kIOReturnBadArgument;*out=nullptr; + auto client=new NativeOfflineGPUClient;if(!client)return kIOReturnNoMemory; + bool attached=false; + if(!client->initWithTask(task,security,type,properties) || !(attached=client->attach(this)) || !client->start(this)) { + if(attached)client->detach(this);client->release();return kIOReturnNotOpen; + } + *out=client;return kIOReturnSuccess; +} +extern "C" IOService *native_offline_gpu_create(void) {return new NativeOfflineGPU;} diff --git a/NativeTurnipSmoke.h b/NativeTurnipSmoke.h new file mode 100644 index 0000000..8450ee7 --- /dev/null +++ b/NativeTurnipSmoke.h @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: MIT +// Turnip smoke test, run by the Adreno Metal plugin from inside the system process that loaded it (under that process's sandbox) once the raw +// userspace test (NativeAdrenoUserTest.h) has passed. It loads the real Mesa Turnip driver (libvulkan_freedreno.dylib, built with the native +// Darwin kernel backend and shipped next to the plugin binary) and drives it through the Vulkan ICD entry points without a loader: +// instance, physical device, device, queue, a host-visible buffer, one vkCmdFillBuffer, a fence, and a read-back of the fill pattern. +// Every stage leaves a progress marker (IOServiceOpen type 0x4147, see NativeAdrenoUserTest.h): 6001 library loaded, 6002 entry points, +// 6003 instance, 6004 physical device, 6005 device, 6007 buffer memory mapped, 6006 command buffer recorded, 6008 submitted, 6009 fence +// signalled, 600a fill pattern verified; then the shader tests: 6100 compute begins, 6101 pipeline, 6102 recorded, 6103 fence, 6104 all words +// right (61fS = compute step S failed); 6200 render begins, 6201 pipeline, 6202 recorded, 6203 fence, 6204 image right (62fS = render step S +// failed); 600b everything destroyed. A setup failure leaves 6fSS (stage SS) followed by 6eRR (RR = -VkResult). +// The outcome is a short text for the kernel's report row: +// "T s r v c g/ ". +#pragma once +#define VK_NO_PROTOTYPES 1 +#include +#include "NativeTurnipShaders.h" // generated: scripts/gen-turnip-smoke-shaders.py +#include +#include +#include +#include +#include +#include +#include +#include + +// Vulkan create-info structs are written as {sType} with every other member zero. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmissing-field-initializers" +namespace turnip { + +static char gFirstError[96]; + +static VKAPI_ATTR VkBool32 VKAPI_CALL onMessage(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT, + const VkDebugUtilsMessengerCallbackDataEXT *data, void *) { + if ((severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) && !gFirstError[0] && data && data->pMessage) + snprintf(gFirstError, sizeof gFirstError, "%s", data->pMessage); + return VK_FALSE; +} + +struct Result { + int stage = 0; // first setup failure (0 = none): 1 library .. 9 fence wait + int vkResult = 0; + int verified = -1; // fill test: words of 1024 that hold the expected value (-1 = not run) + int computeWords = -1; // compute test: words of 1024 written by the shader with the expected value (-1 = not run) + int computeStage = 0; // first failing step of the compute test (0 = none) + int renderTriangle = -1, renderOther = -1; // render test: triangle-colour pixels, pixels that are neither colour (-1 = not run) + int renderStage = 0; + bool ok = false; +}; + +// Device-level entry points the tests use, resolved once through vkGetDeviceProcAddr. +#define TURNIP_DEVICE_FNS(X) \ + X(vkDestroyDevice) X(vkGetDeviceQueue) X(vkCreateCommandPool) X(vkDestroyCommandPool) X(vkAllocateCommandBuffers) \ + X(vkResetCommandBuffer) X(vkBeginCommandBuffer) X(vkEndCommandBuffer) X(vkCmdFillBuffer) X(vkCreateBuffer) X(vkDestroyBuffer) \ + X(vkGetBufferMemoryRequirements) X(vkAllocateMemory) X(vkFreeMemory) X(vkBindBufferMemory) X(vkMapMemory) X(vkUnmapMemory) \ + X(vkCreateFence) X(vkDestroyFence) X(vkQueueSubmit) X(vkWaitForFences) X(vkDeviceWaitIdle) X(vkCreateShaderModule) \ + X(vkDestroyShaderModule) X(vkCreateDescriptorSetLayout) X(vkDestroyDescriptorSetLayout) X(vkCreatePipelineLayout) \ + X(vkDestroyPipelineLayout) X(vkCreateComputePipelines) X(vkCreateGraphicsPipelines) X(vkDestroyPipeline) \ + X(vkCreateDescriptorPool) X(vkDestroyDescriptorPool) X(vkAllocateDescriptorSets) X(vkUpdateDescriptorSets) \ + X(vkCmdBindPipeline) X(vkCmdBindDescriptorSets) X(vkCmdPushConstants) X(vkCmdDispatch) X(vkCmdPipelineBarrier) \ + X(vkCreateImage) X(vkDestroyImage) X(vkGetImageMemoryRequirements) X(vkBindImageMemory) X(vkCreateImageView) \ + X(vkDestroyImageView) X(vkCreateRenderPass) X(vkDestroyRenderPass) X(vkCreateFramebuffer) X(vkDestroyFramebuffer) \ + X(vkCmdBeginRenderPass) X(vkCmdEndRenderPass) X(vkCmdDraw) X(vkCmdCopyImageToBuffer) + +struct Device { + VkDevice device = nullptr; + VkQueue queue = nullptr; + VkCommandPool pool = nullptr; + VkCommandBuffer cmd = nullptr; + VkPhysicalDeviceMemoryProperties memory = {}; +#define TURNIP_DECLARE(name) PFN_##name name = nullptr; + TURNIP_DEVICE_FNS(TURNIP_DECLARE) +#undef TURNIP_DECLARE + + uint32_t memoryType(uint32_t bits, VkMemoryPropertyFlags flags) const { + for (uint32_t i = 0; i < memory.memoryTypeCount; ++i) + if ((bits & (1u << i)) && (memory.memoryTypes[i].propertyFlags & flags) == flags) return i; + return UINT32_MAX; + } + // Records into the one command buffer, submits it with a fresh fence and waits up to five seconds. + VkResult submitAndWait() { + VkFenceCreateInfo fci = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence = nullptr; + VkResult r = vkCreateFence(device, &fci, nullptr, &fence); + if (r != VK_SUCCESS) return r; + VkSubmitInfo si = {VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.commandBufferCount = 1; + si.pCommandBuffers = &cmd; + r = vkQueueSubmit(queue, 1, &si, fence); + if (r == VK_SUCCESS) r = vkWaitForFences(device, 1, &fence, VK_TRUE, 5000000000ull); + vkDestroyFence(device, fence, nullptr); + return r; + } + VkResult begin() { + vkResetCommandBuffer(cmd, 0); + VkCommandBufferBeginInfo bi = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + return vkBeginCommandBuffer(cmd, &bi); + } +}; + +// A host-visible buffer with its memory, mapped for the CPU. +struct HostBuffer { + VkBuffer buffer = nullptr; + VkDeviceMemory memory = nullptr; + volatile uint32_t *words = nullptr; + VkResult create(Device &d, VkDeviceSize bytes, VkBufferUsageFlags usage) { + VkBufferCreateInfo bci = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bci.size = bytes; + bci.usage = usage; + bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VkResult r = d.vkCreateBuffer(d.device, &bci, nullptr, &buffer); + if (r != VK_SUCCESS) return r; + VkMemoryRequirements req = {}; + d.vkGetBufferMemoryRequirements(d.device, buffer, &req); + const uint32_t type = d.memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (type == UINT32_MAX) return VK_ERROR_FEATURE_NOT_PRESENT; + VkMemoryAllocateInfo mai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = req.size; + mai.memoryTypeIndex = type; + r = d.vkAllocateMemory(d.device, &mai, nullptr, &memory); + if (r == VK_SUCCESS) r = d.vkBindBufferMemory(d.device, buffer, memory, 0); + void *mapped = nullptr; + if (r == VK_SUCCESS) r = d.vkMapMemory(d.device, memory, 0, bytes, 0, &mapped); + words = static_cast(mapped); + return r != VK_SUCCESS ? r : (words ? VK_SUCCESS : VK_ERROR_MEMORY_MAP_FAILED); + } + void destroy(Device &d) { + if (words) d.vkUnmapMemory(d.device, memory); + if (buffer) d.vkDestroyBuffer(d.device, buffer, nullptr); + if (memory) d.vkFreeMemory(d.device, memory, nullptr); + *this = HostBuffer(); + } +}; + +static VkShaderModule shaderModule(Device &d, const uint32_t *words, size_t bytes) { + VkShaderModuleCreateInfo ci = {VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + ci.codeSize = bytes; + ci.pCode = words; + VkShaderModule module = nullptr; + return d.vkCreateShaderModule(d.device, &ci, nullptr, &module) == VK_SUCCESS ? module : nullptr; +} + +static void hostBarrier(Device &d, VkPipelineStageFlags from, VkAccessFlags access) { + VkMemoryBarrier barrier = {VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = access; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + d.vkCmdPipelineBarrier(d.cmd, from, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +// Fill: vkCmdFillBuffer of the first half of a 4 KiB buffer (Turnip implements it with its own internal blit). Returns verified words or < 0. +template +static int fillTest(Device &d, Marker &mark, int &failStage, VkResult &failResult) { + constexpr VkDeviceSize kBytes = 4096; + constexpr uint32_t kPattern = 0xc0ffee42u; + HostBuffer b; + VkResult r = b.create(d, kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (r != VK_SUCCESS) {b.destroy(d); failStage = 7; failResult = r; return -1;} + for (unsigned i = 0; i < kBytes / 4; ++i) b.words[i] = 0xdeadbeef; + mark(0x6007); + if ((r = d.begin()) != VK_SUCCESS) {b.destroy(d); failStage = 6; failResult = r; return -1;} + d.vkCmdFillBuffer(d.cmd, b.buffer, 0, kBytes / 2, kPattern); // the first half only: the second half must keep 0xdeadbeef + hostBarrier(d, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + if ((r = d.vkEndCommandBuffer(d.cmd)) != VK_SUCCESS) {b.destroy(d); failStage = 6; failResult = r; return -1;} + mark(0x6006); + mark(0x6008); + if ((r = d.submitAndWait()) != VK_SUCCESS) {b.destroy(d); failStage = 9; failResult = r; return -1;} + mark(0x6009); + int good = 0; + for (unsigned i = 0; i < kBytes / 8; ++i) good += b.words[i] == kPattern; + for (unsigned i = kBytes / 8; i < kBytes / 4; ++i) good += b.words[i] == 0xdeadbeefu; + if (good == int(kBytes / 4)) mark(0x600a); + b.destroy(d); + return good; +} + +// Compute: 16 workgroups of 64 invocations each write v[i] = i * 3 + 0x1000 (push constants) into a storage buffer. +template +static int computeTest(Device &d, Marker &mark, int &stage) { + constexpr uint32_t kWords = 1024, kMul = 3, kAdd = 0x1000; + mark(0x6100); + HostBuffer b; + VkShaderModule module = nullptr; + VkDescriptorSetLayout setLayout = nullptr; + VkPipelineLayout layout = nullptr; + VkPipeline pipeline = nullptr; + VkDescriptorPool pool = nullptr; + int result = -1; + auto fail = [&](int s) {stage = s; mark(0x61f0 | s);}; + do { + if (b.create(d, kWords * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != VK_SUCCESS) {fail(1); break;} + for (uint32_t i = 0; i < kWords; ++i) b.words[i] = 0xdeadbeef; + if (!(module = shaderModule(d, kSpv_smoke_fill_comp, sizeof kSpv_smoke_fill_comp))) {fail(2); break;} + VkDescriptorSetLayoutBinding binding = {0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + VkDescriptorSetLayoutCreateInfo dsl = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + dsl.bindingCount = 1; + dsl.pBindings = &binding; + if (d.vkCreateDescriptorSetLayout(d.device, &dsl, nullptr, &setLayout) != VK_SUCCESS) {fail(3); break;} + VkPushConstantRange push = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8}; + VkPipelineLayoutCreateInfo pl = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.setLayoutCount = 1; + pl.pSetLayouts = &setLayout; + pl.pushConstantRangeCount = 1; + pl.pPushConstantRanges = &push; + if (d.vkCreatePipelineLayout(d.device, &pl, nullptr, &layout) != VK_SUCCESS) {fail(4); break;} + VkComputePipelineCreateInfo cp = {VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + cp.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_COMPUTE_BIT, module, "main", nullptr}; + cp.layout = layout; + if (d.vkCreateComputePipelines(d.device, nullptr, 1, &cp, nullptr, &pipeline) != VK_SUCCESS) {fail(5); break;} + mark(0x6101); + VkDescriptorPoolSize size = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; + VkDescriptorPoolCreateInfo dp = {VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dp.maxSets = 1; + dp.poolSizeCount = 1; + dp.pPoolSizes = &size; + if (d.vkCreateDescriptorPool(d.device, &dp, nullptr, &pool) != VK_SUCCESS) {fail(6); break;} + VkDescriptorSetAllocateInfo da = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + da.descriptorPool = pool; + da.descriptorSetCount = 1; + da.pSetLayouts = &setLayout; + VkDescriptorSet set = nullptr; + if (d.vkAllocateDescriptorSets(d.device, &da, &set) != VK_SUCCESS) {fail(6); break;} + VkDescriptorBufferInfo info = {b.buffer, 0, kWords * 4}; + VkWriteDescriptorSet write = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = set; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &info; + d.vkUpdateDescriptorSets(d.device, 1, &write, 0, nullptr); + if (d.begin() != VK_SUCCESS) {fail(7); break;} + const uint32_t constants[2] = {kMul, kAdd}; + d.vkCmdBindPipeline(d.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + d.vkCmdBindDescriptorSets(d.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 1, &set, 0, nullptr); + d.vkCmdPushConstants(d.cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof constants, constants); + d.vkCmdDispatch(d.cmd, kWords / 64, 1, 1); + hostBarrier(d, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT); + if (d.vkEndCommandBuffer(d.cmd) != VK_SUCCESS) {fail(7); break;} + mark(0x6102); + if (d.submitAndWait() != VK_SUCCESS) {fail(8); break;} + mark(0x6103); + int good = 0; + for (uint32_t i = 0; i < kWords; ++i) good += b.words[i] == i * kMul + kAdd; + result = good; + if (good == int(kWords)) mark(0x6104); + } while (false); + if (pipeline) d.vkDestroyPipeline(d.device, pipeline, nullptr); + if (pool) d.vkDestroyDescriptorPool(d.device, pool, nullptr); + if (layout) d.vkDestroyPipelineLayout(d.device, layout, nullptr); + if (setLayout) d.vkDestroyDescriptorSetLayout(d.device, setLayout, nullptr); + if (module) d.vkDestroyShaderModule(d.device, module, nullptr); + b.destroy(d); + return result; +} + +// Render: a 64x64 RGBA8 colour attachment cleared to blue, one triangle drawn in orange (push constant), copied to a host buffer. +// The triangle covers 512 pixels in exact arithmetic; every other pixel must keep the clear colour. +template +static void renderTest(Device &d, Marker &mark, int &stage, int &triangle, int &other) { + constexpr uint32_t kSize = 64; + mark(0x6200); + HostBuffer b; + VkImage image = nullptr; + VkDeviceMemory imageMemory = nullptr; + VkImageView view = nullptr; + VkRenderPass pass = nullptr; + VkFramebuffer framebuffer = nullptr; + VkShaderModule vs = nullptr, fs = nullptr; + VkPipelineLayout layout = nullptr; + VkPipeline pipeline = nullptr; + auto fail = [&](int s) {stage = s; mark(0x62f0 | s);}; + do { + if (b.create(d, kSize * kSize * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT) != VK_SUCCESS) {fail(1); break;} + for (uint32_t i = 0; i < kSize * kSize; ++i) b.words[i] = 0x5a5a5a5a; + VkImageCreateInfo ic = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + ic.imageType = VK_IMAGE_TYPE_2D; + ic.format = VK_FORMAT_R8G8B8A8_UNORM; + ic.extent = {kSize, kSize, 1}; + ic.mipLevels = 1; + ic.arrayLayers = 1; + ic.samples = VK_SAMPLE_COUNT_1_BIT; + ic.tiling = VK_IMAGE_TILING_OPTIMAL; + ic.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + ic.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + if (d.vkCreateImage(d.device, &ic, nullptr, &image) != VK_SUCCESS) {fail(2); break;} + VkMemoryRequirements req = {}; + d.vkGetImageMemoryRequirements(d.device, image, &req); + uint32_t type = d.memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + if (type == UINT32_MAX) type = d.memoryType(req.memoryTypeBits, 0); + VkMemoryAllocateInfo mai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = req.size; + mai.memoryTypeIndex = type; + if (type == UINT32_MAX || d.vkAllocateMemory(d.device, &mai, nullptr, &imageMemory) != VK_SUCCESS || + d.vkBindImageMemory(d.device, image, imageMemory, 0) != VK_SUCCESS) {fail(2); break;} + VkImageViewCreateInfo vc = {VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + vc.image = image; + vc.viewType = VK_IMAGE_VIEW_TYPE_2D; + vc.format = ic.format; + vc.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + if (d.vkCreateImageView(d.device, &vc, nullptr, &view) != VK_SUCCESS) {fail(3); break;} + VkAttachmentDescription attachment = {0, ic.format, VK_SAMPLE_COUNT_1_BIT, VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE, + VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_DONT_CARE, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL}; + VkAttachmentReference ref = {0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass = {}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &ref; + VkSubpassDependency after = {0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, 0}; + VkRenderPassCreateInfo rp = {VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; + rp.attachmentCount = 1; + rp.pAttachments = &attachment; + rp.subpassCount = 1; + rp.pSubpasses = &subpass; + rp.dependencyCount = 1; + rp.pDependencies = &after; + if (d.vkCreateRenderPass(d.device, &rp, nullptr, &pass) != VK_SUCCESS) {fail(4); break;} + VkFramebufferCreateInfo fb = {VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + fb.renderPass = pass; + fb.attachmentCount = 1; + fb.pAttachments = &view; + fb.width = kSize; + fb.height = kSize; + fb.layers = 1; + if (d.vkCreateFramebuffer(d.device, &fb, nullptr, &framebuffer) != VK_SUCCESS) {fail(4); break;} + vs = shaderModule(d, kSpv_smoke_triangle_vert, sizeof kSpv_smoke_triangle_vert); + fs = shaderModule(d, kSpv_smoke_triangle_frag, sizeof kSpv_smoke_triangle_frag); + if (!vs || !fs) {fail(5); break;} + VkPushConstantRange push = {VK_SHADER_STAGE_FRAGMENT_BIT, 0, 16}; + VkPipelineLayoutCreateInfo pl = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.pushConstantRangeCount = 1; + pl.pPushConstantRanges = &push; + if (d.vkCreatePipelineLayout(d.device, &pl, nullptr, &layout) != VK_SUCCESS) {fail(6); break;} + VkPipelineShaderStageCreateInfo stages[2] = { + {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_VERTEX_BIT, vs, "main", nullptr}, + {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_FRAGMENT_BIT, fs, "main", nullptr}}; + VkPipelineVertexInputStateCreateInfo vi = {VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; + VkPipelineInputAssemblyStateCreateInfo ia = {VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; + ia.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + VkViewport viewport = {0, 0, float(kSize), float(kSize), 0, 1}; + VkRect2D scissor = {{0, 0}, {kSize, kSize}}; + VkPipelineViewportStateCreateInfo vp = {VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; + vp.viewportCount = 1; + vp.pViewports = &viewport; + vp.scissorCount = 1; + vp.pScissors = &scissor; + VkPipelineRasterizationStateCreateInfo rs = {VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; + rs.polygonMode = VK_POLYGON_MODE_FILL; + rs.cullMode = VK_CULL_MODE_NONE; + rs.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + rs.lineWidth = 1.0f; + VkPipelineMultisampleStateCreateInfo ms = {VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; + ms.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + VkPipelineColorBlendAttachmentState blend = {}; + blend.colorWriteMask = 0xf; + VkPipelineColorBlendStateCreateInfo cb = {VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; + cb.attachmentCount = 1; + cb.pAttachments = &blend; + VkGraphicsPipelineCreateInfo gp = {VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; + gp.stageCount = 2; + gp.pStages = stages; + gp.pVertexInputState = &vi; + gp.pInputAssemblyState = &ia; + gp.pViewportState = &vp; + gp.pRasterizationState = &rs; + gp.pMultisampleState = &ms; + gp.pColorBlendState = &cb; + gp.layout = layout; + gp.renderPass = pass; + if (d.vkCreateGraphicsPipelines(d.device, nullptr, 1, &gp, nullptr, &pipeline) != VK_SUCCESS) {fail(7); break;} + mark(0x6201); + if (d.begin() != VK_SUCCESS) {fail(8); break;} + VkClearValue clear = {}; + clear.color.float32[2] = 1.0f; + clear.color.float32[3] = 1.0f; + VkRenderPassBeginInfo rb = {VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; + rb.renderPass = pass; + rb.framebuffer = framebuffer; + rb.renderArea = scissor; + rb.clearValueCount = 1; + rb.pClearValues = &clear; + d.vkCmdBeginRenderPass(d.cmd, &rb, VK_SUBPASS_CONTENTS_INLINE); + d.vkCmdBindPipeline(d.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + const float color[4] = {1.0f, 0.5f, 0.0f, 1.0f}; + d.vkCmdPushConstants(d.cmd, layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof color, color); + d.vkCmdDraw(d.cmd, 3, 1, 0, 0); + d.vkCmdEndRenderPass(d.cmd); + VkBufferImageCopy copy = {}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {kSize, kSize, 1}; + d.vkCmdCopyImageToBuffer(d.cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, b.buffer, 1, ©); + hostBarrier(d, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + if (d.vkEndCommandBuffer(d.cmd) != VK_SUCCESS) {fail(8); break;} + mark(0x6202); + if (d.submitAndWait() != VK_SUCCESS) {fail(9); break;} + mark(0x6203); + // RGBA8 little-endian words: clear = (0, 0, 255, 255), triangle = (255, 127 or 128, 0, 255). + int tri = 0, rest = 0; + for (uint32_t i = 0; i < kSize * kSize; ++i) { + const uint32_t px = b.words[i]; + if (px == 0xffff0000u) continue; + if (px == 0xff0080ffu || px == 0xff007fffu) ++tri; + else ++rest; + } + triangle = tri; + other = rest; + if (rest == 0 && tri >= 480 && tri <= 544) mark(0x6204); + } while (false); + if (pipeline) d.vkDestroyPipeline(d.device, pipeline, nullptr); + if (layout) d.vkDestroyPipelineLayout(d.device, layout, nullptr); + if (vs) d.vkDestroyShaderModule(d.device, vs, nullptr); + if (fs) d.vkDestroyShaderModule(d.device, fs, nullptr); + if (framebuffer) d.vkDestroyFramebuffer(d.device, framebuffer, nullptr); + if (pass) d.vkDestroyRenderPass(d.device, pass, nullptr); + if (view) d.vkDestroyImageView(d.device, view, nullptr); + if (image) d.vkDestroyImage(d.device, image, nullptr); + if (imageMemory) d.vkFreeMemory(d.device, imageMemory, nullptr); + b.destroy(d); +} + +template +static Result run(Marker &&mark, const char *libraryPath) { + Result out; + auto fail = [&](int stage, int result) { + out.stage = stage; + out.vkResult = result; + mark(0x6f00 | stage); + mark(0x6e00 | ((-result) & 0xff)); + return out; + }; + // The Darwin backend reaches the driver the same way the plugin does: the AGX accelerator service with open type 0x4147, ABI methods through + // an admitted selector. Nothing to configure; set here only so the choice is visible. The shader disk cache is off: this runs inside a + // system process whose home directory is not ours to fill. + setenv("TU_DARWIN_SERVICE", "IOAcceleratorES", 0); + setenv("MESA_SHADER_CACHE_DISABLE", "true", 0); + // Concurrent binning (BV running ahead of BR) is a performance feature; its barriers make BR wait for BV timestamps. int-058's + // first fill hung with the ring stopped inside the IB (no fault), before the kernel framed IBs for both threads. Keep it off + // until a run with the framing has passed. + setenv("TU_DEBUG", "nocb", 0); + + void *lib = dlopen(libraryPath, RTLD_NOW | RTLD_LOCAL); + if (!lib) return fail(1, -1); + mark(0x6001); + auto gipa = reinterpret_cast(dlsym(lib, "vk_icdGetInstanceProcAddr")); + if (!gipa) return fail(2, -2); + auto createInstance = reinterpret_cast(gipa(nullptr, "vkCreateInstance")); + if (!createInstance) return fail(2, -3); + mark(0x6002); + + const char *extensions[] = {VK_EXT_DEBUG_UTILS_EXTENSION_NAME}; + VkDebugUtilsMessengerCreateInfoEXT messenger = {VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT}; + messenger.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; + messenger.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT; + messenger.pfnUserCallback = onMessage; + VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; + app.pApplicationName = "NativeAdrenoMetal"; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo ici = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + ici.pNext = &messenger; + ici.pApplicationInfo = &app; + ici.enabledExtensionCount = 1; + ici.ppEnabledExtensionNames = extensions; + VkInstance instance = nullptr; + VkResult r = createInstance(&ici, nullptr, &instance); + if (r != VK_SUCCESS) return fail(3, r); + mark(0x6003); + +#define INSTANCE_FN(name) auto name = reinterpret_cast(gipa(instance, #name)); if (!name) return fail(3, -4) + INSTANCE_FN(vkEnumeratePhysicalDevices); + INSTANCE_FN(vkGetPhysicalDeviceMemoryProperties); + INSTANCE_FN(vkGetPhysicalDeviceQueueFamilyProperties); + INSTANCE_FN(vkCreateDevice); + INSTANCE_FN(vkGetDeviceProcAddr); + INSTANCE_FN(vkDestroyInstance); +#undef INSTANCE_FN + + uint32_t count = 1; + VkPhysicalDevice physical = nullptr; + r = vkEnumeratePhysicalDevices(instance, &count, &physical); + if ((r != VK_SUCCESS && r != VK_INCOMPLETE) || !count || !physical) return fail(4, r == VK_SUCCESS ? -5 : r); + mark(0x6004); + + uint32_t families = 0; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, nullptr); + if (!families) return fail(4, -6); + VkQueueFamilyProperties family[8] = {}; + if (families > 8) families = 8; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, family); + uint32_t queueFamily = families; + for (uint32_t i = 0; i < families && queueFamily == families; ++i) + if (family[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) queueFamily = i; + if (queueFamily == families) return fail(4, -7); + + float priority = 1.0f; + VkDeviceQueueCreateInfo qci = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + qci.queueFamilyIndex = queueFamily; + qci.queueCount = 1; + qci.pQueuePriorities = &priority; + VkDeviceCreateInfo dci = {VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + dci.queueCreateInfoCount = 1; + dci.pQueueCreateInfos = &qci; + Device d; + r = vkCreateDevice(physical, &dci, nullptr, &d.device); + if (r != VK_SUCCESS) return fail(5, r); + mark(0x6005); +#define TURNIP_RESOLVE(name) d.name = reinterpret_cast(vkGetDeviceProcAddr(d.device, #name)); if (!d.name) return fail(5, -8); + TURNIP_DEVICE_FNS(TURNIP_RESOLVE) +#undef TURNIP_RESOLVE + vkGetPhysicalDeviceMemoryProperties(physical, &d.memory); + d.vkGetDeviceQueue(d.device, queueFamily, 0, &d.queue); + VkCommandPoolCreateInfo pci = {VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pci.queueFamilyIndex = queueFamily; + r = d.vkCreateCommandPool(d.device, &pci, nullptr, &d.pool); + if (r != VK_SUCCESS) return fail(6, r); + VkCommandBufferAllocateInfo cai = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + cai.commandPool = d.pool; + cai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cai.commandBufferCount = 1; + r = d.vkAllocateCommandBuffers(d.device, &cai, &d.cmd); + if (r != VK_SUCCESS) return fail(6, r); + + // The three tests are independent: a failed or unverified one does not stop the next, unless the GPU stopped retiring work (fence timeout). + int failStage = 0; + VkResult failResult = VK_SUCCESS; + out.verified = fillTest(d, mark, failStage, failResult); + if (failStage) { + fail(failStage, failResult); + } + if (failStage != 9) { + out.computeWords = computeTest(d, mark, out.computeStage); + if (out.computeStage != 8) renderTest(d, mark, out.renderStage, out.renderTriangle, out.renderOther); + } + + d.vkDeviceWaitIdle(d.device); + d.vkDestroyCommandPool(d.device, d.pool, nullptr); + d.vkDestroyDevice(d.device, nullptr); + vkDestroyInstance(instance, nullptr); + mark(0x600b); + out.ok = !failStage && out.verified == 1024 && out.computeWords == 1024 && out.renderOther == 0 && out.renderTriangle >= 480 && + out.renderTriangle <= 544; + return out; +} + +// Watchdog. run() executes on a worker thread; if it has not returned after 15, 45 and 90 seconds, the calling thread samples the worker and +// sends "W s u/k over 300 ms, then the stack": the PC, LR and frame-pointer return addresses, each +// as (v = libvulkan_freedreno.dylib; offsets resolve against the unstripped build). A worker that is +// running in the kernel (system time only, or no time at all: a core parked by the panic screen) is not suspended, because thread_suspend +// would wait for it forever; its line ends in "kern". +struct Watched { + const char *libraryPath; + void (*mark)(void *, uint32_t); + void *markContext; + volatile uint32_t lastMark; + volatile int done; + Result result; +}; + +static void *watchedWorker(void *argument) { + Watched *w = static_cast(argument); + w->result = run([w](uint32_t value) { w->lastMark = value; w->mark(w->markContext, value); }, w->libraryPath); + __atomic_store_n(&w->done, 1, __ATOMIC_RELEASE); + return nullptr; +} + +static size_t frameText(char *out, size_t size, uint64_t address) { + Dl_info info = {}; + if (!address || !dladdr(reinterpret_cast(address), &info) || !info.dli_fbase) + return (size_t)snprintf(out, size, " ?%llx", (unsigned long long)address); + const char *name = info.dli_fname ? strrchr(info.dli_fname, '/') : nullptr; + name = name ? name + 1 : (info.dli_fname ? info.dli_fname : "?"); + const char tag = !strncmp(name, "libvulkan", 9) ? 'v' : name[0]; + return (size_t)snprintf(out, size, " %c%llx", tag, (unsigned long long)(address - reinterpret_cast(info.dli_fbase))); +} + +static void sample(Watched &w, pthread_t worker, char *text, size_t size) { + const mach_port_t thread = pthread_mach_thread_np(worker); + thread_basic_info_data_t a = {}, b = {}; + mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; + thread_info(thread, THREAD_BASIC_INFO, reinterpret_cast(&a), &count); + usleep(300000); + count = THREAD_BASIC_INFO_COUNT; + thread_info(thread, THREAD_BASIC_INFO, reinterpret_cast(&b), &count); + auto us = [](time_value_t t) { return int64_t(t.seconds) * 1000000 + t.microseconds; }; + const int64_t du = us(b.user_time) - us(a.user_time), dk = us(b.system_time) - us(a.system_time); + size_t at = (size_t)snprintf(text, size, "W %x s%d u%lld/k%lld", unsigned(w.lastMark), b.run_state, (long long)du, (long long)dk); + if (b.run_state == TH_STATE_RUNNING && du <= 0) { + snprintf(text + at, size - at, " kern"); + return; + } + if (thread_suspend(thread) != KERN_SUCCESS) {snprintf(text + at, size - at, " nosusp"); return;} + arm_thread_state64_t state = {}; + count = ARM_THREAD_STATE64_COUNT; + if (thread_get_state(thread, ARM_THREAD_STATE64, reinterpret_cast(&state), &count) == KERN_SUCCESS) { + const uint64_t pc = arm_thread_state64_get_pc(state), lr = arm_thread_state64_get_lr(state); + uint64_t fp = arm_thread_state64_get_fp(state); + at += frameText(text + at, size - at, pc); + if (at < size) at += frameText(text + at, size - at, lr); + const uint64_t low = reinterpret_cast(pthread_get_stackaddr_np(worker)) - pthread_get_stacksize_np(worker); + const uint64_t high = reinterpret_cast(pthread_get_stackaddr_np(worker)); + for (unsigned i = 0; i < 8 && at + 4 < size && fp >= low && fp + 16 <= high && !(fp & 7); ++i) { + const uint64_t next = reinterpret_cast(fp)[0], ret = reinterpret_cast(fp)[1]; + if (ret != lr || i) at += frameText(text + at, size - at, ret); + if (next <= fp) break; + fp = next; + } + } + thread_resume(thread); +} + +template +static Result runWatched(Marker &mark, Reporter &&report, const char *libraryPath) { + Watched *w = new Watched{}; + w->libraryPath = libraryPath; + w->markContext = &mark; + w->mark = [](void *context, uint32_t value) { (*static_cast(context))(value); }; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8u << 20); // Mesa's NIR/ir3 passes recurse; the default 512 KiB secondary stack is small + pthread_t worker; + const bool started = pthread_create(&worker, &attr, watchedWorker, w) == 0; + pthread_attr_destroy(&attr); + if (!started) {Result r; r.stage = 0; r.vkResult = -12; return r;} + static const unsigned checkpoints[] = {15, 45, 90}; + unsigned next = 0; + for (unsigned tenths = 0; !__atomic_load_n(&w->done, __ATOMIC_ACQUIRE); ++tenths) { + usleep(100000); + if (next < 3 && tenths == checkpoints[next] * 10) { + char text[64] = {}; + sample(*w, worker, text, sizeof text); + report(text); + ++next; + } + } + pthread_join(worker, nullptr); + Result result = w->result; + delete w; + return result; +} + +// Path of libvulkan_freedreno.dylib: next to the image this code is part of (the plugin binary inside the bundle). +static bool libraryPath(char *path, size_t size) { + Dl_info info = {}; + if (!dladdr(reinterpret_cast(&libraryPath), &info) || !info.dli_fname) return false; + snprintf(path, size, "%s", info.dli_fname); + char *slash = strrchr(path, '/'); + if (!slash) return false; + snprintf(slash + 1, size - size_t(slash + 1 - path), "libvulkan_freedreno.dylib"); + return true; +} + +} // namespace turnip +#pragma clang diagnostic pop diff --git a/OfflineMetalCommandWire.h b/OfflineMetalCommandWire.h new file mode 100644 index 0000000..a0f4c12 --- /dev/null +++ b/OfflineMetalCommandWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Actual asynchronous host submission and status observation. Native integration +// must separately qualify resource visibility, lifetime and callback delivery. +#pragma once +#include "OfflineMetalFenceWire.h" +enum { OM_COMMAND_SUBMIT=0x12e,OM_COMMAND_STATUS=0x12f }; +static_assert(OM_COMMAND_SUBMIT==OM_TEXTURE_HAZARD+1,"Command extension ordering"); +// Reply is OMCommandResult: non-completed states have unavailable (zero) times. +// Successful completion requires actual status 4 and genuine nonzero GPU times. diff --git a/OfflineMetalDepthStencilWire.h b/OfflineMetalDepthStencilWire.h new file mode 100644 index 0000000..20ec4f4 --- /dev/null +++ b/OfflineMetalDepthStencilWire.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT +// Explicit offline depth/stencil resources, state and render attachments. +#pragma once +#include "OfflineMetalTextureWire.h" +enum { OM_DEPTH_STENCIL_STATE=0x123,OM_SET_DEPTH_STENCIL=0x124, + OM_STENCIL_REFERENCE=0x125,OM_PIPELINE_ATTACHMENTS=0x126,OM_RENDER_ATTACHMENTS=0x127 }; +static_assert(OM_DEPTH_STENCIL_STATE==OM_COPY_TEXTURE+1,"Depth/stencil extension follows texture copy"); +typedef struct {uint64_t enabled,compare,failure,depthFailure,pass,readMask,writeMask;} OMStencilFace; +typedef struct {uint64_t compare,write;OMStencilFace front,back;} OMDepthStencil; +typedef struct {uint64_t encoder,state;} OMSetDepthStencil; +typedef struct {uint64_t encoder,front,back;} OMStencilReference; +typedef struct {OMBlendPipeline colors;uint64_t depthFormat,stencilFormat;} OMAttachmentPipeline; +typedef struct {uint64_t texture,load,store;double clear;} OMDepthAttachment; +typedef struct {uint64_t texture,load,store,clear;} OMStencilAttachment; +typedef struct {OMRender colors;OMDepthAttachment depth;OMStencilAttachment stencil;} OMAttachmentRender; +static_assert(sizeof(OMDepthStencil)==128 && sizeof(OMAttachmentRender)==520,"Bounded attachment/state ABI"); diff --git a/OfflineMetalEventWire.h b/OfflineMetalEventWire.h new file mode 100644 index 0000000..3f2d277 --- /dev/null +++ b/OfflineMetalEventWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Actual GPU timeline events for cross-queue dependencies; CPU bookkeeping +// never substitutes for executing a GPU signal/wait or reports a signaled value. +#pragma once +#include "OfflineMetalCommandWire.h" +enum { OM_EVENT=0x130,OM_EVENT_SIGNAL,OM_EVENT_WAIT }; +static_assert(OM_EVENT==OM_COMMAND_STATUS+1,"Event extension ordering"); +typedef struct {uint64_t command,event,value;} OMEventDependency; +static_assert(sizeof(OMEventDependency)==24,"Bounded real GPU event dependency"); diff --git a/OfflineMetalFenceWire.h b/OfflineMetalFenceWire.h new file mode 100644 index 0000000..a533fcd --- /dev/null +++ b/OfflineMetalFenceWire.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Explicit offline actual GPU fence dependencies; no emulated completion. +#pragma once +#include "OfflineMetalDepthStencilWire.h" +enum { OM_FENCE=0x128,OM_RENDER_FENCE_UPDATE,OM_RENDER_FENCE_WAIT, + OM_BLIT_FENCE_UPDATE,OM_BLIT_FENCE_WAIT,OM_TEXTURE_HAZARD }; +static_assert(OM_FENCE==OM_RENDER_ATTACHMENTS+1,"Fence extension ordering"); +typedef struct {uint64_t encoder,fence,stages;} OMFenceDependency; +typedef struct {OMTexture texture;uint64_t hazard;} OMHazardTexture; +static_assert(sizeof(OMHazardTexture)==96,"Bounded actual texture hazard mode"); +static_assert(sizeof(OMFenceDependency)==24,"Bounded real fence dependency"); diff --git a/OfflineMetalFrontend.mm b/OfflineMetalFrontend.mm new file mode 100644 index 0000000..55fab6b --- /dev/null +++ b/OfflineMetalFrontend.mm @@ -0,0 +1,1001 @@ +// SPDX-License-Identifier: MIT +// Explicit offline native Metal API frontend. Limited supported operations; +// never installed or advertised as a complete system/hardware GPU driver. +#import +#import +#import +#import +#import +#import +#include +#include +#include "OfflineMetalWire.h" +#include "OfflineMetalFunctionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalVertexWire.h" +#include "OfflineMetalFenceWire.h" +#include "OfflineMetalSharedEventWire.h" +#include +#include +#include +#include +using IOMethod=int (*)(unsigned,unsigned,const uint64_t *,unsigned,const void *,size_t,uint64_t *,unsigned *,void *,size_t *); +static IOMethod ioMethod; +static unsigned connection; +static uint32_t sequence; +static NSRecursiveLock *transaction; +static NSDictionary *deviceInfo; +static id remoteDevice; +static std::atomic live{false},asyncEnabled{false},closing{false}; +static NSLock *pendingLock; +static NSMutableSet *pendingCommands; +static NSMutableSet *pendingNotifications; +static NSException *notificationFailure; +static uint64_t notificationsRegistered,notificationsDelivered,notificationsCancelled,notificationsFailed; +static void fail(NSString *reason) { @throw [NSException exceptionWithName:@"OfflineMetalTransport" reason:reason userInfo:nil]; } +static void receiveBytes(void *bytes,size_t size) { + unsigned char *p=(unsigned char *)bytes; + while(size){size_t count=MIN(size,(size_t)4096);int status=ioMethod(connection,1,nullptr,0,nullptr,0,nullptr,nullptr,p,&count); + if(status || !count || count>size)fail(@"Actual console receive failed");p+=count;size-=count;} +} +static NSData *call(uint16_t operation,const void *input,size_t size,OMResult *result) { + [transaction lock]; + @try { + if(!live || size>4080)fail(@"Invalid live transport request"); + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,operation,(uint32_t)size,sequence+1}; + unsigned char bytes[4096];memcpy(bytes,&header,sizeof(header));if(size)memcpy(bytes+sizeof(header),input,size); + int sendStatus=ioMethod(connection,0,nullptr,0,bytes,sizeof(header)+size,nullptr,nullptr,nullptr,nullptr); + if(sendStatus)fail([NSString stringWithFormat:@"Actual transport send failed: %08x",(unsigned)sendStatus]); + sequence++; + OMWireHeader reply={};receiveBytes(&reply,sizeof(reply)); + if(reply.magic!=OM_WIRE_MAGIC || reply.version!=OFFLINE_METAL_VERSION || reply.operation!=(operation|OM_WIRE_RESPONSE) + || reply.request!=sequence || reply.bytesOFFLINE_METAL_MAX_REPLY)fail(@"Invalid actual Metal reply"); + receiveBytes(result,sizeof(*result));if(result->reserved)fail(@"Unknown reply flags"); + NSMutableData *body=[NSMutableData dataWithLength:reply.bytes-sizeof(*result)];if(body.length)receiveBytes(body.mutableBytes,body.length); + return body; + } @finally {[transaction unlock];} +} +static NSData *checked(uint16_t operation,const void *input,size_t size,OMResult *result) { + NSData *body=call(operation,input,size,result);if(result->error)fail([[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding]?:@"Unsupported or failed actual Metal operation");return body; +} +static NSError *operationError(OMResult result,NSData *body) { + NSString *why=[[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding]?:@"Actual offline backend refused this operation"; + return [NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:result.error userInfo:@{NSLocalizedDescriptionKey:why}]; +} +@interface OMObject : NSObject +@property(readonly) id device; +@property(nonatomic) uint64_t handle; +@property(atomic,copy) NSString *label; +@end +@implementation OMObject +- (id)device {return remoteDevice;} +- (void)dealloc { + // Actual host Metal objects retain resources required by their recorded + // command buffers. Drop exactly this proxy's independent table reference. + // Session teardown releases any remaining references after real commits. + [transaction lock]; + @try {if(live && !closing && _handle){OMHandle request={_handle};OMResult result={};checked(OM_RELEASE,&request,sizeof(request),&result);}} + @finally {[transaction unlock];} +} +@end +@interface OMFenceObject : OMObject +@end +@implementation OMFenceObject +@end +@interface OMEventObject : OMObject +@end +@implementation OMEventObject +@end +// These selectors are read by the unchanged target handle constructor. The +// eventPort is a real Mach capability created by the native kernel broker. +@interface MTLSharedEventHandle (OMOriginalSharedHandle) +- (instancetype)initWithSharedEvent:(id)event; +- (mach_port_t)eventPort; +- (uint64_t)labelTraceID; +@end +static NSDictionary *sharedMetadata(NSData *body) { + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil]; + if(![info isKindOfClass:[NSDictionary class]] || ![info[@"device_is_nil"] isKindOfClass:[NSNumber class]] || + ![info[@"device_is_owner"] isKindOfClass:[NSNumber class]] || + ([info[@"device_is_nil"] boolValue]==[info[@"device_is_owner"] boolValue]) || + ![info[@"signaled_value"] isKindOfClass:[NSNumber class]] || + (info[@"label"]!=[NSNull null] && ![info[@"label"] isKindOfClass:[NSString class]]))fail(@"Invalid actual shared-event metadata"); + return info; +} +enum {OMNotificationPending,OMNotificationQueued,OMNotificationRunning,OMNotificationFinished,OMNotificationCancelled}; +@interface OMSharedNotificationState : NSObject +@property(nonatomic) uint64_t handle,value; +@property(nonatomic,strong) id event; +@property(nonatomic,strong) MTLSharedEventListener *listener; +@property(nonatomic,strong) dispatch_queue_t queue; +@property(nonatomic,strong) dispatch_group_t callbackGroup; +@property(nonatomic,copy) MTLSharedEventNotificationBlock block; +// All state transitions and real callback thread identities use pendingLock. +@property(nonatomic) unsigned phase; +@property(nonatomic) pthread_t callbackThread; +- (void)start; +@end +@implementation OMSharedNotificationState +- (void)start { + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED,0),^{@autoreleasepool { + @try { + struct timespec start={},now={};if(clock_gettime(CLOCK_MONOTONIC,&start))fail(@"Actual listener clock unavailable"); + for(;;) { + [pendingLock lock];BOOL cancelled=self.phase==OMNotificationCancelled;[pendingLock unlock];if(cancelled)return; + OMSharedNotification reply={}; + [transaction lock];@try { + if(!live || closing)return; + OMHandle input={self.handle};OMResult result={};NSData *body=checked(OM_SHARED_NOTIFICATION_POLL,&input,sizeof(input),&result); + if(body.length!=sizeof(reply))fail(@"Invalid actual listener reply");memcpy(&reply,body.bytes,sizeof(reply)); + } @finally {[transaction unlock];} + if(reply.ready) { + if(reply.ready!=1 || reply.event!=[(OMObject *)self.event handle] || reply.value=60)fail(@"Bounded actual listener observation timed out");usleep(1000); + } + } @catch(NSException *exception) { + [pendingLock lock];if(self.phase!=OMNotificationCancelled){if(!notificationFailure)notificationFailure=exception;notificationsFailed++;self.phase=OMNotificationFinished;[pendingNotifications removeObject:self];}[pendingLock unlock]; + } + }}); +} +@end +@interface OMSharedEventObject : OMEventObject { + mach_port_t _exportPort; + uint64_t _traceID; +} +@property(nonatomic) BOOL deviceIsNil; +- (mach_port_t)eventPort; +- (uint64_t)labelTraceID; +@end +@implementation OMSharedEventObject +- (id)device {return self.deviceIsNil?nil:remoteDevice;} +- (void)dealloc {if(_exportPort)mach_port_deallocate(mach_task_self(),_exportPort);} +- (uint64_t)signaledValue { + OMHandle input={self.handle};OMResult result={};NSData *body=checked(OM_SHARED_VALUE_GET,&input,sizeof(input),&result); + if(body.length!=8)fail(@"Invalid actual event counter");uint64_t value=0;memcpy(&value,body.bytes,8);return value; +} +- (void)setSignaledValue:(uint64_t)value { + OMSharedEventValue input={self.handle,value};OMResult result={};NSData *body=checked(OM_SHARED_VALUE_SET,&input,sizeof(input),&result); + if(body.length!=8)fail(@"Missing actual event counter write result"); +} +- (void)setLabel:(NSString *)label { + if(label && ![label isKindOfClass:[NSString class]])fail(@"Invalid shared-event label"); + NSData *text=[label dataUsingEncoding:NSUTF8StringEncoding];if(text.length>512)fail(@"Excessive shared-event label"); + OMSharedLabel input={self.handle,(uint64_t)(label!=nil),text.length};NSMutableData *data=[NSMutableData dataWithBytes:&input length:sizeof(input)];if(text)[data appendData:text]; + [transaction lock];@try { + OMResult result={};NSData *body=checked(OM_SHARED_LABEL_SET,data.bytes,data.length,&result);NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil]; + id actual=info[@"label"];if(actual!=[NSNull null] && ![actual isKindOfClass:[NSString class]])fail(@"Missing actual label result"); + [super setLabel:actual==[NSNull null]?nil:actual]; + if(_exportPort){mach_port_deallocate(mach_task_self(),_exportPort);_exportPort=0;_traceID=0;} + } @finally {[transaction unlock];} +} +- (mach_port_t)eventPort { + [transaction lock];@try { + if(!live || closing)fail(@"Shared export on a closing context");if(_exportPort)return _exportPort; + uint64_t output=0,input=self.handle;unsigned count=1;unsigned char bytes[4096];size_t size=sizeof(bytes); + int status=ioMethod(connection,2,&input,1,nullptr,0,&output,&count,bytes,&size); + if(status || count!=1 || !output || output>UINT32_MAX || output==MACH_PORT_DEAD)fail(@"Native shared-event capability export refused"); + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:[NSData dataWithBytes:bytes length:size] options:0 error:nil]; + if(![info[@"trace_id_supported"] boolValue] || ![info[@"trace_id"] isKindOfClass:[NSNumber class]]){mach_port_deallocate(mach_task_self(),(mach_port_t)output);fail(@"Unqualified actual shared-event trace ABI");} + _exportPort=(mach_port_t)output;_traceID=[info[@"trace_id"] unsignedLongLongValue];return _exportPort; + } @finally {[transaction unlock];} +} +- (uint64_t)labelTraceID {(void)self.eventPort;return _traceID;} +- (MTLSharedEventHandle *)newSharedEventHandle { + Class cls=NSClassFromString(@"MTLSharedEventHandle");if(!cls || class_getInstanceSize(cls)!=16)fail(@"Wrong original shared-handle ABI"); + return [[cls alloc] initWithSharedEvent:self]; +} +- (void)notifyListener:(MTLSharedEventListener *)listener atValue:(uint64_t)value block:(MTLSharedEventNotificationBlock)block { + Class cls=NSClassFromString(@"MTLSharedEventListenerInternal");Method getter=class_getInstanceMethod(cls,@selector(dispatchQueue)); + if(!listener || !block || object_getClass(listener)!=cls || class_getInstanceSize(cls)!=24 || !getter || strcmp(method_getTypeEncoding(getter),"@16@0:8"))fail(@"Unqualified original shared listener ABI"); + dispatch_queue_t queue=listener.dispatchQueue;if(!queue)fail(@"Missing actual original listener queue"); + [transaction lock];@try { + if(!live || closing)fail(@"Listener registration on a closing context");OMSharedEventValue input={self.handle,value};OMResult result={};checked(OM_SHARED_NOTIFY,&input,sizeof(input),&result);if(!result.handle)fail(@"Missing actual listener registration"); + OMSharedNotificationState *state=[OMSharedNotificationState new];state.handle=result.handle;state.value=value;state.event=self;state.listener=listener;state.queue=queue;state.block=block;state.callbackGroup=dispatch_group_create(); + [pendingLock lock];if(closing){[pendingLock unlock];fail(@"Listener registration raced context finish");}[pendingNotifications addObject:state];notificationsRegistered++;[pendingLock unlock];[state start]; + } @finally {[transaction unlock];} +} +- (BOOL)waitUntilSignaledValue:(uint64_t)value timeoutMS:(uint64_t)milliseconds { + struct timespec start={},now={};if(clock_gettime(CLOCK_MONOTONIC,&start))fail(@"Actual monotonic clock unavailable"); + for(;;){if(self.signaledValue>=value)return YES;if(clock_gettime(CLOCK_MONOTONIC,&now))fail(@"Actual monotonic clock unavailable"); + uint64_t elapsed=(uint64_t)(now.tv_sec-start.tv_sec)*1000; + if(now.tv_nsec>=start.tv_nsec)elapsed+=(now.tv_nsec-start.tv_nsec)/1000000; + else elapsed-=(start.tv_nsec-now.tv_nsec+999999)/1000000; + if(elapsed>=milliseconds)return NO;usleep(1000);} +} +@end +@interface OMFunctionObject : OMObject +@property(nonatomic,strong) NSString *name; +@property(nonatomic) MTLFunctionType functionType; +@end +@implementation OMFunctionObject +@end +@interface MTLFunctionConstantValues (OMOriginalConstantAccess) +- (NSArray *)newIndexedConstantArray; +- (NSArray *)newNamedConstantArray; +@end +@protocol OMOriginalConstantValue +- (NSUInteger)dataType; +- (void *)data; +- (NSUInteger)index; +- (NSString *)name; +@end +static bool methodABI(id object,SEL selector,const char *encoding) { + Method method=class_getInstanceMethod(object_getClass(object),selector); + return method && !strcmp(method_getTypeEncoding(method),encoding); +} +static NSError *constantError(NSString *reason) { + return [NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 userInfo:@{NSLocalizedDescriptionKey:reason}]; +} +@interface OMLibraryObject : OMObject +@property(nonatomic,strong) NSArray *functionNames; +- (id)newFunctionWithName:(NSString *)name; +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error; +@end +@implementation OMLibraryObject +- (id)newFunctionWithName:(NSString *)name { + if(!name || [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128)return nil; + OMFunction request={};request.library=self.handle;strcpy(request.name,name.UTF8String); + OMResult result={};NSData *body=call(OM_FUNCTION,&request,sizeof(request),&result);if(result.error)return nil; + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![info isKindOfClass:[NSDictionary class]])fail(@"Actual function metadata missing"); + OMFunctionObject *object=[OMFunctionObject new];object.handle=result.handle;object.name=info[@"name"];object.functionType=(MTLFunctionType)[info[@"type"] unsignedIntegerValue]; + return (id)object; +} +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error { + auto refuse=[&](NSString *why)->id {if(error)*error=constantError(why);return nil;}; + if(!name.length || [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128 || + object_getClass(values)!=NSClassFromString(@"MTLFunctionConstantValuesInternal") || + !methodABI(values,@selector(newIndexedConstantArray),"@16@0:8") || !methodABI(values,@selector(newNamedConstantArray),"@16@0:8")) + return refuse(@"Unqualified actual function-constant object or name"); + // Read through the original target's real accessors. Never replace setters, + // interpret opaque storage, or substitute unspecialized shader functions. + MTLFunctionConstantValues *snapshot=[values copy]; + NSArray *indexed=[snapshot newIndexedConstantArray],*named=[snapshot newNamedConstantArray]; + if(![indexed isKindOfClass:[NSArray class]] || ![named isKindOfClass:[NSArray class]] || indexed.count+named.count>OFFLINE_METAL_MAX_CONSTANTS) + return refuse(@"Unqualified or excessive original constants"); + OMSpecializedFunction request={};request.function.library=self.handle;strcpy(request.function.name,name.UTF8String);request.count=indexed.count+named.count; + NSMutableData *packet=[NSMutableData dataWithBytes:&request length:sizeof(request)]; + for(unsigned group=0;group<2;group++)for(id value in (group?named:indexed)) { + if(object_getClass(value)!=NSClassFromString(group?@"MTLNamedConstantValue":@"MTLIndexedConstantValue") || + !methodABI(value,@selector(dataType),"Q16@0:8") || !methodABI(value,@selector(data),"^v16@0:8") || + !methodABI(value,group?@selector(name):@selector(index),group?"@16@0:8":"Q16@0:8")) + return refuse(@"Unqualified actual constant descriptor ABI"); + id descriptor=value; + OMFunctionConstant row={};row.type=[descriptor dataType];row.bytes=OMFunctionConstantBytes(row.type);row.named=group; + const void *data=[descriptor data];if(!row.bytes || !data)return refuse(@"Unsupported original constant type or missing actual value"); + if(group) { + NSString *key=[descriptor name];if(![key isKindOfClass:[NSString class]] || !key.length || [key lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128){ + return refuse(@"Invalid actual named constant");}strcpy(row.name,key.UTF8String); + } else {row.index=[descriptor index];if(row.index>65535)return refuse(@"Original constant index exceeds qualified range");} + memcpy(row.data,data,row.bytes);[packet appendBytes:&row length:sizeof(row)]; + } + OMResult result={};NSData *body=call(OM_FUNCTION,packet.bytes,packet.length,&result); + if(result.error){if(error)*error=operationError(result,body);return nil;} + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![info isKindOfClass:[NSDictionary class]])fail(@"Actual specialized function metadata missing"); + OMFunctionObject *object=[OMFunctionObject new];object.handle=result.handle;object.name=info[@"name"];object.functionType=(MTLFunctionType)[info[@"type"] unsignedIntegerValue]; + if(error)*error=nil;return (id)object; +} +@end +@interface OMTextureObject : OMObject +@property(nonatomic) OMTexture descriptor; +@property(nonatomic) MTLHazardTrackingMode actualHazardMode; +@property(nonatomic) IOSurfaceRef surface; +@property(nonatomic) uint32_t surfaceSeed; +@property(nonatomic) BOOL surfaceSeedValid; +- (void)uploadSurface; +- (void)downloadSurface; +- (void)getBytes:(void *)pixels bytesPerRow:(NSUInteger)stride fromRegion:(MTLRegion)region mipmapLevel:(NSUInteger)level; +@end +@implementation OMTextureObject +- (void)dealloc {if(_surface)CFRelease(_surface);} +- (IOSurfaceRef)iosurface {return self.surface;} +- (NSUInteger)iosurfacePlane {return 0;} +- (void)uploadSurface { + @synchronized(self) { + if(!self.surface)return;uint32_t seed=0; + if(IOSurfaceLock(self.surface,kIOSurfaceLockReadOnly,&seed))fail(@"Actual IOSurface read lock failed"); + @try { + if(self.surfaceSeedValid && seed==self.surfaceSeed)return; + const unsigned char *base=(const unsigned char *)IOSurfaceGetBaseAddress(self.surface);size_t row=IOSurfaceGetBytesPerRow(self.surface); + if(!base)fail(@"Missing actual IOSurface mapping"); + for(uint64_t y=0;ySIZE_MAX/4 + || strideSIZE_MAX/region.size.height)fail(@"Unsupported texture readback layout"); + OMReadTexture request={self.handle,region.origin.x,region.origin.y,region.size.width,region.size.height,level};OMResult result={}; + NSData *body=checked(OM_READ_TEXTURE,&request,sizeof(request),&result); + if(region.size.width>SIZE_MAX/4/region.size.height || body.length!=region.size.width*region.size.height*4)fail(@"Actual pixel extent differs"); + for(NSUInteger row=0;rowself.data.length || range.length>self.data.length-range.location)fail(@"Invalid native buffer modification range");} +- (void)upload { + for(NSUInteger offset=0;offset +- (id)texture; +@end +@interface OMCommandObject : OMObject +@property(nonatomic,strong) id commandQueue; +@property(atomic) OMCommandResult actualResult; +@property(nonatomic) BOOL useAsync; +@property(atomic) BOOL submissionStarted; +@property(atomic) BOOL resourcesVisible; +@property(atomic) pthread_t callbackThread; +@property(atomic,strong) NSException *asynchronousFailure; +@property(atomic,strong) NSException *deliveryFailure; +- (void)commitAsync; +@property(nonatomic) MTLCommandBufferStatus actualEnqueueStatus; +@property(nonatomic,strong) NSMutableArray *completedHandlers; +@property(nonatomic,strong) NSMutableArray *scheduledHandlers; +@property(atomic,strong) dispatch_group_t callbackGroup; +@property(atomic,strong) dispatch_group_t scheduledGroup; +@property(atomic,strong) dispatch_group_t visibilityGroup; +@property(atomic) BOOL scheduledObserved; +@property(atomic) MTLCommandBufferStatus actualScheduledStatus; +@property(nonatomic,strong) NSMutableArray *surfaceTargets; +@property(nonatomic,strong) NSMutableArray *buffers; +@property(nonatomic,strong) NSMutableArray *textures; +@property(nonatomic,strong) NSMutableArray> *presentations; +@property(nonatomic,strong) NSMutableArray *fences; +@property(nonatomic,strong) NSMutableArray *events; +- (void)trackFence:(id)fence; +- (void)trackBuffer:(OMBufferObject *)buffer; +- (void)trackTexture:(OMTextureObject *)texture; +- (void)trackSurfaceTarget:(OMTextureObject *)texture; +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)descriptor; +- (id)blitCommandEncoder; +- (void)commit; +- (void)waitUntilCompleted; +@end +@interface OMEncoderObject : OMObject +@property(nonatomic,strong) OMCommandObject *command; +@property(nonatomic,strong) NSMutableDictionary *vertexBuffers; +@property(nonatomic,strong) NSMutableDictionary *fragmentBuffers; +@end +@implementation OMEncoderObject +- (void)updateFence:(id)fence afterStages:(MTLRenderStages)stages { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign render fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],(uint64_t)stages};OMResult result={};checked(OM_RENDER_FENCE_UPDATE,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)waitForFence:(id)fence beforeStages:(MTLRenderStages)stages { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign render fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],(uint64_t)stages};OMResult result={};checked(OM_RENDER_FENCE_WAIT,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)setDepthStencilState:(id)state { + if(state && ![(id)state isKindOfClass:[OMDepthStateObject class]])fail(@"Foreign depth/stencil state"); + OMSetDepthStencil request={self.handle,state?[(OMObject *)state handle]:0};OMResult result={};checked(OM_SET_DEPTH_STENCIL,&request,sizeof(request),&result); +} +- (void)setStencilFrontReferenceValue:(uint32_t)front backReferenceValue:(uint32_t)back { + OMStencilReference request={self.handle,front,back};OMResult result={};checked(OM_STENCIL_REFERENCE,&request,sizeof(request),&result); +} +- (void)setStencilReferenceValue:(uint32_t)value {[self setStencilFrontReferenceValue:value backReferenceValue:value];} +- (void)setRenderPipelineState:(id)pipeline { + if(![(id)pipeline isKindOfClass:[OMPipelineObject class]])fail(@"Foreign pipeline object"); + OMSetPipeline request={self.handle,[(OMObject *)pipeline handle]};OMResult result={};checked(OM_SET_PIPELINE,&request,sizeof(request),&result); +} +- (void)setCullMode:(MTLCullMode)mode {OMSetCull request={self.handle,(uint64_t)mode};OMResult result={};checked(OM_SET_CULL,&request,sizeof(request),&result);} +- (void)setViewport:(MTLViewport)viewport { + OMViewport request={self.handle,viewport.originX,viewport.originY,viewport.width,viewport.height,viewport.znear,viewport.zfar};OMResult result={};checked(OM_VIEWPORT,&request,sizeof(request),&result); +} +- (void)setScissorRect:(MTLScissorRect)rect {OMScissor request={self.handle,rect.x,rect.y,rect.width,rect.height};OMResult result={};checked(OM_SCISSOR,&request,sizeof(request),&result);} +- (void)setBlendColorRed:(float)red green:(float)green blue:(float)blue alpha:(float)alpha { + OMBlendColor request={self.handle,red,green,blue,alpha};OMResult result={};checked(OM_BLEND_COLOR,&request,sizeof(request),&result); +} +- (void)setTexture:(id)texture index:(NSUInteger)index vertex:(BOOL)vertex { + if(texture && ![(id)texture isKindOfClass:[OMTextureObject class]])fail(@"Foreign texture object"); + OMSetTexture request={self.handle,texture?[(OMObject *)texture handle]:0,index};OMResult result={}; + checked(vertex?OM_VERTEX_TEXTURE:OM_FRAGMENT_TEXTURE,&request,sizeof(request),&result); + if(texture)[self.command trackTexture:(OMTextureObject *)texture]; +} +- (void)setVertexTexture:(id)texture atIndex:(NSUInteger)index {[self setTexture:texture index:index vertex:YES];} +- (void)setFragmentTexture:(id)texture atIndex:(NSUInteger)index {[self setTexture:texture index:index vertex:NO];} +- (void)drawPrimitives:(MTLPrimitiveType)type vertexStart:(NSUInteger)start vertexCount:(NSUInteger)count { + OMDraw request={self.handle,(uint64_t)type,start,count,1};OMResult result={};checked(OM_DRAW,&request,sizeof(request),&result); +} +- (void)setBuffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index vertex:(BOOL)vertex { + if(index>=31 || (buffer && ![(id)buffer isKindOfClass:[OMBufferObject class]]))fail(@"Foreign buffer object or invalid binding index"); + OMSetBuffer request={self.handle,buffer?[(OMObject *)buffer handle]:0,offset,index};OMResult result={}; + checked(vertex?OM_VERTEX_BUFFER:OM_FRAGMENT_BUFFER,&request,sizeof(request),&result); + if(vertex && !self.vertexBuffers)self.vertexBuffers=[NSMutableDictionary dictionary]; + if(!vertex && !self.fragmentBuffers)self.fragmentBuffers=[NSMutableDictionary dictionary]; + NSMutableDictionary *bindings=vertex?self.vertexBuffers:self.fragmentBuffers; + if(buffer)bindings[@(index)]=buffer;else [bindings removeObjectForKey:@(index)]; + if(buffer)[self.command trackBuffer:(OMBufferObject *)buffer]; +} +- (void)setVertexBuffer:(id)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index {[self setBuffer:buffer offset:offset index:index vertex:YES];} +- (void)setFragmentBuffer:(id)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index {[self setBuffer:buffer offset:offset index:index vertex:NO];} +- (void)setVertexBufferOffset:(NSUInteger)offset atIndex:(NSUInteger)index { + OMBufferObject *buffer=self.vertexBuffers[@(index)];if(!buffer)fail(@"No actual vertex buffer bound at this index"); + [self setVertexBuffer:(id)buffer offset:offset atIndex:index]; +} +- (void)setFragmentBufferOffset:(NSUInteger)offset atIndex:(NSUInteger)index { + OMBufferObject *buffer=self.fragmentBuffers[@(index)];if(!buffer)fail(@"No actual fragment buffer bound at this index"); + [self setFragmentBuffer:(id)buffer offset:offset atIndex:index]; +} +- (void)setVertexBytes:(const void *)bytes length:(NSUInteger)length atIndex:(NSUInteger)index { + id buffer=[remoteDevice newBufferWithBytes:bytes length:length options:MTLResourceStorageModeShared]; + if(!buffer)fail(@"Invalid vertex-byte allocation");[self setVertexBuffer:buffer offset:0 atIndex:index]; +} +- (void)setFragmentBytes:(const void *)bytes length:(NSUInteger)length atIndex:(NSUInteger)index { + id buffer=[remoteDevice newBufferWithBytes:bytes length:length options:MTLResourceStorageModeShared]; + if(!buffer)fail(@"Invalid fragment-byte allocation");[self setFragmentBuffer:buffer offset:0 atIndex:index]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset instanceCount:(NSUInteger)instances baseVertex:(NSInteger)baseVertex baseInstance:(NSUInteger)baseInstance { + if(![(id)buffer isKindOfClass:[OMBufferObject class]])fail(@"Foreign index buffer"); + OMDrawIndexed request={self.handle,(uint64_t)type,count,(uint64_t)indexType,[(OMObject *)buffer handle],offset,instances,baseVertex,baseInstance};OMResult result={}; + checked(OM_DRAW_INDEXED,&request,sizeof(request),&result);[self.command trackBuffer:(OMBufferObject *)buffer]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset { + [self drawIndexedPrimitives:type indexCount:count indexType:indexType indexBuffer:buffer indexBufferOffset:offset instanceCount:1 baseVertex:0 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset instanceCount:(NSUInteger)instances { + [self drawIndexedPrimitives:type indexCount:count indexType:indexType indexBuffer:buffer indexBufferOffset:offset instanceCount:instances baseVertex:0 baseInstance:0]; +} +- (void)endEncoding {OMHandle request={self.handle};OMResult result={};checked(OM_END,&request,sizeof(request),&result);} +@end +@interface OMBlitEncoderObject : OMEncoderObject +@end +@implementation OMBlitEncoderObject +- (void)updateFence:(id)fence { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign blit fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],0};OMResult result={};checked(OM_BLIT_FENCE_UPDATE,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)waitForFence:(id)fence { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign blit fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],0};OMResult result={};checked(OM_BLIT_FENCE_WAIT,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)copyFromTexture:(id)source sourceSlice:(NSUInteger)sourceSlice sourceLevel:(NSUInteger)sourceLevel sourceOrigin:(MTLOrigin)sourceOrigin sourceSize:(MTLSize)sourceSize + toTexture:(id)destination destinationSlice:(NSUInteger)destinationSlice destinationLevel:(NSUInteger)destinationLevel destinationOrigin:(MTLOrigin)destinationOrigin { + if(![(id)source isKindOfClass:[OMTextureObject class]] || ![(id)destination isKindOfClass:[OMTextureObject class]])fail(@"Foreign blit texture"); + OMCopyTexture request={self.handle,[(OMObject *)source handle],[(OMObject *)destination handle],sourceSlice,sourceLevel, + {sourceOrigin.x,sourceOrigin.y,sourceOrigin.z},{sourceSize.width,sourceSize.height,sourceSize.depth},destinationSlice,destinationLevel, + {destinationOrigin.x,destinationOrigin.y,destinationOrigin.z}}; + OMResult result={};checked(OM_COPY_TEXTURE,&request,sizeof(request),&result); + [self.command trackTexture:(OMTextureObject *)source];[self.command trackSurfaceTarget:(OMTextureObject *)destination]; +} +- (void)copyFromTexture:(id)source toTexture:(id)destination { + [self copyFromTexture:source sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(source.width,source.height,source.depth) + toTexture:destination destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; +} +- (void)copyFromBuffer:(id)source sourceOffset:(NSUInteger)sourceOffset toBuffer:(id)destination destinationOffset:(NSUInteger)destinationOffset size:(NSUInteger)size { + if(![(id)source isKindOfClass:[OMBufferObject class]] || ![(id)destination isKindOfClass:[OMBufferObject class]])fail(@"Foreign blit buffer"); + OMCopyBuffer request={self.handle,[(OMObject *)source handle],sourceOffset,[(OMObject *)destination handle],destinationOffset,size};OMResult result={}; + checked(OM_COPY_BUFFER,&request,sizeof(request),&result); + [self.command trackBuffer:(OMBufferObject *)source];[self.command trackBuffer:(OMBufferObject *)destination]; +} +@end +@implementation OMCommandObject +- (void)encodeSignalEvent:(id)event value:(uint64_t)value { + if(self.submissionStarted || self.actualResult.status || ![(id)event isKindOfClass:[OMEventObject class]])fail(@"Unqualified event or submitted command"); + OMEventDependency request={self.handle,[(OMObject *)event handle],value};OMResult result={};checked(OM_EVENT_SIGNAL,&request,sizeof(request),&result); + if(!self.events)self.events=[NSMutableArray array];if([self.events indexOfObjectIdenticalTo:event]==NSNotFound)[self.events addObject:(OMEventObject *)event]; +} +- (void)encodeWaitForEvent:(id)event value:(uint64_t)value { + if(self.submissionStarted || self.actualResult.status || ![(id)event isKindOfClass:[OMEventObject class]])fail(@"Unqualified event or submitted command"); + OMEventDependency request={self.handle,[(OMObject *)event handle],value};OMResult result={};checked(OM_EVENT_WAIT,&request,sizeof(request),&result); + if(!self.events)self.events=[NSMutableArray array];if([self.events indexOfObjectIdenticalTo:event]==NSNotFound)[self.events addObject:(OMEventObject *)event]; +} +- (void)commitAsync { + if(self.submissionStarted || self.actualResult.status)fail(@"Command already submitted"); + dispatch_group_t group=dispatch_group_create();dispatch_group_enter(group);self.callbackGroup=group; + self.scheduledGroup=dispatch_group_create();self.visibilityGroup=dispatch_group_create();dispatch_group_enter(self.scheduledGroup);dispatch_group_enter(self.visibilityGroup); + [pendingLock lock]; + if(closing || !live){[pendingLock unlock];dispatch_group_leave(self.scheduledGroup);dispatch_group_leave(self.visibilityGroup);dispatch_group_leave(group);fail(@"Context is closing");} + self.submissionStarted=YES;[pendingCommands addObject:self];[pendingLock unlock]; + @try { + for(OMBufferObject *buffer in self.buffers)[buffer upload]; + for(OMTextureObject *texture in self.textures)[texture uploadSurface]; + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMAND_SUBMIT,&request,sizeof(request),&result); + if(body.length!=sizeof(OMCommandResult))fail(@"Missing actual asynchronous submission");OMCommandResult initial={};memcpy(&initial,body.bytes,sizeof(initial)); + if(initial.statusMTLCommandBufferStatusCompleted || + (initial.status=MTLCommandBufferStatusScheduled){self.actualScheduledStatus=(MTLCommandBufferStatus)initial.status;self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + NSArray *scheduled=[self.scheduledHandlers copy],*completed=[self.completedHandlers copy],*presentations=[self.presentations copy]; + self.scheduledHandlers=nil;self.completedHandlers=nil;self.presentations=nil; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{@autoreleasepool { + @try { + OMCommandResult value=initial;unsigned polls=0; + while(value.status!=MTLCommandBufferStatusCompleted) { + if(++polls>4096)fail(@"Actual GPU completion observation exceeded its bound"); + OMResult observed={};NSData *reply=checked(OM_COMMAND_STATUS,&request,sizeof(request),&observed); + if(reply.length!=sizeof(value))fail(@"Missing actual GPU status");memcpy(&value,reply.bytes,sizeof(value)); + if(value.statusMTLCommandBufferStatusCompleted || + (value.status=MTLCommandBufferStatusScheduled && !self.scheduledObserved){self.actualScheduledStatus=(MTLCommandBufferStatus)value.status;self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + if(value.status!=MTLCommandBufferStatusCompleted){self.actualResult=value;usleep(1000);} + } + if(!value.start_ns || value.end_ns<=value.start_ns)fail(@"Missing actual completed GPU timing"); + for(OMTextureObject *texture in self.surfaceTargets)[texture downloadSurface]; + for(OMBufferObject *buffer in self.buffers)[buffer download]; + self.actualResult=value;self.resourcesVisible=YES; + @try {for(id drawable in presentations)[drawable present];} + @catch(NSException *exception){self.deliveryFailure=exception;} + } @catch(NSException *exception){self.asynchronousFailure=exception;} + if(!self.scheduledObserved){self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + dispatch_group_leave(self.visibilityGroup); + // Callbacks are deliberately late: real GPU completion and CPU visibility + // precede success delivery. A transport/GPU failure has a real error and + // never delivers a fabricated successful command or display timestamp. + self.callbackThread=pthread_self(); + @try { + if(self.resourcesVisible)for(MTLCommandBufferHandler handler in scheduled)handler((id)self); + for(MTLCommandBufferHandler handler in completed)handler((id)self); + } @catch(NSException *exception){self.deliveryFailure=exception;} + @finally { + self.callbackThread=nullptr;[pendingLock lock];[pendingCommands removeObject:self];[pendingLock unlock];dispatch_group_leave(group); + } + }}); + } @catch(NSException *exception) { + self.asynchronousFailure=exception;if(!self.scheduledObserved){self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);}dispatch_group_leave(self.visibilityGroup);[pendingLock lock];[pendingCommands removeObject:self];[pendingLock unlock];dispatch_group_leave(group);@throw; + } +} +- (void)trackFence:(id)fence { + if(!self.fences)self.fences=[NSMutableArray array];if([self.fences indexOfObjectIdenticalTo:fence]==NSNotFound)[self.fences addObject:(OMFenceObject *)fence]; +} +- (void)presentDrawable:(id)drawable { + if(self.submissionStarted || self.actualResult.status || !drawable || ![(id)drawable conformsToProtocol:@protocol(MTLDrawable)] || + !methodABI(drawable,@selector(texture),"@16@0:8") || !methodABI(drawable,@selector(present),"v16@0:8"))fail(@"Unqualified drawable or already completed command"); + id texture=[(id)drawable texture]; + if(![(id)texture isKindOfClass:[OMTextureObject class]] || texture.device!=remoteDevice || !texture.iosurface) + fail(@"Drawable requires this device's actual IOSurface texture"); + if(!self.presentations)self.presentations=[NSMutableArray array]; + if(self.presentations.count>=8 || [self.presentations indexOfObjectIdenticalTo:drawable]!=NSNotFound)fail(@"Duplicate or excessive actual drawable presentation"); + [self trackSurfaceTarget:(OMTextureObject *)texture];[self.presentations addObject:drawable]; +} +- (void)addCompletedHandler:(MTLCommandBufferHandler)handler { + if(!handler || self.submissionStarted || self.actualResult.status)fail(@"Completion handler requires a valid uncommitted command"); + if(!self.completedHandlers)self.completedHandlers=[NSMutableArray array];[self.completedHandlers addObject:[handler copy]]; +} +- (void)addScheduledHandler:(MTLCommandBufferHandler)handler { + if(!handler || self.submissionStarted || self.actualResult.status)fail(@"Scheduled handler requires a valid uncommitted command"); + if(!self.scheduledHandlers)self.scheduledHandlers=[NSMutableArray array];[self.scheduledHandlers addObject:[handler copy]]; +} +- (void)enqueue { + if(self.submissionStarted || self.actualResult.status || self.actualEnqueueStatus)fail(@"Command already enqueued or submitted"); + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMAND_ENQUEUE,&request,sizeof(request),&result); + if(body.length!=sizeof(uint64_t))fail(@"Missing actual enqueue status");uint64_t status=0;memcpy(&status,body.bytes,sizeof(status)); + if(status!=MTLCommandBufferStatusEnqueued)fail(@"Actual GPU command was not enqueued");self.actualEnqueueStatus=(MTLCommandBufferStatus)status; +} +- (void)trackBuffer:(OMBufferObject *)buffer { + if(!self.buffers)self.buffers=[NSMutableArray array];if(![self.buffers containsObject:buffer])[self.buffers addObject:buffer]; +} +- (void)trackTexture:(OMTextureObject *)texture { + if(!self.textures)self.textures=[NSMutableArray array];if(![self.textures containsObject:texture])[self.textures addObject:texture]; +} +- (void)trackSurfaceTarget:(OMTextureObject *)texture { + [self trackTexture:texture]; + if(texture.surface){if(!self.surfaceTargets)self.surfaceTargets=[NSMutableArray array];if(![self.surfaceTargets containsObject:texture])[self.surfaceTargets addObject:texture];} +} +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)descriptor { + if(!descriptor)fail(@"Missing render pass descriptor"); + OMAttachmentRender extended={};OMRender &request=extended.colors;request.command=self.handle; + for(unsigned i=0;i<8;i++) { + MTLRenderPassColorAttachmentDescriptor *a=descriptor.colorAttachments[i];if(!a.texture)continue; + if(![(id)a.texture isKindOfClass:[OMTextureObject class]] || a.level || a.slice || a.depthPlane || a.resolveTexture)fail(@"Unsupported or foreign render attachment"); + request.colors[i].texture=[(OMObject *)a.texture handle];request.colors[i].load=a.loadAction;request.colors[i].store=a.storeAction; + OMTextureObject *texture=(OMTextureObject *)a.texture; + [self trackSurfaceTarget:texture]; + if(texture.surface)[texture uploadSurface]; + MTLClearColor color=a.clearColor;request.colors[i].clear[0]=color.red;request.colors[i].clear[1]=color.green;request.colors[i].clear[2]=color.blue;request.colors[i].clear[3]=color.alpha; + } + MTLRenderPassDepthAttachmentDescriptor *depth=descriptor.depthAttachment; + if(depth.texture) { + if(![(id)depth.texture isKindOfClass:[OMTextureObject class]] || depth.level || depth.slice || depth.depthPlane || depth.resolveTexture)fail(@"Unsupported or foreign depth attachment"); + extended.depth={[(OMObject *)depth.texture handle],(uint64_t)depth.loadAction,(uint64_t)depth.storeAction,depth.clearDepth};[self trackTexture:(OMTextureObject *)depth.texture]; + } + MTLRenderPassStencilAttachmentDescriptor *stencil=descriptor.stencilAttachment; + if(stencil.texture) { + if(![(id)stencil.texture isKindOfClass:[OMTextureObject class]] || stencil.level || stencil.slice || stencil.depthPlane || stencil.resolveTexture)fail(@"Unsupported or foreign stencil attachment"); + extended.stencil={[(OMObject *)stencil.texture handle],(uint64_t)stencil.loadAction,(uint64_t)stencil.storeAction,stencil.clearStencil};[self trackTexture:(OMTextureObject *)stencil.texture]; + } + bool attachments=depth.texture || stencil.texture; + OMResult result={};checked(attachments?OM_RENDER_ATTACHMENTS:OM_RENDER,attachments?(const void *)&extended:(const void *)&request,attachments?sizeof(extended):sizeof(request),&result); + OMEncoderObject *object=[OMEncoderObject new];object.handle=result.handle;object.command=self;return (id)object; +} +- (id)blitCommandEncoder { + OMHandle request={self.handle};OMResult result={};checked(OM_BLIT,&request,sizeof(request),&result); + OMBlitEncoderObject *object=[OMBlitEncoderObject new];object.handle=result.handle;object.command=self;return (id)object; +} +- (void)commit { + if(self.useAsync){[self commitAsync];return;} + if(self.actualResult.status)fail(@"Command already submitted"); + for(OMBufferObject *buffer in self.buffers)[buffer upload]; + for(OMTextureObject *texture in self.textures)[texture uploadSurface]; + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMIT,&request,sizeof(request),&result); + if(body.length!=sizeof(OMCommandResult))fail(@"Missing actual command result");OMCommandResult value={};memcpy(&value,body.bytes,sizeof(value)); + if(value.status!=MTLCommandBufferStatusCompleted || !value.start_ns || value.end_ns<=value.start_ns)fail(@"Actual GPU command did not complete"); + for(OMTextureObject *texture in self.surfaceTargets)[texture downloadSurface]; + for(OMBufferObject *buffer in self.buffers)[buffer download]; + self.actualResult=value;self.resourcesVisible=YES; + // Present the real original drawable only after actual GPU completion and + // visibility in its original IOSurface. Its real framework owns presentation + // scheduling and timestamps; GPU completion is never a fabricated display + // completion. Preserve actual GPU status/callbacks even if presentation fails. + NSArray> *presentations=self.presentations;self.presentations=nil; + NSException *presentationFailure=nil; + @try {for(id drawable in presentations)[drawable present];} + @catch(NSException *exception){presentationFailure=exception;} + // This transport is synchronous: submission, execution and surface/buffer + // readback have all actually finished. Deliver each registered block exactly + // once on a real dispatch worker, with the real completed command. Completion + // waits also wait for that delivery, without invoking callbacks under the + // original renderer's encoding/commit call stack. + // Scheduled callbacks are deliberately late here; no asynchronous timing or + // fabricated Scheduled/Completed state is advertised. + NSArray *scheduled=self.scheduledHandlers,*completed=self.completedHandlers; + self.scheduledHandlers=nil;self.completedHandlers=nil; + dispatch_group_t group=nil; + if(scheduled.count || completed.count){group=dispatch_group_create();dispatch_group_enter(group);self.callbackGroup=group;} + if(group)dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{ + self.callbackThread=pthread_self(); + @try { + for(MTLCommandBufferHandler handler in scheduled)handler((id)self); + for(MTLCommandBufferHandler handler in completed)handler((id)self); + } @finally {self.callbackThread=nullptr;dispatch_group_leave(group);} + }); + if(presentationFailure)@throw presentationFailure; +} +- (void)waitUntilScheduled { + if(self.useAsync) { + if(!self.submissionStarted || !self.scheduledGroup)fail(@"Command has not actually been submitted"); + if(dispatch_group_wait(self.scheduledGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual scheduling observation timed out"); + if(self.asynchronousFailure)@throw self.asynchronousFailure; + if(self.actualScheduledStatus=MTLCommandBufferStatusCommitted && self.actualResult.status<=MTLCommandBufferStatusCompleted; + [self waitUntilScheduled];return self.actualResult.status==MTLCommandBufferStatusCompleted; +} +- (void)waitUntilCompleted { + if(self.useAsync) { + if(!self.submissionStarted || !self.callbackGroup)fail(@"Command has not actually been submitted"); + bool ownCallback=self.callbackThread && pthread_equal(self.callbackThread,pthread_self()); + if(!ownCallback && dispatch_group_wait(self.callbackGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual asynchronous completion/visibility/callback wait timed out"); + if(self.asynchronousFailure)@throw self.asynchronousFailure;if(self.deliveryFailure)@throw self.deliveryFailure; + if(!self.resourcesVisible || self.actualResult.status!=MTLCommandBufferStatusCompleted)fail(@"Actual GPU resources are not completed and visible");return; + } + if(self.actualResult.status!=MTLCommandBufferStatusCompleted)fail(@"Command has not actually completed"); + if(self.callbackGroup && !(self.callbackThread && pthread_equal(self.callbackThread,pthread_self())) && dispatch_group_wait(self.callbackGroup,dispatch_time(DISPATCH_TIME_NOW,10*NSEC_PER_SEC)))fail(@"Actual completion-handler delivery timed out"); +} +- (MTLCommandBufferStatus)status { + if(self.asynchronousFailure)return MTLCommandBufferStatusError; + OMCommandResult value=self.actualResult; + if(self.useAsync && value.status==MTLCommandBufferStatusCompleted && !self.resourcesVisible){if(dispatch_group_wait(self.visibilityGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual resource visibility timed out");if(self.asynchronousFailure)return MTLCommandBufferStatusError;value=self.actualResult;} + return value.status?(MTLCommandBufferStatus)value.status:self.actualEnqueueStatus; +} +- (NSError *)error { + if(self.asynchronousFailure)return [NSError errorWithDomain:@"org.blu-sharky.offline-metal.async" code:-1 userInfo:@{NSLocalizedDescriptionKey:self.asynchronousFailure.reason?:@"Actual asynchronous transport/GPU failure"}]; + return nil; +} +- (CFTimeInterval)GPUStartTime {return self.actualResult.start_ns/1e9;} +- (CFTimeInterval)GPUEndTime {return self.actualResult.end_ns/1e9;} +@end +@interface OMQueueObject : OMObject +@end +@implementation OMQueueObject +- (NSUInteger)readPriority:(BOOL)background { + OMQueuePriority request={self.handle,(uint64_t)background,0};OMResult result={};NSData *body=checked(OM_QUEUE_PRIORITY_GET,&request,sizeof(request),&result); + if(body.length!=sizeof(uint64_t))fail(@"Missing actual queue priority");uint64_t value=0;memcpy(&value,body.bytes,sizeof(value));return value; +} +- (BOOL)writePriority:(NSUInteger)priority background:(BOOL)background { + // Exact original MTLIOAccelCommandQueue validation: 0..5, except 3 in foreground. + if(priority>5 || (!background && priority==3))return NO; + OMQueuePriority request={self.handle,(uint64_t)background,priority};OMResult result={};NSData *body=call(OM_QUEUE_PRIORITY_SET,&request,sizeof(request),&result); + if(result.error)return NO;if(body.length!=sizeof(uint64_t))fail(@"Missing actual queue priority-change result");uint64_t value=0;memcpy(&value,body.bytes,sizeof(value)); + if(value>1)fail(@"Invalid actual priority-change result");return value!=0; +} +- (NSUInteger)getGPUPriority {return [self readPriority:NO];} +- (NSUInteger)getBackgroundGPUPriority {return [self readPriority:YES];} +- (BOOL)setGPUPriority:(NSUInteger)priority {return [self writePriority:priority background:NO];} +- (BOOL)setBackgroundGPUPriority:(NSUInteger)priority {return [self writePriority:priority background:YES];} +- (id)commandBuffer { + OMHandle request={self.handle};OMResult result={};checked(OM_COMMAND,&request,sizeof(request),&result); + OMCommandObject *object=[OMCommandObject new];object.handle=result.handle;object.commandQueue=(id)self;object.useAsync=asyncEnabled;return (id)object; +} +@end +// This model contributes only own implementations to a dynamically allocated +// real _MTLDevice subclass. No original method implementation is overwritten. +@interface OMDeviceMethods : NSObject +@end +@implementation OMDeviceMethods +- (NSString *)name {return deviceInfo[@"name"];} +- (uint64_t)registryID {return [deviceInfo[@"registryID"] unsignedLongLongValue];} +- (NSUInteger)maxBufferLength {return [deviceInfo[@"maxBufferLength"] unsignedIntegerValue];} +- (BOOL)isLowPower {return [deviceInfo[@"lowPower"] boolValue];} +- (BOOL)isRemovable {return [deviceInfo[@"removable"] boolValue];} +- (BOOL)isHeadless {return [deviceInfo[@"headless"] boolValue];} +// This partial offline device does not implement any complete GPU family or +// feature set. The original base implementation needs Apple capability tables +// which this independent device neither owns nor populates. +- (BOOL)supportsFamily:(MTLGPUFamily)family {(void)family;return NO;} +- (BOOL)supportsFeatureSet:(MTLFeatureSet)featureSet {(void)featureSet;return NO;} +- (id)newEvent { + OMResult result={};call(OM_EVENT,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual GPU event identity");OMEventObject *object=[OMEventObject new];object.handle=result.handle;return (id)object; +} +- (id)newSharedEvent { + OMResult result={};NSData *body=call(OM_SHARED_EVENT,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual shared-event identity");NSDictionary *info=sharedMetadata(body); + OMSharedEventObject *object=[OMSharedEventObject new];object.handle=result.handle;object.deviceIsNil=[info[@"device_is_nil"] boolValue]; + // Initialize without writing metadata back to the actual event. + ((void (*)(id,SEL,id))class_getMethodImplementation([OMObject class],@selector(setLabel:)))(object,@selector(setLabel:),info[@"label"]==[NSNull null]?nil:info[@"label"]); + return object; +} +- (id)newSharedEventWithHandle:(MTLSharedEventHandle *)handle { + Class cls=NSClassFromString(@"MTLSharedEventHandle");if(!handle || object_getClass(handle)!=cls || class_getInstanceSize(cls)!=16)return nil; + mach_port_t port=handle.eventPort;if(!MACH_PORT_VALID(port))return nil; + [transaction lock];@try { + if(!live || closing)return nil;uint64_t input=port,output=0;unsigned count=1;unsigned char bytes[4096];size_t size=sizeof(bytes); + int status=ioMethod(connection,3,&input,1,nullptr,0,&output,&count,bytes,&size);if(status)return nil; + if(count!=1 || !output)fail(@"Missing actual imported shared event");NSDictionary *info=sharedMetadata([NSData dataWithBytes:bytes length:size]); + OMSharedEventObject *object=[OMSharedEventObject new];object.handle=output;object.deviceIsNil=[info[@"device_is_nil"] boolValue]; + ((void (*)(id,SEL,id))class_getMethodImplementation([OMObject class],@selector(setLabel:)))(object,@selector(setLabel:),info[@"label"]==[NSNull null]?nil:info[@"label"]); + return object; + } @finally {[transaction unlock];} +} +- (id)newFence { + OMResult result={};call(OM_FENCE,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual GPU fence identity");OMFenceObject *object=[OMFenceObject new];object.handle=result.handle;return (id)object; +} +- (id)newCommandQueue { + OMResult result={};checked(OM_QUEUE,nullptr,0,&result);OMQueueObject *object=[OMQueueObject new];object.handle=result.handle;return (id)object; +} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options { + if(!length || length>MIN((NSUInteger)OFFLINE_METAL_MAX_BUFFER,self.maxBufferLength) || options)return nil; + OMBuffer request={length,(uint64_t)options};OMResult result={};call(OM_BUFFER,&request,sizeof(request),&result);if(result.error)return nil; + OMBufferObject *object=[OMBufferObject new];object.handle=result.handle;object.data=[NSMutableData dataWithLength:length];return (id)object; +} +- (id)newBufferWithBytes:(const void *)bytes length:(NSUInteger)length options:(MTLResourceOptions)options { + if(!bytes)return nil;OMBufferObject *object=(OMBufferObject *)[self newBufferWithLength:length options:options];if(!object)return nil; + memcpy(object.data.mutableBytes,bytes,length);return (id)object; +} +- (id)newPipelineLibraryWithFilePath:(NSString *)path error:(NSError **)error { + // This optional original directory contains Apple-family pipeline entries. + // No backend qualification exists for that private archive format. Refuse + // explicitly so the unchanged caller can take its real AIR-library path, + // rather than invoking the Apple base implementation with absent internals. + (void)path; + if(error){*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 + userInfo:@{NSLocalizedDescriptionKey:@"Apple-family pipeline libraries are not qualified for this independent device; use portable AIR input"}];} + return nil; +} +- (id)newLibraryWithFile:(NSString *)path error:(NSError **)error { + NSError *failure=nil;NSData *bytes=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:&failure]:nil; + if(!bytes){if(error)*error=failure?:[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"A readable actual library file is required"}];return nil;} + dispatch_data_t data=dispatch_data_create(bytes.bytes,bytes.length,dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{(void)bytes.length;}); + return [(id)self newLibraryWithData:data error:error]; +} +- (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"An actual local library file URL is required"}];return nil;} + return [(id)self newLibraryWithFile:url.path error:error]; +} +- (id)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(NSError **)error { + NSURL *url=[bundle URLForResource:@"default" withExtension:@"metallib"]; + if(!url){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"The actual bundle has no default Metal library"}];return nil;} + return [(id)self newLibraryWithURL:url error:error]; +} +- (id)newLibraryWithData:(dispatch_data_t)data error:(NSError **)error { + const void *bytes=nullptr;size_t size=0;dispatch_data_t mapped=dispatch_data_create_map(data,&bytes,&size);(void)mapped; + if(!bytes || !size || size>64*1024*1024){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:nil];return nil;} + OMLibrary request={};CC_SHA256(bytes,(CC_LONG)size,request.sha256);OMResult result={};NSData *body=call(OM_LIBRARY,&request,sizeof(request),&result); + if(result.error){if(error)*error=operationError(result,body);return nil;} + NSArray *names=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![names isKindOfClass:[NSArray class]])fail(@"Missing actual library metadata"); + OMLibraryObject *object=[OMLibraryObject new];object.handle=result.handle;object.functionNames=names;if(error)*error=nil;return (id)object; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d { + if(!d || d.hazardTrackingMode>MTLHazardTrackingModeTracked || !d.allowGPUOptimizedContents)return nil; + OMTexture request={(uint64_t)d.textureType,(uint64_t)d.pixelFormat,d.width,d.height,d.depth,d.mipmapLevelCount,d.sampleCount,d.arrayLength,(uint64_t)d.usage,(uint64_t)d.storageMode,(uint64_t)d.cpuCacheMode}; + OMHazardTexture extended={request,(uint64_t)d.hazardTrackingMode};OMResult result={};NSData *body=call(OM_TEXTURE_HAZARD,&extended,sizeof(extended),&result);if(result.error)return nil; + if(body.length!=sizeof(uint64_t))fail(@"Missing actual texture hazard mode");uint64_t actual=0;memcpy(&actual,body.bytes,sizeof(actual)); + if(actual>MTLHazardTrackingModeTracked || (extended.hazard && actual!=extended.hazard))fail(@"Actual texture hazard mode differs"); + OMTextureObject *object=[OMTextureObject new];object.handle=result.handle;object.descriptor=request;object.actualHazardMode=(MTLHazardTrackingMode)actual;return (id)object; +} +- (id)newDepthStencilStateWithDescriptor:(MTLDepthStencilDescriptor *)d { + if(!d)return nil; + auto face=[](MTLStencilDescriptor *s)->OMStencilFace { + if(!s)return {};return {1,(uint64_t)s.stencilCompareFunction,(uint64_t)s.stencilFailureOperation,(uint64_t)s.depthFailureOperation, + (uint64_t)s.depthStencilPassOperation,s.readMask,s.writeMask}; + }; + OMDepthStencil request={(uint64_t)d.depthCompareFunction,(uint64_t)d.depthWriteEnabled,face(d.frontFaceStencil),face(d.backFaceStencil)}; + OMResult result={};call(OM_DEPTH_STENCIL_STATE,&request,sizeof(request),&result);if(result.error)return nil; + OMDepthStateObject *object=[OMDepthStateObject new];object.handle=result.handle;return (id)object; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d iosurface:(IOSurfaceRef)surface plane:(NSUInteger)plane { + if(!surface || CFGetTypeID(surface)!=IOSurfaceGetTypeID() || !d || plane || IOSurfaceGetPlaneCount(surface) || d.storageMode!=MTLStorageModeShared)return nil; + bool format=((d.pixelFormat==MTLPixelFormatR32Float && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_OneComponent32Float) + || (d.pixelFormat==MTLPixelFormatRGBA8Unorm && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_32RGBA) + || (d.pixelFormat==MTLPixelFormatBGRA8Unorm && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_32BGRA)); + if(!format + || !d.width || !d.height || d.width!=IOSurfaceGetWidth(surface) || d.height!=IOSurfaceGetHeight(surface) + || d.width>SIZE_MAX/4 || IOSurfaceGetBytesPerElement(surface)!=4 || IOSurfaceGetBytesPerRow(surface)SIZE_MAX/d.height || IOSurfaceGetAllocSize(surface))object; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + static_assert(MTLVertexFormatFloat4==31 && MTLVertexFormatHalf4==27 && MTLVertexStepFunctionPerInstance==2,"Actual target SDK vertex enum ABI"); + bool vertexInputs=false;OMVertexPipeline vertexPipeline={}; + // Read real original descriptors. Inactive attributes/layouts carry zeroes; + // static stepping is transported without modifying any original descriptor. + if(d.vertexDescriptor)for(unsigned i=0;i<31;i++) { + MTLVertexAttributeDescriptor *a=d.vertexDescriptor.attributes[i];if(a.format==MTLVertexFormatInvalid)continue; + vertexInputs=true;vertexPipeline.attributes[i]={(uint64_t)a.format,a.offset,a.bufferIndex}; + if(a.bufferIndex<31) { + MTLVertexBufferLayoutDescriptor *l=d.vertexDescriptor.layouts[a.bufferIndex]; + vertexPipeline.layouts[a.bufferIndex]={l.stride,(uint64_t)l.stepFunction,l.stepRate}; + } + } + if(![(id)d.vertexFunction isKindOfClass:[OMFunctionObject class]] || ![(id)d.fragmentFunction isKindOfClass:[OMFunctionObject class]] + || (vertexInputs && !OMVertexInputsValid(vertexPipeline)) || !d.rasterizationEnabled || d.alphaToCoverageEnabled || d.alphaToOneEnabled) { + if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 userInfo:nil];return nil; + } + OMAttachmentPipeline attachments={};OMBlendPipeline &extended=attachments.colors;OMPipeline &request=extended.pipeline;bool blending=false; + attachments.depthFormat=d.depthAttachmentPixelFormat;attachments.stencilFormat=d.stencilAttachmentPixelFormat; + request.vertex=[(OMObject *)d.vertexFunction handle];request.fragment=[(OMObject *)d.fragmentFunction handle];request.samples=d.rasterSampleCount; + for(unsigned i=0;i<8;i++) { + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];request.formats[i]=a.pixelFormat;request.masks[i]=a.writeMask; + blending|=a.blendingEnabled;extended.colors[i]={(uint64_t)a.blendingEnabled,(uint64_t)a.sourceRGBBlendFactor,(uint64_t)a.destinationRGBBlendFactor, + (uint64_t)a.rgbBlendOperation,(uint64_t)a.sourceAlphaBlendFactor,(uint64_t)a.destinationAlphaBlendFactor,(uint64_t)a.alphaBlendOperation}; + } + bool attached=attachments.depthFormat || attachments.stencilFormat; + vertexPipeline.pipeline=attachments; + OMResult result={};NSData *body=call((vertexInputs || attached)?OM_PIPELINE_ATTACHMENTS:blending?OM_PIPELINE_BLEND:OM_PIPELINE, + vertexInputs?(const void *)&vertexPipeline:attached?(const void *)&attachments:blending?(const void *)&extended:(const void *)&request, + vertexInputs?sizeof(vertexPipeline):attached?sizeof(attachments):blending?sizeof(extended):sizeof(request),&result);if(result.error){if(error)*error=operationError(result,body);return nil;} + OMPipelineObject *object=[OMPipelineObject new];object.handle=result.handle;return (id)object; +} +@end +extern "C" id offline_metal_frontend_open(void); +extern "C" id offline_metal_frontend_open_service(const char *); +extern "C" void offline_metal_frontend_finish(void); +id offline_metal_frontend_open(void) { + return offline_metal_frontend_open_service("NativeVirtioConsole"); +} +id offline_metal_frontend_open_service(const char *serviceName) { + if(!serviceName || (strcmp(serviceName,"NativeVirtioConsole") && strcmp(serviceName,"NativeOfflineGPU")))return nil; + if(live)return nil;void *iokit=dlopen("/System/Library/Frameworks/IOKit.framework/IOKit",RTLD_NOW|RTLD_LOCAL);if(!iokit)return nil; + auto matching=(void *(*)(const char *))dlsym(iokit,"IOServiceMatching");auto services=(int (*)(unsigned,void *,unsigned *))dlsym(iokit,"IOServiceGetMatchingServices"); + auto next=(unsigned (*)(unsigned))dlsym(iokit,"IOIteratorNext");auto release=(int (*)(unsigned))dlsym(iokit,"IOObjectRelease");auto open=(int (*)(unsigned,unsigned,unsigned,unsigned *))dlsym(iokit,"IOServiceOpen"); + ioMethod=(IOMethod)dlsym(iokit,"IOConnectCallMethod");if(!matching || !services || !next || !release || !open || !ioMethod)return nil; + unsigned iterator=0;if(services(0,matching(serviceName),&iterator) || !iterator)return nil;unsigned service=next(iterator); + unsigned extra=next(iterator);int openStatus=-1; + if(service && !extra)openStatus=open(service,mach_task_self(),0,&connection); + if(extra)release(extra);if(service)release(service);release(iterator);if(openStatus)return nil; + transaction=[NSRecursiveLock new];pendingLock=[NSLock new];pendingCommands=[NSMutableSet set];pendingNotifications=[NSMutableSet set];notificationFailure=nil;asyncEnabled=false;closing=false;live=true;OMResult result={};NSData *body=checked(OM_DEVICE,nullptr,0,&result); + deviceInfo=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![deviceInfo isKindOfClass:[NSDictionary class]])fail(@"Actual device metadata missing"); + Class base=objc_getClass("_MTLDevice");if(!base || class_getInstanceSize(base)!=536)fail(@"Wrong original Metal base ABI"); + Class cls=objc_allocateClassPair(base,"NativeOfflineMetalDevice",0);if(!cls)fail(@"Own device class allocation failed"); + unsigned count=0;Method *methods=class_copyMethodList([OMDeviceMethods class],&count); + for(unsigned i=0;i=1 && format<=12)return 2+(format-1)%3; + if(format>=13 && format<=24)return 2*(2+(format-13)%3); + if(format>=25 && format<=27)return 2*(format-23); + if(format>=28 && format<=39)return 4*(1+(format-28)%4); + if(format>=40 && format<=42)return 4; + if(format>=45 && format<=48)return 1; + if(format>=49 && format<=53)return 2; + if(format==54 || format==55)return 4; + return 0; +} +static inline bool OMVertexInputsValid(const OMVertexPipeline &p) { + bool used[31]={};unsigned count=0; + for(const auto &a:p.attributes) { + if(!a.format){if(a.offset || a.buffer)return false;continue;} + unsigned bytes=OMVertexFormatBytes(a.format); + if(!bytes || a.buffer>=31 || a.offset>4096-bytes)return false; + used[a.buffer]=true;count++; + } + if(!count)return false; + for(unsigned i=0;i<31;i++) { + const auto &l=p.layouts[i]; + if(!used[i]){if(l.stride || l.step || l.rate)return false;continue;} + if(l.stride>OFFLINE_METAL_MAX_VERTEX_STRIDE || l.step>2 || l.rate>65535 || (l.step && (!l.rate || !l.stride)))return false; + } + return true; +} diff --git a/OfflineMetalWire.h b/OfflineMetalWire.h new file mode 100644 index 0000000..90f644e --- /dev/null +++ b/OfflineMetalWire.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT +// Offline Metal command transport. No deployment GPU or page-table ABI. +#pragma once +#include +enum { OM_WIRE_MAGIC=0x5550474e, OM_WIRE_RESPONSE=0x8000 }; +typedef struct {uint32_t magic;uint16_t version,operation;uint32_t bytes,request;} OMWireHeader; +static_assert(sizeof(OMWireHeader)==16,"Native Metal wire header"); +enum { OFFLINE_METAL_VERSION=2, OFFLINE_METAL_MAX_REPLY=16*1024*1024, + OM_DEVICE=0x100,OM_QUEUE,OM_LIBRARY,OM_FUNCTION,OM_TEXTURE,OM_PIPELINE, + OM_COMMAND,OM_RENDER,OM_SET_PIPELINE,OM_SET_CULL,OM_DRAW,OM_END, + OM_COMMIT,OM_READ_TEXTURE,OM_RELEASE,OM_FINISH,OM_WRITE_TEXTURE, + OM_BUFFER,OM_WRITE_BUFFER,OM_READ_BUFFER,OM_VERTEX_BUFFER,OM_FRAGMENT_BUFFER, + OM_DRAW_INDEXED,OM_BLIT,OM_COPY_BUFFER,OM_PIPELINE_BLEND,OM_VERTEX_TEXTURE,OM_FRAGMENT_TEXTURE, + OM_SCISSOR,OM_VIEWPORT,OM_BLEND_COLOR }; +enum { OFFLINE_METAL_MAX_BUFFER=64*1024*1024 }; +typedef struct {int32_t error;uint32_t reserved;uint64_t handle;} OMResult; +typedef struct {uint64_t handle;} OMHandle; +typedef struct {uint8_t sha256[32];} OMLibrary; +typedef struct {uint64_t library;char name[128];} OMFunction; +typedef struct {uint64_t type,format,width,height,depth,levels,samples,array,usage,storage,cache;} OMTexture; +typedef struct {uint64_t vertex,fragment,samples,formats[8],masks[8];} OMPipeline; +typedef struct {uint64_t texture,load,store;double clear[4];} OMColor; +typedef struct {uint64_t command;OMColor colors[8];} OMRender; +typedef struct {uint64_t encoder,pipeline;} OMSetPipeline; +typedef struct {uint64_t encoder,mode;} OMSetCull; +typedef struct {uint64_t encoder,primitive,start,count,instances;} OMDraw; +typedef struct {uint64_t status,start_ns,end_ns;} OMCommandResult; +typedef struct {uint64_t texture,x,y,width,height,level;} OMReadTexture; +typedef struct {uint64_t texture,x,y,width,height,level,stride;} OMWriteTexture; +typedef struct {uint64_t bytes,options;} OMBuffer; +typedef struct {uint64_t buffer,offset,bytes;} OMBufferRegion; +typedef struct {uint64_t encoder,buffer,offset,index;} OMSetBuffer; +typedef struct {uint64_t encoder,primitive,count,type,buffer,offset,instances;int64_t baseVertex;uint64_t baseInstance;} OMDrawIndexed; +typedef struct {uint64_t encoder,source,sourceOffset,destination,destinationOffset,bytes;} OMCopyBuffer; +typedef struct {uint64_t enabled,sourceRGB,destinationRGB,rgbOperation,sourceAlpha,destinationAlpha,alphaOperation;} OMBlend; +typedef struct {OMPipeline pipeline;OMBlend colors[8];} OMBlendPipeline; +typedef struct {uint64_t encoder,texture,index;} OMSetTexture; +typedef struct {uint64_t encoder,x,y,width,height;} OMScissor; +typedef struct {uint64_t encoder;double x,y,width,height,nearDepth,farDepth;} OMViewport; +typedef struct {uint64_t encoder;float red,green,blue,alpha;} OMBlendColor; diff --git a/OriginalAcceleratorABI.h b/OriginalAcceleratorABI.h new file mode 100644 index 0000000..a052b09 --- /dev/null +++ b/OriginalAcceleratorABI.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +#pragma once +// Exact supplied iOS17 base, independently audited against its complete +// original vtable and protected constructor/destructor bodies. No extra fields +// or operational overrides beyond IOService. This declares a real ancestry; +// no original class, method, vtable or metaclass is replaced. +#include +class IOAcceleratorES : public IOService { + OSDeclareAbstractStructors(IOAcceleratorES) +}; +static_assert(sizeof(IOService)==136 && sizeof(IOAcceleratorES)==136,"Qualified original accelerator base extent"); diff --git a/OriginalGPUProviderABI.h b/OriginalGPUProviderABI.h new file mode 100644 index 0000000..78ecf5b --- /dev/null +++ b/OriginalGPUProviderABI.h @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +#pragma once +// Unpublished ABI instrument only. The native service prefix and opaque storage +// match the exact original abstract base. Non-pure tail entries are bound to +// their ORIGINAL entry addresses, with no C++ caller supplied for those opaque +// source signatures. Required tail probes capture their actual register frame; +// descriptor slot170 may return NULL only for an explicit missing-resource +// control. No unknown signature, capability or successful +// operational implementation is claimed. This header is not a system driver API. +#include "OriginalAcceleratorABI.h" +class IOMemoryDescriptor; +class IOGPU : public IOAcceleratorES { + OSDeclareAbstractStructors(IOGPU) + unsigned char originalPrivateState[600]; +protected: + void free() override; +public: + IOReturn configureReport(IOReportChannelList *,IOReportConfigureAction,void *,void *) override; + IOReturn updateReport(IOReportChannelList *,IOReportUpdateAction,void *,void *) override; + bool start(IOService *) override; + void stop(IOService *) override; + IOWorkLoop *getWorkLoop() const override; + using IOAcceleratorES::newUserClient; + IOReturn newUserClient(task_t,void *,UInt32,IOUserClient **) override; +protected: + virtual void abiSlot168(); + virtual void abiSlot169(); + virtual IOMemoryDescriptor *abiSlot170(UInt32 *) = 0; + virtual void abiSlot171(); + virtual void abiSlot172(); + virtual void abiSlot173(); + virtual void abiSlot174() = 0; + virtual void abiSlot175(); + virtual void abiSlot176(); + virtual void abiSlot177(); + virtual void abiSlot178(); + virtual void abiSlot179(); + virtual void abiSlot180(); + virtual void abiSlot181(); + virtual void abiSlot182(); + virtual void abiSlot183(); + virtual void abiSlot184(); + virtual void abiSlot185(); + virtual void abiSlot186(); + virtual void abiSlot187(); + virtual void abiSlot188(); + virtual void abiSlot189() = 0; + virtual void abiSlot190(); + virtual void abiSlot191(); + virtual void abiSlot192(void *) = 0; + virtual void abiSlot193(); + virtual void abiSlot194(); + virtual bool abiSlot195(IOService *) = 0; + virtual void abiSlot196() = 0; + virtual void abiSlot197() = 0; + virtual void abiSlot198(); + virtual void abiSlot199(); + virtual void abiSlot200() = 0; + virtual void abiSlot201(); + virtual void abiSlot202() = 0; + virtual void abiSlot203(); + virtual void abiSlot204(); + virtual void abiSlot205() = 0; + virtual void abiSlot206(); + virtual void abiSlot207(); + virtual void abiSlot208(); + virtual void abiSlot209(); + virtual void abiSlot210(); + virtual void abiSlot211(); + virtual void abiSlot212(); + virtual void abiSlot213(); + virtual void abiSlot214(); +}; +static_assert(sizeof(IOGPU)==736,"Exact original provider storage extent"); diff --git a/README.md b/README.md new file mode 100644 index 0000000..e41fd1d --- /dev/null +++ b/README.md @@ -0,0 +1,1074 @@ +# NativeMetal + +## Scheduling and completion separation (offline, 2026-10-20) + +The native frontend now forwards scheduled callbacks and `waitUntilScheduled` +to the runtime's actual scheduling transition. Previously it called scheduled +handlers from completion and implemented the scheduling wait as a completion +wait. That also serialized `commitAndWaitUntilSubmitted` with GPU completion. +Completed callbacks keep their existing completion-queue behavior; scheduled +callbacks finish before the runtime advances to completion. Frontend encoding +refusals and commits refused by the runtime unblock waits without announcing a +scheduling event. A refused commit releases its callback ownership reference. + +`bridge/scheduled_host.mm` uses a bounded semaphore to hold a completed handler: +the scheduling wait must return independently, and the scheduled handler must +observe `Scheduled`, before completion. It covers the runtime thread, an explicit +serial completion queue, a frontend encoding refusal, and an open-encoder commit +refusal. The old frontend fails the successful-command case. Telemetry adds +separate scheduling/completion wait counts and elapsed times (`J` rows). + +The host qualification and ARM64/iOS17 cross-build use metal2vulkan revision +`720ef1ce9ae2` with the umbrella's recorded patches. These checks +qualify API ordering over a host Vulkan driver, not Adreno timing, system UI +frame rate, panel presentation, or cold-start shader coverage. This is a +confirmed serialization defect, not proof of the sole cause of hardware lag. + +The render pipeline creation entry points also forward descriptor +`binaryArchives` and the raw `MTLPipelineOption` bits to the runtime. Previously +the archive could be populated/serialized but creation always used default +options, silently compiling even when a listed empty archive should fail under +`FailOnBinaryArchiveMiss`. Foreign archive objects are refused. The C ABI keeps +the original default entry point and adds a counted archive list for creation. +`bridge/archive_host.mm` proves empty/changed-descriptor strict misses, then +serializes and reloads an archive in a second process with an independent fresh +shader-cache directory, and checks the rendered pixels over the host GPU. + +These archives contain the translator's artifacts, not Adreno machine code. +The runtime does not consult them for later draw variants, and Vulkan driver +pipeline compilation still requires its separate, identity-matched cache. +This change repairs render archive consumption; it does not seed the system +image or qualify complete CoreAnimation shader/variant coverage or a cold-frame +latency bound. Compute descriptor archive consumption remains separate work. + +Timing schema v2 attributes encoder API counts to the command buffer that +received them, rather than subtracting global counts at commit. Commit indices +are assigned under the ring lock so simultaneous commits cannot reuse an index. +The historical `gpu=` field is now `ctc=` (CPU commit-to-completion-callback). +The latest four buffers also report creation-to-commit age, which includes +caller idle time, and commit-to-scheduled-callback time (`-1` if not observed). +Pipeline/library/function and Vulkan/cache totals occupy separate rows to fit +the 90-character transport limit. The worst-case table stays within 48 rows. + +`bridge/telemetry_host.mm` interleaves render/blit encoding on two queues and +commits in reverse order, then checks both the attributed work and actual host +GPU pixels. The previous global-delta implementation fails this test. Counts +describe API requests, not proof that the GPU executed every requested command. + +## Resource-specific completion waits (offline, 2026-10-20) + +The resource SPI `waitUntilComplete` no longer drains every in-flight command +buffer. It asks the runtime whether an already-committed buffer references the +resource's storage. Texture views resolve to their root or backing buffer; +heap resources conservatively share a dependency, and imported host ranges are +compared for overlap without reading their bytes. An unknown bridge result +preserves the conservative wait. The frontend has no live remap/rebind API; +this query is not a remapping lifetime protocol. Completion and actual GPU +results remain the runtime's. + +`bridge/patches/0002-resource-wait-storage-dependencies.patch` records the exact +runtime extension. The umbrella builder verifies the complete source diff for +both patches. This changes the runtime source digest and therefore the shader +cache build identity; artifacts must be produced for this exact build. + +The bounded `resource_wait_host.mm` test holds pending work behind an unsignaled +shared event. The previous frontend blocks even for a resource the work never +uses. The fixed frontend returns for that resource and keeps waiting for the +actual buffer, a buffer-backed texture view, nested shared/private texture +views, two imports of one IOSurface, heap storage and overlapping no-copy +buffers. It then signals the event and checks the actual host GPU bytes, +using GPU readback for private resources. J rows report resource wait time and +dependency checks/skips; the maximum table now occupies 47 of 48 rows. + +This removes a reproduced serialization defect. Static target-code callsites +do not establish its per-frame frequency, hardware speedup, stable 120 FPS or +the sole cause of lag. No hardware boot or timing qualification is implied. + +Native ARM64/iOS 17 Metal API frontend for the original iPadOS graphics stack. +The device is published through MTLAddDevice. The frontend translates AIR with +metal2vulkan and submits through Turnip and NativeAdreno. The desktop is +GPU-composited but slow; shader compilation is not fixed. + +`tools/MetalVulkanProbe.rs` is an explicit offline consumer of metal2vulkan's +public compiler API. The umbrella pins the upstream source and locked Cargo +dependencies. The probe selects `exact_answers()` and separately selects +exact sampling; it still exports all precision notes, resource requirements, +specialization and runtime dependencies. Those switches are not an assertion +of numerical equivalence. The executable links no Vulkan loader and never +publishes a Metal device. It emits SPIR-V for independent validation, not +Adreno instructions or completion events. + +The accompanying vertex layout is an authored diagnostic interleaving of +float4/float2/float4 attributes at locations 0/1/3, with a 48-byte stride. +It is supplied explicitly by the caller, not inferred as a system runtime +layout. Missing descriptors/constants and unsupported stages remain errors. +Fragment compilation requires an explicit attachment map. The coverage +attachment fixture enables R32Float at attachment 2; silently attaching only +target 0 would let these coverage shaders optimize into empty fragments. +The umbrella records actual SPIR-V Output-variable stores and can require +them for selected fragment tests. A store instruction is still not proof of +shader execution or correct pixel values. +Target parsing, pipeline translation and SPIR-V validation require separate +evidence from shader execution, native GPU4K translation, submission/fences, +IOSurface sharing and display presentation. + +The partial device explicitly returns false for every GPU-family and feature-set +query. Its implemented subset does not qualify a complete family. This also +avoids the original base class dereferencing Apple capability tables which +this independent device does not initialize. The original RenderBox RBDevice +initializer, device getter and internal dispatch-queue getter now execute in +both actual original-runtime diagnostic workers; their subsequent original +IOSurface sharing and four real GPU completions still succeed. This qualifies +construction only, not original RenderBox rendering or system device discovery. + +Queue foreground/background priority queries and setters forward to the actual +host queue after validating its real Objective-C method ABI. No priority or +successful change is synthesized. Host qualification changes and restores a +real foreground priority, preserves a peer queue, and refuses foreign/wrong +objects and invalid arguments. Original RenderBox executes both real +priority queries in each native diagnostic worker. No RenderBox frame is qualified. +The broker explicitly allows the two new bounded operation IDs; its +native-4K candidate retains all 29 original executable component hashes. + +An interrupted backend session now retains its actual operation/completion +ledger with transport_healthy=false and session_finished=false. The caller +must return failure for such a report. A real disconnect control verifies +the incomplete report contains no fabricated command completion. + +Explicit command-buffer enqueue now reserves the real host Metal queue and +returns its actual Enqueued status. The synchronous wire permits one pending +reservation per queue; duplicate/foreign requests and unsupported predecessor +ordering refuse before entering a blocking host wait. Independent queues can +both reserve and execute. Host qualification completes four GPU commands and +verifies all returned pixels, including retrying a second reservation after +its real predecessor finishes. Actual original RenderBox enqueues both +native workers' real buffers. Pending reservations are never reported as GPU completions. + +Registered completion and scheduled handlers execute once on a genuine +dispatch worker after verified GPU completion and original surface/buffer +synchronization. waitUntilCompleted also waits for those handlers with a +bounded timeout. Scheduled callbacks are late in this synchronous transport; +no early asynchronous scheduling timing is claimed. The original target's +BOOL-returning commitAndWaitUntilSubmitted interface derives success only +after a real successful commit. The two-process surface regression exercises +both public/private commit paths, real enqueue status, ordered once-only +callbacks with actual completion/timestamps, shared-surface seed coherence and +post-peer-exit GPU work. All five native broker/permission controls also pass +against the same new candidate. Original RenderBox's convenience image path +can return an image with only empty command buffers; that image is explicitly +not qualified as GPU rendering. Actual renderer resource use/output must still +be established before normal device discovery or any desktop claim. + +The frontend dynamically derives from the supplied target's actual 536-byte +_MTLDevice class and adds only its own method implementations. Original +implementations are not overwritten. Actual original descriptor objects +are marshalled through bounded typed requests over NativeVirtioConsole's +administrator userclient. Host replies carry real object handles, GPU +completion/timing and texture bytes. Unsupported operations refuse. + +Implemented operations create queues, known original libraries, named +functions, simple render pipelines, shared R32Float/BGRA8Unorm/RGBA8Unorm 2D textures, command +buffers and render encoders. Pipeline/cull/draw/end/commit operations call +real host Metal APIs. Commit waits for actual GPU completion; texture reads +copy actual getBytes output. Proxy deallocation drops its real host table +reference, while actual Metal command buffers retain their own resources. +Session finish drops remaining references. There is no CPU pixel renderer. + +RGBA8Unorm now pairs with real original 32RGBA IOSurfaces as requested by +original RenderBox. Upload/readback retain actual channel bytes without CPU +swizzling. The native two-process sharing/callback regression uses RGBA: +four real GPU completions, five full readbacks, original seed coherence, +Mach authorization/refusal and post-peer-exit work pass with correct channel +order. The separate BGRA two-process/permission regression also passes on the +same source/candidate. Original RenderBox allocates both actual RGBA surface +textures before its next real private RG16Float resource refusal; no original +renderer frame is qualified by those allocations. + +Private RG16Float textures and actual bounded texture blits are now implemented. +Private textures never use CPU getBytes/replaceRegion; shared staging is filled +by a genuine Metal blit. The native two-worker resource instrument clears each +private texture and a contrasting shared destination on the GPU, copies between +them, and checks every half-float result. Invalid bounds and direct private CPU +reads refuse in both workers. Together with the original RGBA surface sequence, +six actual commands, twelve ordered callbacks and seven complete readbacks pass, +with distinct live native-4K worker CPU contexts and no pending reservations. +All five broker/permission controls pass on the fresh texture-copy candidate. +Blit destinations backed by original IOSurfaces join the real post-completion +readback/seed path; no CPU pixel conversion or private-memory mapping is used. +Private Stencil8 is now supported by the attachment/state extension below. + +Private Stencil8/Depth32Float resources, real depth/stencil states, reference +values and pipeline/render attachments are implemented. Both actual native +workers execute the unchanged original shader pair through stencil mismatch/ +match and depth reject/pass controls, checking every returned pixel. Together +with the RGBA surface sequence, twelve real commands, twenty-four ordered +callbacks and thirteen complete readbacks pass; private stencil CPU reads and +wrong-type state bindings refuse. All five current-candidate broker controls +pass. Combined Depth32Float_Stencil8 also passes the corresponding real host +GPU tests, but has no separate native combined-attachment qualification yet. +Original RenderBox now reaches its inherited optional Apple pipeline-library +loader and faults on a null internal object. No original renderer frame is +qualified. An interrupted active encoder also exposes a host teardown assertion; +both are retained explicit failures, to fix before progressing the renderer. + +Shared/default-cache buffers now have real host allocations and stable native +contents mirrors. Bound buffers are retained by their command; their actual +current bytes upload immediately before GPU commit, and actual buffer bytes +download after completion. Thus CPU writes after encoding remain visible to +the GPU. This is explicit copying, not shared physical GPU memory. Allocation +is bounded at 64 MiB and each diagnostic session at 8192 operations; arbitrary +large workloads/session combinations are not qualified. Other storage/options +refuse. Vertex/fragment bindings, inline byte bindings, indexed draws and +aligned nonoverlapping GPU buffer blits use actual corresponding Metal calls. + +The native resource/presentation instrument qualifies 84-byte original +RenderBox uniforms and 12-byte indices. A real GPU blit replaces a distinct +destination pattern, and that actual destination becomes the original +accumulator_gradient_vertex input. Indexed triangles produce the right-half +image, then a uniform update after encoding produces a full image. All pixels +reach actual GOP and are read back. Exit 163 and absent/default controls pass; +this does not establish complete Metal capabilities, system UI or Adreno. + +Original texture-read shaders now execute through real vertex/fragment +texture bindings. Commands retain referenced textures and recheck real +IOSurface generation seeds before commit. GPU blend pipelines carry actual +per-attachment factors/operations; viewport, bounded scissor and constant +blend color use real Metal state. Unsupported multisampling and invalid +clipping refuse before issuing the unsupported GPU operation. + +The native composition instrument creates its two source images on the GPU, +then uses unchanged read_surf_vert/read_surf_frag to copy and blend them. It +qualifies full blue, clipped premultiplied composition and constant-factor +composition through actual IOSurface and GOP, checking every pixel. Exit 167 +and both device/default controls pass. This is actual GPU composition, not a +CPU-generated image or the original system desktop. Filtered samplers, +function constants, complete resource/format/capability contracts, system +discovery and original framebuffer swap integration remain incomplete. + +Basic nonplanar R32Float guest IOSurfaces now synchronize with host textures. +BGRA8Unorm also supports the original framework's actual 32BGRA surfaces; +descriptor/surface format mismatches and unsupported planes refuse. A genuine +GPU clear and the unchanged original attachment_clear_frag_lpf shader operate +on this format, with full native surface/readback hashes and actual ordered +GPU timing. The BGRA instrument exits 151. The bounded host session allows +8192 operations so full display-size surface uploads can fit; individual +request/reply and texture extent limits remain unchanged. This does not add +system device discovery, display presentation or physical Adreno execution. +The frontend uses the original IOSurface framework's actual locks, mapped +base/pitch/allocation and generation seed. Bounded byte uploads preserve +existing native surface contents; actual completed GPU output is copied back +before the frontend reports completion. This is explicit offline transfer, +not shared guest/host GPU physical memory or deployment zero-copy. Unsupported +planes/formats/extents refuse. Command buffers retain their surface targets. + +The parent native IOSurface instrument actually creates a 64x64 surface, +exports/imports its real Mach port and verifies both references see the GPU +output. Initial float-two pixels upload unchanged; a real GPU clear returns +float-one to the surface, which becomes the next texture's actual upload. +An original shader draw then returns zeros through that same original surface. +The actual native exit is 149, with active CPU 4K and original AMFI/root +retained. Full IOSurface cross-process/presentation semantics remain open. + +Libraries use a bounded content-addressed cache of the exact original +QuartzCore and RenderBox files supplied to the host instrument. The guest +hashes complete actual newLibraryWithData input. Unknown identities refuse; +known input selects the already hash-qualified unchanged AIR slice of that +same original file. Arbitrary named functions are obtained from the real +host library. No function body or target metadata is rewritten. General +library uploads, specialization constants and source compilation remain open. + +The parent native instrument performs an actual clear and an original +QuartzCore shader draw using normal Metal API calls, checks every returned +pixel and GPU timing, and exits 119. Unknown library/function controls refuse. +Automatic original MTLCreateSystemDefaultDevice still returns nil: this +explicit partial object is not advertised as a completed system driver. +Active CPU translations remain native 4 KiB; original AMFI/root checks +remain active. No active GPU SMMU geometry is inferred from host Metal. + +Not yet: full Metal feature/device capabilities, general texture formats, +general render descriptors, cross-process resource sharing, normal +bundle/service packaging, events/fences, heaps, argument buffers, indirect +commands, drawables and presentation, or a native Adreno backend. +Keep binary inputs, extracted shader files and observation artifacts outside +this source repository. + +The separate version-3 offline session protocol adds independent real host +object tables and per-session request ordering. Host resource identifiers +are unique across sessions, but lookup/release/bind operations resolve only +inside the selected table. Closing one session drops only its independent +references; another session can still submit real GPU work and read its own +output. Existing version-2 framing remains available without multiplexing. +The bounded host instrument opens exactly two sessions and checks seven +actual cross-session, stale/unknown-session and replay refusals, three real +GPU completions and complete texture readbacks. Run the umbrella +check-host-metal-originals.py followed by check-host-metal-sessions.py. + +Session selection belongs to the owned kernel broker, never caller-supplied +Metal request bytes. The host-only checkpoint remains distinct from the +native runtime evidence below. It establishes no guest display, active +Adreno SMMU mapping, hardware execution or original desktop. + +NativeOfflineGPU.cpp is the explicit private kernel component of this +partial offline stack. It holds a real exclusive byte-transport open, +serializes each complete host request/reply and buffers replies separately +for each actual native IOUserClient. The kernel selects an independent host +namespace; public request bytes contain no selectable session identity. +Framing, ordering, reply bounds, unread-reply ownership and teardown have +real failure paths. A transport failure prevents reuse of a partial stream +by another client. Closing a namespace releases only its own actual object +references. This component registers only its private test class, not an +IOAcceleratorES service or a system Metal device. + +The umbrella native instrument actually spawns two original-runtime child +processes. Each uses the real Metal object/command frontend and a distinct +original IOSurface. Three actual M1 GPU submissions complete, all surface +pixels are checked, a foreign texture handle is refused, and one process +continues GPU execution after the peer exits. At each actual userclient +creation the broker records live TCR/TTBR0/TTBR1/SCTLR and the actual task/PID, +and refuses nonnative-4K CPU contexts. Both workers report a 4096-byte ABI; +their actual TTBR0/task identities differ. These are CPU observations, not +active native-4K Adreno SMMU qualification. + +Only the own signed test process has the exact service/IOKit permissions +required by the original Sandbox: no-container and an allowlist containing +NativeOfflineGPUClient and IOSurfaceRootUserClient. Removing either +permission genuinely refuses spawn or device access. Original Sandbox, +AMFI, code signatures and both original trust modules remain unchanged. +Run the umbrella build-offline-gpu-broker.py, Metal build/stage/probe helpers +with --broker-processes, and check-offline-metal-processes.py. Authorized +general concurrent IOSurface sharing/coherence, normal system discovery, +complete Metal/presentation contracts, the original desktop and native Adreno +execution remain open. Scoped synchronous sharing and explicit own bundle +packaging are separately qualified below. + +## Explicit own bundle factory and normal-path offline view + +`OfflineMetalPlugin.mm` packages this partial frontend as an own signed bundle. +Its real principal class derives from the target's original `_MTLDevice`. +`initWithAcceleratorPort:` rejects null and unrelated services, requires the +actual unique private NativeOfflineGPU registry entry, and opens that same +service through the real frontend. The returned device reports the actual +provided guest service registry ID; host metadata retains the separate backend +identity. No original class implementation is replaced. This factory still +uses the private diagnostic broker, not an IOAcceleratorES implementation. + +The umbrella addon instrument mounts an existing source-built XNU bindfs +read-only union view at `/System/Library/Extensions`. Only own signed code is +added; actual lower-volume AGX metadata remains accessible. Its backing is +mutable own RAM Data, so it is not an immutable authenticated production addon +volume. Original NSBundle loads the own bundle in two real original-runtime +children, accepts its principal class and invokes the factory with real ports. +Both children then perform genuine Mach-authorized original IOSurface sharing, +seed-driven refresh, four actual GPU completions and peer execution after the +creator exits. The own helper has only its scoped no-container/IOKit permissions +and the original bindfs-allow permission; original applications retain theirs. + +`python3 scripts/check-offline-metal-addon.py` in the umbrella checks this +checkpoint and two real controls. Identical own machine instructions lose +mount access without bindfs-allow. A one-byte change to the own factory's code, +retaining its original signature and explicitly trusted hash, triggers an +actual bad code-page count and cs_invalid_page before any GPU access. Both +original trust modules and every original executable remain unchanged. + +This is explicit offline factory/packaging qualification. It does not publish +a normal Metal device, invoke normal Metal registry discovery, qualify ordinary +application access, asynchronous shared fences or a desktop. Original default +Metal remains nil. The complete Adreno backend and its active native-4K SMMU +remain unqualified. + + +## Actual file libraries and interrupted encoders + +The explicit device loads actual local file/URL/default-bundle libraries through +its existing exact-input hash qualification. Successful loads clear stale errors; +unreadable files, non-file URLs and readable non-library inputs fail honestly. +The optional Apple-family pipeline-library directory is explicitly unsupported. +Unchanged original RenderBox handles that error and genuinely loads its original +portable AIR library, then requests actual function-constant specialization. That +next interface remains absent at this checkpoint; no renderer frame is claimed. + +The host tracks actual open render/blit encoders. Ending an encoder invalidates +further encoder calls; duplicate end, post-end draw/state/copy, concurrent encoder +creation and commit with active encoding refuse before host Metal APIs. On actual +session closure or transport loss, cleanup only ends abandoned encoding, without +submitting the interrupted command or manufacturing GPU completion. The complete +partial ledger survives with explicit failure. Host unfinished/ended/disconnect +controls and native two-worker library/depth/stencil regressions pass. These are +explicit offline results, not normal system discovery, desktop or Adreno proof. + + +## Original function constants and RenderBox GPU drawing + +The explicit frontend reads a copy of the actual iOS17 constant-values object +through its original typed array/data/type/index/name methods, with exact runtime +class and ABI checks. It transfers bounded original bytes to actual host Metal +specialization; it never substitutes an unspecialized function for that request. +Scalar and two/four-component types have explicit bounded sizes; matrices, +three-component alignment profiles and unsupported values refuse. The extension +uses a distinct bounded payload on the existing function operation. The host +checks exact lengths, counts, identities, types, sizes and duplicate keys before +calling real Metal, and records actual values and compiler outcomes. + +Both actual native workers now run the unchanged original RenderBox renderer, +which specializes original plane__vertex/plane_color_fragment for both shapes, +binds real original buffer offsets and executes GPU indexed draws. Every pixel +of each returned original image matches left-red/right-blue, independently of +the complete host GPU readback hashes. Original surface uploads are all zero, +so the shapes do not originate in CPU-painted texture data. Six real commands +complete including the separate four-command surface-sharing/lifetime sequence. +This qualifies the explicit original renderer path, not the system desktop. + +The native depth/stencil fixture additionally qualifies both original named and +indexed constant accessors, source-object preservation, stale-error clearing, +unknown-function refusal and unsupported-type refusal. An empty constants set +and its pipeline are genuinely accepted by this exact host backend; this does +not qualify unspecified shader-state semantics. Earlier incorrect negative-test +expectations and actual observations remain retained separately. The host-only +function-constants control compares indexed/named red output with white output +after one real UInt4 bit changes, verifies every pixel and checks nine genuine +argument/isolation/compiler refusals. Native buffer-offset methods retain the +actual previously bound resource and rebind its real offset through the existing +checked path. Complete normal discovery, asynchronous contracts, presentation, +the original desktop, Adreno execution and active native-4K GPU SMMU remain open. + + +## Unpublished original accelerator-base lifecycle + +OriginalAcceleratorABI.h declares only the exact supplied IOAcceleratorES base: +136 bytes and the existing 168-entry IOService table, with no new operational +methods. The umbrella audit derives its real protected-compatible constructor +aliases, destructor and metaclass from the unchanged original IOGPU component. +NativeOfflineAccelerator uses those genuine bodies and its own compiler-produced +type metadata. No original class/vtable/method is replaced. This object is never +attached, started or registered as a system service. + +The opt-in kernel lifecycle fixture actually allocates/initializes the own object, +confirms its actual original parent and rejects unrelated IOGPU/private-broker +ancestry, then releases it. Real instance counts go 0 -> 1 -> 0. Omitting original +IOGPUFamily load genuinely refuses before allocation; default-off creates no +object. The complete native table and exact base imports are independently audited. +Both controls share the same immutable candidate. This is early CPU-4K/kernel ABI +evidence before root acceptance or userland; no GPU client, backend response, +normal Metal publication, original desktop or Adreno execution is qualified. +Run the umbrella build-offline-gpu-broker.py --accelerator-abi and the dedicated +probe-accelerator-abi.py / check-accelerator-abi.py tools. The build option defaults +off. Complete normal device/client/resource/queue/surface/presentation contracts +remain required before original system discovery can publish this partial stack. + +## Original CAMetalLayer drawable present requests + +The explicit frontend now retains each actual drawable through standard +presentDrawable:, validates its protocol, method ABI, own-device IOSurface +texture and bounded identity, and refuses duplicate or completed submissions. +After real GPU completion and actual IOSurface downloads, commit invokes the +original drawable's present method. It preserves real GPU completion callbacks +if that original method raises. It creates no display timestamp or fake fence. + +The target's unchanged CAMetalLayer creates an actual CAMetalDrawable on this +explicit device. Its original present implementation remains at its original +cache offset. Two actual GPU commands produce the original RenderBox RGBA image +and convert it to the drawable's BGRA texture. Every one of 4096 pixels and both +independent full GPU readback hashes agree; all initial texture uploads are zero. +Invalid-object, duplicate and post-completion requests refuse, and both actual +command callbacks execute once. The actual first drawable ID and presentedTime +are zero. This detached layer performs no framebuffer swap or retirement and +qualifies a genuine present request only, not display completion. Time/duration +presentation variants, normal system discovery, original desktop, Adreno and +active native-4K GPU SMMU remain open. The umbrella --metal-layer fixture and +check-original-metal-layer.py retain exact source, signed-input and CPU-4K evidence. + +## Genuine GPU fences and explicit texture hazard modes + +OfflineMetalFenceWire.h extends the bounded transport with real fence creation, +render stage updates/waits and blit updates/waits. The frontend implements the +complete MTLFence object protocol, owns the actual device identity and retains +fence proxies through commands. Object labels use atomic copy semantics. The +backend invokes real Metal fence APIs on actual per-session objects; it never +marks a fence signaled in CPU state. It refuses foreign/released handles, wrong +or ended encoders, unsupported stages, an update-then-wait cycle in one pass, +missing producers and unsent cross-command dependencies. This synchronous subset +accepts an earlier ended producer pass in the same command or a genuinely +completed producer on another queue; asynchronous pending producers remain open. +Producer bookkeeping releases with the independent fence handle or namespace. + +The texture extension preserves old request layouts and explicitly accepts +Default/Tracked/Untracked descriptors. It queries and returns the actual backend +mode and refuses an explicit mode mismatch. Texture resourceOptions reflect the +actual storage/cache/hazard attributes. Default really resolves to Tracked on +the observed backend. Neither resource attributes nor host Metal fences qualify +Adreno residency, interrupts, fences or an active native-4K GPU SMMU context. + +The host-only check-host-metal-fences.py executes four explicitly Untracked +resources through real render -> blit -> original QuartzCore shader rendering, +then a real completed-producer dependency on another queue. Both full readbacks +match all 8192 pixels, with no texture uploads. Eight actual fence calls and two +GPU completions pass; twelve argument, namespace, lifetime and dependency controls +refuse. An encoded but unsubmitted producer/control is never submitted by cleanup. +The existing original function-specialization and ended-encoder regressions pass. + +The native --fences fixture uses an actual private Untracked GPU target, a real +render-update -> blit-wait/update -> render-wait chain and unchanged original +QuartzCore shaders. The private result is GPU-copied to the actual original +CAMetalDrawable, then GPU-sampled in its final render pass. Both full readback +hashes and all 4096 drawable pixels agree. Five native fence refusals, real +protocol/copy-label behavior and existing drawable/callback controls pass. +Normal present invokes the original implementation only after real completion +and IOSurface visibility. The detached layer still has presentedTime zero and +no framebuffer swap. This is explicit offline synchronization qualification, +not a system desktop, asynchronous general queue contract or native Adreno proof. + +## Asynchronous host submission and real cross-queue GPU events + +OfflineMetalCommandWire.h adds separate submit/status operations. Submit invokes +actual Metal commit and returns without waitUntilCompleted. Status reads the real +command state; completion times remain unavailable until genuine completion. +Completion records are unique even across repeated queries. Namespace close and +transport-loss cleanup wait for already submitted work and record its real GPU +result before releasing references. They never submit merely encoded controls. +Transport loss remains unhealthy/unfinished even when its actual GPU work completes. + +An initial pending-producer cross-queue MTLFence experiment returned an all-zero +copy while the original shader output was red; both buffers reported Completed. +The exact failed source, executable, GPU readback and timing ledger are retained. +This is a failed qualification, not successful asynchronous resource visibility. +Cross-queue fence waits therefore still require a genuinely completed producer. +Same-queue waits can consume actually committed producers. The supplied host +profile uses actual MTLEvent timeline signals/waits for overlapping queues. + +OfflineMetalEventWire.h creates genuine per-session GPU events and encodes real +command-buffer signals/waits. CPU state records only encoded producers; it never +reports a signaled value. Missing, foreign, unsubmitted, same-command, zero/future +and nonmonotonic dependencies refuse. No CPU fence or precomputed completion is +substituted. The current bounded profile requires the producer already committed +before encoding its consumer wait; future unsubmitted producers remain unsupported. + +check-host-metal-async.py --api-validation and its --disconnect-control execute +three genuinely nonblocking submissions, six fence calls and five GPU timeline +event calls on two actual queues. Initial real states are Committed; queries also +observe Scheduled and Completed with actual GPU times. Every one of 8192 pixels +matches after the original shader and cross-queue event dependency. Thirty-one +argument, namespace, state and lifetime controls refuse. An independently released +submitted command finishes during real cleanup; unsubmitted controls never gain +GPU submissions. The disconnect case exits with failure and keeps its truthful +incomplete transport ledger. Eight additional bounded API-validation runs pass +24 GPU commands and 65536 pixels, including eight waits on actually in-flight +producers and four disconnect cases, with no unaccounted validation messages. +The original synchronous fence regression also passes. + +That initial checkpoint was host-only. The separate opt-in native integration +below now qualifies its own visibility/callback/lifetime subset. It does not +qualify system discovery, original desktop, native Adreno events/fences or active +native-4K GPU SMMU state. + +## Opt-in original-runtime asynchronous commands + +The private broker now accepts the exact bounded command/event extensions. The +frontend explicitly opts in through offline_metal_frontend_enable_async; default +contexts retain synchronous commit. Each new command captures that mode. Actual +submission returns independently of a dispatch worker that observes real GPU +status, synchronizes actual IOSurface/buffer data, invokes the original drawable +present and delivers callbacks. Event/fence/resource proxies stay retained through +completion. Proxy seed/data access is synchronized across workers. + +Scheduling, resource visibility and callback completion have independent groups. +waitUntilScheduled only waits for actually observed scheduling; it cannot wait +for a callback while an original caller holds its lock. The original private +commitAndWaitUntilSubmitted B16 ABI returns actual accepted submission in async +mode. Successful completion waits include resource visibility and callback +execution; callback reentrant waits use their actual worker identity. Scheduled +callbacks remain deliberately late. Real transport/GPU failures retain errors; +presentation/callback exceptions remain separate from real GPU completion. + +Context finish refuses an active callback caller, waits pending work and all its +callbacks, then closes the actual namespace/userclient. Atomic lifecycle flags, +a recursive transport lock and release/close handoff keep a proxy destructor +from sending a stale release after namespace close. Finish never submits merely +encoded work. It permits the real completion callback's readback while closing. + +The --async-commands native fixture enables the own device before unchanged +original RenderBox initialization/drawing. All four actual GPU commands use +nonblocking submit, including that original renderer. Native workers deliver +five own callbacks away from the caller thread, two reentrant completion waits +and one genuine callback-close refusal. An explicit caller-held lock blocks its +scheduled callback while waitUntilScheduled still returns, qualifying the +scheduling/callback separation. A final GPU clear has its caller reference +released and no explicit wait; context finish waits its callback, which checks +every one of 4096 real green pixels before close. Original drawable/peer images +also match all pixels and independent GPU hashes. Both GPU event values execute +on genuine commands; no CPU signal is substituted. + +The same kernel passes the existing default synchronous drawable/fence path and +real default-off original launchd main, preserving actual root, original AMFI/ +signatures/trust and active native CPU 4K. Use the umbrella dedicated async probe +and checker, including --default-off and --synchronous-control. This is explicit +offline original-runtime integration, not normal device/client authorization, +window-server composition, display timing, original desktop or native Adreno. +The actual detached drawable presentedTime remains zero and no framebuffer swap +occurs. Shared-event Mach export/import and complete concurrent/system contracts +remain separate requirements before normal discovery publication. + +## Actual host shared events, listeners and exported Metal handles + +OfflineMetalSharedEventWire.h adds actual newSharedEvent, value access, labels, +real listener registrations/polls and internal export/import/drop operations. +Only original Metal callbacks mark a notification ready. Pending registrations +are cancelled at namespace teardown without fabricating delivery. A separate CPU +setter control is explicitly recorded and never drives the rendered dependencies. +Shared-event aliases retain actual Metal handles and common producer metadata; +closing an owner retires its producer references. Imported waits can use the real +hardware counter or an actually committed producer, never a CPU signaled flag. + +The backend imports with actual newSharedEventWithHandle after the original +namespace closes. Its imported object continues GPU work after the actual export +handle is released. Export references keep the private transport alive until +released. Internal control framing is distinct from client sequence accounting; +a forbidden client control request refuses without desynchronizing its sequence. +Public native raw submission must exclude these internal integer export IDs and +mediate them through actual native Mach capabilities before this feature connects. +At that host-only checkpoint the native broker still ended at ordinary GPU event operations. + +check-host-metal-shared-events.py --api-validation executes three GPU tasks, +five shared GPU signals/waits, all 12288 original-shader/peer pixels and eighteen +refusals. Three actual listener callbacks and two genuinely unfired cancellations +pass. The hardware timeline reaches 1/2/3 from GPU commands; only an unrelated +control event uses a real CPU setter to 7. Actual shared-event device metadata +reports this owner device on the tested host; nil is never assumed from a type +name. Its actual exported handle copies the label and has a real supported trace +getter returning zero. Previous async and disconnect regressions still pass. + +The umbrella audit derives the unchanged target's original 16-byte secure handle, +24-byte private data, typed eventPort/label/labelTraceID getters, label copy and +real mach_port_mod_refs send-right increment. Its original listener construction +ABI is also pinned. That is source analysis only: native Mach transfer, original +handle construction/coding, recipient import and process-exit lifetime must be +qualified separately. No raw host export ID is a native permission or completion. +System desktop, native Adreno and active native-4K GPU SMMU remain unqualified. + +The native private broker now mediates shared-event export/import through real +IKOT_IOKIT_IDENT OSObject Mach capabilities, with independent internal host and +public request sequences. Raw host export identifiers are never accepted. Token +no-senders destruction drops the actual retained host Metal handle exactly once, +independently of creator userclient lifetime. Both custom selectors verify the +actual owning task, genuine token class/controller and bounded metadata buffers. + +The frontend calls the unchanged original MTLSharedEventHandle constructor, whose +real send-right reference count rises and falls 2 -> 3 -> 2. Host-derived event +metadata preserves a genuine nil or owner device (which varies with actual host +API validation). Counter reads, bounded waits, copied labels and GPU wait/signal +encoding are backed by actual Metal operations. At this capability-only checkpoint native listeners explicitly +refuse until independently qualified. No original code/methods are replaced. + +Run the umbrella build/stage --shared-events, probe-native-metal-shared-events.py +and check-native-metal-shared-events.py. Two real original-runtime native-4K +workers prove recipient import after creator waitpid/exit, three actual GPU tasks, +all 12288 pixels with no texture uploads, and continued imported GPU execution +after the final original handle/Mach right is destroyed. Wrong/stale Mach rights +and guessed raw host imports refuse without breaking subsequent real requests. +One export/import/drop and zero live tokens remain at finish. The own received- +right adapter uses the actual original constructor; it does not qualify original +NSXPC/NSSecureCoding serialization. System Metal/desktop and Adreno remain open. + +The native shared-notification extension now validates the unchanged original +MTLSharedEventListenerInternal 24-byte class and real typed dispatchQueue getter. +Only genuine host Metal listener ready records can schedule a native user block, +on that original listener's actual queue, with exact event identity/counter. +Bounded workers record failures separately. Pending/queued/running callbacks have +distinct states under the lifecycle lock. Context finish cancels unfired or queued +registrations, waits genuinely running callbacks and rejects reentrant finish from +one of those callbacks. It never waits a queued block behind a caller-held queue. +After close a cancelled queued block skips delivery; no new transport call escapes +past namespace close. A cancelled or timed-out registration is never successful. + +Build/stage --shared-notifications, probe-native-metal-shared-events.py +--notifications --api-validation, then the matching checker flags. Two callbacks +run on distinct original listener queues away from the caller, reenter real +counter access and refuse callback-local finish. Two value-99 registrations never +fire. A third real host callback reaches a deliberately blocked original queue: +finish cancels it without deadlock, and draining the queue confirms no user block. +The host independently records three actual GPU-triggered notifications/two +unfired ones; native counters record two delivered/three cancelled/zero failed. +The existing creator-exit/Mach rights/12288-pixel GPU controls still pass. The new +frontend and shared-event kernel also pass the full original RenderBox async +primary path (four actual GPU tasks, original images, event/fence/callback controls). +Original secure coding, normal system publication/authorization, desktop and +native Adreno/SMMU remain independent unqualified work. + +The umbrella --shared-coding instrument independently qualifies genuine original +same-process NSXPCListener/NSXPCConnection shared-handle coding and original +non-NSXPC-coder refusal. --shared-cross-process additionally transports a genuine +anonymous endpoint via original libxpc serialization and the original endpoint +accessors, with real peer PID checks. It uses the supplied target's actual third +receive flags argument, not an obsolete two-argument declaration. The peer's +original NSXPCCoder decodes a real Mach send right; the native broker imports its +actual event. After creator waitpid/exit, that NSXPC event executes the real blue +GPU wait/signal task. A distinct genuine Mach import sees its hardware counter +and continues with green GPU work after final handle destruction. All 12288 real +GPU pixels, the original handle reference controls, wrong/stale/raw refusals, +three real imports, exact final cleanup, native CPU 4K/root/AMFI and GPU API +Validation pass. No new bootstrap or deployment authorization is implied. +Normal system discovery/client policy, full compositor UI/desktop and actual +Adreno/SMMU qualification remain open. + +The unpublished NativeIOGPUProviderProbe now derives from the actual abstract +736-byte/215-entry IOGPU. Its real original protected C2/D2 and all inherited +operational entry addresses remain intact. The ten required private hooks have +non-returning register captures; a separate publication guard also never returns. +Opaque tail declarations bind original entries directly without introducing C++ +callers or claiming recovered source signatures. It is an introspection object, +not a system driver. + +Native QEMU observes original construction/init/free/D2 restoring instance count +0 -> 1 -> 0. In the independent start observation, actual original IOGPU::start +reaches required slot 195 with genuine this/provider and the exact original +caller LR. The hook records live CPU translation registers, reports Unsupported +and deliberately stops; startup does not return, and this start object remains +live at the checkpoint. Missing original registration refuses before allocation; +default-off executes original launchd with no probe object. No normal publication, +successful provider startup, system Metal/UI, Adreno execution or active native +GPU SMMU is qualified. + +## Original QuartzCore vertex attributes and real buffer geometry + +OfflineMetalVertexWire.h adds a 2104-byte, size-discriminated extension of the +existing attachment-pipeline operation. No new kernel selector or permission is +needed. The actual original vertex descriptor is serialized into 31 attributes +and 31 static layouts; the host constructs real Metal descriptors and pipelines. +Known scalar/vector/packed formats are bounded explicitly; unused entries must +be zero. Dynamic strides, patch stepping and out-of-range formats/indices/extents +refuse. No unsupported vertex layout is silently replaced with an empty one. + +The original unchanged QuartzCore std_vert0_lpf and attachment_clear_frag_lpf +now execute through actual float4/float2 attributes, a 40-byte vertex stride, +real MVP buffer and updated vertex bytes. Host API Validation qualifies two +GPU draws/all 8192 pixels plus ten framing/layout/foreign-namespace refusals. +Two actual native iPadOS workers then qualify four original-shader GPU draws, +all 16384 geometry-dependent pixels, descriptor mutation independence and real +shared-surface/foreign-resource lifetime controls. All 137 actual requests and +144974 reply bytes reconcile independently; seven GPU submissions complete, +and both workers exit zero with distinct native CPU 4K task/TTBR0 contexts. +New geometry targets never receive uploaded CPU pixels. + +The original RenderBox asynchronous regression also passes on the same source +kernel/frontend/backend: original GPU output, genuine events/callback visibility +and cleanup remain intact. Nonempty static vertex input is newly supported; +ordinary system discovery, full display-server composition, original desktop, +Adreno execution and active native-4K GPU SMMU are still unqualified. + +## Original client absent-descriptor refusal and cleanup + +The unpublished original-provider instrument now declares slot170's actual +descriptor return and uint32 output pointer. The supplied original concrete +method writes a uint32 through x1 and returns its real descriptor through x0; +the actual original client retains it and invokes native createMappingInTask. +This register contract is derived from this target's unchanged code. Other +opaque tail source signatures remain unqualified. + +The naked slot170 entry saves the same actual register frame before C++ code. +Its normal observation remains non-returning. Only the explicit own +client-refusal instrument returns NULL for the genuinely absent backend +descriptor, restoring the original LR/SP. All other required callbacks and +publication remain non-returning captures. No descriptor or device success is +supplied, and no original method or class identity is replaced. + +The original client factory, custom owning-task/subtype initializer and client +start now execute together. An absent descriptor returns through original error +handling, cleanup, detach and release. QEMU checks actual type1 and high16 +subtype controls: the original client count rises by one at the real callback +and returns to its baseline; the own provider count also returns to baseline. +The original factory returns its actual failure with NULL output, no live +attachment/publication and zero GPU protocol traffic. The read-only observer +never dereferences freed client storage. Separate non-returning observations +still inspect the real live original 304-byte client and its stored subtype. + +This is genuine refusal/lifetime qualification for the unpublished instrument. +Real shared descriptors, device/resource/queue implementations, successful +original startup, normal system discovery, desktop and native Adreno/SMMU +remain required. + +## Native frontend over the bridge + +`NativeBridgeMetalFrontend.mm` is the Metal API frontend of the native path: +the metal2vulkan runtime (`bridge/NativeMetalObjects.rs`, a C ABI of the +runtime's device, resources, states, queues, command buffers and encoders) +on Turnip and NativeAdreno. The device is a dynamic subclass of the target's +_MTLDevice adding only its own methods, as in the offline frontend. Implemented: +buffers (also no-copy), textures (also over buffers and over an IOSurface's +own pages, without a copy), metallib and AIR libraries, function-constant +specialization through the target's own accessors, compute and render +pipelines with vertex descriptors and blending, samplers, depth/stencil states, +queues, command buffers with scheduled/completed handlers and GPU times, and +compute, render and blit encoders. An encoder call the runtime refuses fails +its command buffer (status Error, handlers still run); an unimplemented +selector is reported through a hook before the usual exception. Not yet: +events/fences, heaps, argument buffers, indirect commands, drawables and +presentation. No GPU family or feature set is claimed. + +`scripts/check-native-metal-frontend-host.py` (umbrella) qualifies it on the +Mac through the public Metal API only, over MoltenVK as the runtime's Vulkan +driver: compute, render with blit readback, rendering into a real IOSurface +read by the CPU, sampling that surface into a second surface, texture compute, +handler order, a refusal, function constants, and translation of the target's +QuartzCore library. That is a host observation of the frontend and runtime; +it does not qualify Turnip, the Adreno 740, its SMMU context or presentation. + +Compute descriptor creation and archive population use the same supported +descriptor fields: function constants, thread limit/promise, indirect/binary +function support, call stack depth and buffer mutability. Explicit archive +lists and pipeline option bits reach the runtime for both compute and render +creation. Nonempty compute stage inputs and linked/dynamic libraries are +refused here; empty descriptors materialized by Metal retain their default +meaning. A separate-process, empty-cache archive regression checks strict +hits/misses, descriptor/function-constant identity and actual host GPU output. +The archives contain translator artifacts, not Adreno machine code; later +draw/dispatch variants and a complete cold UI shader pack remain unqualified. + +The default compute-function path is separately checked over Turnip and the +software kernel peer in `bridge/driver_cache_host.cpp`. A cold process writes +translation artifacts and a real driver-pipeline record. Another process with +the same directory must report artifact/driver hits and no new misses. The C +bridge exposes the two layers' separate counters for this check. Mesa disk +shader caching is disabled. Commands retire through the software peer, and +their output is unchecked; actual host GPU pixels have separate regressions. +Matching these host caches does not qualify them for a different iOS driver +build or cover every target UI function/constant/dispatch variant. + +Bound buffer/texture owners are retained separately from the runtime's GPU +storage handles. The former owns no-copy deallocators, IOSurface references +and buffer/view/heap ancestry. Compute/render bindings, blits, index buffers, +pass attachments and visibility buffers retain those owners until real +completion (or a local refusal which submitted no work). Completion also drops +the IOSurface visibility set, so keeping a completed command does not keep its +old frame storage pinned. Discarding an uncommitted command releases its owners. + +`bridge/resource_lifetime_host.mm` drops all caller-owned references while +actual host GPU work is gated on an unsignaled event. It checks raw-buffer +copies/fills, nested buffer-backed views, compute input and a clear-only render +attachment, plus local refusal and uncommitted discard. The no-copy callback +must remain pending and then run exactly once; successful work checks actual +GPU output. The old frontend invokes the callback before submission in all +seven cases. The test owns its mapping separately so that the old failure +never submits a dangling pointer. This qualifies host ownership, not safe +native GPU kernel resource leases, scanout-map reuse or 120 FPS. + +IOSurface cache maintenance deduplicates exact live CPU address/length pairs +within each command phase. Independent imports and nested texture views of +one surface previously repeated the same full-range operation before commit +and after completion. Range sorting removes those duplicates, keeping every +distinct range's maintenance and carrying no address identity across phases +or frames. The command's retained owners keep all mappings live during each +phase. No hardware coherence assumption or cross-frame mapping cache is added. + +`bridge/surface_flush_host.mm` forwards every intercepted maintenance call to +libSystem and checks real host GPU writes, independent alias reads and shared +surface bytes. For 128 aliases of one allocation the old code makes 256 calls +covering 4 MiB; the new code makes two covering 32 KiB. Two distinct allocations +require four calls. A single alias still requires both phases. Telemetry reports +actual maintenance calls/bytes/time and removed duplicates; the plugin streams +all 48 supported rows. Actual native workload frequency and hardware frame-time +benefit remain unqualified. + +IOSurface texture creation now reuses a still-owned no-copy buffer for an exact +surface ID, CPU base and rounded allocation size. The buffer retains the +surface through its own release. A mutex covers lookup and first import; the +cache has weak values and at most 256 associations, so it cannot keep an old +frame's GPU allocation alive. An evicted association may be recreated while +its original resources remain live. General caller-owned no-copy buffers keep +their own objects and deallocators; no remap or rebind API is added. + +`bridge/surface_import_host.mm` counts actual frontend import/release calls +while forwarding every call to the unmodified bridge. The separate host-test +`import_count_loader.cpp` forwards all Vulkan requests to the real driver and +counts successful external-host-memory allocations. These are distinct counts: +the runtime already shares sequential live imports. For 128 aliases of one +surface, frontend calls fall from 128 to one while actual Vulkan imports stay +one. A 64-way concurrent first creation instead reproduced seven actual imports +and stale host GPU texels on the previous frontend; the fixed path makes one +and reads the correct GPU output. This also avoids the runtime's documented +concurrent first-import window for this frontend's IOSurface path. The common +native bridge also covers raw no-copy callers, as described below; direct +runtime clients outside that C ABI still need their own synchronization. + +Distinct surfaces retain distinct imports. Parallel creation, owner expiry, +work gated behind a real unsignaled GPU event and dropping the caller's last +surface reference all pass. Completed commands release their owners even while +retained. A 514-alias/257-surface case crosses the metadata bound: 258 frontend +objects and 257 Vulkan imports, all GPU bytes correct and all owners released. +The tests do not qualify Adreno timing, target UI concurrency, repeated-blur +frame deadlines, or active native-4K GPU translation. + +The common `nmb_buffer_new_no_copy` entry now gates its runtime construction per +device. This covers both `newBufferWithBytesNoCopy:` and IOSurface textures, +including concurrent mixtures, while returning a separate buffer object for +every raw call. The gate covers the live-import lookup, allocation and join; +it ends before returning an object/error, and resource release, GPU work, +completion waits and other APIs do not take it. The C ABI supplies neither a +dma-buf source nor a user runtime deallocator. Frontend deallocators still run +outside the bridge. No upstream runtime patch or shader cache identity change +is introduced by this boundary fix. Distinct no-copy constructors on the same +device share the CPU construction gate; this is not a claim about their timing. + +`bridge/no_copy_import_host.mm` starts sixteen simultaneous public Metal calls +in each fresh process. The forwarding loader inserts a one-shot 25 ms delay +before the first actual host-memory allocation to expose the lookup window, +then calls the real driver unchanged. Old code makes sixteen imports for +sixteen raw aliases and nine for eight raw buffers mixed with eight surface +textures. Both now make one. The GPU producer completes before alias readback, +so the check assumes no implicit synchronization for arbitrary raw Metal +aliases. All GPU bytes match, every raw object remains distinct, and each +deallocator runs exactly once. A separate callback creates a buffer from +another thread and waits for it, qualifying release-time reentry without the +construction gate held. These are host allocation/ownership controls, not +Adreno timing or stable UI frame evidence. + +`bridge/quartzcore_blur_host.mm` separates bare-descriptor failures from valid +pipeline coverage in the unchanged target library. Its 167 entries comprise +41 vertex, 111 fragment and 15 kernel functions. Four ordinary compute entries +compile; eleven report a tile-attachment requirement when incorrectly requested +as ordinary compute. Of 22 name-matched render pairs, the bare probe's eight +failures are six missing vertex descriptions and two missing MRT attachments. +Compatible authored layouts/formats make all 22 pipelines, without changing +the source library or advertising tile support. These are selected descriptor +recipes, not the target UI's complete function-constant/format/state inventory. + +The same control executes the original float/half narrow-blur functions on the +host GPU. An authored seven-tap binomial stencil provides an independent +reference; horizontal and vertical passes run six times for each precision, +checking every output component. No further pipeline making or cache miss is +allowed during the repeated draws. The native frontend exposes structured +work/cache counters for this qualification; cumulative creation diagnostics +must also be positive so disabled logging cannot masquerade as reuse. + +The umbrella's `check-quartzcore-blur-host.py` runs cold producer, archive-only +and provisioned-cache processes, with strict empty/changed-descriptor refusals. +The archive contains 22 compatible pairs and the two blur recipes. Archive-only +reload avoids translation but still reports 20 driver misses in the observed +host run. Copying the matching producer cache reports 24 driver feedback hits +and no translation or driver miss. All 24 pipeline objects still have to be +created, and MoltenVK's feedback may describe cached stages; this is not proof +of a native Adreno first-frame deadline. Archives, driver records and raw logs +remain ignored. A different iOS/Adreno profile needs its own artifacts; the +complete initial-image UI pack and stable 120 FPS remain unqualified. + +`bridge/quartzcore_cache.mm` runs the same 24 authored descriptor recipes over +actual Turnip/IR3 and the explicitly required software peer. Its offline-only +`graphics_cache_loader.cpp` forwards real calls, enables the queried Vulkan +cache-control feature and can forbid compilation after device initialization. +It independently records real compiler results and feedback. The negative +UUID control mutates a private copy of loaded data; producer files stay intact. +No draw is encoded. Device entry-point priming submits barriers/timestamps +to the software peer, which executes no shader. This is compiler evidence, +separate from the original-blur host GPU pixel control. + +The creating API can return a state even when a Vulkan prewarming attempt +failed. Therefore nonnil states cannot qualify driver-cache readiness. In the +observed Turnip graphics-library path, 24 states use 65 actual graphics +creations, 33 with shader application caches. The strict consumer succeeds in +all 65 and reports valid application hits in all 33; empty and UUID-mismatched +caches produce actual compile-required results. The runtime's aggregate +shader-cache hit statistics do not report this graphics-library branch, so the +independent call counters are required. The umbrella records a separate Mesa +feedback fix: its graphics hit flag's AND reduction previously began false, +concealing successful cache hits. + +Device initialization itself makes a graphics pipeline before the strict UI +scope and submits initialization work. It is explicitly outside the consumer's +compile-forbidden claim. These checks do not qualify every actual draw variant, +initial-image completeness, native iOS/Adreno shader execution, active GPU 4K +translation or any frame deadline. The control loader is never deployed. + +`bridge/quartzcore_cache_ios.cpp` is an own QEMU-only PID 1 harness for this +compiler instrument. It finds the exact own RAM Data UUID and NativeRamBlock +ancestry, establishes the existing readonly union extension view, and spawns +four separate normally linked workers. Actual original dyld/AMFI and 4K +CodeDirectories remain in force. Parent and workers require the genuine 4096 +byte ABI; no page-size shim is used. The parent hashes the source library, +archive and every sorted cache file, calls fsync/sync and stops at its own EL0 +checkpoint. Its small manifest and complete worker log fit the existing bounded +saved-register captures. Existing bounded own-Data export supplies the full +cache bytes without widening a user-memory read. + +The iOS Turnip build passes the same 24-recipe strict compiler controls: 65 +successful graphics creations and 33 real application shader-cache hits, with +UUID/empty-cache refusals in separate workers. The actual selected pack has +153 files; readonly extraction matches every byte against the paused iOS +parent's independent manifest. The backing is mutable test Data, and these +workers produce the pack during that boot. A fresh boot provisioned from these +bytes, complete UI/draw variants, authenticated production packaging, Adreno +execution/active GPU 4K translation and display deadlines remain unqualified. + +`bridge/quartzcore_cache_provisioned_ios.cpp` reuses the exact volume, manifest +and checkpoint helpers with a consumer-only main. The staged image already +holds the exported iOS archive/cache at the normal consumer paths. Before any +worker starts, the parent hashes that initial pack. Only consumer, wrong-UUID +and empty-driver workers run, in distinct processes; their final pack manifest +must equal the initial one byte for byte. Actual iOS first-consumer checks pass +all 65 compile-forbidden UI graphics creations and all 33 shader cache hits, +with zero translation misses/writes or driver record saves and no boot producer. +Original signing/AMFI/trust, native CPU4K and real worker page ABI checks pass. +This qualifies the selected recipes in an owned test image only. + +The separately observed device initialization is exit-guard priming in the +runtime's `device/prime.rs`: it calls the driver entry points before registering +the teardown hook, makes an initially empty pipeline cache, and discards that +cache's bytes. It includes compute and graphics pipeline creation and a +barrier/timestamp submission. The render/compute recording is not submitted; +this is not a semantic GPU probe or a source of measured shader output. +Caching that startup work must preserve its complete priming and teardown +ordering. Device initialization remains outside the UI consumer's strict scope; +complete variant coverage, production authenticated packaging and native +Adreno frame deadlines remain unqualified. + +`bridge/patches/0003-cache-device-entry-point-priming.patch` now keeps that +startup cache after all priming succeeds and reloads it in later devices. +Its raw disk/cache path creates no worker and enters no exit-guard section +before priming. Every primed call, the barrier/timestamp batch, waits, failure +cleanup and post-priming hook registration remain. Raw loader words are restored +before each added cache entry point. The optional generated helper also uses +the shared priming cache; generated-command execution is not qualified by the +current controls. + +Records reuse the existing bounded, checksummed, atomic driver-cache format. +Their keys include the driver/build UUID, compile profile, instance settings, +priming implementation bytes and relevant feature/queue choices. Invalid or +refused data starts empty; failed priming never persists partial cache data. +An unchanged loaded record is not rewritten. This startup store has separate +counters from the ordinary shader-cache store, so independent driver calls are +used to qualify startup hits. + +`bridge/priming_cache_loader.cpp` extends the offline graphics-cache forwarding +loader without changing its previously qualified source. It adds actual compute +feedback and a compiler-forbidden device-open policy. `priming_cache_host.cpp` +opens/closes the device without authored shader work. Both compute and graphics +helpers now report real application-cache hits with compilation forbidden. +Cold, wrong-UUID, truncated and bad-checksum controls refuse and leave zero live +software-peer objects. An exact preserved pre-fix executable still fails during +device open with the same populated cache. Every case retains the one real +priming barrier/timestamp submission; no shader execution is claimed. + +The full host GPU/frontend and Turnip controls pass. The instrument and complete +plugin cross-build for ordinary ARM64/iOS17 with own 4K signatures. New priming +cache execution under iOS and a fresh initial image still need qualification. +Earlier iOS UI packs remain evidence for their earlier runtime; regenerate and +verify artifacts for this build rather than silently treating them as current. +These results do not qualify all UI variants, native Adreno execution/4K SMMU, +physical display deadlines or stable 120 FPS. The test loader is never deployed. + +`bridge/priming_cache_ios.cpp` now runs the exact priming instrument in actual +iOS/QEMU workers. Four cold processes qualify empty-cache refusal, production, +strict import and wrong-UUID refusal. A second fresh image prepopulates the +actual native record and selects a consumer-only main path: its first worker +hits both compute and graphics helpers while compilation is forbidden; no +producer runs in that boot. The same bounded manifest helpers independently +hash the initial/final record and the readonly exported Data snapshot. The +8,977-byte native record has its actual iOS driver identity, not a relabelled +host header. Original AMFI/signatures/trust/root, active CPU4K and real worker +page ABI checks pass. Both positive and refused workers retain priming and +release all software-peer resources. The peer executes no shader. + +This proves the runtime's public priming helper path only. A separate source +audit finds Turnip device creation initializes fifteen clear/blit shader +variants and its empty shaders before these public pipeline calls. Its IR3 +compiler is explicitly created with `disable_cache=true`, independent of the +runtime's new cache. That path needs compiler-level measurement and separate +optimization; public Vulkan compile-forbidden flags cannot qualify it. No +complete startup-compilation, UI-variant, native GPU4K or frame-rate claim follows. diff --git a/bridge/FrontendChecks.h b/bridge/FrontendChecks.h new file mode 100644 index 0000000..fdc1014 --- /dev/null +++ b/bridge/FrontendChecks.h @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: MIT +// The native Metal frontend's qualification through the public Metal API only, shared by the host test (frontend_host.mm, over MoltenVK) and +// the plugin (NativeMetalBridgePhase.h, over Turnip on the Adreno 740). Every check reads back what the GPU wrote. Steps (markers 0x6601..): +// 1 compute; 2 render to a private texture with a blit readback; 3 render into a real IOSurface read by the CPU, 4 sampling it into a +// second surface (the composition shape); 5 a compute kernel reading and writing textures; 6 completion handler order; 7 an encoder refusal +// failing its command buffer; 8 the target's QuartzCore library (when a path is given); 9 function constants (when that fixture is given); +// 10 a shared event ordering two queues (GPU wait, GPU signal, CPU signal, listener notification) and a fence between two encoders; +// 11 heaps (automatic and placement), a visibility result buffer and setDepthBias. +#pragma once +#import +#import +#import +#include +#include +#include +#include + +namespace frontendchecks { +struct Report { + unsigned failed, last; + char first[96]; + unsigned qcFunctions, qcKernels, qcKernelsOk, qcPairs, qcPairsOk; + void (*mark)(uint32_t); + bool hostSimulation; // the host simulation's IOKit shim hides IOSurface (steps 3-4 skipped) and Metal's listener (none is made) +}; +static Report *gReport; +__attribute__((format(printf, 2, 3))) static void fail(Report *r, const char *format, ...) { + char text[160]; + va_list args; + va_start(args, format); + vsnprintf(text, sizeof text, format, args); + va_end(args); + std::printf("FAIL %s\n", text); + if (!r->failed++) snprintf(r->first, sizeof r->first, "%x %s", r->last, text); +} +static void step(Report *r, uint32_t marker) { + r->last = marker; + if (r->mark) r->mark(marker); +} +#define CHECK(c, ...) do { if (!(c)) fail(gReport, __VA_ARGS__); } while (0) +static bool completed(id cb, const char *what) { + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusCompleted) { + fail(gReport, "%s: status %lu %s", what, (unsigned long)cb.status, cb.error.localizedDescription.UTF8String ?: ""); + return false; + } + return true; +} +static bool finish(id cb, const char *what) { + [cb commit]; + return completed(cb, what); +} +// The fixture's picture: top-left quarter (255, 51, 0, 255), the rest cleared blue; `bgra` swaps red and blue in memory. +static unsigned wrongTexels(const uint8_t *bytes, size_t row, unsigned side, bool bgra = false) { + unsigned bad = 0; + for (unsigned y = 0; y < side; ++y) + for (unsigned x = 0; x < side; ++x) { + const bool inside = x < side / 2 && y < side / 2; + uint8_t e[4] = {inside ? (uint8_t)255 : (uint8_t)0, inside ? (uint8_t)51 : (uint8_t)0, inside ? (uint8_t)0 : (uint8_t)255, 255}; + if (bgra) std::swap(e[0], e[2]); + if (memcmp(bytes + y * row + x * 4, e, 4)) ++bad; + } + return bad; +} +static IOSurfaceRef surface(unsigned side, OSType format) { + NSDictionary *p = @{(id)kIOSurfaceWidth: @(side), (id)kIOSurfaceHeight: @(side), (id)kIOSurfaceBytesPerElement: @4, + (id)kIOSurfacePixelFormat: @(format)}; + return IOSurfaceCreate((__bridge CFDictionaryRef)p); +} +static id pipeline(id device, id library, NSString *fragment, MTLPixelFormat format) { + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = [library newFunctionWithName:@"colored_vertex"]; + d.fragmentFunction = [library newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat = format; + NSError *error = nil; + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + CHECK(state, "render pipeline %s: %s", fragment.UTF8String, error.localizedDescription.UTF8String); + return state; +} +// Draws the fixture quad (vertex positions/colors at buffers 0/1) into `target`, cleared blue first. +static void drawQuad(id cb, id state, id target, id sampled, + const float (*positions)[2], unsigned count) { + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 1, 1); + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:state]; + [e setVertexBytes:positions length:sizeof(float) * 2 * count atIndex:0]; + std::vector colors; + for (unsigned i = 0; i < count; ++i) colors.insert(colors.end(), {1.0f, 0.2f, 0.0f, 1.0f}); + [e setVertexBytes:colors.data() length:colors.size() * sizeof(float) atIndex:1]; + if (sampled) [e setFragmentTexture:sampled atIndex:0]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:count]; + [e endEncoding]; +} +static const float kQuarter[6][2] = {{-1, 1}, {0, 1}, {-1, 0}, {0, 1}, {0, 0}, {-1, 0}}; +static const float kFull[6][2] = {{-1, 1}, {1, 1}, {-1, -1}, {1, 1}, {1, -1}, {-1, -1}}; + +// Runs every step on `device` (the frontend's); `library` is the bridge's runtime_examples fixture, `fcLibrary` (may be nil) the +// function-constant fixture, `qcPath` (may be nil) the target's QuartzCore default.metallib. +static void run(id device, id library, id fcLibrary, NSString *qcPath, Report *r) { + gReport = r; + @autoreleasepool { + CHECK([library.functionNames containsObject:@"sampled_fragment"], "function names"); + CHECK(![library newFunctionWithName:@"no_such_function"], "a missing function must be nil"); + id queue = [device newCommandQueue]; + CHECK(queue, "command queue"); + if (!queue) return; + + step(r, 0x6601); + // 1. compute: out[i] = 3 * in[i] (the fixture's scale_and_offset with scale 2) + { + NSError *error = nil; + id f = [library newFunctionWithName:@"scale_and_offset"]; + CHECK(f.functionType == MTLFunctionTypeKernel, "kernel type %lu", (unsigned long)f.functionType); + id state = [device newComputePipelineStateWithFunction:f error:&error]; + CHECK(state, "compute pipeline: %s", error.localizedDescription.UTF8String); + std::vector in(1000); + for (unsigned i = 0; i < in.size(); ++i) in[i] = (float)i; + id input = [device newBufferWithBytes:in.data() length:in.size() * 4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:in.size() * 4 options:MTLResourceStorageModeShared]; + const float scale = 2; + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setBuffer:input offset:0 atIndex:0]; + [e setBuffer:output offset:0 atIndex:1]; + [e setBytes:&scale length:4 atIndex:2]; + [e dispatchThreads:MTLSizeMake(in.size(), 1, 1) threadsPerThreadgroup:MTLSizeMake(64, 1, 1)]; + [e endEncoding]; + if (finish(cb, "compute")) { + const float *v = (const float *)output.contents; + unsigned bad = 0; + for (unsigned i = 0; i < in.size(); ++i) bad += v[i] != 3.0f * i; + CHECK(!bad, "compute: %u wrong outputs", bad); + } + } + id colored = pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + step(r, 0x6602); + // 2. render into a private texture, blit to a buffer + { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget; + d.storageMode = MTLStorageModePrivate; + id target = [device newTextureWithDescriptor:d]; + CHECK(target && target.width == 8 && target.pixelFormat == MTLPixelFormatRGBA8Unorm, "private texture"); + id readback = [device newBufferWithLength:8 * 8 * 4 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + drawQuad(cb, colored, target, nil, kQuarter, 6); + id b = [cb blitCommandEncoder]; + [b copyFromTexture:target sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(8, 8, 1) + toBuffer:readback destinationOffset:0 destinationBytesPerRow:32 destinationBytesPerImage:256]; + [b endEncoding]; + if (finish(cb, "render")) CHECK(!wrongTexels((const uint8_t *)readback.contents, 32, 8), "render: wrong texels"); + } + step(r, 0x6603); + // 3. render into an IOSurface (its pages imported, no copy), then 4. sample it into a second surface + IOSurfaceRef layer = r->hostSimulation ? nullptr : surface(64, 'RGBA'), screen = r->hostSimulation ? nullptr : surface(64, 'RGBA'); + CHECK(r->hostSimulation || (layer && screen), "IOSurfaceCreate"); + if (layer && screen) { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:64 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + d.storageMode = MTLStorageModeShared; + id layerTexture = [device newTextureWithDescriptor:d iosurface:layer plane:0]; + id screenTexture = [device newTextureWithDescriptor:d iosurface:screen plane:0]; + CHECK(layerTexture && screenTexture && layerTexture.iosurface == layer, "IOSurface textures"); + id cb = [queue commandBuffer]; + drawQuad(cb, colored, layerTexture, nil, kQuarter, 6); + if (finish(cb, "surface render")) { + IOSurfaceLock(layer, kIOSurfaceLockReadOnly, nullptr); + CHECK(!wrongTexels((const uint8_t *)IOSurfaceGetBaseAddress(layer), IOSurfaceGetBytesPerRow(layer), 64), "surface render: wrong texels"); + IOSurfaceUnlock(layer, kIOSurfaceLockReadOnly, nullptr); + } + step(r, 0x6604); + // sampled_fragment samples its texture at position * 1/8 (normalized), so an 8x8 viewport of the screen reads the layer whole + id sampled = pipeline(device, library, @"sampled_fragment", MTLPixelFormatRGBA8Unorm); + cb = [queue commandBuffer]; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = screenTexture; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 1, 0, 1); + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:sampled]; + [e setViewport:(MTLViewport){0, 0, 8, 8, 0, 1}]; + [e setVertexBytes:kFull length:sizeof kFull atIndex:0]; + float colors[6][4] = {}; + [e setVertexBytes:colors length:sizeof colors atIndex:1]; + [e setFragmentTexture:layerTexture atIndex:0]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [e endEncoding]; + if (finish(cb, "composition")) { + IOSurfaceLock(screen, kIOSurfaceLockReadOnly, nullptr); + const uint8_t *p = (const uint8_t *)IOSurfaceGetBaseAddress(screen); + std::vector corner(8 * 8 * 4); + for (unsigned y = 0; y < 8; ++y) memcpy(&corner[y * 32], p + y * IOSurfaceGetBytesPerRow(screen), 32); + const unsigned bad = wrongTexels(corner.data(), 32, 8); + const uint8_t *outside = p + 20 * IOSurfaceGetBytesPerRow(screen) + 20 * 4; + CHECK(!bad, "composition: %u wrong texels in the sampled corner", bad); + CHECK(outside[0] == 0 && outside[1] == 255 && outside[2] == 0, "composition: the rest keeps its clear color"); + IOSurfaceUnlock(screen, kIOSurfaceLockReadOnly, nullptr); + } + } + step(r, 0x6605); + // 5. compute over textures: invert_texels writes 1 - rgb of its input texel + { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + d.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite; + id src = [device newTextureWithDescriptor:d], dst = [device newTextureWithDescriptor:d]; + std::vector texels(16 * 16 * 4); + for (size_t i = 0; i < texels.size(); ++i) texels[i] = (uint8_t)(i * 7); + [src replaceRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0 withBytes:texels.data() bytesPerRow:64]; + NSError *error = nil; + id state = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"invert_texels"] error:&error]; + CHECK(state, "invert pipeline: %s", error.localizedDescription.UTF8String); + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setTexture:src atIndex:0]; + [e setTexture:dst atIndex:1]; + [e dispatchThreads:MTLSizeMake(16, 16, 1) threadsPerThreadgroup:MTLSizeMake(8, 8, 1)]; + [e endEncoding]; + if (finish(cb, "texture compute")) { + std::vector out(texels.size()); + [dst getBytes:out.data() bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + unsigned bad = 0; + for (size_t i = 0; i < out.size(); ++i) { + const int expected = i % 4 == 3 ? texels[i] : 255 - texels[i]; + bad += abs(out[i] - expected) > 1; + } + CHECK(!bad, "texture compute: %u wrong channels", bad); + } + } + step(r, 0x6606); + // 6. handlers: scheduled before completed, each once, after the GPU finished + { + __block int order = 0, scheduled = -1, done = -1; + id cb = [queue commandBuffer]; + id b = [cb blitCommandEncoder]; + id buffer = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + [b fillBuffer:buffer range:NSMakeRange(0, 4096) value:0xa5]; + [b endEncoding]; + [cb addScheduledHandler:^(id) {scheduled = order++;}]; + __block MTLCommandBufferStatus seen = MTLCommandBufferStatusNotEnqueued; + [cb addCompletedHandler:^(id c) {seen = c.status; done = order++;}]; + if (finish(cb, "handlers")) { + CHECK(scheduled == 0 && done == 1 && order == 2, "handler order %d %d", scheduled, done); + CHECK(seen == MTLCommandBufferStatusCompleted, "status inside the completed handler"); + const uint8_t *p = (const uint8_t *)buffer.contents; + CHECK(p[0] == 0xa5 && p[4095] == 0xa5, "fill"); + } + } + step(r, 0x6607); + // 7. a refused encoder call fails its command buffer (no silent drop), and the handlers still run + { + id cb = [queue commandBuffer]; + id e = nil; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = [device newTextureWithDescriptor:d]; + e = [cb renderCommandEncoderWithDescriptor:pass]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; // no pipeline set + [e endEncoding]; + __block bool ran = false; + [cb addCompletedHandler:^(id) {ran = true;}]; + [cb commit]; + [cb waitUntilCompleted]; + CHECK(cb.status == MTLCommandBufferStatusError && cb.error, "a draw without a pipeline must fail the command buffer"); + CHECK(ran, "the completed handler runs for a failed command buffer"); + } + step(r, 0x6609); + // 9. function constants (newFunctionWithName:constantValues:error:): the runtime's fixture kernel_function_constant_gate_fold, whose + // out[24] is in * 1000 * mode for mode 1..3 and in otherwise. + if (fcLibrary) { + for (int mode : {0, 2, 3}) { + MTLFunctionConstantValues *values = [MTLFunctionConstantValues new]; + [values setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + NSError *error = nil; + id f = [fcLibrary newFunctionWithName:@"fcfold" constantValues:values error:&error]; + CHECK(f, "specialized function: %s", error.localizedDescription.UTF8String); + id state = f ? [device newComputePipelineStateWithFunction:f error:&error] : nil; + CHECK(state, "specialized pipeline: %s", error.localizedDescription.UTF8String); + if (!state) continue; + const float in = 2; + id input = [device newBufferWithBytes:&in length:4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:32 * 4 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setBuffer:input offset:0 atIndex:0]; + [e setBuffer:output offset:0 atIndex:1]; + [e dispatchThreads:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [e endEncoding]; + if (finish(cb, "specialized compute")) { + const float got = ((const float *)output.contents)[24], expected = mode ? in * 1000 * mode : in; + CHECK(got == expected, "mode %d: out[24] %g, expected %g", mode, got, expected); + } + } + } + step(r, 0x6608); + // 8. the target's own QuartzCore library through newLibraryWithURL: (a universal metallib; its AIR64 slice): every kernel into a + // compute pipeline and every _vert_

/ _frag_

pair into a BGRA8 render pipeline. Refusals are counted, not hidden. + if (qcPath) { + NSError *error = nil; + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:qcPath] error:&error]; + CHECK(qc, "QuartzCore library: %s", error.localizedDescription.UTF8String); + unsigned kernels = 0, kernelsOk = 0, pairs = 0, pairsOk = 0; + NSArray *names = qc.functionNames; + for (NSString *name in names) { + id f = [qc newFunctionWithName:name]; + if (f.functionType == MTLFunctionTypeKernel) { + ++kernels; + kernelsOk += [device newComputePipelineStateWithFunction:f error:nil] != nil; + } else if (f.functionType == MTLFunctionTypeVertex) { + NSString *fragment = [name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if ([fragment isEqualToString:name] || ![names containsObject:fragment]) continue; + ++pairs; + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = f; + d.fragmentFunction = [qc newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + pairsOk += [device newRenderPipelineStateWithDescriptor:d error:nil] != nil; + } + } + CHECK(names.count && kernelsOk + pairsOk, "QuartzCore: nothing translated"); + r->qcFunctions = (unsigned)names.count; r->qcKernels = kernels; r->qcKernelsOk = kernelsOk; r->qcPairs = pairs; r->qcPairsOk = pairsOk; + } + step(r, 0x660a); + // 10. A second queue's command buffer waits for value 1, then copies; the first queue fills and signals 1. The waiting one is committed + // first, so its copy only sees the fill if the GPU wait held. A third waits for 3, which only the CPU signals. Then a fence orders two + // blit encoders of one command buffer, and the listener is notified at 3. + { + id event = [device newSharedEvent]; + id fence = [device newFence]; + id other = [device newCommandQueue]; + CHECK(event && fence && other, "event, fence, second queue"); + id a = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id b = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id c = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + if (event && fence && other) { + id waiting = [other commandBuffer]; + [waiting encodeWaitForEvent:event value:1]; + id copy = [waiting blitCommandEncoder]; + [copy copyFromBuffer:a sourceOffset:0 toBuffer:b destinationOffset:0 size:4096]; + [copy endEncoding]; + [waiting commit]; + id signalling = [queue commandBuffer]; + id fill = [signalling blitCommandEncoder]; + [fill fillBuffer:a range:NSMakeRange(0, 4096) value:0x3c]; + [fill endEncoding]; + [signalling encodeSignalEvent:event value:1]; + id cpu = [other commandBuffer]; + [cpu encodeWaitForEvent:event value:3]; + id second = [cpu blitCommandEncoder]; + [second fillBuffer:c range:NSMakeRange(0, 64) value:0x77]; + [second updateFence:fence]; + [second endEncoding]; + id third = [cpu blitCommandEncoder]; + [third waitForFence:fence]; + [third copyFromBuffer:c sourceOffset:0 toBuffer:c destinationOffset:2048 size:64]; + [third endEncoding]; + [cpu commit]; + if (finish(signalling, "event signal") && completed(waiting, "event wait")) { + CHECK(((const uint8_t *)b.contents)[4095] == 0x3c, "the GPU wait did not order the copy after the fill"); + CHECK(event.signaledValue == 1, "signaled value %llu", (unsigned long long)event.signaledValue); + } + CHECK(cpu.status != MTLCommandBufferStatusCompleted, "a wait for 3 completed before 3 was signaled"); + __block uint64_t heard = 0; + dispatch_semaphore_t notified = dispatch_semaphore_create(0); + MTLSharedEventListener *listener = r->hostSimulation ? nil : [[MTLSharedEventListener alloc] init]; + [event notifyListener:listener atValue:3 block:^(id, uint64_t value) {heard = value; dispatch_semaphore_signal(notified);}]; + event.signaledValue = 3; + if (completed(cpu, "CPU-signaled wait")) { + const uint8_t *p = (const uint8_t *)c.contents; + CHECK(p[2048] == 0x77 && p[2111] == 0x77, "the fence did not order the copy after the fill"); + } + CHECK(!dispatch_semaphore_wait(notified, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)) && heard >= 3, + "listener notification %llu", (unsigned long long)heard); + CHECK([event waitUntilSignaledValue:3 timeoutMS:10], "waitUntilSignaledValue"); + } + } + step(r, 0x660b); + // 11. heaps, a visibility result and a depth bias. An automatic heap's buffer is a buffer of that heap (`heap`, `usedSize`); a placement + // heap's two buffers are distinct bytes at their offsets; a pass counts the visible quad into its visibility result buffer (Boolean: 1); + // setDepthBias reaches the runtime without failing the command buffer. + { + MTLHeapDescriptor *hd = [MTLHeapDescriptor new]; + hd.size = 65536; + hd.storageMode = MTLStorageModeShared; + id heap = [device newHeapWithDescriptor:hd]; + CHECK(heap && heap.size >= 65536, "automatic heap"); + if (heap) { + id hb = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + CHECK(hb && hb.heap == heap, "a heap buffer names its heap"); + CHECK(heap.usedSize >= 4096, "heap used size %lu", (unsigned long)heap.usedSize); + if (hb) { + id cb = [queue commandBuffer]; + id fill = [cb blitCommandEncoder]; + [fill fillBuffer:hb range:NSMakeRange(0, 4096) value:0x5a]; + [fill endEncoding]; + if (finish(cb, "heap fill")) CHECK(((const uint8_t *)hb.contents)[0] == 0x5a && ((const uint8_t *)hb.contents)[4095] == 0x5a, "heap buffer contents"); + } + } + MTLHeapDescriptor *pd = [MTLHeapDescriptor new]; + pd.size = 65536; + pd.type = MTLHeapTypePlacement; + pd.storageMode = MTLStorageModeShared; + id placed = [device newHeapWithDescriptor:pd]; + CHECK(placed && placed.type == MTLHeapTypePlacement, "placement heap"); + if (placed) { + id first = [placed newBufferWithLength:4096 options:MTLResourceStorageModeShared offset:0]; + id second = [placed newBufferWithLength:4096 options:MTLResourceStorageModeShared offset:4096]; + CHECK(first && second && second.heapOffset == 4096, "placed buffers"); + if (first && second) { + id cb = [queue commandBuffer]; + id fill = [cb blitCommandEncoder]; + [fill fillBuffer:first range:NSMakeRange(0, 4096) value:0x11]; + [fill fillBuffer:second range:NSMakeRange(0, 4096) value:0x22]; + [fill endEncoding]; + if (finish(cb, "placement fill")) + CHECK(((const uint8_t *)first.contents)[4095] == 0x11 && ((const uint8_t *)second.contents)[0] == 0x22, "placed buffers overlap"); + } + } + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + td.storageMode = MTLStorageModePrivate; + id target = [device newTextureWithDescriptor:td]; + id visible = [device newBufferWithLength:8 options:MTLResourceStorageModeShared]; + if (target && visible) { + memset(visible.contents, 0, 8); + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.visibilityResultBuffer = visible; + id cb = [queue commandBuffer]; + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:colored]; + [e setVisibilityResultMode:MTLVisibilityResultModeBoolean offset:0]; + [e setDepthBias:0 slopeScale:0 clamp:0]; + std::vector colors; + for (unsigned i = 0; i < 6; ++i) colors.insert(colors.end(), {1.0f, 0.2f, 0.0f, 1.0f}); + [e setVertexBytes:kQuarter length:sizeof(float) * 2 * 6 atIndex:0]; + [e setVertexBytes:colors.data() length:colors.size() * sizeof(float) atIndex:1]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [e endEncoding]; + if (finish(cb, "visibility result")) CHECK(*(const uint64_t *)visible.contents != 0, "visibility result stayed 0 for a visible quad"); + } + } + if (layer) CFRelease(layer); + if (screen) CFRelease(screen); + } + step(r, 0x660f); + gReport = nullptr; +} +#undef CHECK +} // namespace frontendchecks diff --git a/bridge/NativeMetalBridge.h b/bridge/NativeMetalBridge.h new file mode 100644 index 0000000..7d6bbf0 --- /dev/null +++ b/bridge/NativeMetalBridge.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* C ABI of the native Metal bridge (NativeMetalBridge.rs): the metal2vulkan runtime over Turnip. */ +#ifndef NATIVE_METAL_BRIDGE_H +#define NATIVE_METAL_BRIDGE_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +struct NmbSelftest { + uint32_t stage; /* 0 when every step ran, else the failing step's marker (0x6501..0x6506) */ + uint32_t compute_bad; /* wrong outputs of 1000 */ + uint32_t render_bad; /* wrong texels of 64 (8x8) */ + uint32_t import_bad; /* wrong texels of 4096 (64x64 over imported host pages) */ + uint32_t qc_functions; + uint32_t qc_kernels_ok, qc_kernels_failed; + uint32_t qc_pairs_ok, qc_pairs_failed; + uint32_t reserved; + char message[96]; +}; +/* Markers: 0x6501 device, 0x6502 fixture library, 0x6503 compute, 0x6504 render, 0x6505 import, 0x6506 QuartzCore, 0x650f done. */ +uint32_t nmb_selftest(const char *loader, const char *cache_dir, const uint8_t *quartzcore, size_t quartzcore_bytes, + void (*mark)(uint32_t), struct NmbSelftest *out); +#ifdef __cplusplus +} +#endif +#endif diff --git a/bridge/NativeMetalBridge.rs b/bridge/NativeMetalBridge.rs new file mode 100644 index 0000000..40a753f --- /dev/null +++ b/bridge/NativeMetalBridge.rs @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: MIT +//! Native Metal bridge: the metal2vulkan runtime (the Metal API on Vulkan) over Turnip, behind a C ABI the Objective-C +//! Metal frontend and the plugin's self-test call. Turnip is opened as the runtime's Vulkan "loader" (it exports +//! vkGetInstanceProcAddr; iPadOS has no Khronos loader). The pinned runtime is GPL-2.0-or-later OR LGPL-3.0-or-later and is +//! linked unmodified from its own source; nothing of it is copied here. +//! +//! The self-test runs real Metal-level work and checks the GPU's results: a compute kernel, a render pass, a render into +//! host pages imported without a copy (the IOSurface path), and translation of the target's own QuartzCore library into +//! A740 pipelines. Each step reports a marker before it starts so a hang names the step. + +#[path = "NativeMetalObjects.rs"] +pub mod objects; + +use metal2vulkan::metal::{PixelFormat, PrimitiveType, TextureUsage}; +use metal2vulkan::pipeline::{ColorAttachmentDescriptor, RenderPipelineDescriptor, StageSource}; +use metal2vulkan_runtime::{ + BlitOption, ClearColor, CommandBuffer, CommandBufferStatus, Device, DeviceOptions, HostMemory, LoadAction, MetalLevel, + Origin, PipelineOptions, RenderPassDescriptor, ResourceOptions, Size, StoreAction, TextureDescriptor, +}; +use std::ffi::{c_char, c_void, CStr}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::time::Duration; + +/// The runtime's own example library (validation/fixtures/public/examples/runtime_examples.ll), assembled to AIR at build time. +static FIXTURE_AIR: &[u8] = include_bytes!(env!("NMB_FIXTURE_AIR")); + +unsafe extern "C" { + /// libkern (libSystem): clean and invalidate the data cache over a range, from user space. + fn sys_dcache_flush(start: *mut c_void, len: usize); +} + +/// Result of nmb_selftest; mirrors NativeMetalBridge.h. +#[repr(C)] +pub struct NmbSelftest { + /// 0 when every step ran; otherwise the marker of the step that failed. + pub stage: u32, + pub compute_bad: u32, + pub render_bad: u32, + pub import_bad: u32, + pub qc_functions: u32, + pub qc_kernels_ok: u32, + pub qc_kernels_failed: u32, + pub qc_pairs_ok: u32, + pub qc_pairs_failed: u32, + pub reserved: u32, + pub message: [u8; 96], +} + +type Mark = Option; + +struct Run<'a> { + out: &'a mut NmbSelftest, + mark: Mark, +} + +impl Run<'_> { + fn step(&mut self, marker: u32) { + if let Some(mark) = self.mark { + // SAFETY: the caller's callback, valid for the call. + unsafe { mark(marker) }; + } + } + fn fail(&mut self, marker: u32, why: impl std::fmt::Display) -> u32 { + self.out.stage = marker; + let text = format!("{why}"); + let n = text.len().min(self.out.message.len() - 1); + self.out.message[..n].copy_from_slice(&text.as_bytes()[..n]); + self.out.message[n] = 0; + marker + } +} + +const WAIT: Duration = Duration::from_secs(10); + +fn finish(command_buffer: &CommandBuffer) -> Result<(), String> { + command_buffer.commit().map_err(|e| format!("commit: {e:?}"))?; + command_buffer.wait_until_completed_for(WAIT).map_err(|_| "no completion in 10 s".to_string())?; + match command_buffer.status() { + CommandBufferStatus::Completed => Ok(()), + status => Err(format!("{status:?} {:?}", command_buffer.error())), + } +} + +fn open(loader: &CStr, cache: &CStr) -> Result { + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + // No macOS release names this host; answer as the newest measured Metal. + options.metal_level = Some(MetalLevel::NEWEST); + options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + Device::system_default_with(&options).map_err(|e| format!("device: {e:?}")) +} + +fn compute(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const COUNT: usize = 1000; + let function = library.new_function_with_name("scale_and_offset").map_err(|e| format!("{e:?}"))?; + let state = device + .new_compute_pipeline_state_with_function(&function, &PipelineOptions::default()) + .map_err(|e| format!("pipeline: {e:?}"))?; + let input: Vec = (0..COUNT).flat_map(|i| (i as f32).to_le_bytes()).collect(); + let input = device.new_buffer_with_bytes(&input, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let output = device.new_buffer((COUNT * 4) as u64, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let scale = device.new_buffer_with_bytes(&2.0f32.to_le_bytes(), ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let queue = device.new_command_queue().map_err(|e| format!("{e:?}"))?; + let command_buffer = queue.command_buffer().map_err(|e| format!("{e:?}"))?; + let mut encoder = command_buffer.compute_command_encoder().map_err(|e| format!("{e:?}"))?; + encoder.set_compute_pipeline_state(&state).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&input), 0, 0).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&output), 0, 1).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&scale), 0, 2).map_err(|e| format!("{e:?}"))?; + encoder.dispatch_threads(Size::new(COUNT as u64, 1, 1), Size::new(64, 1, 1)).map_err(|e| format!("{e:?}"))?; + encoder.end_encoding().map_err(|e| format!("{e:?}"))?; + finish(&command_buffer)?; + let contents = output.contents().ok_or("no contents")?; + // SAFETY: the buffer holds COUNT floats and no command buffer writes it now. + let values = unsafe { std::slice::from_raw_parts(contents.as_ptr() as *const f32, COUNT) }; + Ok(values.iter().enumerate().filter(|(i, v)| **v != 3.0 * *i as f32).count() as u32) +} + +/// Draws the fixture's top-left-quarter quad into `target` (cleared blue) and returns the encoded command buffer's result. +fn draw_quad(device: &Device, library: &metal2vulkan_runtime::Library, target: &metal2vulkan_runtime::Texture, + readback: Option<(&metal2vulkan_runtime::Buffer, u64, u64)>) -> Result<(), String> { + let vertex = library.new_function_with_name("colored_vertex").map_err(|e| format!("{e:?}"))?; + let fragment = library.new_function_with_name("colored_fragment").map_err(|e| format!("{e:?}"))?; + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function(vertex.function_descriptor())); + descriptor.fragment = Some(StageSource::Function(fragment.function_descriptor())); + descriptor.color_attachments[0] = Some(ColorAttachmentDescriptor::new(PixelFormat::RGBA8Unorm)); + let state = device.new_render_pipeline_state(&descriptor, &PipelineOptions::default()).map_err(|e| format!("pipeline: {e:?}"))?; + let positions: [[f32; 2]; 6] = [[-1.0, 1.0], [0.0, 1.0], [-1.0, 0.0], [0.0, 1.0], [0.0, 0.0], [-1.0, 0.0]]; + let positions: Vec = positions.iter().flatten().flat_map(|v| v.to_le_bytes()).collect(); + let colors: Vec = (0..6).flat_map(|_| [1.0f32, 0.2, 0.0, 1.0]).flat_map(|v| v.to_le_bytes()).collect(); + let positions = device.new_buffer_with_bytes(&positions, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let colors = device.new_buffer_with_bytes(&colors, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let mut pass = RenderPassDescriptor::new(); + let color = &mut pass.color_attachments[0]; + color.texture = Some(target.clone()); + color.load_action = LoadAction::Clear; + color.store_action = StoreAction::Store; + color.clear_color = ClearColor::new(0.0, 0.0, 1.0, 1.0); + let queue = device.new_command_queue().map_err(|e| format!("{e:?}"))?; + let command_buffer = queue.command_buffer().map_err(|e| format!("{e:?}"))?; + let mut encoder = command_buffer.render_command_encoder(&pass).map_err(|e| format!("{e:?}"))?; + encoder.set_render_pipeline_state(&state).map_err(|e| format!("{e:?}"))?; + encoder.set_vertex_buffer(Some(&positions), 0, 0).map_err(|e| format!("{e:?}"))?; + encoder.set_vertex_buffer(Some(&colors), 0, 1).map_err(|e| format!("{e:?}"))?; + encoder.draw_primitives(PrimitiveType::Triangle, 0, 6, 1, 0).map_err(|e| format!("{e:?}"))?; + encoder.end_encoding().map_err(|e| format!("{e:?}"))?; + if let Some((buffer, row, side)) = readback { + let mut blit = command_buffer.blit_command_encoder().map_err(|e| format!("{e:?}"))?; + blit.copy_from_texture_to_buffer(target, 0, 0, Origin::new(0, 0, 0), Size::new(side, side, 1), buffer, 0, row, + row * side, BlitOption::None).map_err(|e| format!("{e:?}"))?; + blit.end_encoding().map_err(|e| format!("{e:?}"))?; + } + finish(&command_buffer) +} + +/// Texels off the expected picture: the top-left quarter drawn (255, 51, 0, 255), the rest cleared blue. +fn wrong_texels(bytes: &[u8], row: usize, side: usize) -> u32 { + let mut bad = 0; + for y in 0..side { + for x in 0..side { + let at = y * row + x * 4; + let inside = x < side / 2 && y < side / 2; + let expected: [u8; 4] = if inside { [255, 51, 0, 255] } else { [0, 0, 255, 255] }; + if bytes[at..at + 4] != expected { + bad += 1; + } + } + } + bad +} + +fn render(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const SIDE: u64 = 8; + let mut texture = TextureDescriptor::texture_2d(PixelFormat::RGBA8Unorm, SIDE, SIDE, false); + texture.usage = TextureUsage::RENDER_TARGET; + texture.resource_options = ResourceOptions::private(); + let target = device.new_texture(&texture).map_err(|e| format!("{e:?}"))?; + let readback = device.new_buffer(SIDE * 4 * SIDE, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + draw_quad(device, library, &target, Some((&readback, SIDE * 4, SIDE)))?; + let contents = readback.contents().ok_or("no contents")?; + // SAFETY: SIDE rows of SIDE * 4 bytes, no longer written. + let bytes = unsafe { std::slice::from_raw_parts(contents.as_ptr(), (SIDE * 4 * SIDE) as usize) }; + Ok(wrong_texels(bytes, (SIDE * 4) as usize, SIDE as usize)) +} + +/// The IOSurface path without an IOSurface: page-aligned host pages imported without a copy (the kernel pins them, Turnip maps +/// them into the GPU), a linear render target over them, CPU caches cleaned before and invalidated after the GPU's use. +fn import(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const SIDE: u64 = 64; + const ROW: u64 = 256; + let bytes = (ROW * SIDE) as usize; + // SAFETY: an anonymous private mapping of whole pages, unmapped below after every GPU use completed. + let pages = unsafe { + libc::mmap(std::ptr::null_mut(), bytes, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_PRIVATE | libc::MAP_ANON, -1, 0) + }; + if pages == libc::MAP_FAILED { + return Err("mmap".into()); + } + let result = (|| { + // SAFETY: the pages were just mapped; nothing else uses them. + unsafe { std::ptr::write_bytes(pages as *mut u8, 0x5a, bytes) }; + // SAFETY: the mapping is writable, page aligned and outlives the buffer (dropped before the unmap). + let memory = unsafe { HostMemory::new(NonNull::new(pages as *mut u8).ok_or("null")?) }; + // SAFETY: as above, for `bytes` bytes. + let buffer = unsafe { device.new_buffer_with_bytes_no_copy(memory, bytes as u64, ResourceOptions::shared(), None) } + .map_err(|e| format!("no-copy buffer: {e:?}"))?; + let mut texture = TextureDescriptor::texture_2d(PixelFormat::RGBA8Unorm, SIDE, SIDE, false); + texture.usage = TextureUsage::RENDER_TARGET | TextureUsage::SHADER_READ; + texture.resource_options = ResourceOptions::shared(); + let target = buffer.new_texture(&texture, 0, ROW).map_err(|e| format!("texture over pages: {e:?}"))?; + // SAFETY: the range is this mapping. + unsafe { sys_dcache_flush(pages, bytes) }; + draw_quad(device, library, &target, None)?; + // SAFETY: the GPU finished writing; drop any line the CPU fetched meanwhile. + unsafe { sys_dcache_flush(pages, bytes) }; + // SAFETY: the pages hold `bytes` bytes and no command buffer writes them now. + let view = unsafe { std::slice::from_raw_parts(pages as *const u8, bytes) }; + let bad = wrong_texels(view, ROW as usize, SIDE as usize); + drop(target); + drop(buffer); + Ok(bad) + })(); + // SAFETY: the buffer and texture over the pages are gone. + unsafe { libc::munmap(pages, bytes) }; + result +} + +/// The AIR slice of a universal metallib (a fat container, big endian, of an AIR64 metallib and an Apple-GPU binary, as the target's +/// frameworks ship them), as Metal itself picks it; a thin metallib is returned as it is. +pub fn air_slice(bytes: &[u8]) -> Option<&[u8]> { + const AIR64: u32 = 0x0100_0017; + let word = |at: usize| bytes.get(at..at + 4).map(|w| u32::from_be_bytes([w[0], w[1], w[2], w[3]])); + if word(0)? != 0xcafe_babe { + return Some(bytes); + } + let count = word(4)? as usize; + for i in 0..count.min(16) { + let at = 8 + 20 * i; + if word(at)? == AIR64 { + let (offset, size) = (word(at + 8)? as usize, word(at + 12)? as usize); + return bytes.get(offset..offset.checked_add(size)?); + } + } + None +} + +/// Translates the target's own QuartzCore library: every kernel into a compute pipeline, and the vertex/fragment pairs named +/// alike into render pipelines over a BGRA8 attachment. Counts only; refusals (function constants, tile functions) are expected. +fn quartzcore(run: &mut Run, device: &Device, bytes: &[u8]) -> Result<(), String> { + let bytes = air_slice(bytes).ok_or("QC library: no AIR64 slice")?; + let library = device.new_library_with_data(bytes).map_err(|e| format!("QC library: {e:?}"))?; + let names = library.function_names(); + let list = std::env::var_os("NMB_LIST").is_some(); + run.out.qc_functions = names.len() as u32; + let mut vertices = Vec::new(); + let mut fragments = Vec::new(); + for name in &names { + let Ok(function) = library.new_function_with_name(name) else { continue }; + if list { + eprintln!("fn {name} {:?}", function.function_type()); + } + match function.function_type() { + Ok(metal2vulkan::metal::FunctionType::Kernel) => { + match device.new_compute_pipeline_state_with_function(&function, &PipelineOptions::default()) { + Ok(_) => run.out.qc_kernels_ok += 1, + Err(e) => { + run.out.qc_kernels_failed += 1; + if list { + eprintln!("kernel {name}: {e:?}"); + } + } + } + } + Ok(metal2vulkan::metal::FunctionType::Vertex) => vertices.push((function, name.clone())), + Ok(metal2vulkan::metal::FunctionType::Fragment) => fragments.push((function, name.clone())), + _ => {} + } + } + // The library's own pairs: `_vert_` with `_frag_`. + for (vertex, vname) in &vertices { + let Some(fname) = [vname.replace("_vert_", "_frag_"), vname.replacen("_vert", "_frag", 1)] + .into_iter() + .find(|candidate| fragments.iter().any(|(_, n)| n == candidate)) + else { + continue; + }; + let fragment = &fragments.iter().find(|(_, n)| *n == fname).unwrap().0; + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function(vertex.function_descriptor())); + descriptor.fragment = Some(StageSource::Function(fragment.function_descriptor())); + descriptor.color_attachments[0] = Some(ColorAttachmentDescriptor::new(PixelFormat::BGRA8Unorm)); + match device.new_render_pipeline_state(&descriptor, &PipelineOptions::default()) { + Ok(_) => run.out.qc_pairs_ok += 1, + Err(e) => { + run.out.qc_pairs_failed += 1; + if list { + eprintln!("pair {vname} {fname}: {e:?}"); + } + } + } + } + Ok(()) +} + +fn selftest(run: &mut Run, loader: &CStr, cache: &CStr, qc: &[u8]) -> u32 { + run.step(0x6501); + let device = match open(loader, cache) { + Ok(device) => device, + Err(why) => return run.fail(0x6501, why), + }; + run.step(0x6502); + let library = match metal2vulkan::library::Library::from_air(FIXTURE_AIR) { + Ok(library) => device.new_library_with_compile_library(library), + Err(why) => return run.fail(0x6502, format!("{why:?}")), + }; + run.step(0x6503); + match compute(&device, &library) { + Ok(bad) => run.out.compute_bad = bad, + Err(why) => return run.fail(0x6503, why), + } + run.step(0x6504); + match render(&device, &library) { + Ok(bad) => run.out.render_bad = bad, + Err(why) => return run.fail(0x6504, why), + } + run.step(0x6505); + match import(&device, &library) { + Ok(bad) => run.out.import_bad = bad, + Err(why) => return run.fail(0x6505, why), + } + if !qc.is_empty() { + run.step(0x6506); + if let Err(why) = quartzcore(run, &device, qc) { + return run.fail(0x6506, why); + } + } + run.step(0x650f); + 0 +} + +/// Runs the self-test; returns 0 or the marker of the failed step (also in out->stage, with out->message). +/// +/// # Safety +/// `loader` and `cache` are C strings; `qc` points at `qc_len` bytes (or is null with 0); `out` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_selftest(loader: *const c_char, cache: *const c_char, qc: *const u8, qc_len: usize, mark: Mark, + out: *mut NmbSelftest) -> u32 { + if loader.is_null() || cache.is_null() || out.is_null() { + return 0x65ff; + } + // SAFETY: the caller's contract. + let out = unsafe { &mut *out }; + // SAFETY: as above. + let (loader, cache) = unsafe { (CStr::from_ptr(loader), CStr::from_ptr(cache)) }; + let qc: &[u8] = if qc.is_null() || qc_len == 0 { &[] } else { unsafe { std::slice::from_raw_parts(qc, qc_len) } }; + // SAFETY: plain integers and bytes. + *out = unsafe { std::mem::zeroed() }; + let mut run = Run { out, mark }; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| selftest(&mut run, loader, cache, qc))) { + Ok(code) => code, + Err(_) => run.fail(0x65fe, "panic"), + } +} diff --git a/bridge/NativeMetalObjects.h b/bridge/NativeMetalObjects.h new file mode 100644 index 0000000..2ba32ff --- /dev/null +++ b/bridge/NativeMetalObjects.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: MIT */ +/* C ABI of the bridge's Metal objects (NativeMetalObjects.rs): the metal2vulkan runtime's device, resources, states, queues, command buffers + * and encoders as opaque handles, for the Objective-C frontend (NativeBridgeMetalFrontend.mm). Enumerations and option words are Metal's raw + * values. A call that fails returns 0/NULL and leaves its reason in nmb_last_error() (per thread). Every handle is released once with its + * release call; an encoder is released by its end call (or by nmb_encoder_discard when it is never ended). */ +#ifndef NATIVE_METAL_OBJECTS_H +#define NATIVE_METAL_OBJECTS_H +#include +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +typedef struct NmbDevice NmbDevice; +typedef struct NmbBuffer NmbBuffer; +typedef struct NmbHeap NmbHeap; +typedef struct NmbTexture NmbTexture; +typedef struct NmbLibrary NmbLibrary; +typedef struct NmbFunction NmbFunction; +typedef struct NmbComputePipeline NmbComputePipeline; +typedef struct NmbRenderPipeline NmbRenderPipeline; +typedef struct NmbSampler NmbSampler; +typedef struct NmbDepthStencil NmbDepthStencil; +typedef struct NmbQueue NmbQueue; +typedef struct NmbCommandBuffer NmbCommandBuffer; +typedef struct NmbEncoder NmbEncoder; +typedef struct NmbEvent NmbEvent; +typedef struct NmbFence NmbFence; + +const char *nmb_last_error(void); +/* The bridge's embedded AIR fixtures: the runtime's own runtime_examples and kernel_function_constant_gate_fold. */ +const void *nmb_fixture_air(size_t *length); +const void *nmb_function_constant_air(size_t *length); + +/* loader: a library exporting vkGetInstanceProcAddr (Turnip on the tablet). cache: a writable directory for translated shaders. */ +NmbDevice *nmb_device_open(const char *loader, const char *cache); +void nmb_device_release(NmbDevice *); +size_t nmb_device_name(const NmbDevice *, char *out, size_t size); +uint64_t nmb_device_max_buffer_length(const NmbDevice *); +// Pipeline makings (count, total us, longest us, waited us, failed), sink messages (info, warning+), shader cache hits/misses/writes. +bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]); +/* Artifact lookups and driver pipeline records are separate cache layers. */ +struct NmbShaderCacheStats { + uint64_t hits, misses, rejected, read_failures, writes, write_failures, evictions; + uint64_t driver_loads, driver_hits, driver_misses, driver_saves, driver_truncated, driver_mismatched, driver_unchanged; +}; +bool nmb_device_shader_cache_stats(const NmbDevice *, struct NmbShaderCacheStats *); +size_t nmb_last_warning(char *out, size_t size); +bool nmb_device_max_threads_per_threadgroup(const NmbDevice *, uint64_t out[3]); +/* Device queries the runtime answers by Metal selector: Apple's private counts and flags where the device has an answer of its own, + * and the public queries. A selector's kind says its result and argument. */ +enum {NMB_ANSWER_COUNT = 0, NMB_ANSWER_FLAG = 1, NMB_ANSWER_COUNT_ARG = 2, NMB_ANSWER_FLAG_ARG = 3}; +const char *nmb_device_answer_selector(size_t index, uint32_t *kind /* NMB_ANSWER_* */); /* NULL past the last */ +bool nmb_device_answer(const NmbDevice *, const char *selector, uint64_t arg, uint64_t *out); /* false: no answer, nmb_last_error says why */ +size_t nmb_device_vendor_name(const NmbDevice *, char *out, size_t size); +uint64_t nmb_device_linear_bytes_per_row(const NmbDevice *, uint64_t format, uint64_t width); /* 0: refused */ + +typedef struct NmbBinaryArchive NmbBinaryArchive; +NmbBinaryArchive *nmb_binary_archive_new(NmbDevice *, const char *path /* NULL: empty */); +void nmb_binary_archive_release(NmbBinaryArchive *); +bool nmb_binary_archive_serialize(NmbBinaryArchive *, const char *path); + +NmbBuffer *nmb_buffer_new(NmbDevice *, uint64_t length, uint64_t options, const void *bytes /* NULL: zeroed */); +/* Wraps page-aligned memory the caller keeps mapped until the buffer is released (an IOSurface's base, host pages). */ +NmbBuffer *nmb_buffer_new_no_copy(NmbDevice *, void *pointer, uint64_t length, uint64_t options); +void nmb_buffer_release(NmbBuffer *); +void *nmb_buffer_contents(const NmbBuffer *); +uint64_t nmb_buffer_length(const NmbBuffer *); +uint64_t nmb_buffer_gpu_address(const NmbBuffer *); + +struct NmbTextureDescriptor { + uint64_t type, format, width, height, depth, levels, samples, array_length, usage, options; +}; +/* Heaps: type 0 automatic / 1 placement; offset UINT64_MAX asks an automatic heap to place the resource. */ +NmbHeap *nmb_heap_new(NmbDevice *, uint64_t type, uint64_t size, uint64_t options); +void nmb_heap_release(NmbHeap *); +uint64_t nmb_heap_size(const NmbHeap *); +uint64_t nmb_heap_used_size(const NmbHeap *); +uint64_t nmb_heap_allocated_size(const NmbHeap *); +uint64_t nmb_heap_max_available_size(const NmbHeap *, uint64_t alignment); +uint64_t nmb_heap_set_purgeable_state(const NmbHeap *, uint64_t state); +NmbBuffer *nmb_heap_new_buffer(const NmbHeap *, uint64_t length, uint64_t options, uint64_t offset); +NmbTexture *nmb_heap_new_texture(const NmbHeap *, const struct NmbTextureDescriptor *, uint64_t offset); +NmbTexture *nmb_texture_new(NmbDevice *, const struct NmbTextureDescriptor *); +/* A linear texture over a buffer's bytes; iosurface marks the buffer as standing for a surface without planes (Metal's IOSurface rules). */ +NmbTexture *nmb_texture_new_on_buffer(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, bool iosurface); +NmbTexture *nmb_texture_new_linear(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image); +/* whole: a view of every level and slice as format; else levels and slices are {start, count} and kind the view's texture type. */ +NmbTexture *nmb_texture_new_view(NmbTexture *, uint64_t format, uint64_t kind, const uint64_t levels[2], const uint64_t slices[2], bool whole); +/* setPurgeableState: with Metal's raw state; the answer's raw state, 0 where it fails. */ +uint64_t nmb_texture_set_purgeable_state(const NmbTexture *, uint64_t state); +uint64_t nmb_buffer_set_purgeable_state(const NmbBuffer *, uint64_t state); +void nmb_texture_release(NmbTexture *); +void nmb_texture_describe(const NmbTexture *, struct NmbTextureDescriptor *out); +bool nmb_texture_replace(NmbTexture *, const uint64_t region[6], uint64_t level, uint64_t slice, const void *bytes, uint64_t bytes_per_row, + uint64_t bytes_per_image, uint64_t length); +bool nmb_texture_get_bytes(const NmbTexture *, void *bytes, uint64_t bytes_per_row, uint64_t bytes_per_image, uint64_t length, + const uint64_t region[6], uint64_t level, uint64_t slice); + +/* A metallib (a universal one gives its AIR64 slice), or with air set bare AIR bitcode. */ +NmbLibrary *nmb_library_new(NmbDevice *, const void *bytes, size_t length, bool air); +void nmb_library_release(NmbLibrary *); +uint32_t nmb_library_function_count(const NmbLibrary *); +size_t nmb_library_function_name(const NmbLibrary *, uint32_t index, char *out, size_t size); +NmbFunction *nmb_function_new(NmbLibrary *, const char *name); +/* A function specialized with constant values (newFunctionWithName:constantValues:error:): type is Metal's MTLDataType, name is null for an + * indexed value; bytes holds the value's bytes. */ +struct NmbConstant {uint64_t type, index; const char *name; const void *bytes; uint64_t length;}; +NmbFunction *nmb_function_new_specialized(NmbLibrary *, const char *name, const struct NmbConstant *constants, uint32_t count); +void nmb_function_release(NmbFunction *); +/* Metal's MTLFunctionType (1 vertex, 2 fragment, 3 kernel, ...). */ +uint64_t nmb_function_type(const NmbFunction *); + +NmbComputePipeline *nmb_compute_pipeline_new(NmbDevice *, NmbFunction *); +void nmb_compute_pipeline_release(NmbComputePipeline *); +uint64_t nmb_compute_pipeline_max_threads(const NmbComputePipeline *); +uint64_t nmb_compute_pipeline_execution_width(const NmbComputePipeline *); + +struct NmbColorAttachment { + uint64_t format, write_mask, blending, source_rgb, destination_rgb, rgb_operation, source_alpha, destination_alpha, alpha_operation; +}; +struct NmbVertexAttribute {uint64_t format, offset, buffer;}; +struct NmbVertexLayout {uint64_t stride, step_function, step_rate;}; +struct NmbRenderPipelineDescriptor { + NmbFunction *vertex, *fragment; + struct NmbColorAttachment colors[8]; + uint64_t depth_format, stencil_format, samples, topology, rasterization; + uint64_t vertex_inputs; /* 0: no vertex descriptor */ + struct NmbVertexAttribute attributes[31]; + struct NmbVertexLayout layouts[31]; +}; +NmbRenderPipeline *nmb_render_pipeline_new(NmbDevice *, const struct NmbRenderPipelineDescriptor *); +/* Descriptor binaryArchives and MTLPipelineOption bits. NULL options means defaults. + * The archive array and its handles must remain valid for the duration of this call. */ +struct NmbPipelineOptions {uint64_t flags; NmbBinaryArchive *const *archives; size_t archive_count;}; +struct NmbComputePipelineDescriptor { + NmbFunction *function; + uint64_t max_threads, multiple_of_execution_width, indirect, max_call_stack_depth, adding_binary_functions; + uint64_t mutability[31]; +}; +NmbComputePipeline *nmb_compute_pipeline_new_with_options(NmbDevice *, NmbFunction *, const struct NmbPipelineOptions *); +NmbComputePipeline *nmb_compute_pipeline_new_with_descriptor(NmbDevice *, const struct NmbComputePipelineDescriptor *, const struct NmbPipelineOptions *); +NmbRenderPipeline *nmb_render_pipeline_new_with_options(NmbDevice *, const struct NmbRenderPipelineDescriptor *, const struct NmbPipelineOptions *); +/* The archive makes the state as the matching new call does and keeps what it used (addRender/ComputePipelineFunctionsWithDescriptor:). */ +bool nmb_binary_archive_add_render(NmbBinaryArchive *, const struct NmbRenderPipelineDescriptor *); +bool nmb_binary_archive_add_compute(NmbBinaryArchive *, NmbFunction *); +bool nmb_binary_archive_add_compute_descriptor(NmbBinaryArchive *, const struct NmbComputePipelineDescriptor *); +void nmb_render_pipeline_release(NmbRenderPipeline *); + +struct NmbSamplerDescriptor { + uint64_t min_filter, mag_filter, mip_filter, s_address, t_address, r_address, border_color, normalized, max_anisotropy, compare, lod_average; + float lod_min, lod_max; +}; +NmbSampler *nmb_sampler_new(NmbDevice *, const struct NmbSamplerDescriptor *); +void nmb_sampler_release(NmbSampler *); + +struct NmbStencil {uint64_t compare, failure, depth_failure, pass, read_mask, write_mask;}; +struct NmbDepthStencilDescriptor {uint64_t compare, write; struct NmbStencil front, back;}; +NmbDepthStencil *nmb_depth_stencil_new(NmbDevice *, const struct NmbDepthStencilDescriptor *); +void nmb_depth_stencil_release(NmbDepthStencil *); + +NmbQueue *nmb_queue_new(NmbDevice *); +void nmb_queue_release(NmbQueue *); +NmbCommandBuffer *nmb_command_buffer_new(NmbQueue *); +void nmb_command_buffer_release(NmbCommandBuffer *); +/* 1 storage dependency, 0 unrelated/already complete, -1 unknown (caller must wait). + * Includes texture roots, buffer-backed textures, heaps and overlapping imported host ranges. + * This frontend does not rebind/remap live backing ranges. */ +int nmb_command_buffer_uses_buffer(const NmbCommandBuffer *, const NmbBuffer *); +int nmb_command_buffer_uses_texture(const NmbCommandBuffer *, const NmbTexture *); +/* The handler runs once, on a runtime thread, after the command buffer completed or failed. */ +bool nmb_command_buffer_on_completed(NmbCommandBuffer *, void (*handler)(void *context), void *context); +/* Runs on actual scheduling, before completion. Not called if submission fails before scheduling. + * The caller keeps context alive until completion (or a refused commit). */ +bool nmb_command_buffer_on_scheduled(NmbCommandBuffer *, void (*handler)(void *context), void *context); +bool nmb_command_buffer_commit(NmbCommandBuffer *); +/* Wait for scheduling and scheduled handlers only; false on a refused wait. */ +bool nmb_command_buffer_wait_scheduled(NmbCommandBuffer *); +/* 0 completed in time, 1 timed out (timeout_ms 0 waits without a bound). */ +int nmb_command_buffer_wait(NmbCommandBuffer *, uint64_t timeout_ms); +/* Metal's MTLCommandBufferStatus (0 not enqueued .. 4 completed, 5 error). */ +uint64_t nmb_command_buffer_status(const NmbCommandBuffer *); +size_t nmb_command_buffer_error(const NmbCommandBuffer *, char *out, size_t size); +double nmb_command_buffer_gpu_time(const NmbCommandBuffer *, bool end); + +/* Events: an MTLEvent (shared false) or an MTLSharedEvent within this process. */ +NmbEvent *nmb_event_new(NmbDevice *, bool shared); +void nmb_event_release(NmbEvent *); +uint64_t nmb_shared_event_value(const NmbEvent *); +bool nmb_shared_event_set_value(NmbEvent *, uint64_t value); +/* true once the value is reached within timeout_ms */ +bool nmb_shared_event_wait(NmbEvent *, uint64_t value, uint64_t timeout_ms); +/* handler(context, value) runs once on the bridge's listener thread when the value is reached (at once if it already is). */ +bool nmb_shared_event_notify(NmbEvent *, uint64_t value, void (*handler)(void *context, uint64_t value), void *context); +bool nmb_command_buffer_signal_event(NmbCommandBuffer *, NmbEvent *, uint64_t value); +bool nmb_command_buffer_wait_event(NmbCommandBuffer *, NmbEvent *, uint64_t value); +NmbFence *nmb_fence_new(NmbDevice *); +void nmb_fence_release(NmbFence *); +/* stages: Metal's MTLRenderStages for a render encoder, ignored otherwise */ +bool nmb_encoder_update_fence(NmbEncoder *, NmbFence *, uint64_t stages); +bool nmb_encoder_wait_fence(NmbEncoder *, NmbFence *, uint64_t stages); + +NmbEncoder *nmb_compute_encoder_new(NmbCommandBuffer *); +NmbEncoder *nmb_blit_encoder_new(NmbCommandBuffer *); +struct NmbPassAttachment { + NmbTexture *texture; + uint64_t level, slice, depth_plane, load, store; + double clear[4]; /* color, or clear[0] depth / stencil */ +}; +struct NmbRenderPassDescriptor { + struct NmbPassAttachment colors[8], depth, stencil; + uint64_t width, height, array_length, samples; + NmbBuffer *visibility; /* the pass's visibility result buffer, or null */ +}; +NmbEncoder *nmb_render_encoder_new(NmbCommandBuffer *, const struct NmbRenderPassDescriptor *); +bool nmb_encoder_end(NmbEncoder *); +void nmb_encoder_discard(NmbEncoder *); + +/* stage: 0 compute, 1 vertex, 2 fragment */ +bool nmb_encoder_set_buffer(NmbEncoder *, uint32_t stage, NmbBuffer *, uint64_t offset, uint32_t index); +bool nmb_encoder_set_buffer_offset(NmbEncoder *, uint32_t stage, uint64_t offset, uint32_t index); +bool nmb_encoder_set_bytes(NmbEncoder *, uint32_t stage, const void *bytes, size_t length, uint32_t index); +bool nmb_encoder_set_texture(NmbEncoder *, uint32_t stage, NmbTexture *, uint32_t index); +bool nmb_encoder_set_sampler(NmbEncoder *, uint32_t stage, NmbSampler *, uint32_t index); + +bool nmb_compute_set_pipeline(NmbEncoder *, NmbComputePipeline *); +bool nmb_compute_set_threadgroup_memory(NmbEncoder *, uint32_t length, uint32_t index); +bool nmb_compute_dispatch(NmbEncoder *, const uint64_t size[3], const uint64_t per_group[3], bool threads); + +bool nmb_render_set_pipeline(NmbEncoder *, NmbRenderPipeline *); +bool nmb_render_set_depth_stencil(NmbEncoder *, NmbDepthStencil *); +bool nmb_render_set_viewport(NmbEncoder *, const double viewport[6]); +bool nmb_render_set_scissor(NmbEncoder *, const uint64_t rect[4]); +bool nmb_render_set_raster(NmbEncoder *, uint64_t cull_mode, uint64_t winding, uint64_t fill_mode); +bool nmb_render_set_stencil_reference(NmbEncoder *, uint32_t front, uint32_t back); +bool nmb_render_set_blend_color(NmbEncoder *, const float color[4]); +bool nmb_render_set_depth_bias(NmbEncoder *, float bias, float slope_scale, float clamp); +bool nmb_render_set_line_width(NmbEncoder *, float width); +bool nmb_render_set_visibility_mode(NmbEncoder *, uint64_t mode, uint64_t offset); +bool nmb_render_draw(NmbEncoder *, uint64_t primitive, uint32_t start, uint32_t count, uint32_t instances, uint32_t base_instance); +bool nmb_render_draw_indexed(NmbEncoder *, uint64_t primitive, uint32_t count, uint64_t index_type, NmbBuffer *indices, uint64_t offset, + uint32_t instances, int32_t base_vertex, uint32_t base_instance); + +bool nmb_blit_copy_buffer(NmbEncoder *, NmbBuffer *source, uint64_t source_offset, NmbBuffer *destination, uint64_t destination_offset, + uint64_t size); +bool nmb_blit_fill_buffer(NmbEncoder *, NmbBuffer *, uint64_t start, uint64_t end, uint8_t value); +/* origins and sizes are x,y,z / width,height,depth */ +bool nmb_blit_copy_texture(NmbEncoder *, NmbTexture *source, uint64_t source_slice, uint64_t source_level, const uint64_t source_origin[3], + const uint64_t size[3], NmbTexture *destination, uint64_t destination_slice, uint64_t destination_level, + const uint64_t destination_origin[3]); +bool nmb_blit_buffer_to_texture(NmbEncoder *, NmbBuffer *source, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image, + const uint64_t size[3], NmbTexture *destination, uint64_t slice, uint64_t level, const uint64_t origin[3]); +bool nmb_blit_texture_to_buffer(NmbEncoder *, NmbTexture *source, uint64_t slice, uint64_t level, const uint64_t origin[3], + const uint64_t size[3], NmbBuffer *destination, uint64_t offset, uint64_t bytes_per_row, + uint64_t bytes_per_image); +bool nmb_blit_generate_mipmaps(NmbEncoder *, NmbTexture *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/bridge/NativeMetalObjects.rs b/bridge/NativeMetalObjects.rs new file mode 100644 index 0000000..1ac4904 --- /dev/null +++ b/bridge/NativeMetalObjects.rs @@ -0,0 +1,1987 @@ +// SPDX-License-Identifier: MIT +//! The bridge's Metal objects behind a C ABI (NativeMetalObjects.h): the metal2vulkan runtime's device, resources, states, queues, +//! command buffers and encoders as boxed handles for the Objective-C frontend. Raw Metal enumerations go through the runtime's own +//! checked conversions; a value it does not know fails the call (never a substitute). Failures leave their reason in a per-thread +//! string, and no call unwinds into C. + +use metal2vulkan::bind::SamplerDescriptor; +use metal2vulkan::metal::state::{DepthStencilDescriptor, StencilDescriptor}; +use metal2vulkan::library::{FunctionConstantValue, FunctionConstantValues}; +use metal2vulkan::metal::{ + BlendFactor, BlendOperation, ColorWriteMask, CompareFunction, CullMode, DataType, IndexType, PixelFormat, PrimitiveTopologyClass, + PrimitiveType, RenderStages, SamplerAddressMode, SamplerBorderColor, SamplerMinMagFilter, SamplerMipFilter, StencilOperation, TextureType, + TextureUsage, TriangleFillMode, VertexFormat, VertexStepFunction, Winding, +}; +use metal2vulkan::pipeline::{ + ColorAttachmentDescriptor, ComputePipelineDescriptor, RenderPipelineDescriptor, StageSource, VertexAttributeDescriptor, VertexBufferLayoutDescriptor, + VertexDescriptor, +}; +use metal2vulkan_runtime::{ + BinaryArchive, BinaryArchiveDescriptor, BlitCommandEncoder, FamilyRow, FeatureRow, GpuFamily, RowState, BlitOption, Buffer, ClearColor, CommandBuffer, CommandQueue, ComputeCommandEncoder, + ComputePipelineState, DebugMessage, DepthStencilState, Device, DeviceOptions, MessageSeverity, MessageSink, Event, EventRef, Fence, Function, Heap, HeapDescriptor, HeapType, HostMemory, IoSurfacePlane, Library, LoadAction, + MetalLevel, Origin, PipelineOption, PipelineOptions, PurgeableState, PrivateCount, PrivateFlag, PrivateProperties, MAX_COLOR_ATTACHMENTS, Region, RenderCommandEncoder, RenderPassDescriptor, RenderPipelineState, ResourceOptions, + SamplerState, ScissorRect, SharedEvent, SharedEventListener, Size, StoreAction, Texture, TextureDescriptor, Viewport, VisibilityResultMode, +}; +use std::cell::RefCell; +use std::ffi::{c_char, c_void, CStr}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::time::Duration; + +thread_local! { + static LAST: RefCell> = const { RefCell::new(Vec::new()) }; +} + +fn set_error(why: String) { + LAST.with(|last| { + let mut last = last.borrow_mut(); + last.clear(); + last.extend(why.bytes().filter(|b| *b != 0)); + last.push(0); + }); +} + +/// Runs `body`, turning an error or a panic into the per-thread reason and `fallback`. +fn guard(fallback: T, body: impl FnOnce() -> Result) -> T { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)) { + Ok(Ok(value)) => value, + Ok(Err(why)) => { + set_error(why); + fallback + } + Err(_) => { + set_error("panic in the Metal runtime".into()); + fallback + } + } +} + +fn boxed(value: T) -> *mut T { + Box::into_raw(Box::new(value)) +} + +/// The object a handle points at. +fn get<'a, T>(handle: *const T) -> Result<&'a T, String> { + // SAFETY: handles come from `boxed` and live until their release call (the header's contract). + unsafe { handle.as_ref() }.ok_or_else(|| "null handle".to_string()) +} + +fn get_mut<'a, T>(handle: *mut T) -> Result<&'a mut T, String> { + // SAFETY: as in `get`; an encoder is used by one thread at a time (Metal's rule). + unsafe { handle.as_mut() }.ok_or_else(|| "null handle".to_string()) +} + +fn release(handle: *mut T) { + if !handle.is_null() { + // SAFETY: from `boxed`, released once. + drop(unsafe { Box::from_raw(handle) }); + } +} + +fn copy_out(text: &str, out: *mut c_char, size: usize) -> usize { + if !out.is_null() && size > 0 { + let n = text.len().min(size - 1); + // SAFETY: the caller's buffer holds `size` bytes. + unsafe { + std::ptr::copy_nonoverlapping(text.as_ptr(), out as *mut u8, n); + *out.add(n) = 0; + } + } + text.len() +} + +fn raw>(value: u64, what: &str) -> Result +where + T::Error: std::fmt::Debug, +{ + u32::try_from(value).ok().and_then(|v| T::try_from(v).ok()).ok_or_else(|| format!("{what} {value} is not a Metal value")) +} + +fn e(what: &'static str) -> impl FnOnce(E) -> String { + move |error| format!("{what}: {error:?}") +} + +fn options(value: u64) -> Result { + ResourceOptions::from_raw(value).map_err(e("resource options")) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_last_error() -> *const c_char { + LAST.with(|last| { + let last = last.borrow(); + if last.is_empty() { c"".as_ptr() } else { last.as_ptr() as *const c_char } + }) +} + +/// The runtime's kernel_function_constant_gate_fold fixture, assembled to AIR at build time. +static FUNCTION_CONSTANT_AIR: &[u8] = include_bytes!(env!("NMB_FC_AIR")); + +/// # Safety +/// `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_fixture_air(length: *mut usize) -> *const c_void { + // SAFETY: the caller's contract. + if let Some(length) = unsafe { length.as_mut() } { + *length = crate::FIXTURE_AIR.len(); + } + crate::FIXTURE_AIR.as_ptr() as *const c_void +} + +/// # Safety +/// `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_constant_air(length: *mut usize) -> *const c_void { + // SAFETY: the caller's contract. + if let Some(length) = unsafe { length.as_mut() } { + *length = FUNCTION_CONSTANT_AIR.len(); + } + FUNCTION_CONSTANT_AIR.as_ptr() as *const c_void +} + +// ---- device ---- + +pub struct NmbDevice(Device, std::sync::OnceLock, std::sync::Mutex<()>); + +impl NmbDevice { + /// The runtime judges the private properties by the family rows once. + fn private(&self) -> &PrivateProperties { + self.1.get_or_init(|| self.0.private_properties()) + } +} + +/// # Safety +/// `loader` and `cache` are C strings. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_char) -> *mut NmbDevice { + guard(std::ptr::null_mut(), || { + if loader.is_null() || cache.is_null() { + return Err("loader and cache are required".into()); + } + // SAFETY: the caller's contract. + let (loader, cache) = unsafe { (CStr::from_ptr(loader), CStr::from_ptr(cache)) }; + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + options.metal_level = Some(MetalLevel::NEWEST); + options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + options.messages = Some(MessageSink::new(observe_message)); + Ok(boxed(NmbDevice(Device::system_default_with(&options).map_err(e("device"))?, std::sync::OnceLock::new(), std::sync::Mutex::new(())))) + }) +} + +// What the runtime says through the device's sink, kept as numbers for the plugin's telemetry: each render VkPipeline making the +// creation log (METAL2VULKAN_LOG_PIPELINES) reports ("... made in ms[ after waiting ms for its turn]"), and the other +// messages by severity with the last warning's text. +static PIPELINE_MAKINGS: [std::sync::atomic::AtomicU64; 5] = [const { std::sync::atomic::AtomicU64::new(0) }; 5]; // n, us, max us, waited us, failed +static OTHER_MESSAGES: [std::sync::atomic::AtomicU64; 2] = [const { std::sync::atomic::AtomicU64::new(0) }; 2]; // info, warning or worse +static LAST_WARNING: std::sync::Mutex = std::sync::Mutex::new(String::new()); + +fn millis_after(text: &str, marker: &str) -> Option { + let at = text.find(marker)? + marker.len(); + let rest = &text[at..]; + let end = rest.find(' ').unwrap_or(rest.len()); + rest[..end].parse().ok() +} + +fn observe_message(message: &DebugMessage) { + use std::sync::atomic::Ordering::Relaxed; + if message.id_name == "metal2vulkan-pipeline-creation" { + let made = millis_after(&message.text, " made in "); + let us = made.or_else(|| millis_after(&message.text, " failed in ")).map_or(0, |ms| (ms * 1000.0) as u64); + PIPELINE_MAKINGS[0].fetch_add(1, Relaxed); + PIPELINE_MAKINGS[1].fetch_add(us, Relaxed); + PIPELINE_MAKINGS[2].fetch_max(us, Relaxed); + PIPELINE_MAKINGS[3].fetch_add(millis_after(&message.text, " after waiting ").map_or(0, |ms| (ms * 1000.0) as u64), Relaxed); + if made.is_none() { + PIPELINE_MAKINGS[4].fetch_add(1, Relaxed); + } + return; + } + if message.severity >= MessageSeverity::Warning { + OTHER_MESSAGES[1].fetch_add(1, Relaxed); + if let Ok(mut last) = LAST_WARNING.lock() { + *last = message.text.chars().take(160).collect(); + } + } else { + OTHER_MESSAGES[0].fetch_add(1, Relaxed); + } +} + +/// The sink's counts into the caller's ten words: pipeline makings (count, total us, longest us, waited us, failed), messages (info, +/// warning or worse), and the device's shader cache hits, misses and writes. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_work_stats(device: *const NmbDevice, out: *mut u64) -> bool { + use std::sync::atomic::Ordering::Relaxed; + guard(false, || { + if out.is_null() { + return Err("null stats".into()); + } + // SAFETY: the header's ten words. + let out = unsafe { std::slice::from_raw_parts_mut(out, 10) }; + for (i, v) in PIPELINE_MAKINGS.iter().enumerate() { + out[i] = v.load(Relaxed); + } + out[5] = OTHER_MESSAGES[0].load(Relaxed); + out[6] = OTHER_MESSAGES[1].load(Relaxed); + let cache = get(device)?.0.shader_cache_statistics(); + out[7] = cache.hits; + out[8] = cache.misses; + out[9] = cache.writes; + Ok(true) + }) +} + +/// Separate artifact and driver-record counters, mirroring the C ABI. +#[repr(C)] +pub struct NmbShaderCacheStats { + pub hits: u64, + pub misses: u64, + pub rejected: u64, + pub read_failures: u64, + pub writes: u64, + pub write_failures: u64, + pub evictions: u64, + pub driver_loads: u64, + pub driver_hits: u64, + pub driver_misses: u64, + pub driver_saves: u64, + pub driver_truncated: u64, + pub driver_mismatched: u64, + pub driver_unchanged: u64, +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_shader_cache_stats(device: *const NmbDevice, out: *mut NmbShaderCacheStats) -> bool { + guard(false, || { + let out = get_mut(out)?; + let stats = get(device)?.0.shader_cache_statistics(); + *out = NmbShaderCacheStats { + hits: stats.hits, misses: stats.misses, rejected: stats.rejected, + read_failures: stats.read_failures, writes: stats.writes, write_failures: stats.write_failures, evictions: stats.evictions, + driver_loads: stats.driver_loads, driver_hits: stats.driver_pipeline_hits, driver_misses: stats.driver_pipeline_misses, + driver_saves: stats.driver_saves, driver_truncated: stats.driver_data_truncated, + driver_mismatched: stats.driver_data_mismatched, driver_unchanged: stats.driver_data_unchanged, + }; + Ok(true) + }) +} + +/// The last warning the device's sink received, as a C string; returns its length. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_last_warning(out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(copy_out(&LAST_WARNING.lock().map(|s| s.clone()).unwrap_or_default(), out, size))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_release(device: *mut NmbDevice) { + release(device) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_name(device: *const NmbDevice, out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(copy_out(get(device)?.0.name(), out, size))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_max_buffer_length(device: *const NmbDevice) -> u64 { + guard(0, || Ok(get(device)?.0.properties().max_buffer_length)) +} + +/// `maxThreadsPerThreadgroup` into the caller's three words. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_max_threads_per_threadgroup(device: *const NmbDevice, out: *mut u64) -> bool { + guard(false, || { + if out.is_null() { + return Err("null size".into()); + } + let s = get(device)?.0.properties().max_threads_per_threadgroup; + // SAFETY: the header's three words. + unsafe { std::slice::from_raw_parts_mut(out, 3) }.copy_from_slice(&[s.width, s.height, s.depth]); + Ok(true) + }) +} + +/// The device selectors `nmb_device_answer` takes beyond Apple's private counts and flags: the selector and its kind +/// (the header's NMB_ANSWER_* values). +const PUBLIC_ANSWERS: &[(&str, u32)] = &[ + ("maxThreadgroupMemoryLength", 0), + ("argumentBuffersSupport", 0), + ("readWriteTextureSupport", 0), + ("maxColorAttachments", 0), + ("minConstantBufferAlignmentBytes", 0), + ("deviceLinearReadOnlyTextureAlignmentBytes", 0), + ("areRasterOrderGroupsSupported", 1), + ("supportsRasterOrderGroups", 1), + ("isDepth24Stencil8PixelFormatSupported", 1), + ("areProgrammableSamplePositionsSupported", 1), + ("supports32BitFloatFiltering", 1), + ("supportsMemorylessRenderTargets", 1), + ("supportsSamplerReductionMode", 1), + ("supportsYCBCRFormats12", 1), + ("supportsYCBCRPackedFormats12", 1), + ("supportsSIMDGroup", 1), + ("minimumLinearTextureAlignmentForPixelFormat:", 2), + ("supportsTextureSampleCount:", 3), + ("supportsVertexAmplificationCount:", 3), + ("supportsRasterizationRateMapWithLayerCount:", 3), + ("supportsFamily:", 3), + ("supportsPrimitiveType:", 3), + ("supportsTextureWriteRoundingMode:", 3), +]; + +/// Every selector `nmb_device_answer` takes, with its kind, as C strings. +fn answer_selectors() -> &'static [(std::ffi::CString, u32)] { + static ALL: std::sync::OnceLock> = std::sync::OnceLock::new(); + ALL.get_or_init(|| { + let counts = PrivateCount::ALL.into_iter().map(|c| (c.selector(), 0)); + let flags = PrivateFlag::ALL.into_iter().map(|f| (f.selector(), 1)); + counts + .chain(flags) + .chain(PUBLIC_ANSWERS.iter().copied()) + .map(|(s, kind)| (std::ffi::CString::new(s).expect("selector"), kind)) + .collect() + }) +} + +/// The selector at `index` of the ones `nmb_device_answer` takes and its kind; null past the last. +/// +/// # Safety +/// `kind` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_answer_selector(index: usize, kind: *mut u32) -> *const c_char { + let Some((selector, k)) = answer_selectors().get(index) else { return std::ptr::null() }; + // SAFETY: the caller's contract. + if let Some(kind) = unsafe { kind.as_mut() } { + *kind = *k; + } + selector.as_ptr() +} + +fn device_answer(device: &NmbDevice, selector: &str, arg: u64) -> Result { + let d = &device.0; + let p = d.properties(); + let none = || format!("the device has no answer for {selector}"); + if let Some(c) = PrivateCount::ALL.into_iter().find(|c| c.selector() == selector) { + return device.private().count(c).value().ok_or_else(none); + } + if let Some(f) = PrivateFlag::ALL.into_iter().find(|f| f.selector() == selector) { + return device.private().flag(f).value().map(u64::from).ok_or_else(none); + } + let gap = |r: Result| r.map(u64::from).map_err(|g| format!("{selector}: {g:?}")); + let format_supported = |f: PixelFormat| d.pixel_format_capabilities(f) != metal2vulkan_runtime::FormatCapabilities::NONE; + Ok(match selector { + "maxThreadgroupMemoryLength" => p.max_threadgroup_memory_length, + "argumentBuffersSupport" => p.argument_buffers_support.as_raw() as u64, + "readWriteTextureSupport" => p.read_write_texture_support.as_raw() as u64, + // The runtime's pass takes MAX_COLOR_ATTACHMENTS; the device's Vulkan limits bound what a pipeline links. + "maxColorAttachments" => { + let l = &p.description.limits; + (MAX_COLOR_ATTACHMENTS as u64).min(l.max_color_attachments.min(l.max_fragment_output_attachments) as u64) + } + // The runtime binds a buffer at any offset within it (render.rs, setVertexBuffer); 4 is a scalar's alignment, which the shader's + // own loads need. + "minConstantBufferAlignmentBytes" => 4, + "deviceLinearReadOnlyTextureAlignmentBytes" => { + device.private().count(PrivateCount::DeviceLinearTextureAlignmentBytes).value().ok_or_else(none)? + } + "areRasterOrderGroupsSupported" | "supportsRasterOrderGroups" => gap(p.are_raster_order_groups_supported())?, + "isDepth24Stencil8PixelFormatSupported" => gap(p.is_depth24_stencil8_pixel_format_supported())?, + "areProgrammableSamplePositionsSupported" => gap(p.are_programmable_sample_positions_supported())?, + "supports32BitFloatFiltering" => p.supports_32bit_float_filtering as u64, + // A Memoryless texture is a transient render attachment (texture.rs), which every device makes. + "supportsMemorylessRenderTargets" => 1, + "supportsSamplerReductionMode" => gap(p.supports_sampler_reduction_mode())?, + // Apple's SIMD-group functions: the family's SIMD-scoped barrier and permute rows are exact on the device. + "supportsSIMDGroup" => { + let rows = d.family_rows(GpuFamily::Apple9); + let exact = |r: FeatureRow| rows.iter().any(|a| a.row == FamilyRow::Feature(r) && a.state == RowState::Exact); + (exact(FeatureRow::SimdBarrier) && exact(FeatureRow::SimdPermute)) as u64 + } + "supportsYCBCRFormats12" => format_supported(PixelFormat::YCBCR124202P) as u64, + "supportsYCBCRPackedFormats12" => format_supported(PixelFormat::YCBCR124202PPACKED) as u64, + "minimumLinearTextureAlignmentForPixelFormat:" => p + .minimum_linear_texture_alignment_for_pixel_format(raw::(arg, "pixel format")?) + .map_err(e("minimum linear texture alignment"))?, + "supportsTextureSampleCount:" => gap(p.supports_texture_sample_count(arg))?, + "supportsVertexAmplificationCount:" => gap(p.supports_vertex_amplification_count(arg))?, + "supportsRasterizationRateMapWithLayerCount:" => gap(p.supports_rasterization_rate_map_with_layer_count(arg))?, + "supportsFamily:" => d.supports_family(arg as i64) as u64, + "supportsPrimitiveType:" => p.supports_primitive_type(arg) as u64, + "supportsTextureWriteRoundingMode:" => p.supports_texture_write_rounding_mode(arg) as u64, + _ => return Err(format!("{selector} is not a device query of the bridge")), + }) +} + +/// The runtime's answer on the device to `selector` (one `nmb_device_answer_selector` lists), with `arg` for a selector that takes one. +/// False, with the reason, where the device has no answer of its own. +/// +/// # Safety +/// `selector` is a C string; `out` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_answer(device: *const NmbDevice, selector: *const c_char, arg: u64, out: *mut u64) -> bool { + guard(false, || { + if selector.is_null() || out.is_null() { + return Err("null selector or answer".into()); + } + // SAFETY: the caller's contract. + let selector = unsafe { CStr::from_ptr(selector) }.to_str().map_err(e("selector"))?; + let answer = device_answer(get(device)?, selector, arg)?; + // SAFETY: as above. + unsafe { *out = answer }; + Ok(true) + }) +} + +/// `requiredLinearTextureBytesPerRowForDescriptor:` (Apple SPI): a row of `width` texels of `format` rounded up to the format's +/// linear alignment, from the runtime's format description; 0, with the reason, for a format no buffer can back. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_linear_bytes_per_row(device: *const NmbDevice, format: u64, width: u64) -> u64 { + guard(0, || { + let d = &get(device)?.0; + let alignment = d + .properties() + .minimum_linear_texture_alignment_for_pixel_format(raw::(format, "pixel format")?) + .map_err(e("linear texture"))?; + let info = d.pixel_format_info(format); + // Flags bit 0x400: a format of blocks, which the alignment query has refused already. + if info.flags & 0x400 != 0 || info.words[0] == 0 { + return Err(format!("pixel format {format} has no texel size")); + } + Ok((width * info.words[0]).div_ceil(alignment) * alignment) + }) +} + +/// `vendorName`: the Vulkan device's PCI vendor, named where the ID is a known GPU vendor's, else the ID itself. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_vendor_name(device: *const NmbDevice, out: *mut c_char, size: usize) -> usize { + guard(0, || { + let id = get(device)?.0.properties().vendor_id; + let name = match id { + 0x5143 => "Qualcomm".to_string(), + 0x106b => "Apple".to_string(), + 0x10de => "NVIDIA".to_string(), + 0x1002 => "AMD".to_string(), + 0x8086 => "Intel".to_string(), + 0x13b5 => "ARM".to_string(), + _ => format!("0x{id:04x}"), + }; + Ok(copy_out(&name, out, size)) + }) +} + +// ---- buffers ---- + +pub struct NmbBuffer(Buffer); + +/// # Safety +/// `bytes` is null or holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_buffer_new(device: *mut NmbDevice, length: u64, options_word: u64, bytes: *const c_void) -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let options = options(options_word)?; + let buffer = if bytes.is_null() { + device.new_buffer(length, options) + } else { + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; + device.new_buffer_with_bytes(bytes, options) + }; + Ok(boxed(NmbBuffer(buffer.map_err(e("buffer"))?))) + }) +} + +/// # Safety +/// `pointer` is page aligned, holds `length` bytes and stays mapped until the buffer is released. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_buffer_new_no_copy(device: *mut NmbDevice, pointer: *mut c_void, length: u64, options_word: u64) + -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let device = get(device)?; + // SAFETY: the caller's contract. + let memory = unsafe { HostMemory::new(NonNull::new(pointer as *mut u8).ok_or("null memory")?) }; + let modes = options(options_word)?; + // The runtime's live-import lookup and subsequent join are separate: + // simultaneous first constructors can allocate distinct imports of the + // same pages. Cover every native ABI no-copy call, including mixtures + // of raw buffers and IOSurface textures. Each call still gets its own + // buffer object. This ABI passes neither a dma-buf source nor a runtime + // deallocator; no caller callback runs under this gate. Drop it before + // returning/formatting a failure; GPU work and resource release never + // take it. A device's unrelated APIs and other devices remain separate. + let buffer = { + let _gate = device.2.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + // SAFETY: as above. + unsafe { device.0.new_buffer_with_bytes_no_copy(memory, length, modes, None) } + }; + Ok(boxed(NmbBuffer(buffer.map_err(e("no-copy buffer"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_release(buffer: *mut NmbBuffer) { + release(buffer) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_contents(buffer: *const NmbBuffer) -> *mut c_void { + guard(std::ptr::null_mut(), || Ok(get(buffer)?.0.contents().map_or(std::ptr::null_mut(), |p| p.as_ptr() as *mut c_void))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_length(buffer: *const NmbBuffer) -> u64 { + guard(0, || Ok(get(buffer)?.0.length())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_gpu_address(buffer: *const NmbBuffer) -> u64 { + guard(0, || Ok(get(buffer)?.0.gpu_address().0)) +} + +// ---- heaps ---- + +pub struct NmbHeap(Heap); + +/// `newHeapWithDescriptor:`: type (0 automatic, 1 placement), size, resource options. Sparse heaps are not made here. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_new(device: *mut NmbDevice, heap_type: u64, size: u64, options_word: u64) -> *mut NmbHeap { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let heap_type = HeapType::from_raw(heap_type).map_err(e("heap type"))?; + if heap_type == HeapType::Sparse { + return Err("sparse heaps are not implemented".into()); + } + let descriptor = HeapDescriptor::new(heap_type, size, options(options_word)?); + Ok(boxed(NmbHeap(device.new_heap(&descriptor).map_err(e("heap"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_release(heap: *mut NmbHeap) { + release(heap) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_size(heap: *const NmbHeap) -> u64 { + guard(0, || Ok(get(heap)?.0.size())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_used_size(heap: *const NmbHeap) -> u64 { + guard(0, || Ok(get(heap)?.0.used_size())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_allocated_size(heap: *const NmbHeap) -> u64 { + guard(0, || Ok(get(heap)?.0.current_allocated_size())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_max_available_size(heap: *const NmbHeap, alignment: u64) -> u64 { + guard(0, || Ok(get(heap)?.0.max_available_size(alignment).map_err(e("heap available size"))?)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_set_purgeable_state(heap: *const NmbHeap, state: u64) -> u64 { + guard(0, || { + let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; + Ok(get(heap)?.0.set_purgeable_state(state).map_err(e("heap purgeable state"))?.raw()) + }) +} + +/// A buffer in the heap: automatic heaps place it, a placement heap takes `offset` (`u64::MAX` asks for automatic placement). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_new_buffer(heap: *const NmbHeap, length: u64, options_word: u64, offset: u64) -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let heap = &get(heap)?.0; + let options = options(options_word)?; + let buffer = if offset == u64::MAX { heap.new_buffer(length, options) } else { heap.new_buffer_at_offset(length, options, offset) }; + Ok(boxed(NmbBuffer(buffer.map_err(e("heap buffer"))?))) + }) +} + +/// A texture in the heap, placed as `nmb_heap_new_buffer` places a buffer. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_new_texture(heap: *const NmbHeap, descriptor: *const NmbTextureDescriptor, offset: u64) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let heap = &get(heap)?.0; + let descriptor = texture_descriptor(descriptor)?; + let texture = if offset == u64::MAX { heap.new_texture(&descriptor) } else { heap.new_texture_at_offset(&descriptor, offset) }; + Ok(boxed(NmbTexture(texture.map_err(e("heap texture"))?))) + }) +} + +// ---- textures ---- + +#[repr(C)] +pub struct NmbTextureDescriptor { + pub kind: u64, + pub format: u64, + pub width: u64, + pub height: u64, + pub depth: u64, + pub levels: u64, + pub samples: u64, + pub array_length: u64, + pub usage: u64, + pub options: u64, +} + +pub struct NmbTexture(Texture); + +fn texture_descriptor(d: *const NmbTextureDescriptor) -> Result { + let d = get(d)?; + let mut out = TextureDescriptor::default(); + out.texture_type = raw::(d.kind, "texture type")?; + out.pixel_format = raw::(d.format, "pixel format")?; + out.width = d.width; + out.height = d.height; + out.depth = d.depth; + out.mipmap_level_count = d.levels; + out.sample_count = d.samples; + out.array_length = d.array_length; + out.usage = raw::(d.usage, "texture usage")?; + out.resource_options = options(d.options)?; + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new(device: *mut NmbDevice, descriptor: *const NmbTextureDescriptor) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + Ok(boxed(NmbTexture(get(device)?.0.new_texture(&descriptor).map_err(e("texture"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_on_buffer(buffer: *mut NmbBuffer, descriptor: *const NmbTextureDescriptor, offset: u64, + bytes_per_row: u64, iosurface: bool) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + let buffer = &get(buffer)?.0; + let texture = if iosurface { + buffer.new_texture_with_iosurface_plane(&descriptor, IoSurfacePlane::new(offset, bytes_per_row, false)) + } else { + buffer.new_texture(&descriptor, offset, bytes_per_row) + }; + Ok(boxed(NmbTexture(texture.map_err(e("texture over buffer"))?))) + }) +} + +/// `newLinearTextureWithDescriptor:offset:bytesPerRow:bytesPerImage:` (Apple SPI). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_linear(buffer: *mut NmbBuffer, descriptor: *const NmbTextureDescriptor, offset: u64, bytes_per_row: u64, + bytes_per_image: u64) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + let texture = get(buffer)?.0.new_linear_texture(&descriptor, offset, bytes_per_row, bytes_per_image); + Ok(boxed(NmbTexture(texture.map_err(e("linear texture"))?))) + }) +} + +/// A view of `texture`: of the whole texture as `format` where `whole`, else of `levels` and `slices` (start, count) as `format` and `kind`. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_view(texture: *mut NmbTexture, format: u64, kind: u64, levels: *const u64, slices: *const u64, whole: bool) + -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let t = &get(texture)?.0; + let format = raw::(format, "pixel format")?; + let view = if whole { + t.new_texture_view(format) + } else { + if levels.is_null() || slices.is_null() { + return Err("null levels or slices".into()); + } + // SAFETY: the header's two words each. + let (l, s) = unsafe { (std::slice::from_raw_parts(levels, 2), std::slice::from_raw_parts(slices, 2)) }; + let ranges = (l[0]..l[0].checked_add(l[1]).ok_or("levels overflow")?, s[0]..s[0].checked_add(s[1]).ok_or("slices overflow")?); + t.new_texture_view_with_levels_slices(format, raw::(kind, "texture type")?, ranges.0, ranges.1) + }; + Ok(boxed(NmbTexture(view.map_err(e("texture view"))?))) + }) +} + +/// `setPurgeableState:` on a texture: the runtime's answer as Metal's raw state, 0 where it fails. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_set_purgeable_state(texture: *const NmbTexture, state: u64) -> u64 { + guard(0, || { + let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; + Ok(get(texture)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw()) + }) +} + +/// `setPurgeableState:` on a buffer, as on a texture. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_set_purgeable_state(buffer: *const NmbBuffer, state: u64) -> u64 { + guard(0, || { + let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; + Ok(get(buffer)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_release(texture: *mut NmbTexture) { + release(texture) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_describe(texture: *const NmbTexture, out: *mut NmbTextureDescriptor) { + guard((), || { + let t = &get(texture)?.0; + let out = get_mut(out)?; + *out = NmbTextureDescriptor { + kind: t.texture_type().as_raw() as u64, + format: t.pixel_format().as_raw() as u64, + width: t.width(), + height: t.height(), + depth: t.depth(), + levels: t.mipmap_level_count(), + samples: t.sample_count(), + array_length: t.array_length(), + usage: t.usage().as_raw() as u64, + options: t.resource_options().raw(), + }; + Ok(()) + }) +} + +fn region(r: *const u64) -> Result { + if r.is_null() { + return Err("null region".into()); + } + // SAFETY: the header's six words. + let r = unsafe { std::slice::from_raw_parts(r, 6) }; + Ok(Region::new_3d(r[0], r[1], r[2], r[3], r[4], r[5])) +} + +fn triple(v: *const u64) -> Result<[u64; 3], String> { + if v.is_null() { + return Err("null origin or size".into()); + } + // SAFETY: the header's three words. + let v = unsafe { std::slice::from_raw_parts(v, 3) }; + Ok([v[0], v[1], v[2]]) +} + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_texture_replace(texture: *mut NmbTexture, region_words: *const u64, level: u64, slice: u64, + bytes: *const c_void, bytes_per_row: u64, bytes_per_image: u64, length: u64) -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; + get(texture)?.0.replace_region(region(region_words)?, level, slice, bytes, bytes_per_row, bytes_per_image) + .map_err(e("replace region"))?; + Ok(true) + }) +} + +/// # Safety +/// `bytes` holds `length` writable bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_texture_get_bytes(texture: *const NmbTexture, bytes: *mut c_void, bytes_per_row: u64, bytes_per_image: u64, + length: u64, region_words: *const u64, level: u64, slice: u64) -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts_mut(bytes as *mut u8, length as usize) }; + get(texture)?.0.get_bytes(bytes, bytes_per_row, bytes_per_image, region(region_words)?, level, slice) + .map_err(e("get bytes"))?; + Ok(true) + }) +} + +// ---- libraries and functions ---- + +pub struct NmbLibrary(Library, Vec); +pub struct NmbFunction(Function); + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_library_new(device: *mut NmbDevice, bytes: *const c_void, length: usize, air: bool) -> *mut NmbLibrary { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + if bytes.is_null() || length == 0 { + return Err("no library bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length) }; + let library = if air { + device.new_library_with_compile_library(metal2vulkan::library::Library::from_air(bytes).map_err(e("AIR"))?) + } else { + device.new_library_with_data(crate::air_slice(bytes).ok_or("metallib without an AIR64 slice")?).map_err(e("metallib"))? + }; + let names = library.function_names(); + Ok(boxed(NmbLibrary(library, names))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_release(library: *mut NmbLibrary) { + release(library) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_function_count(library: *const NmbLibrary) -> u32 { + guard(0, || Ok(get(library)?.1.len() as u32)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_function_name(library: *const NmbLibrary, index: u32, out: *mut c_char, size: usize) -> usize { + guard(0, || { + let names = &get(library)?.1; + Ok(copy_out(names.get(index as usize).ok_or("function index")?, out, size)) + }) +} + +/// # Safety +/// `name` is a C string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_new(library: *mut NmbLibrary, name: *const c_char) -> *mut NmbFunction { + guard(std::ptr::null_mut(), || { + if name.is_null() { + return Err("null function name".into()); + } + // SAFETY: the caller's contract. + let name = unsafe { CStr::from_ptr(name) }.to_str().map_err(e("function name"))?; + Ok(boxed(NmbFunction(get(library)?.0.new_function_with_name(name).map_err(e("function"))?))) + }) +} + +#[repr(C)] +pub struct NmbConstant { + pub kind: u64, + pub index: u64, + pub name: *const c_char, + pub bytes: *const c_void, + pub length: u64, +} + +/// # Safety +/// `name` is a C string; `constants` holds `count` entries whose names are C strings (or null) and whose bytes hold `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_new_specialized(library: *mut NmbLibrary, name: *const c_char, constants: *const NmbConstant, + count: u32) -> *mut NmbFunction { + guard(std::ptr::null_mut(), || { + if name.is_null() || (constants.is_null() && count > 0) { + return Err("null function name or constants".into()); + } + // SAFETY: the caller's contract. + let name = unsafe { CStr::from_ptr(name) }.to_str().map_err(e("function name"))?; + let constants = if count == 0 { &[][..] } else { unsafe { std::slice::from_raw_parts(constants, count as usize) } }; + let mut values = FunctionConstantValues::defaults(); + for c in constants { + if c.bytes.is_null() { + return Err("constant without bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(c.bytes as *const u8, c.length as usize) }; + let value = FunctionConstantValue::new(raw::(c.kind, "constant type")?, bytes.to_vec()); + if c.name.is_null() { + values.by_index.insert(u32::try_from(c.index).map_err(e("constant index"))?, value); + } else { + // SAFETY: the caller's contract. + let key = unsafe { CStr::from_ptr(c.name) }.to_str().map_err(e("constant name"))?; + values.by_name.insert(key.to_string(), value); + } + } + let function = get(library)?.0.new_function_with_name_constant_values(name, &values).map_err(e("specialized function"))?; + Ok(boxed(NmbFunction(function))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_release(function: *mut NmbFunction) { + release(function) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_type(function: *const NmbFunction) -> u64 { + guard(0, || Ok(get(function)?.0.function_type().map_err(e("function type"))?.as_raw() as u64)) +} + +// ---- pipelines and states ---- + +pub struct NmbComputePipeline(ComputePipelineState); +pub struct NmbRenderPipeline(RenderPipelineState); +pub struct NmbSampler(SamplerState); +pub struct NmbDepthStencil(DepthStencilState); + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_new(device: *mut NmbDevice, function: *mut NmbFunction) -> *mut NmbComputePipeline { + // SAFETY: null options names no array, the existing handles follow this ABI. + unsafe { nmb_compute_pipeline_new_with_options(device, function, std::ptr::null()) } +} + +/// # Safety +/// `options` is null or a valid record with a counted array of live archive handles. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_compute_pipeline_new_with_options(device: *mut NmbDevice, function: *mut NmbFunction, + options: *const NmbPipelineOptions) -> *mut NmbComputePipeline { + guard(std::ptr::null_mut(), || { + let resolved = unsafe { pipeline_options(options)? }; + let state = get(device)?.0.new_compute_pipeline_state_with_function(&get(function)?.0, &resolved); + Ok(boxed(NmbComputePipeline(state.map_err(e("compute pipeline"))?))) + }) +} + +#[repr(C)] +pub struct NmbComputePipelineDescriptor { + pub function: *mut NmbFunction, + pub max_threads: u64, + pub multiple_of_execution_width: u64, + pub indirect: u64, + pub max_call_stack_depth: u64, + pub adding_binary_functions: u64, + pub mutability: [u64; 31], +} + +fn compute_descriptor(descriptor: *const NmbComputePipelineDescriptor) -> Result { + let d = get(descriptor)?; + let mut out = ComputePipelineDescriptor::new(StageSource::Function(get(d.function)?.0.function_descriptor())); + out.max_total_threads_per_threadgroup = if d.max_threads == 0 { None } else { Some(u32::try_from(d.max_threads).map_err(e("compute max threads"))?) }; + out.threadgroup_size_is_multiple_of_execution_width = d.multiple_of_execution_width != 0; + out.support_indirect_command_buffers = d.indirect != 0; + out.max_call_stack_depth = u32::try_from(d.max_call_stack_depth).map_err(e("compute stack depth"))?; + out.support_adding_binary_functions = d.adding_binary_functions != 0; + for (index, &mutability) in d.mutability.iter().enumerate() { + if mutability != 0 { + out.buffers.insert(index as u32, raw::(mutability, "buffer mutability")?); + } + } + Ok(out) +} + +/// # Safety +/// `options` is null or a valid record with a counted array of live archive handles. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_compute_pipeline_new_with_descriptor(device: *mut NmbDevice, descriptor: *const NmbComputePipelineDescriptor, + options: *const NmbPipelineOptions) -> *mut NmbComputePipeline { + guard(std::ptr::null_mut(), || { + let resolved = unsafe { pipeline_options(options)? }; + let state = get(device)?.0.new_compute_pipeline_state(&compute_descriptor(descriptor)?, &resolved).map_err(e("compute pipeline"))?; + Ok(boxed(NmbComputePipeline(state))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_release(pipeline: *mut NmbComputePipeline) { + release(pipeline) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_max_threads(pipeline: *const NmbComputePipeline) -> u64 { + guard(0, || Ok(get(pipeline)?.0.max_total_threads_per_threadgroup() as u64)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_execution_width(pipeline: *const NmbComputePipeline) -> u64 { + guard(0, || Ok(get(pipeline)?.0.thread_execution_width() as u64)) +} + +#[repr(C)] +pub struct NmbColorAttachment { + pub format: u64, + pub write_mask: u64, + pub blending: u64, + pub source_rgb: u64, + pub destination_rgb: u64, + pub rgb_operation: u64, + pub source_alpha: u64, + pub destination_alpha: u64, + pub alpha_operation: u64, +} + +#[repr(C)] +pub struct NmbVertexAttribute { + pub format: u64, + pub offset: u64, + pub buffer: u64, +} + +#[repr(C)] +pub struct NmbVertexLayout { + pub stride: u64, + pub step_function: u64, + pub step_rate: u64, +} + +#[repr(C)] +pub struct NmbRenderPipelineDescriptor { + pub vertex: *mut NmbFunction, + pub fragment: *mut NmbFunction, + pub colors: [NmbColorAttachment; 8], + pub depth_format: u64, + pub stencil_format: u64, + pub samples: u64, + pub topology: u64, + pub rasterization: u64, + pub vertex_inputs: u64, + pub attributes: [NmbVertexAttribute; 31], + pub layouts: [NmbVertexLayout; 31], +} + +fn optional_format(value: u64) -> Result, String> { + if value == 0 { Ok(None) } else { raw::(value, "pixel format").map(Some) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_pipeline_new(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor) -> *mut NmbRenderPipeline { + // SAFETY: null options names no array; the device/descriptor handles follow the existing ABI. + unsafe { nmb_render_pipeline_new_with_options(device, descriptor, std::ptr::null()) } +} + +#[repr(C)] +pub struct NmbPipelineOptions { + pub flags: u64, + pub archives: *const *mut NmbBinaryArchive, + pub archive_count: usize, +} + +/// # Safety +/// The options record and its counted archive array are valid for this call. +unsafe fn pipeline_options(options: *const NmbPipelineOptions) -> Result { + let mut resolved = PipelineOptions::default(); + if let Some(options) = unsafe { options.as_ref() } { + resolved.options = PipelineOption::from_raw(options.flags); + if options.archive_count > 0 { + if options.archives.is_null() || options.archive_count > isize::MAX as usize / std::mem::size_of::<*mut NmbBinaryArchive>() { + return Err("invalid binary archive array".into()); + } + let archives = unsafe { std::slice::from_raw_parts(options.archives, options.archive_count) }; + for &archive in archives { + resolved.binary_archives.push(get(archive)?.0.clone()); + } + } + } + Ok(resolved) +} + +/// # Safety +/// `options` is null or a valid record; its array holds `archive_count` live archive handles. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_render_pipeline_new_with_options(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor, + options: *const NmbPipelineOptions) -> *mut NmbRenderPipeline { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let resolved = unsafe { pipeline_options(options)? }; + let state = device.new_render_pipeline_state(&render_descriptor(descriptor)?, &resolved).map_err(e("render pipeline"))?; + Ok(boxed(NmbRenderPipeline(state))) + }) +} + +fn render_descriptor(descriptor: *const NmbRenderPipelineDescriptor) -> Result { + let d = get(descriptor)?; + let mut out = RenderPipelineDescriptor::new(StageSource::Function(get(d.vertex)?.0.function_descriptor())); + if !d.fragment.is_null() { + out.fragment = Some(StageSource::Function(get(d.fragment)?.0.function_descriptor())); + } + for (slot, c) in out.color_attachments.iter_mut().zip(d.colors.iter()) { + if c.format == 0 { + continue; + } + let mut a = ColorAttachmentDescriptor::new(raw::(c.format, "color format")?); + a.write_mask = raw::(c.write_mask, "write mask")?; + a.blending_enabled = c.blending != 0; + a.source_rgb_blend_factor = raw::(c.source_rgb, "blend factor")?; + a.destination_rgb_blend_factor = raw::(c.destination_rgb, "blend factor")?; + a.rgb_blend_operation = raw::(c.rgb_operation, "blend operation")?; + a.source_alpha_blend_factor = raw::(c.source_alpha, "blend factor")?; + a.destination_alpha_blend_factor = raw::(c.destination_alpha, "blend factor")?; + a.alpha_blend_operation = raw::(c.alpha_operation, "blend operation")?; + *slot = Some(a); + } + out.depth_format = optional_format(d.depth_format)?; + out.stencil_format = optional_format(d.stencil_format)?; + out.raster_sample_count = d.samples.max(1) as u32; + out.input_primitive_topology = raw::(d.topology, "topology")?; + out.rasterization_enabled = d.rasterization != 0; + if d.vertex_inputs != 0 { + let mut inputs = VertexDescriptor::new(); + for (i, a) in d.attributes.iter().enumerate() { + if a.format == 0 { + continue; + } + inputs.attributes.insert(i as u32, VertexAttributeDescriptor::new(raw::(a.format, "vertex format")?, + a.offset as u32, a.buffer as u32)); + if let Some(l) = d.layouts.get(a.buffer as usize) { + let mut layout = VertexBufferLayoutDescriptor::new(Some(l.stride as u32)); + layout.step_function = raw::(l.step_function, "step function")?; + layout.step_rate = l.step_rate as u32; + inputs.layouts.insert(a.buffer as u32, layout); + } + } + out.vertex_descriptor = Some(inputs); + } + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_pipeline_release(pipeline: *mut NmbRenderPipeline) { + release(pipeline) +} + +// ---- binary archives ---- + +pub struct NmbBinaryArchive(BinaryArchive); + +/// # Safety +/// `path` is null (an empty archive) or a C string naming the archive's file. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_binary_archive_new(device: *mut NmbDevice, path: *const c_char) -> *mut NmbBinaryArchive { + guard(std::ptr::null_mut(), || { + // SAFETY: the caller's contract. + let url = (!path.is_null()).then(|| PathBuf::from(unsafe { CStr::from_ptr(path) }.to_string_lossy().into_owned())); + let archive = get(device)?.0.new_binary_archive(&BinaryArchiveDescriptor::new(url)).map_err(e("binary archive"))?; + Ok(boxed(NmbBinaryArchive(archive))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_release(archive: *mut NmbBinaryArchive) { + release(archive) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_add_render(archive: *mut NmbBinaryArchive, descriptor: *const NmbRenderPipelineDescriptor) -> bool { + guard(false, || { + get(archive)?.0.add_render_pipeline_functions(&render_descriptor(descriptor)?).map_err(e("archive render pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_add_compute(archive: *mut NmbBinaryArchive, function: *mut NmbFunction) -> bool { + guard(false, || { + let descriptor = ComputePipelineDescriptor::new(StageSource::Function(get(function)?.0.function_descriptor())); + get(archive)?.0.add_compute_pipeline_functions(&descriptor).map_err(e("archive compute pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_add_compute_descriptor(archive: *mut NmbBinaryArchive, descriptor: *const NmbComputePipelineDescriptor) -> bool { + guard(false, || { + get(archive)?.0.add_compute_pipeline_functions(&compute_descriptor(descriptor)?).map_err(e("archive compute pipeline"))?; + Ok(true) + }) +} + +/// # Safety +/// `path` is a C string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_binary_archive_serialize(archive: *mut NmbBinaryArchive, path: *const c_char) -> bool { + guard(false, || { + if path.is_null() { + return Err("null path".into()); + } + // SAFETY: the caller's contract. + let path = PathBuf::from(unsafe { CStr::from_ptr(path) }.to_string_lossy().into_owned()); + get(archive)?.0.serialize_to_url(&path).map_err(e("archive serialize"))?; + Ok(true) + }) +} + +#[repr(C)] +pub struct NmbSamplerDescriptor { + pub min_filter: u64, + pub mag_filter: u64, + pub mip_filter: u64, + pub s_address: u64, + pub t_address: u64, + pub r_address: u64, + pub border_color: u64, + pub normalized: u64, + pub max_anisotropy: u64, + pub compare: u64, + pub lod_average: u64, + pub lod_min: f32, + pub lod_max: f32, +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_sampler_new(device: *mut NmbDevice, descriptor: *const NmbSamplerDescriptor) -> *mut NmbSampler { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut out = SamplerDescriptor::new(); + out.min_filter = raw::(d.min_filter, "min filter")?; + out.mag_filter = raw::(d.mag_filter, "mag filter")?; + out.mip_filter = raw::(d.mip_filter, "mip filter")?; + out.s_address_mode = raw::(d.s_address, "address mode")?; + out.t_address_mode = raw::(d.t_address, "address mode")?; + out.r_address_mode = raw::(d.r_address, "address mode")?; + out.border_color = raw::(d.border_color, "border color")?; + out.normalized_coordinates = d.normalized != 0; + out.max_anisotropy = d.max_anisotropy.max(1) as u32; + out.compare_function = raw::(d.compare, "compare function")?; + out.lod_average = d.lod_average != 0; + out.lod_min_clamp = d.lod_min; + out.lod_max_clamp = d.lod_max; + Ok(boxed(NmbSampler(get(device)?.0.new_sampler_state(&out).map_err(e("sampler"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_sampler_release(sampler: *mut NmbSampler) { + release(sampler) +} + +#[repr(C)] +pub struct NmbStencil { + pub compare: u64, + pub failure: u64, + pub depth_failure: u64, + pub pass: u64, + pub read_mask: u64, + pub write_mask: u64, +} + +#[repr(C)] +pub struct NmbDepthStencilDescriptor { + pub compare: u64, + pub write: u64, + pub front: NmbStencil, + pub back: NmbStencil, +} + +fn stencil(s: &NmbStencil) -> Result { + let mut out = StencilDescriptor::default(); + out.stencil_compare_function = raw::(s.compare, "stencil compare")?; + out.stencil_failure_operation = raw::(s.failure, "stencil operation")?; + out.depth_failure_operation = raw::(s.depth_failure, "stencil operation")?; + out.depth_stencil_pass_operation = raw::(s.pass, "stencil operation")?; + out.read_mask = s.read_mask as u32; + out.write_mask = s.write_mask as u32; + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_depth_stencil_new(device: *mut NmbDevice, descriptor: *const NmbDepthStencilDescriptor) -> *mut NmbDepthStencil { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut out = DepthStencilDescriptor::default(); + out.depth_compare_function = raw::(d.compare, "depth compare")?; + out.depth_write_enabled = d.write != 0; + out.front_face_stencil = stencil(&d.front)?; + out.back_face_stencil = stencil(&d.back)?; + Ok(boxed(NmbDepthStencil(get(device)?.0.new_depth_stencil_state(&out).map_err(e("depth/stencil state"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_depth_stencil_release(state: *mut NmbDepthStencil) { + release(state) +} + +// ---- queues and command buffers ---- + +pub struct NmbQueue(CommandQueue); +pub struct NmbCommandBuffer(CommandBuffer); + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_queue_new(device: *mut NmbDevice) -> *mut NmbQueue { + guard(std::ptr::null_mut(), || Ok(boxed(NmbQueue(get(device)?.0.new_command_queue().map_err(e("queue"))?)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_queue_release(queue: *mut NmbQueue) { + release(queue) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_new(queue: *mut NmbQueue) -> *mut NmbCommandBuffer { + guard(std::ptr::null_mut(), || Ok(boxed(NmbCommandBuffer(get(queue)?.0.command_buffer().map_err(e("command buffer"))?)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_release(command_buffer: *mut NmbCommandBuffer) { + release(command_buffer) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_uses_buffer(command_buffer: *const NmbCommandBuffer, buffer: *const NmbBuffer) -> i32 { + guard(-1, || { + let used = get(command_buffer)?.0.used_resources(); + Ok(i32::from(used.references_resource(get(buffer)?.0.as_resource()))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_uses_texture(command_buffer: *const NmbCommandBuffer, texture: *const NmbTexture) -> i32 { + guard(-1, || { + let used = get(command_buffer)?.0.used_resources(); + Ok(i32::from(used.references_resource(get(texture)?.0.as_resource()))) + }) +} + +struct Context(*mut c_void); +// SAFETY: the context is the caller's, handed back on the runtime's thread as the header states. +unsafe impl Send for Context {} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_on_completed(command_buffer: *mut NmbCommandBuffer, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + get(command_buffer)?.0.add_completed_handler(Box::new(move |_| { + let context = context; + handler(context.0) + })).map_err(e("completed handler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_on_scheduled(command_buffer: *mut NmbCommandBuffer, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + get(command_buffer)?.0.add_scheduled_handler(Box::new(move |_| { + let context = context; + handler(context.0) + })).map_err(e("scheduled handler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait_scheduled(command_buffer: *mut NmbCommandBuffer) -> bool { + guard(false, || { + get(command_buffer)?.0.wait_until_scheduled().map_err(e("wait scheduled"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffer) -> bool { + guard(false, || { + get(command_buffer)?.0.commit().map_err(e("commit"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait(command_buffer: *mut NmbCommandBuffer, timeout_ms: u64) -> i32 { + guard(-1, || { + let command_buffer = &get(command_buffer)?.0; + if timeout_ms == 0 { + command_buffer.wait_until_completed().map_err(e("wait"))?; + return Ok(0); + } + Ok(if command_buffer.wait_until_completed_for(Duration::from_millis(timeout_ms)).is_ok() { 0 } else { 1 }) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_status(command_buffer: *const NmbCommandBuffer) -> u64 { + guard(5, || { + Ok(get(command_buffer)?.0.status().raw()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_error(command_buffer: *const NmbCommandBuffer, out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(get(command_buffer)?.0.error_description().map_or(0, |text| copy_out(&text, out, size)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_gpu_time(command_buffer: *const NmbCommandBuffer, end: bool) -> f64 { + guard(0.0, || { + let command_buffer = &get(command_buffer)?.0; + let time = if end { command_buffer.gpu_end_time() } else { command_buffer.gpu_start_time() }; + time.map_err(e("GPU time")) + }) +} + +// ---- events and fences ---- + +pub enum NmbEvent { + Event(Event), + Shared(SharedEvent), +} +pub struct NmbFence(Fence); + +impl NmbEvent { + fn as_ref(&self) -> EventRef<'_> { + match self { + NmbEvent::Event(e) => EventRef::Event(e), + NmbEvent::Shared(s) => EventRef::Shared(s), + } + } +} + +fn shared<'a>(event: *const NmbEvent) -> Result<&'a SharedEvent, String> { + match get(event)? { + NmbEvent::Shared(s) => Ok(s), + NmbEvent::Event(_) => Err("not a shared event".into()), + } +} + +/// One listener thread for every notification of this process (Metal's listener only names the queue a block runs on; the frontend +/// forwards to it). +fn listener() -> Result<&'static SharedEventListener, String> { + static LISTENER: std::sync::OnceLock> = std::sync::OnceLock::new(); + LISTENER.get_or_init(|| SharedEventListener::new().map_err(e("listener"))).as_ref().map_err(|why| why.clone()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_event_new(device: *mut NmbDevice, is_shared: bool) -> *mut NmbEvent { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + Ok(boxed(if is_shared { + NmbEvent::Shared(device.new_shared_event().map_err(e("shared event"))?) + } else { + NmbEvent::Event(device.new_event().map_err(e("event"))?) + })) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_event_release(event: *mut NmbEvent) { + release(event) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_value(event: *const NmbEvent) -> u64 { + guard(0, || Ok(shared(event)?.signaled_value())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_set_value(event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + shared(event)?.set_signaled_value(value); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_wait(event: *mut NmbEvent, value: u64, timeout_ms: u64) -> bool { + guard(false, || Ok(shared(event)?.wait_until_signaled_value(value, Duration::from_millis(timeout_ms)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_notify(event: *mut NmbEvent, value: u64, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + shared(event)?.notify_listener(listener()?, value, Box::new(move |_, reached| { + let context = context; + handler(context.0, reached) + })); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_signal_event(command_buffer: *mut NmbCommandBuffer, event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + get(command_buffer)?.0.encode_signal_event(get(event)?.as_ref(), value).map_err(e("signal event"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait_event(command_buffer: *mut NmbCommandBuffer, event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + get(command_buffer)?.0.encode_wait_for_event(get(event)?.as_ref(), value).map_err(e("wait for event"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_fence_new(device: *mut NmbDevice) -> *mut NmbFence { + guard(std::ptr::null_mut(), || Ok(boxed(NmbFence(get(device)?.0.new_fence())))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_fence_release(fence: *mut NmbFence) { + release(fence) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_update_fence(encoder: *mut NmbEncoder, fence: *mut NmbFence, stages: u64) -> bool { + guard(false, || { + let fence = &get(fence)?.0; + match get_mut(encoder)? { + NmbEncoder::Compute(c) => c.update_fence(fence), + NmbEncoder::Blit(b) => b.update_fence(fence), + NmbEncoder::Render(r) => r.update_fence(fence, raw::(stages, "render stages")?), + } + .map_err(e("update fence"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_wait_fence(encoder: *mut NmbEncoder, fence: *mut NmbFence, stages: u64) -> bool { + guard(false, || { + let fence = &get(fence)?.0; + match get_mut(encoder)? { + NmbEncoder::Compute(c) => c.wait_for_fence(fence), + NmbEncoder::Blit(b) => b.wait_for_fence(fence), + NmbEncoder::Render(r) => r.wait_for_fence(fence, raw::(stages, "render stages")?), + } + .map_err(e("wait for fence"))?; + Ok(true) + }) +} + +// ---- encoders ---- + +pub enum NmbEncoder { + Compute(ComputeCommandEncoder), + Render(RenderCommandEncoder), + Blit(BlitCommandEncoder), +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_encoder_new(command_buffer: *mut NmbCommandBuffer) -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || { + Ok(boxed(NmbEncoder::Compute(get(command_buffer)?.0.compute_command_encoder().map_err(e("compute encoder"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_encoder_new(command_buffer: *mut NmbCommandBuffer) -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || Ok(boxed(NmbEncoder::Blit(get(command_buffer)?.0.blit_command_encoder().map_err(e("blit encoder"))?)))) +} + +#[repr(C)] +pub struct NmbPassAttachment { + pub texture: *mut NmbTexture, + pub level: u64, + pub slice: u64, + pub depth_plane: u64, + pub load: u64, + pub store: u64, + pub clear: [f64; 4], +} + +#[repr(C)] +pub struct NmbRenderPassDescriptor { + pub colors: [NmbPassAttachment; 8], + pub depth: NmbPassAttachment, + pub stencil: NmbPassAttachment, + pub width: u64, + pub height: u64, + pub array_length: u64, + pub samples: u64, + pub visibility: *const NmbBuffer, +} + +fn texture_of(a: &NmbPassAttachment) -> Result, String> { + if a.texture.is_null() { Ok(None) } else { Ok(Some(get(a.texture)?.0.clone())) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, descriptor: *const NmbRenderPassDescriptor) + -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut pass = RenderPassDescriptor::new(); + for (slot, a) in pass.color_attachments.iter_mut().zip(d.colors.iter()) { + slot.texture = texture_of(a)?; + slot.level = a.level; + slot.slice = a.slice; + slot.depth_plane = a.depth_plane; + slot.load_action = LoadAction::from_raw(a.load).map_err(e("load action"))?; + slot.store_action = StoreAction::from_raw(a.store).map_err(e("store action"))?; + slot.clear_color = ClearColor::new(a.clear[0], a.clear[1], a.clear[2], a.clear[3]); + } + let depth = &mut pass.depth_attachment; + depth.texture = texture_of(&d.depth)?; + depth.level = d.depth.level; + depth.slice = d.depth.slice; + depth.depth_plane = d.depth.depth_plane; + depth.load_action = LoadAction::from_raw(d.depth.load).map_err(e("load action"))?; + depth.store_action = StoreAction::from_raw(d.depth.store).map_err(e("store action"))?; + depth.clear_depth = d.depth.clear[0]; + let stencil = &mut pass.stencil_attachment; + stencil.texture = texture_of(&d.stencil)?; + stencil.level = d.stencil.level; + stencil.slice = d.stencil.slice; + stencil.depth_plane = d.stencil.depth_plane; + stencil.load_action = LoadAction::from_raw(d.stencil.load).map_err(e("load action"))?; + stencil.store_action = StoreAction::from_raw(d.stencil.store).map_err(e("store action"))?; + stencil.clear_stencil = d.stencil.clear[0] as u32; + pass.render_target_width = d.width; + pass.render_target_height = d.height; + pass.render_target_array_length = d.array_length; + pass.default_raster_sample_count = d.samples; + if !d.visibility.is_null() { + pass.visibility_result_buffer = Some(get(d.visibility)?.0.clone()); + } + let encoder = get(command_buffer)?.0.render_command_encoder(&pass).map_err(e("render encoder"))?; + Ok(boxed(NmbEncoder::Render(encoder))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_end(encoder: *mut NmbEncoder) -> bool { + guard(false, || { + if encoder.is_null() { + return Err("null encoder".into()); + } + // SAFETY: from `boxed`; ending releases it. + let encoder = unsafe { Box::from_raw(encoder) }; + match *encoder { + NmbEncoder::Compute(c) => c.end_encoding(), + NmbEncoder::Render(r) => r.end_encoding(), + NmbEncoder::Blit(b) => b.end_encoding(), + } + .map_err(e("end encoding"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_discard(encoder: *mut NmbEncoder) { + release(encoder) +} + +fn compute<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut ComputeCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Compute(c) => Ok(c), + _ => Err("not a compute encoder".into()), + } +} + +fn render<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut RenderCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Render(r) => Ok(r), + _ => Err("not a render encoder".into()), + } +} + +fn blit<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut BlitCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Blit(b) => Ok(b), + _ => Err("not a blit encoder".into()), + } +} + +fn optional<'a, T>(handle: *const T) -> Result, String> { + if handle.is_null() { Ok(None) } else { get(handle).map(Some) } +} + +const COMPUTE: u32 = 0; +const VERTEX: u32 = 1; +const FRAGMENT: u32 = 2; + +fn bad_stage(stage: u32) -> String { + format!("stage {stage} does not fit this encoder") +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_buffer(encoder: *mut NmbEncoder, stage: u32, buffer: *mut NmbBuffer, offset: u64, index: u32) -> bool { + guard(false, || { + let buffer = optional(buffer)?.map(|b| &b.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer(buffer, offset, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer(buffer, offset, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_buffer(buffer, offset, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set buffer"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_buffer_offset(encoder: *mut NmbEncoder, stage: u32, offset: u64, index: u32) -> bool { + guard(false, || { + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer_offset(offset, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer_offset(offset, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_buffer_offset(offset, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set buffer offset"))?; + Ok(true) + }) +} + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_encoder_set_bytes(encoder: *mut NmbEncoder, stage: u32, bytes: *const c_void, length: usize, index: u32) + -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length) }; + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_bytes(bytes, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_bytes(bytes, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_bytes(bytes, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set bytes"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_texture(encoder: *mut NmbEncoder, stage: u32, texture: *mut NmbTexture, index: u32) -> bool { + guard(false, || { + let texture = optional(texture)?.map(|t| &t.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_texture(texture, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_texture(texture, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_texture(texture, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set texture"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_sampler(encoder: *mut NmbEncoder, stage: u32, sampler: *mut NmbSampler, index: u32) -> bool { + guard(false, || { + let sampler = optional(sampler)?.map(|s| &s.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_sampler_state(sampler, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_sampler_state(sampler, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_sampler_state(sampler, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set sampler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_set_pipeline(encoder: *mut NmbEncoder, pipeline: *mut NmbComputePipeline) -> bool { + guard(false, || { + compute(encoder)?.set_compute_pipeline_state(&get(pipeline)?.0).map_err(e("compute pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_set_threadgroup_memory(encoder: *mut NmbEncoder, length: u32, index: u32) -> bool { + guard(false, || { + compute(encoder)?.set_threadgroup_memory_length(length, index).map_err(e("threadgroup memory"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_dispatch(encoder: *mut NmbEncoder, size: *const u64, per_group: *const u64, threads: bool) -> bool { + guard(false, || { + let (s, g) = (triple(size)?, triple(per_group)?); + let (s, g) = (Size::new(s[0], s[1], s[2]), Size::new(g[0], g[1], g[2])); + let c = compute(encoder)?; + if threads { c.dispatch_threads(s, g) } else { c.dispatch_threadgroups(s, g) }.map_err(e("dispatch"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_pipeline(encoder: *mut NmbEncoder, pipeline: *mut NmbRenderPipeline) -> bool { + guard(false, || { + render(encoder)?.set_render_pipeline_state(&get(pipeline)?.0).map_err(e("render pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_depth_stencil(encoder: *mut NmbEncoder, state: *mut NmbDepthStencil) -> bool { + guard(false, || { + render(encoder)?.set_depth_stencil_state(optional(state)?.map(|s| &s.0)).map_err(e("depth/stencil state"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_viewport(encoder: *mut NmbEncoder, viewport: *const f64) -> bool { + guard(false, || { + if viewport.is_null() { + return Err("null viewport".into()); + } + // SAFETY: the header's six values. + let v = unsafe { std::slice::from_raw_parts(viewport, 6) }; + render(encoder)?.set_viewport(Viewport::new(v[0], v[1], v[2], v[3], v[4], v[5])).map_err(e("viewport"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_scissor(encoder: *mut NmbEncoder, rect: *const u64) -> bool { + guard(false, || { + if rect.is_null() { + return Err("null scissor".into()); + } + // SAFETY: the header's four values. + let r = unsafe { std::slice::from_raw_parts(rect, 4) }; + render(encoder)?.set_scissor_rect(ScissorRect::new(r[0], r[1], r[2], r[3])).map_err(e("scissor"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_raster(encoder: *mut NmbEncoder, cull_mode: u64, winding: u64, fill_mode: u64) -> bool { + guard(false, || { + let r = render(encoder)?; + r.set_cull_mode(raw::(cull_mode, "cull mode")?); + r.set_front_facing_winding(raw::(winding, "winding")?); + r.set_triangle_fill_mode(raw::(fill_mode, "fill mode")?).map_err(e("fill mode"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_stencil_reference(encoder: *mut NmbEncoder, front: u32, back: u32) -> bool { + guard(false, || { + render(encoder)?.set_stencil_reference_values(front, back); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_depth_bias(encoder: *mut NmbEncoder, bias: f32, slope_scale: f32, clamp: f32) -> bool { + guard(false, || { + render(encoder)?.set_depth_bias(bias, slope_scale, clamp).map_err(e("depth bias"))?; + Ok(true) + }) +} + +/// SPI `setLineWidth:`: the runtime accepts Metal's default width of 1 and refuses any other (Apple's wide-line rule is unmeasured). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_line_width(encoder: *mut NmbEncoder, width: f32) -> bool { + guard(false, || { + render(encoder)?.set_line_width(width).map_err(e("line width"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_visibility_mode(encoder: *mut NmbEncoder, mode: u64, offset: u64) -> bool { + guard(false, || { + let mode = VisibilityResultMode::from_raw(mode).map_err(e("visibility result mode"))?; + render(encoder)?.set_visibility_result_mode(mode, offset).map_err(e("visibility result mode"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_blend_color(encoder: *mut NmbEncoder, color: *const f32) -> bool { + guard(false, || { + if color.is_null() { + return Err("null color".into()); + } + // SAFETY: the header's four values. + let c = unsafe { std::slice::from_raw_parts(color, 4) }; + render(encoder)?.set_blend_color(c[0], c[1], c[2], c[3]); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_draw(encoder: *mut NmbEncoder, primitive: u64, start: u32, count: u32, instances: u32, base_instance: u32) + -> bool { + guard(false, || { + render(encoder)?.draw_primitives(raw::(primitive, "primitive")?, start, count, instances, base_instance) + .map_err(e("draw"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_render_draw_indexed(encoder: *mut NmbEncoder, primitive: u64, count: u32, index_type: u64, indices: *mut NmbBuffer, + offset: u64, instances: u32, base_vertex: i32, base_instance: u32) -> bool { + guard(false, || { + render(encoder)?.draw_indexed_primitives(raw::(primitive, "primitive")?, count, + raw::(index_type, "index type")?, &get(indices)?.0, offset, instances, + base_vertex, base_instance).map_err(e("indexed draw"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_copy_buffer(encoder: *mut NmbEncoder, source: *mut NmbBuffer, source_offset: u64, destination: *mut NmbBuffer, + destination_offset: u64, size: u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_buffer_to_buffer(&get(source)?.0, source_offset, &get(destination)?.0, destination_offset, size) + .map_err(e("buffer copy"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_fill_buffer(encoder: *mut NmbEncoder, buffer: *mut NmbBuffer, start: u64, end: u64, value: u8) -> bool { + guard(false, || { + blit(encoder)?.fill_buffer(&get(buffer)?.0, start..end, value).map_err(e("fill"))?; + Ok(true) + }) +} + +fn origin(v: *const u64) -> Result { + let v = triple(v)?; + Ok(Origin::new(v[0], v[1], v[2])) +} + +fn size(v: *const u64) -> Result { + let v = triple(v)?; + Ok(Size::new(v[0], v[1], v[2])) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_copy_texture(encoder: *mut NmbEncoder, source: *mut NmbTexture, source_slice: u64, source_level: u64, + source_origin: *const u64, extent: *const u64, destination: *mut NmbTexture, + destination_slice: u64, destination_level: u64, destination_origin: *const u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_texture_to_texture(&get(source)?.0, source_slice, source_level, origin(source_origin)?, size(extent)?, + &get(destination)?.0, destination_slice, destination_level, + origin(destination_origin)?).map_err(e("texture copy"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_buffer_to_texture(encoder: *mut NmbEncoder, source: *mut NmbBuffer, offset: u64, bytes_per_row: u64, + bytes_per_image: u64, extent: *const u64, destination: *mut NmbTexture, slice: u64, + level: u64, destination_origin: *const u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_buffer_to_texture(&get(source)?.0, offset, bytes_per_row, bytes_per_image, size(extent)?, + &get(destination)?.0, slice, level, origin(destination_origin)?, BlitOption::None) + .map_err(e("buffer to texture"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_texture_to_buffer(encoder: *mut NmbEncoder, source: *mut NmbTexture, slice: u64, level: u64, + source_origin: *const u64, extent: *const u64, destination: *mut NmbBuffer, offset: u64, + bytes_per_row: u64, bytes_per_image: u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_texture_to_buffer(&get(source)?.0, slice, level, origin(source_origin)?, size(extent)?, + &get(destination)?.0, offset, bytes_per_row, bytes_per_image, BlitOption::None) + .map_err(e("texture to buffer"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_generate_mipmaps(encoder: *mut NmbEncoder, texture: *mut NmbTexture) -> bool { + guard(false, || { + blit(encoder)?.generate_mipmaps(&get(texture)?.0).map_err(e("mipmaps"))?; + Ok(true) + }) +} diff --git a/bridge/archive_host.mm b/bridge/archive_host.mm new file mode 100644 index 0000000..f77e973 --- /dev/null +++ b/bridge/archive_host.mm @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: MIT +// Two processes, distinct initially empty caches: archive selection cannot be satisfied by +// the producer's live pipeline memo. A strict empty/changed-descriptor miss must refuse. +#include "FrontendChecks.h" +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" + +int main(int argc, char **argv) { + if (argc != 5) return 2; + @autoreleasepool { + const bool produce = !strcmp(argv[4], "produce"); + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + if (!library) return 2; + auto descriptor = [MTLRenderPipelineDescriptor new]; + descriptor.vertexFunction = [library newFunctionWithName:@"colored_vertex"]; + descriptor.fragmentFunction = [library newFunctionWithName:@"colored_fragment"]; + descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatRGBA8Unorm; + auto archiveDescriptor = [MTLBinaryArchiveDescriptor new]; + NSURL *url = [NSURL fileURLWithPath:@(argv[3])]; + if (!produce) archiveDescriptor.url = url; + NSError *error = nil; + id archive = [device newBinaryArchiveWithDescriptor:archiveDescriptor error:&error]; + if (!archive) {std::printf("FAIL open archive: %s\n", error.localizedDescription.UTF8String);return 1;} + descriptor.binaryArchives = @[archive]; + if (produce) { + id state = [device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if (state || !error) {std::puts("FAIL empty archive miss was silently compiled");return 1;} + if (![archive addRenderPipelineFunctionsWithDescriptor:descriptor error:&error] || ![archive serializeToURL:url error:&error]) { + std::printf("FAIL archive add/save: %s\n", error.localizedDescription.UTF8String);return 1; + } + } + error = nil; + id state = [device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if (!state || error) {std::printf("FAIL strict archive hit: %s\n", error.localizedDescription.UTF8String);return 1;} + // A format not added to the archive must still miss, even after a valid pipeline is live. + descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + if ([device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error) { + std::puts("FAIL changed-descriptor archive miss");return 1; + } + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id target = [device newTextureWithDescriptor:td]; + id cb = [[device newCommandQueue] commandBuffer]; + if (!target || !cb) return 2; + frontendchecks::Report report = {}; + frontendchecks::gReport = &report; + frontendchecks::drawQuad(cb, state, target, nil, frontendchecks::kQuarter, 6); + if (!frontendchecks::finish(cb, "archived draw")) return 1; + uint8_t pixels[16 * 16 * 4] = {}; + [target getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + if (frontendchecks::wrongTexels(pixels, 64, 16)) {std::puts("FAIL archived render pixels");return 1;} + std::printf("PASS: %s render archive, strict hit/miss and GPU pixel check (host only)\n", produce ? "produced" : "reloaded"); + } +} diff --git a/bridge/bridge_smoke.cpp b/bridge/bridge_smoke.cpp new file mode 100644 index 0000000..b89220c --- /dev/null +++ b/bridge/bridge_smoke.cpp @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MIT +// Host run of the bridge self-test over the host Turnip and the host kernel stack (dyld-interposed IOKit shim). The software CP executes +// nothing, so result counts are not expected to be zero here; the run proves every step builds, submits and retires. +#include "NativeMetalBridge.h" +#include +#include +#include +static void mark(uint32_t value) {std::printf("mark %x\n",value);std::fflush(stdout);} +int main(int argc,char **argv) { + if(argc<3)return 2; + std::vector qc; + if(argc>3) { + FILE *f=std::fopen(argv[3],"rb");if(!f)return 3; + std::fseek(f,0,SEEK_END);qc.resize(size_t(std::ftell(f)));std::fseek(f,0,SEEK_SET); + if(std::fread(qc.data(),1,qc.size(),f)!=qc.size())return 4;std::fclose(f); + } + NmbSelftest r{}; + const uint32_t code=nmb_selftest(argv[1],argv[2],qc.data(),qc.size(),mark,&r); + std::printf("code %x stage %x compute_bad %u render_bad %u import_bad %u qc %u kernels %u/%u pairs %u/%u msg '%s'\n",code,r.stage, + r.compute_bad,r.render_bad,r.import_bad,r.qc_functions,r.qc_kernels_ok,r.qc_kernels_failed,r.qc_pairs_ok,r.qc_pairs_failed,r.message); + return code?1:0; +} diff --git a/bridge/cache_export_host.cpp b/bridge/cache_export_host.cpp new file mode 100644 index 0000000..30b4f9f --- /dev/null +++ b/bridge/cache_export_host.cpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: MIT +// Host check of the pipeline-cache export/import format (CacheExport.h): a directory tree becomes the export blob, the blob is written where +// the image would carry it, and importPack rebuilds the tree in a second directory; every file must come back byte for byte, an existing +// file of equal size is kept, and damaged packs import nothing. argv: source dir, destination dir, blob path. No kernel call is made. +#define CACHEEXPORT_FORMAT_ONLY +#include +#include "../CacheExport.h" +#include +#include + +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) + +static std::vector slurp(const std::string &path) { + std::vector out;FILE *f=fopen(path.c_str(),"rb");if(!f)return out; + uint8_t b[4096];for(size_t n;(n=fread(b,1,sizeof b,f))>0;)out.insert(out.end(),b,b+n);fclose(f);return out; +} +static void spill(const std::string &path,const std::vector &data) {FILE *f=fopen(path.c_str(),"wb");CHECK(f);CHECK(fwrite(data.data(),1,data.size(),f)==data.size());fclose(f);} + +int main(int argc,char **argv) { + CHECK(argc==4); + const std::string source=argv[1],dest=argv[2],blobPath=argv[3]; + std::vector files;cacheexport::scan(source,"",files,0); + CHECK(files.size()>=3); + const uint64_t sig=cacheexport::signature(files); + std::vector again;cacheexport::scan(source,"",again,0); + CHECK(cacheexport::signature(again)==sig); + std::vector blob;CHECK(cacheexport::build(source,files,blob)); + /* the export's stream: as long as announced, and written out as the uncompressed export blob for the host's readers */ + std::vector raw={'S','H','E','X','R','0','0','1',0,0,0,0,0,0,0,0}; + CHECK(cacheexport::stream(source,files,[&](const uint8_t *p,size_t n){raw.insert(raw.end(),p,p+n);return true;})); + CHECK(raw.size()-16==cacheexport::streamBytes(source,files)); + {const uint32_t n=(uint32_t)(raw.size()-16);std::memcpy(&raw[8],&n,4);} + spill(blobPath+".raw",raw); + mkdir(dest.c_str(),0700);mkdir((dest+"-damaged").c_str(),0700); /* the plugin makes its directory before importing */ + spill(blobPath,blob); + CHECK(cacheexport::baked().empty()); + CHECK(cacheexport::importPack(blobPath.c_str(),dest)==(int)files.size()); + CHECK(cacheexport::baked().size()==files.size()); /* every entry of a pack read to its end is known as baked */ + for(const auto &f:files)CHECK(slurp(source+"/"+f.path)==slurp(dest+"/"+f.path)); + CHECK(cacheexport::importPack(blobPath.c_str(),dest)==0); /* all present with their sizes: nothing rewritten */ + /* damaged packs: truncated, bad magic, flipped compressed byte */ + std::vector bad=blob;bad.resize(bad.size()/2);spill(blobPath+".t",bad); + CHECK(cacheexport::importPack((blobPath+".t").c_str(),dest+"-damaged")==-1); + bad=blob;bad[0]='X';spill(blobPath+".m",bad);CHECK(cacheexport::importPack((blobPath+".m").c_str(),dest+"-damaged")==-1); + bad=blob;bad[bad.size()-5]^=0x55;spill(blobPath+".f",bad);CHECK(cacheexport::importPack((blobPath+".f").c_str(),dest+"-damaged")==-1); + CHECK(cacheexport::importPack((blobPath+".missing").c_str(),dest)==0); + std::vector left;cacheexport::scan(dest+"-damaged","",left,0);CHECK(left.empty()); + CHECK(cacheexport::baked().size()==files.size()); /* damaged packs add nothing */ + /* the export leaves baked files out: only a new file, or one whose size changed, goes out */ + std::vector now;cacheexport::scan(dest,"",now,0);cacheexport::dropBaked(now);CHECK(now.empty()); + spill(dest+"/new-entry",{1,2,3}); + { std::vector grown=slurp(dest+"/"+files[0].path);grown.push_back(7);spill(dest+"/"+files[0].path,grown); } + now.clear();cacheexport::scan(dest,"",now,0);cacheexport::dropBaked(now); + CHECK(now.size()==2); + std::printf("PASS: cache export %zu files, %zu blob bytes; import exact, idempotent, 3 damaged packs refused, baked files left out of the export\n",files.size(),blob.size()); + return 0; +} diff --git a/bridge/carenderer_host.mm b/bridge/carenderer_host.mm new file mode 100644 index 0000000..c7d6348 --- /dev/null +++ b/bridge/carenderer_host.mm @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: MIT +// Host instrument: the Mac's own QuartzCore renders a layer tree through CARenderer into a texture of the native frontend's device (bridge +// over a host Vulkan driver, MoltenVK on the Mac). CoreAnimation's Metal renderer then drives the frontend as the target's render server +// will, and the pixels it produced are checked. With NMB_DEVICE_BASE_NSOBJECT=1 every device selector the renderer sends that the frontend +// lacks is reported as missing. The Mac's QuartzCore is not the target's: this names gaps and checks results, it qualifies nothing on iPadOS. +#import +#import +#import +#import +#import +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include +#include +#import +extern "C" size_t native_bridge_metal_telemetry(char *, size_t); + +static std::set missing; +static void onMissing(const char *cls, const char *sel) { + if (missing.insert(std::string(cls) + " " + sel).second) std::fprintf(stderr, "missing selector: %s %s\n", cls, sel); +} + +// NMB_DISCOVER=1 (instrument only): an unimplemented selector is logged and answers zero instead of raising, so one run lists every gap on +// the renderer's path. Results after a zero answer mean nothing; the frontend itself never answers this way. +static const char *protocolTypes(SEL selector) { + Protocol *protocols[] = {@protocol(MTLDevice), @protocol(MTLTexture), @protocol(MTLBuffer), @protocol(MTLResource), @protocol(MTLCommandQueue), + @protocol(MTLCommandBuffer), @protocol(MTLRenderCommandEncoder), @protocol(MTLBlitCommandEncoder), + @protocol(MTLComputeCommandEncoder), @protocol(MTLCommandEncoder), @protocol(MTLLibrary), @protocol(MTLFunction), + @protocol(MTLRenderPipelineState), @protocol(MTLComputePipelineState), @protocol(MTLSamplerState), + @protocol(MTLDepthStencilState), @protocol(MTLHeap), @protocol(MTLFence), @protocol(MTLEvent), @protocol(MTLSharedEvent)}; + for (Protocol *p : protocols) + for (BOOL required : {YES, NO}) { + objc_method_description m = protocol_getMethodDescription(p, selector, required, YES); + if (m.types) return m.types; + } + return nullptr; +} +static NSMethodSignature *discoverSignature(id self, SEL _cmd, SEL selector) { + const char *types = protocolTypes(selector); + if (!types) {onMissing(class_getName(object_getClass(self)), (std::string(sel_getName(selector)) + " (no protocol signature)").c_str()); types = "@@:";} + return [NSMethodSignature signatureWithObjCTypes:types]; +} +static void discoverForward(id self, SEL _cmd, NSInvocation *invocation) { + onMissing(class_getName(object_getClass(self)), sel_getName(invocation.selector)); + const NSUInteger length = invocation.methodSignature.methodReturnLength; + if (length) {std::vector zero(length); [invocation setReturnValue:zero.data()];} +} +static void discover(Class cls) { + class_addMethod(cls, @selector(methodSignatureForSelector:), (IMP)discoverSignature, "@@::"); + class_addMethod(cls, @selector(forwardInvocation:), (IMP)discoverForward, "v@:@"); +} + +static CGColorRef rgba(double r, double g, double b, double a) { + CGColorSpaceRef space = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); + const CGFloat c[4] = {r, g, b, a}; + CGColorRef color = CGColorCreate(space, c); + CGColorSpaceRelease(space); + return color; +} + +// argv: loader, shader cache. Prints PASS or the first wrong pixel. +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + // NMB_SYSTEM_DEVICE=1: the Mac's own Metal device, the instrument's baseline. + id device = getenv("NMB_SYSTEM_DEVICE") ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, onMissing); + if (!device) {std::printf("FAIL device\n"); return 1;} + if (getenv("NMB_DISCOVER")) {discover(object_getClass(device)); discover(objc_getClass("NBObject"));} + const NSUInteger side = 64; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:side height:side mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + d.storageMode = MTLStorageModeShared; + id target = [device newTextureWithDescriptor:d]; + id queue = [device newCommandQueue]; + if (!target || !queue) {std::printf("FAIL target or queue\n"); return 1;} + + // Blue background; a red square in the top-left quarter; a half-transparent green square in the bottom-right quarter. + CALayer *root = [CALayer layer]; + root.frame = CGRectMake(0, 0, side, side); + root.backgroundColor = rgba(0, 0, 1, 1); + CALayer *red = [CALayer layer]; + red.frame = CGRectMake(0, 0, side / 2, side / 2); + red.backgroundColor = rgba(1, 0, 0, 1); + CALayer *green = [CALayer layer]; + green.frame = CGRectMake(side / 2, side / 2, side / 2, side / 2); + green.backgroundColor = rgba(0, 1, 0, 1); + green.opacity = 0.5; + [root addSublayer:red]; + [root addSublayer:green]; + + CARenderer *renderer = [CARenderer rendererWithMTLTexture:target options:@{kCARendererMetalCommandQueue: queue}]; + if (!renderer) {std::printf("FAIL CARenderer refused the frontend's texture\n"); return 1;} + renderer.layer = root; + renderer.bounds = CGRectMake(0, 0, side, side); + // The layers' implicit transaction would commit at the run loop's end; the renderer sees only committed trees. + // NMB_BLUR_TIMING=1: also render a blurred, shadowed layer for several frames and print the frontend's creation telemetry (pipeline + // counts and times, VkPipeline makings), so the cost of the pipelines CoreAnimation's own renderer asks for can be measured. + if (getenv("NMB_BLUR_TIMING")) { + CALayer *blurred = [CALayer layer]; + blurred.frame = CGRectMake(4, 4, side - 8, side - 8); + blurred.backgroundColor = rgba(1, 1, 0, 0.7); + blurred.cornerRadius = 9; + blurred.shadowOpacity = 0.8; + blurred.shadowRadius = 5; + CIFilter *blur = [CIFilter filterWithName:@"CIGaussianBlur"]; + [blur setValue:@6 forKey:kCIInputRadiusKey]; + blurred.filters = @[blur]; + [root addSublayer:blurred]; + [CATransaction flush]; + for (int frame = 0; frame < 3; ++frame) { + const uint64_t t0 = clock_gettime_nsec_np(CLOCK_UPTIME_RAW); + [renderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [renderer addUpdateRect:renderer.bounds]; + [renderer render]; + [renderer endFrame]; + id fence = [queue commandBuffer]; + [fence commit]; + [fence waitUntilCompleted]; + std::printf("BLUR frame %d %.1f ms\n", frame, (clock_gettime_nsec_np(CLOCK_UPTIME_RAW) - t0) / 1e6); + } + static char table[16384]; + native_bridge_metal_telemetry(table, sizeof table); + for (char *line = std::strtok(table, "\n"); line; line = std::strtok(nullptr, "\n")) + if (line[0] == 'T' || line[0] == 'V' || line[0] == 'S' || line[0] == 'W') std::printf("TELEMETRY %s\n", line); + [blurred removeFromSuperlayer]; + [CATransaction flush]; + } + [renderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [renderer addUpdateRect:renderer.bounds]; + [renderer render]; + [renderer endFrame]; + // CARenderer commits its own command buffers; one more on the same queue completes after them. + id last = [queue commandBuffer]; + [last commit]; + [last waitUntilCompleted]; + + std::vector pixels(side * side * 4); + [target getBytes:pixels.data() bytesPerRow:side * 4 fromRegion:MTLRegionMake2D(0, 0, side, side) mipmapLevel:0]; + // BGRA. Row 0 of the texture is the layer tree's y 0 (the Mac's Metal device renders it so, NMB_SYSTEM_DEVICE=1): red top-left, + // green bottom-right. + auto at = [&](NSUInteger x, NSUInteger y) {return &pixels[(y * side + x) * 4];}; + struct Probe {NSUInteger x, y; int b, g, r;} probes[] = { + {8, 8, 0, 0, 255}, // red quarter + {side - 8, side - 8, 128, 128, 0}, // green at 0.5 over blue: (0, 0.5, 0.5) + {side - 8, 8, 255, 0, 0}, // background + {8, side - 8, 255, 0, 0}, // background + }; + bool ok = true; + for (auto &p : probes) { + const uint8_t *px = at(p.x, p.y); + if (std::abs(px[0] - p.b) > 3 || std::abs(px[1] - p.g) > 3 || std::abs(px[2] - p.r) > 3 || px[3] != 255) { + std::printf("FAIL pixel (%lu,%lu) = b%u g%u r%u a%u, want b%d g%d r%d\n", (unsigned long)p.x, (unsigned long)p.y, px[0], px[1], px[2], px[3], + p.b, p.g, p.r); + ok = false; + } + } + std::printf("missing selectors: %zu\n", missing.size()); + if (!ok) return 1; + std::printf("PASS: the Mac's CARenderer rendered a layer tree on %s\n", device.name.UTF8String); + return 0; + } +} diff --git a/bridge/compute_archive_host.mm b/bridge/compute_archive_host.mm new file mode 100644 index 0000000..3a0ce49 --- /dev/null +++ b/bridge/compute_archive_host.mm @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT +// Separate processes/cache directories distinguish archive consumption from live memo hits. +#import +#import +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" +#include +#include + +#define REQUIRE(condition) do {if (!(condition)) {std::printf("FAIL line %d: %s; %s\n", __LINE__, #condition, error.localizedDescription.UTF8String ?: "");return 1;}} while (0) + +int main(int argc, char **argv) { + if (argc != 5) return 2; + @autoreleasepool { + NSError *error = nil; + const bool produce = !strcmp(argv[4], "produce"); + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + REQUIRE(device); + size_t bytes = 0; + const void *air = nmb_function_constant_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + REQUIRE(library); + auto constants = [MTLFunctionConstantValues new]; + int mode = 2; + [constants setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + id function = [library newFunctionWithName:@"fcfold" constantValues:constants error:&error]; + REQUIRE(function && !error); + REQUIRE([device respondsToSelector:@selector(newComputePipelineStateWithDescriptor:options:reflection:error:)]); + auto descriptor = [MTLComputePipelineDescriptor new]; + descriptor.computeFunction = function; + descriptor.maxTotalThreadsPerThreadgroup = 32; + descriptor.buffers[1].mutability = MTLMutabilityMutable; + descriptor.label = @"archived compute"; + auto archiveDescriptor = [MTLBinaryArchiveDescriptor new]; + NSURL *url = [NSURL fileURLWithPath:@(argv[3])]; + if (!produce) archiveDescriptor.url = url; + id archive = [device newBinaryArchiveWithDescriptor:archiveDescriptor error:&error]; + REQUIRE(archive && !error); + descriptor.binaryArchives = @[archive]; + if (produce) { + id miss = [device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if (miss || !error) { + std::printf("FAIL strict empty archive: state=%s error=%s\n", miss ? "created" : "nil", error.localizedDescription.UTF8String ?: "nil"); + return 1; + } + REQUIRE([archive addComputePipelineFunctionsWithDescriptor:descriptor error:&error] && !error); + REQUIRE([archive serializeToURL:url error:&error] && !error); + } + id state = [device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + REQUIRE(state && !error && state.maxTotalThreadsPerThreadgroup == 32); + REQUIRE([state.label isEqualToString:descriptor.label]); + // Both descriptor fields and function constants must retain their archive identity. + descriptor.maxTotalThreadsPerThreadgroup = 16; + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error); + descriptor.maxTotalThreadsPerThreadgroup = 32; + descriptor.buffers[1].mutability = MTLMutabilityDefault; + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error); + descriptor.buffers[1].mutability = MTLMutabilityMutable; + mode = 3; + [constants setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + descriptor.computeFunction = [library newFunctionWithName:@"fcfold" constantValues:constants error:&error]; + REQUIRE(descriptor.computeFunction && !error); + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error); + descriptor.computeFunction = function; + descriptor.stageInputDescriptor = [MTLStageInputOutputDescriptor stageInputOutputDescriptor]; + descriptor.stageInputDescriptor.attributes[0].format = MTLAttributeFormatFloat; + // Unsupported descriptor state must be refused by both population and creation. + REQUIRE(![archive addComputePipelineFunctionsWithDescriptor:descriptor error:&error] && error); + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionNone reflection:nil error:&error] && error); + descriptor.stageInputDescriptor = nil; + const float inputValue = 2; + id input = [device newBufferWithBytes:&inputValue length:4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:128 options:MTLResourceStorageModeShared]; + id command = [[device newCommandQueue] commandBuffer]; + id encoder = [command computeCommandEncoder]; + REQUIRE(input && output && encoder); + [encoder setComputePipelineState:state]; + [encoder setBuffer:input offset:0 atIndex:0]; + [encoder setBuffer:output offset:0 atIndex:1]; + [encoder dispatchThreads:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [encoder endEncoding]; + [command commit]; + [command waitUntilCompleted]; + REQUIRE(command.status == MTLCommandBufferStatusCompleted); + REQUIRE(((const float *)output.contents)[24] == 4000); + // Check the no-list behavior only after the cold archive consumer and GPU verdict. + REQUIRE([device newComputePipelineStateWithFunction:function options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && !error); + std::printf("PASS: %s compute archive, strict descriptor misses and host GPU output\n", produce ? "produced" : "reloaded"); + } +} diff --git a/bridge/driver_cache_host.cpp b/bridge/driver_cache_host.cpp new file mode 100644 index 0000000..9eac391 --- /dev/null +++ b/bridge/driver_cache_host.cpp @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT +// The default compute-function path used by the target; no explicit binary archive. +// Runs over Turnip and the software kernel peer: no shader output is claimed. +#include "NativeMetalObjects.h" +#include +#include +#include + +#define REQUIRE(x) do {if (!(x)) {std::fprintf(stderr,"FAIL line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while (0) + +int main(int argc,char **argv) { + if(argc!=4)return 2; + const bool consume=!std::strcmp(argv[3],"consume"); + REQUIRE(consume||!std::strcmp(argv[3],"produce")); + auto submissions=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_submissions");REQUIRE(submissions); + NmbDevice *device=nmb_device_open(argv[1],argv[2]);REQUIRE(device); + NmbShaderCacheStats before={},after={};REQUIRE(nmb_device_shader_cache_stats(device,&before)); + size_t size=0;const void *air=nmb_fixture_air(&size); + NmbLibrary *library=nmb_library_new(device,air,size,true);REQUIRE(library); + NmbFunction *function=nmb_function_new(library,"scale_and_offset");REQUIRE(function); + NmbComputePipeline *pipeline=nmb_compute_pipeline_new(device,function);REQUIRE(pipeline); + float input[64]={}; + NmbBuffer *source=nmb_buffer_new(device,sizeof input,0,input); + NmbBuffer *destination=nmb_buffer_new(device,sizeof input,0,nullptr);REQUIRE(source&&destination); + NmbQueue *queue=nmb_queue_new(device);REQUIRE(queue); + NmbCommandBuffer *command=nmb_command_buffer_new(queue);REQUIRE(command); + NmbEncoder *encoder=nmb_compute_encoder_new(command);REQUIRE(encoder); + REQUIRE(nmb_compute_set_pipeline(encoder,pipeline)); + REQUIRE(nmb_encoder_set_buffer(encoder,0,source,0,0)); + REQUIRE(nmb_encoder_set_buffer(encoder,0,destination,0,1)); + const float scale=2;REQUIRE(nmb_encoder_set_bytes(encoder,0,&scale,sizeof scale,2)); + const uint64_t threads[]={64,1,1}; + REQUIRE(nmb_compute_dispatch(encoder,threads,threads,true)); + REQUIRE(nmb_encoder_end(encoder)); + const unsigned submitted=submissions(); + REQUIRE(nmb_command_buffer_commit(command)); + REQUIRE(nmb_command_buffer_wait(command,10000)==0); + REQUIRE(nmb_command_buffer_status(command)==4&&submissions()>submitted); + REQUIRE(nmb_device_shader_cache_stats(device,&after)); + std::printf("cache %s: artifacts hit=%llu miss=%llu write=%llu; driver load=%llu hit=%llu miss=%llu save=%llu same=%llu; rejected=%llu readfail=%llu writefail=%llu truncated=%llu mismatched=%llu\n",argv[3], + (unsigned long long)(after.hits-before.hits),(unsigned long long)(after.misses-before.misses),(unsigned long long)(after.writes-before.writes), + (unsigned long long)(after.driver_loads-before.driver_loads),(unsigned long long)(after.driver_hits-before.driver_hits), + (unsigned long long)(after.driver_misses-before.driver_misses),(unsigned long long)(after.driver_saves-before.driver_saves), + (unsigned long long)(after.driver_unchanged-before.driver_unchanged),(unsigned long long)after.rejected,(unsigned long long)after.read_failures, + (unsigned long long)after.write_failures,(unsigned long long)after.driver_truncated,(unsigned long long)after.driver_mismatched); + if(consume){ + REQUIRE(after.hits>before.hits&&after.misses==before.misses); + REQUIRE(after.driver_loads>before.driver_loads&&after.driver_hits>before.driver_hits); + REQUIRE(after.driver_misses==before.driver_misses); + }else{ + REQUIRE(after.misses>before.misses&&after.writes>before.writes); + REQUIRE(after.driver_misses>before.driver_misses&&after.driver_saves>before.driver_saves); + } + REQUIRE(!after.read_failures&&!after.write_failures&&!after.rejected&&!after.driver_truncated&&!after.driver_mismatched); + nmb_command_buffer_release(command);nmb_queue_release(queue); + nmb_buffer_release(destination);nmb_buffer_release(source);nmb_compute_pipeline_release(pipeline); + nmb_function_release(function);nmb_library_release(library);nmb_device_release(device); + std::printf("PASS: default compute %s uses separate artifact/Turnip caches; software transport, results unchecked\n",argv[3]); + return 0; +} diff --git a/bridge/frontend_host.mm b/bridge/frontend_host.mm new file mode 100644 index 0000000..39f0414 --- /dev/null +++ b/bridge/frontend_host.mm @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +// Host run of the frontend checks (FrontendChecks.h) over the bridge and a host Vulkan driver as the loader (MoltenVK on the Mac: an offline +// instrument; on the tablet the plugin runs the same checks over Turnip). Prints one PASS line or the failures. +#include "FrontendChecks.h" +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +static std::vector missing; +static void onMissing(const char *cls, const char *sel) {missing.push_back(std::string(cls) + " " + sel);} + +// argv: loader, shader cache, [QuartzCore default.metallib or ""] +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, onMissing); + if (!device) {std::printf("FAIL device\n"); return 1;} + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + air = nmb_function_constant_air(&bytes); + id fcLibrary = native_bridge_metal_library_with_air(air, bytes); + if (!library || !fcLibrary) {std::printf("FAIL fixture libraries\n"); return 1;} + frontendchecks::Report r = {}; + frontendchecks::run(device, library, fcLibrary, argc > 3 && argv[3][0] ? @(argv[3]) : nil, &r); + if (r.qcFunctions) std::printf("QuartzCore bare-descriptor probe: %u functions, kernels %u/%u, pairs %u/%u (not complete shader coverage)\n", r.qcFunctions, r.qcKernelsOk, r.qcKernels, r.qcPairsOk, r.qcPairs); + for (auto &m : missing) std::printf("missing selector: %s\n", m.c_str()); + if (r.failed) {std::printf("FAILED %u checks, first: %s\n", r.failed, r.first); return 1;} + std::printf("PASS: native Metal frontend over the bridge on %s: compute, render, IOSurface render, surface sampling, texture compute, " + "handlers, refusal, function constants, events and fences\n", device.name.UTF8String); + return 0; + } +} diff --git a/bridge/graphics_cache_loader.cpp b/bridge/graphics_cache_loader.cpp new file mode 100644 index 0000000..4d71bcc --- /dev/null +++ b/bridge/graphics_cache_loader.cpp @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: MIT */ +/* Offline qualification only: forward to the actual Vulkan compiler, enabling + * its reported cache-control feature and optionally forbidding compilation. + * A negative control corrupts only a private copy of a cache UUID. Never deploy. */ +#include +#include +#include +#include +#include +#include +#include + +static PFN_vkGetInstanceProcAddr realInstance; +static std::atomic realDevice; +static std::once_flag initialized; +static std::atomic activeInstance; +static std::atomic strict, badUuid; +// calls, success, compile-required, valid feedback, whole-cache hit, UUID mutations, devices, +// calls with an application cache, their valid feedback and their cache hits +static std::atomic counts[10]; +static void initialize() { + std::call_once(initialized,[]{ + const char *path=std::getenv("NMB_TEST_REAL_VULKAN_LOADER"); + void *library=path?dlopen(path,RTLD_NOW|RTLD_LOCAL):nullptr; + if(!library)std::abort(); + realInstance=reinterpret_cast(dlsym(library,"vkGetInstanceProcAddr")); + if(!realInstance)realInstance=reinterpret_cast(dlsym(library,"vk_icdGetInstanceProcAddr")); + if(!realInstance)std::abort(); + }); +} +static VKAPI_ATTR VkResult VKAPI_CALL createDevice(VkPhysicalDevice physical,const VkDeviceCreateInfo *info, + const VkAllocationCallbacks *allocator,VkDevice *device) { + initialize(); + // The qualified runtime requests Vulkan 1.2. Refuse an unexpected chain + // rather than duplicate a promoted feature or alter caller-owned structures. + for(auto p=static_cast(info->pNext);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES || + p->sType==VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES)return VK_ERROR_FEATURE_NOT_PRESENT; + auto query=reinterpret_cast(realInstance(activeInstance.load(),"vkGetPhysicalDeviceFeatures2")); + if(!query)return VK_ERROR_FEATURE_NOT_PRESENT; + VkPhysicalDevicePipelineCreationCacheControlFeatures supported={};supported.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES; + VkPhysicalDeviceFeatures2 features={};features.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;features.pNext=&supported; + query(physical,&features);if(!supported.pipelineCreationCacheControl)return VK_ERROR_FEATURE_NOT_PRESENT; + std::vector extensions; + if(info->enabledExtensionCount)extensions.assign(info->ppEnabledExtensionNames,info->ppEnabledExtensionNames+info->enabledExtensionCount); + bool listed=false;for(auto name:extensions)listed|=!std::strcmp(name,VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME); + if(!listed)extensions.push_back(VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME); + supported.pNext=const_cast(info->pNext); + VkDeviceCreateInfo copy=*info;copy.pNext=&supported;copy.enabledExtensionCount=(uint32_t)extensions.size();copy.ppEnabledExtensionNames=extensions.data(); + auto real=reinterpret_cast(realInstance(activeInstance.load(),"vkCreateDevice")); + if(!real)return VK_ERROR_INITIALIZATION_FAILED; + const auto result=real(physical,©,allocator,device);if(result==VK_SUCCESS)counts[6]++; + return result; +} +static VKAPI_ATTR VkResult VKAPI_CALL createCache(VkDevice device,const VkPipelineCacheCreateInfo *info, + const VkAllocationCallbacks *allocator,VkPipelineCache *cache) { + VkPipelineCacheCreateInfo copy=*info;std::vector bytes; + if(badUuid.load() && info->initialDataSize) { + if(info->initialDataSize<32)return VK_ERROR_INITIALIZATION_FAILED; + const auto *source=static_cast(info->pInitialData); + bytes.assign(source,source+info->initialDataSize);bytes[16]^=1;copy.pInitialData=bytes.data();counts[5]++; + } + auto real=reinterpret_cast(realDevice.load()(device,"vkCreatePipelineCache")); + return real(device,©,allocator,cache); +} +static VKAPI_ATTR VkResult VKAPI_CALL createGraphics(VkDevice device,VkPipelineCache cache,uint32_t count, + const VkGraphicsPipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) { + if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED; + std::vector copies(infos,infos+count); + std::vector feedback(count); + std::vector additions(count); + std::vector observed(count); + for(uint32_t i=0;i(copies[i].pNext);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO) + observed[i]=reinterpret_cast(p)->pPipelineCreationFeedback; + if(!observed[i]) { + auto &a=additions[i];a.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO; + a.pNext=copies[i].pNext;a.pPipelineCreationFeedback=&feedback[i];copies[i].pNext=&a;observed[i]=&feedback[i]; + } + } + auto real=reinterpret_cast(realDevice.load()(device,"vkCreateGraphicsPipelines")); + counts[0]+=count;if(cache)counts[7]+=count;const auto result=real(device,cache,count,copies.data(),allocator,pipelines); + if(result==VK_PIPELINE_COMPILE_REQUIRED)counts[2]++; + for(uint32_t i=0;iflags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) { + counts[3]++;if(cache)counts[8]++; + if(observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT){counts[4]++;if(cache)counts[9]++;} + } + } + return result; +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) { + initialize(); + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(createGraphics); + if(!std::strcmp(name,"vkCreatePipelineCache"))return reinterpret_cast(createCache); + return realDevice.load()(device,name); +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) { + initialize(); + if(instance){ + activeInstance=instance; + realDevice=reinterpret_cast(realInstance(instance,"vkGetDeviceProcAddr")); + if(!realDevice.load())std::abort(); + } + if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast(vkGetDeviceProcAddr); + if(!std::strcmp(name,"vkCreateDevice"))return reinterpret_cast(createDevice); + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(createGraphics); + if(!std::strcmp(name,"vkCreatePipelineCache"))return reinterpret_cast(createCache); + return realInstance(instance,name); +} +extern "C" void nmb_test_graphics_cache_control(bool forbidCompilation,bool corruptUuid) {strict=forbidCompilation;badUuid=corruptUuid;} +extern "C" void nmb_test_graphics_cache_counts(uint64_t out[10]) {for(unsigned i=0;i<10;++i)out[i]=counts[i].load();} diff --git a/bridge/import_count_loader.cpp b/bridge/import_count_loader.cpp new file mode 100644 index 0000000..a98c4a1 --- /dev/null +++ b/bridge/import_count_loader.cpp @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: MIT */ +/* Host-test Vulkan forwarding loader. Every request reaches the real driver; + * count successful external-host-memory imports without changing any request, + * code, driver identity, memory contents, fence or GPU result. Never deployed. */ +#include +#include +#include +#include +#include +#include +#include +#include + +static PFN_vkGetInstanceProcAddr realInstance; +static PFN_vkGetDeviceProcAddr realDevice; +static std::once_flag initialized; +static std::atomic imported; +static std::atomic importDelayMs; +static void initialize() { + std::call_once(initialized,[]{ + const char *path=std::getenv("NMB_TEST_REAL_VULKAN_LOADER"); + void *library=path?dlopen(path,RTLD_NOW|RTLD_LOCAL):nullptr; + if(!library)std::abort(); + realInstance=reinterpret_cast(dlsym(library,"vkGetInstanceProcAddr")); + realDevice=reinterpret_cast(dlsym(library,"vkGetDeviceProcAddr")); + if(!realInstance || !realDevice)std::abort(); + }); +} +static VKAPI_ATTR VkResult VKAPI_CALL allocate(VkDevice device,const VkMemoryAllocateInfo *info, + const VkAllocationCallbacks *allocator,VkDeviceMemory *memory) { + initialize(); + bool hostImport=false; + for(auto node=static_cast(info->pNext);node;node=node->pNext) + if(node->sType==VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT){hostImport=true;break;} + // A bounded, one-shot delay makes a concurrent first-lookup miss + // reproducible. The actual allocation and its result are unchanged. + if(hostImport) { + const unsigned delay=importDelayMs.exchange(0); + if(delay)std::this_thread::sleep_for(std::chrono::milliseconds(delay)); + } + auto real=reinterpret_cast(realDevice(device,"vkAllocateMemory")); + const VkResult result=real(device,info,allocator,memory); + if(result==VK_SUCCESS && hostImport)imported++; + return result; +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) { + initialize(); + if(!std::strcmp(name,"vkAllocateMemory"))return reinterpret_cast(allocate); + return realDevice(device,name); +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) { + initialize(); + if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast(vkGetDeviceProcAddr); + if(!std::strcmp(name,"vkAllocateMemory"))return reinterpret_cast(allocate); + return realInstance(instance,name); +} +extern "C" uint64_t nmb_test_vk_host_import_count() {return imported.load();} +extern "C" void nmb_test_vk_host_import_delay(unsigned milliseconds) { + importDelayMs.store(milliseconds<=100?milliseconds:100); +} diff --git a/bridge/nil_args_host.mm b/bridge/nil_args_host.mm new file mode 100644 index 0000000..09dd2be --- /dev/null +++ b/bridge/nil_args_host.mm @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +// Host check: a blit encoder call with a nil resource (what a refused allocation hands the caller) fails its command buffer instead of raising an +// exception. An exception escaping a worker thread of the host process aborts the process; on the tablet that killed SpringBoard (int-075/083). +#import +#import +#include +#include "../NativeBridgeMetalFrontend.h" + +__attribute__((noinline)) static id nothing() {return nil;} // not a literal: the compiler rejects nil for these nonnull parameters + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + id queue = [device newCommandQueue]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + id texture = [device newTextureWithDescriptor:d]; + int failed = 0; + for (int variant = 0; variant < 4; ++variant) { + @try { + id cb = [queue commandBuffer]; + id blit = [cb blitCommandEncoder]; + switch (variant) { + case 0: [blit copyFromBuffer:(id)nothing() sourceOffset:0 toBuffer:buffer destinationOffset:0 size:16]; break; + case 1: [blit fillBuffer:(id)nothing() range:NSMakeRange(0, 16) value:0]; break; + case 2: [blit copyFromTexture:(id)nothing() sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(8, 8, 1) + toBuffer:buffer destinationOffset:0 destinationBytesPerRow:32 destinationBytesPerImage:256]; break; + default: [blit copyFromTexture:texture toTexture:(id)nothing()]; break; + } + [blit endEncoding]; + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusError) {std::printf("FAIL variant %d: status %ld, not an error\n", variant, (long)cb.status); ++failed;} + } @catch (NSException *e) { + std::printf("FAIL variant %d raised %s\n", variant, e.name.UTF8String); ++failed; + } + } + if (failed) return 1; + std::printf("PASS: a nil resource fails the command buffer in every blit variant, no exception\n"); + } + return 0; +} diff --git a/bridge/no_copy_import_host.mm b/bridge/no_copy_import_host.mm new file mode 100644 index 0000000..c00441c --- /dev/null +++ b/bridge/no_copy_import_host.mm @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +// Actual public Metal no-copy calls through the native bridge. Every buffer +// remains a distinct owner; count real Vulkan imports and check real GPU bytes. +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +constexpr unsigned workers=16; +struct State {std::array,workers> released{};std::atomic errors{0};}; + +static bool creation(id device,bool mixed,uint64_t (*imports)(),void (*delay)(unsigned)) { + auto state=std::make_shared();bool gpu=true; + void *mapping=nullptr;size_t bytes=0;IOSurfaceRef surface=nullptr; + uint64_t allocated=0; + @autoreleasepool { + if(mixed) { + surface=IOSurfaceCreate((__bridge CFDictionaryRef)@{(id)kIOSurfaceWidth:@64,(id)kIOSurfaceHeight:@16, + (id)kIOSurfaceBytesPerElement:@4,(id)kIOSurfaceBytesPerRow:@256,(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}); + if(!surface)return false; + mapping=IOSurfaceGetBaseAddress(surface);bytes=IOSurfaceGetAllocSize(surface); + } else { + bytes=(size_t)getpagesize();mapping=mmap(nullptr,bytes,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(mapping==MAP_FAILED)return false; + } + if(!mapping || bytes<4096)return false; + memset(mapping,0xcc,bytes); + auto buffers=[NSMutableArray array],textures=[NSMutableArray array]; + std::mutex startLock;std::condition_variable ready;unsigned waiting=0;bool start=false; + std::vector threads; + const uint64_t before=imports();delay(25); + for(unsigned i=0;i lock(startLock);++waiting;ready.notify_all();ready.wait(lock,[&]{return start;});} + @autoreleasepool { + if(mixed && (i&1)) { + auto d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:16 mipmapped:NO]; + d.storageMode=MTLStorageModeShared;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite; + id texture=[device newTextureWithDescriptor:d iosurface:surface plane:0]; + if(!texture){state->errors++;return;} + @synchronized(textures){[textures addObject:texture];} + } else { + auto owner=state;const unsigned ordinal=i; + id buffer=[device newBufferWithBytesNoCopy:mapping length:bytes options:MTLResourceStorageModeShared + deallocator:^(void *,NSUInteger){owner->released[ordinal]++;}]; + if(!buffer){state->errors++;return;} + @synchronized(buffers){[buffers addObject:buffer];} + } + } + }); + {std::unique_lock lock(startLock);ready.wait(lock,[&]{return waiting==workers;});start=true;ready.notify_all();} + for(auto &thread:threads)thread.join(); + allocated=imports()-before; + const unsigned expected=mixed?workers/2:workers; + if(state->errors || buffers.count!=expected || textures.count!=(mixed?workers/2:0))return false; + @autoreleasepool { + // The API still returns distinct buffer objects and independent + // callbacks; only the driver's import is shared. + if([NSSet setWithArray:buffers].count!=expected)return false; + } + id queue=[device newCommandQueue]; + id write=[queue commandBuffer];id fill=[write blitCommandEncoder]; + [fill fillBuffer:buffers[0] range:NSMakeRange(0,4096) value:0x5a];[fill endEncoding];[write commit];[write waitUntilCompleted]; + // Use an explicit completed producer, so correctness here does not + // assume implicit synchronization of arbitrary no-copy Metal aliases. + id output=[device newBufferWithLength:workers*4096 options:MTLResourceStorageModeShared]; + id read=[queue commandBuffer];id copy=[read blitCommandEncoder]; + if(!queue || !write || !fill || !output || !read || !copy)return false; + NSUInteger at=0; + for(id buffer in buffers){[copy copyFromBuffer:buffer sourceOffset:0 toBuffer:output destinationOffset:at size:4096];at+=4096;} + for(id texture in textures){ + [copy copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1) + toBuffer:output destinationOffset:at destinationBytesPerRow:256 destinationBytesPerImage:4096];at+=4096; + } + [copy endEncoding];[read commit];[read waitUntilCompleted]; + gpu=write.status==MTLCommandBufferStatusCompleted && read.status==MTLCommandBufferStatusCompleted && at==workers*4096; + for(unsigned i=0;ireleased[i].load()!=(mixed&&(i&1)?0u:1u))owners=false; + if(owners)break;usleep(1000); + } + if(surface)CFRelease(surface);else munmap(mapping,bytes); + std::printf("mode=%s buffers=%u vk_imports=%llu gpu=%s independent_owners=%s\n",mixed?"mixed":"raw",mixed?workers/2:workers, + (unsigned long long)allocated,gpu?"pass":"FAIL",owners?"pass":"FAIL"); + return allocated==1 && gpu && owners; +} + +static bool reentry(id device) { + const size_t page=(size_t)getpagesize();void *mapping=mmap(nullptr,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(mapping==MAP_FAILED)return false; + auto finished=dispatch_semaphore_create(0);auto state=std::make_shared(); + @autoreleasepool { + id outer=[device newBufferWithBytesNoCopy:mapping length:page options:MTLResourceStorageModeShared + deallocator:^(void *,NSUInteger){ + state->released[0]++; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{ + @autoreleasepool { + id nested=[device newBufferWithBytesNoCopy:(char *)mapping+page length:page options:MTLResourceStorageModeShared + deallocator:^(void *,NSUInteger){state->released[1]++;}]; + if(!nested)state->errors++; + } + dispatch_semaphore_signal(finished); + }); + if(dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,3*NSEC_PER_SEC)))state->errors++; + }]; + if(!outer)return false; + } + const bool ok=!state->errors && state->released[0]==1 && state->released[1]==1; + // On a timeout keep the test-owned mapping alive until process exit; + // never turn a lifetime failure into a submitted dangling pointer. + if(ok)munmap(mapping,page*2); + std::printf("deallocator cross-thread reentry=%s\n",ok?"pass":"FAIL");return ok; +} + +int main(int argc,char **argv) { + if(argc!=4)return 2; + @autoreleasepool { + id device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2; + void *loader=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!loader)return 2; + auto imports=(uint64_t (*)())dlsym(loader,"nmb_test_vk_host_import_count"); + auto delay=(void (*)(unsigned))dlsym(loader,"nmb_test_vk_host_import_delay"); + if(!imports || !delay)return 2; + bool ok=false; + if(!strcmp(argv[3],"raw"))ok=creation(device,false,imports,delay); + else if(!strcmp(argv[3],"mixed"))ok=creation(device,true,imports,delay); + else if(!strcmp(argv[3],"reentry"))ok=reentry(device); + if(!ok)return 1; + std::puts("PASS: native no-copy construction coalesces real imports while retaining independent objects, GPU bytes and callbacks"); + } +} diff --git a/bridge/patches/0001-optional-constant-output-members.patch b/bridge/patches/0001-optional-constant-output-members.patch new file mode 100644 index 0000000..8042565 --- /dev/null +++ b/bridge/patches/0001-optional-constant-output-members.patch @@ -0,0 +1,59 @@ +diff --git a/crates/metal2vulkan-air/src/canon/sccp.rs b/crates/metal2vulkan-air/src/canon/sccp.rs +index e47e683..f89e2fc 100644 +--- a/crates/metal2vulkan-air/src/canon/sccp.rs ++++ b/crates/metal2vulkan-air/src/canon/sccp.rs +@@ -995,6 +995,30 @@ impl<'b> Solver<'b> { + self.set(i, new, m, held) + } + ++ /// Whether `v` is an aggregate built by `insertvalue` chains whose only unsupplied-constant ++ /// taint sits in individual inserted members, so those members alone can become `undef` ++ /// (Metal leaves an unsupplied optional constant's value indeterminate) while the rest of ++ /// the aggregate keeps its computed members. ++ fn leafwise(&self, v: Value) -> bool { ++ let mut cur = v; ++ loop { ++ if !matches!(self.lat(cur), Lat::Fc(_)) { ++ return true; ++ } ++ let Value::Inst(i) = cur else { return false }; ++ let Some(inst) = self.body.insts.get(i) else { ++ return false; ++ }; ++ let InstKind::InsertValue { agg, val, .. } = &inst.kind else { ++ return false; ++ }; ++ if matches!(self.lat(*val), Lat::Fc(_)) && !matches!(val, Value::Inst(_)) { ++ return false; ++ } ++ cur = *agg; ++ } ++ } ++ + /// After the fixpoint: every reachable non-pure use of an unsupplied constant needs the + /// caller's value, and a reachable branch on a literal `undef`/`poison` is refused. + fn check(&self, cx: &Context<'_>) -> Result<(), Error> { +@@ -1031,6 +1055,11 @@ impl<'b> Solver<'b> { + ))); + } + } ++ if let InstKind::Ret(Some(v)) = &inst.kind { ++ if self.leafwise(*v) { ++ continue; ++ } ++ } + let mut needs = None; + inst.kind.for_each_value(|v| { + if let Lat::Fc(index) = self.lat(v) { +@@ -1070,7 +1099,11 @@ impl<'b> Solver<'b> { + let deletable = is_pure(&inst.kind) + || is_normalize(&inst.kind) + || is_defined_arg(&inst.kind, cx.folder.consts, cx.fcs).is_some(); ++ let leaf_chain = matches!(inst.kind, InstKind::InsertValue { .. }) ++ && matches!(lat, Lat::Fc(_)) ++ && self.leafwise(Value::Inst(i)); + match (&inst.kind, lat) { ++ _ if leaf_chain => {} + (_, Lat::Const(c)) if deletable => edit.replace(i, Value::Const(c)), + (_, Lat::Fc(_)) if deletable => { + let undef = cx.folder.consts.intern( diff --git a/bridge/patches/0002-resource-wait-storage-dependencies.patch b/bridge/patches/0002-resource-wait-storage-dependencies.patch new file mode 100644 index 0000000..6177488 --- /dev/null +++ b/bridge/patches/0002-resource-wait-storage-dependencies.patch @@ -0,0 +1,67 @@ +diff --git a/crates/metal2vulkan-runtime/src/used_resources.rs b/crates/metal2vulkan-runtime/src/used_resources.rs +index 7521345..03001a8 100644 +--- a/crates/metal2vulkan-runtime/src/used_resources.rs ++++ b/crates/metal2vulkan-runtime/src/used_resources.rs +@@ -58,6 +58,62 @@ pub struct UsedResources { + pub heaps: Vec, + } + ++// Compare host address ranges without touching their bytes. Object identity covers private ++// memory; host ranges additionally cover distinct imports of an IOSurface or overlapping ++// no-copy buffers. Unknown arithmetic is conservative. Rebinding is not performed by this ++// consumer, and callers must not remap these ranges while the resources are in use. ++fn buffers_overlap(a: &Buffer, b: &Buffer) -> bool { ++ if a == b { ++ return true; ++ } ++ let (Some(a_start), Some(b_start)) = (a.contents(), b.contents()) else { ++ return false; ++ }; ++ let (Ok(a_len), Ok(b_len)) = (usize::try_from(a.length()), usize::try_from(b.length())) else { ++ return true; ++ }; ++ let a_start = a_start.as_ptr() as usize; ++ let b_start = b_start.as_ptr() as usize; ++ let (Some(a_end), Some(b_end)) = (a_start.checked_add(a_len), b_start.checked_add(b_len)) else { ++ return true; ++ }; ++ a_start < b_end && b_start < a_end ++} ++ ++impl UsedResources { ++ fn references_buffer(&self, buffer: &Buffer) -> bool { ++ if buffer.heap().is_some_and(|heap| self.heaps.contains(heap)) { ++ return true; ++ } ++ self.buffers.iter().any(|other| buffers_overlap(buffer, other)) ++ || self.heaps.iter().any(|heap| heap.bytes().is_some_and(|bytes| buffers_overlap(buffer, bytes))) ++ } ++ ++ /// Whether this list names a resource's storage, including a view's root, a buffer ++ /// texture's backing buffer, a heap's resources and overlapping imported host ranges. ++ /// A caller may skip waiting only while this is false. This consumer does not rebind ++ /// or remap live backing ranges. Indirect command buffers remain conservative. ++ pub fn references_resource(&self, resource: super::resource::ResourceRef<'_>) -> bool { ++ use super::resource::ResourceRef; ++ match resource { ++ ResourceRef::Buffer(buffer) => self.references_buffer(buffer), ++ ResourceRef::Texture(texture) => match texture.memory() { ++ TextureMemory::Heap(heap) => self.references_heap(heap), ++ TextureMemory::Buffer(buffer) => self.references_buffer(buffer), ++ TextureMemory::Own(root) => self.textures.contains(root), ++ }, ++ ResourceRef::IndirectCommandBuffer(_) => true, ++ } ++ } ++ ++ /// Heap storage is treated as one dependency, conservatively including its disjoint ++ /// members. Imports over its host range are dependencies too. ++ pub fn references_heap(&self, heap: &Heap) -> bool { ++ self.heaps.contains(heap) ++ || heap.bytes().is_some_and(|bytes| self.references_buffer(bytes)) ++ } ++} ++ + /// The kind of a command encoder ([`EndedEncoder::kind`]), by the Metal protocol it is. + #[non_exhaustive] + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] diff --git a/bridge/patches/0003-cache-device-entry-point-priming.patch b/bridge/patches/0003-cache-device-entry-point-priming.patch new file mode 100644 index 0000000..47bd1ea --- /dev/null +++ b/bridge/patches/0003-cache-device-entry-point-priming.patch @@ -0,0 +1,253 @@ +diff --git a/crates/metal2vulkan-runtime/src/device/mod.rs b/crates/metal2vulkan-runtime/src/device/mod.rs +index 990c892..00a6e55 100644 +--- a/crates/metal2vulkan-runtime/src/device/mod.rs ++++ b/crates/metal2vulkan-runtime/src/device/mod.rs +@@ -962,6 +962,21 @@ impl Device { + // SAFETY: the device is live, of `physical`, and enables timeline semaphores, and + // `bufferDeviceAddress` where `device_address` says so; `queues` holds at least one of + // its queues, which nothing uses yet. ++ let priming_has = prime::Has { ++ timestamps: timestamp_bits > 0, ++ device_addresses: device_address, ++ generated_commands: generated_commands.is_some(), ++ sparse_binding: sparse.binds(), ++ }; ++ // Raw disk/cache I/O only: starting the ordinary guarded cache or its ++ // worker before priming would change the destructor ordering it protects. ++ let priming_cache = cache.priming( ++ &unsafe { instance.raw.get_physical_device_properties(physical) }, ++ compile.profile_fingerprint(), ++ instance.settings, ++ prime::cache_key(priming_has, queue_family), ++ instance.sink().cloned(), ++ ); + if let Err(error) = unsafe { + prime::prime( + &instance.raw, +@@ -971,13 +986,9 @@ impl Device { + queue_family, + &queues, + Sharing::new(&queue_families), +- prime::Has { +- timestamps: timestamp_bits > 0, +- device_addresses: device_address, +- generated_commands: generated_commands.is_some(), +- sparse_binding: sparse.binds(), +- }, ++ priming_has, + generated_commands.as_ref(), ++ priming_cache, + ) + } { + scheduler.stop(); +diff --git a/crates/metal2vulkan-runtime/src/device/prime.rs b/crates/metal2vulkan-runtime/src/device/prime.rs +index 91791ba..50856aa 100644 +--- a/crates/metal2vulkan-runtime/src/device/prime.rs ++++ b/crates/metal2vulkan-runtime/src/device/prime.rs +@@ -52,9 +52,11 @@ use crate::generated_commands::{ + use crate::loader_word::LoaderWord; + use crate::memory; + use crate::release::{self, Generated, Released}; ++use crate::shader_cache::PrimingCache; + use crate::sharing::Sharing; + use crate::tracker::Barrier; + use ash::vk; ++use sha2::{Digest, Sha256}; + use std::time::{Duration, Instant}; + + use super::QueueSlot; +@@ -184,7 +186,7 @@ const VERTEX_SHADER: [u32; 29] = [ + + /// What the device primed has: a queue family that writes timestamps, device addresses, + /// `VK_EXT_device_generated_commands`, and sparse binding on its queues. +-#[derive(Clone, Copy)] ++#[derive(Clone, Copy, Debug)] + pub(crate) struct Has { + pub(crate) timestamps: bool, + pub(crate) device_addresses: bool, +@@ -192,6 +194,16 @@ pub(crate) struct Has { + pub(crate) sparse_binding: bool, + } + ++/// Identity includes every byte of the helper implementation, enabled compile ++/// profile (in Records), instance settings, driver identity, and these choices. ++/// No pointer/handle or host path enters the persistent key. ++pub(super) fn cache_key(has: Has, queue_family: u32) -> metal2vulkan::cache::Hash256 { ++ crate::shader_cache::driver::Making::new("entry-point-priming-v1") ++ .digest(&Sha256::digest(include_bytes!("prime.rs")).into()) ++ .text(&(has, queue_family)) ++ .finish() ++} ++ + /// Prime `device` of `physical`, made from `instance`, whose queues are `queues`, the first of + /// its primary family `queue_family`, its buffers and images shared as `sharing` says, on this + /// thread; `has` says what the device has, and `generated` holds the commands of +@@ -213,6 +225,7 @@ pub(super) unsafe fn prime( + sharing: Sharing<'_>, + has: Has, + generated: Option<&Commands>, ++ mut disk: PrimingCache, + ) -> Result<(), Error> { + let mut primer = Primer { + device, +@@ -222,6 +235,8 @@ pub(super) unsafe fn prime( + memory: unsafe { instance.get_physical_device_memory_properties(physical) }, + calls: 0, + held: Vec::new(), ++ disk: &mut disk, ++ pipeline_cache: vk::PipelineCache::null(), + }; + // SAFETY: the caller's. + let primed = unsafe { +@@ -239,6 +254,10 @@ pub(super) unsafe fn prime( + unsafe { + let _ = device.device_wait_idle(); + primer.made(Call::DeviceWaitIdle); ++ if primed.is_ok() && primer.pipeline_cache != vk::PipelineCache::null() { ++ primer.disk.keep(device, loader_word, primer.pipeline_cache); ++ primer.made(Call::GetPipelineCacheData); ++ } + while let Some(release) = primer.held.pop() { + let call = release(device); + primer.made(call); +@@ -303,6 +322,8 @@ struct Primer<'a> { + calls: u128, + /// What was made, released last first. + held: Vec, ++ disk: &'a mut PrimingCache, ++ pipeline_cache: vk::PipelineCache, + } + + impl Primer<'_> { +@@ -575,7 +596,7 @@ impl Primer<'_> { + .layout(layout) + .push_next(&mut flags); + let pipeline = device +- .create_compute_pipelines(vk::PipelineCache::null(), &[info], None) ++ .create_compute_pipelines(self.pipeline_cache, &[info], None) + .map_err(|(_, error)| vulkan(error))?[0]; + self.hold(pipeline); + let pipelines = ExecutionSetPipelineInfo::new(pipeline, 1); +@@ -848,9 +869,8 @@ impl Primer<'_> { + let module = device.create_shader_module(&info, None).map_err(vulkan)?; + self.made(Call::CreateShaderModule); + self.hold(module); +- let cache = device +- .create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) +- .map_err(vulkan)?; ++ let cache = self.disk.open(device, self.loader_word).map_err(vulkan)?; ++ self.pipeline_cache = cache; + self.made(Call::CreatePipelineCache); + self.hold(cache); + let stage = vk::PipelineShaderStageCreateInfo::default() +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +index 88e0fad..513fe37 100644 +--- a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs ++++ b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +@@ -282,6 +282,106 @@ impl CacheSetup { + } + } + ++/// Disk access for entry-point priming, before any exit hook is registered. ++/// Deliberately no worker, exit-guard section, or ordinary driver::make call. ++/// Priming keeps its raw call/cleanup ordering and records every entry point. ++pub(super) struct PrimingCache { ++ store: Store, ++ identity: driver::Identity, ++ name: String, ++ loaded: Option>, ++} ++ ++impl CacheSetup { ++ pub(super) fn priming( ++ &self, ++ properties: &vk::PhysicalDeviceProperties, ++ profile: Hash256, ++ settings: Hash256, ++ key: Hash256, ++ sink: Option, ++ ) -> PrimingCache { ++ let identity = driver::Identity::of(properties); ++ let name = Records { ++ identity, ++ settings, ++ profile, ++ } ++ .name(Making::new("device-entry-point-priming").digest(&key)); ++ PrimingCache { ++ store: Store::new(self.directory.clone(), sink), ++ identity, ++ name, ++ loaded: None, ++ } ++ } ++} ++ ++impl PrimingCache { ++ /// The caller owns the raw cache and primes/releases its entry points. ++ /// No exit_guard::called/enter may happen here, before priming is finished. ++ pub(super) unsafe fn open( ++ &mut self, ++ raw: &ash::Device, ++ word: &crate::loader_word::LoaderWord, ++ ) -> Result { ++ let seed = match self.store.load_content(&self.name, |bytes| { ++ let data = driver::open(&self.identity, &bytes)?; ++ Some(bytes[data].to_vec()) ++ }) { ++ Lookup::Hit(bytes) => Some(bytes), ++ _ => None, ++ }; ++ if let Some(bytes) = seed { ++ let info = vk::PipelineCacheCreateInfo::default().initial_data(&bytes); ++ // SAFETY: the caller owns the live device and the info's bytes outlive the call. ++ if let Ok(cache) = unsafe { ++ word.restore(); ++ raw.create_pipeline_cache(&info, None) ++ } { ++ self.loaded = Some(bytes); ++ return Ok(cache); ++ } ++ } ++ // Invalid/absent/refused data is never treated as a successful load. ++ // SAFETY: as above; the empty create info outlives the call. ++ unsafe { ++ word.restore(); ++ raw.create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) ++ } ++ } ++ ++ /// Called after all priming (including graphics/generated helpers) succeeds, ++ /// before its raw cache is destroyed and before exit-hook registration. ++ pub(super) unsafe fn keep( ++ &self, ++ raw: &ash::Device, ++ word: &crate::loader_word::LoaderWord, ++ cache: vk::PipelineCache, ++ ) { ++ // SAFETY: priming owns the live device/cache and every making has returned. ++ let Ok(data) = (unsafe { ++ word.restore(); ++ raw.get_pipeline_cache_data(cache) ++ }) else { ++ return; ++ }; ++ if self.loaded.as_deref() == Some(data.as_slice()) { ++ return; ++ } ++ if self.identity.check_vulkan_header(&data).is_err() { ++ return; ++ } ++ if self ++ .store ++ .write_record(&self.name, &driver::seal(&self.identity, &data)) ++ .is_err() ++ { ++ self.store.write_failed(); ++ } ++ } ++} ++ + /// One device's shader cache: its memo, its view of the directory, what names its makings' + /// records, and the cache thread. + pub(super) struct ShaderCache { diff --git a/bridge/patches/0004-fold-constant-global-field-loads.patch b/bridge/patches/0004-fold-constant-global-field-loads.patch new file mode 100644 index 0000000..5a79314 --- /dev/null +++ b/bridge/patches/0004-fold-constant-global-field-loads.patch @@ -0,0 +1,205 @@ +diff --git a/crates/metal2vulkan-air/src/canon/const_loads.rs b/crates/metal2vulkan-air/src/canon/const_loads.rs +index 7b3dfd7..98976df 100644 +--- a/crates/metal2vulkan-air/src/canon/const_loads.rs ++++ b/crates/metal2vulkan-air/src/canon/const_loads.rs +@@ -1,4 +1,4 @@ +-//! Pointers loaded from fields of constant globals. ++//! Values loaded from fields of constant globals. + //! + //! A constant global whose initializer is known once function constants are bound and the static + //! initializers have run may hold pointers: a program-scope `constexpr sampler` taken by value is +@@ -9,18 +9,28 @@ + //! constant and threadgroup spaces, so such a module was refused, under the values that made the + //! select one global. + //! ++//! The same holds for every other first-class field. An uber-shader takes its whole configuration ++//! as one struct of function constants that a static initializer copies into an internal ++//! `constant` global (QuartzCore's `fc_fragment_shader_config`, read field by field after a ++//! function-constant `select` between it and a buffer). SCCP folds that select to the global but ++//! not the field loads, so every branch the configuration decides survived: QuartzCore's ++//! `fixed_frag` became about 46000 SPIR-V instructions where its specialized form is a few ++//! hundred, and the driver compiled each such pipeline for about a second. Metal's compiler ++//! specializes these reads away, so a module that reads them is translated as Metal compiles it. ++//! + //! This pass reads such a load as the initializer's constant at the field: a non-volatile, +-//! non-atomic load of a pointer whose address is a field of a constant global at constant indices +-//! (a `getelementptr` instruction or constant expression over the global's own value type, its +-//! first index 0), where the field's type is the load's. A copy LLVM made through an integer (a +-//! load of the field as an integer of the pointer's size, then `inttoptr` to the field's type) +-//! reads the same constant at its `inttoptr`. The global is never written (it is `constant`, and +-//! initialized by no one else), so every run reads that constant: the address of the sampler +-//! state, which the sampler family reads as the sampler it names. Only pointers are read this +-//! way: a load of any other type from a constant global is lowered from the global's form (its +-//! typed private copy, or the constant blob), and reading it here would change those modules +-//! without changing what they compute. SCCP then runs again over a body this pass changed, so +-//! what the constants decide (an aggregate of them, a phi of them, a member of either) folds too. ++//! non-atomic load of a scalar, vector or pointer whose address is a field of a constant global at ++//! constant indices (a `getelementptr` instruction or constant expression over the global's own ++//! value type, its first index 0), where the field's type is the load's. A copy LLVM made through ++//! an integer (a load of a pointer field as an integer of the pointer's size, then `inttoptr` to ++//! the field's type) reads the same constant at its `inttoptr`. The global is never written (it ++//! is `constant`, and initialized by no one else), so every run reads that constant: for a ++//! pointer, the address of the sampler state, which the sampler family reads as the sampler it ++//! names. An aggregate load, a load of another type than the field's, and a field the initializer ++//! leaves `undef` (an unsupplied constant's) are left to the global's lowered form. SCCP then runs ++//! again over a body this pass changed, so what the constants decide (a branch, an aggregate of ++//! them, a phi of them, a member of either) folds too, and the two alternate until a round folds ++//! nothing: a decided branch can expose a further select of the global. + + use crate::budget::Meter; + use crate::error::Error; +@@ -29,9 +39,9 @@ use crate::ir::{InstId, InstKind, ModuleParts, Type, TypeId, Value}; + + use super::rewrite::{apply, scoped, BodyEdit}; + +-/// Read every load of a pointer from a field of a constant global in `body`, and every +-/// `inttoptr` of such a field's integer copy, as the field's constant (module docs), and say +-/// whether there was one. Returns `body` itself when there is none. ++/// Read every first-class load from a field of a constant global in `body`, and every `inttoptr` ++/// of a pointer field's integer copy, as the field's constant (module docs), and say whether there ++/// was one. Returns `body` itself when there is none. + pub(super) fn fold( + body: Body, + parts: &mut ModuleParts, +@@ -61,8 +71,8 @@ pub(super) fn fold( + } + } + +-/// Each instruction of `body` that reads a pointer field of a constant global (module docs), with +-/// the field's constant. ++/// Each instruction of `body` that reads a first-class field of a constant global (module docs), ++/// with the field's constant. + fn find( + body: &Body, + parts: &mut ModuleParts, +@@ -71,24 +81,27 @@ fn find( + ) -> Result<(), Error> { + for (id, inst) in body.insts.iter() { + m.tick(1)?; +- let Some(space) = parts.types.ptr_space(inst.ty) else { +- continue; +- }; +- if !matches!(parts.types.get(inst.ty), Some(Type::Ptr { .. })) { +- continue; +- } ++ let first_class = parts ++ .types ++ .get(inst.ty) ++ .is_some_and(|t| t.is_sized() && !t.is_aggregate()); ++ let pointer = matches!(parts.types.get(inst.ty), Some(Type::Ptr { .. })); + let ptr = match inst.kind { + InstKind::Load { + ptr, + volatile: false, + atomic: None, + .. +- } => ptr, ++ } if first_class => ptr, + // `inttoptr` of an integer load of the pointer's size: a copy through an integer. + InstKind::Cast { + op: CastOp::IntToPtr, + v: Value::Inst(l), +- } => match body.insts.get(l).map(|l| (&l.kind, l.ty)) { ++ } if pointer => { ++ let Some(space) = parts.types.ptr_space(inst.ty) else { ++ continue; ++ }; ++ match body.insts.get(l).map(|l| (&l.kind, l.ty)) { + Some(( + &InstKind::Load { + ptr, +@@ -104,7 +117,8 @@ fn find( + ptr + } + _ => continue, +- }, ++ } ++ } + _ => continue, + }; + let Some((g, path)) = field_address(body, parts, ptr) else { +diff --git a/crates/metal2vulkan-air/src/canon/mod.rs b/crates/metal2vulkan-air/src/canon/mod.rs +index bd05d94..e3bd590 100644 +--- a/crates/metal2vulkan-air/src/canon/mod.rs ++++ b/crates/metal2vulkan-air/src/canon/mod.rs +@@ -567,11 +567,17 @@ pub fn specialize_dynamic( + }; + body.release(m); + let body = solved; +- // A pointer loaded from a field of a constant global is the field's constant: a `constexpr +- // sampler` that SCCP's folded select picked is its sampler state. SCCP runs again over a body +- // that changed, to fold what those constants decide. +- let (body, folded) = const_loads::fold(body, &mut parts, m)?; +- let body = if folded { ++ // A value loaded from a field of a constant global is the field's constant: a `constexpr ++ // sampler` that SCCP's folded select picked is its sampler state, and an uber-shader's ++ // configuration struct of function constants is its members. SCCP runs again over a body that ++ // changed, to fold what those constants decide; the two alternate until a round folds nothing. ++ let mut body = body; ++ loop { ++ let (next, folded) = const_loads::fold(body, &mut parts, m)?; ++ if !folded { ++ body = next; ++ break; ++ } + let solved = { + let mut cx = sccp::Context { + folder: fold::Folder { +@@ -583,13 +589,11 @@ pub fn specialize_dynamic( + fcs: &fcs, + name: name.clone(), + }; +- sccp::run(&body, &mut cx, m)? ++ sccp::run(&next, &mut cx, m)? + }; +- body.release(m); +- solved +- } else { +- body +- }; ++ next.release(m); ++ body = solved; ++ } + matmul2d::refuse_left(&body, &parts, &ep.stage, &name, m)?; + post_check(&body, &parts, &stubs, &name, &control_points(ep), m)?; + // Rasterization-rate maps: each `float2` map call becomes Apple's GPU decoder of the parameter +diff --git a/crates/metal2vulkan-air/src/canon/tests.rs b/crates/metal2vulkan-air/src/canon/tests.rs +index fb0dad5..93edc0a 100644 +--- a/crates/metal2vulkan-air/src/canon/tests.rs ++++ b/crates/metal2vulkan-air/src/canon/tests.rs +@@ -1658,7 +1658,18 @@ fn a_deeply_nested_written_global_is_rebuilt_on_a_small_stack() { + static_init(&mut m, "init", |f| store(f, five, gaddr)); + let fty = m.fn_ty(void, &[dev]); + let mut f = m.define("k", fty); +- let v = load(&mut f, i32t, gaddr); ++ // A volatile read keeps the global: a plain one now reads the rebuilt initializer's leaf ++ // as its constant, and the unused global is pruned. ++ let v = f.push( ++ i32t, ++ InstKind::Load { ++ ptr: gaddr, ++ align: Align::from_bytes(4), ++ volatile: true, ++ atomic: None, ++ range: None, ++ }, ++ ); + store(&mut f, v, arg(0)); + f.ret(None); + let k = f.finish(); +@@ -2359,9 +2370,12 @@ fn a_pointer_loaded_from_a_constant_globals_field_is_the_fields_constant() { + assert!(text.contains(kept) && !text.contains(gone), "{text}"); + } + } +- // A field of any other type is left to the memory plan: its load stays. +- let text = body_text(&picked_field(Some(1), 1).unwrap()); +- assert!(text.contains("getelementptr"), "{text}"); ++ // A field of any other first-class type reads the same way: the picked struct's own `i64`. ++ for (flag, kept, gone) in [(1, "333", "444"), (0, "444", "333")] { ++ let text = body_text(&picked_field(Some(flag), 1).unwrap()); ++ assert!(!text.contains("getelementptr") && !text.contains("load"), "{text}"); ++ assert!(text.contains(kept) && !text.contains(gone), "{text}"); ++ } + // Unsupplied, the select stays undecided, as before. + assert_eq!( + picked_field(None, 0).unwrap_err(), diff --git a/bridge/pipeline_timing_host.mm b/bridge/pipeline_timing_host.mm new file mode 100644 index 0000000..f010fd4 --- /dev/null +++ b/bridge/pipeline_timing_host.mm @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: MIT +// Host instrument: how long the target's own QuartzCore shaders take through the native frontend (metal2vulkan, then the Vulkan driver given as +// loader: the Adreno Turnip over the host simulation, or MoltenVK). Every _vert_

/ _frag_

pair becomes a BGRA8 render pipeline +// and every kernel a compute pipeline, each timed; with METAL2VULKAN_LOG_PIPELINES=1 the runtime also prints the VkPipeline creation time of each, +// which separates the driver's compile from the translation. The host CPU is not the tablet's: compare the split, not the absolute numbers. +// argv: loader, shader cache dir, QuartzCore default.metallib, [limit], [draw]. With draw=1 each render state made is drawn once (three +// vertices into a 64x64 BGRA8 target, a zeroed buffer at every vertex and fragment index 0..7), which makes its VkPipeline: the runtime +// makes it at the draw, so without a draw the creation time is the translation alone. +#import +#import +#include +#include +#include +#include +#include +#include +#include "NativeBridgeMetalFrontend.h" + +extern "C" size_t native_bridge_metal_telemetry(char *, size_t); + +static double nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1e6;} +static void missing(const char *c, const char *s) {std::fprintf(stderr, "missing %s %s\n", c, s);} + +int main(int argc, char **argv) { + if (argc < 4) {std::fprintf(stderr, "usage: loader cache metallib [limit]\n"); return 2;} + @autoreleasepool { + const unsigned limit = argc > 4 ? (unsigned)std::atoi(argv[4]) : 0; + const bool draw = argc > 5 && std::atoi(argv[5]) == 1; + double t0 = nowMs(); + id device = native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed\n"); return 1;} + std::printf("open %.1f ms\n", nowMs() - t0); + // draw=2: the plugin's on-device benchmark (native_bridge_metal_benchmark) with the same pair count, for the host side of the comparison. + if (argc > 5 && std::atoi(argv[5]) == 2) { + static char rows[512]; + native_bridge_metal_benchmark(device, argv[3], limit ? limit : 8, rows, sizeof rows); + std::printf("%s", rows); + return 0; + } + NSError *error = nil; + t0 = nowMs(); + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:@(argv[3])] error:&error]; + if (!qc) {std::printf("library failed: %s\n", error.localizedDescription.UTF8String); return 1;} + NSArray *names = qc.functionNames; + std::printf("library %.1f ms, %lu functions\n", nowMs() - t0, (unsigned long)names.count); + std::vector function, pipeline, drawn; + id queue = [device newCommandQueue]; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:64 height:64 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id target = [device newTextureWithDescriptor:td]; + id zeros = [device newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + unsigned made = 0, refused = 0, n = 0; + for (NSString *name in names) { + if (limit && n >= limit) break; + double f0 = nowMs(); + id f = [qc newFunctionWithName:name]; + double f1 = nowMs(); + function.push_back(f1 - f0); + if (f.functionType == MTLFunctionTypeKernel) { + ++n; + double p0 = nowMs(); + id state = [device newComputePipelineStateWithFunction:f error:&error]; + double p1 = nowMs(); + pipeline.push_back(p1 - p0); + std::printf("K %-60s fn=%7.1f pipe=%8.1f %s\n", name.UTF8String, f1 - f0, p1 - p0, state ? "ok" : error.localizedDescription.UTF8String); + state ? ++made : ++refused; + } else if (f.functionType == MTLFunctionTypeVertex) { + NSString *fragmentName = [name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if ([fragmentName isEqualToString:name] || ![names containsObject:fragmentName]) continue; + ++n; + double g0 = nowMs(); + id fragment = [qc newFunctionWithName:fragmentName]; + double g1 = nowMs(); + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = f; + d.fragmentFunction = fragment; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + double p0 = nowMs(); + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + double p1 = nowMs(); + pipeline.push_back(p1 - p0); + double took = 0; + if (state && draw) { + double d0 = nowMs(); + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + id commands = [queue commandBuffer]; + id encoder = [commands renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:state]; + for (NSUInteger i = 0; i < 8; ++i) { + [encoder setVertexBuffer:zeros offset:0 atIndex:i]; + [encoder setFragmentBuffer:zeros offset:0 atIndex:i]; + } + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; + [encoder endEncoding]; + [commands commit]; + [commands waitUntilCompleted]; + took = nowMs() - d0; + drawn.push_back(took); + } + std::printf("R %-60s fn=%7.1f+%7.1f pipe=%8.1f draw=%8.1f %s\n", name.UTF8String, f1 - f0, g1 - g0, p1 - p0, took, + state ? "ok" : error.localizedDescription.UTF8String); + state ? ++made : ++refused; + } + std::fflush(stdout); + } + std::sort(pipeline.begin(), pipeline.end()); + double total = 0; + for (double v : pipeline) total += v; + double ftotal = 0; + for (double v : function) ftotal += v; + if (!pipeline.empty()) + std::printf("SUMMARY pipelines=%zu made=%u refused=%u total=%.0f ms median=%.1f ms p90=%.1f ms max=%.1f ms | functions=%zu total=%.0f ms\n", + pipeline.size(), made, refused, total, pipeline[pipeline.size() / 2], pipeline[pipeline.size() * 9 / 10], pipeline.back(), + function.size(), ftotal); + if (!drawn.empty()) { + std::sort(drawn.begin(), drawn.end()); + double dt = 0; + for (double v : drawn) dt += v; + static char table[8192]; + native_bridge_metal_telemetry(table, sizeof table); + for (char *line = std::strtok(table, "\n"); line; line = std::strtok(nullptr, "\n")) + if (line[0] == 'V' || line[0] == 'W' || line[0] == 'S') std::printf("TELEMETRY %s\n", line); + std::printf("DRAWS n=%zu total=%.0f ms median=%.1f ms max=%.1f ms\n", drawn.size(), dt, drawn[drawn.size() / 2], drawn.back()); + } + } + return 0; +} diff --git a/bridge/plugin_sequence_host.mm b/bridge/plugin_sequence_host.mm new file mode 100644 index 0000000..6dafabe --- /dev/null +++ b/bridge/plugin_sequence_host.mm @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +// The plugin's bridge phase (NativeMetalBridgePhase.h) in order over the host simulation: the bridge self-test, then the native frontend on a +// second device over the same Turnip and the frontend checks. The software CP executes nothing, so results are wrong by design; this +// proves every step builds, submits and retires over the host kernel stack and that the second device opens after the first is gone. +// IOSurface steps are skipped (the simulation's IOKit shim hides that service). Prints "frontend failed first <..> last ". +#include "bridge/FrontendChecks.h" +#include "bridge/NativeMetalBridge.h" +#include "bridge/NativeMetalObjects.h" +#include "NativeBridgeMetalFrontend.h" +static void mark(uint32_t v) {std::printf("mark %x\n", v); + std::fflush(stdout);} +static void missing(const char *c, const char *s) {std::printf("missing %s %s\n", c, s);} +int main(int, char **argv) { + NmbSelftest r{}; + uint32_t code = nmb_selftest(argv[1], argv[2], nullptr, 0, mark, &r); + std::printf("selftest code %x stage %x msg %s\n", code, r.stage, r.message); + id device = native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed: %s\n", nmb_last_error()); return 1;} + size_t n = 0; + const void *a = nmb_fixture_air(&n); + id lib = native_bridge_metal_library_with_air(a, n); + a = nmb_function_constant_air(&n); + id fc = native_bridge_metal_library_with_air(a, n); + frontendchecks::Report rep = {}; + rep.mark = mark; + rep.hostSimulation = true; + frontendchecks::run(device, lib, fc, nil, &rep); + std::printf("frontend failed %u first %s last %x\n", rep.failed, rep.first, rep.last); + return 0; +} diff --git a/bridge/present_host.mm b/bridge/present_host.mm new file mode 100644 index 0000000..23a91ef --- /dev/null +++ b/bridge/present_host.mm @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +// Host instrument for drivers/NativeMetal/GpuScanoutPresenter.h: anonymous pages stand for the scanout and for the compositor's surface, +// the GPU copy runs on the native frontend's device (bridge over MoltenVK) and every pixel of the scanout is compared with the surface. +// With NMB_SYSTEM_DEVICE=1 the Mac's own device runs the same code as the baseline. This is a host check of the copy's contract, not an +// Adreno run and not the kernel presenter. +#import +#import +#include +#include +#include +#include "../GpuScanoutPresenter.h" +#include "../NativeBridgeMetalFrontend.h" + +static void *pages(size_t bytes) { + void *p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + return p == MAP_FAILED ? nullptr : p; +} +static size_t roundUp(size_t n) {const size_t page = (size_t)getpagesize(); return (n + page - 1) & ~(page - 1);} + +// One frame at a given surface pitch and scanout pitch; every pixel of the scanout must equal the surface's pixel, padding must stay untouched. +static bool run(id device, uint32_t w, uint32_t h, size_t srcStride, size_t dstStride, uint32_t seed, const char *name) { + const size_t srcBytes = roundUp(srcStride * h), dstBytes = roundUp(dstStride * h); + uint8_t *src = (uint8_t *)pages(srcBytes), *dst = (uint8_t *)pages(dstBytes); + if (!src || !dst) {std::printf("FAIL %s pages\n", name); return false;} + for (size_t i = 0; i < srcBytes; ++i) src[i] = (uint8_t)((i * 131 + seed) ^ (i >> 7)); + for (size_t i = 0; i < dstBytes; ++i) dst[i] = 0xa5; + gpupresent::Presenter presenter(device, {dst, dstBytes, dstStride, w, h}); + if (!presenter.ready()) {std::printf("FAIL %s scanout import\n", name); return false;} + const gpupresent::Result r = presenter.present({src, srcBytes, srcStride, w, h}); + if (r != gpupresent::kOk) {std::printf("FAIL %s present %d\n", name, (int)r); return false;} + for (uint32_t y = 0; y < h; ++y) { + if (memcmp(dst + y * dstStride, src + y * srcStride, (size_t)w * 4)) {std::printf("FAIL %s row %u differs\n", name, y); return false;} + for (size_t x = (size_t)w * 4; x < dstStride; ++x) + if (dst[y * dstStride + x] != 0xa5) {std::printf("FAIL %s row %u padding written at %zu\n", name, y, x); return false;} + } + std::printf("ok %s %ux%u pitch %zu->%zu\n", name, w, h, srcStride, dstStride); + munmap(src, srcBytes); munmap(dst, dstBytes); + return true; +} + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = getenv("NMB_SYSTEM_DEVICE") ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + bool ok = true; + ok &= run(device, 256, 64, 256 * 4, 256 * 4, 1, "equal pitch"); + ok &= run(device, 250, 61, 250 * 4 + 8, 256 * 4, 2, "surface pitch smaller"); + ok &= run(device, 250, 61, 256 * 4, 250 * 4 + 40, 3, "scanout pitch smaller"); + ok &= run(device, 254, 33, 254 * 4 + 8, 254 * 4 + 64, 4, "unaligned pitches"); + if (!ok) return 1; + std::printf("PASS: the GPU copied every pixel of four frames into the scanout pages, padding untouched\n"); + } + return 0; +} diff --git a/bridge/priming_cache_host.cpp b/bridge/priming_cache_host.cpp new file mode 100644 index 0000000..cbf9332 --- /dev/null +++ b/bridge/priming_cache_host.cpp @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* Device open/close only. Actual Turnip compiler; mandatory software peer. + * No authored command, shader execution, timing or hardware claim. */ +#include "NativeMetalObjects.h" +#include +#include +#include +#include +#include +#include +#include +#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL priming line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while(0) +int main(int argc,char **argv) { + if(argc!=4)return 2; + const bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),uuid=!std::strcmp(argv[3],"uuid-miss"),cold=!std::strcmp(argv[3],"cold-miss"); + REQUIRE(produce||consume||uuid||cold); +#if TARGET_OS_IPHONE + REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE")); +#endif + REQUIRE(!setenv("NMB_TEST_REAL_VULKAN_LOADER",argv[1],1) && !setenv("MESA_SHADER_CACHE_DISABLE","true",1) && !setenv("NMB_TEST_FORCE_PRIME_STRICT",produce?"0":"1",1)); + const char *path=getenv("NMB_TEST_PRIMING_CACHE_LOADER");REQUIRE(path); + void *loader=dlopen(path,RTLD_NOW|RTLD_LOCAL);REQUIRE(loader); + auto control=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_control")); + auto graphics=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_counts")); + auto compute=reinterpret_cast(dlsym(loader,"nmb_test_prime_compute_counts")); + auto live=reinterpret_cast(dlsym(RTLD_DEFAULT,"host_gpu_live_objects")); + auto submitted=reinterpret_cast(dlsym(RTLD_DEFAULT,"host_gpu_submissions")); + REQUIRE(control&&graphics&&compute&&live&&submitted);control(false,uuid); + NmbDevice *device=nmb_device_open(path,argv[2]);uint64_t c[5]={},g[10]={};compute(c);graphics(g); + if(cold||uuid){REQUIRE(!device && c[2]>0);if(uuid)REQUIRE(g[5]>0);} + else { + REQUIRE(device && c[0] && c[1]==c[0] && !c[2] && c[3]==c[0] && g[0] && g[1]==g[0] && !g[2] && g[3]==g[0]); + if(consume)REQUIRE(c[4]==c[0] && g[4]==g[0]); + nmb_device_release(device); + } + REQUIRE(live()==0); + std::printf("PASS: device priming %s pid=%d compute=%llu/%llu/%llu/%llu/%llu graphics=%llu/%llu/%llu/%llu/%llu uuid_mutations=%llu submissions=%u live=0; software peer, no shader execution\n",argv[3],getpid(), + (unsigned long long)c[0],(unsigned long long)c[1],(unsigned long long)c[2],(unsigned long long)c[3],(unsigned long long)c[4], + (unsigned long long)g[0],(unsigned long long)g[1],(unsigned long long)g[2],(unsigned long long)g[3],(unsigned long long)g[4],(unsigned long long)g[5],submitted()); +} diff --git a/bridge/priming_cache_ios.cpp b/bridge/priming_cache_ios.cpp new file mode 100644 index 0000000..ef857af --- /dev/null +++ b/bridge/priming_cache_ios.cpp @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: MIT */ +/* Owned offline QEMU harness for the real iOS device-priming cache. Shared + * helpers retain exact RAM-volume identity, readonly view and bounded capture. + * The optional first-consumer marker is part of the hashed initial test image. */ +#define main unused_ui_cold_suite_main +#include "quartzcore_cache_ios.cpp" +#undef main + +int main(int argc,char **argv) { + (void)argv;prefix="/private/var/tmp/priming";workerLog="/private/var/tmp/priming-worker.log"; + bool first=false; + if(FILE *file=fopen("/private/var/tmp/priming-first-consumer","rb")) { + char marker[32]={};const auto n=fread(marker,1,sizeof marker,file);fclose(file); + if(n!=strlen("priming-first-v1\n") || memcmp(marker,"priming-first-v1\n",n))stop(96); + first=true; + } else if(errno!=ENOENT)stop(97); + result[0]=first?0x5052494d494f5332ull:0x5052494d494f5331ull; + result[2]=getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE); + if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1); + mountCompilerView(); + if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") || + setenv("NMB_TEST_PRIMING_CACHE_LOADER","/System/Library/Extensions/OfflineTurnipCompiler/libOfflinePrimingCacheControl.dylib",1))stop(2); + const std::string cache=std::string(prefix)+".cache",empty=std::string(prefix)+".empty",copy=std::string(prefix)+".archive"; + std::array initial={}; + if(first){summarize();memcpy(initial.data(),snapshot,initial.size());} + else {struct stat old={};if(!lstat(copy.c_str(),&old) || errno!=ENOENT)stop(3);} + if(unlink(workerLog) && errno!=ENOENT)stop(5); + posix_spawn_file_actions_t actions; + if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) || + posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4); + const char *coldModes[]={"cold-miss","produce","consume","uuid-miss"},*firstModes[]={"consume","uuid-miss","cold-miss"}; + const unsigned cases=first?3:4; + for(unsigned i=0;i("compiler-worker"),const_cast(driverPath),const_cast(directory.c_str()),const_cast(mode),nullptr}; + pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ); + result[12+3*i]=child;result[13+3*i]=error;if(error || child<=1)stop(10+i); + int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i); + result[14+3*i]=status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1; + } + posix_spawn_file_actions_destroy(&actions); + // A diagnostic duplicate of the sole actual driver record, not a Metal + // binary archive. The full directory is independently hashed by summarize. + if(!first) { + std::error_code ec;std::filesystem::path record;unsigned files=0; + std::filesystem::recursive_directory_iterator iterator(cache,ec),end;if(ec)stop(98); + for(;iterator!=end;iterator.increment(ec)) { + if(ec)stop(99);const auto status=iterator->symlink_status(ec); + if(ec || std::filesystem::is_symlink(status))stop(100); + if(std::filesystem::is_directory(status))continue; + if(!std::filesystem::is_regular_file(status))stop(101); + record=iterator->path();++files; + } + if(ec || files!=1 || !std::filesystem::copy_file(record,copy,std::filesystem::copy_options::none,ec) || ec)stop(102); + } + summarize(); + if(first) { + if(memcmp(initial.data(),snapshot,initial.size()))stop(103); + memcpy(snapshot+256,initial.data(),initial.size());result[6]=384;result[45]=2;result[46]=1; + } + stop(0); +} diff --git a/bridge/priming_cache_loader.cpp b/bridge/priming_cache_loader.cpp new file mode 100644 index 0000000..405990c --- /dev/null +++ b/bridge/priming_cache_loader.cpp @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* Offline only. Preserve the existing graphics/UUID counters and forward real + * calls, additionally forbid/observe compute and graphics during device open. */ +#define vkGetInstanceProcAddr baselineGetInstanceProcAddr +#define vkGetDeviceProcAddr baselineGetDeviceProcAddr +#include "graphics_cache_loader.cpp" +#undef vkGetInstanceProcAddr +#undef vkGetDeviceProcAddr + +static std::atomic computeCounts[5]; // calls, successes, required, valid, app hits +static bool forceCompilationForbidden() { + const char *value=std::getenv("NMB_TEST_FORCE_PRIME_STRICT");return value && !std::strcmp(value,"1"); +} +static bool hasFlags2(const void *next) { + for(auto p=static_cast(next);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR)return true; + return false; +} +static VKAPI_ATTR VkResult VKAPI_CALL primeGraphics(VkDevice device,VkPipelineCache cache,uint32_t count, + const VkGraphicsPipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) { + if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED; + std::vector copies(infos,infos+count); + if(forceCompilationForbidden())for(auto &info:copies) { + // Explicit refusal rather than silently missing flags promoted to pNext. + if(hasFlags2(info.pNext))return VK_ERROR_FEATURE_NOT_PRESENT; + info.flags|=VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT; + } + return createGraphics(device,cache,count,copies.data(),allocator,pipelines); +} +static VKAPI_ATTR VkResult VKAPI_CALL primeCompute(VkDevice device,VkPipelineCache cache,uint32_t count, + const VkComputePipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) { + if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED; + std::vector copies(infos,infos+count); + std::vector feedback(count); + std::vector additions(count); + std::vector observed(count); + for(uint32_t i=0;i(copies[i].pNext);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO) + observed[i]=reinterpret_cast(p)->pPipelineCreationFeedback; + if(!observed[i]) { + auto &a=additions[i];a.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO; + a.pNext=copies[i].pNext;a.pPipelineCreationFeedback=&feedback[i];copies[i].pNext=&a;observed[i]=&feedback[i]; + } + } + auto real=reinterpret_cast(realDevice.load()(device,"vkCreateComputePipelines")); + computeCounts[0]+=count;const auto result=real(device,cache,count,copies.data(),allocator,pipelines); + if(result==VK_PIPELINE_COMPILE_REQUIRED)computeCounts[2]++; + for(uint32_t i=0;iflags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) { + computeCounts[3]++; + if(observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT)computeCounts[4]++; + } + } + return result; +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) { + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(primeGraphics); + if(!std::strcmp(name,"vkCreateComputePipelines"))return reinterpret_cast(primeCompute); + return baselineGetDeviceProcAddr(device,name); +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) { + const auto result=baselineGetInstanceProcAddr(instance,name); + if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast(vkGetDeviceProcAddr); + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(primeGraphics); + if(!std::strcmp(name,"vkCreateComputePipelines"))return reinterpret_cast(primeCompute); + return result; +} +extern "C" void nmb_test_prime_compute_counts(uint64_t out[5]) {for(unsigned i=0;i<5;++i)out[i]=computeCounts[i].load();} diff --git a/bridge/quartzcore_blur_host.mm b/bridge/quartzcore_blur_host.mm new file mode 100644 index 0000000..3049353 --- /dev/null +++ b/bridge/quartzcore_blur_host.mm @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT +// Execute unchanged target blur functions on a real host GPU. The reference +// is an authored seven-tap binomial convolution, not a deployment renderer. +#import +#import +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +constexpr unsigned width=32,height=24; +struct Vertex {float position[4],uv[2],unusedColor[4];}; +static const Vertex vertices[]={ + {{-1,-1,0,1},{0,1},{}},{{1,-1,0,1},{1,1},{}},{{-1,1,0,1},{0,0},{}}, + {{-1,1,0,1},{0,0},{}},{{1,-1,0,1},{1,1},{}},{{1,1,0,1},{1,0},{}}}; +struct alignas(16) UniformF {float offset[8][2],weight[8],divide;unsigned char padding[12];}; +struct alignas(8) UniformH {float offset[8][2];_Float16 weight[8],divide;unsigned char padding[6];}; +static_assert(sizeof(UniformF)==112 && sizeof(UniformH)==88); + +static MTLRenderPipelineDescriptor *blurDescriptor(id library,bool half) { + NSString *suffix=half?@"lph":@"lpf";auto d=[MTLRenderPipelineDescriptor new]; + d.vertexFunction=[library newFunctionWithName:[@"downsample_blur_vert_" stringByAppendingString:suffix]]; + d.fragmentFunction=[library newFunctionWithName:[@"narrow_blur_7_frag_" stringByAppendingString:suffix]]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatRGBA16Float; + auto vd=[MTLVertexDescriptor vertexDescriptor]; + const unsigned indices[]={0,1,3},offsets[]={0,16,24}; + const MTLVertexFormat formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat4}; + for(unsigned i=0;i<3;++i){vd.attributes[indices[i]].format=formats[i];vd.attributes[indices[i]].offset=offsets[i];vd.attributes[indices[i]].bufferIndex=0;} + vd.layouts[0].stride=sizeof(Vertex);d.vertexDescriptor=vd;return d; +} +static MTLRenderPipelineDescriptor *namedDescriptor(id library,NSString *vertex,NSString *fragment) { + auto d=[MTLRenderPipelineDescriptor new];d.vertexFunction=[library newFunctionWithName:vertex];d.fragmentFunction=[library newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm; + auto vd=[MTLVertexDescriptor vertexDescriptor]; + const MTLVertexFormat formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat2,MTLVertexFormatFloat4,MTLVertexFormatFloat}; + const unsigned offsets[]={0,16,24,32,48}; + for(unsigned i=0;i<5;++i){vd.attributes[i].format=formats[i];vd.attributes[i].bufferIndex=0;vd.attributes[i].offset=offsets[i];} + vd.layouts[0].stride=64;d.vertexDescriptor=vd; + if([vertex hasPrefix:@"path_blit_"]){d.colorAttachments[1].pixelFormat=MTLPixelFormatBGRA8Unorm;d.colorAttachments[2].pixelFormat=MTLPixelFormatR32Float;} + return d; +} +static bool cacheState(const char *phase) { + NmbShaderCacheStats s={};uint64_t work[10]={}; + if(!native_bridge_metal_shader_cache_stats(&s) || !native_bridge_metal_work_stats(work))return false; + std::printf("cache %s artifact_hit=%llu miss=%llu write=%llu driver_load=%llu hit=%llu miss=%llu save=%llu same=%llu vk_makings=%llu\n",phase, + (unsigned long long)s.hits,(unsigned long long)s.misses,(unsigned long long)s.writes, + (unsigned long long)s.driver_loads,(unsigned long long)s.driver_hits,(unsigned long long)s.driver_misses, + (unsigned long long)s.driver_saves,(unsigned long long)s.driver_unchanged,(unsigned long long)work[0]); + return !s.rejected && !s.read_failures && !s.write_failures && !s.driver_truncated && !s.driver_mismatched && !work[4]; +} +static bool inventory(id device,id library) { + unsigned vertex=0,fragment=0,kernels=0,ordinary=0,tile=0,pairs=0,bare=0,attributes=0,attachments=0; + for(NSString *name in library.functionNames) { + id f=[library newFunctionWithName:name];if(!f)return false; + NSError *error=nil; + if(f.functionType==MTLFunctionTypeKernel) { + ++kernels; + if([device newComputePipelineStateWithFunction:f error:&error])++ordinary; + else if([error.localizedDescription containsString:@"NeedsInput(TileAttachments)"])++tile; + else {std::printf("FAIL unclassified compute %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return false;} + } else if(f.functionType==MTLFunctionTypeFragment)++fragment; + else if(f.functionType==MTLFunctionTypeVertex) { + ++vertex;NSString *other=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if([other isEqualToString:name] || ![library.functionNames containsObject:other])continue; + ++pairs;auto d=[MTLRenderPipelineDescriptor new];d.vertexFunction=f;d.fragmentFunction=[library newFunctionWithName:other]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm; + if([device newRenderPipelineStateWithDescriptor:d error:&error])++bare; + else if([error.localizedDescription containsString:@"InvalidDescriptor(VertexAttribute"])++attributes; + else if([error.localizedDescription containsString:@"InvalidDescriptor(ColorAttachment"])++attachments; + else {std::printf("FAIL unclassified render %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return false;} + } else return false; + } + std::printf("inventory functions=167 vertex=%u fragment=%u kernels=%u ordinary=%u tile_requires=%u pairs=%u bare_ok=%u missing_vertex=%u missing_attachment=%u\n", + vertex,fragment,kernels,ordinary,tile,pairs,bare,attributes,attachments); + return vertex==41 && fragment==111 && kernels==15 && ordinary==4 && tile==11 && pairs==22 && bare==14 && attributes==6 && attachments==2; +} +static std::vector<_Float16> convolution(const std::vector<_Float16> &input,bool vertical) { + // Independent integer stencil: coefficients sum to 64, clamp at the edge. + constexpr unsigned weight[]={1,6,15,20,15,6,1};std::vector<_Float16> result(input.size()); + for(unsigned y=0;y command,id state,id source, + id destination,id sampler,bool half,bool vertical) { + auto pass=[MTLRenderPassDescriptor renderPassDescriptor];pass.colorAttachments[0].texture=destination; + pass.colorAttachments[0].loadAction=MTLLoadActionDontCare;pass.colorAttachments[0].storeAction=MTLStoreActionStore; + id encoder=[command renderCommandEncoderWithDescriptor:pass];if(!encoder)return false; + [encoder setRenderPipelineState:state];[encoder setVertexBytes:vertices length:sizeof vertices atIndex:0]; + const float identity[]={1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1},texmat[]={1,1,0,0}; + [encoder setVertexBytes:identity length:sizeof identity atIndex:2];[encoder setVertexBytes:texmat length:sizeof texmat atIndex:3]; + [encoder setFragmentTexture:source atIndex:3];[encoder setFragmentSamplerState:sampler atIndex:0]; + // Two bilinear pairs exactly represent the seven authored stencil taps. + UniformF f={};UniformH h={}; + f.offset[0][vertical?1:0]=0.6f/(vertical?height:width); + f.offset[1][vertical?1:0]=(15.0f/7)/(vertical?height:width); + f.weight[0]=25.0f/64;f.weight[1]=7.0f/64; + if(half) { + memcpy(h.offset,f.offset,sizeof h.offset);h.weight[0]=(_Float16)f.weight[0];h.weight[1]=(_Float16)f.weight[1]; + const _Float16 scale=1;[encoder setFragmentBytes:&h length:sizeof h atIndex:1];[encoder setFragmentBytes:&scale length:sizeof scale atIndex:6]; + } else { + const float scale=1;[encoder setFragmentBytes:&f length:sizeof f atIndex:1];[encoder setFragmentBytes:&scale length:sizeof scale atIndex:6]; + } + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6];[encoder endEncoding];return true; +} +static bool runBlur(id device,id state,bool half) { + std::vector<_Float16> data(width*height*4); + for(unsigned y=0;y input=[device newTextureWithDescriptor:td],middle=[device newTextureWithDescriptor:td],output=[device newTextureWithDescriptor:td]; + auto sd=[MTLSamplerDescriptor new];sd.minFilter=MTLSamplerMinMagFilterLinear;sd.magFilter=MTLSamplerMinMagFilterLinear; + sd.sAddressMode=MTLSamplerAddressModeClampToEdge;sd.tAddressMode=MTLSamplerAddressModeClampToEdge; + id sampler=[device newSamplerStateWithDescriptor:sd];id queue=[device newCommandQueue]; + if(!input || !middle || !output || !sampler || !queue)return false; + [input replaceRegion:MTLRegionMake2D(0,0,width,height) mipmapLevel:0 withBytes:data.data() bytesPerRow:width*8]; + float worst=0; + for(unsigned round=0;round<6;++round){@autoreleasepool { + NmbShaderCacheStats before={},after={};uint64_t workBefore[10]={},workAfter[10]={}; + if(!native_bridge_metal_shader_cache_stats(&before) || !native_bridge_metal_work_stats(workBefore))return false; + id command=[queue commandBuffer]; + if(!encode(command,state,input,middle,sampler,half,false) || !encode(command,state,middle,output,sampler,half,true))return false; + [command commit];[command waitUntilCompleted]; + if(command.status!=MTLCommandBufferStatusCompleted){std::printf("FAIL blur command: %s\n",command.error.localizedDescription.UTF8String);return false;} + std::vector<_Float16> got(data.size());[output getBytes:got.data() bytesPerRow:width*8 fromRegion:MTLRegionMake2D(0,0,width,height) mipmapLevel:0]; + for(size_t i=0;i0.003f){std::printf("FAIL blur %s round=%u element=%zu got=%g expected=%g\n",half?"half":"float",round,i,(double)got[i],(double)expected[i]);return false;} + worst=std::max(worst,error); + } + if(!native_bridge_metal_shader_cache_stats(&after) || !native_bridge_metal_work_stats(workAfter))return false; + std::printf("round %s %u vk_makings=%llu artifact_hit=%llu miss=%llu driver_load=%llu hit=%llu miss=%llu\n",half?"half":"float",round, + (unsigned long long)(workAfter[0]-workBefore[0]),(unsigned long long)(after.hits-before.hits),(unsigned long long)(after.misses-before.misses), + (unsigned long long)(after.driver_loads-before.driver_loads),(unsigned long long)(after.driver_hits-before.driver_hits),(unsigned long long)(after.driver_misses-before.driver_misses)); + if(round && (workAfter[0]!=workBefore[0] || after.misses!=before.misses || after.driver_misses!=before.driver_misses)) { + std::puts("FAIL repeated identical blur performs new pipeline/translation work");return false; + } + }} + std::printf("blur %s rounds=6 passes=12 texels=%u max_error=%g GPU=pass\n",half?"half":"float",width*height,worst);return true; +} + +int main(int argc,char **argv) { + if(argc!=6 || (strcmp(argv[5],"produce") && strcmp(argv[5],"consume") && strcmp(argv[5],"inventory")))return 2; + if(setenv("METAL2VULKAN_LOG_PIPELINES","1",1))return 2; + const bool produce=!strcmp(argv[5],"produce"); + @autoreleasepool { + id device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2; + NSError *error=nil;id library=[device newLibraryWithFile:@(argv[3]) error:&error];if(!library)return 2; + if(library.functionNames.count!=167)return 1; + if(!strcmp(argv[5],"inventory"))return inventory(device,library)?0:1; + auto ad=[MTLBinaryArchiveDescriptor new];NSURL *url=[NSURL fileURLWithPath:@(argv[4])];if(!produce)ad.url=url; + id archive=[device newBinaryArchiveWithDescriptor:ad error:&error];if(!archive)return 1; + if(!cacheState("opened"))return 1; + if(produce) { + auto empty=blurDescriptor(library,false);empty.binaryArchives=@[archive];error=nil; + if([device newRenderPipelineStateWithDescriptor:empty options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error)return 1; + } + unsigned pairs=0; + for(NSString *name in library.functionNames) { + id function=[library newFunctionWithName:name];if(function.functionType!=MTLFunctionTypeVertex)continue; + NSString *fragment=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if([fragment isEqualToString:name] || ![library.functionNames containsObject:fragment])continue; + auto d=namedDescriptor(library,name,fragment);d.binaryArchives=@[archive]; + if(produce && ![archive addRenderPipelineFunctionsWithDescriptor:d error:&error]){ + std::printf("FAIL archive add %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return 1;} + error=nil; + if(![device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || error){ + std::printf("FAIL strict pair %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return 1;} + ++pairs; + } + if(pairs!=22)return 1; + std::printf("valid descriptor pairs=%u strict archive=pass\n",pairs); + for(bool half:{false,true}) { + auto d=blurDescriptor(library,half);d.binaryArchives=@[archive]; + if(produce && ![archive addRenderPipelineFunctionsWithDescriptor:d error:&error])return 1; + error=nil;id state=[device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if(!state){std::printf("FAIL blur pipeline: %s\n",error.localizedDescription.UTF8String);return 1;} + const auto format=d.colorAttachments[0].pixelFormat;d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm;error=nil; + if([device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error)return 1; + d.colorAttachments[0].pixelFormat=format; + if(!runBlur(device,state,half))return 1; + } + if(produce && ![archive serializeToURL:url error:&error])return 1; + if(!cacheState("finished"))return 1; + uint64_t work[10]={};if(!native_bridge_metal_work_stats(work) || !work[0])return 1; // logging must be active, never a zero-counter shortcut + std::puts("PASS: unchanged target blur functions execute repeated separable Gaussian reference cases on the host GPU"); + } +} diff --git a/bridge/quartzcore_cache.mm b/bridge/quartzcore_cache.mm new file mode 100644 index 0000000..5b24ca8 --- /dev/null +++ b/bridge/quartzcore_cache.mm @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: MIT +// Actual compiler/cache instrument. No draw is encoded. The runtime may initialize +// neutral resources over the mandatory software peer; it executes no shader. +#import +#include +#include +#include "NativeMetalObjects.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL UI cache line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while(0) +static NmbRenderPipelineDescriptor descriptor(NmbLibrary *library,const std::string &vertex,const std::string &fragment,bool blur) { + NmbRenderPipelineDescriptor d={};d.vertex=nmb_function_new(library,vertex.c_str());d.fragment=nmb_function_new(library,fragment.c_str()); + d.colors[0].format=blur?MTLPixelFormatRGBA16Float:MTLPixelFormatBGRA8Unorm; + if(vertex.rfind("path_blit_",0)==0){d.colors[1].format=MTLPixelFormatBGRA8Unorm;d.colors[2].format=MTLPixelFormatR32Float;} + for(auto &c:d.colors)if(c.format){c.write_mask=MTLColorWriteMaskAll;c.source_rgb=MTLBlendFactorOne;c.source_alpha=MTLBlendFactorOne;} + d.samples=1;d.rasterization=1;d.vertex_inputs=1; + if(blur) { + d.attributes[0]={MTLVertexFormatFloat4,0,0};d.attributes[1]={MTLVertexFormatFloat2,16,0};d.attributes[3]={MTLVertexFormatFloat4,24,0}; + d.layouts[0]={40,MTLVertexStepFunctionPerVertex,1}; + } else { + const uint64_t formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat2,MTLVertexFormatFloat4,MTLVertexFormatFloat}; + const uint64_t offsets[]={0,16,24,32,48};for(unsigned i=0;i<5;++i)d.attributes[i]={formats[i],offsets[i],0}; + d.layouts[0]={64,MTLVertexStepFunctionPerVertex,1}; + } + return d; +} +int main(int argc,char **argv) { + if(argc!=4)return 2; + const bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),uuid=!std::strcmp(argv[3],"uuid-miss"),empty=!std::strcmp(argv[3],"empty-driver"); + REQUIRE(produce||consume||uuid||empty); +#if TARGET_OS_IPHONE + REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE")); +#endif + REQUIRE(!setenv("MESA_SHADER_CACHE_DISABLE","true",1) && !setenv("METAL2VULKAN_LOG_PIPELINES","1",1) && !setenv("NMB_TEST_REAL_VULKAN_LOADER",argv[1],1)); + std::string path=argv[1];path=path.substr(0,path.find_last_of('/')+1)+"libOfflineGraphicsCacheControl.dylib"; + if(const char *override=getenv("NMB_TEST_GRAPHICS_CACHE_LOADER"))path=override; + void *loader=dlopen(path.c_str(),RTLD_NOW|RTLD_LOCAL);REQUIRE(loader); + auto control=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_control")); + auto counters=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_counts")); + auto submitted=reinterpret_cast(dlsym(RTLD_DEFAULT,"host_gpu_submissions"));REQUIRE(control&&counters&&submitted); + const unsigned submissions=submitted();control(false,false); + const std::string prefix=argv[2],archivePath=prefix+".archive",cachePath=prefix+(empty?".empty":".cache"); + NmbDevice *device=nmb_device_open(path.c_str(),cachePath.c_str());REQUIRE(device); + uint64_t deviceCounts[10]={};counters(deviceCounts);control(!produce,uuid); + const char *libraryPath="/System/Library/Frameworks/QuartzCore.framework/default.metallib"; + if(const char *override=getenv("NMB_TEST_QUARTZCORE_LIBRARY"))libraryPath=override; + std::ifstream input(libraryPath,std::ios::binary);REQUIRE(input); + std::vector bytes(9670832);input.read(bytes.data(),bytes.size());REQUIRE(input.good() && input.peek()==EOF); + unsigned char digest[CC_SHA256_DIGEST_LENGTH];CC_SHA256(bytes.data(),(CC_LONG)bytes.size(),digest); + char digestHex[65]={};for(unsigned i=0;i<32;++i)std::snprintf(digestHex+2*i,3,"%02x",digest[i]); + REQUIRE(!std::strcmp(digestHex,"606f913adf520420f9b4bb666efb782a68c830f082f1e559e17b20d0d13adbc9")); + NmbLibrary *library=nmb_library_new(device,bytes.data(),bytes.size(),false);REQUIRE(library&&nmb_library_function_count(library)==167); + std::vector names;for(unsigned i=0;i<167;++i){char name[256]={};REQUIRE(nmb_library_function_name(library,i,name,sizeof name) recipes; + for(const auto &name:names) { + const auto marker=name.find("_vert_");if(marker==std::string::npos)continue; + auto fragment=name;fragment.replace(marker,6,"_frag_");if(std::find(names.begin(),names.end(),fragment)==names.end())continue; + recipes.push_back(descriptor(library,name,fragment,false)); + } + REQUIRE(recipes.size()==22); + for(const auto *suffix:{"lpf","lph"})recipes.push_back(descriptor(library,std::string("downsample_blur_vert_")+suffix,std::string("narrow_blur_7_frag_")+suffix,true)); + for(const auto &d:recipes)REQUIRE(d.vertex&&d.fragment&&nmb_function_type(d.vertex)==1&&nmb_function_type(d.fragment)==2); + NmbBinaryArchive *archive=nmb_binary_archive_new(device,produce?nullptr:archivePath.c_str());REQUIRE(archive); + NmbPipelineOptions options={MTLPipelineOptionFailOnBinaryArchiveMiss,&archive,1}; + uint64_t coldRequired=0;bool coldObject=false; + if(produce) { + // Exercise the real driver's compile-forbidden cold refusal, separately + // from strict binary-archive misses. Clear only the test policy afterwards. + control(true,false);auto cold=nmb_render_pipeline_new(device,&recipes.back()); + coldObject=cold!=nullptr;if(cold)nmb_render_pipeline_release(cold); + uint64_t counts[10]={};counters(counts);coldRequired=counts[2];REQUIRE(coldRequired>0);control(false,false); + } + uint64_t before[10]={};counters(before);unsigned made=0; + for(const auto &d:recipes) { + if(produce)REQUIRE(nmb_binary_archive_add_render(archive,&d)); + NmbRenderPipeline *state=nmb_render_pipeline_new_with_options(device,&d,&options); + if(uuid||empty){if(state)nmb_render_pipeline_release(state);break;} + REQUIRE(state);nmb_render_pipeline_release(state);++made; + } + uint64_t counts[10]={};counters(counts); + std::fprintf(stderr,"graphics results %s: calls=%llu success=%llu required=%llu valid=%llu hits=%llu cached=%llu/%llu/%llu\n",argv[3], + (unsigned long long)(counts[0]-before[0]),(unsigned long long)(counts[1]-before[1]),(unsigned long long)(counts[2]-before[2]), + (unsigned long long)(counts[3]-before[3]),(unsigned long long)(counts[4]-before[4]),(unsigned long long)(counts[7]-before[7]), + (unsigned long long)(counts[8]-before[8]),(unsigned long long)(counts[9]-before[9])); + if(uuid||empty) {REQUIRE(counts[2]>before[2]);if(uuid)REQUIRE(counts[5]>before[5]);} + else REQUIRE(made==24 && counts[1]>before[1]); + if(consume)REQUIRE(counts[2]==before[2] && counts[7]>before[7] && counts[8]-before[8]==counts[7]-before[7] && counts[9]-before[9]==counts[7]-before[7]); + if(produce)REQUIRE(nmb_binary_archive_serialize(archive,archivePath.c_str())); + NmbShaderCacheStats stats={};REQUIRE(nmb_device_shader_cache_stats(device,&stats)); + REQUIRE(!stats.rejected&&!stats.read_failures&&!stats.write_failures&&!stats.driver_truncated&&!stats.driver_mismatched); + if(consume)REQUIRE(!stats.misses&&!stats.writes&&!stats.driver_misses&&stats.driver_loads); + const unsigned initializationSubmissions=submitted()-submissions; + std::printf("PASS: QuartzCore compiler %s pid=%d recipes=%u calls=%llu success=%llu required=%llu valid=%llu hits=%llu uuid_mutations=%llu devices=%llu artifacts=%llu/%llu/%llu driver=%llu/%llu/%llu/%llu cached_calls=%llu/%llu/%llu device_graphics=%llu cold_required=%llu cold_object=%u init_submissions=%u; no shader execution\n",argv[3],getpid(),made, + (unsigned long long)(counts[0]-before[0]),(unsigned long long)(counts[1]-before[1]),(unsigned long long)(counts[2]-before[2]), + (unsigned long long)(counts[3]-before[3]),(unsigned long long)(counts[4]-before[4]),(unsigned long long)(counts[5]-before[5]),(unsigned long long)counts[6], + (unsigned long long)stats.hits,(unsigned long long)stats.misses,(unsigned long long)stats.writes, + (unsigned long long)stats.driver_loads,(unsigned long long)stats.driver_hits,(unsigned long long)stats.driver_misses,(unsigned long long)stats.driver_saves,(unsigned long long)(counts[7]-before[7]), + (unsigned long long)(counts[8]-before[8]),(unsigned long long)(counts[9]-before[9]),(unsigned long long)deviceCounts[0],(unsigned long long)coldRequired,(unsigned)coldObject,initializationSubmissions); + nmb_binary_archive_release(archive);for(const auto &d:recipes){nmb_function_release(d.vertex);nmb_function_release(d.fragment);} + nmb_library_release(library);nmb_device_release(device); + return 0; +} diff --git a/bridge/quartzcore_cache_ios.cpp b/bridge/quartzcore_cache_ios.cpp new file mode 100644 index 0000000..fb1f62f --- /dev/null +++ b/bridge/quartzcore_cache_ios.cpp @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: MIT */ +/* Owned QEMU-only PID 1 harness. Run the actual iOS UI compiler in four cold + * children, flush own mutable Data, then expose a bounded manifest checkpoint. + * Original dyld/AMFI, signatures, read-only OS and exact RAM-volume selection + * remain in force. The linked software peer executes no shader. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; +static uint64_t result[48]; +alignas(4096) static unsigned char snapshot[4096]; +alignas(4096) static unsigned char workerOutput[4096]; +static const char *workerLog="/private/var/tmp/quartzcore-compiler-worker.log"; +static const char *driverPath="/System/Library/Extensions/OfflineTurnipCompiler/libvulkan_freedreno.dylib"; +static const char *prefix="/private/var/tmp/quartzcore-ui"; + +[[noreturn]] static void stop(unsigned failure) { + result[1]=failure; + if(FILE *file=fopen(workerLog,"rb")) { + result[11]=fread(workerOutput,1,sizeof workerOutput-1,file); + if(fgetc(file)!=EOF || ferror(file))result[11]=sizeof workerOutput; + fclose(file); + } + __asm__ volatile("mov x13, %0\nmov x14, %1\nmov x15, %2\nbrk #0"::"r"(result),"r"(snapshot),"r"(workerOutput):"x13","x14","x15","memory"); + _exit(99); +} +static void mountCompilerView() { + char uuid[37]={};FILE *file=fopen("/private/var/tmp/compiler-data-uuid","rb"); + if(!file)stop(60); + const size_t count=fread(uuid,1,37,file);fclose(file); + if(count!=36 || uuid[36])stop(61); + CFStringRef identity=CFStringCreateWithCString(nullptr,uuid,kCFStringEncodingASCII); + CFMutableDictionaryRef match=IOServiceMatching("IOMedia");if(!identity || !match)stop(62); + const void *keys[]={CFSTR("UUID")},*values[]={identity}; + CFDictionaryRef properties=CFDictionaryCreate(nullptr,keys,values,1,&kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks); + if(!properties)stop(63); + CFDictionarySetValue(match,CFSTR("IOPropertyMatch"),properties);CFRelease(properties); + io_iterator_t iterator=0;if(IOServiceGetMatchingServices(0,match,&iterator) || !iterator)stop(64); + char device[64]={};unsigned found=0; + while(io_service_t media=IOIteratorNext(iterator)) { + ++found; + CFTypeRef observed=IORegistryEntryCreateCFProperty(media,CFSTR("UUID"),nullptr,0); + CFTypeRef name=IORegistryEntryCreateCFProperty(media,CFSTR("BSD Name"),nullptr,0); + if(!observed || !CFEqual(observed,identity) || !name || CFGetTypeID(name)!=CFStringGetTypeID())stop(65); + char bsd[56]={};if(!CFStringGetCString((CFStringRef)name,bsd,sizeof bsd,kCFStringEncodingASCII) || strncmp(bsd,"disk",4))stop(66); + for(const char *p=bsd;*p;++p)if(!((*p>='a'&&*p<='z')||(*p>='0'&&*p<='9')))stop(67); + bool backed=false;io_registry_entry_t node=media; + for(unsigned depth=0;node && depth<64;++depth) { + if(IOObjectConformsTo(node,"NativeRamBlock")){backed=true;break;} + io_registry_entry_t above=0;auto error=IORegistryEntryGetParentEntry(node,"IOService",&above); + if(node!=media)IOObjectRelease(node);node=error?0:above; + } + if(node && node!=media)IOObjectRelease(node);if(!backed)stop(68); + snprintf(device,sizeof device,"/dev/%s",bsd);CFRelease(observed);CFRelease(name);IOObjectRelease(media); + } + IOObjectRelease(iterator);CFRelease(identity);if(found!=1 || !device[0])stop(69); + const char *point="/System/Library/Extensions",*directory="/private/var/tmp/native-compiler-addons"; + struct stat block={},source={},before={},after={},original={},code={};struct statfs fs={}; + if(stat(device,&block) || !S_ISBLK(block.st_mode) || stat(point,&before) || stat(directory,&source) || source.st_dev!=block.st_rdev)stop(70); + if(mount("bindfs",point,MNT_RDONLY|MNT_UNION,(void *)directory))stop(71); + if(statfs(point,&fs) || (fs.f_flags&(MNT_RDONLY|MNT_UNION))!=(MNT_RDONLY|MNT_UNION) || + strcmp(fs.f_mntfromname,directory) || strcmp(fs.f_fstypename,"bindfs") || stat(point,&after) || + after.st_dev==before.st_dev || stat(driverPath,&code) || code.st_dev!=after.st_dev || + stat("/System/Library/Extensions/AGXMetalA10.bundle/Info.plist",&original) || original.st_dev!=before.st_dev)stop(72); + result[38]=fs.f_flags;result[39]=(uint32_t)before.st_dev;result[40]=(uint32_t)after.st_dev; + result[41]=(uint32_t)original.st_dev;result[42]=(uint32_t)code.st_dev;result[43]=found;result[44]=1; +} +static std::array digestFile(const std::filesystem::path &path,uint64_t *length) { + struct stat st={};if(lstat(path.c_str(),&st) || !S_ISREG(st.st_mode) || st.st_size<0 || st.st_size>16*1024*1024)stop(80); + FILE *file=fopen(path.c_str(),"rb");if(!file)stop(81); + CC_SHA256_CTX hash;CC_SHA256_Init(&hash);unsigned char buffer[16384];uint64_t total=0; + for(;;){const size_t n=fread(buffer,1,sizeof buffer,file);if(n){CC_SHA256_Update(&hash,buffer,(CC_LONG)n);total+=n;}if(n digest={};CC_SHA256_Final(digest.data(),&hash);*length=total;return digest; +} +static void summarize() { + const std::filesystem::path directory=std::string(prefix)+".cache",archive=std::string(prefix)+".archive"; + std::error_code ec;std::vector names; + std::filesystem::recursive_directory_iterator iterator(directory,std::filesystem::directory_options::none,ec),end; + if(ec)stop(83); + for(;iterator!=end;iterator.increment(ec)) { + if(ec)stop(84); + const auto status=iterator->symlink_status(ec);if(ec || std::filesystem::is_symlink(status))stop(85); + if(std::filesystem::is_directory(status))continue; + if(!std::filesystem::is_regular_file(status) || names.size()>=2048)stop(86); + auto name=iterator->path().lexically_relative(directory).generic_string(); + if(name.empty() || name.size()>192 || name.find("..")!=std::string::npos)stop(87); + for(unsigned char c:name)if(c<33 || c>126)stop(88); + names.push_back(name); + } + if(ec || names.empty())stop(89);std::sort(names.begin(),names.end()); + CC_SHA256_CTX manifest;CC_SHA256_Init(&manifest);uint64_t total=0; + for(const auto &name:names) { + uint64_t size=0;const auto digest=digestFile(directory/name,&size);total+=size;if(total>64*1024*1024)stop(90); + const uint32_t length=(uint32_t)name.size();CC_SHA256_Update(&manifest,&length,sizeof length); + CC_SHA256_Update(&manifest,name.data(),length);CC_SHA256_Update(&manifest,&size,sizeof size);CC_SHA256_Update(&manifest,digest.data(),digest.size()); + } + uint64_t archiveBytes=0,sourceBytes=0;const auto archiveDigest=digestFile(archive,&archiveBytes); + const auto sourceDigest=digestFile("/System/Library/Frameworks/QuartzCore.framework/default.metallib",&sourceBytes); + auto *words=reinterpret_cast(snapshot);words[0]=0x515455495041434bull;words[1]=archiveBytes; + memcpy(snapshot+16,archiveDigest.data(),32);words[6]=names.size();words[7]=total; + CC_SHA256_Final(snapshot+64,&manifest);memcpy(snapshot+96,sourceDigest.data(),32); + result[6]=128;result[7]=sourceBytes;result[8]=names.size();result[9]=total;result[10]=archiveBytes; + int fd=open(archive.c_str(),O_RDONLY);if(fd<0 || fsync(fd))stop(91);close(fd); + sync();result[45]=1;result[47]=1; +} +int main(int argc,char **argv) { + (void)argv;result[0]=0x515443504b494f31ull;result[2]=(uint64_t)getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE); + if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1); + mountCompilerView(); + if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") || + unsetenv("NMB_TEST_GRAPHICS_CACHE_LOADER") || unsetenv("NMB_TEST_QUARTZCORE_LIBRARY"))stop(2); + // A fresh fixture must never reuse the earlier run's products. + struct stat old={};if(!lstat((std::string(prefix)+".archive").c_str(),&old) || errno!=ENOENT)stop(3); + if(unlink(workerLog) && errno!=ENOENT)stop(5); + posix_spawn_file_actions_t actions; + if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) || + posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4); + const char *modes[]={"produce","consume","uuid-miss","empty-driver"}; + for(unsigned i=0;i<4;++i) { + char *arguments[]={const_cast("compiler-worker"),const_cast(driverPath),const_cast(prefix),const_cast(modes[i]),nullptr}; + pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ); + result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;if(error || child<=1)stop(10+i); + int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i); + result[14+3*i]=(uint64_t)status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1; + } + posix_spawn_file_actions_destroy(&actions);summarize();stop(0); +} diff --git a/bridge/quartzcore_cache_provisioned_ios.cpp b/bridge/quartzcore_cache_provisioned_ios.cpp new file mode 100644 index 0000000..dffb162 --- /dev/null +++ b/bridge/quartzcore_cache_provisioned_ios.cpp @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* QEMU-only first-consumer harness. Reuse the exact volume/manifest/checkpoint + * helpers, but never call the cold producer suite. Initial Data already holds + * the actual iOS-generated archive and caches at the consumer's normal paths. */ +#define main unused_cold_suite_main +#include "quartzcore_cache_ios.cpp" +#undef main + +int main(int argc,char **argv) { + (void)argv;result[0]=0x515443504b494f32ull;result[2]=(uint64_t)getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE); + if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1); + mountCompilerView(); + if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") || + unsetenv("NMB_TEST_GRAPHICS_CACHE_LOADER") || unsetenv("NMB_TEST_QUARTZCORE_LIBRARY"))stop(2); + // Hash the actual initial-image bytes before any compiler worker is spawned. + summarize();std::array initial={};memcpy(initial.data(),snapshot,initial.size()); + if(unlink(workerLog) && errno!=ENOENT)stop(5); + posix_spawn_file_actions_t actions; + if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) || + posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4); + const char *modes[]={"consume","uuid-miss","empty-driver"}; + for(unsigned i=0;i<3;++i) { + char *arguments[]={const_cast("compiler-worker"),const_cast(driverPath),const_cast(prefix),const_cast(modes[i]),nullptr}; + pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ); + result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;if(error || child<=1)stop(10+i); + int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i); + result[14+3*i]=(uint64_t)status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1; + } + posix_spawn_file_actions_destroy(&actions);summarize(); + if(memcmp(initial.data(),snapshot,initial.size()))stop(95); + memcpy(snapshot+256,initial.data(),initial.size());result[6]=384;result[45]=2;result[46]=1;stop(0); +} diff --git a/bridge/register_host.mm b/bridge/register_host.mm new file mode 100644 index 0000000..2a2a333 --- /dev/null +++ b/bridge/register_host.mm @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +// Host check: the system's own device registration (MTLAddDevice, the call the target's discovery makes with a plugin's device) accepts the frontend's +// device. It asserts that the device conforms to Metal's device protocols and runs the base class's initLimits/initFeatureQueries/initWorkarounds on it; +// the Mac's version differs from the target's, but it caught the missing MTLDeviceSPI conformance that the target's _MTLAddDevice also asserts. +#import +#import +#include +#import +#include +#include "../NativeBridgeMetalFrontend.h" + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + void *add = dlsym(dlopen("/System/Library/Frameworks/Metal.framework/Metal", RTLD_NOW), "MTLAddDevice"); + if (!add) {std::printf("SKIP: this system has no MTLAddDevice\n"); return 0;} + id device = native_bridge_metal_open(argv[1], argv[2], 1, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + if (![(id)device conformsToProtocol:objc_getProtocol("MTLDeviceSPI")]) {std::printf("FAIL MTLDeviceSPI conformance\n"); return 1;} + ((void (*)(id))add)(device); + bool listed = false; + for (id d in MTLCopyAllDevices()) listed |= (d == device); + if (!listed) {std::printf("FAIL the registered device is not among the system's devices\n"); return 1;} + std::printf("PASS: the system's device registration accepted the frontend's device and lists it\n"); + } + return 0; +} diff --git a/bridge/resource_lifetime_host.mm b/bridge/resource_lifetime_host.mm new file mode 100644 index 0000000..b530625 --- /dev/null +++ b/bridge/resource_lifetime_host.mm @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +// Actual host GPU work waits on an unsignaled event while every caller-owned +// no-copy buffer/view reference is dropped. Its deallocator must not run yet. +#import +#import +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +struct State {std::atomic released{0};}; + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + unsigned failures = 0; + size_t airBytes = 0; + const void *air = nmb_fixture_air(&airBytes); + id library = native_bridge_metal_library_with_air(air, airBytes); + id pipeline = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"scale_and_offset"] error:nil]; + if (!pipeline) return 2; + for (unsigned variant = 0; variant < 7; ++variant) { + auto state = std::make_shared(); + const size_t page = (size_t)getpagesize(); + void *pages = mmap(nullptr, page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (pages == MAP_FAILED) return 2; + memset(pages, variant == 6 ? 0xcc : 0x5a, page); + if (variant == 5) for (unsigned i = 0; i < 256; ++i) ((float *)pages)[i] = (float)i; + id output = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id event = [device newSharedEvent]; + id commands = [[device newCommandQueue] commandBuffer]; + __weak id weakBuffer; + if (!output || !event || !commands) return 2; + @autoreleasepool { + // Deliberately keep the mapping owned by the test, so the old + // early-deallocator failure cannot make the pending GPU read + // unmapped memory. No pixel or GPU completion is synthesized. + id buffer = [device newBufferWithBytesNoCopy:pages length:page options:MTLResourceStorageModeShared + deallocator:^(void *, NSUInteger){state->released.fetch_add(1);}]; + if (!buffer) return 2; + weakBuffer = buffer; + [commands encodeWaitForEvent:event value:1]; + if (variant == 5) { + id compute = [commands computeCommandEncoder]; + [compute setComputePipelineState:pipeline]; + [compute setBuffer:buffer offset:0 atIndex:0]; + [compute setBuffer:output offset:0 atIndex:1]; + const float scale = 2; + [compute setBytes:&scale length:sizeof scale atIndex:2]; + [compute dispatchThreads:MTLSizeMake(256,1,1) threadsPerThreadgroup:MTLSizeMake(64,1,1)]; + [compute endEncoding]; + } else if (variant == 1 || variant == 6) { + auto d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + d.storageMode = MTLStorageModeShared; + d.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView | MTLTextureUsageRenderTarget; + id root = [buffer newTextureWithDescriptor:d offset:0 bytesPerRow:256]; + id view = [[root newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] + newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + if (!view) return 2; + if (variant == 6) { + auto pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = view; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(90.0/255,90.0/255,90.0/255,90.0/255); + [[commands renderCommandEncoderWithDescriptor:pass] endEncoding]; + } + if (variant == 1) { + id blit = [commands blitCommandEncoder]; + [blit copyFromTexture:view sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(16,16,1) + toBuffer:output destinationOffset:0 destinationBytesPerRow:64 destinationBytesPerImage:1024]; + [blit endEncoding]; + } + } else { + id blit = [commands blitCommandEncoder]; + if (variant == 2) [blit fillBuffer:buffer range:NSMakeRange(0, 1024) value:0x37]; + [blit copyFromBuffer:buffer sourceOffset:0 toBuffer:output destinationOffset:0 size:1024]; + id missing = nil; + if (variant == 3) [blit fillBuffer:missing range:NSMakeRange(0, 16) value:0]; // local refusal, never submit + [blit endEncoding]; + } + } + const unsigned before = state->released.load(); + if (before || !weakBuffer) ++failures; + unsigned pending = before; + if (variant != 4) { + [commands commit]; + // Completion must remain pending until the actual event is + // signaled, except a locally refused command which submitted + // no GPU work. Check with bounded dispatch waits. + auto finished = dispatch_semaphore_create(0); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + @autoreleasepool {[commands waitUntilCompleted];} + dispatch_semaphore_signal(finished); + }); + const bool early = dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, 100 * NSEC_PER_MSEC)) == 0; + pending = state->released.load(); + if (variant != 3 && (early || pending || !weakBuffer)) ++failures; + event.signaledValue = 1; + if (!early && dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC))) return 1; + if (commands.status != (variant == 3 ? MTLCommandBufferStatusError : MTLCommandBufferStatusCompleted)) ++failures; + if (variant != 3) for (unsigned i = 0; i < (variant == 5 ? 256 : 1024); ++i) { + const bool bad = variant == 5 ? ((const float *)output.contents)[i] != 3.0f*i + : variant == 6 ? ((const unsigned char *)pages)[(i / 64) * 256 + i % 64] != 0x5a + : ((const unsigned char *)output.contents)[i] != (variant == 2 ? 0x37 : 0x5a); + if (bad) {++failures;break;} + } + if (state->released.load() != 1 || weakBuffer) ++failures; // even while commands remains retained + } + commands = nil; + // The real completion callback can release its own reference just + // after waitUntilCompleted returns. Bound that scheduling margin. + for (unsigned i = 0; i < 300 && state->released.load() == 0; ++i) usleep(1000); + if (state->released.load() != 1 || weakBuffer) ++failures; + std::printf("case %u before=%u pending=%u released=%u\n", variant, before, pending, state->released.load()); + munmap(pages, page); + } + if (failures) {std::printf("FAIL: resource owner lifetime (%u checks)\n", failures);return 1;} + std::puts("PASS: no-copy owners/views survive encoding and gated GPU work, release once after completion/refusal/discard; actual GPU bytes match"); + } +} diff --git a/bridge/resource_wait_host.mm b/bridge/resource_wait_host.mm new file mode 100644 index 0000000..bd6790b --- /dev/null +++ b/bridge/resource_wait_host.mm @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +// Keep real work pending behind an unsignaled shared event. Waiting for an unrelated +// resource must return; waiting for its storage, views or IOSurface alias must not. +#import +#import +#import +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +@protocol ResourceWait +- (void)waitUntilComplete; +@end + +static dispatch_time_t seconds(unsigned n) {return dispatch_time(DISPATCH_TIME_NOW, n * NSEC_PER_SEC);} + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + for (unsigned variant = 0; variant < 9; ++variant) { + @autoreleasepool { + id output = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id input = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id query = output; + id target = nil; + IOSurfaceRef surface = nullptr; + void *pages = nullptr; + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.storageMode = MTLStorageModeShared; + td.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView; + if (variant == 0) query = input; // never used by the pending fill + if (variant == 2) { + id texture = [output newTextureWithDescriptor:td offset:0 bytesPerRow:256]; + query = [texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + } + if (variant == 3 || variant == 7) { + if (variant == 7) td.storageMode = MTLStorageModePrivate; + target = [device newTextureWithDescriptor:td]; + query = [[target newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + } + if (variant == 4) { + NSDictionary *p = @{(id)kIOSurfaceWidth:@16, (id)kIOSurfaceHeight:@16, + (id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}; + surface = IOSurfaceCreate((__bridge CFDictionaryRef)p); + target = [device newTextureWithDescriptor:td iosurface:surface plane:0]; + query = [device newTextureWithDescriptor:td iosurface:surface plane:0]; + } + if (variant == 5) { + auto hd = [MTLHeapDescriptor new];hd.size = 65536;hd.storageMode = MTLStorageModeShared; + id heap = [device newHeapWithDescriptor:hd]; + output = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + query = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + } + if (variant == 6) { + const size_t page = (size_t)getpagesize(); + pages = mmap(nullptr, page * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (pages == MAP_FAILED) return 2; + auto keep = std::shared_ptr(pages, [page](void *p){munmap(p, page * 2);}); + auto release = ^(void *, NSUInteger){(void)keep;}; + output = [device newBufferWithBytesNoCopy:pages length:page * 2 options:MTLResourceStorageModeShared deallocator:release]; + query = [device newBufferWithBytesNoCopy:(char *)pages + page length:page options:MTLResourceStorageModeShared deallocator:release]; + } + if (variant == 8) { + output = [device newBufferWithLength:4096 options:MTLResourceStorageModePrivate]; + query = output; + } + if (!output || !input || !query || ((variant == 3 || variant == 4 || variant == 7) && !target)) { + std::printf("FAIL resource allocation %u\n", variant);return 2; + } + memset(input.contents, 0x5a, 4096); + id event = [device newSharedEvent]; + id commands = [[device newCommandQueue] commandBuffer]; + [commands encodeWaitForEvent:event value:1]; + id blit = [commands blitCommandEncoder]; + if (target) [blit copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:64 sourceBytesPerImage:1024 sourceSize:MTLSizeMake(16,16,1) + toTexture:target destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; + else [blit fillBuffer:output range:NSMakeRange(0, output.length) value:0x5a]; + [blit endEncoding];[commands commit]; + auto entered = dispatch_semaphore_create(0), returned = dispatch_semaphore_create(0); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + dispatch_semaphore_signal(entered); + [(id)query waitUntilComplete]; + dispatch_semaphore_signal(returned); + }); + if (dispatch_semaphore_wait(entered, seconds(3))) return 1; + const bool early = dispatch_semaphore_wait(returned, dispatch_time(DISPATCH_TIME_NOW, 250 * NSEC_PER_MSEC)) == 0; + event.signaledValue = 1; + if (!early && dispatch_semaphore_wait(returned, seconds(3))) {std::puts("FAIL resource wait stuck");return 1;} + [commands waitUntilCompleted]; + if (commands.status != MTLCommandBufferStatusCompleted || early != (variant == 0)) { + std::printf("FAIL variant=%u early=%d status=%lu\n", variant, early, (unsigned long)commands.status);return 1; + } + if (target) { + unsigned char pixels[1024] = {}; + if (variant == 7) { + memset(input.contents, 0, 4096); + id readback = [[device newCommandQueue] commandBuffer]; + id copy = [readback blitCommandEncoder]; + [copy copyFromTexture:target sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(16,16,1) + toBuffer:input destinationOffset:0 destinationBytesPerRow:64 destinationBytesPerImage:1024]; + [copy endEncoding];[readback commit];[readback waitUntilCompleted]; + if (readback.status != MTLCommandBufferStatusCompleted) return 1; + memcpy(pixels, input.contents, sizeof pixels); + } else [target getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0,0,16,16) mipmapLevel:0]; + for (unsigned char pixel : pixels) if (pixel != 0x5a) return 1; + } else if (variant == 8) { + memset(input.contents, 0, 4096); + id readback = [[device newCommandQueue] commandBuffer]; + id copy = [readback blitCommandEncoder]; + [copy copyFromBuffer:output sourceOffset:0 toBuffer:input destinationOffset:0 size:4096]; + [copy endEncoding];[readback commit];[readback waitUntilCompleted]; + if (readback.status != MTLCommandBufferStatusCompleted) return 1; + for (NSUInteger i = 0; i < 4096; ++i) if (((const unsigned char *)input.contents)[i] != 0x5a) return 1; + } else for (NSUInteger i = 0; i < output.length; ++i) if (((const unsigned char *)output.contents)[i] != 0x5a) return 1; + // The shared deallocator owns no-copy pages until both imports release them. + query = nil;output = nil;target = nil; + if (surface) CFRelease(surface); + } + } + std::puts("PASS: unrelated resource wait returns while work is gated; storage, views, heap and IOSurface/no-copy aliases wait for real completion"); + } +} diff --git a/bridge/scheduled_host.mm b/bridge/scheduled_host.mm new file mode 100644 index 0000000..3c53a26 --- /dev/null +++ b/bridge/scheduled_host.mm @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT +// A blocked completed handler must not hold waitUntilScheduled. No hardware timing claim: +// the semaphores create a deterministic distinction between scheduling and completion. +#import +#import +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +static dispatch_time_t deadline() {return dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC);} +__attribute__((noinline)) static id nothing() {return nil;} +@protocol CompletionQueue +- (void)setCompletionQueue:(dispatch_queue_t)queue; +@end + +int main(int argc, char **argv) { + if (argc < 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + id queue = [device newCommandQueue]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + if (!queue || !buffer) return 2; + for (unsigned variant : {0u, 1u, 2u, 3u}) { + const bool refused = variant == 1 || variant == 3; + [(id)queue setCompletionQueue:variant == 2 ? dispatch_queue_create("scheduled-test", DISPATCH_QUEUE_SERIAL) : nil]; + id cb = [queue commandBuffer]; + auto entered = dispatch_semaphore_create(0), release = dispatch_semaphore_create(0); + auto returned = dispatch_semaphore_create(0); + std::atomic scheduledCount{0}, completedCount{0}; + auto scheduled = &scheduledCount, completed = &completedCount; + [cb addScheduledHandler:^(id command) { + // This also catches callbacks deferred to Completed by the frontend. + if (command.status == MTLCommandBufferStatusScheduled) ++*scheduled; + }]; + [cb addCompletedHandler:^(id) { + ++*completed; + dispatch_semaphore_signal(entered); + dispatch_semaphore_wait(release, deadline()); + }]; + id blit = [cb blitCommandEncoder]; + if (variant == 1) { + // A frontend encoding failure has no scheduling event to announce. + [blit fillBuffer:(id)nothing() range:NSMakeRange(0, 16) value:0]; + } else { + [blit fillBuffer:buffer range:NSMakeRange(0, 256) value:0x5a]; + } + if (variant != 3) [blit endEncoding]; // variant 3: runtime refuses commit with an encoder open + [cb commit]; + if (dispatch_semaphore_wait(entered, deadline())) {std::puts("FAIL completion never arrived");return 1;} + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + [cb waitUntilScheduled]; + dispatch_semaphore_signal(returned); + }); + const bool independent = dispatch_semaphore_wait(returned, dispatch_time(DISPATCH_TIME_NOW, 300 * NSEC_PER_MSEC)) == 0; + dispatch_semaphore_signal(release); + if (!independent && dispatch_semaphore_wait(returned, deadline())) {std::puts("FAIL scheduled wait stuck");return 1;} + [cb waitUntilCompleted]; + if (variant == 3) [blit endEncoding]; + if (!independent || completedCount != 1 || scheduledCount != (refused ? 0u : 1u) || + cb.status != (refused ? MTLCommandBufferStatusError : MTLCommandBufferStatusCompleted)) { + std::printf("FAIL variant=%u independent=%d scheduled=%u completed=%u status=%lu\n", + variant, independent, scheduledCount.load(), completedCount.load(), (unsigned long)cb.status); + return 1; + } + } + std::puts("PASS: scheduling precedes completion on runtime/serial queues; scheduled waits ignore blocked completed handlers; frontend/runtime refusals announce no scheduling"); + } +} diff --git a/bridge/selector_coverage_host.mm b/bridge/selector_coverage_host.mm new file mode 100644 index 0000000..78f6317 --- /dev/null +++ b/bridge/selector_coverage_host.mm @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +// Host instrument: which of the Metal methods the target's QuartzCore references (scripts/audit-quartzcore-metal-calls.py, JSON path as the third +// argument) the native frontend's live objects respond to. One object of each kind is made on the frontend's device (bridge over MoltenVK); a +// kind that cannot be made is reported as such. Prints every selector an object lacks. A reference proves QuartzCore can send it; it does not +// show that the system UI does. +#import +#import +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "FrontendChecks.h" +#include "../bridge/NativeMetalObjects.h" + +static id N(id object) {return object ?: (id)[NSNull null];} + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 4) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + NSData *json = [NSData dataWithContentsOfFile:[NSString stringWithUTF8String:argv[3]]]; + NSDictionary *all = [NSJSONSerialization JSONObjectWithData:json options:0 error:nil][@"kinds"]; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + id queue = [device newCommandQueue]; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:8 height:8 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + id texture = [device newTextureWithDescriptor:td]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = texture; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + id render = [cb renderCommandEncoderWithDescriptor:pass]; + [render endEncoding]; + id blit = [cb blitCommandEncoder]; + [blit endEncoding]; + id compute = [cb computeCommandEncoder]; + [compute endEncoding]; + id function = [library newFunctionWithName:@"scale_and_offset"]; + id computePipeline = [device newComputePipelineStateWithFunction:function error:nil]; + id renderPipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + id sampler = [device newSamplerStateWithDescriptor:[MTLSamplerDescriptor new]]; + id depth = [device newDepthStencilStateWithDescriptor:[MTLDepthStencilDescriptor new]]; + id archive = [device newBinaryArchiveWithDescriptor:[MTLBinaryArchiveDescriptor new] error:nil]; + MTLHeapDescriptor *hd = [MTLHeapDescriptor new]; + hd.size = 65536; + id heap = [device respondsToSelector:@selector(newHeapWithDescriptor:)] ? [device newHeapWithDescriptor:hd] : nil; + NSDictionary *objects = @{@"device": N(device), @"texture": N(texture), @"buffer": N(buffer), @"heap": N(heap), @"command queue": N(queue), + @"command buffer": N(cb), @"render encoder": N(render), @"blit encoder": N(blit), @"compute encoder": N(compute), + @"render pipeline": N(renderPipeline), @"compute pipeline": N(computePipeline), @"sampler": N(sampler), + @"depth stencil": N(depth), @"library": N(library), @"function": N(function), @"binary archive": N(archive)}; + unsigned total = 0; + for (NSString *kind in [all.allKeys sortedArrayUsingSelector:@selector(compare:)]) { + id object = objects[kind]; + NSArray *selectors = [all[kind][@"selectors"] allKeys]; + if (!object || object == [NSNull null]) {std::printf("%-18s cannot be created (%lu selectors referenced)\n", kind.UTF8String, (unsigned long)selectors.count); ++total; continue;} + NSMutableArray *missing = [NSMutableArray array]; + for (id entry in selectors) { + NSString *name = entry; + if ([name hasPrefix:@"."]) continue; + if (![object respondsToSelector:NSSelectorFromString(name)]) [missing addObject:name]; + } + total += missing.count; + std::printf("%-18s %lu referenced, %lu missing%s%s\n", kind.UTF8String, (unsigned long)selectors.count, (unsigned long)missing.count, + missing.count ? ": " : "", [missing componentsJoinedByString:@" "].UTF8String); + } + std::printf("%s: %u selectors the target's QuartzCore references are not answered by the frontend's objects\n", total ? "GAPS" : "PASS", total); + } + return 0; +} diff --git a/bridge/surface_flush_host.mm b/bridge/surface_flush_host.mm new file mode 100644 index 0000000..e88c932 --- /dev/null +++ b/bridge/surface_flush_host.mm @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT +// Count actual frontend cache-maintenance calls while forwarding every call +// to libSystem. GPU writes and alias reads must still reach the real pages. +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +static std::atomic calls, bytes; +extern "C" void sys_dcache_flush(void *address, size_t length) { + using Flush = void (*)(void *, size_t); + static Flush real = reinterpret_cast(dlsym(RTLD_NEXT, "sys_dcache_flush")); + if (!real || real == sys_dcache_flush) std::abort(); + calls++;bytes += length; + real(address, length); +} + +static bool run(id device, unsigned aliases, unsigned allocations) { + @autoreleasepool { + constexpr unsigned width = 64, height = 16, row = width * 4, frameBytes = row * height; + std::vector surfaces; + auto textures = [NSMutableArray array]; + auto descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm + width:width height:height mipmapped:NO]; + descriptor.storageMode = MTLStorageModeShared; + descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView; + for (unsigned i = 0; i < allocations; ++i) { + NSDictionary *properties = @{(id)kIOSurfaceWidth:@(width), (id)kIOSurfaceHeight:@(height), + (id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfaceBytesPerRow:@(row), + (id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}; + IOSurfaceRef surface = IOSurfaceCreate((__bridge CFDictionaryRef)properties); + if (!surface || IOSurfaceLock(surface, 0, nullptr)) return false; + memset(IOSurfaceGetBaseAddress(surface), i ? 0xa7 : 0xcc, IOSurfaceGetAllocSize(surface)); + if (IOSurfaceUnlock(surface, 0, nullptr)) return false; + surfaces.push_back(surface); + } + for (unsigned i = 0; i < aliases; ++i) { + // Include independent imports as well as nested views of an import. + id texture = [device newTextureWithDescriptor:descriptor iosurface:surfaces[i % allocations] plane:0]; + if (i & 1) texture = [[texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] + newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + if (!texture) return false; + [textures addObject:texture]; + } + std::vector upload(frameBytes, 0x5a); + id input = [device newBufferWithBytes:upload.data() length:frameBytes options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:aliases * frameBytes options:MTLResourceStorageModeShared]; + id command = [[device newCommandQueue] commandBuffer]; + id blit = [command blitCommandEncoder]; + if (!input || !output || !blit) return false; + [blit copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:row sourceBytesPerImage:frameBytes + sourceSize:MTLSizeMake(width,height,1) toTexture:textures[0] destinationSlice:0 destinationLevel:0 + destinationOrigin:MTLOriginMake(0,0,0)]; + for (unsigned i = 0; i < aliases; ++i) + [blit copyFromTexture:textures[i] sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) + sourceSize:MTLSizeMake(width,height,1) toBuffer:output destinationOffset:i * frameBytes + destinationBytesPerRow:row destinationBytesPerImage:frameBytes]; + [blit endEncoding]; + const uint64_t beforeCalls = calls.load(), beforeBytes = bytes.load(); + [command commit];[command waitUntilCompleted]; + uint64_t expectedBytes = 0; + for (IOSurfaceRef surface : surfaces) expectedBytes += 2 * IOSurfaceGetAllocSize(surface); + bool ok = command.status == MTLCommandBufferStatusCompleted; + for (unsigned i = 0; i < aliases; ++i) for (unsigned j = 0; j < frameBytes; ++j) + if (((const unsigned char *)output.contents)[i * frameBytes + j] != (i % allocations ? 0xa7 : 0x5a)) ok = false; + // The GPU also changed the first surface; check its actual shared bytes. + for (unsigned y = 0; y < height; ++y) for (unsigned x = 0; x < row; ++x) + if (((const unsigned char *)IOSurfaceGetBaseAddress(surfaces[0]))[y * IOSurfaceGetBytesPerRow(surfaces[0]) + x] != 0x5a) ok = false; + const uint64_t actualCalls = calls.load() - beforeCalls, actualBytes = bytes.load() - beforeBytes; + std::printf("aliases=%u allocations=%u flushes=%llu bytes=%llu expected=%u/%llu pixels=%s\n", aliases, allocations, + (unsigned long long)actualCalls, (unsigned long long)actualBytes, 2 * allocations, + (unsigned long long)expectedBytes, ok ? "pass" : "FAIL"); + ok &= actualCalls == 2 * allocations && actualBytes == expectedBytes; + for (IOSurfaceRef surface : surfaces) CFRelease(surface); + return ok; + } +} + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + bool ok = run(device, 1, 1); + ok &= run(device, 128, 1); + ok &= run(device, 128, 2); + char telemetry[6144] = {}; + native_bridge_metal_telemetry(telemetry, sizeof telemetry); + const char *line = strstr(telemetry, "J surface cache "); + unsigned long long count = 0, maintained = 0, duplicates = 0; + if (!line || sscanf(line, "J surface cache n=%llu bytes=%llu dup=%llu", &count, &maintained, &duplicates) != 3 || + count != calls.load() || maintained != bytes.load() || duplicates != 506) ok = false; + if (!ok) {std::puts("FAIL: duplicate IOSurface cache maintenance");return 1;} + std::puts("PASS: each live surface range flushed once per phase, independent imports/views deduplicated, distinct surfaces and GPU bytes preserved"); + } +} diff --git a/bridge/surface_import_host.mm b/bridge/surface_import_host.mm new file mode 100644 index 0000000..820a828 --- /dev/null +++ b/bridge/surface_import_host.mm @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: MIT +// Count real frontend no-copy imports/releases, forwarding every call to the +// unmodified bridge. Use actual GPU pixels and bounded pending-work lifetimes. +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +static std::atomic imports, releases; +static std::mutex handlesLock; +static std::set handles; +static uint64_t (*vkImports)(); +extern "C" NmbBuffer *tracked_surface_import(NmbDevice *d, void *p, uint64_t n, uint64_t options) { + NmbBuffer *buffer = nmb_buffer_new_no_copy(d,p,n,options); + if (buffer) {std::lock_guard lock(handlesLock);handles.insert(buffer);imports++;} + return buffer; +} +extern "C" void tracked_buffer_release(NmbBuffer *buffer) { + {std::lock_guard lock(handlesLock);if(handles.erase(buffer))releases++;} + nmb_buffer_release(buffer); +} +static bool drained(unsigned expected) { + for(unsigned i=0;i<300 && releases.load()!=expected;++i)usleep(1000); + return releases.load()==expected; +} +static IOSurfaceRef makeSurface(unsigned byte) { + IOSurfaceRef surface=IOSurfaceCreate((__bridge CFDictionaryRef)@{ + (id)kIOSurfaceWidth:@64,(id)kIOSurfaceHeight:@16,(id)kIOSurfaceBytesPerElement:@4, + (id)kIOSurfaceBytesPerRow:@256,(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}); + if(!surface || IOSurfaceLock(surface,0,nullptr))return nullptr; + memset(IOSurfaceGetBaseAddress(surface),byte,IOSurfaceGetAllocSize(surface)); + if(IOSurfaceUnlock(surface,0,nullptr)){CFRelease(surface);return nullptr;} + return surface; +} +static MTLTextureDescriptor *descriptor() { + auto d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:16 mipmapped:NO]; + d.storageMode=MTLStorageModeShared; + d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite|MTLTextureUsagePixelFormatView; + return d; +} + +static bool aliases(id device,unsigned count,unsigned allocations,bool parallel) { + const unsigned before=imports.load(),released=releases.load();bool ok=true; + const uint64_t vkBefore=vkImports?vkImports():0; + @autoreleasepool { + std::vector surfaces; + for(unsigned i=0;i failures{0};auto *failed=&failures; + const IOSurfaceRef surface=surfaces[0]; + dispatch_apply(count,dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^(size_t i){ + (void)i; + @autoreleasepool { + id texture=[device newTextureWithDescriptor:d iosurface:surface plane:0]; + if(!texture){(*failed)++;return;} + @synchronized(textures){[textures addObject:texture];} + } + }); + ok=failures.load()==0; + } else for(unsigned i=0;i texture=[device newTextureWithDescriptor:d iosurface:surfaces[i%allocations] plane:0]; + if(i&1)texture=[[texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + if(!texture)return false;[textures addObject:texture]; + } + if(textures.count!=count)return false; + std::vector data(4096,0x5a); + id input=[device newBufferWithBytes:data.data() length:data.size() options:MTLResourceStorageModeShared]; + id output=[device newBufferWithLength:count*4096 options:MTLResourceStorageModeShared]; + id command=[[device newCommandQueue] commandBuffer];id copy=[command blitCommandEncoder]; + if(!input || !output || !copy)return false; + [copy copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:256 sourceBytesPerImage:4096 sourceSize:MTLSizeMake(64,16,1) + toTexture:textures[0] destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; + for(unsigned i=0;i256?(made>=allocations && made<=count):made==allocations; + const uint64_t actualVk=vkImports?vkImports()-vkBefore:0; + std::printf("aliases=%u surfaces=%u parallel=%d imports=%u releases=%u vk_imports=%llu gpu=%s retired=%s reuse=%s\n", + count,allocations,parallel,made,releases.load()-released,(unsigned long long)actualVk,ok?"pass":"FAIL",retired?"pass":"FAIL",reuse?"pass":"FAIL"); + return ok && retired && reuse && (!vkImports || actualVk==allocations); +} + +static bool expires(id device) { + const unsigned before=imports.load(),released=releases.load();IOSurfaceRef surface=makeSurface(0x35);if(!surface)return false; + bool ok=true; + for(unsigned i=0;i<2;++i) { + @autoreleasepool {id texture=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];if(!texture)ok=false;} + ok &= drained(released+i+1) && imports.load()==before+i+1; + } + CFRelease(surface); + std::printf("expired owners imports=%u releases=%u result=%s\n",imports.load()-before,releases.load()-released,ok?"pass":"FAIL"); + return ok; +} + +static bool pending(id device) { + const unsigned before=imports.load(),released=releases.load();bool ok=true; + @autoreleasepool { + IOSurfaceRef surface=makeSurface(0x79);if(!surface)return false; + id event=[device newSharedEvent];id command=[[device newCommandQueue] commandBuffer]; + id output=[device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + if(!event || !command || !output)return false; + @autoreleasepool { + id texture=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];if(!texture)return false; + [command encodeWaitForEvent:event value:1];id copy=[command blitCommandEncoder]; + [copy copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1) + toBuffer:output destinationOffset:0 destinationBytesPerRow:256 destinationBytesPerImage:4096]; + [copy endEncoding];[command commit]; + } + auto finished=dispatch_semaphore_create(0); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{@autoreleasepool{[command waitUntilCompleted];}dispatch_semaphore_signal(finished);}); + if(!dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,100*NSEC_PER_MSEC)))ok=false; + @autoreleasepool { + id alias=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0]; + if(!alias || imports.load()!=before+1 || releases.load()!=released)ok=false; + // Drop the caller's last surface reference before the GPU reads it. + CFRelease(surface);surface=nullptr; + } + event.signaledValue=1; + if(dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,3*NSEC_PER_SEC)))return false; + if(command.status!=MTLCommandBufferStatusCompleted)ok=false; + for(unsigned i=0;i<4096;++i)if(((const unsigned char *)output.contents)[i]!=0x79)ok=false; + // Completed commands must release the import even if still retained. + ok &= drained(released+1); + } + std::printf("pending owners imports=%u releases=%u result=%s\n",imports.load()-before,releases.load()-released,ok?"pass":"FAIL"); + return ok; +} + +int main(int argc,char **argv) { + if(argc!=3 && !(argc==4 && !strcmp(argv[3],"parallel")))return 2; + @autoreleasepool { + id device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2; + void *loader=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!loader)return 2; + vkImports=reinterpret_cast(dlsym(loader,"nmb_test_vk_host_import_count")); + if(getenv("NMB_TEST_REAL_VULKAN_LOADER") && !vkImports)return 2; + if(argc==4)return aliases(device,64,1,true)?0:1; + bool ok=aliases(device,1,1,false); + ok &= aliases(device,128,1,false); + ok &= aliases(device,128,2,false); + ok &= aliases(device,64,1,true); + ok &= expires(device); + ok &= pending(device); + ok &= aliases(device,514,257,false); + if(!ok || imports.load()!=releases.load()){std::puts("FAIL: live IOSurface import reuse");return 1;} + std::puts("PASS: live IOSurface aliases reuse imports with real GPU bytes; distinct, parallel, expired and pending owners preserve lifetimes"); + } +} diff --git a/bridge/telemetry_host.mm b/bridge/telemetry_host.mm new file mode 100644 index 0000000..1368bb1 --- /dev/null +++ b/bridge/telemetry_host.mm @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Interleave encoding on two queues, then reverse commit order. Global deltas assign +// the first buffer's render work to the second buffer's blits; per-buffer counts must not. +#include "FrontendChecks.h" +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" +#include +#include + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + size_t length = 0; + const void *air = nmb_fixture_air(&length); + id library = native_bridge_metal_library_with_air(air, length); + frontendchecks::Report report = {}; + frontendchecks::gReport = &report; + id pipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id texture = [device newTextureWithDescriptor:td]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + id render = [[device newCommandQueue] commandBuffer]; + id copy = [[device newCommandQueue] commandBuffer]; + if (!library || !pipeline || !texture || !buffer || !render || !copy) return 2; + frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6); + for (unsigned i = 0; i < 3; ++i) { + id blit = [copy blitCommandEncoder]; + [blit fillBuffer:buffer range:NSMakeRange(0, 256) value:0x5a]; + [blit endEncoding]; + } + frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6); + [copy commit];[render commit]; + if (!frontendchecks::completed(copy, "blit") || !frontendchecks::completed(render, "render")) return 1; + char text[4608] = {}; + native_bridge_metal_telemetry(text, sizeof text); + std::map> counts; + for (char *line = strtok(text, "\n"); line; line = strtok(nullptr, "\n")) { + unsigned index, sec, ms, passes, draws, kpx, dispatches, blits; + if (sscanf(line, "K%u t=%u.%us rp%u dr%u kpx%u cs%u bl%u", &index, &sec, &ms, &passes, &draws, &kpx, &dispatches, &blits) != 8) continue; + std::printf("%s\n", line); + counts[index] = {passes, blits}; + if (draws != (index == 2 ? 2u : 0u)) {std::puts("FAIL cross-buffer draw attribution");return 1;} + } + if (counts.size() != 2 || counts[1] != std::make_pair(0u, 3u) || counts[2] != std::make_pair(2u, 0u)) { + std::puts("FAIL interleaved render/blit attribution");return 1; + } + uint8_t pixels[16 * 16 * 4] = {}; + [texture getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + if (frontendchecks::wrongTexels(pixels, 64, 16)) return 1; + for (unsigned i = 0; i < 256; ++i) if (((const uint8_t *)buffer.contents)[i] != 0x5a) return 1; + std::puts("PASS: opposite encode/commit orders retain per-buffer render/blit counts and correct host GPU pixels"); + } +} diff --git a/bridge/uber_diff_host.mm b/bridge/uber_diff_host.mm new file mode 100644 index 0000000..359ee0f --- /dev/null +++ b/bridge/uber_diff_host.mm @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT +// Host instrument: QuartzCore's own uber-shader (fixed_vert_lph_gen with fixed_frag_lph_cph / fixed_frag_lph_cpf) under recipes of its +// function constants, through the native frontend. Each recipe makes one render pipeline, draws it once over fixed pseudo-random inputs +// (every buffer and texture the stages may read, all values finite) into a 64x64 BGRA8 target, and prints the pipeline time, the draw time +// (the runtime makes the VkPipeline at the draw) and a hash of the pixels. Two runs of the same recipes over two builds of the translator, +// on the same loader, must print the same hashes: the instrument behind a change to what the translator emits for these modules. Over the +// Adreno Turnip host simulation the CP executes nothing, so there only the times mean anything. +// argv: loader (or apple), shader cache dir, QuartzCore AIR metallib, recipe file, [pixel dump dir]. A recipe line: fragment name, then name:type:value +// constants (bool, uchar, uchar4 as a,b,c,d); fixed_vert_lph_gen takes the ones it declares. +#import +#import +#include +#include +#include +#include +#include +#include +#include "NativeBridgeMetalFrontend.h" + +extern "C" size_t native_bridge_metal_telemetry(char *, size_t); + +static double nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1e6;} +static void missing(const char *c, const char *s) {std::fprintf(stderr, "missing %s %s\n", c, s);} +static uint32_t lcg(uint32_t &s) {s = s * 1664525u + 1013904223u; return s >> 8;} +// A half in [0, 1): sign 0, exponent 0..14, so any pair of them read as a float is finite too. +static uint16_t unitHalf(uint32_t &s) {return uint16_t((lcg(s) % 15) << 10 | (lcg(s) & 0x3ff));} +static uint64_t fnv(const uint8_t *p, size_t n) {uint64_t h = 1469598103934665603ull; while (n--) h = (h ^ *p++) * 1099511628211ull; return h;} + +static MTLFunctionConstantValues *constants(const std::vector &specs, MTLFunctionConstantValues *v) { + for (const std::string &spec : specs) { + size_t a = spec.find(':'), b = spec.find(':', a + 1); + NSString *name = @(spec.substr(0, a).c_str()); + std::string type = spec.substr(a + 1, b - a - 1), value = spec.substr(b + 1); + if (type == "bool") {bool x = value == "true" || value == "1"; [v setConstantValue:&x type:MTLDataTypeBool withName:name];} + else if (type == "uchar") {uint8_t x = uint8_t(std::strtoul(value.c_str(), nullptr, 0)); [v setConstantValue:&x type:MTLDataTypeUChar withName:name];} + else if (type == "uchar4") { + uint8_t x[4] = {}; + const char *p = value.c_str(); + for (int i = 0; i < 4 && *p; ++i) {char *e; x[i] = uint8_t(std::strtoul(p, &e, 0)); p = *e ? e + 1 : e;} + [v setConstantValue:x type:MTLDataTypeUChar4 withName:name]; + } + } + return v; +} + +int main(int argc, char **argv) { + if (argc < 5) {std::fprintf(stderr, "usage: loader cache metallib recipes [dump dir]\n"); return 2;} + @autoreleasepool { + // Loader "apple": the host's own Metal device compiles the same AIR, the reference the translator's results are held to. + const bool apple = !std::strcmp(argv[1], "apple"); + id device = apple ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed\n"); return 1;} + NSError *error = nil; + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:@(argv[3])] error:&error]; + if (!qc) {std::printf("library failed: %s\n", error.localizedDescription.UTF8String); return 1;} + FILE *recipes = std::fopen(argv[4], "r"); + if (!recipes) {std::printf("no recipes\n"); return 1;} + const char *dump = argc > 5 ? argv[5] : nullptr; + id queue = [device newCommandQueue]; + constexpr unsigned W = 64, H = 64; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:W height:H mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + td.storageMode = MTLStorageModeShared; + id target = [device newTextureWithDescriptor:td]; + // Inputs, the same in every run: a full-screen pair of triangles (SurfaceVertexXC: float4 position, two float2 texcoords, half4 + // color, padding; 48 bytes), the identity MVP and texture matrices, a zero vertex config (layout 0), and finite pseudo-random + // words for every other buffer and texel. + uint32_t seed = 0x5eed1234u; + struct Vertex {float position[4], tc0[2], tc1[2]; uint16_t color[4]; uint8_t pad[8];}; + static_assert(sizeof(Vertex) == 48); + const float corners[6][2] = {{-1, -1}, {1, -1}, {-1, 1}, {-1, 1}, {1, -1}, {1, 1}}; + Vertex vertices[6] = {}; + for (int i = 0; i < 6; ++i) { + vertices[i].position[0] = corners[i][0]; vertices[i].position[1] = corners[i][1]; vertices[i].position[3] = 1; + vertices[i].tc0[0] = vertices[i].tc1[1] = (corners[i][0] + 1) / 2; + vertices[i].tc0[1] = vertices[i].tc1[0] = (1 - corners[i][1]) / 2; + for (int c = 0; c < 4; ++c) vertices[i].color[c] = unitHalf(seed); + } + id vertexBuffer = [device newBufferWithBytes:vertices length:sizeof vertices options:MTLResourceStorageModeShared]; + const float identity[16] = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + id matrix = [device newBufferWithBytes:identity length:sizeof identity options:MTLResourceStorageModeShared]; + id zeros = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + std::vector> words; + for (int i = 0; i < 9; ++i) { + std::vector w(131072 / 2 + 512); + for (uint16_t &x : w) x = unitHalf(seed); + if (i == 1) { + // CA::OGL::Metal::Uniform at its own layout: floats in [0, 1) at [6 x float4] (0..96), [2 x float4] and float4 + // (128..176); halves in [0, 1) elsewhere; its two flag bytes (120, 121) and the trailing [8 x i8] zero. + auto *bytes = reinterpret_cast(w.data()); + auto unitFloat = [&](size_t at) {float f = float(lcg(seed) % 65536) / 65536.0f; std::memcpy(bytes + at, &f, 4);}; + for (size_t at = 0; at < 96; at += 4) unitFloat(at); + for (size_t at = 128; at < 176; at += 4) unitFloat(at); + bytes[120] = bytes[121] = 0; + std::memset(bytes + 216, 0, 8); + } + words.push_back([device newBufferWithBytes:w.data() length:w.size() * 2 options:MTLResourceStorageModeShared]); + } + MTLTextureDescriptor *xd = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA16Float width:16 height:16 mipmapped:NO]; + xd.usage = MTLTextureUsageShaderRead; + std::vector> textures; + for (int i = 0; i < 12; ++i) { + id t = [device newTextureWithDescriptor:xd]; + std::vector texels(16 * 16 * 4); + for (uint16_t &x : texels) x = unitHalf(seed); + [t replaceRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0 withBytes:texels.data() bytesPerRow:16 * 8]; + textures.push_back(t); + } + MTLSamplerDescriptor *sd = [MTLSamplerDescriptor new]; + sd.minFilter = sd.magFilter = MTLSamplerMinMagFilterLinear; + id sampler = [device newSamplerStateWithDescriptor:sd]; + char line[4096]; + unsigned n = 0, made = 0; + double pipeTotal = 0, drawTotal = 0; + while (std::fgets(line, sizeof line, recipes)) { + std::vector tokens; + for (char *t = std::strtok(line, " \t\n"); t; t = std::strtok(nullptr, " \t\n")) tokens.emplace_back(t); + if (tokens.empty() || tokens[0][0] == '#') continue; + ++n; + std::vector specs(tokens.begin() + 1, tokens.end()); + double f0 = nowMs(); + id vertex = [qc newFunctionWithName:@"fixed_vert_lph_gen" constantValues:constants(specs, [MTLFunctionConstantValues new]) error:&error]; + id fragment = vertex ? [qc newFunctionWithName:@(tokens[0].c_str()) constantValues:constants(specs, [MTLFunctionConstantValues new]) error:&error] : nil; + double f1 = nowMs(); + if (!fragment) {std::printf("R%03u function refused: %s\n", n, error.localizedDescription.UTF8String); continue;} + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = vertex; + d.fragmentFunction = fragment; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + double p0 = nowMs(); + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + double p1 = nowMs(); + if (!state) {std::printf("R%03u pipeline refused: %s\n", n, error.localizedDescription.UTF8String); continue;} + ++made; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0); + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + double d0 = nowMs(); + id commands = [queue commandBuffer]; + id encoder = [commands renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:state]; + [encoder setVertexBuffer:zeros offset:0 atIndex:0]; + [encoder setVertexBuffer:vertexBuffer offset:0 atIndex:1]; + [encoder setVertexBuffer:matrix offset:0 atIndex:2]; + [encoder setVertexBuffer:matrix offset:0 atIndex:3]; + [encoder setVertexBuffer:matrix offset:0 atIndex:4]; + for (NSUInteger i = 0; i < 9; ++i) [encoder setFragmentBuffer:words[i] offset:0 atIndex:i]; + for (NSUInteger i = 0; i < 12; ++i) [encoder setFragmentTexture:textures[i] atIndex:i]; + [encoder setFragmentSamplerState:sampler atIndex:0]; + [encoder setFragmentSamplerState:sampler atIndex:1]; + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [encoder endEncoding]; + [commands commit]; + [commands waitUntilCompleted]; + double d1 = nowMs(); + std::vector pixels(W * H * 4); + [target getBytes:pixels.data() bytesPerRow:W * 4 fromRegion:MTLRegionMake2D(0, 0, W, H) mipmapLevel:0]; + if (dump) { + char path[1024]; + std::snprintf(path, sizeof path, "%s/R%03u.bgra", dump, n); + if (FILE *out = std::fopen(path, "wb")) {std::fwrite(pixels.data(), 1, pixels.size(), out); std::fclose(out);} + } + pipeTotal += p1 - p0; + drawTotal += d1 - d0; + std::printf("R%03u %s fn=%.1f pipe=%.1f draw=%.1f status=%ld hash=%016llx\n", n, tokens[0].c_str(), f1 - f0, p1 - p0, d1 - d0, + (long)commands.status, (unsigned long long)fnv(pixels.data(), pixels.size())); + std::fflush(stdout); + } + static char table[8192]; + if (!apple) native_bridge_metal_telemetry(table, sizeof table); + for (char *l = std::strtok(table, "\n"); l; l = std::strtok(nullptr, "\n")) + if (l[0] == 'V' || l[0] == 'T') std::printf("TELEMETRY %s\n", l); + std::printf("SUMMARY recipes=%u made=%u pipe=%.0f ms draw=%.0f ms\n", n, made, pipeTotal, drawTotal); + } + return 0; +} diff --git a/shaders/smoke_fill.comp b/shaders/smoke_fill.comp new file mode 100644 index 0000000..42f392a --- /dev/null +++ b/shaders/smoke_fill.comp @@ -0,0 +1,9 @@ +#version 450 +// Shader-execution check: every invocation writes a value derived from its global index and two push constants. +layout(local_size_x = 64) in; +layout(std430, set = 0, binding = 0) buffer Out { uint v[]; } o; +layout(push_constant) uniform P { uint mul; uint add; } p; +void main() { + uint i = gl_GlobalInvocationID.x; + o.v[i] = i * p.mul + p.add; +} diff --git a/shaders/smoke_triangle.frag b/shaders/smoke_triangle.frag new file mode 100644 index 0000000..908bb17 --- /dev/null +++ b/shaders/smoke_triangle.frag @@ -0,0 +1,4 @@ +#version 450 +layout(location = 0) out vec4 color; +layout(push_constant) uniform P { vec4 color; } p; +void main() { color = p.color; } diff --git a/shaders/smoke_triangle.vert b/shaders/smoke_triangle.vert new file mode 100644 index 0000000..9aa6974 --- /dev/null +++ b/shaders/smoke_triangle.vert @@ -0,0 +1,6 @@ +#version 450 +// Rasterisation check: one triangle from the vertex index, no vertex buffers. +void main() { + const vec2 p[3] = vec2[](vec2(-0.5, -0.5), vec2(0.5, -0.5), vec2(0.0, 0.5)); + gl_Position = vec4(p[gl_VertexIndex], 0.0, 1.0); +} diff --git a/tools/MetalVulkanProbe.rs b/tools/MetalVulkanProbe.rs new file mode 100644 index 0000000..11321fa --- /dev/null +++ b/tools/MetalVulkanProbe.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Offline consumer of a pinned translator's public API; never opens a GPU. +use metal2vulkan::device::{Device, DeviceDescription, DevicePolicy, SamplingPolicy}; +use metal2vulkan::library::{FunctionStage, Library}; +use metal2vulkan::metal::PixelFormat; +use metal2vulkan::pipeline::{ + ColorAttachmentDescriptor, ComputePipelineDescriptor, FunctionDescriptor, + RenderPipelineDescriptor, StageModule, StageSource, VertexDescriptor, +}; +use metal2vulkan::Budget; +use serde_json::{json, Value}; +use std::{collections::BTreeMap, env, fs, process}; + +fn stage(module: &StageModule) -> Value { + json!({ + "stage": format!("{:?}", module.stage), + "entry_point": module.entry_point, + "spirv": &*module.spirv, + "precision_notes": module.precision_notes.iter().map(|n| format!("{n:?}")).collect::>(), + "required_subgroup_size": module.required_subgroup_size, + "specialization": format!("{:?}", module.specialization), + "constant_blob_bytes": module.constant_blob.as_ref().map_or(0, |b| b.len()), + }) +} + +fn run(args: &[String]) -> Result> { + if args.len() == 2 && args[0] == "--list" { + let library = Library::parse(&fs::read(&args[1])?)?; + return Ok(json!({"functions": library.functions().map(|f| json!({ + "name": f.name, "stage": format!("{:?}", f.stage) + })).collect::>(), "gpu_execution": false})); + } + let mut args = args; + let mut vertex_descriptor = None; + let mut color_attachments = None; + while args.first().is_some_and(|v| v.starts_with("--")) { + let path = args.get(1).ok_or("missing descriptor path")?; + match args[0].as_str() { + "--vertex-descriptor" if vertex_descriptor.is_none() => { + vertex_descriptor = Some(serde_json::from_slice::(&fs::read( + path, + )?)?); + } + "--color-attachments" if color_attachments.is_none() => { + let colors = + serde_json::from_slice::>(&fs::read(path)?)?; + if colors.is_empty() || colors.keys().any(|index| *index >= 8) { + return Err("color attachments require indices in 0..8".into()); + } + color_attachments = Some(colors); + } + _ => return Err("unknown or duplicate descriptor option".into()), + } + args = &args[2..]; + } + if args.len() != 3 && args.len() != 5 { + return Err( + "expected --list INPUT, or PROFILE INPUT FUNCTION [FRAGMENT_INPUT FUNCTION]".into(), + ); + } + let description = DeviceDescription::from_khronos_json(&fs::read_to_string(&args[0])?, None)?; + let mut policy = DevicePolicy::default().exact_answers(); + // exact_answers deliberately keeps sampling unchanged in this upstream revision. + policy.sampling = SamplingPolicy::Exact; + let device = Device::new(description, policy)?; + let library = Library::parse(&fs::read(&args[1])?)?; + let function = library.function(&args[2])?; + let budget = Budget::standard(); + let (stages, layout, interface, dependencies, requirements) = match function.info().stage { + FunctionStage::Kernel if args.len() == 3 => { + if vertex_descriptor.is_some() || color_attachments.is_some() { + return Err("compute pipeline does not take render descriptors".into()); + } + let descriptor = ComputePipelineDescriptor::new(StageSource::Function( + FunctionDescriptor::new(function), + )); + let p = device.compile_compute_pipeline(&descriptor, &budget)?; + ( + vec![stage(&p.stage)], + format!("{:?}", p.layout), + format!("{:?}", p.interface), + format!("{:?}", p.dependencies), + format!("{:?}", p.requirements), + ) + } + FunctionStage::Vertex => { + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function( + FunctionDescriptor::new(function), + )); + descriptor.vertex_descriptor = vertex_descriptor; + if args.len() == 5 { + let fragment_library = Library::parse(&fs::read(&args[3])?)?; + let fragment = fragment_library.function(&args[4])?; + if fragment.info().stage != FunctionStage::Fragment { + return Err("fragment input is not a fragment function".into()); + } + descriptor.fragment = + Some(StageSource::Function(FunctionDescriptor::new(fragment))); + let colors = color_attachments + .as_ref() + .ok_or("fragment translation requires explicit color attachments")?; + for (&index, &format) in colors { + descriptor.color_attachments[index] = + Some(ColorAttachmentDescriptor::new(format)); + } + } else if color_attachments.is_some() { + return Err("color attachments require a fragment function in this probe".into()); + } + let p = device.compile_render_pipeline(&descriptor, &budget)?; + ( + p.stages.iter().map(stage).collect(), + format!("{:?}", p.layout), + format!("{:?}", p.interface), + format!("{:?}", p.dependencies), + format!("{:?}", p.requirements), + ) + } + _ => { + return Err( + "stage needs a matching pipeline descriptor; standalone fragment refused".into(), + ) + } + }; + Ok(json!({ + "stages": stages, "layout": layout, "interface": interface, + "dependencies": dependencies, "requirements": requirements, + "policy": "exact_answers+exact_sampling", + "validation_envs": device.profile().spirv_validation().target_envs, + "validation_relaxations": device.profile().spirv_validation().relaxations, + "profile_is_hardware_observation": false, + "gpu_execution": false, "metal_device_published": false, + })) +} + +fn main() { + match run(&env::args().skip(1).collect::>()) { + Ok(value) => println!("{value}"), + Err(error) => { + eprintln!("REFUSED: {error}"); + process::exit(2); + } + } +} diff --git a/tools/metal-vulkan-probe.lock b/tools/metal-vulkan-probe.lock new file mode 100644 index 0000000..3800b8d --- /dev/null +++ b/tools/metal-vulkan-probe.lock @@ -0,0 +1,208 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "metal2vulkan" +version = "0.2.0" +dependencies = [ + "metal2vulkan-air", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "metal2vulkan-air" +version = "0.2.0" +dependencies = [ + "serde", + "sha2", +] + +[[package]] +name = "native-metal-vulkan-probe" +version = "0.1.0" +dependencies = [ + "metal2vulkan", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/probe-coverage-attachment.json b/tools/probe-coverage-attachment.json new file mode 100644 index 0000000..d75ed15 --- /dev/null +++ b/tools/probe-coverage-attachment.json @@ -0,0 +1 @@ +{"2": "R32Float"} diff --git a/tools/probe-vertex-layout.json b/tools/probe-vertex-layout.json new file mode 100644 index 0000000..72deaf8 --- /dev/null +++ b/tools/probe-vertex-layout.json @@ -0,0 +1,10 @@ +{ + "layouts": { + "0": {"stride": 48, "step_function": "PerVertex", "step_rate": 1} + }, + "attributes": { + "0": {"format": "Float4", "offset": 0, "buffer_index": 0}, + "1": {"format": "Float2", "offset": 16, "buffer_index": 0}, + "3": {"format": "Float4", "offset": 32, "buffer_index": 0} + } +}