From 5a34e353ed67bf12ba8bf3a310c85fcde896bb87 Mon Sep 17 00:00:00 2001 From: bluseliu50 <265648257+bluseliu50@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:12:06 +0800 Subject: [PATCH] Import NativeMetal Co-authored-by: rpone --- CacheExport.h | 313 +++ GpuScanoutClient.h | 122 + GpuScanoutPresenter.h | 88 + HostMetalCommandServer.h | 4 + HostMetalCommandServer.mm | 658 ++++++ LICENSE | 21 + NativeAdrenoMetalPlugin.mm | 351 +++ NativeAdrenoProbeDaemon.cpp | 25 + NativeAdrenoUserTest.h | 628 ++++++ NativeBridgeMetalFrontend.h | 28 + NativeBridgeMetalFrontend.mm | 1521 +++++++++++++ NativeIOGPUProviderProbe.cpp | 56 + NativeMetalBridgePhase.h | 197 ++ NativeOfflineAccelerator.cpp | 11 + NativeOfflineGPU.cpp | 312 +++ NativeTurnipSmoke.h | 653 ++++++ OfflineMetalCommandWire.h | 9 + OfflineMetalDepthStencilWire.h | 16 + OfflineMetalEventWire.h | 9 + OfflineMetalFenceWire.h | 11 + OfflineMetalFrontend.mm | 1001 +++++++++ OfflineMetalFunctionWire.h | 28 + OfflineMetalPlugin.mm | 32 + OfflineMetalQueueWire.h | 7 + OfflineMetalSessionWire.h | 8 + OfflineMetalSharedEventWire.h | 14 + OfflineMetalTextureWire.h | 11 + OfflineMetalVertexWire.h | 38 + OfflineMetalWire.h | 40 + OriginalAcceleratorABI.h | 11 + OriginalGPUProviderABI.h | 74 + README.md | 1074 +++++++++ bridge/FrontendChecks.h | 460 ++++ bridge/NativeMetalBridge.h | 27 + bridge/NativeMetalBridge.rs | 363 +++ bridge/NativeMetalObjects.h | 260 +++ bridge/NativeMetalObjects.rs | 1987 +++++++++++++++++ bridge/archive_host.mm | 58 + bridge/bridge_smoke.cpp | 22 + bridge/cache_export_host.cpp | 57 + bridge/carenderer_host.mm | 165 ++ bridge/compute_archive_host.mm | 90 + bridge/driver_cache_host.cpp | 60 + bridge/frontend_host.mm | 33 + bridge/graphics_cache_loader.cpp | 115 + bridge/import_count_loader.cpp | 60 + bridge/nil_args_host.mm | 44 + bridge/no_copy_import_host.mm | 144 ++ ...001-optional-constant-output-members.patch | 59 + ...2-resource-wait-storage-dependencies.patch | 67 + ...003-cache-device-entry-point-priming.patch | 253 +++ ...004-fold-constant-global-field-loads.patch | 205 ++ bridge/pipeline_timing_host.mm | 130 ++ bridge/plugin_sequence_host.mm | 30 + bridge/present_host.mm | 55 + bridge/priming_cache_host.cpp | 40 + bridge/priming_cache_ios.cpp | 62 + bridge/priming_cache_loader.cpp | 74 + bridge/quartzcore_blur_host.mm | 202 ++ bridge/quartzcore_cache.mm | 112 + bridge/quartzcore_cache_ios.cpp | 144 ++ bridge/quartzcore_cache_provisioned_ios.cpp | 32 + bridge/register_host.mm | 27 + bridge/resource_lifetime_host.mm | 127 ++ bridge/resource_wait_host.mm | 127 ++ bridge/scheduled_host.mm | 70 + bridge/selector_coverage_host.mm | 73 + bridge/surface_flush_host.mm | 102 + bridge/surface_import_host.mm | 173 ++ bridge/telemetry_host.mm | 56 + bridge/uber_diff_host.mm | 178 ++ shaders/smoke_fill.comp | 9 + shaders/smoke_triangle.frag | 4 + shaders/smoke_triangle.vert | 6 + tools/MetalVulkanProbe.rs | 143 ++ tools/metal-vulkan-probe.lock | 208 ++ tools/probe-coverage-attachment.json | 1 + tools/probe-vertex-layout.json | 10 + 78 files changed, 14095 insertions(+) create mode 100644 CacheExport.h create mode 100644 GpuScanoutClient.h create mode 100644 GpuScanoutPresenter.h create mode 100644 HostMetalCommandServer.h create mode 100644 HostMetalCommandServer.mm create mode 100644 LICENSE create mode 100644 NativeAdrenoMetalPlugin.mm create mode 100644 NativeAdrenoProbeDaemon.cpp create mode 100644 NativeAdrenoUserTest.h create mode 100644 NativeBridgeMetalFrontend.h create mode 100644 NativeBridgeMetalFrontend.mm create mode 100644 NativeIOGPUProviderProbe.cpp create mode 100644 NativeMetalBridgePhase.h create mode 100644 NativeOfflineAccelerator.cpp create mode 100644 NativeOfflineGPU.cpp create mode 100644 NativeTurnipSmoke.h create mode 100644 OfflineMetalCommandWire.h create mode 100644 OfflineMetalDepthStencilWire.h create mode 100644 OfflineMetalEventWire.h create mode 100644 OfflineMetalFenceWire.h create mode 100644 OfflineMetalFrontend.mm create mode 100644 OfflineMetalFunctionWire.h create mode 100644 OfflineMetalPlugin.mm create mode 100644 OfflineMetalQueueWire.h create mode 100644 OfflineMetalSessionWire.h create mode 100644 OfflineMetalSharedEventWire.h create mode 100644 OfflineMetalTextureWire.h create mode 100644 OfflineMetalVertexWire.h create mode 100644 OfflineMetalWire.h create mode 100644 OriginalAcceleratorABI.h create mode 100644 OriginalGPUProviderABI.h create mode 100644 README.md create mode 100644 bridge/FrontendChecks.h create mode 100644 bridge/NativeMetalBridge.h create mode 100644 bridge/NativeMetalBridge.rs create mode 100644 bridge/NativeMetalObjects.h create mode 100644 bridge/NativeMetalObjects.rs create mode 100644 bridge/archive_host.mm create mode 100644 bridge/bridge_smoke.cpp create mode 100644 bridge/cache_export_host.cpp create mode 100644 bridge/carenderer_host.mm create mode 100644 bridge/compute_archive_host.mm create mode 100644 bridge/driver_cache_host.cpp create mode 100644 bridge/frontend_host.mm create mode 100644 bridge/graphics_cache_loader.cpp create mode 100644 bridge/import_count_loader.cpp create mode 100644 bridge/nil_args_host.mm create mode 100644 bridge/no_copy_import_host.mm create mode 100644 bridge/patches/0001-optional-constant-output-members.patch create mode 100644 bridge/patches/0002-resource-wait-storage-dependencies.patch create mode 100644 bridge/patches/0003-cache-device-entry-point-priming.patch create mode 100644 bridge/patches/0004-fold-constant-global-field-loads.patch create mode 100644 bridge/pipeline_timing_host.mm create mode 100644 bridge/plugin_sequence_host.mm create mode 100644 bridge/present_host.mm create mode 100644 bridge/priming_cache_host.cpp create mode 100644 bridge/priming_cache_ios.cpp create mode 100644 bridge/priming_cache_loader.cpp create mode 100644 bridge/quartzcore_blur_host.mm create mode 100644 bridge/quartzcore_cache.mm create mode 100644 bridge/quartzcore_cache_ios.cpp create mode 100644 bridge/quartzcore_cache_provisioned_ios.cpp create mode 100644 bridge/register_host.mm create mode 100644 bridge/resource_lifetime_host.mm create mode 100644 bridge/resource_wait_host.mm create mode 100644 bridge/scheduled_host.mm create mode 100644 bridge/selector_coverage_host.mm create mode 100644 bridge/surface_flush_host.mm create mode 100644 bridge/surface_import_host.mm create mode 100644 bridge/telemetry_host.mm create mode 100644 bridge/uber_diff_host.mm create mode 100644 shaders/smoke_fill.comp create mode 100644 shaders/smoke_triangle.frag create mode 100644 shaders/smoke_triangle.vert create mode 100644 tools/MetalVulkanProbe.rs create mode 100644 tools/metal-vulkan-probe.lock create mode 100644 tools/probe-coverage-attachment.json create mode 100644 tools/probe-vertex-layout.json diff --git a/CacheExport.h b/CacheExport.h new file mode 100644 index 0000000..9fb72ff --- /dev/null +++ b/CacheExport.h @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: MIT +// Pipeline-cache export (int-147). The render server's runtime keeps its translated shaders and the Vulkan driver's pipeline records in its +// cache directory, but that directory lives on the RAM Data disk: every boot compiled the whole UI again (int-146: 98 pipelines, 63 s, +// up to 1.6 s each, also while the user opened App Library). This thread sends the directory to the kernel's UFS log (method 0x7100 on a +// hand-off connection, kits/sheng/BootLog.c export region) whenever it changed and then stayed unchanged for one period; the host reads it +// back with the log and builds the next image's Data with it, so those pipelines start as cache hits. +// A boot whose image carried a pack exports only the files the pack did not have (see baked()). +// Stream: "M2VCACH1", u32 directory length, directory, u32 file count, then per file u32 path length, u32 mode, u32 uid, u32 gid, u64 size, +// path (relative to the directory), contents. Export blob: "SHEXR001", u32 stream bytes, u32 0, the stream as it is: it goes out in pieces +// while the files are read, so the render server never holds more than a piece (int-149: one boot's new variants were about 100 MB). +// Image pack: "SHEXP001", u32 stream bytes, u32 compressed bytes, the stream zlib-compressed (scripts/add-gpu-plugin-data.py merges exports +// into one). The export only reads. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef CACHEEXPORT_FORMAT_ONLY // the format alone (host check, bridge/cache_export_host.cpp); the plugin also gets the sender +#include "NativeAdrenoUserTest.h" +#endif + +namespace cacheexport { + +// kLimit: the largest export blob, one half of the kernel's 256 MiB export region. +constexpr uint32_t kMethod = 0x7100, kPiece = 3968, kLimit = 128u << 20; +struct File {std::string path; struct stat st;}; + +// A pack in the image is the union of several boots' exports (scripts/add-gpu-plugin-data.py); it is read as a stream, so only its own +// file and the stream bounds limit it. +constexpr uint64_t kPackLimit = 1ull << 30; + +// Files the image's pack carried (path and size). The export leaves them out: the host has them already and merges each boot's export +// with the earlier ones, so a boot sends only what it compiled itself (int-148: the whole directory outgrew the export area once the UI's +// own variants were added, and the export refused the build, marker 0x69f2). +static std::set &baked() {static std::set files; return files;} +static std::string bakedKey(const std::string &path, uint64_t size) {return path + '\n' + std::to_string(size);} +static void dropBaked(std::vector &files) { + if (baked().empty()) return; + std::vector fresh; + for (File &f : files) if (!baked().count(bakedKey(f.path, (uint64_t)f.st.st_size))) fresh.push_back(std::move(f)); + files.swap(fresh); +} + +static void scan(const std::string &root, const std::string &relative, std::vector &out, unsigned depth) { + if (depth > 8) return; + DIR *dir = opendir((root + "/" + relative).c_str()); + if (!dir) return; + while (dirent *entry = readdir(dir)) { + if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue; + const std::string path = relative.empty() ? std::string(entry->d_name) : relative + "/" + entry->d_name; + struct stat st = {}; + if (lstat((root + "/" + path).c_str(), &st) != 0) continue; + if (S_ISDIR(st.st_mode)) scan(root, path, out, depth + 1); + else if (S_ISREG(st.st_mode)) out.push_back({path, st}); + } + closedir(dir); +} + +// Count, bytes and newest modification of the directory: equal on two scans = nothing is being written. +static uint64_t signature(const std::vector &files) { + uint64_t s = files.size() * 0x9e3779b97f4a7c15ull; + for (const File &f : files) s = s * 31 + (uint64_t)f.st.st_size * 1000003 + (uint64_t)f.st.st_mtimespec.tv_sec * 7 + (uint64_t)f.st.st_mtimespec.tv_nsec; + return s; +} + +template static void put(std::vector &b, T v) {const uint8_t *p = (const uint8_t *)&v; b.insert(b.end(), p, p + sizeof v);} + +// The stream's length for these files, from their sizes as scanned. +static uint64_t streamBytes(const std::string &root, const std::vector &files) { + uint64_t n = 8 + 4 + root.size() + 4; + for (const File &f : files) n += 24 + f.path.size() + (uint64_t)f.st.st_size; + return n; +} + +// The stream, handed to emit(bytes, count) in order (emit returns false to stop); false when a file vanished or no longer has the size +// it was scanned with, or emit stopped. +template static bool stream(const std::string &root, const std::vector &files, Emit &&emit) { + std::vector head; + {static const char magic[] = "M2VCACH1"; head.insert(head.end(), magic, magic + 8);} + put(head, (uint32_t)root.size()); + head.insert(head.end(), root.begin(), root.end()); + put(head, (uint32_t)files.size()); + if (!emit(head.data(), head.size())) return false; + std::vector chunk(65536); + for (const File &f : files) { + FILE *in = fopen((root + "/" + f.path).c_str(), "rb"); + if (!in) return false; + head.clear(); + put(head, (uint32_t)f.path.size()); + put(head, (uint32_t)f.st.st_mode); + put(head, (uint32_t)f.st.st_uid); + put(head, (uint32_t)f.st.st_gid); + put(head, (uint64_t)f.st.st_size); + head.insert(head.end(), f.path.begin(), f.path.end()); + bool ok = emit(head.data(), head.size()); + uint64_t left = (uint64_t)f.st.st_size; + while (ok && left) { + const size_t want = left < chunk.size() ? (size_t)left : chunk.size(); + ok = fread(chunk.data(), 1, want, in) == want && emit(chunk.data(), want); + left -= want; + } + ok = ok && fgetc(in) == EOF; // grown since the scan: refuse rather than send a short copy + fclose(in); + if (!ok) return false; + } + return true; +} + +#ifdef CACHEEXPORT_FORMAT_ONLY +// The image pack format (SHEXP001, compressed), for the host check; scripts/add-gpu-plugin-data.py builds the same. False when a file +// vanished or zlib is missing. +static bool build(const std::string &root, const std::vector &files, std::vector &out) { + using BoundFn = unsigned long (*)(unsigned long); + using CompressFn = int (*)(uint8_t *, unsigned long *, const uint8_t *, unsigned long, int); + static void *z = dlopen("/usr/lib/libz.1.dylib", RTLD_NOW | RTLD_LOCAL); + if (!z) return false; + auto bound = (BoundFn)dlsym(z, "compressBound"); + auto compress = (CompressFn)dlsym(z, "compress2"); + if (!bound || !compress) return false; + std::vector raw; + if (!stream(root, files, [&](const uint8_t *p, size_t n) {raw.insert(raw.end(), p, p + n); return true;})) return false; + unsigned long zbytes = bound(raw.size()); + std::vector zdata(zbytes); + if (compress(zdata.data(), &zbytes, raw.data(), raw.size(), 6) != 0) return false; + out.clear(); + {static const char magic[] = "SHEXP001"; out.insert(out.end(), magic, magic + 8);} + put(out, (uint32_t)raw.size()); + put(out, (uint32_t)zbytes); + out.insert(out.end(), zdata.begin(), zdata.begin() + (long)zbytes); + return true; +} +#endif + +// Reads a pack's stream: inflated as it is consumed, so neither the pack nor the stream is held in memory. +struct PackReader { + using InitFn = int (*)(z_streamp, const char *, int); + using InflateFn = int (*)(z_streamp, int); + using EndFn = int (*)(z_streamp); + FILE *in = nullptr; + z_stream z = {}; + InflateFn inflateFn = nullptr; + EndFn endFn = nullptr; + bool open = false, ended = false; + uint64_t left = 0, produced = 0, expect = 0; // compressed bytes not yet read from the file, stream bytes produced, stream bytes announced + uint8_t input[65536]; + ~PackReader() {if (open) endFn(&z); if (in) fclose(in);} + bool start(const char *path) { + static void *lib = dlopen("/usr/lib/libz.1.dylib", RTLD_NOW | RTLD_LOCAL); + auto init = lib ? (InitFn)dlsym(lib, "inflateInit_") : nullptr; + inflateFn = lib ? (InflateFn)dlsym(lib, "inflate") : nullptr; + endFn = lib ? (EndFn)dlsym(lib, "inflateEnd") : nullptr; + if (!init || !inflateFn || !endFn || !(in = fopen(path, "rb"))) return false; + struct stat st = {}; + uint8_t head[16]; + uint32_t raw = 0, zbytes = 0; + if (fstat(fileno(in), &st) != 0 || (uint64_t)st.st_size > kPackLimit || fread(head, 1, 16, in) != 16 || memcmp(head, "SHEXP001", 8)) return false; + memcpy(&raw, head + 8, 4); + memcpy(&zbytes, head + 12, 4); + if ((uint64_t)zbytes != (uint64_t)st.st_size - 16) return false; + left = zbytes; expect = raw; + open = init(&z, ZLIB_VERSION, (int)sizeof z) == Z_OK; + return open; + } + bool read(void *out, size_t n) { + z.next_out = static_cast(out); + z.avail_out = (uInt)n; + while (z.avail_out) { + if (ended) return false; + if (!z.avail_in && left) { + const size_t want = left < sizeof input ? (size_t)left : sizeof input; + if (fread(input, 1, want, in) != want) return false; + left -= want; + z.next_in = input; + z.avail_in = (uInt)want; + } + const uInt before = z.avail_out; + const int r = inflateFn(&z, Z_NO_FLUSH); + produced += before - z.avail_out; + if (r == Z_STREAM_END) ended = true; + else if (r != Z_OK) return false; + if (produced > expect) return false; + } + return true; + } + bool skip(uint64_t n) { + uint8_t scratch[16384]; + while (n) {const size_t k = n < sizeof scratch ? (size_t)n : sizeof scratch; if (!read(scratch, k)) return false; n -= k;} + return true; + } + // The whole stream was consumed: zlib's check passed at its end and nothing follows it. + bool finished() { + if (!ended) {uint8_t b; if (read(&b, 1)) return false;} + return ended && produced == expect && !left && !z.avail_in; + } +}; + +// Import, before the device opens: the image may carry a pack beside the bundle (NativeAdrenoMetal.m2vpack, put there by +// scripts/add-gpu-plugin-data.py --pipeline-pack). Its files go into this process's own cache directory, so they carry this process's owner +// and mode; a file that is already there with the same size is kept. Files are written beside their place and renamed only once the +// whole pack has been read and checked: a damaged pack imports nothing. Returns the number of files written, or -1. +static int importPack(const char *pack, const std::string &directory) { + if (access(pack, R_OK) != 0) return 0; + PackReader r; + if (!r.start(pack)) return -1; + std::vector written; + auto fail = [&]() {for (const std::string &f : written) unlink((f + ".import").c_str()); return -1;}; + char magic[8]; + uint32_t rootLength = 0, count = 0; + if (!r.read(magic, 8) || memcmp(magic, "M2VCACH1", 8) || !r.read(&rootLength, 4) || rootLength > 4096 || !r.skip(rootLength) || + !r.read(&count, 4) || count > 1u << 20) return fail(); + std::set seen; + std::vector chunk(65536); + for (uint32_t i = 0; i < count; ++i) { + uint32_t pathLength = 0, mode = 0, uid = 0, gid = 0; + uint64_t size = 0; + if (!r.read(&pathLength, 4) || !r.read(&mode, 4) || !r.read(&uid, 4) || !r.read(&gid, 4) || !r.read(&size, 8)) return fail(); + if (pathLength == 0 || pathLength > 1024 || size > r.expect) return fail(); + std::string path(pathLength, '\0'); + if (!r.read(&path[0], pathLength)) return fail(); + if (path[0] == '/' || path.find("..") != std::string::npos || path.find('\0') != std::string::npos) return fail(); + seen.insert(bakedKey(path, size)); + const std::string full = directory + "/" + path; + struct stat st = {}; + if (stat(full.c_str(), &st) == 0 && (uint64_t)st.st_size == size) {if (!r.skip(size)) return fail(); continue;} + for (size_t slash = full.find('/', directory.size() + 1); slash != std::string::npos; slash = full.find('/', slash + 1)) + mkdir(full.substr(0, slash).c_str(), 0700); + written.push_back(full); + FILE *out = fopen((full + ".import").c_str(), "wb"); + if (!out) return fail(); + bool ok = true; + for (uint64_t left = size; ok && left;) { + const size_t k = left < chunk.size() ? (size_t)left : chunk.size(); + ok = r.read(chunk.data(), k) && fwrite(chunk.data(), 1, k, out) == k; + left -= k; + } + ok = fclose(out) == 0 && ok; + if (!ok) return fail(); + } + if (!r.finished()) return fail(); + int renamed = 0; + for (const std::string &f : written) if (rename((f + ".import").c_str(), f.c_str()) == 0) ++renamed; else unlink((f + ".import").c_str()); + baked().insert(seen.begin(), seen.end()); // only a pack read to its end counts + return renamed; +} + +#ifndef CACHEEXPORT_FORMAT_ONLY +// Markers 0x69xx: 00 started, 01 connection, 02 an export went out (then 0x6Bnn: its size in MiB, saturated), f0 no connection, +// f1 no selector, f2 a file changed or vanished while it was read, f3 a piece refused, f4 the new files exceed one export. +static void run(Gpu &marker, std::string root) { + while (root.size() > 1 && root.back() == '/') root.pop_back(); + marker.mark(0x6900); + Gpu link; + link.call = marker.call; link.openFn = marker.openFn; link.closeFn = marker.closeFn; link.port = marker.port; + if (!link.openFn || link.openFn(link.port, mach_task_self(), 0x4148, &link.conn) != 0 || !link.conn) {marker.mark(0x69f0); return;} + if (!link.probe()) {marker.mark(0x69f1); return;} + marker.mark(0x6901); + uint64_t previous = 0, exported = 0; + uint32_t generation = 0; + std::vector buffer(24 + kPiece); + for (;;) { + sleep(15); + std::vector files; + scan(root, "", files, 0); + dropBaked(files); + const uint64_t now = signature(files); + const bool stable = now == previous; + previous = now; + if (!stable || now == exported || files.empty()) continue; + const uint64_t total = 16 + streamBytes(root, files); + if (total > kLimit) {marker.mark(0x69f4); exported = now; continue;} + ++generation; + uint32_t offset = 0, filled = 0; + bool sent = true; + auto flush = [&]() { + const uint32_t header[6] = {kTunnelMagic, kMethod, generation, (uint32_t)total, offset, filled}; + memcpy(buffer.data(), header, sizeof header); + sent = link.call(link.conn, link.selector, buffer.data(), sizeof header + filled, nullptr, nullptr) == 0; + offset += filled; + filled = 0; + return sent; + }; + auto emit = [&](const uint8_t *p, size_t n) { + while (n) { + const size_t k = n < kPiece - filled ? n : kPiece - filled; + memcpy(buffer.data() + 24 + filled, p, k); + filled += (uint32_t)k; p += k; n -= k; + if (filled == kPiece && !flush()) return false; + } + return true; + }; + uint8_t head[16] = {'S', 'H', 'E', 'X', 'R', '0', '0', '1'}; + const uint32_t streamed = (uint32_t)(total - 16); + memcpy(head + 8, &streamed, 4); + const bool read = emit(head, sizeof head) && stream(root, files, emit); + if (read && filled) flush(); + if (!sent) {marker.mark(0x69f3); continue;} // retried after the next change or period + if (!read || offset != total) {marker.mark(0x69f2); continue;} + exported = now; + marker.mark(0x6902); + marker.mark(0x6B00 | (uint32_t)(total >> 20 > 0xff ? 0xff : total >> 20)); + } +} + +#endif + +} // namespace cacheexport diff --git a/GpuScanoutClient.h b/GpuScanoutClient.h new file mode 100644 index 0000000..6ffcb6d --- /dev/null +++ b/GpuScanoutClient.h @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: MIT +// Userspace side of the GPU hand-off (boot-arg sheng-fb-gpu=1): serves the display driver's frames with the GPU. For every offered frame the +// GPU copies the compositor's surface pages into the scanout mapping (GpuScanoutPresenter.h), and the swap is completed with the GPU's +// verdict. A failed copy is reported as failed; the display driver then falls back to its own path and counts it. +// The calls reach the kernel through a Transport. In the plugin that is the Adreno accelerator's connection (the original sandbox profiles +// admit that class, not the presenter's own service): methods 0x7000 + NGP_METHOD_* through the tunnel selector, scanout as memory type 0x5343. +#pragma once +#import +#include +#include +#include +#include "GpuScanoutPresenter.h" +#include "../NativeMobileDisplay/NativeGOPWire.h" + +namespace gpuscanout { + +struct Transport { + void *context; + // NGP_METHOD_MODE / WAIT_SWAP / COMPLETE_SWAP with the wire records; returns the IOReturn (0 = success). + int (*call)(void *context, uint32_t method, const void *in, size_t inSize, void *out, size_t outSize); + // The scanout, writable, mapped into this task. + int (*mapScanout)(void *context, uint64_t *address, uint64_t *length); + // Progress marker (the plugin shows the latest ones on the boot page, which survives a panic that parks the calling core). + void (*mark)(void *context, uint32_t value); +}; + +struct Stats { + uint64_t frames, failures, timeouts; + int lastResult; // gpupresent::Result of the last frame + int step; // 0 idle, 2 mode refused, 3 scanout not mappable/importable, 4 serving + int error; // the failing call's return, when there is one + uint64_t verified; // frames whose scanout was read back and compared with the surface + uint64_t mismatchRows; // rows of those frames that differ (CPU reads only: nothing is rendered or copied by the CPU) + uint32_t lastMs; // the GPU copy's time for the last frame + uint64_t totalMs; // and for all frames (the mean is totalMs / (frames + failures)) +}; + +class Client { +public: + // Reads the mode and maps the scanout; false with stats.step telling where it stopped. + bool open(id device, const Transport &transport, Stats *stats) { + transport_ = transport; + stats_ = stats; + NGPMode mode = {}; + note(0x6710); + int status = transport_.call(transport_.context, NGP_METHOD_MODE, nullptr, 0, &mode, sizeof mode); + note(0x6711); + if (status || mode.version != NGP_VERSION || mode.format != NGP_BGRX8888 || !mode.width || !mode.height || + mode.stride < mode.width * 4 || mode.bytes < mode.stride * mode.height) return fail(2, status); + const size_t page = (size_t)getpagesize(); + uint64_t address = 0, length = 0; + note(0x6712); + status = transport_.mapScanout(transport_.context, &address, &length); + note(0x6713); + if (status || !address || length < mode.bytes) return fail(3, status); + // The mapping covers whole pages; the framebuffer allocation ends on a page boundary, the copy never writes past the last row. + gpupresent::Scanout scanout = {(void *)(uintptr_t)address, (size_t)((mode.bytes + page - 1) & ~(page - 1)), (size_t)mode.stride, + (uint32_t)mode.width, (uint32_t)mode.height}; + note(0x6714); + scanout_ = scanout; + presenter_ = new gpupresent::Presenter(device, scanout); + note(0x6715); + if (!presenter_->ready()) return fail(3, 0); + stats_->step = 4; + return true; + } + // One wait (at most timeoutMs) and, when the display driver offered a frame, one GPU copy and its completion. Returns false on a + // protocol error that makes further serving pointless. + bool serveOne(uint64_t timeoutMs) { + NGPWait wait = {timeoutMs}; + NGPSwap swap = {}; + if (!stats_->timeouts && !stats_->frames) note(0x6720); // first wait + const int status = transport_.call(transport_.context, NGP_METHOD_WAIT_SWAP, &wait, sizeof wait, &swap, sizeof swap); + if (status) { + if ((unsigned)status == 0xe00002d6u /* kIOReturnTimeout */) {++stats_->timeouts; return true;} + ++stats_->failures; + stats_->error = status; + return false; + } + const size_t page = (size_t)getpagesize(); + note(0x6730); // a frame was offered + const uint64_t began = mach_absolute_time(); + const gpupresent::Result r = presenter_->present({(const void *)(uintptr_t)swap.address, (size_t)((swap.bytes + page - 1) & ~(page - 1)), + (size_t)swap.stride, swap.width, swap.height}); + { + static mach_timebase_info_data_t base; + if (!base.denom) mach_timebase_info(&base); + stats_->lastMs = (uint32_t)(((mach_absolute_time() - began) * base.numer / base.denom) / 1000000); + stats_->totalMs += stats_->lastMs; + } + stats_->lastResult = (int)r; + if (r == gpupresent::kOk && stats_->verified < 4) verify(swap); + note(0x6731); // the GPU copy returned + NGPComplete done = {swap.sequence, r == gpupresent::kOk ? 0u : 0xe00002bcu /* kIOReturnError */, 0}; + const int completed = transport_.call(transport_.context, NGP_METHOD_COMPLETE_SWAP, &done, sizeof done, nullptr, 0); + if (completed) {++stats_->failures; stats_->error = completed; return false;} + if (r == gpupresent::kOk) ++stats_->frames; else ++stats_->failures; + return true; + } +private: + // Reads the scanout back with aligned 32-bit loads (the framebuffer is device memory; wider loads can fault) and compares the top rows with the + // surface the GPU copied. The lower part of the panel is skipped: the diagnostic renderer paints there. Evidence only; nothing is written. + void verify(const NGPSwap &swap) { + const uint32_t rows = swap.height < 400 ? swap.height : 400, words = swap.width; + const volatile uint32_t *shown = (const volatile uint32_t *)scanout_.base; + const volatile uint32_t *surface = (const volatile uint32_t *)(uintptr_t)swap.address; + for (uint32_t y = 0; y < rows; ++y) { + const volatile uint32_t *a = shown + (size_t)y * (scanout_.stride / 4), *b = surface + (size_t)y * (swap.stride / 4); + for (uint32_t x = 0; x < words; ++x) + if (a[x] != b[x]) {++stats_->mismatchRows; break;} + } + ++stats_->verified; + } + void note(uint32_t value) {if (transport_.mark) transport_.mark(transport_.context, value);} + bool fail(int step, int error) {stats_->step = step; stats_->error = error; return false;} + Transport transport_ = {}; + gpupresent::Presenter *presenter_ = nullptr; + gpupresent::Scanout scanout_ = {}; + Stats *stats_ = nullptr; +}; + +} // namespace gpuscanout diff --git a/GpuScanoutPresenter.h b/GpuScanoutPresenter.h new file mode 100644 index 0000000..9a9e938 --- /dev/null +++ b/GpuScanoutPresenter.h @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT +// GPU presentation: the GPU copies a finished BGRA frame (the compositor's IOSurface pages) into the firmware scanout memory. Both are +// imported without a copy (newBufferWithBytesNoCopy: pinned by the kernel and mapped into the GPU's 4 KiB context), the copy is a blit on +// the GPU's queue, and the call returns after the GPU retired it. No CPU touches a pixel; there is no staging copy and no software fallback +// here. Used through the public Metal API, on any device (the native frontend on the tablet, the Mac's device for the host check). +#pragma once +#import +#include +#include +#include + +namespace gpupresent { + +struct Scanout { + void *base; // writable mapping of the firmware framebuffer (page aligned) + size_t bytes; // whole extent, a multiple of the page size + size_t stride; // bytes per row + uint32_t width, height; +}; + +struct Frame { + const void *base; // page-aligned mapping of the surface pages + size_t bytes; // allocation size, a multiple of the page size + size_t stride; + uint32_t width, height; +}; + +enum Result { kOk = 0, kBadFrame, kBadScanout, kNoResource, kGpuFailed }; + +class Presenter { +public: + Presenter(id device, const Scanout &scanout) : device_(device), scanout_(scanout) { + queue_ = [device newCommandQueue]; + if (!validScanout() || !queue_) return; + target_ = [device newBufferWithBytesNoCopy:scanout.base length:scanout.bytes options:MTLResourceStorageModeShared deallocator:nil]; + } + bool ready() const { return target_ != nil; } + // One frame: encodes the GPU copy, commits, waits for the GPU. Rows are copied as whole rows of width*4 bytes. + Result present(const Frame &frame) { + if (!ready()) return kBadScanout; + const size_t page = (size_t)getpagesize(), row = (size_t)frame.width * 4; + if (!frame.base || ((uintptr_t)frame.base & (page - 1)) || !frame.bytes || (frame.bytes & (page - 1)) || + frame.width != scanout_.width || frame.height != scanout_.height || frame.stride < row || + (size_t)frame.height > frame.bytes / frame.stride || frame.stride * (frame.height - 1) + row > frame.bytes) return kBadFrame; + id source = [device_ newBufferWithBytesNoCopy:(void *)frame.base length:frame.bytes options:MTLResourceStorageModeShared deallocator:nil]; + if (!source) return kNoResource; + id commands = [queue_ commandBuffer]; + id blit = commands ? [commands blitCommandEncoder] : nil; + if (!blit) return kNoResource; + if (frame.stride == scanout_.stride) { + // Equal pitch: the rows are one contiguous run. + [blit copyFromBuffer:source sourceOffset:0 toBuffer:target_ destinationOffset:0 size:frame.stride * (frame.height - 1) + row]; + } else { + // A linear texture over the surface pages reads them with the surface's pitch and writes the scanout's pitch. + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:frame.width + height:frame.height mipmapped:NO]; + d.storageMode = MTLStorageModeShared; + d.usage = MTLTextureUsageShaderRead; + if (frame.stride % [device_ minimumLinearTextureAlignmentForPixelFormat:MTLPixelFormatBGRA8Unorm]) { + for (uint32_t y = 0; y < frame.height; ++y) + [blit copyFromBuffer:source sourceOffset:y * frame.stride toBuffer:target_ destinationOffset:y * scanout_.stride size:row]; + } else { + id texture = [source newTextureWithDescriptor:d offset:0 bytesPerRow:frame.stride]; + if (!texture) {[blit endEncoding]; return kNoResource;} + [blit copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) + sourceSize:MTLSizeMake(frame.width, frame.height, 1) toBuffer:target_ destinationOffset:0 + destinationBytesPerRow:scanout_.stride destinationBytesPerImage:scanout_.stride * frame.height]; + } + } + [blit endEncoding]; + [commands commit]; + [commands waitUntilCompleted]; + return commands.status == MTLCommandBufferStatusCompleted ? kOk : kGpuFailed; + } +private: + bool validScanout() const { + const size_t page = (size_t)getpagesize(); + return scanout_.base && !((uintptr_t)scanout_.base & (page - 1)) && scanout_.bytes && !(scanout_.bytes & (page - 1)) && + scanout_.width && scanout_.height && scanout_.stride >= (size_t)scanout_.width * 4 && + scanout_.stride * scanout_.height <= scanout_.bytes; + } + id device_; + Scanout scanout_; + id queue_; + id target_; +}; + +} // namespace gpupresent diff --git a/HostMetalCommandServer.h b/HostMetalCommandServer.h new file mode 100644 index 0000000..6c5502d --- /dev/null +++ b/HostMetalCommandServer.h @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: MIT +#import +#import +NSDictionary *ServeOfflineMetalCommands(const char *,id,NSArray> *,NSArray *,unsigned expectedSessions=0); diff --git a/HostMetalCommandServer.mm b/HostMetalCommandServer.mm new file mode 100644 index 0000000..bd7d548 --- /dev/null +++ b/HostMetalCommandServer.mm @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: MIT +// Actual host Metal objects and commands. Guest requests select operations; +// no pre-scripted draw, completion or texture content is returned. +#import "HostMetalCommandServer.h" +#import +#import +#import +#include "OfflineMetalWire.h" +#include "OfflineMetalFunctionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalVertexWire.h" +#include "OfflineMetalFenceWire.h" +#include "OfflineMetalCommandWire.h" +#include "OfflineMetalEventWire.h" +#include "OfflineMetalSharedEventWire.h" +#include "OfflineMetalSessionWire.h" +#include +#include +#include +#include +#include +#include +#include +#include +static bool exact(int fd,void *bytes,size_t size,bool sending) { + unsigned char *p=(unsigned char *)bytes; + while(size){ssize_t n=sending?write(fd,p,size):read(fd,p,size);if(n<0 && errno==EINTR)continue;if(n<=0)return false;p+=n;size-=n;}return true; +} +static NSString *hashBytes(const void *bytes,size_t size) { + if(size>UINT32_MAX)return nil;unsigned char hash[32];CC_SHA256(bytes,(CC_LONG)size,hash); + NSMutableString *s=[NSMutableString string];for(unsigned i=0;i<32;i++)[s appendFormat:@"%02x",hash[i]];return s; +} +static NSData *jsonBytes(id value) {return [NSJSONSerialization dataWithJSONObject:value options:0 error:nil];} +NSDictionary *ServeOfflineMetalCommands(const char *path,id device,NSArray> *libraries,NSArray *libraryRows,unsigned expectedSessions) { + if(expectedSessions>128)return nil; + if(strlen(path)>=sizeof(((sockaddr_un *)nullptr)->sun_path))return nil; + NSMutableDictionary *fenceProducers=[NSMutableDictionary dictionary],*eventProducers=[NSMutableDictionary dictionary]; + NSMutableArray *eventsCreated=[NSMutableArray array],*eventOperations=[NSMutableArray array],*sharedValues=[NSMutableArray array],*sharedLabels=[NSMutableArray array],*notificationJournal=[NSMutableArray array],*exportJournal=[NSMutableArray array]; + NSMutableDictionary *sharedExports=[NSMutableDictionary dictionary]; + NSMutableDictionary *sharedStates=[NSMutableDictionary dictionary]; + NSMutableDictionary *notifications=[NSMutableDictionary dictionary]; + NSLock *notificationLock=[NSLock new];MTLSharedEventListener *sharedListener=[[MTLSharedEventListener alloc] initWithDispatchQueue:dispatch_get_global_queue(QOS_CLASS_DEFAULT,0)]; + uint64_t nextExport=1,nextNotification=1;uint32_t controlRequest=1; + NSMutableArray *fencesCreated=[NSMutableArray array],*fenceOperations=[NSMutableArray array],*textureHazards=[NSMutableArray array]; + int listener=socket(AF_UNIX,SOCK_STREAM,0);if(listener<0)return nil; + sockaddr_un address={};address.sun_family=AF_UNIX;strcpy(address.sun_path,path); + if(bind(listener,(sockaddr *)&address,sizeof(address)) || listen(listener,1)){close(listener);return nil;} + fprintf(stderr,"OFFLINE_METAL_READY\n");fflush(stderr); + int fd=accept(listener,nullptr,nullptr);close(listener);if(fd<0)return nil; + timeval timeout={90,0};setsockopt(fd,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout));setsockopt(fd,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout)); + int yes=1;setsockopt(fd,SOL_SOCKET,SO_NOSIGPIPE,&yes,sizeof(yes)); + NSMutableDictionary *objects=[NSMutableDictionary dictionary];uint64_t next=1; + NSMutableDictionary *sessions=[NSMutableDictionary dictionary]; + NSMutableDictionary *sessionRequests=[NSMutableDictionary dictionary]; + uint64_t nextSession=1;unsigned openedSessions=0,closedSessions=0; + NSMutableArray *completed=[NSMutableArray array],*readbacks=[NSMutableArray array],*operations=[NSMutableArray array],*uploads=[NSMutableArray array]; + NSMutableArray *bufferWrites=[NSMutableArray array],*bufferReads=[NSMutableArray array],*bufferCopies=[NSMutableArray array]; + NSMutableArray *queuePriorities=[NSMutableArray array]; + NSMutableArray *enqueued=[NSMutableArray array]; + NSMutableDictionary *pendingEnqueues=[NSMutableDictionary dictionary]; + NSMutableArray *textureCopies=[NSMutableArray array]; + NSMutableArray *depthStates=[NSMutableArray array],*stencilReferences=[NSMutableArray array],*attachmentPasses=[NSMutableArray array]; + NSMutableDictionary *openEncoders=[NSMutableDictionary dictionary];NSMutableArray *abortedEncoders=[NSMutableArray array]; + NSMutableArray *specializations=[NSMutableArray array]; + NSMutableDictionary *asyncCommands=[NSMutableDictionary dictionary]; + NSMutableArray *submitted=[NSMutableArray array],*commandObservations=[NSMutableArray array],*failedCommands=[NSMutableArray array]; + NSMutableSet *recordedCompletions=[NSMutableSet set],*recordedFailures=[NSMutableSet set]; + auto observeCommand=[&](uint64_t handle,id command,NSString *reason)->OMCommandResult { + OMCommandResult value={(uint64_t)command.status,0,0}; + if(value.status==MTLCommandBufferStatusCompleted) { + value.start_ns=(uint64_t)(command.GPUStartTime*1e9);value.end_ns=(uint64_t)(command.GPUEndTime*1e9); + if(value.start_ns && value.end_ns>value.start_ns && ![recordedCompletions containsObject:@(handle)]) { + [recordedCompletions addObject:@(handle)];[completed addObject:@{@"command":@(handle),@"status":@(value.status),@"gpu_start_ns":@(value.start_ns),@"gpu_end_ns":@(value.end_ns),@"observed_by":reason}]; + } + } else if(value.status==MTLCommandBufferStatusError && ![recordedFailures containsObject:@(handle)]) { + [recordedFailures addObject:@(handle)];[failedCommands addObject:@{@"command":@(handle),@"status":@(value.status),@"actual_error":command.error.localizedDescription?:@""}]; + } + return value; + }; + auto finishSubmitted=[&](uint64_t selected,bool all,NSString *reason) { + for(NSNumber *handle in asyncCommands.allKeys) { + NSDictionary *row=asyncCommands[handle];if(!all && [row[@"session"] unsignedLongLongValue]!=selected)continue; + id command=row[@"command"];[command waitUntilCompleted]; + observeCommand(handle.unsignedLongLongValue,command,reason);[asyncCommands removeObjectForKey:handle]; + } + }; + auto abortEncoders=[&](uint64_t selected,bool all,NSString *reason) { + for(NSNumber *handle in openEncoders.allKeys) { + NSDictionary *row=openEncoders[handle];if(!all && [row[@"session"] unsignedLongLongValue]!=selected)continue; + id encoder=row[@"encoder"];id command=row[@"command_object"]; + uint64_t status=command.status;[encoder endEncoding]; + [abortedEncoders addObject:@{@"encoder":handle,@"command":row[@"command_handle"],@"session":row[@"session"], + @"actual_command_status":@(status),@"reason":reason,@"encoding_ended":@YES,@"gpu_submission_created_by_cleanup":@NO}]; + [openEncoders removeObjectForKey:handle]; + } + }; + auto hasOpenEncoder=[&](uint64_t command)->bool { + for(NSDictionary *row in openEncoders.allValues)if([row[@"command_handle"] unsignedLongLongValue]==command)return true;return false; + }; + auto cancelNotifications=[&](uint64_t selected,bool all) { + [notificationLock lock]; + for(NSNumber *key in notifications.allKeys) { + NSMutableDictionary *record=notifications[key];if(!all && [record[@"session"] unsignedLongLongValue]!=selected)continue; + record[@"cancelled"]=@YES;[notificationJournal addObject:@{@"notification":key,@"session":record[@"session"],@"event":record[@"event"],@"requested":record[@"requested"],@"actually_fired":record[@"ready"],@"actual_value":record[@"value"]}];[notifications removeObjectForKey:key]; + } + [notificationLock unlock]; + }; + NSMutableDictionary *renderExtents=[NSMutableDictionary dictionary];NSMutableArray *blendPipelines=[NSMutableArray array],*textureBindings=[NSMutableArray array],*vertexPipelines=[NSMutableArray array]; + uint32_t bootBytes=0;bool finished=false,healthy=true; + auto save=[&](id object)->uint64_t {if(!object)return 0;uint64_t key=next++;objects[@(key)]=object;return key;}; + for(uint32_t request=1;request<=8192 && healthy && !finished;request++) { + @autoreleasepool { + OMWireHeader header={}; + if(request==1) { + uint32_t prefix=0,count=0; + do {unsigned char byte=0;if(!exact(fd,&byte,1,false)){healthy=false;break;} + prefix=(prefix>>8)|((uint32_t)byte<<24);count++;if(count>65536){healthy=false;break;} + } while(count<4 || prefix!=OM_WIRE_MAGIC); + if(!healthy)break;bootBytes=count-4;header.magic=prefix; + if(!exact(fd,(unsigned char *)&header+4,sizeof(header)-4,false)){healthy=false;break;} + } else if(!exact(fd,&header,sizeof(header),false)){healthy=false;break;} + if(header.magic!=OM_WIRE_MAGIC || header.version!=(expectedSessions?OFFLINE_METAL_SESSION_VERSION:OFFLINE_METAL_VERSION) + || (!expectedSessions && header.request!=request) || header.bytes>4080){healthy=false;break;} + uint64_t session=0;bool validSession=true; + if(expectedSessions) { + if(!exact(fd,&session,sizeof(session),false)){healthy=false;break;} + if(header.operation==OM_SESSION_OPEN)validSession=!session && header.request==1 && !header.bytes; + else if(!session && header.operation==OM_SHARED_EXPORT_DROP){validSession=header.request==controlRequest;if(validSession)controlRequest++;} + else { + objects=sessions[@(session)];NSNumber *expected=sessionRequests[@(session)]; + validSession=objects && expected && header.request==expected.unsignedIntValue; + if(validSession)sessionRequests[@(session)]=@(header.request+1); + } + } + std::vector input(header.bytes); + if(!input.empty() && !exact(fd,input.data(),input.size(),false)){healthy=false;break;} + OMResult result={};NSData *body=nil;NSError *error=nil; + auto get=[&](uint64_t handle,Protocol *protocol)->id {id object=objects[@(handle)];return object && [object conformsToProtocol:protocol]?object:nil;}; + auto getEncoder=[&](uint64_t handle,Protocol *protocol)->id {return openEncoders[@(handle)]?get(handle,protocol):nil;}; +#define REQUEST(Type) if(input.size()!=sizeof(Type)){result.error=-2;break;} Type q={};memcpy(&q,input.data(),sizeof(q)) + if(!validSession)result.error=-2; + else @try { + switch(header.operation) { + case OM_SESSION_OPEN: + if(!expectedSessions || openedSessions>=expectedSessions){result.error=-2;break;} + result.handle=nextSession++;sessions[@(result.handle)]=[NSMutableDictionary dictionary]; + sessionRequests[@(result.handle)]=@1;openedSessions++;break; + case OM_SESSION_CLOSE: + if(!expectedSessions || !input.empty()){result.error=-2;break;} + // Remove only this real namespace's independent references. Command + // buffers retain their actual GPU dependencies according to Metal. + abortEncoders(session,false,@"session-close"); + finishSubmitted(session,false,@"session-close"); + cancelNotifications(session,false); + for(NSNumber *handle in objects.allKeys){[renderExtents removeObjectForKey:handle];[fenceProducers removeObjectForKey:handle];[eventProducers removeObjectForKey:handle];NSMutableDictionary *shared=sharedStates[handle];if([shared[@"producer"][@"session"] unsignedLongLongValue]==session)[shared removeObjectForKey:@"producer"];[sharedStates removeObjectForKey:handle];} + for(NSNumber *handle in pendingEnqueues.allKeys)if([pendingEnqueues[handle][@"session"] unsignedLongLongValue]==session)[pendingEnqueues removeObjectForKey:handle]; + [objects removeAllObjects];[sessions removeObjectForKey:@(session)];[sessionRequests removeObjectForKey:@(session)]; + closedSessions++;finished=openedSessions==expectedSessions && !sessions.count && !sharedExports.count;break; + case OM_DEVICE: + if(!input.empty()){result.error=-2;break;} + body=jsonBytes(@{@"name":device.name,@"registryID":@(device.registryID),@"maxBufferLength":@(MIN(device.maxBufferLength,(NSUInteger)OFFLINE_METAL_MAX_BUFFER)), + @"lowPower":@(device.lowPower),@"removable":@(device.removable),@"headless":@(device.headless), + @"maxThreads":@[@(device.maxThreadsPerThreadgroup.width),@(device.maxThreadsPerThreadgroup.height),@(device.maxThreadsPerThreadgroup.depth)]});break; + case OM_QUEUE: + if(!input.empty()){result.error=-2;break;}result.handle=save([device newCommandQueue]);if(!result.handle)result.error=-3;break; + case OM_QUEUE_PRIORITY_GET:case OM_QUEUE_PRIORITY_SET: { + REQUEST(OMQueuePriority);id queue=get(q.queue,@protocol(MTLCommandQueue)); + if(!queue || q.background>1 || (header.operation==OM_QUEUE_PRIORITY_GET && q.priority)){result.error=-2;break;} + bool writing=header.operation==OM_QUEUE_PRIORITY_SET; + if(writing && (q.priority>5 || (!q.background && q.priority==3))){result.error=-4;break;} + SEL getter=sel_registerName(q.background?"getBackgroundGPUPriority":"getGPUPriority"); + SEL setter=sel_registerName(q.background?"setBackgroundGPUPriority:":"setGPUPriority:"); + Method readMethod=class_getInstanceMethod(object_getClass(queue),getter),writeMethod=class_getInstanceMethod(object_getClass(queue),setter); + if(!readMethod || strcmp(method_getTypeEncoding(readMethod),"Q16@0:8") || + (writing && (!writeMethod || strcmp(method_getTypeEncoding(writeMethod),"B24@0:8Q16")))){result.error=-4;break;} + uint64_t before=((NSUInteger(*)(id,SEL))objc_msgSend)(queue,getter),value=before; + if(writing)value=((BOOL(*)(id,SEL,NSUInteger))objc_msgSend)(queue,setter,q.priority)?1:0; + uint64_t after=((NSUInteger(*)(id,SEL))objc_msgSend)(queue,getter); + body=[NSData dataWithBytes:&value length:sizeof(value)]; + [queuePriorities addObject:@{@"queue":@(q.queue),@"session":@(session),@"background":@(q.background),@"writing":@(writing), + @"requested":@(q.priority),@"before":@(before),@"after":@(after),@"result":@(value)}];break; + } + case OM_LIBRARY: { + REQUEST(OMLibrary);NSMutableString *hash=[NSMutableString string];for(unsigned i=0;i<32;i++)[hash appendFormat:@"%02x",q.sha256[i]]; + NSUInteger i=0;for(;i library=libraries[i];result.handle=save(library);body=jsonBytes(library.functionNames);break; + } + case OM_FUNCTION: { + OMFunction q={};OMSpecializedFunction extended={};bool specialized=input.size()!=sizeof(q); + if(!specialized)memcpy(&q,input.data(),sizeof(q)); + else { + if(input.size()OFFLINE_METAL_MAX_CONSTANTS || input.size()!=sizeof(extended)+extended.count*sizeof(OMFunctionConstant)){result.error=-2;break;} + } + id library=get(q.library,@protocol(MTLLibrary)); + if(!library || !memchr(q.name,0,sizeof(q.name))){result.error=-2;break;} + NSString *name=[NSString stringWithUTF8String:q.name];if(!name.length){result.error=-2;break;} + id function=nil; + if(specialized) { + MTLFunctionConstantValues *values=[MTLFunctionConstantValues new];NSMutableArray *rows=[NSMutableArray array];NSMutableSet *keys=[NSMutableSet set]; + for(uint64_t i=0;i1 || row.index>65535 || + (row.named && (!memchr(row.name,0,sizeof(row.name)) || row.index)) || (!row.named && row.name[0])){result.error=-2;break;} + NSString *constantName=row.named?[NSString stringWithUTF8String:row.name]:nil; + NSString *key=row.named?[NSString stringWithFormat:@"name:%@",constantName]:[NSString stringWithFormat:@"index:%llu",row.index]; + if((row.named && !constantName.length) || [keys containsObject:key]){result.error=-2;break;}[keys addObject:key]; + if(row.named)[values setConstantValue:row.data type:(MTLDataType)row.type withName:constantName]; + else [values setConstantValue:row.data type:(MTLDataType)row.type atIndex:row.index]; + NSMutableString *hex=[NSMutableString string];for(uint64_t j=0;jMTLHazardTrackingModeTracked){result.error=-2;break;} + bool attachment=q.format==MTLPixelFormatStencil8 || q.format==MTLPixelFormatDepth32Float || q.format==MTLPixelFormatDepth32Float_Stencil8; + if(q.type!=MTLTextureType2D || (!attachment && q.format!=MTLPixelFormatR32Float && q.format!=MTLPixelFormatRG16Float && q.format!=MTLPixelFormatBGRA8Unorm && q.format!=MTLPixelFormatRGBA8Unorm) || !q.width || !q.height || q.width>2048 || q.height>2048 + || q.depth!=1 || q.levels!=1 || q.samples!=1 || q.array!=1 || (q.storage!=MTLStorageModeShared && q.storage!=MTLStorageModePrivate) || q.cache!=MTLCPUCacheModeDefaultCache){result.error=-4;break;} + if(attachment && (q.storage!=MTLStorageModePrivate || q.usage!=MTLTextureUsageRenderTarget)){result.error=-4;break;} + MTLTextureDescriptor *d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:(MTLPixelFormat)q.format width:q.width height:q.height mipmapped:NO]; + d.usage=q.usage;d.storageMode=(MTLStorageMode)q.storage;d.hazardTrackingMode=(MTLHazardTrackingMode)hazard; + id texture=[device newTextureWithDescriptor:d];if(!texture || (hazard && texture.hazardTrackingMode!=hazard)){result.error=-4;break;} + result.handle=save(texture); + if(header.operation==OM_TEXTURE_HAZARD){uint64_t actual=texture.hazardTrackingMode;body=[NSData dataWithBytes:&actual length:sizeof(actual)];[textureHazards addObject:@{@"texture":@(result.handle),@"session":@(session),@"requested":@(hazard),@"actual":@(actual)}];} + break; + } + case OM_PIPELINE:case OM_PIPELINE_BLEND:case OM_PIPELINE_ATTACHMENTS: { + OMPipeline q={};OMBlend blends[8]={};uint64_t depth=0,stencil=0;OMVertexPipeline vertexInput={};bool hasVertexInput=false; + if(header.operation==OM_PIPELINE) {if(input.size()!=sizeof(q)){result.error=-2;break;}memcpy(&q,input.data(),sizeof(q));} + else if(header.operation==OM_PIPELINE_BLEND){if(input.size()!=sizeof(OMBlendPipeline)){result.error=-2;break;}OMBlendPipeline ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.pipeline;memcpy(blends,ext.colors,sizeof(blends));} + else { + if(input.size()==sizeof(vertexInput)) {memcpy(&vertexInput,input.data(),sizeof(vertexInput));if(!OMVertexInputsValid(vertexInput)){result.error=-2;break;}hasVertexInput=true;} + else if(input.size()!=sizeof(OMAttachmentPipeline)){result.error=-2;break;} + OMAttachmentPipeline ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.colors.pipeline;memcpy(blends,ext.colors.colors,sizeof(blends));depth=ext.depthFormat;stencil=ext.stencilFormat; + } + id vertex=get(q.vertex,@protocol(MTLFunction)),fragment=get(q.fragment,@protocol(MTLFunction)); + if(!vertex || !fragment || q.samples!=1 || (depth && depth!=MTLPixelFormatDepth32Float && depth!=MTLPixelFormatDepth32Float_Stencil8) || + (stencil && stencil!=MTLPixelFormatStencil8 && stencil!=MTLPixelFormatDepth32Float_Stencil8)){result.error=-2;break;} + MTLRenderPipelineDescriptor *d=[MTLRenderPipelineDescriptor new];d.vertexFunction=vertex;d.fragmentFunction=fragment;d.rasterSampleCount=q.samples; + d.depthAttachmentPixelFormat=(MTLPixelFormat)depth;d.stencilAttachmentPixelFormat=(MTLPixelFormat)stencil; + NSMutableArray *actualAttributes=[NSMutableArray array],*actualLayouts=[NSMutableArray array]; + if(hasVertexInput) { + MTLVertexDescriptor *v=[MTLVertexDescriptor vertexDescriptor];bool used[31]={}; + for(unsigned i=0;i<31;i++) { + const auto &a=vertexInput.attributes[i];if(!a.format)continue; + v.attributes[i].format=(MTLVertexFormat)a.format;v.attributes[i].offset=a.offset;v.attributes[i].bufferIndex=a.buffer;used[a.buffer]=true; + [actualAttributes addObject:@{@"index":@(i),@"format":@(v.attributes[i].format),@"offset":@(v.attributes[i].offset),@"buffer":@(v.attributes[i].bufferIndex)}]; + } + for(unsigned i=0;i<31;i++)if(used[i]) { + const auto &l=vertexInput.layouts[i];v.layouts[i].stride=l.stride;v.layouts[i].stepFunction=(MTLVertexStepFunction)l.step;v.layouts[i].stepRate=l.rate; + [actualLayouts addObject:@{@"index":@(i),@"stride":@(v.layouts[i].stride),@"step":@(v.layouts[i].stepFunction),@"rate":@(v.layouts[i].stepRate)}]; + } + d.vertexDescriptor=v; + } + for(unsigned i=0;i<8;i++) { + d.colorAttachments[i].pixelFormat=(MTLPixelFormat)q.formats[i];d.colorAttachments[i].writeMask=(MTLColorWriteMask)q.masks[i]; + if(header.operation!=OM_PIPELINE) { + const OMBlend &b=blends[i];if(b.enabled>1 || b.sourceRGB>MTLBlendFactorOneMinusSource1Alpha || b.destinationRGB>MTLBlendFactorOneMinusSource1Alpha || + b.sourceAlpha>MTLBlendFactorOneMinusSource1Alpha || b.destinationAlpha>MTLBlendFactorOneMinusSource1Alpha || b.rgbOperation>MTLBlendOperationMax || b.alphaOperation>MTLBlendOperationMax){result.error=-2;break;} + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];a.blendingEnabled=b.enabled; + a.sourceRGBBlendFactor=(MTLBlendFactor)b.sourceRGB;a.destinationRGBBlendFactor=(MTLBlendFactor)b.destinationRGB;a.rgbBlendOperation=(MTLBlendOperation)b.rgbOperation; + a.sourceAlphaBlendFactor=(MTLBlendFactor)b.sourceAlpha;a.destinationAlphaBlendFactor=(MTLBlendFactor)b.destinationAlpha;a.alphaBlendOperation=(MTLBlendOperation)b.alphaOperation; + } + } + if(result.error)break;result.handle=save([device newRenderPipelineStateWithDescriptor:d error:&error]);if(!result.handle)result.error=-3; + if(result.handle && hasVertexInput)[vertexPipelines addObject:@{@"pipeline":@(result.handle),@"session":@(session),@"vertex":vertex.name,@"fragment":fragment.name,@"attributes":actualAttributes,@"layouts":actualLayouts}]; + if(result.handle && header.operation==OM_PIPELINE_BLEND)[blendPipelines addObject:@{@"pipeline":@(result.handle),@"vertex":vertex.name,@"fragment":fragment.name}];break; + } + case OM_FENCE: + if(!input.empty()){result.error=-2;break;} + result.handle=save([device newFence]);if(!result.handle){result.error=-4;break;} + [fencesCreated addObject:@{@"fence":@(result.handle),@"session":@(session)}];break; + case OM_RENDER_FENCE_UPDATE:case OM_RENDER_FENCE_WAIT:case OM_BLIT_FENCE_UPDATE:case OM_BLIT_FENCE_WAIT: { + REQUEST(OMFenceDependency); + bool render=header.operation==OM_RENDER_FENCE_UPDATE || header.operation==OM_RENDER_FENCE_WAIT; + bool update=header.operation==OM_RENDER_FENCE_UPDATE || header.operation==OM_BLIT_FENCE_UPDATE; + id encoder=getEncoder(q.encoder,render?@protocol(MTLRenderCommandEncoder):@protocol(MTLBlitCommandEncoder)); + id fence=get(q.fence,@protocol(MTLFence));NSDictionary *active=openEncoders[@(q.encoder)]; + if(!encoder || !fence || !active || (render?(!q.stages || (q.stages&~3ULL)):q.stages!=0)){result.error=-2;break;} + id command=active[@"command_object"];NSDictionary *producer=fenceProducers[@(q.fence)]; + if(!update) { + // Same-queue producers must be actually committed. Cross-queue fences + // stay limited to completed producers after a retained real pending- + // producer readback failure. Use actual GPU events for overlap across + // queues; CPU encoding metadata never represents a signaled fence. + id source=producer[@"command"];MTLCommandBufferStatus actual=source.status; + if(!producer || [producer[@"encoder"] unsignedLongLongValue]==q.encoder || openEncoders[producer[@"encoder"]] || + (source!=command && (actualMTLCommandBufferStatusCompleted || + (source.commandQueue!=command.commandQueue && actual!=MTLCommandBufferStatusCompleted)))){result.error=-4;break;} + } + if(render) { + if(update)[(id)encoder updateFence:fence afterStages:(MTLRenderStages)q.stages]; + else [(id)encoder waitForFence:fence beforeStages:(MTLRenderStages)q.stages]; + } else { + if(update)[(id)encoder updateFence:fence];else [(id)encoder waitForFence:fence]; + } + [fenceOperations addObject:@{@"fence":@(q.fence),@"encoder":@(q.encoder),@"session":@(session),@"command":active[@"command_handle"],@"stages":@(q.stages),@"render":@(render),@"update":@(update)}]; + if(update)fenceProducers[@(q.fence)]=@{@"encoder":@(q.encoder),@"command":command};break; + } + case OM_SHARED_EVENT: { + if(!input.empty()){result.error=-2;break;}id event=[device newSharedEvent];if(!event){result.error=-4;break;} + if(event.device && event.device.registryID!=device.registryID){result.error=-4;break;} + result.handle=save(event);sharedStates[@(result.handle)]=[NSMutableDictionary dictionary];[eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session),@"shared":@YES,@"device_is_nil":@(event.device==nil)}]; + body=jsonBytes(@{@"device_is_nil":@(event.device==nil),@"device_is_owner":@(event.device && event.device.registryID==device.registryID),@"signaled_value":@(event.signaledValue),@"label":event.label?:[NSNull null]});break; + } + case OM_SHARED_VALUE_GET: { + REQUEST(OMHandle);id event=get(q.handle,@protocol(MTLSharedEvent));if(!event){result.error=-2;break;} + uint64_t value=event.signaledValue;body=[NSData dataWithBytes:&value length:sizeof(value)]; + [sharedValues addObject:@{@"event":@(q.handle),@"session":@(session),@"value":@(value),@"cpu_set":@NO}];break; + } + case OM_SHARED_VALUE_SET: { + REQUEST(OMSharedEventValue);id event=get(q.event,@protocol(MTLSharedEvent));if(!event || q.value event=get(q.event,@protocol(MTLSharedEvent)); + if(!event || q.present>1 || q.bytes>512 || input.size()!=sizeof(q)+q.bytes || (!q.present && q.bytes)){result.error=-2;break;} + NSString *label=q.present?[[NSString alloc] initWithBytes:input.data()+sizeof(q) length:q.bytes encoding:NSUTF8StringEncoding]:nil; + if(q.present && !label){result.error=-2;break;}event.label=label;body=jsonBytes(@{@"label":event.label?:[NSNull null]}); + [sharedLabels addObject:@{@"event":@(q.event),@"session":@(session),@"actual_label":event.label?:[NSNull null]}];break; + } + case OM_SHARED_NOTIFY: { + REQUEST(OMSharedEventValue);id event=get(q.event,@protocol(MTLSharedEvent));if(!event || !sharedListener){result.error=-2;break;} + uint64_t identifier=nextNotification++;NSMutableDictionary *record=[@{@"session":@(session),@"event":@(q.event),@"requested":@(q.value),@"ready":@NO,@"value":@0,@"cancelled":@NO,@"identity_match":@NO} mutableCopy]; + [notificationLock lock];notifications[@(identifier)]=record;[notificationLock unlock];__weak id expected=event; + [event notifyListener:sharedListener atValue:q.value block:^(id actual,uint64_t value) { + [notificationLock lock];if(![record[@"cancelled"] boolValue]){record[@"ready"]=@YES;record[@"value"]=@(value);record[@"identity_match"]=@(actual==expected);}[notificationLock unlock]; + }];result.handle=identifier;break; + } + case OM_SHARED_NOTIFICATION_POLL: { + REQUEST(OMHandle);[notificationLock lock];NSMutableDictionary *record=notifications[@(q.handle)]; + if(!record || [record[@"session"] unsignedLongLongValue]!=session){result.error=-2;[notificationLock unlock];break;} + OMSharedNotification value={};value.ready=[record[@"ready"] boolValue]; + if(value.ready){if(![record[@"identity_match"] boolValue] || [record[@"value"] unsignedLongLongValue]<[record[@"requested"] unsignedLongLongValue])result.error=-3;else {value.event=[record[@"event"] unsignedLongLongValue];value.value=[record[@"value"] unsignedLongLongValue];}} + [notificationLock unlock];if(!result.error)body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_SHARED_EXPORT: { + REQUEST(OMHandle);id event=get(q.handle,@protocol(MTLSharedEvent));if(!expectedSessions || !event || !sharedStates[@(q.handle)]){result.error=-2;break;} + MTLSharedEventHandle *handle=[event newSharedEventHandle];if(!handle){result.error=-4;break;}result.handle=nextExport++;sharedExports[@(result.handle)]=@{@"handle":handle,@"state":sharedStates[@(q.handle)]}; + Method trace=class_getInstanceMethod(object_getClass(handle),sel_registerName("labelTraceID"));BOOL supported=trace && !strcmp(method_getTypeEncoding(trace),"Q16@0:8"); + uint64_t traceID=supported?((uint64_t (*)(id,SEL))objc_msgSend)(handle,sel_registerName("labelTraceID")):0; + body=jsonBytes(@{@"label":handle.label?:[NSNull null],@"trace_id_supported":@(supported),@"trace_id":supported?@(traceID):[NSNull null]}); + [exportJournal addObject:@{@"operation":@"export",@"export":@(result.handle),@"event":@(q.handle),@"session":@(session)}];break; + } + case OM_SHARED_IMPORT: { + REQUEST(OMHandle);NSDictionary *exported=sharedExports[@(q.handle)];MTLSharedEventHandle *handle=exported[@"handle"];if(!session || !handle){result.error=-2;break;} + id event=[device newSharedEventWithHandle:handle];if(!event || (event.device && event.device.registryID!=device.registryID)){result.error=-4;break;}result.handle=save(event);sharedStates[@(result.handle)]=exported[@"state"]; + body=jsonBytes(@{@"device_is_nil":@(event.device==nil),@"device_is_owner":@(event.device && event.device.registryID==device.registryID),@"signaled_value":@(event.signaledValue),@"label":event.label?:[NSNull null]}); + [eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session),@"shared":@YES,@"imported":@YES,@"device_is_nil":@(event.device==nil)}]; + [exportJournal addObject:@{@"operation":@"import",@"export":@(q.handle),@"event":@(result.handle),@"session":@(session)}];break; + } + case OM_SHARED_EXPORT_DROP: { + REQUEST(OMHandle);if(!expectedSessions || session || !sharedExports[@(q.handle)]){result.error=-2;break;}[sharedExports removeObjectForKey:@(q.handle)]; + [exportJournal addObject:@{@"operation":@"drop",@"export":@(q.handle),@"session":@0}];finished=openedSessions==expectedSessions && !sessions.count && !sharedExports.count;break; + } + case OM_EVENT: + if(!input.empty()){result.error=-2;break;} + result.handle=save([device newEvent]);if(!result.handle){result.error=-4;break;} + [eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session)}];break; + case OM_EVENT_SIGNAL:case OM_EVENT_WAIT: { + REQUEST(OMEventDependency);id command=get(q.command,@protocol(MTLCommandBuffer));id event=get(q.event,@protocol(MTLEvent)); + if(!command || !event || !q.value || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.command)){result.error=-2;break;} + NSMutableDictionary *shared=sharedStates[@(q.event)];NSDictionary *producer=shared?shared[@"producer"]:eventProducers[@(q.event)];bool signal=header.operation==OM_EVENT_SIGNAL; + uint64_t previous=[producer[@"value"] unsignedLongLongValue],actualValue=shared?[(id)event signaledValue]:0; + uint64_t actualProducer=0; + if(signal) { + if(q.value<=previous || (shared && q.value<=actualValue)){result.error=-2;break;} + [command encodeSignalEvent:event value:q.value];NSDictionary *record=@{@"value":@(q.value),@"command":command,@"session":@(session)};if(shared)shared[@"producer"]=record;else eventProducers[@(q.event)]=record; + } else { + id source=producer[@"command"];actualProducer=source.status; + if(!(shared && actualValue>=q.value) && (!producer || q.value>previous || source==command || actualProducerMTLCommandBufferStatusCompleted)){result.error=-4;break;} + [command encodeWaitForEvent:event value:q.value]; + } + [eventOperations addObject:@{@"event":@(q.event),@"command":@(q.command),@"session":@(session),@"value":@(q.value),@"signal":@(signal),@"shared":@(shared!=nil),@"actual_signaled_value":@(actualValue),@"actual_producer_status":@(actualProducer)}];break; + } + case OM_COMMAND: { + REQUEST(OMHandle);id queue=get(q.handle,@protocol(MTLCommandQueue)); + if(!queue){result.error=-2;break;}result.handle=save([queue commandBuffer]);if(!result.handle)result.error=-3;break; + } + case OM_COMMAND_ENQUEUE: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer)); + if(!command || command.status!=MTLCommandBufferStatusNotEnqueued){result.error=-2;break;} + // This synchronous transport currently supports one pending reservation + // per real queue. Refuse unsupported ordering before blocking the server + // behind a command whose predecessor cannot yet receive its commit. + for(NSDictionary *row in pendingEnqueues.allValues)if([row[@"command"] commandQueue]==command.commandQueue){result.error=-4;break;} + if(result.error)break;[command enqueue];uint64_t status=command.status; + if(status!=MTLCommandBufferStatusEnqueued){result.error=-3;break;} + pendingEnqueues[@(q.handle)]=@{@"command":command,@"session":@(session)}; + body=[NSData dataWithBytes:&status length:sizeof(status)]; + [enqueued addObject:@{@"command":@(q.handle),@"session":@(session),@"status":@(status),@"operation_request":@(request)}];break; + } + case OM_RENDER:case OM_RENDER_ATTACHMENTS: { + OMRender q={};OMDepthAttachment depth={};OMStencilAttachment stencil={}; + if(header.operation==OM_RENDER){if(input.size()!=sizeof(q)){result.error=-2;break;}memcpy(&q,input.data(),sizeof(q));} + else {if(input.size()!=sizeof(OMAttachmentRender)){result.error=-2;break;}OMAttachmentRender ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.colors;depth=ext.depth;stencil=ext.stencil;} + id command=get(q.command,@protocol(MTLCommandBuffer));if(!command || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.command)){result.error=-2;break;} + MTLRenderPassDescriptor *d=[MTLRenderPassDescriptor renderPassDescriptor];NSUInteger width=0,height=0; + for(unsigned i=0;i<8;i++)if(q.colors[i].texture) { + id texture=get(q.colors[i].texture,@protocol(MTLTexture));if(!texture || q.colors[i].load>MTLLoadActionClear || q.colors[i].store>MTLStoreActionStore || (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height; + d.colorAttachments[i].texture=texture;d.colorAttachments[i].loadAction=(MTLLoadAction)q.colors[i].load;d.colorAttachments[i].storeAction=(MTLStoreAction)q.colors[i].store; + const double *v=q.colors[i].clear;d.colorAttachments[i].clearColor=MTLClearColorMake(v[0],v[1],v[2],v[3]); + } + if(depth.texture) { + id texture=get(depth.texture,@protocol(MTLTexture)); + if(!texture || (texture.pixelFormat!=MTLPixelFormatDepth32Float && texture.pixelFormat!=MTLPixelFormatDepth32Float_Stencil8) || + depth.load>MTLLoadActionClear || depth.store>MTLStoreActionStore || !std::isfinite(depth.clear) || depth.clear<0 || depth.clear>1 || + (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height;d.depthAttachment.texture=texture;d.depthAttachment.loadAction=(MTLLoadAction)depth.load; + d.depthAttachment.storeAction=(MTLStoreAction)depth.store;d.depthAttachment.clearDepth=depth.clear; + } + if(stencil.texture) { + id texture=get(stencil.texture,@protocol(MTLTexture)); + if(!texture || (texture.pixelFormat!=MTLPixelFormatStencil8 && texture.pixelFormat!=MTLPixelFormatDepth32Float_Stencil8) || + stencil.load>MTLLoadActionClear || stencil.store>MTLStoreActionStore || stencil.clear>255 || + (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height;d.stencilAttachment.texture=texture;d.stencilAttachment.loadAction=(MTLLoadAction)stencil.load; + d.stencilAttachment.storeAction=(MTLStoreAction)stencil.store;d.stencilAttachment.clearStencil=(uint32_t)stencil.clear; + } + if(!width || !height){result.error=-2;break;} + if(result.error)break;result.handle=save([command renderCommandEncoderWithDescriptor:d]);if(!result.handle)result.error=-3; + else { + renderExtents[@(result.handle)]=@[@(width),@(height)]; + openEncoders[@(result.handle)]=@{@"encoder":objects[@(result.handle)],@"command_object":command,@"command_handle":@(q.command),@"session":@(session)}; + if(depth.texture || stencil.texture)[attachmentPasses addObject:@{@"command":@(q.command),@"encoder":@(result.handle),@"session":@(session), + @"depth":@(depth.texture),@"stencil":@(stencil.texture),@"depth_clear":@(depth.clear),@"stencil_clear":@(stencil.clear), + @"depth_load":@(depth.load),@"stencil_load":@(stencil.load),@"width":@(width),@"height":@(height)}]; + }break; + } + case OM_DEPTH_STENCIL_STATE: { + REQUEST(OMDepthStencil); + auto validFace=[](const OMStencilFace &f)->bool { + if(f.enabled>1)return false; + if(!f.enabled)return !(f.compare|f.failure|f.depthFailure|f.pass|f.readMask|f.writeMask); + return f.compare<=MTLCompareFunctionAlways && f.failure<=MTLStencilOperationDecrementWrap && + f.depthFailure<=MTLStencilOperationDecrementWrap && f.pass<=MTLStencilOperationDecrementWrap && f.readMask<=UINT32_MAX && f.writeMask<=UINT32_MAX; + }; + if(q.compare>MTLCompareFunctionAlways || q.write>1 || !validFace(q.front) || !validFace(q.back)){result.error=-2;break;} + auto face=[](const OMStencilFace &f)->MTLStencilDescriptor * { + if(!f.enabled)return nil;MTLStencilDescriptor *s=[MTLStencilDescriptor new];s.stencilCompareFunction=(MTLCompareFunction)f.compare; + s.stencilFailureOperation=(MTLStencilOperation)f.failure;s.depthFailureOperation=(MTLStencilOperation)f.depthFailure;s.depthStencilPassOperation=(MTLStencilOperation)f.pass; + s.readMask=(uint32_t)f.readMask;s.writeMask=(uint32_t)f.writeMask;return s; + }; + MTLDepthStencilDescriptor *d=[MTLDepthStencilDescriptor new];d.depthCompareFunction=(MTLCompareFunction)q.compare;d.depthWriteEnabled=q.write; + d.frontFaceStencil=face(q.front);d.backFaceStencil=face(q.back);result.handle=save([device newDepthStencilStateWithDescriptor:d]); + if(!result.handle){result.error=-3;break;} + [depthStates addObject:@{@"state":@(result.handle),@"session":@(session),@"depth_compare":@(q.compare),@"depth_write":@(q.write), + @"front_compare":@(q.front.compare),@"back_compare":@(q.back.compare),@"front_write_mask":@(q.front.writeMask),@"back_write_mask":@(q.back.writeMask)}];break; + } + case OM_SET_DEPTH_STENCIL: { + REQUEST(OMSetDepthStencil);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id state=q.state?get(q.state,@protocol(MTLDepthStencilState)):nil; + if(!encoder || (q.state && !state)){result.error=-2;break;}[encoder setDepthStencilState:state];break; + } + case OM_STENCIL_REFERENCE: { + REQUEST(OMStencilReference);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || q.front>UINT32_MAX || q.back>UINT32_MAX){result.error=-2;break;} + [encoder setStencilFrontReferenceValue:(uint32_t)q.front backReferenceValue:(uint32_t)q.back]; + [stencilReferences addObject:@{@"encoder":@(q.encoder),@"session":@(session),@"front":@(q.front),@"back":@(q.back)}];break; + } + case OM_SET_PIPELINE: { + REQUEST(OMSetPipeline);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id pipeline=get(q.pipeline,@protocol(MTLRenderPipelineState)); + if(!encoder || !pipeline){result.error=-2;break;}[encoder setRenderPipelineState:pipeline];break; + } + case OM_SET_CULL: { + REQUEST(OMSetCull);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));if(!encoder || q.mode>MTLCullModeBack){result.error=-2;break;}[encoder setCullMode:(MTLCullMode)q.mode];break; + } + case OM_DRAW: { + REQUEST(OMDraw);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || q.primitive>MTLPrimitiveTypeTriangleStrip || !q.count || q.count>65536 || q.start>65536 || !q.instances || q.instances>1024){result.error=-2;break;} + [encoder drawPrimitives:(MTLPrimitiveType)q.primitive vertexStart:q.start vertexCount:q.count instanceCount:q.instances];break; + } + case OM_END: { + REQUEST(OMHandle);id encoder=get(q.handle,@protocol(MTLCommandEncoder));if(!encoder || !openEncoders[@(q.handle)]){result.error=-2;break;}[encoder endEncoding];[openEncoders removeObjectForKey:@(q.handle)];break; + } + case OM_COMMIT:case OM_COMMAND_SUBMIT: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer));if(!command || hasOpenEncoder(q.handle)){result.error=-2;break;} + if(command.status>MTLCommandBufferStatusEnqueued){result.error=-2;break;} + for(NSNumber *handle in pendingEnqueues)if(handle.unsignedLongLongValue!=q.handle && [pendingEnqueues[handle][@"command"] commandQueue]==command.commandQueue){result.error=-4;break;} + if(result.error)break;[command commit];[pendingEnqueues removeObjectForKey:@(q.handle)]; + if(header.operation==OM_COMMIT)[command waitUntilCompleted]; + else asyncCommands[@(q.handle)]=@{@"session":@(session),@"command":command}; + OMCommandResult value=observeCommand(q.handle,command,header.operation==OM_COMMIT?@"blocking-commit":@"nonblocking-submit");error=command.error; + if(value.statusMTLCommandBufferStatusCompleted || error || + (value.status==MTLCommandBufferStatusCompleted && (!value.start_ns || value.end_ns<=value.start_ns))){result.error=-3;break;} + if(header.operation==OM_COMMAND_SUBMIT)[submitted addObject:@{@"command":@(q.handle),@"session":@(session),@"actual_status":@(value.status),@"wait_until_completed_called":@NO}]; + body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_COMMAND_STATUS: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer)); + if(!command || command.statusMTLCommandBufferStatusCompleted || error || (value.status==MTLCommandBufferStatusCompleted && (!value.start_ns || value.end_ns<=value.start_ns))){result.error=-3;break;} + body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_READ_TEXTURE: { + REQUEST(OMReadTexture);id texture=get(q.texture,@protocol(MTLTexture)); + if(!texture || texture.storageMode!=MTLStorageModeShared || q.level || !q.width || !q.height || q.x>texture.width || q.width>texture.width-q.x + || q.y>texture.height || q.height>texture.height-q.y || (texture.pixelFormat!=MTLPixelFormatR32Float && texture.pixelFormat!=MTLPixelFormatRG16Float && texture.pixelFormat!=MTLPixelFormatBGRA8Unorm && texture.pixelFormat!=MTLPixelFormatRGBA8Unorm) + || q.width>OFFLINE_METAL_MAX_REPLY/4/q.height){result.error=-2;break;} + NSMutableData *pixels=[NSMutableData dataWithLength:q.width*q.height*4]; + [texture getBytes:pixels.mutableBytes bytesPerRow:q.width*4 fromRegion:MTLRegionMake2D(q.x,q.y,q.width,q.height) mipmapLevel:0];body=pixels; + [readbacks addObject:@{@"texture":@(q.texture),@"bytes":@(pixels.length),@"sha256":hashBytes(pixels.bytes,pixels.length)}];break; + } + case OM_WRITE_TEXTURE: { + if(input.size() texture=get(q.texture,@protocol(MTLTexture)); + if(!texture || texture.storageMode!=MTLStorageModeShared || q.level || !q.width || !q.height || q.x>texture.width || q.width>texture.width-q.x + || q.y>texture.height || q.height>texture.height-q.y || (texture.pixelFormat!=MTLPixelFormatR32Float && texture.pixelFormat!=MTLPixelFormatRG16Float && texture.pixelFormat!=MTLPixelFormatBGRA8Unorm && texture.pixelFormat!=MTLPixelFormatRGBA8Unorm) + || q.width>SIZE_MAX/4 || q.stride!=q.width*4 || q.height>(input.size()-sizeof(q))/q.stride + || q.height*q.stride!=input.size()-sizeof(q)){result.error=-2;break;} + const void *pixels=input.data()+sizeof(q); + [texture replaceRegion:MTLRegionMake2D(q.x,q.y,q.width,q.height) mipmapLevel:0 withBytes:pixels bytesPerRow:q.stride]; + [uploads addObject:@{@"texture":@(q.texture),@"x":@(q.x),@"y":@(q.y),@"width":@(q.width),@"height":@(q.height), + @"bytes":@(q.stride*q.height),@"sha256":hashBytes(pixels,q.stride*q.height)}];break; + } + case OM_BUFFER: { + REQUEST(OMBuffer); + if(!q.bytes || q.bytes>OFFLINE_METAL_MAX_BUFFER || q.bytes>device.maxBufferLength || q.options){result.error=-4;break;} + result.handle=save([device newBufferWithLength:q.bytes options:MTLResourceStorageModeShared]);if(!result.handle)result.error=-3;break; + } + case OM_WRITE_BUFFER: { + if(input.size() buffer=get(q.buffer,@protocol(MTLBuffer)); + if(!buffer || !q.bytes || q.offset>buffer.length || q.bytes>buffer.length-q.offset || q.bytes!=input.size()-sizeof(q)){result.error=-2;break;} + memcpy((unsigned char *)buffer.contents+q.offset,input.data()+sizeof(q),q.bytes); + [bufferWrites addObject:@{@"buffer":@(q.buffer),@"offset":@(q.offset),@"bytes":@(q.bytes),@"sha256":hashBytes(input.data()+sizeof(q),q.bytes)}];break; + } + case OM_READ_BUFFER: { + REQUEST(OMBufferRegion);id buffer=get(q.buffer,@protocol(MTLBuffer)); + if(!buffer || !q.bytes || q.bytes>OFFLINE_METAL_MAX_REPLY || q.offset>buffer.length || q.bytes>buffer.length-q.offset){result.error=-2;break;} + body=[NSData dataWithBytes:(unsigned char *)buffer.contents+q.offset length:q.bytes]; + [bufferReads addObject:@{@"buffer":@(q.buffer),@"offset":@(q.offset),@"bytes":@(q.bytes),@"sha256":hashBytes(body.bytes,body.length)}];break; + } + case OM_VERTEX_BUFFER:case OM_FRAGMENT_BUFFER: { + REQUEST(OMSetBuffer);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + id buffer=q.buffer?get(q.buffer,@protocol(MTLBuffer)):nil; + if(!encoder || q.index>=31 || (q.buffer && !buffer) || (!buffer && q.offset) || (buffer && q.offset>=buffer.length)){result.error=-2;break;} + if(header.operation==OM_VERTEX_BUFFER)[encoder setVertexBuffer:buffer offset:q.offset atIndex:q.index]; + else [encoder setFragmentBuffer:buffer offset:q.offset atIndex:q.index];break; + } + case OM_DRAW_INDEXED: { + REQUEST(OMDrawIndexed);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id buffer=get(q.buffer,@protocol(MTLBuffer)); + uint64_t stride=q.type==MTLIndexTypeUInt16?2:4; + if(!encoder || !buffer || q.primitive>MTLPrimitiveTypeTriangleStrip || q.type>MTLIndexTypeUInt32 || !q.count || + q.count>65536 || !q.instances || q.instances>1024 || q.offset%stride || q.offset>buffer.length || q.count>(buffer.length-q.offset)/stride){result.error=-2;break;} + [encoder drawIndexedPrimitives:(MTLPrimitiveType)q.primitive indexCount:q.count indexType:(MTLIndexType)q.type + indexBuffer:buffer indexBufferOffset:q.offset instanceCount:q.instances baseVertex:q.baseVertex baseInstance:q.baseInstance];break; + } + case OM_BLIT: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer));if(!command || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.handle)){result.error=-2;break;} + result.handle=save([command blitCommandEncoder]);if(!result.handle)result.error=-3; + else openEncoders[@(result.handle)]=@{@"encoder":objects[@(result.handle)],@"command_object":command,@"command_handle":@(q.handle),@"session":@(session)};break; + } + case OM_COPY_BUFFER: { + REQUEST(OMCopyBuffer);id encoder=getEncoder(q.encoder,@protocol(MTLBlitCommandEncoder)); + id source=get(q.source,@protocol(MTLBuffer)),destination=get(q.destination,@protocol(MTLBuffer)); + if(!encoder || !source || !destination || !q.bytes || ((q.sourceOffset|q.destinationOffset|q.bytes)&3) || q.sourceOffset>source.length || q.bytes>source.length-q.sourceOffset || + q.destinationOffset>destination.length || q.bytes>destination.length-q.destinationOffset || + (source==destination && q.sourceOffset encoder=getEncoder(q.encoder,@protocol(MTLBlitCommandEncoder)); + id source=get(q.source,@protocol(MTLTexture)),destination=get(q.destination,@protocol(MTLTexture)); + if(!encoder || !source || !destination || source==destination || source.pixelFormat!=destination.pixelFormat || + source.textureType!=MTLTextureType2D || destination.textureType!=MTLTextureType2D || + q.sourceSlice || q.destinationSlice || q.sourceLevel || q.destinationLevel || q.sourceOrigin[2] || q.destinationOrigin[2] || q.sourceSize[2]!=1 || + !q.sourceSize[0] || !q.sourceSize[1] || q.sourceOrigin[0]>source.width || q.sourceSize[0]>source.width-q.sourceOrigin[0] || + q.sourceOrigin[1]>source.height || q.sourceSize[1]>source.height-q.sourceOrigin[1] || + q.destinationOrigin[0]>destination.width || q.sourceSize[0]>destination.width-q.destinationOrigin[0] || + q.destinationOrigin[1]>destination.height || q.sourceSize[1]>destination.height-q.destinationOrigin[1]){result.error=-2;break;} + if(source.pixelFormat!=MTLPixelFormatR32Float && source.pixelFormat!=MTLPixelFormatRG16Float && source.pixelFormat!=MTLPixelFormatRGBA8Unorm && source.pixelFormat!=MTLPixelFormatBGRA8Unorm){result.error=-4;break;} + [encoder copyFromTexture:source sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(q.sourceOrigin[0],q.sourceOrigin[1],0) + sourceSize:MTLSizeMake(q.sourceSize[0],q.sourceSize[1],1) toTexture:destination destinationSlice:0 destinationLevel:0 + destinationOrigin:MTLOriginMake(q.destinationOrigin[0],q.destinationOrigin[1],0)]; + [textureCopies addObject:@{@"source":@(q.source),@"destination":@(q.destination),@"session":@(session),@"format":@(source.pixelFormat), + @"source_storage":@(source.storageMode),@"destination_storage":@(destination.storageMode),@"width":@(q.sourceSize[0]),@"height":@(q.sourceSize[1])}];break; + } + case OM_VERTEX_TEXTURE:case OM_FRAGMENT_TEXTURE: { + REQUEST(OMSetTexture);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id texture=q.texture?get(q.texture,@protocol(MTLTexture)):nil; + if(!encoder || q.index>=128 || (q.texture && !texture)){result.error=-2;break;} + if(header.operation==OM_VERTEX_TEXTURE)[encoder setVertexTexture:texture atIndex:q.index];else [encoder setFragmentTexture:texture atIndex:q.index]; + [textureBindings addObject:@{@"encoder":@(q.encoder),@"texture":@(q.texture),@"index":@(q.index),@"vertex":@(header.operation==OM_VERTEX_TEXTURE)}];break; + } + case OM_SCISSOR: { + REQUEST(OMScissor);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));NSArray *extent=renderExtents[@(q.encoder)]; + if(!encoder || !extent || !q.width || !q.height || q.x>[extent[0] unsignedLongLongValue] || q.y>[extent[1] unsignedLongLongValue] || + q.width>[extent[0] unsignedLongLongValue]-q.x || q.height>[extent[1] unsignedLongLongValue]-q.y){result.error=-2;break;} + [encoder setScissorRect:(MTLScissorRect){(NSUInteger)q.x,(NSUInteger)q.y,(NSUInteger)q.width,(NSUInteger)q.height}];break; + } + case OM_VIEWPORT: { + REQUEST(OMViewport);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || !std::isfinite(q.x) || !std::isfinite(q.y) || !std::isfinite(q.width) || !std::isfinite(q.height) || + !std::isfinite(q.nearDepth) || !std::isfinite(q.farDepth) || q.width<=0 || q.height<=0 || q.nearDepth<0 || q.farDepth>1 || q.nearDepth>q.farDepth){result.error=-2;break;} + [encoder setViewport:(MTLViewport){q.x,q.y,q.width,q.height,q.nearDepth,q.farDepth}];break; + } + case OM_BLEND_COLOR: { + REQUEST(OMBlendColor);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || !std::isfinite(q.red) || !std::isfinite(q.green) || !std::isfinite(q.blue) || !std::isfinite(q.alpha)){result.error=-2;break;} + [encoder setBlendColorRed:q.red green:q.green blue:q.blue alpha:q.alpha];break; + } + case OM_RELEASE: {REQUEST(OMHandle);if(!objects[@(q.handle)]){result.error=-2;break;}[objects removeObjectForKey:@(q.handle)];[renderExtents removeObjectForKey:@(q.handle)];[fenceProducers removeObjectForKey:@(q.handle)];[eventProducers removeObjectForKey:@(q.handle)];[sharedStates removeObjectForKey:@(q.handle)];break;} + case OM_FINISH:if(expectedSessions || !input.empty()){result.error=-2;break;}finishSubmitted(0,true,@"finish");finished=true;break; + default:result.error=-4;break; + } + } @catch(NSException *exception) {result.error=-5;body=[exception.reason dataUsingEncoding:NSUTF8StringEncoding];} +#undef REQUEST + if(error)body=[error.localizedDescription dataUsingEncoding:NSUTF8StringEncoding]; + if(body.length>OFFLINE_METAL_MAX_REPLY){result.error=-2;body=nil;} + if(result.error) { + NSString *reason=[[NSString alloc] initWithData:body?:[NSData data] encoding:NSUTF8StringEncoding]; + fprintf(stderr,"OFFLINE_METAL_ERROR request=%u operation=%u error=%d %s\n",request,header.operation,result.error, + reason?[[reason substringToIndex:MIN(reason.length,(NSUInteger)512)] UTF8String]:"");fflush(stderr); + } + [operations addObject:@{@"request":@(request),@"client_request":@(header.request),@"session":@(session),@"operation":@(header.operation),@"input_bytes":@(input.size()),@"error":@(result.error),@"handle":@(result.handle),@"result_bytes":@(body.length)}]; + header.operation|=OM_WIRE_RESPONSE;header.bytes=(uint32_t)(sizeof(result)+body.length); + healthy=exact(fd,&header,sizeof(header),true) && (!expectedSessions || exact(fd,&session,sizeof(session),true)) + && exact(fd,&result,sizeof(result),true) && (!body.length || exact(fd,(void *)body.bytes,body.length,true)); + } + } + close(fd);unlink(path); + // End abandoned encoding before releasing its real objects. Never commit + // interrupted buffers or invent a GPU completion to make teardown succeed. + abortEncoders(0,true,healthy?@"unclosed-at-finish":@"transport-lost"); + finishSubmitted(0,true,healthy?@"finish":@"transport-lost"); + cancelNotifications(0,true); + return @{@"host_device":device.name,@"operations":operations,@"completed_commands":completed,@"texture_readbacks":readbacks,@"texture_uploads":uploads, + @"transport_healthy":@(healthy),@"session_finished":@(finished),@"enqueued_commands":enqueued,@"pending_enqueues":@(pendingEnqueues.count), + @"aborted_encoders":abortedEncoders,@"open_encoders":@(openEncoders.count), + @"shared_event_values":sharedValues,@"shared_event_labels":sharedLabels,@"shared_event_notifications":notificationJournal,@"shared_event_exports":exportJournal,@"live_shared_exports":@(sharedExports.count), + @"events_created":eventsCreated,@"event_operations":eventOperations, + @"submitted_commands":submitted,@"command_observations":commandObservations,@"failed_commands":failedCommands,@"pending_submissions":@(asyncCommands.count), + @"function_specializations":specializations,@"fences_created":fencesCreated,@"fence_operations":fenceOperations,@"texture_hazard_modes":textureHazards, + @"buffer_uploads":bufferWrites,@"buffer_readbacks":bufferReads,@"buffer_copies":bufferCopies,@"queue_priorities":queuePriorities,@"texture_copies":textureCopies, + @"depth_stencil_states":depthStates,@"stencil_references":stencilReferences,@"attachment_passes":attachmentPasses, + @"blend_pipelines":blendPipelines,@"texture_bindings":textureBindings,@"vertex_pipelines":vertexPipelines, + @"boot_console_prefix_bytes":@(bootBytes),@"isolated_sessions":@(expectedSessions!=0),@"opened_sessions":@(openedSessions),@"closed_sessions":@(closedSessions), + @"cpu_pixel_renderer":@NO,@"guest_system_device_discovery":@NO, + @"adreno_execution":@NO,@"original_desktop":@NO}; +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..424da21 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 NativeMetal contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NativeAdrenoMetalPlugin.mm b/NativeAdrenoMetalPlugin.mm new file mode 100644 index 0000000..00b8b9b --- /dev/null +++ b/NativeAdrenoMetalPlugin.mm @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: MIT +// Adreno Metal plugin, reached through the original discovery (IOAcceleratorES -> MetalPluginName -> NSBundle -> principal class -> +// initWithAcceleratorPort:). Default (stage 1): it runs the shared userspace GPU test (NativeAdrenoUserTest.h) from inside a real system +// process, under that process's sandbox, and then returns nil. No device is registered, so the original software rendering keeps running unchanged. +// Published (the image carries the file NativeAdrenoMetal.publish beside the bundle): backboardd, the one process that owns the GPU, returns the native +// Metal frontend's device (the metal2vulkan runtime on Turnip, on the Adreno 740), which the original discovery registers; the CARenderServer then +// renders the system UI through it, and the display driver's frames are written to the scanout by the GPU (GpuScanoutClient.h). Every other process +// returns nil and keeps the software renderer. +#import +#include +#include +#include +#include +#include +#include "NativeAdrenoUserTest.h" +#include "NativeMetalBridgePhase.h" +#include "CacheExport.h" + +@interface _MTLDevice : NSObject +@end +@interface NativeAdrenoMetalDeviceBase : NSObject // only a lock object for the published mode's one-time setup +@end +@implementation NativeAdrenoMetalDeviceBase +@end +@interface NativeAdrenoMetalDevice : _MTLDevice +- (id)initWithAcceleratorPort:(unsigned)port; +@end + +// ---- published mode ---- +namespace publish { +// The image's marker: a file beside the bundle in the union view of /System/Library/Extensions (the bundle's own seal stays untouched). +// A file beside the bundle: its path in path, true when it is readable. +static bool requestedPath(const char *name, char *path, size_t size) { + Dl_info info = {}; + if (!dladdr(reinterpret_cast(&requestedPath), &info) || !info.dli_fname) return false; + snprintf(path, size, "%s", info.dli_fname); + char *slash = strrchr(path, '/'); // the bundle's executable: /NativeAdrenoMetal.bundle/NativeAdrenoMetal + if (!slash) return false; + *slash = 0; + slash = strrchr(path, '/'); + if (!slash) return false; + snprintf(slash + 1, size - size_t(slash + 1 - path), "%s", name); + return access(path, R_OK) == 0; +} +static bool requested() { + char path[1024]; + return requestedPath("NativeAdrenoMetal.publish", path, sizeof path); +} +static Gpu *makeGpu(unsigned port) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return nullptr; + Gpu *gpu = new Gpu; + gpu->call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod"); + gpu->map = (MapFn)dlsym(iokit, "IOConnectMapMemory64"); + gpu->unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64"); + gpu->openFn = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen"); + gpu->closeFn = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose"); + gpu->port = port; + return gpu; +} +static std::string gCacheDirectory; // the runtime's cache directory (the one that opened the device), exported by CacheExport.h +// Progress markers 0x68xx (GPU6): 6800 entered, 6801 Turnip found, 6802 device made, 6803 returned to the discovery, 68f1 no Turnip, 68f2 no device. +static unsigned gDiagBits; +// Reads of the render server's own state for the framebuffer-format question (diagnostic only, for the exact QuartzCore of the target image; a read +// that fails or a build that differs reports zeros). Each value is one marker, 0x7Kvv: K picks the item, vv the value. +static bool readTask(uintptr_t address, void *out, size_t size) { + vm_size_t got = 0; + return vm_read_overwrite(mach_task_self(), address, size, (vm_address_t) out, &got) == KERN_SUCCESS && got == size; +} +static bool writeTask(uintptr_t address, const void *data, size_t size) { + return vm_write(mach_task_self(), address, (vm_offset_t) data, (mach_msg_type_number_t) size) == KERN_SUCCESS; +} +// EXPERIMENT (second report onward): this display path takes plain 8-bit BGRA only, so tell the render server's display that it has neither the deep +// (10-bit) framebuffer nor universal buffer compression. Whether the state it reads those from is the whole story is what the markers show. +static unsigned gReports; +struct DisplaySample {unsigned flags = 0, mode = 0, behaviour = 0, globals = 0; uint16_t display = 0; uintptr_t object = 0;}; +static DisplaySample sampleDisplay(id first, intptr_t slide) { + DisplaySample out; + uintptr_t a = first ? (uintptr_t) *(void **) ((char *) (__bridge void *) first + 8) : 0, b = 0, d = 0; + if (a && readTask(a + 0x40, &b, 8) && b && readTask(b + 0x60, &d, 8) && d) { + uint32_t f = 0; uint64_t m = 0; uint8_t byte[4] = {}; + readTask(d + 0x238, &f, 4); + readTask(d + 0x258, &out.display, 2); + readTask(d + 0x248, &m, 8); + readTask(d + 0x6c17, &byte[0], 1); + readTask(d + 0x6658 + 0x57d, &byte[1], 1); + readTask(d + 0x6658 + 0x58a, &byte[2], 1); + readTask(d + 0x6658 + 0x59b, &byte[3], 1); + out.object = d; + out.flags = f & 0xff; + out.mode = (unsigned) ((m >> 55) & 31) | (m ? 0x20 : 0); + out.behaviour = (byte[0] ? 1 : 0) | (byte[1] ? 2 : 0) | (byte[2] ? 4 : 0) | (byte[3] ? 8 : 0); + } + uint8_t v[4] = {}; + readTask(0x1f491ae10 + slide, &v[0], 1); // the render server's deep-framebuffer flag + readTask(0x1ed7abf10 + slide, &v[1], 1); // framebuffer compression in effect + readTask(0x1f4919dc0 + slide, &v[2], 1); // its default + readTask(0x1f4919c20 + slide, &v[3], 1); // universal buffer compression + out.globals = (v[0] ? 1 : 0) | (v[1] ? 2 : 0) | (v[2] ? 4 : 0) | (v[3] ? 8 : 0); + return out; +} +static void markSample(Gpu &gpu, const DisplaySample &x, unsigned base) { + gpu.mark(base | (x.flags << 4) | x.globals); // display flags (bit 3 deep), globals: deep/compression/default/universal + gpu.mark((base + 0x1000) | (x.display >> 4)); // display limits word: 0x200 compression support, 0x1c00 type + gpu.mark((base + 0x2000) | (x.mode << 4) | x.behaviour); // mode colour field, ICC/other behaviour bytes +} +static void reportDisplayState(Gpu &gpu) { + ++gReports; + static DisplaySample before; + static bool changed; + Class serverClass = NSClassFromString(@"CAWindowServer"); + void *start = dlsym(RTLD_DEFAULT, "CARenderServerStart"); + if (!serverClass || !start) return; + const intptr_t slide = (intptr_t) start - 0x189651efc; + @try { + id server = ((id (*)(id, SEL)) objc_msgSend)(serverClass, NSSelectorFromString(@"serverIfRunning")); + NSArray *displays = server ? ((id (*)(id, SEL)) objc_msgSend)(server, NSSelectorFromString(@"displays")) : nil; + id first = displays.count ? displays[0] : nil; + DisplaySample now = sampleDisplay(first, slide); + if (!changed) { + before = now; + if (gReports >= 2 && now.object) { + uint32_t f = 0; + if (readTask(now.object + 0x238, &f, 4) && (f & 8)) {f &= ~8u; writeTask(now.object + 0x238, &f, 4);} + uint8_t zero = 0; + writeTask(0x1f4919c20 + slide, &zero, 1); + changed = true; + now = sampleDisplay(first, slide); + } + } + markSample(gpu, before, 0x7000); + markSample(gpu, now, 0xA000); + } @catch (NSException *) {} +} +static id deviceForPort(unsigned port) { + static id device; + static bool tried; + @synchronized([NativeAdrenoMetalDeviceBase class]) { + if (tried) return device; + tried = true; + // One process owns the GPU (the driver grants one connection): the render server. The others keep the software renderer. + if (strcmp(getprogname(), "backboardd") != 0) return nil; + // The discovery releases the accelerator port when it is done; the monitor and the scanout client keep using it. + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); + } + Gpu *gpu = makeGpu(port); + if (!gpu) return nil; + gpu->mark(0x6800); + char turnip[1024]; + if (!turnip::libraryPath(turnip, sizeof turnip) || access(turnip, R_OK) != 0) {gpu->mark(0x68f1); return nil;} + gpu->mark(0x6801); + // The runtime refuses to open without a writable shader-cache directory, and which directories backboardd's sandbox lets it write is not + // known from here: try the user cache and temp directories, then /tmp and /var/tmp, and use the first that opens a device. + uint64_t registryID = 0; + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto entryID = (int (*)(unsigned, uint64_t *))dlsym(iokit, "IORegistryEntryGetRegistryEntryID")) entryID(port, ®istryID); + } + // What the render server sees for its framebuffer choice (marker 0x6Axx): bit0 the deep-framebuffer preference is set, bit1 it is false, + // bit2 MobileGestalt's answer behind the render server's deep flag is true, bit3 it exists, bit4 the compression capability dictionary exists, + // bit5 the process is not root. + { + unsigned bits = 0; + if (CFTypeRef value = CFPreferencesCopyAppValue(CFSTR("CAEnableDeepFramebuffer"), kCFPreferencesCurrentApplication)) { + bits |= 1; + if (CFGetTypeID(value) == CFBooleanGetTypeID() && !CFBooleanGetValue((CFBooleanRef) value)) bits |= 2; + CFRelease(value); + } + if (auto copyAnswer = (CFTypeRef (*)(CFStringRef)) dlsym(RTLD_DEFAULT, "MGCopyAnswer")) { + if (CFTypeRef deep = copyAnswer(CFSTR("Aixt/MEN2O2B7f+8m4TxUA"))) { + bits |= 8; + if (CFGetTypeID(deep) == CFBooleanGetTypeID() && CFBooleanGetValue((CFBooleanRef) deep)) bits |= 4; + CFRelease(deep); + } + if (CFTypeRef caps = copyAnswer(CFSTR("AoKnINTLPoKML3ctoP0AZg"))) {bits |= 16; CFRelease(caps);} + } + if (getuid() != 0) bits |= 32; + gDiagBits = bits; + gpu->mark(0x6A00 | bits); + } + // The render server compresses its framebuffer (an Apple/AGX layout this GPU does not write) unless told not to; the display driver takes plain rows. + setenv("CA_DISABLE_FRAMEBUFFER_COMPRESSION", "1", 1); + // The runtime's creation log reports each VkPipeline making's time to the device's sink, which the telemetry's V row sums. + setenv("METAL2VULKAN_LOG_PIPELINES", "1", 0); + CFPreferencesSetAppValue(CFSTR("CAEnableFramebufferCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); + CFPreferencesSetAppValue(CFSTR("CAEnableIOSurfaceCompression"), kCFBooleanFalse, kCFPreferencesCurrentApplication); + char candidates[4][512] = {}; + if (!confstr(_CS_DARWIN_USER_CACHE_DIR, candidates[0], sizeof candidates[0] - 16)) candidates[0][0] = 0; + if (!confstr(_CS_DARWIN_USER_TEMP_DIR, candidates[1], sizeof candidates[1] - 16)) candidates[1][0] = 0; + strlcpy(candidates[2], "/tmp/", sizeof candidates[2]); + strlcpy(candidates[3], "/private/var/tmp/", sizeof candidates[3]); + for (unsigned i = 0; i < 4 && !device; ++i) { + if (!candidates[i][0]) continue; + strlcat(candidates[i], "m2v", sizeof candidates[i]); + mkdir(candidates[i], 0700); + if (access(candidates[i], W_OK) != 0) continue; + // The image's pipeline pack (an earlier boot's export), into this directory before the runtime reads it. Marker 0x6Cnn: files written + // (saturated at 0xfe), 0x6Cff the pack was refused. + if (char pack[1024]; requestedPath("NativeAdrenoMetal.m2vpack", pack, sizeof pack)) { + const int imported = cacheexport::importPack(pack, candidates[i]); + if (imported) gpu->mark(0x6C00 | (imported < 0 ? 0xff : imported > 0xfe ? 0xfe : imported)); + } + @try { + device = native_bridge_metal_open(turnip, candidates[i], registryID, nullptr); + } @catch (NSException *exception) { + device = nil; + } + if (device) {gpu->mark(0x6810 + i); gCacheDirectory = candidates[i];} // which directory served + } + if (!device) { + gpu->mark(0x68f2); + char text[64]; + snprintf(text, sizeof text, "PUB no device: %.40s", nmb_last_error()); + streamText(*gpu, text); + return nil; + } + gpu->mark(0x6802); + // The status line (GPU8) every ten seconds, and the display hand-off client on its own thread: both only read or serve, neither is the discovery's. + pthread_t monitor, scanout; + pthread_attr_t attributes; + pthread_attr_init(&attributes); + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + pthread_attr_setstacksize(&attributes, 1u << 20); + pthread_create(&monitor, &attributes, [](void *arg) -> void * { + Gpu *g = static_cast(arg); + for (;;) { + sleep(10); + char status[64], text[64]; + static unsigned phase; + // status (counters, then the last two refusals' texts), formats; the stream carries 63 characters per line, so a long status goes + // out in 57-character pieces on consecutive lines (the UFS log keeps every line). + char full[420]; + native_bridge_metal_status(full, sizeof full); + const size_t length = strlen(full); + const unsigned pieces = (unsigned)((length + 56) / 57); + const unsigned step = phase++ % (1 + (pieces ? (pieces > 7 ? 7 : pieces) : 1)); + if (step == 0) { + native_bridge_metal_formats(status, sizeof status); + snprintf(text, sizeof text, "PUB %.58s", status); + } else { + snprintf(text, sizeof text, "ST%u %.57s", step - 1, full + (size_t)(step - 1) * 57); + } + streamText(*g, text); + reportDisplayState(*g); + g->mark(0x6A00 | gDiagBits); // the diagnostics page lists the last few markers only + } + return nullptr; + }, gpu); + // The fixed CPU/memory/pipeline workload of native_bridge_metal_benchmark, once while the render server starts and once on an idle + // desktop, against the host's run of the same (pipeline_timing_host draw=2): how much slower the tablet is, and where. + pthread_t benchmark; + pthread_create(&benchmark, &attributes, [](void *arg) -> void * { + id dev = (__bridge id)arg; + sleep(3); + native_bridge_metal_benchmark(dev, "/System/Library/Frameworks/QuartzCore.framework/default.metallib", 8, nullptr, 0); + sleep(150); + native_bridge_metal_benchmark(dev, "/System/Library/Frameworks/QuartzCore.framework/default.metallib", 8, nullptr, 0); + return nullptr; + }, (__bridge void *)device); + pthread_t telemetry; + pthread_create(&telemetry, &attributes, [](void *arg) -> void * { + Gpu *g = static_cast(arg); + sleep(5); + // The bridge's telemetry table, resent whole every four seconds as markers (two characters each): 0xC000|row starts a row of the + // kernel's table, 0x8000|(a<<7)|b appends two characters. Rows are at most 90 characters; 48 rows. + for (;;) { + static char table[6144]; + native_bridge_metal_telemetry(table, sizeof table); + unsigned row = 0; + for (char *line = table; *line && row < 48;) { + char *end = strchr(line, '\n'); + if (end) *end = 0; + g->mark(0xC000u | row); + const size_t length = strlen(line) < 90 ? strlen(line) : 90; + for (size_t i = 0; i < length; i += 2) { + const unsigned a = (unsigned char)line[i] & 0x7f, b = i + 1 < length ? (unsigned char)line[i + 1] & 0x7f : 0; + g->mark(0x8000u | (a << 7) | b); + } + ++row; + if (!end) break; + line = end + 1; + } + sleep(6); + } + return nullptr; + }, gpu); + pthread_create(&scanout, &attributes, [](void *arg) -> void * { + Gpu *g = static_cast(arg); + sleep(3); + @try { + bridgephase::serveScanout(*g, device); + } @catch (NSException *exception) { + g->mark(0x68f3); + } + return nullptr; + }, gpu); + pthread_t exporter; + pthread_create(&exporter, &attributes, [](void *arg) -> void * { + cacheexport::run(*static_cast(arg), gCacheDirectory); + return nullptr; + }, gpu); + pthread_attr_destroy(&attributes); + gpu->mark(0x6803); + return device; + } +} +} // namespace publish + +@implementation NativeAdrenoMetalDevice +- (id)initWithAcceleratorPort:(unsigned)port { + self = [super init]; + if (!self || !port) return nil; + if (publish::requested()) return publish::deviceForPort(port); + static bool attempted; + if (attempted) return nil; + attempted = true; + // The test runs on its own thread: a kernel call that never returns (int-042: BoFree) then stalls that thread only, not SpringBoard's + // main thread (the UI froze with the test running inline). + // The discovery code releases the accelerator port as soon as this returns nil, which is before the test thread has started: the first + // marker (and every later open) then went to a dead name and never reached the kernel (K0 in int-043..052, with the plugin file opened by + // SpringBoard). Keep one extra user reference on the port, and let the thread look the service up itself as well. + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); + } + struct Job {unsigned port; char name[32];}; + Job *job = new Job{port, {}}; + strlcpy(job->name, getprogname(), sizeof job->name); + pthread_t thread; + pthread_attr_t attributes; + pthread_attr_init(&attributes); + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + if (pthread_create(&thread, &attributes, [](void *arg) -> void * { + Job *j = static_cast(arg); + unsigned port = j->port; + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + auto matching = (void *(*)(const char *))dlsym(iokit, "IOServiceMatching"); + auto getService = (unsigned (*)(unsigned, void *))dlsym(iokit, "IOServiceGetMatchingService"); + if (matching && getService) { + const unsigned own = getService(0, matching("IOAcceleratorES")); + if (own) port = own; + } + } + gAfterTurnip = bridgephase::run; + adrenoUserTest(port, j->name); + delete j; + return nullptr; + }, job) != 0) delete job; + pthread_attr_destroy(&attributes); + return nil; +} +@end diff --git a/NativeAdrenoProbeDaemon.cpp b/NativeAdrenoProbeDaemon.cpp new file mode 100644 index 0000000..a5f3b29 --- /dev/null +++ b/NativeAdrenoProbeDaemon.cpp @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT +// Adreno probe daemon: started by launchd from an owner job (a union view over /System/Library/LaunchDaemons, see the boot-time addon +// mount), so it runs outside every sandbox profile. It waits for the Adreno accelerator service to appear, runs the shared userspace GPU +// test once and exits. The result reaches the kernel through the driver's progress markers and SetMetadata. +#include +#include "NativeAdrenoUserTest.h" + +using MatchingFn = void *(*)(const char *); +using GetServiceFn = unsigned (*)(unsigned, void *); + +int main() { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return 1; + auto matching = (MatchingFn)dlsym(iokit, "IOServiceMatching"); + auto getService = (GetServiceFn)dlsym(iokit, "IOServiceGetMatchingService"); + if (!matching || !getService) return 2; + // The service is published a few seconds into boot (after the GPU is powered); poll for up to two minutes. + unsigned port = 0; + for (unsigned i = 0; i < 240 && !port; ++i) { + port = getService(0, matching("IOAcceleratorES")); + if (!port) usleep(500000); + } + if (!port) return 3; + return adrenoUserTest(port, "adrenoprobed") >= 0 ? 0 : 4; +} diff --git a/NativeAdrenoUserTest.h b/NativeAdrenoUserTest.h new file mode 100644 index 0000000..2ed55dd --- /dev/null +++ b/NativeAdrenoUserTest.h @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: MIT +// The userspace GPU test shared by the Adreno Metal plugin (runs inside whichever system process the original Metal discovery loads it +// into, under that process's sandbox) and the launchd probe daemon (no sandbox profile). It proves the whole userspace path to the GPU: +// open the driver, move every ABI method through an admitted selector (or the mailbox page), map buffer objects write-combined or +// uncached, write an indirect buffer from userspace, submit it, wait for the fence, read the result back, and report one line to the +// kernel through SetMetadata. Progress markers are IOServiceOpen calls with type 0x4147, which the driver counts and refuses. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "NativeAdrenoABI.h" +#include "NativeTurnipSmoke.h" +#if __has_include("TurnipReplay.h") +#include "TurnipReplay.h" +#define NATIVE_TURNIP_REPLAY 1 +#endif +extern "C" { +#include "AdrenoPM4.h" +} + +namespace { +using IOServiceOpenFn = int (*)(unsigned, unsigned, uint32_t, unsigned *); +using IOServiceCloseFn = int (*)(unsigned); +using CallFn = int (*)(unsigned, uint32_t, const void *, size_t, void *, size_t *); +using MapFn = int (*)(unsigned, uint32_t, unsigned, uint64_t *, uint64_t *, uint32_t); +using UnmapFn = int (*)(unsigned, uint32_t, unsigned, uint64_t); + +// The sandbox admits only some external-method selector numbers for this user-client class, and which ones is decided by the +// profile, not by us. Every ABI method therefore travels through one admitted selector as {magic, method, payload}; the driver +// recognises the magic before its own selector table. Probing 0..255 finds an admitted selector without knowing the profile. +constexpr uint32_t kTunnelMagic = 0x4e544e4c, kProbe = 0xffffffffu; +struct Gpu { + unsigned conn = 0; + CallFn call = nullptr; + MapFn map = nullptr; + UnmapFn unmap = nullptr; + uint32_t selector = 0; + unsigned admitted = 0; + uint64_t lastTicks = 0; + unsigned port = 0; + IOServiceOpenFn openFn = nullptr; + IOServiceCloseFn closeFn = nullptr; + // Progress marker: an IOServiceOpen with type 0x4147. The driver counts it and refuses it, so it needs no selector, no + // mapping and no connection, and survives a sandbox that admits nothing else or a crash after it. + void mark(uint32_t value) { + unsigned c = 0; + if (openFn && port && openFn(port, mach_task_self(), 0x41470000u | (value & 0xffff), &c) == 0 && c && closeFn) closeFn(c); + } + volatile uint32_t *mailbox = nullptr; + uint32_t mailboxSeq = 0; + int mailboxMethod(uint32_t m, const void *in, size_t is, void *out, size_t os) { + if (!mailbox || is > 0x700 || os > 0x700) return -1; + if (is) memcpy(const_cast(mailbox) + 0x10, in, is); + mailbox[3] = m; + mailbox[5] = (uint32_t)is; + mailbox[6] = (uint32_t)os; + __sync_synchronize(); + const uint32_t seq = ++mailboxSeq; + mailbox[1] = seq; + __sync_synchronize(); + for (unsigned i = 0; i < 6000 && mailbox[2] != seq; ++i) usleep(500); // the kernel poller runs at about 1 kHz + if (mailbox[2] != seq) return -2; + __sync_synchronize(); + const int status = (int)mailbox[4]; + if (!status && os) memcpy(out, const_cast(mailbox) + 0x200, os); + return status; + } + int method(uint32_t m, const void *in, size_t is, void *out, size_t os) { + if (mailbox) return mailboxMethod(m, in, is, out, os); + uint8_t buffer[8 + sizeof(NativeAdrenoAbiSubmit)]; + if (is > sizeof(NativeAdrenoAbiSubmit)) return -1; + const uint32_t header[2] = {kTunnelMagic, m}; + memcpy(buffer, header, 8); + if (is) memcpy(buffer + 8, in, is); + size_t o = os; + return call(conn, selector, buffer, 8 + is, os ? out : nullptr, os ? &o : nullptr); + } + // No selector is admitted: use the mailbox page (IOConnectMapMemory64 is not covered by the selector filter). + bool openMailbox() { + uint64_t address = 0, length = 0; + if (map(conn, 0xC0DE, mach_task_self(), &address, &length, 1) != 0 || length < 4096) return false; + mailbox = reinterpret_cast(address); + return mailbox[0] == 0x4d41494c; + } + // Returns true when some selector was admitted; the first one becomes the tunnel selector. + bool probe() { + const uint32_t header[2] = {kTunnelMagic, kProbe}; + bool found = false; + for (uint32_t s = 0; s < 256; ++s) { + if (call(conn, s, header, sizeof header, nullptr, nullptr) == 0) { + if (!found) selector = s; + found = true; + ++admitted; + } + } + return found; + } +}; + +void report(Gpu &gpu, const char *process, int step, int error, int verified, int verifiedWc, uint64_t ticks) { + // SetMetadata with a bogus handle: the driver records the text before it validates the handle. + NativeAdrenoAbiMetadata meta = {}; + meta.handle = 1; + char text[NativeAdrenoAbiMetadataBytes] = {}; + snprintf(text, sizeof text, "%.9s s%d e%x v%d/%d t%llx a%u/%u", process, step, error, verified, verifiedWc, + (unsigned long long)ticks, gpu.mailbox ? 0xffffu : gpu.admitted, gpu.selector); + meta.bytes = (uint32_t)strlen(text); + memcpy(meta.text, text, meta.bytes); + gpu.method(NativeAdrenoMethodSetMetadata, &meta, sizeof meta, nullptr, 0); +} + +// Everything the Turnip backend does for a first submission, in the same order; returns the verified dword count or -1. +int runTest(Gpu &gpu, uint32_t mapFlags, int &step, int &error) { + auto check = [&](int status) { + if (status && !error) error = status; + if (!status) ++step; + gpu.mark(status ? 0x4000 | (status & 0xfff) : 0x3000 | (mapFlags == 0x401 ? 0x100 : 0) | step); + return status == 0; + }; + NativeAdrenoAbiParams params = {}; + if (!check(gpu.method(NativeAdrenoMethodQuery, nullptr, 0, ¶ms, sizeof params))) return -1; + NativeAdrenoAbiDeviceInfo info = {}; + if (!check(gpu.method(NativeAdrenoMethodGetDeviceInfo, nullptr, 0, &info, sizeof info))) return -1; + NativeAdrenoAbiBoAlloc alloc = {4096, 12, 0}; + NativeAdrenoAbiBoAllocReply ib = {}, target = {}; + if (!check(gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &ib, sizeof ib)) || + !check(gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &target, sizeof target))) return -1; + const uint64_t ibIova = info.va_start, tgtIova = info.va_start + 0x1000; + NativeAdrenoAbiBoMap mapIb = {ib.handle, ibIova, 1, 0, 2, 0}, mapTgt = {target.handle, tgtIova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + if (!check(gpu.method(NativeAdrenoMethodBoMap, &mapIb, sizeof mapIb, &mr, sizeof mr)) || + !check(gpu.method(NativeAdrenoMethodBoMap, &mapTgt, sizeof mapTgt, &mr, sizeof mr))) return -1; + NativeAdrenoAbiBoCpuMap cmIb = {ib.handle}, cmTgt = {target.handle}; + NativeAdrenoAbiBoCpuMapReply crIb = {}, crTgt = {}; + if (!check(gpu.method(NativeAdrenoMethodBoCpuMap, &cmIb, sizeof cmIb, &crIb, sizeof crIb)) || + !check(gpu.method(NativeAdrenoMethodBoCpuMap, &cmTgt, sizeof cmTgt, &crTgt, sizeof crTgt))) return -1; + uint64_t ibAddr = 0, ibLen = 0, tgtAddr = 0, tgtLen = 0; + if (!check(gpu.map(gpu.conn, crIb.token, mach_task_self(), &ibAddr, &ibLen, mapFlags)) || + !check(gpu.map(gpu.conn, crTgt.token, mach_task_self(), &tgtAddr, &tgtLen, mapFlags))) return -1; + if (ibLen < 4096 || tgtLen < 4096) {error = 0xbad1; return -1;} + volatile uint32_t *iw = reinterpret_cast(ibAddr); + volatile uint32_t *tw = reinterpret_cast(tgtAddr); + for (unsigned i = 0; i < 1024; ++i) tw[i] = 0xdeadbeef; + const uint32_t values[3] = {0x0adc0de1u, 0xfeed0002u, 0x600d0003u}; + uint32_t words[12]; + for (unsigned i = 0; i < 3; ++i) { + if (AdrenoPM4Type7(0x3d, 3, &words[i * 4]) != AdrenoPM4Success) {error = 0xbad2; return -1;} + words[i * 4 + 1] = uint32_t(tgtIova + i * 4); + words[i * 4 + 2] = uint32_t(tgtIova >> 32); + words[i * 4 + 3] = values[i]; + } + for (unsigned i = 0; i < 12; ++i) iw[i] = words[i]; + __asm__ volatile("dsb sy" ::: "memory"); // the mapping is non-cacheable and the GPU is an outer-shareable observer + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {ib.handle, ibIova, sizeof words}; + NativeAdrenoAbiSubmitReply sr = {}; + if (!check(gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr))) return -1; + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + if (!check(gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0))) return -1; + __asm__ volatile("dsb sy" ::: "memory"); + const int verified = (tw[0] == values[0]) + (tw[1] == values[1]) + (tw[2] == values[2]) + (tw[3] == 0xdeadbeef); + NativeAdrenoAbiTimestamp ts = {}; + uint64_t ticks = 0; + if (check(gpu.method(NativeAdrenoMethodGetTimestamp, nullptr, 0, &ts, sizeof ts))) ticks = ts.ticks; + gpu.unmap(gpu.conn, crIb.token, mach_task_self(), ibAddr); + gpu.unmap(gpu.conn, crTgt.token, mach_task_self(), tgtAddr); + NativeAdrenoAbiBoCpuUnmap ui = {ib.handle, crIb.token, 0}, ut = {target.handle, crTgt.token, 0}; + check(gpu.method(NativeAdrenoMethodBoCpuUnmap, &ui, sizeof ui, nullptr, 0)); + check(gpu.method(NativeAdrenoMethodBoCpuUnmap, &ut, sizeof ut, nullptr, 0)); + NativeAdrenoAbiBoFree fi = {ib.handle}, ft = {target.handle}; + check(gpu.method(NativeAdrenoMethodBoFree, &fi, sizeof fi, nullptr, 0)); + check(gpu.method(NativeAdrenoMethodBoFree, &ft, sizeof ft, nullptr, 0)); + gpu.lastTicks = ticks; + return verified; +} +} + +// A report line as a marker stream (0x7fff, then 0x70cc per character): works while another connection holds the driver's single slot. +// Two threads of one process may report (the published mode's status monitor and the scanout client): the driver keeps one text cursor, so a line +// goes out whole. +static pthread_mutex_t gStreamLock = PTHREAD_MUTEX_INITIALIZER; +static void streamText(Gpu &gpu, const char *text) { + pthread_mutex_lock(&gStreamLock); + gpu.mark(0x7fff); + for (unsigned i = 0; i < 63 && text[i]; ++i) gpu.mark(0x7000 | (uint8_t)text[i]); + pthread_mutex_unlock(&gStreamLock); +} + +// Line into report (GPU7) instead of report2 (GPU8). +__attribute__((unused)) static void streamText7(Gpu &gpu, const char *text) { + pthread_mutex_lock(&gStreamLock); + gpu.mark(0x7ffe); + for (unsigned i = 0; i < 63 && text[i]; ++i) gpu.mark(0x7000 | (uint8_t)text[i]); + pthread_mutex_unlock(&gStreamLock); +} + +// Remap probe (int-064): does the CP read the new pages after a GPU address is moved to other physical pages? int-063's W1 run (the +// replay IB on fresh buffers at the same addresses) stopped both CP threads in the SQE's unknown-opcode trap, i.e. on wrong data. A +// CP_MEM_TO_MEM from a remapped page into a result page reports what the CP sees instead of executing it, so no variant can stop the CP: +// v0 first mapping, v1 remap to a page allocated before the old one is freed (so the old page keeps its old pattern), +// v2 as v1 with a UCHE invalidate (CP_EVENT_WRITE7 0x31) and CP_WAIT_FOR_IDLE before the copy, +// v3 as v1 but remapped while the CP is busy spinning on a flag (CP_WAIT_REG_MEM), then released: GX stays up across the TLB invalidate. +// Each result: '1' the new pattern, 'o' the previous mapping's pattern, 'z' zero, 'd' result not written, 'g' other, 'x' setup failed, +// 't' fence timeout. Reported as "P" in front of the replay line. The addresses (0x5f000000..) are never used elsewhere. +struct ProbeBo { + uint64_t handle = 0, address = 0; + uint32_t token = 0; + volatile uint32_t *words = nullptr; +}; +static int probeAlloc(Gpu &gpu, ProbeBo &b, uint32_t fill) { + NativeAdrenoAbiBoAlloc alloc = {4096, 12, 0}; + NativeAdrenoAbiBoAllocReply reply = {}; + if (int e = gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &reply, sizeof reply)) return e; + b.handle = reply.handle; + NativeAdrenoAbiBoCpuMap cm = {b.handle}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + b.token = cr.token; + uint64_t length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &b.address, &length, 0x101)) return e; + if (length < 4096) return 0xbad5; + b.words = reinterpret_cast(b.address); + for (unsigned i = 0; i < 1024; ++i) b.words[i] = fill; + __asm__ volatile("dsb sy" ::: "memory"); + return 0; +} +static int probeMap(Gpu &gpu, const ProbeBo &b, uint64_t iova) { + NativeAdrenoAbiBoMap map = {b.handle, iova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + return gpu.method(NativeAdrenoMethodBoMap, &map, sizeof map, &mr, sizeof mr); +} +static void probeFree(Gpu &gpu, ProbeBo &b) { + if (!b.handle) return; + if (b.address) { + gpu.unmap(gpu.conn, b.token, mach_task_self(), b.address); + NativeAdrenoAbiBoCpuUnmap cu = {b.handle, b.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + } + NativeAdrenoAbiBoFree f = {b.handle}; + gpu.method(NativeAdrenoMethodBoFree, &f, sizeof f, nullptr, 0); + b = ProbeBo(); +} +static char gProbe[6] = "P----"; +__attribute__((unused)) static void remapProbe(Gpu &gpu) { + constexpr uint64_t kSource = 0x5f000000ull, kResult = 0x5f010000ull, kIb = 0x5f020000ull, kFlag = 0x5f030000ull; + const uint32_t patterns[4] = {0x0a1a1a10u, 0x0b2b2b20u, 0x0c3c3c30u, 0x0d4d4d40u}; + ProbeBo ib, result, flag, source; + bool stuck = false; + if (probeAlloc(gpu, ib, 0) || probeAlloc(gpu, result, 0xdeadbeef) || probeAlloc(gpu, flag, 0) || + probeAlloc(gpu, source, patterns[0]) || probeMap(gpu, ib, kIb) || probeMap(gpu, result, kResult) || + probeMap(gpu, flag, kFlag) || probeMap(gpu, source, kSource)) { + for (unsigned v = 1; v < 5; ++v) gProbe[v] = 'x'; + probeFree(gpu, source); probeFree(gpu, flag); probeFree(gpu, result); probeFree(gpu, ib); + return; + } + for (unsigned v = 0; v < 4 && !stuck; ++v) { + gpu.mark(0x6d00 | v); + ProbeBo next; + if (v) { + // The new page exists before the old one is freed, so the two differ and the old one still holds the old pattern. + if (probeAlloc(gpu, next, patterns[v])) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + } + uint32_t words[32], n = 0; + auto packet = [&](uint32_t opcode, std::initializer_list payload) { + if (AdrenoPM4Type7(opcode, uint32_t(payload.size()), &words[n]) != AdrenoPM4Success) return false; + ++n; + for (uint32_t w : payload) words[n++] = w; + return true; + }; + bool ok = true; + if (v == 3) ok = packet(0x3c, {3u | (1u << 4), uint32_t(kFlag), uint32_t(kFlag >> 32), 1u, 0xffffffffu, 16u}); // wait flag == 1 + if (v == 2) ok = packet(0x46, {0x31u}) && packet(0x26, {}); + ok = ok && packet(0x73, {0u, uint32_t(kResult), uint32_t(kResult >> 32), uint32_t(kSource), uint32_t(kSource >> 32)}); + if (!ok) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + for (uint32_t i = 0; i < n; ++i) ib.words[i] = words[i]; + result.words[0] = 0xdeadbeef; + flag.words[0] = 0; + __asm__ volatile("dsb sy" ::: "memory"); + if (v && v != 3) { + probeFree(gpu, source); + if (probeMap(gpu, next, kSource)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); break;} + source = next; + } + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {ib.handle, kIb, uint64_t(n) * 4}; + NativeAdrenoAbiSubmitReply sr = {}; + if (gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + if (v == 3) { + usleep(20000); // the CP is spinning on the flag + probeFree(gpu, source); + if (probeMap(gpu, next, kSource)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); stuck = true; break;} + source = next; + flag.words[0] = 1; + __asm__ volatile("dsb sy" ::: "memory"); + } + int waited = -1; + for (unsigned t = 0; t < 10 && waited; ++t) { + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + waited = gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0); + } + if (waited) {gProbe[1 + v] = 't'; stuck = true; break;} + const uint32_t got = result.words[0], want = patterns[v]; + gProbe[1 + v] = got == want ? '1' : (v && got == patterns[v - 1]) ? 'o' : got == 0 ? 'z' : got == 0xdeadbeef ? 'd' : 'g'; + } + // A stuck CP may still reference the buffers. + if (!stuck) {probeFree(gpu, source); probeFree(gpu, flag); probeFree(gpu, result); probeFree(gpu, ib);} +} + +#ifdef NATIVE_TURNIP_REPLAY +// Prefix replay of Turnip's first submission (recorded on the host by scripts/gen-turnip-replay.py with the tablet's 4 KiB page size, so +// the GPU addresses are the ones Turnip itself uses on the tablet): its buffers are mapped at the recorded GPU addresses and growing +// prefixes of its IB, cut after each TYPE7 packet, are submitted one by one, each waited for up to 2 s. Reported on GPU7 as +// "B / t o c e", markers +// 0x6b00 | thread variant, then 0x6a00 | prefix. +struct Replay { + static constexpr unsigned kCount = sizeof turnipreplay::kBuffers / sizeof turnipreplay::kBuffers[0]; + uint64_t handles[kCount] = {}; + unsigned ibIndex = kCount; + bool hung = false; +}; +// Allocates and maps every recorded buffer and fills it through a CPU mapping with the given flags: 0x101 (Device, aligned volatile +// stores, as the prefix replay always did) or 0x401 (write combine filled with memcpy/memset, as Turnip fills its BOs). +static int replayLoad(Gpu &gpu, Replay &r, uint32_t mapFlags) { + using namespace turnipreplay; + for (unsigned i = 0; i < Replay::kCount; ++i) { + const Buffer &b = kBuffers[i]; + NativeAdrenoAbiBoAlloc alloc = {b.bytes, 12, 0}; + NativeAdrenoAbiBoAllocReply reply = {}; + if (int e = gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &reply, sizeof reply)) return e; + r.handles[i] = reply.handle; + NativeAdrenoAbiBoMap map = {reply.handle, b.iova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoMap, &map, sizeof map, &mr, sizeof mr)) return e; + NativeAdrenoAbiBoCpuMap cm = {reply.handle}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + uint64_t address = 0, length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, mapFlags)) return e; + if (length < b.bytes) return 0xbad3; + if (mapFlags == 0x401) { + memcpy(reinterpret_cast(address), b.data, b.words * 4); + memset(reinterpret_cast(address) + b.words * 4, 0, b.bytes - b.words * 4); + } else { + volatile uint32_t *w = reinterpret_cast(address); + for (uint64_t k = 0; k < b.bytes / 4; ++k) w[k] = k < b.words ? b.data[k] : 0; + } + __asm__ volatile("dsb sy" ::: "memory"); + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + NativeAdrenoAbiBoCpuUnmap cu = {reply.handle, cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + if (b.iova <= kIbIova && kIbIova < b.iova + b.bytes) r.ibIndex = i; + } + return r.ibIndex == Replay::kCount ? 0xbad4 : 0; +} +// Submits the first dwords of the IB and waits up to 2 s; a timeout leaves the buffers referenced by the stuck CP. +static int replayRun(Gpu &gpu, Replay &r, uint32_t dwords) { + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {r.handles[r.ibIndex], turnipreplay::kIbIova, uint64_t(dwords) * 4}; + NativeAdrenoAbiSubmitReply sr = {}; + if (int e = gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr)) return e; + int waited = -1; + for (unsigned t = 0; t < 10 && waited; ++t) { + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + waited = gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0); + } + if (waited) r.hung = true; + return waited; +} +// A stuck CP still references the buffers; they are only released when everything retired. +static void replayFree(Gpu &gpu, Replay &r) { + if (r.hung) return; + for (uint64_t &h : r.handles) + if (h) {NativeAdrenoAbiBoFree f = {h}; gpu.method(NativeAdrenoMethodBoFree, &f, sizeof f, nullptr, 0); h = 0;} +} +// One whole-IB control run on fresh buffers at the same GPU addresses: '1' retired, '0' stuck, 'x' setup failed. Marker 0x6c00 | tag. +static char replayControl(Gpu &gpu, uint32_t mapFlags, uint32_t tag, bool &hung) { + Replay r; + gpu.mark(0x6c00 | tag); + char result = 'x'; + if (!replayLoad(gpu, r, mapFlags)) result = replayRun(gpu, r, turnipreplay::kCuts[sizeof turnipreplay::kCuts / sizeof turnipreplay::kCuts[0] - 1]) ? (r.hung ? '0' : 'x') : '1'; + hung = r.hung; + replayFree(gpu, r); + return result; +} +static char gReplayControls[4] = "---"; // W1 W2 N, appended to the B line +static char gReplayLine[64]; +// Whole-IB runs before any prefix (int-065). int-064's remap probe read the new pages in every variant (P1111), yet the whole IB on fresh +// buffers after the prefix series stopped the CP again (W1). These runs separate "a fresh first run stops" (prefix runs had left GPU-written +// memory behind) from "a second whole run stops": E, r1 the whole IB on fresh buffers before anything else, r2 the same +// buffers again, r3 fresh buffers at the same addresses (W1's case); '1' retired, '0' stuck, 'x' setup failed, '-' not run. After r1 +// retires, "d" lists how many words of each buffer differ from the recording and "f" the first changed word outside the fill target, as +// @=, so the memory the GPU wrote is known. +static char gEarly[48] = "E---"; +__attribute__((unused)) static bool earlyRuns(Gpu &gpu) { + using namespace turnipreplay; + const uint32_t whole = kCuts[sizeof kCuts / sizeof kCuts[0] - 1]; + char diff[40] = ""; + Replay r; + gpu.mark(0x6e01); + if (replayLoad(gpu, r, 0x101)) {gEarly[1] = 'x'; replayFree(gpu, r); return true;} + gEarly[1] = replayRun(gpu, r, whole) ? (r.hung ? '0' : 'x') : '1'; + if (gEarly[1] == '1') { + unsigned counts[Replay::kCount] = {}, firstBuffer = 0, firstWord = 0; + uint32_t firstValue = 0; + bool found = false; + for (unsigned i = 0; i < Replay::kCount; ++i) { + const Buffer &b = kBuffers[i]; + NativeAdrenoAbiBoCpuMap cm = {r.handles[i]}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) continue; + uint64_t address = 0, length = 0; + if (!gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, 0x101)) { + const volatile uint32_t *w = reinterpret_cast(address); + for (uint64_t k = 0; k < b.bytes / 4 && k < length / 4; ++k) { + const uint32_t want = k < b.words ? b.data[k] : 0; + if (w[k] == want) continue; + ++counts[i]; + // The fill target is the recording's empty buffer; its change is expected. + if (!found && b.words) {found = true; firstBuffer = i; firstWord = unsigned(k); firstValue = w[k];} + } + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + } + NativeAdrenoAbiBoCpuUnmap cu = {r.handles[i], cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + } + int at = snprintf(diff, sizeof diff, " d"); + for (unsigned i = 0; i < Replay::kCount && at < int(sizeof diff); ++i) + at += snprintf(diff + at, sizeof diff - at, "%s%x", i ? "," : "", counts[i]); + if (found && at < int(sizeof diff)) snprintf(diff + at, sizeof diff - at, " f%u@%x=%x", firstBuffer, firstWord, firstValue); + } + if (gEarly[1] == '1') {gpu.mark(0x6e02); gEarly[2] = replayRun(gpu, r, whole) ? (r.hung ? '0' : 'x') : '1';} + const bool stuck = r.hung; + replayFree(gpu, r); + if (!stuck && gEarly[2] == '1') { + Replay fresh; + gpu.mark(0x6e03); + gEarly[3] = replayLoad(gpu, fresh, 0x101) ? 'x' : replayRun(gpu, fresh, whole) ? (fresh.hung ? '0' : 'x') : '1'; + replayFree(gpu, fresh); + if (fresh.hung) gEarly[3] = '0'; + } + snprintf(gEarly + 4, sizeof gEarly - 4, "%s", diff); + streamText7(gpu, gEarly); + return gEarly[1] == '1' && gEarly[2] == '1' && gEarly[3] == '1'; +} +// Fresh-buffer prefix scan (int-066). int-065: the whole IB on fresh buffers, before any other replay, stopped the CP (E0--), while the +// prefix series on one set of buffers retires every prefix. Here every prefix runs on freshly loaded buffers, so memory written by +// earlier runs cannot help it; the first prefix that stops names the packet that needs that memory. When every prefix retires, CP state +// left by the earlier runs is what helped, and the whole IB then runs once more on fresh buffers (e). Reported on GPU7 as +// "F / o c e". +static bool freshScan(Gpu &gpu) { + using namespace turnipreplay; + constexpr unsigned kPrefixes = sizeof kCuts / sizeof kCuts[0]; + unsigned retired = 0; + bool stuck = false; + char whole = '-'; + for (unsigned p = 0; p < kPrefixes && !stuck; ++p) { + Replay r; + gpu.mark(0x6f80 | p); + int e = replayLoad(gpu, r, 0x101); + if (!e) e = replayRun(gpu, r, kCuts[p]); + stuck = r.hung; + replayFree(gpu, r); + if (e) break; + ++retired; + } + if (retired == kPrefixes) { + Replay r; + gpu.mark(0x6fa0); + whole = replayLoad(gpu, r, 0x101) ? 'x' : replayRun(gpu, r, kCuts[kPrefixes - 1]) ? (r.hung ? '0' : 'x') : '1'; + stuck = r.hung; + replayFree(gpu, r); + } + char text[64]; + snprintf(text, sizeof text, "F %u/%u o%x c%u e%c", retired, kPrefixes, retired < kPrefixes ? kCutOps[retired] : 0, + retired < kPrefixes ? kCuts[retired] : 0, whole); + streamText7(gpu, text); + return !stuck && whole == '1'; +} +static bool replayPhase(Gpu &gpu) { + using namespace turnipreplay; + constexpr unsigned kPrefixes = sizeof kCuts / sizeof kCuts[0]; + Replay r; + unsigned retired = 0, variants = 0; + int error = replayLoad(gpu, r, 0x101); + // The first packet is CP_THREAD_CONTROL(BOTH) and int-060 stopped on it alone. Before the prefixes, the same two dwords run with + // BR only (1) and BV only (2), so a hang names the thread: t. + auto patch = [&](uint32_t value) { + const Buffer &b = kBuffers[r.ibIndex]; + NativeAdrenoAbiBoCpuMap cm = {r.handles[r.ibIndex]}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + uint64_t address = 0, length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, 0x101)) return e; + reinterpret_cast(address)[(kIbIova - b.iova) / 4 + 1] = value; + __asm__ volatile("dsb sy" ::: "memory"); + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + NativeAdrenoAbiBoCpuUnmap cu = {r.handles[r.ibIndex], cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + return 0; + }; + const uint32_t original = r.ibIndex < Replay::kCount ? kBuffers[r.ibIndex].data[(kIbIova - kBuffers[r.ibIndex].iova) / 4 + 1] : 0; + for (uint32_t thread = 1; thread <= 2 && !error; ++thread) { + gpu.mark(0x6b00 | thread); + if (!(error = patch(thread)) && !(error = replayRun(gpu, r, 2))) ++variants; + } + if (!error) error = patch(original); + for (unsigned p = 0; p < kPrefixes && !error; ++p) { + gpu.mark(0x6a00 | p); + if ((error = replayRun(gpu, r, kCuts[p]))) break; + ++retired; + } + replayFree(gpu, r); + // Turnip's own fill of this IB stopped in int-062 while this replay retired. The control runs separate the remaining differences, each + // on fresh buffers at the same addresses: W1 Device-mapped stores (address reuse after a free), W2 write-combine memcpy (how Turnip + // fills its BOs); the caller adds N, a second connection (a second context entry on a CP that already ran work). + bool stuck = r.hung; + if (retired == kPrefixes && !stuck) gReplayControls[0] = replayControl(gpu, 0x101, 1, stuck); + if (retired == kPrefixes && !stuck) gReplayControls[1] = replayControl(gpu, 0x401, 2, stuck); + snprintf(gReplayLine, sizeof gReplayLine, "%.4s B %u/%u t%u o%x c%u e%x ", gEarly, retired, kPrefixes, variants, + retired < kPrefixes ? kCutOps[retired] : 0, retired < kPrefixes ? kCuts[retired] : 0, unsigned(error)); + char text[64]; + snprintf(text, sizeof text, "%s%s", gReplayLine, gReplayControls); + streamText7(gpu, text); + return retired == kPrefixes && !stuck; +} +// N: the whole IB on a second connection, whose first submission carries the A7XX context entry again (Turnip's connection does that +// after the replay). Returns false when the CP stopped. +static bool replaySecondClient(Gpu &gpu) { + unsigned conn = 0; + if (gpu.openFn(gpu.port, mach_task_self(), 0x4147, &conn) != 0 || !conn) {gReplayControls[2] = 'o'; return true;} + const unsigned first = gpu.conn; + volatile uint32_t *mailbox = gpu.mailbox; + gpu.conn = conn; + gpu.mailbox = nullptr; + bool ok = !mailbox || gpu.openMailbox(); + bool stuck = false; + gReplayControls[2] = ok ? replayControl(gpu, 0x101, 3, stuck) : 'm'; + gpu.closeFn(conn); + gpu.conn = first; + gpu.mailbox = mailbox; + char text[64]; + snprintf(text, sizeof text, "%s%s", gReplayLine, gReplayControls); + streamText7(gpu, text); + return !stuck; +} +#endif + +// Run after a passing Turnip phase by the plugin (the Metal bridge self-test); null in the probe daemon. +static void (*gAfterTurnip)(Gpu &gpu, const char *turnipPath); + +// Turnip phase: only where libvulkan_freedreno.dylib sits next to this code (the plugin bundle), never in the probe daemon. +static void turnipPhase(Gpu &gpu) { + char path[1024]; + if (!turnip::libraryPath(path, sizeof path) || access(path, R_OK) != 0) return; + gpu.mark(0x6000); + auto marker = [&](uint32_t value) { gpu.mark(value); }; + const turnip::Result result = turnip::runWatched(marker, [&](const char *text) { streamText(gpu, text); }, path); + char text[NativeAdrenoAbiMetadataBytes] = {}; + snprintf(text, sizeof text, "T s%d r%d v%d c%d%s g%d/%d%s %.30s", result.stage, result.vkResult, result.verified, result.computeWords, + result.computeStage ? "!" : "", result.renderTriangle, result.renderOther, result.renderStage ? "!" : "", turnip::gFirstError); + streamText(gpu, text); + gpu.mark(result.ok ? 0x6fff : 0x6ffe); + // Also when results are wrong but every Vulkan step ran: one boot then shows both layers. + if (result.stage == 0 && result.vkResult == 0 && gAfterTurnip) gAfterTurnip(gpu, path); +} + +// Returns the GPU-verified dword count of the uncached run, or a negative number; every outcome is also reported to the kernel. +static int adrenoUserTest(unsigned port, const char *process) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return -1; + auto open = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen"); + auto close = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose"); + Gpu gpu; + gpu.call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod"); + gpu.map = (MapFn)dlsym(iokit, "IOConnectMapMemory64"); + gpu.unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64"); + gpu.port = port; + gpu.openFn = open; + gpu.closeFn = close; + gpu.mark(1); + if (!open || !close || !gpu.call || !gpu.map || !gpu.unmap) return -1; + gpu.mark(2); + // 0x4147 ('AG') is the open type the driver serves; the original IOGPU library's type-1 opens are refused. + const int openResult = open(port, mach_task_self(), 0x4147, &gpu.conn); + if (openResult != 0 || !gpu.conn) { + // The exact kern_return_t, as two markers (high and low half), after the failure marker. + gpu.mark(0x4f01); + gpu.mark(((uint32_t)openResult >> 16) & 0xffff ? ((uint32_t)openResult >> 16) & 0xffff : 0xffff); + gpu.mark((uint32_t)openResult & 0xffff ? (uint32_t)openResult & 0xffff : 0xffff); + return -1; + } + gpu.mark(3); + const bool probed = gpu.probe(); + gpu.mark(0x1000 | (gpu.admitted & 0xfff)); + bool mailboxOk = false; + if (!probed) {mailboxOk = gpu.openMailbox(); gpu.mark(mailboxOk ? 0x2001 : 0x2000);} + if (!probed && !mailboxOk) {close(gpu.conn); return -2;} // neither channel works: the driver's E row shows what reached it + // mapFlags are what the CLIENT asks for: IOUserClient::mapClientMemory64 lets them override the cache mode the driver returns, so + // the cache bits must be set here. 0x101 = anywhere + inhibit cache (aligned dword accesses only), 0x401 = anywhere + write + // combine (Normal non-cacheable; what a Vulkan driver needs, since it uses memcpy and unaligned stores). + int step = 0, error = 0; + const int verified = runTest(gpu, 0x101, step, error); + int stepWc = 0, errorWc = 0; + const int verifiedWc = error ? -2 : runTest(gpu, 0x401, stepWc, errorWc); + report(gpu, process, step + stepWc, error ? error : errorWc, verified, verifiedWc, gpu.lastTicks); + gpu.mark(0x5000 | (verified & 0xff)); + bool replayed = true; +#ifdef NATIVE_TURNIP_REPLAY + // Plugin only (the Turnip library sits next to it): a stuck replay leaves the CP stopped, so Turnip is not started after it. + char turnipPath[1024]; + if (verified == 4 && verifiedWc == 4 && turnip::libraryPath(turnipPath, sizeof turnipPath) && access(turnipPath, R_OK) == 0) { + replayed = freshScan(gpu) && replayPhase(gpu); + } +#endif + close(gpu.conn); +#ifdef NATIVE_TURNIP_REPLAY + if (replayed && verified == 4 && verifiedWc == 4) replayed = replaySecondClient(gpu); +#endif + if (verified == 4 && verifiedWc == 4 && replayed) turnipPhase(gpu); + return verified; +} diff --git a/NativeBridgeMetalFrontend.h b/NativeBridgeMetalFrontend.h new file mode 100644 index 0000000..5c0c7ca --- /dev/null +++ b/NativeBridgeMetalFrontend.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// Entry points of the native Metal frontend over the bridge (NativeBridgeMetalFrontend.mm). +#pragma once +#include +#include +#ifdef __OBJC__ +#import +extern "C" { +// Opens the process's one device: loader exports vkGetInstanceProcAddr (Turnip), cache is a writable shader-cache directory, registryID is the +// accelerator service's registry entry. missing (may be null) is told the class and selector of every unimplemented call. +id native_bridge_metal_open(const char *loader, const char *cache, uint64_t registryID, void (*missing)(const char *, const char *)); +// Bare AIR bitcode as a library (the bridge's fixtures; Metal itself accepts only metallibs). +id native_bridge_metal_library_with_air(const void *bytes, size_t length); +// One line for the plugin's status report: command buffers committed/failed locally, pipelines made, refusals, the last refusal's text. +size_t native_bridge_metal_status(char *out, size_t size); +// The pixel formats of the textures asked for (first eight distinct) and how many texture requests the runtime refused. +size_t native_bridge_metal_formats(char *out, size_t size); +size_t native_bridge_metal_telemetry(char *out, size_t size); +// Structured cumulative artifact/driver-cache counters for offline qualification. +struct NmbShaderCacheStats; +bool native_bridge_metal_shader_cache_stats(struct NmbShaderCacheStats *out); +bool native_bridge_metal_work_stats(uint64_t out[10]); +// The same fixed CPU, memory and pipeline workload on the tablet and the host, so their times compare directly (int-145): compute, first-touch +// page faults, memset bandwidth, CommonCrypto SHA-256, then the first `pairs` vertex/fragment pairs of the given QuartzCore metallib made into +// BGRA8 render states and drawn once each (pipeline_timing_host's draw=1 order). The results become the telemetry's Y rows; also returned. +size_t native_bridge_metal_benchmark(id device, const char *metallib, unsigned pairs, char *out, size_t size); +} +#endif diff --git a/NativeBridgeMetalFrontend.mm b/NativeBridgeMetalFrontend.mm new file mode 100644 index 0000000..569750c --- /dev/null +++ b/NativeBridgeMetalFrontend.mm @@ -0,0 +1,1521 @@ +// SPDX-License-Identifier: MIT +// Native Metal API frontend over the bridge (bridge/NativeMetalObjects.h): the metal2vulkan runtime on Turnip, on the Adreno 740 through +// NativeAdreno. Same shape as the offline frontend: the device is a dynamically allocated subclass of the target's own _MTLDevice that +// only adds this file's implementations; every other object is an own class answering the Metal protocol methods it implements. +// Every result comes from the runtime and the GPU: a refused call returns nil/NO with the runtime's reason, an encoder call the runtime +// refuses fails its command buffer (status Error, handlers run) instead of being dropped. A selector this frontend does not implement is +// reported through the missing-selector hook before the usual exception, so a hardware run names the gap. +// Shared events work within this process (no cross-process handle yet). Not yet implemented here: heaps, argument buffers, indirect commands, drawables/presentation. +#import +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include "bridge/NativeMetalObjects.h" +#include "NativeBridgeMetalFrontend.h" + +extern "C" void sys_dcache_flush(void *start, size_t length); + +static NmbDevice *gDevice; +static id gDeviceObject; +static uint64_t gRegistryID; +static NSString *gName; +static void (*gMissing)(const char *, const char *); + +// What this device has been asked and refused, for the plugin's status line (native_bridge_metal_status): commits, render/compute pipelines made, +// refusals, and the last two refusals' texts. Counters only; nothing here changes a result. +static std::atomic gCommits,gCommitFailures,gPipelines,gProblems,gCompleted; +// What the render server actually asks of the GPU (int-141: 59 commits in 200 s, and a desktop that behaves like CPU compositing): render passes, +// draws, the pixels of their first colour attachment, compute dispatches, blits and the largest pass (pixels). +static std::atomic gPasses,gDraws,gPassPixels,gDispatches,gBlits,gBiggestPass,gFullScreenPasses; +// Where the time goes on the CPU side of the bridge: pipeline, library and function creation (count, total and slowest, in microseconds), and the +// last commits with what each held and how long the GPU took (commit to completion). +static uint64_t nowUs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW)/1000;} +struct Timed {std::atomic count{0},total{0},slowest{0}; + void add(uint64_t began){const uint64_t took=nowUs()-began;count++;total+=took;uint64_t seen=slowest.load();while(took>seen && !slowest.compare_exchange_weak(seen,took)){}}}; +static Timed gTimePipeline,gTimeLibrary,gTimeFunction,gTimeEncodeToCommit; +static Timed gTimeWaitScheduled,gTimeWaitCompleted; +static Timed gTimeResourceWait; +static std::atomic gResourceChecks,gResourceSkipped; +static Timed gTimeSurfaceFlush; +static std::atomic gSurfaceFlushBytes,gSurfaceFlushDuplicates; +struct EncodedWork {uint32_t passes,draws,dispatches,blits;uint64_t pixels;}; +struct CommitRecord {uint64_t committedUs=0,callbackUs=0,ageUs=0;int64_t scheduledUs=-1;uint32_t index=0;EncodedWork work={};}; +static CommitRecord gCommitRing[32]; +static std::mutex gCommitLock; +static std::mutex gProblemLock; +static char gLastProblems[2][200]; +static void noteProblem(NSString *what) { + gProblems++; + std::lock_guard lock(gProblemLock); + memcpy(gLastProblems[1],gLastProblems[0],sizeof gLastProblems[0]); + strlcpy(gLastProblems[0],what.UTF8String?:"?",sizeof gLastProblems[0]); +} +#define NBLOG(...) do{NSString *nbText=[NSString stringWithFormat:__VA_ARGS__];NSLog(@"%@",nbText);noteProblem(nbText);}while(0) +// The pixel formats textures were asked for (first eight distinct), with how many were refused: native_bridge_metal_formats. +static uint32_t gFormats[8];static unsigned gFormatCount;static std::atomic gTextureRefusals; +static void noteFormat(uint32_t format,bool made) { + std::lock_guard lock(gProblemLock); + if(!made)gTextureRefusals++; + for(unsigned i=0;i)device {return gDeviceObject;} +- (void)doesNotRecognizeSelector:(SEL)selector { + if(gMissing)gMissing(class_getName(object_getClass(self)),sel_getName(selector)); + noteProblem([NSString stringWithFormat:@"missing %s %s",class_getName(object_getClass(self)),sel_getName(selector)]); + [super doesNotRecognizeSelector:selector]; +} +@end + +// ---- resources ---- +// The wait takes a strong snapshot of already-committed buffers; new commits require the +// caller's usual resource synchronization. Storage dependency checks are in the runtime. +static NSMutableSet *gInFlight; +static void waitForResource(id resource); +// Resource SPI the runtime has no state for: no protected content (Metal's default options), and no owner transfer +// (memory attribution is the kernel driver's, by the importing task), refused as not supported. +#define NB_RESOURCE_SPI \ +- (uint64_t)protectionOptions {return 0;} \ +- (id)heap {return _parentHeap;} \ +- (BOOL)isAliasable {return NO;} \ +- (int)setOwnerWithIdentity:(unsigned)identity {(void)identity;return 46; /* KERN_NOT_SUPPORTED */} \ +- (void)waitUntilComplete {waitForResource(self);} +@interface NBBuffer : NBObject +@property(nonatomic) NmbBuffer *handle; +@property(nonatomic) MTLResourceOptions options; +@property(nonatomic,copy) void (^deallocator)(void *,NSUInteger); +@property(nonatomic) void *wrapped; +@property(nonatomic) IOSurfaceRef surfaceOwner; // keeps a shared import's CPU mapping alive +@property(nonatomic,strong) id parentHeap; // the heap that made the buffer, nil otherwise +@property(nonatomic) NSUInteger heapOffset; +@end +@interface NBTexture : NBObject +@property(nonatomic) NmbTexture *handle; +@property(nonatomic) NmbTextureDescriptor descriptor; +@property(nonatomic,strong) NBBuffer *buffer; +@property(nonatomic) IOSurfaceRef surface; +@property(nonatomic) NSUInteger offset,rowBytes; +@property(nonatomic,strong) id parent; +@property(nonatomic,strong) id parentHeap; +@property(nonatomic) NSUInteger heapOffset; +@end +@implementation NBBuffer +- (void)dealloc { + const NSUInteger length=nmb_buffer_length(_handle);nmb_buffer_release(_handle); + if(_deallocator)_deallocator(_wrapped,length); + if(_surfaceOwner)CFRelease(_surfaceOwner); +} +- (NSUInteger)length {return nmb_buffer_length(_handle);} +- (void *)contents {return nmb_buffer_contents(_handle);} +- (uint64_t)gpuAddress {return nmb_buffer_gpu_address(_handle);} +- (MTLResourceOptions)resourceOptions {return _options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (MTLHazardTrackingMode)hazardTrackingMode {return (MTLHazardTrackingMode)((_options&MTLResourceHazardTrackingModeMask)>>MTLResourceHazardTrackingModeShift);} +- (void)didModifyRange:(NSRange)range {(void)range;} +- (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { + const uint64_t answer=nmb_buffer_set_purgeable_state(_handle,state); + if(!answer)NBLOG(@"native-metal: buffer setPurgeableState refused: %@",lastError()); + return (MTLPurgeableState)answer; +} +- (NSUInteger)allocatedSize {return nmb_buffer_length(_handle);} +- (IOSurfaceRef)iosurface {return nullptr;} // a buffer is never the backing of a surface here (surfaces back textures) +NB_RESOURCE_SPI +@end +// Reuse only an import still held by a texture/view or pending command. Weak +// values never extend a GPU allocation's lifetime. The strong buffer snapshot +// and its retained surface prevent object/address reuse during a hit. A new +// mapping, allocation identity or size needs a separate import; ordinary +// caller-owned no-copy buffers do not enter this cache. +static NSMapTable *gSurfaceImports; +static std::mutex gSurfaceImportLock; +static NBBuffer *surfaceBuffer(IOSurfaceRef surface,void *base,size_t bytes) { + struct Key {uint64_t identity,address,bytes;}; + const Key value={IOSurfaceGetID(surface),(uintptr_t)base,bytes}; + NSValue *key=value.identity?[NSValue valueWithBytes:&value objCType:@encode(Key)]:nil; + std::lock_guard lock(gSurfaceImportLock); + if(!gSurfaceImports)gSurfaceImports=[NSMapTable strongToWeakObjectsMapTable]; + NBBuffer *buffer=key?[gSurfaceImports objectForKey:key]:nil; + if(buffer)return buffer; + NmbBuffer *handle=nmb_buffer_new_no_copy(gDevice,base,bytes,MTLResourceStorageModeShared); + if(!handle){NBLOG(@"native-metal: surface import refused: %@",lastError());return nil;} + buffer=[NBBuffer new];buffer.handle=handle;buffer.options=MTLResourceStorageModeShared; + buffer.surfaceOwner=(IOSurfaceRef)CFRetain(surface); + if(key) { + // Bound metadata without retaining old frames. Evicting an association is + // harmless: its owners still retain the actual import until their own end. + if(gSurfaceImports.count>=256) { + for(NSValue *old in gSurfaceImports.keyEnumerator.allObjects) + if(![gSurfaceImports objectForKey:old])[gSurfaceImports removeObjectForKey:old]; + if(gSurfaceImports.count>=256)[gSurfaceImports removeObjectForKey:gSurfaceImports.keyEnumerator.nextObject]; + } + [gSurfaceImports setObject:buffer forKey:key]; + } + return buffer; +} +static bool textureDescriptor(MTLTextureDescriptor *d,NmbTextureDescriptor *out) { + if(!d)return false; + *out={(uint64_t)d.textureType,(uint64_t)d.pixelFormat,d.width,d.height,d.depth,d.mipmapLevelCount,d.sampleCount,d.arrayLength,(uint64_t)d.usage,(uint64_t)d.resourceOptions}; + return true; +} +@implementation NBTexture +- (void)dealloc {nmb_texture_release(_handle);if(_surface)CFRelease(_surface);} +- (MTLTextureType)textureType {return (MTLTextureType)_descriptor.type;} +- (MTLPixelFormat)pixelFormat {return (MTLPixelFormat)_descriptor.format;} +- (NSUInteger)width {return _descriptor.width;} +- (NSUInteger)height {return _descriptor.height;} +- (NSUInteger)depth {return _descriptor.depth;} +- (NSUInteger)mipmapLevelCount {return _descriptor.levels;} +- (NSUInteger)sampleCount {return _descriptor.samples;} +- (NSUInteger)arrayLength {return _descriptor.array_length;} +- (MTLTextureUsage)usage {return (MTLTextureUsage)_descriptor.usage;} +- (MTLResourceOptions)resourceOptions {return (MTLResourceOptions)_descriptor.options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_descriptor.options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_descriptor.options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (BOOL)isFramebufferOnly {return NO;} +- (IOSurfaceRef)iosurface {return _surface;} +- (NSUInteger)iosurfacePlane {return 0;} +- (id)buffer {return _surface?nil:(id)_buffer;} +- (NSUInteger)bufferOffset {return _offset;} +- (NSUInteger)bufferBytesPerRow {return _rowBytes;} +- (id)parentTexture {return _parent;} +- (NSUInteger)rotation {return 0;} +- (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { + const uint64_t answer=nmb_texture_set_purgeable_state(_handle,state); + if(!answer)NBLOG(@"native-metal: texture setPurgeableState refused: %@",lastError()); + return (MTLPurgeableState)answer; +} +NB_RESOURCE_SPI +- (id)view:(NmbTexture *)handle { + if(!handle){NBLOG(@"native-metal: texture view refused: %@",lastError());return nil;} + NBTexture *view=[NBTexture new];view.handle=handle;NmbTextureDescriptor descriptor;nmb_texture_describe(handle,&descriptor);view.descriptor=descriptor; + view.parent=(id)self;view.buffer=_buffer;view.offset=_offset;view.rowBytes=_rowBytes; + if(_surface)view.surface=(IOSurfaceRef)CFRetain(_surface); + return (id)view; +} +- (id)newTextureViewWithPixelFormat:(MTLPixelFormat)format { + return [self view:nmb_texture_new_view(_handle,format,0,nullptr,nullptr,true)]; +} +- (id)newTextureViewWithPixelFormat:(MTLPixelFormat)format textureType:(MTLTextureType)type levels:(NSRange)levels slices:(NSRange)slices { + const uint64_t l[2]={levels.location,levels.length},s[2]={slices.location,slices.length}; + return [self view:nmb_texture_new_view(_handle,format,type,l,s,false)]; +} +- (void)replaceRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level slice:(NSUInteger)slice withBytes:(const void *)bytes bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image { + const uint64_t region[6]={r.origin.x,r.origin.y,r.origin.z,r.size.width,r.size.height,r.size.depth}; + const uint64_t length=(r.size.depth>1?image*(r.size.depth-1):0)+row*r.size.height; + if(!nmb_texture_replace(_handle,region,level,slice,bytes,row,image,length))NBLOG(@"native-metal: replaceRegion refused: %@",lastError()); +} +- (void)replaceRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level withBytes:(const void *)bytes bytesPerRow:(NSUInteger)row { + [self replaceRegion:r mipmapLevel:level slice:0 withBytes:bytes bytesPerRow:row bytesPerImage:row*r.size.height]; +} +- (void)getBytes:(void *)bytes bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image fromRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level slice:(NSUInteger)slice { + const uint64_t region[6]={r.origin.x,r.origin.y,r.origin.z,r.size.width,r.size.height,r.size.depth}; + const uint64_t length=(r.size.depth>1?image*(r.size.depth-1):0)+row*r.size.height; + if(!nmb_texture_get_bytes(_handle,bytes,row,image,length,region,level,slice))NBLOG(@"native-metal: getBytes refused: %@",lastError()); +} +- (void)getBytes:(void *)bytes bytesPerRow:(NSUInteger)row fromRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level { + [self getBytes:bytes bytesPerRow:row bytesPerImage:row*r.size.height fromRegion:r mipmapLevel:level slice:0]; +} +@end +// A command can use many independent imports/views of one IOSurface. Their +// owners keep the mapping live through this phase. Maintain each exact CPU +// range once; no address identity or decision is carried into another phase +// or command, and distinct/partially overlapping ranges retain their calls. +static void flushSurfaces(NSMutableSet *surfaces) { + if(!surfaces.count)return; + std::vector> ranges; + ranges.reserve(surfaces.count); + for(NBTexture *texture in surfaces)if(texture.surface) + ranges.emplace_back(reinterpret_cast(IOSurfaceGetBaseAddress(texture.surface)),IOSurfaceGetAllocSize(texture.surface)); + std::sort(ranges.begin(),ranges.end()); + const auto last=std::unique(ranges.begin(),ranges.end()); + gSurfaceFlushDuplicates+=(uint64_t)(ranges.end()-last); + for(auto it=ranges.begin();it!=last;++it) { + const uint64_t began=nowUs(); + sys_dcache_flush(reinterpret_cast(it->first),it->second); + gSurfaceFlushBytes+=it->second;gTimeSurfaceFlush.add(began); + } +} +static NBTexture *textureOnBuffer(NBBuffer *buffer,MTLTextureDescriptor *d,NSUInteger offset,NSUInteger row,bool surface) { + NmbTextureDescriptor descriptor;if(!buffer || !textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new_on_buffer(buffer.handle,&descriptor,offset,row,surface); + if(!handle){NBLOG(@"native-metal: texture over buffer refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.buffer=buffer;texture.offset=offset;texture.rowBytes=row;return texture; +} +@implementation NBBuffer (Textures) +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset bytesPerRow:(NSUInteger)row { + return (id)textureOnBuffer(self,d,offset,row,false); +} +- (id)newLinearTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new_linear(self.handle,&descriptor,offset,row,image); + if(!handle){NBLOG(@"native-metal: linear texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.buffer=self;texture.offset=offset;texture.rowBytes=row;return (id)texture; +} +@end + +// ---- libraries, functions, states ---- +@interface NBLibrary : NBObject +@property(nonatomic) NmbLibrary *handle; +@end +@interface NBFunction : NBObject +@property(nonatomic) NmbFunction *handle; +@property(nonatomic,copy) NSString *name; +@property(nonatomic) MTLFunctionType functionType; +@end +@implementation NBFunction +- (void)dealloc {nmb_function_release(_handle);} +- (MTLFunctionOptions)options {return MTLFunctionOptionNone;} +@end +// The target's own accessors of MTLFunctionConstantValues (as the offline frontend reads them); their ABI is checked before any call. +@interface MTLFunctionConstantValues (NBOriginalConstantAccess) +- (NSArray *)newIndexedConstantArray; +- (NSArray *)newNamedConstantArray; +@end +@protocol NBOriginalConstantValue +- (NSUInteger)dataType; +- (void *)data; +- (NSUInteger)index; +- (NSString *)name; +@end +static bool methodABI(id object,SEL selector,const char *encoding) { + Method method=class_getInstanceMethod(object_getClass(object),selector); + return method && !strcmp(method_getTypeEncoding(method),encoding); +} +// Bytes of a function constant of MTLDataType `type` (scalars and 2/4-wide vectors; 3-wide vectors are not qualified). +static uint64_t constantBytes(uint64_t type) { + switch(type) { + case 3:case 29:case 33:return 4; + case 4:case 30:case 34:return 8; + case 6:case 32:case 36:return 16; + case 16:case 37:case 41:return 2; + case 17:case 38:case 42:return 4; + case 19:case 40:case 44:return 8; + case 45:case 49:case 53:return 1; + case 46:case 50:case 54:return 2; + case 48:case 52:case 56:return 4; + case 81:case 85:return 8; + case 82:case 86:return 16; + case 84:case 88:return 32; + default:return 0; + } +} +@implementation NBLibrary +- (void)dealloc {nmb_library_release(_handle);} +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error { + if(!name || !values || !methodABI(values,@selector(newIndexedConstantArray),"@16@0:8") || !methodABI(values,@selector(newNamedConstantArray),"@16@0:8")) { + if(error)*error=nbError(@"unqualified function constant values");return nil; + } + MTLFunctionConstantValues *snapshot=[values copy]; + NSArray *indexed=[snapshot newIndexedConstantArray],*named=[snapshot newNamedConstantArray]; + std::vector constants; + for(unsigned group=0;group<2;group++)for(id value in (group?named:indexed)) { + if(!methodABI(value,@selector(dataType),"Q16@0:8") || !methodABI(value,@selector(data),"^v16@0:8") || + !methodABI(value,group?@selector(name):@selector(index),group?"@16@0:8":"Q16@0:8")) { + if(error)*error=nbError(@"unqualified constant value ABI");return nil; + } + id v=value; + const uint64_t bytes=constantBytes(v.dataType); + if(!bytes || !v.data){if(error)*error=nbError([NSString stringWithFormat:@"constant type %lu is not qualified",(unsigned long)v.dataType]);return nil;} + constants.push_back({v.dataType,group?0:v.index,group?v.name.UTF8String:nullptr,v.data,bytes}); + } + const uint64_t functionBegan=nowUs(); + NmbFunction *handle=nmb_function_new_specialized(_handle,name.UTF8String,constants.data(),(uint32_t)constants.size());gTimeFunction.add(functionBegan); + if(!handle){if(error)*error=nbError([NSString stringWithFormat:@"specialized function: %@",lastError()]);return nil;} + NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + if(error)*error=nil;return (id)function; +} +- (void)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values completionHandler:(void (^)(id,NSError *))handler { + NSError *error=nil;id function=[self newFunctionWithName:name constantValues:values error:&error]; + handler(function,error); +} +- (MTLLibraryType)type {return MTLLibraryTypeExecutable;} +// The descriptor's name and constants; its specialized name only labels Metal's function and its options ask for compile-time work the +// runtime does when the function is made. +- (id)newFunctionWithDescriptor:(MTLFunctionDescriptor *)d error:(NSError **)error { + if(!d.name){if(error)*error=nbError(@"a function name is required");return nil;} + if(d.constantValues)return [self newFunctionWithName:d.name constantValues:d.constantValues error:error]; + id function=[self newFunctionWithName:d.name]; + if(!function){if(error)*error=nbError([NSString stringWithFormat:@"function %@: %@",d.name,lastError()]);return nil;} + if(error)*error=nil;return function; +} +- (NSArray *)functionNames { + NSMutableArray *names=[NSMutableArray array]; + for(uint32_t i=0,n=nmb_library_function_count(_handle);i)newFunctionWithName:(NSString *)name { + const uint64_t functionBegan=nowUs(); + NmbFunction *handle=name?nmb_function_new(_handle,name.UTF8String):nullptr;gTimeFunction.add(functionBegan);if(!handle)return nil; + NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + return (id)function; +} +@end +@interface NBComputePipeline : NBObject +@property(nonatomic) NmbComputePipeline *handle; +@end +@implementation NBComputePipeline +- (void)dealloc {nmb_compute_pipeline_release(_handle);} +- (NSUInteger)maxTotalThreadsPerThreadgroup {return nmb_compute_pipeline_max_threads(_handle);} +- (NSUInteger)threadExecutionWidth {return nmb_compute_pipeline_execution_width(_handle);} +@end +@interface NBRenderPipeline : NBObject +@property(nonatomic) NmbRenderPipeline *handle; +@end +@implementation NBRenderPipeline +- (void)dealloc {nmb_render_pipeline_release(_handle);} +// The threadgroup size a tile dispatch of this pipeline could use: the device's limit (no tile shaders are offered). +- (NSUInteger)maxTotalThreadsPerThreadgroup { + uint64_t s[3]={};if(!nmb_device_max_threads_per_threadgroup(gDevice,s))return 0; + return s[0]; +} +@end +@interface NBSampler : NBObject +@property(nonatomic) NmbSampler *handle; +@end +@implementation NBSampler +- (void)dealloc {nmb_sampler_release(_handle);} +- (NSUInteger)pixelFormat {return MTLPixelFormatInvalid;} // SPI: a sampler built from a plain descriptor names no format +@end +@interface NBDepthStencil : NBObject +@property(nonatomic) NmbDepthStencil *handle; +@end +@implementation NBDepthStencil +- (void)dealloc {nmb_depth_stencil_release(_handle);} +@end + +// ---- events and fences ---- +@interface NBEvent : NBObject +@property(nonatomic) NmbEvent *handle; +@end +@implementation NBEvent +- (void)dealloc {nmb_event_release(_handle);} +@end +@interface NBSharedEvent : NBEvent +@end +static void notified(void *context,uint64_t value); +@implementation NBSharedEvent +- (uint64_t)signaledValue {return nmb_shared_event_value(self.handle);} +- (void)setSignaledValue:(uint64_t)value {nmb_shared_event_set_value(self.handle,value);} +- (BOOL)waitUntilSignaledValue:(uint64_t)value timeoutMS:(uint64_t)milliseconds {return nmb_shared_event_wait(self.handle,value,milliseconds);} +// The block runs on the listener's dispatch queue (Metal's contract); the bridge's listener thread only forwards it there. +- (void)notifyListener:(MTLSharedEventListener *)listener atValue:(uint64_t)value block:(MTLSharedEventNotificationBlock)block { + if(!block)return; + dispatch_queue_t queue=listener.dispatchQueue?:dispatch_get_global_queue(QOS_CLASS_DEFAULT,0); + NSArray *job=@[self,queue,[block copy]]; + if(!nmb_shared_event_notify(self.handle,value,notified,(__bridge_retained void *)job)) { + (void)(__bridge_transfer NSArray *)(__bridge void *)job; + NBLOG(@"native-metal: notifyListener refused: %@",lastError()); + } +} +// A handle for another process needs a kernel-backed event this bridge does not have yet: refused, never a handle that could not work. +- (MTLSharedEventHandle *)newSharedEventHandle { + if(gMissing)gMissing("NBSharedEvent","newSharedEventHandle (cross-process)"); + return nil; +} +@end +static void notified(void *context,uint64_t value) { + NSArray *job=(__bridge_transfer NSArray *)context; + id event=job[0];MTLSharedEventNotificationBlock block=job[2]; + dispatch_async(job[1],^{block(event,value);}); +} +@interface NBFence : NBObject +@property(nonatomic) NmbFence *handle; +@end +@implementation NBFence +- (void)dealloc {nmb_fence_release(_handle);} +@end + +// ---- command buffers and encoders ---- +@interface NBQueue : NBObject +@property(nonatomic) NmbQueue *handle; +@property(atomic,strong) dispatch_queue_t completionQueue; // SPI: where command buffers' handlers run (nil: the runtime's thread) +@end +@interface NBCommandBuffer : NBObject { +@public + EncodedWork encodedWork; // the command buffer's single encoder thread, not global counter deltas + uint64_t createdUs; +} +@property(nonatomic) NmbCommandBuffer *handle; +@property(nonatomic,strong) NBQueue *queue; +@property(atomic,copy) NSString *encodeFailure; +@property(atomic) BOOL committed,failedLocally; +@property(nonatomic) uint64_t committedUs; +@property(nonatomic) uint32_t commitIndex; +@property(nonatomic,strong) NSMutableArray *scheduledHandlers,*completedHandlers; +@property(nonatomic,strong) NSMutableSet *surfaces; +// Runtime handles retain GPU storage, but only the Objective-C owner retains +// a no-copy deallocator, its memory owner and texture/heap ancestry. +@property(nonatomic,strong) NSMutableSet *resourceOwners; +@property(nonatomic,strong) dispatch_group_t handlers; +@property(nonatomic,strong) dispatch_group_t schedulingHandlers; +@property(nonatomic) BOOL schedulingDelivered; +- (void)refuse:(NSString *)what; +@end +@interface NBEncoder : NBObject +@property(nonatomic) NmbEncoder *handle; +@property(nonatomic,strong) NBCommandBuffer *commandBuffer; +@end +static void waitForResource(id resource) { + const uint64_t began=nowUs(); + NSArray *pending;@synchronized(gInFlight){pending=gInFlight.allObjects;} + for(NBCommandBuffer *commandBuffer in pending) { + int dependency=-1; + if([resource isKindOfClass:[NBBuffer class]])dependency=nmb_command_buffer_uses_buffer(commandBuffer.handle,[(NBBuffer *)resource handle]); + else if([resource isKindOfClass:[NBTexture class]])dependency=nmb_command_buffer_uses_texture(commandBuffer.handle,[(NBTexture *)resource handle]); + gResourceChecks++; + // An unknown result retains the old conservative wait. False means no matching storage + // (or GPU work already completed), not a fabricated completion for a pending dependency. + if(dependency!=0)[(id)commandBuffer waitUntilCompleted];else gResourceSkipped++; + } + gTimeResourceWait.add(began); +} +@implementation NBEncoder +- (void)dealloc {if(_handle)nmb_encoder_discard(_handle);} +- (bool)check:(bool)ok what:(const char *)what { + if(!ok)[_commandBuffer refuse:[NSString stringWithFormat:@"%s: %@",what,lastError()]]; + return ok; +} +- (void)endEncoding { + NmbEncoder *handle=_handle;_handle=nullptr; + if(!handle){[_commandBuffer refuse:@"endEncoding twice"];return;} + [self check:nmb_encoder_end(handle) what:"endEncoding"]; +} +- (void)insertDebugSignpost:(NSString *)s {(void)s;} +- (void)pushDebugGroup:(NSString *)s {(void)s;} +- (void)popDebugGroup {} +// Stage 0 compute, 1 vertex, 2 fragment (the bridge's numbering). +- (void)buffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { + if(buffer && ![(id)buffer isKindOfClass:[NBBuffer class]]){[_commandBuffer refuse:@"foreign buffer"];return;} + if(buffer)[_commandBuffer.resourceOwners addObject:buffer]; + [self check:nmb_encoder_set_buffer(_handle,stage,buffer?[(NBBuffer *)buffer handle]:nullptr,offset,(uint32_t)index) what:"setBuffer"]; +} +- (void)texture:(id)texture index:(NSUInteger)index stage:(uint32_t)stage { + if(texture && ![(id)texture isKindOfClass:[NBTexture class]]){[_commandBuffer refuse:@"foreign texture"];return;} + if(texture)[_commandBuffer.resourceOwners addObject:texture]; + if(texture && [(NBTexture *)texture surface])[_commandBuffer.surfaces addObject:texture]; + [self check:nmb_encoder_set_texture(_handle,stage,texture?[(NBTexture *)texture handle]:nullptr,(uint32_t)index) what:"setTexture"]; +} +- (void)sampler:(id)sampler index:(NSUInteger)index stage:(uint32_t)stage { + if(sampler && ![(id)sampler isKindOfClass:[NBSampler class]]){[_commandBuffer refuse:@"foreign sampler"];return;} + [self check:nmb_encoder_set_sampler(_handle,stage,sampler?[(NBSampler *)sampler handle]:nullptr,(uint32_t)index) what:"setSamplerState"]; +} +- (void)bytes:(const void *)bytes length:(NSUInteger)length index:(NSUInteger)index stage:(uint32_t)stage { + [self check:nmb_encoder_set_bytes(_handle,stage,bytes,length,(uint32_t)index) what:"setBytes"]; +} +- (void)fence:(id)fence stages:(uint64_t)stages update:(bool)update { + if(![(id)fence isKindOfClass:[NBFence class]]){[_commandBuffer refuse:@"foreign fence"];return;} + NmbFence *handle=[(NBFence *)fence handle]; + [self check:update?nmb_encoder_update_fence(_handle,handle,stages):nmb_encoder_wait_fence(_handle,handle,stages) what:update?"updateFence":"waitForFence"]; +} +- (void)updateFence:(id)fence {[self fence:fence stages:0 update:true];} +- (void)waitForFence:(id)fence {[self fence:fence stages:0 update:false];} +- (void)offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { + [self check:nmb_encoder_set_buffer_offset(_handle,stage,offset,(uint32_t)index) what:"setBufferOffset"]; +} +@end + +@interface NBComputeEncoder : NBEncoder +@end +@implementation NBComputeEncoder +- (MTLDispatchType)dispatchType {return MTLDispatchTypeSerial;} +- (void)setComputePipelineState:(id)state { + if(![(id)state isKindOfClass:[NBComputePipeline class]]){[self.commandBuffer refuse:@"foreign compute pipeline"];return;} + [self check:nmb_compute_set_pipeline(self.handle,[(NBComputePipeline *)state handle]) what:"setComputePipelineState"]; +} +- (void)setBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:0];} +- (void)setBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:0];} +- (void)setBuffers:(const id __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k)t atIndex:(NSUInteger)i {[self texture:t index:i stage:0];} +- (void)setTextures:(const id __unsafe_unretained [])t withRange:(NSRange)r { + for(NSUInteger k=0;k)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:0];} +- (void)setThreadgroupMemoryLength:(NSUInteger)n atIndex:(NSUInteger)i { + [self check:nmb_compute_set_threadgroup_memory(self.handle,(uint32_t)n,(uint32_t)i) what:"setThreadgroupMemoryLength"]; +} +- (void)dispatch:(MTLSize)size group:(MTLSize)group threads:(bool)threads { + gDispatches++;self.commandBuffer->encodedWork.dispatches++; + const uint64_t s[3]={size.width,size.height,size.depth},g[3]={group.width,group.height,group.depth}; + [self check:nmb_compute_dispatch(self.handle,s,g,threads) what:"dispatch"]; +} +- (void)dispatchThreads:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:true];} +- (void)dispatchThreadgroups:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:false];} +- (void)useResource:(id)r usage:(MTLResourceUsage)u {(void)r;(void)u;} +- (void)memoryBarrierWithScope:(MTLBarrierScope)s {(void)s;} +@end + +@interface NBRenderEncoder : NBEncoder +@end +@implementation NBRenderEncoder +- (void)setRenderPipelineState:(id)state { + if(![(id)state isKindOfClass:[NBRenderPipeline class]]){[self.commandBuffer refuse:@"foreign render pipeline"];return;} + [self check:nmb_render_set_pipeline(self.handle,[(NBRenderPipeline *)state handle]) what:"setRenderPipelineState"]; +} +- (void)setVertexBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:1];} +- (void)setFragmentBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:2];} +- (void)setVertexBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:1];} +- (void)setFragmentBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:2];} +- (void)setVertexBuffers:(const id __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k)t atIndex:(NSUInteger)i {[self texture:t index:i stage:1];} +- (void)setFragmentTexture:(id)t atIndex:(NSUInteger)i {[self texture:t index:i stage:2];} +- (void)setFragmentTextures:(const id __unsafe_unretained [])t withRange:(NSRange)r { + for(NSUInteger k=0;k)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:1];} +- (void)setFragmentSamplerState:(id)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:2];} +- (void)setDepthStencilState:(id)state { + if(state && ![(id)state isKindOfClass:[NBDepthStencil class]]){[self.commandBuffer refuse:@"foreign depth/stencil state"];return;} + [self check:nmb_render_set_depth_stencil(self.handle,state?[(NBDepthStencil *)state handle]:nullptr) what:"setDepthStencilState"]; +} +- (void)setViewport:(MTLViewport)v { + const double values[6]={v.originX,v.originY,v.width,v.height,v.znear,v.zfar}; + [self check:nmb_render_set_viewport(self.handle,values) what:"setViewport"]; +} +- (void)setScissorRect:(MTLScissorRect)r { + const uint64_t values[4]={r.x,r.y,r.width,r.height}; + [self check:nmb_render_set_scissor(self.handle,values) what:"setScissorRect"]; +} +// The runtime takes cull mode, winding and fill mode together; this encoder keeps Metal's defaults until each is set. +- (void)raster { + NSNumber *cull=objc_getAssociatedObject(self,"cull"),*winding=objc_getAssociatedObject(self,"winding"),*fill=objc_getAssociatedObject(self,"fill"); + [self check:nmb_render_set_raster(self.handle,cull.unsignedLongLongValue,winding.unsignedLongLongValue,fill.unsignedLongLongValue) what:"raster state"]; +} +- (void)setCullMode:(MTLCullMode)m {objc_setAssociatedObject(self,"cull",@(m),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setFrontFacingWinding:(MTLWinding)w {objc_setAssociatedObject(self,"winding",@(w),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setTriangleFillMode:(MTLTriangleFillMode)f {objc_setAssociatedObject(self,"fill",@(f),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setDepthBias:(float)bias slopeScale:(float)slope clamp:(float)clamp { + [self check:nmb_render_set_depth_bias(self.handle,bias,slope,clamp) what:"setDepthBias"]; +} +- (void)setLineWidth:(float)width {[self check:nmb_render_set_line_width(self.handle,width) what:"setLineWidth"];} +- (void)setVisibilityResultMode:(MTLVisibilityResultMode)mode offset:(NSUInteger)offset { + [self check:nmb_render_set_visibility_mode(self.handle,mode,offset) what:"setVisibilityResultMode"]; +} +- (void)setFragmentTexture:(id)t atTextureIndex:(NSUInteger)ti samplerState:(id)s atSamplerIndex:(NSUInteger)si { + [self texture:t index:ti stage:2];[self sampler:s index:si stage:2]; +} +// Tile shaders and imageblock tile memory (Apple GPU families 4 and up) are not offered: supportsTileShaders answers no, and a call that +// reaches one anyway fails its command buffer. +- (NSUInteger)tileWidth {return 0;} +- (NSUInteger)tileHeight {return 0;} +- (void)setTileTexture:(id)t atIndex:(NSUInteger)i {(void)t;(void)i;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)setTileBytes:(const void *)p length:(NSUInteger)n atIndex:(NSUInteger)i {(void)p;(void)n;(void)i;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)dispatchThreadsPerTile:(MTLSize)size {(void)size;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)dispatchThreadsPerTile:(MTLSize)size inRegion:(MTLRegion)region {(void)size;(void)region;[self.commandBuffer refuse:@"tile shaders are not supported"];} +- (void)setStencilReferenceValue:(uint32_t)v {[self check:nmb_render_set_stencil_reference(self.handle,v,v) what:"setStencilReferenceValue"];} +- (void)setStencilFrontReferenceValue:(uint32_t)f backReferenceValue:(uint32_t)b { + [self check:nmb_render_set_stencil_reference(self.handle,f,b) what:"setStencilReferenceValues"]; +} +- (void)setBlendColorRed:(float)r green:(float)g blue:(float)b alpha:(float)a { + const float c[4]={r,g,b,a};[self check:nmb_render_set_blend_color(self.handle,c) what:"setBlendColor"]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n instanceCount:(NSUInteger)i baseInstance:(NSUInteger)b { + gDraws++;self.commandBuffer->encodedWork.draws++; + [self check:nmb_render_draw(self.handle,t,(uint32_t)s,(uint32_t)n,(uint32_t)i,(uint32_t)b) what:"drawPrimitives"]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n instanceCount:(NSUInteger)i { + [self drawPrimitives:t vertexStart:s vertexCount:n instanceCount:i baseInstance:0]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n { + [self drawPrimitives:t vertexStart:s vertexCount:n instanceCount:1 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o + instanceCount:(NSUInteger)i baseVertex:(NSInteger)v baseInstance:(NSUInteger)bi { + if(![(id)b isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:@"foreign index buffer"];return;} + [self.commandBuffer.resourceOwners addObject:b]; + gDraws++;self.commandBuffer->encodedWork.draws++; + [self check:nmb_render_draw_indexed(self.handle,t,(uint32_t)n,k,[(NBBuffer *)b handle],o,(uint32_t)i,(int32_t)v,(uint32_t)bi) what:"drawIndexedPrimitives"]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o + instanceCount:(NSUInteger)i { + [self drawIndexedPrimitives:t indexCount:n indexType:k indexBuffer:b indexBufferOffset:o instanceCount:i baseVertex:0 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o { + [self drawIndexedPrimitives:t indexCount:n indexType:k indexBuffer:b indexBufferOffset:o instanceCount:1 baseVertex:0 baseInstance:0]; +} +- (void)useResource:(id)r usage:(MTLResourceUsage)u stages:(MTLRenderStages)s {(void)r;(void)u;(void)s;} +- (void)updateFence:(id)fence afterStages:(MTLRenderStages)stages {[self fence:fence stages:stages update:true];} +- (void)waitForFence:(id)fence beforeStages:(MTLRenderStages)stages {[self fence:fence stages:stages update:false];} +@end + +@interface NBBlitEncoder : NBEncoder +@end +@implementation NBBlitEncoder +// The arguments are passed as a plain array: an NSArray literal raises on a nil element, and a nil resource (a refused allocation) must fail the +// command buffer like any invalid argument, not abort the host process. +#define NB_OBJECTS(...) ((__unsafe_unretained id[]){__VA_ARGS__}) +- (bool)buffers:(__unsafe_unretained id *)objects count:(size_t)count { + for(size_t i=0;i)s sourceOffset:(NSUInteger)so toBuffer:(id)d destinationOffset:(NSUInteger)dO size:(NSUInteger)n { + if(![self buffers:NB_OBJECTS(s,d) count:2])return; + [self check:nmb_blit_copy_buffer(self.handle,[(NBBuffer *)s handle],so,[(NBBuffer *)d handle],dO,n) what:"copyFromBuffer:toBuffer:"]; +} +- (void)fillBuffer:(id)b range:(NSRange)r value:(uint8_t)v { + if(![self buffers:NB_OBJECTS(b) count:1])return; + [self check:nmb_blit_fill_buffer(self.handle,[(NBBuffer *)b handle],r.location,r.location+r.length,v) what:"fillBuffer"]; +} +- (void)copyFromTexture:(id)s sourceSlice:(NSUInteger)ss sourceLevel:(NSUInteger)sl sourceOrigin:(MTLOrigin)so sourceSize:(MTLSize)z + toTexture:(id)d destinationSlice:(NSUInteger)ds destinationLevel:(NSUInteger)dl destinationOrigin:(MTLOrigin)dO { + if(![self textures:NB_OBJECTS(s,d) count:2])return; + const uint64_t a[3]={so.x,so.y,so.z},size[3]={z.width,z.height,z.depth},b[3]={dO.x,dO.y,dO.z}; + [self check:nmb_blit_copy_texture(self.handle,[(NBTexture *)s handle],ss,sl,a,size,[(NBTexture *)d handle],ds,dl,b) what:"copyFromTexture:toTexture:"]; +} +- (void)copyFromTexture:(id)s toTexture:(id)d { + if(![self textures:NB_OBJECTS(s,d) count:2])return; + const NSUInteger levels=MIN(s.mipmapLevelCount,d.mipmapLevelCount),slices=MIN(s.arrayLength,d.arrayLength); + for(NSUInteger slice=0;slice>level,(NSUInteger)1),MAX(s.height>>level,(NSUInteger)1),MAX(s.depth>>level,(NSUInteger)1)); + [self copyFromTexture:s sourceSlice:slice sourceLevel:level sourceOrigin:MTLOriginMake(0,0,0) sourceSize:size toTexture:d destinationSlice:slice + destinationLevel:level destinationOrigin:MTLOriginMake(0,0,0)]; + } +} +- (void)copyFromBuffer:(id)s sourceOffset:(NSUInteger)o sourceBytesPerRow:(NSUInteger)row sourceBytesPerImage:(NSUInteger)image + sourceSize:(MTLSize)z toTexture:(id)d destinationSlice:(NSUInteger)slice destinationLevel:(NSUInteger)level + destinationOrigin:(MTLOrigin)dO { + if(![self buffers:NB_OBJECTS(s) count:1] || ![self textures:NB_OBJECTS(d) count:1])return; + const uint64_t size[3]={z.width,z.height,z.depth},b[3]={dO.x,dO.y,dO.z}; + [self check:nmb_blit_buffer_to_texture(self.handle,[(NBBuffer *)s handle],o,row,image,size,[(NBTexture *)d handle],slice,level,b) what:"copyFromBuffer:toTexture:"]; +} +- (void)copyFromTexture:(id)s sourceSlice:(NSUInteger)slice sourceLevel:(NSUInteger)level sourceOrigin:(MTLOrigin)so sourceSize:(MTLSize)z + toBuffer:(id)d destinationOffset:(NSUInteger)o destinationBytesPerRow:(NSUInteger)row destinationBytesPerImage:(NSUInteger)image { + if(![self textures:NB_OBJECTS(s) count:1] || ![self buffers:NB_OBJECTS(d) count:1])return; + const uint64_t a[3]={so.x,so.y,so.z},size[3]={z.width,z.height,z.depth}; + [self check:nmb_blit_texture_to_buffer(self.handle,[(NBTexture *)s handle],slice,level,a,size,[(NBBuffer *)d handle],o,row,image) what:"copyFromTexture:toBuffer:"]; +} +- (void)generateMipmapsForTexture:(id)t { + if(![self textures:NB_OBJECTS(t) count:1])return; + [self check:nmb_blit_generate_mipmaps(self.handle,[(NBTexture *)t handle]) what:"generateMipmapsForTexture"]; +} +- (void)synchronizeResource:(id)r {(void)r;} +@end + +static void completed(void *context); +static void scheduled(void *context); +@implementation NBCommandBuffer +- (instancetype)init { + if((self=[super init])){_scheduledHandlers=[NSMutableArray array];_completedHandlers=[NSMutableArray array];_surfaces=[NSMutableSet set];_resourceOwners=[NSMutableSet set];_handlers=dispatch_group_create();_schedulingHandlers=dispatch_group_create();} + if(self)createdUs=nowUs(); + return self; +} +- (void)dealloc {nmb_command_buffer_release(_handle);} +- (id)commandQueue {return (id)_queue;} +- (BOOL)retainedReferences {return YES;} +// Protected content is not supported (the runtime's AppleOnlyCall::SetProtectionOptions takes Metal's default only): any other option fails +// the command buffer. +- (uint64_t)protectionOptions {return 0;} +- (void)setProtectionOptions:(uint64_t)options {if(options)[self refuse:[NSString stringWithFormat:@"protection options 0x%llx",options]];} +- (void)refuse:(NSString *)what { + if(!self.encodeFailure){self.encodeFailure=what;NBLOG(@"native-metal: command buffer refused: %@",what);} +} +- (MTLCommandBufferStatus)status { + if(self.failedLocally)return MTLCommandBufferStatusError; + return (MTLCommandBufferStatus)nmb_command_buffer_status(_handle); +} +- (NSError *)error { + if(self.failedLocally)return nbError(self.encodeFailure); + if(self.status!=MTLCommandBufferStatusError)return nil; + char text[512]={};nmb_command_buffer_error(_handle,text,sizeof text); + return nbError([NSString stringWithUTF8String:text]); +} +// Debug groups and cache hints name work for tools and the driver's caches; they change nothing the GPU computes here. +- (void)pushDebugGroup:(NSString *)name {(void)name;} +- (void)popDebugGroup {} +- (void)encodeCacheHintTag:(uint64_t)tag resourceGroups:(const id __unsafe_unretained [])groups count:(NSUInteger)count {(void)tag;(void)groups;(void)count;} +- (void)encodeCacheHintFinalize:(uint64_t)tag resourceGroups:(const id __unsafe_unretained [])groups count:(NSUInteger)count {(void)tag;(void)groups;(void)count;} +// The kernel driver's own submission times are not recorded: 0 is Metal's answer where they are unavailable. +- (CFTimeInterval)kernelStartTime {return 0;} +- (CFTimeInterval)kernelEndTime {return 0;} +- (BOOL)commitAndWaitUntilSubmitted {[(id)self commit];[self waitUntilScheduled];return !self.failedLocally;} +- (CFTimeInterval)GPUStartTime {return nmb_command_buffer_gpu_time(_handle,false);} +- (CFTimeInterval)GPUEndTime {return nmb_command_buffer_gpu_time(_handle,true);} +- (id)encoder:(NBEncoder *)encoder handle:(NmbEncoder *)handle what:(NSString *)what { + if(self.committed){[self refuse:[what stringByAppendingString:@" after commit"]];return nil;} + if(!handle){[self refuse:[NSString stringWithFormat:@"%@: %@",what,lastError()]];return nil;} + encoder.handle=handle;encoder.commandBuffer=self;return encoder; +} +- (id)computeCommandEncoder { + return [self encoder:[NBComputeEncoder new] handle:self.committed?nullptr:nmb_compute_encoder_new(_handle) what:@"computeCommandEncoder"]; +} +- (id)computeCommandEncoderWithDispatchType:(MTLDispatchType)type { + if(type!=MTLDispatchTypeSerial){[self refuse:@"concurrent compute dispatch"];return nil;} + return [self computeCommandEncoder]; +} +- (id)blitCommandEncoder { + gBlits++;encodedWork.blits++; + return [self encoder:[NBBlitEncoder new] handle:self.committed?nullptr:nmb_blit_encoder_new(_handle) what:@"blitCommandEncoder"]; +} +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)d { + NmbRenderPassDescriptor pass={}; + { + id first=d.colorAttachments[0].texture; + if(first) { + const uint64_t pixels=(uint64_t)first.width*first.height; + gPasses++;gPassPixels+=pixels;encodedWork.passes++;encodedWork.pixels+=pixels; + uint64_t seen=gBiggestPass.load();while(pixels>seen && !gBiggestPass.compare_exchange_weak(seen,pixels)){} + if(pixels>=2000000)gFullScreenPasses++; + } + } + auto attachment=[&](MTLRenderPassAttachmentDescriptor *a,NmbPassAttachment &out)->bool { + if(!a.texture)return true; + if(![(id)a.texture isKindOfClass:[NBTexture class]])return false; + NBTexture *texture=(NBTexture *)a.texture;if(texture.surface)[_surfaces addObject:texture]; + [_resourceOwners addObject:texture]; + out.texture=texture.handle;out.level=a.level;out.slice=a.slice;out.depth_plane=a.depthPlane;out.load=a.loadAction;out.store=a.storeAction; + return true; + }; + for(unsigned i=0;i<8;i++) { + MTLRenderPassColorAttachmentDescriptor *c=d.colorAttachments[i]; + if(!attachment(c,pass.colors[i])){[self refuse:@"foreign color attachment"];return nil;} + pass.colors[i].clear[0]=c.clearColor.red;pass.colors[i].clear[1]=c.clearColor.green;pass.colors[i].clear[2]=c.clearColor.blue;pass.colors[i].clear[3]=c.clearColor.alpha; + } + if(!attachment(d.depthAttachment,pass.depth) || !attachment(d.stencilAttachment,pass.stencil)){[self refuse:@"foreign depth/stencil attachment"];return nil;} + pass.depth.clear[0]=d.depthAttachment.clearDepth;pass.stencil.clear[0]=d.stencilAttachment.clearStencil; + pass.width=d.renderTargetWidth;pass.height=d.renderTargetHeight;pass.array_length=d.renderTargetArrayLength;pass.samples=d.defaultRasterSampleCount; + if(d.visibilityResultBuffer) { + if(![(id)d.visibilityResultBuffer isKindOfClass:[NBBuffer class]]){[self refuse:@"foreign visibility result buffer"];return nil;} + [_resourceOwners addObject:d.visibilityResultBuffer]; + pass.visibility=[(NBBuffer *)d.visibilityResultBuffer handle]; + } + return [self encoder:[NBRenderEncoder new] handle:self.committed?nullptr:nmb_render_encoder_new(_handle,&pass) what:@"renderCommandEncoder"]; +} +- (void)addScheduledHandler:(MTLCommandBufferHandler)block {if(block)@synchronized(self){[_scheduledHandlers addObject:[block copy]];}} +- (void)addCompletedHandler:(MTLCommandBufferHandler)block {if(block)@synchronized(self){[_completedHandlers addObject:[block copy]];}} +- (void)enqueue {} +- (void)event:(id)event value:(uint64_t)value signal:(bool)signal { + if(![(id)event isKindOfClass:[NBEvent class]]){[self refuse:@"foreign event"];return;} + if(self.committed){[self refuse:@"event encoded after commit"];return;} + NmbEvent *handle=[(NBEvent *)event handle]; + if(!(signal?nmb_command_buffer_signal_event(_handle,handle,value):nmb_command_buffer_wait_event(_handle,handle,value))) + [self refuse:[NSString stringWithFormat:@"%s: %@",signal?"encodeSignalEvent":"encodeWaitForEvent",lastError()]]; +} +- (void)encodeSignalEvent:(id)event value:(uint64_t)value {[self event:event value:value signal:true];} +- (void)encodeWaitForEvent:(id)event value:(uint64_t)value {[self event:event value:value signal:false];} +// The runtime has submitted the work. Deliver scheduling without waiting for GPU completion. +- (void)didSchedule { + { + const uint64_t took=nowUs()-self.committedUs; + std::lock_guard lock(gCommitLock); + CommitRecord &r=gCommitRing[self.commitIndex%32]; + if(r.index==self.commitIndex)r.scheduledUs=took; + } + NSArray *handlers; + @synchronized(self){ + _schedulingDelivered=YES;handlers=[_scheduledHandlers copy];[_scheduledHandlers removeAllObjects]; + } + void (^run)(void)=^{ + for(MTLCommandBufferHandler block in handlers)block((id)self); + dispatch_group_leave(self->_schedulingHandlers); + }; + // Run on the runtime's scheduled-handler thread, preserving its ordering and own-wait + // deadlock checks. The queue's completionQueue routes completed handlers only. + run(); +} +// After completion (or a refusal): invalidate the CPU's view of imported surfaces, then completed handlers, once. +- (void)finish { + flushSurfaces(_surfaces); + // Actual GPU completion (or a local refusal that never submitted) is the + // release point. A retained completed command must not pin every old frame. + [_surfaces removeAllObjects];[_resourceOwners removeAllObjects]; + @synchronized(gInFlight){[gInFlight removeObject:self];} + NSArray *completedHandlers; + @synchronized(self){ + // A frontend refusal or a failed submission never scheduled. Unblock its waiter without inventing a scheduled event. + if(!_schedulingDelivered){_schedulingDelivered=YES;[_scheduledHandlers removeAllObjects];dispatch_group_leave(_schedulingHandlers);} + completedHandlers=[_completedHandlers copy];[_completedHandlers removeAllObjects]; + } + void (^run)(void)=^{ + for(MTLCommandBufferHandler block in completedHandlers)block((id)self); + dispatch_group_leave(self->_handlers); + }; + dispatch_queue_t queue=_queue.completionQueue; + if(queue)dispatch_async(queue,run);else run(); +} +- (void)commit { + if(self.committed){NBLOG(@"native-metal: commit twice");return;} + self.committed=YES; + self.committedUs=nowUs(); + { + std::lock_guard lock(gCommitLock); + // Assign once under the same lock as publication. A concurrent commit cannot reuse this index. + const uint32_t index=(uint32_t)(gCommits.fetch_add(1)+1);self.commitIndex=index; + CommitRecord &r=gCommitRing[index%32]; + r={};r.index=index;r.committedUs=self.committedUs;r.ageUs=self.committedUs-createdUs;r.work=encodedWork; + } + dispatch_group_enter(_handlers); + dispatch_group_enter(_schedulingHandlers); + @synchronized(gInFlight){[gInFlight addObject:self];} + if(!self.encodeFailure) { + flushSurfaces(_surfaces); + // Completion owns this reference; the runtime orders the non-owning scheduled callback before it. + void *context=(__bridge_retained void *)self; + if(nmb_command_buffer_on_completed(_handle,completed,context) && + nmb_command_buffer_on_scheduled(_handle,scheduled,context) && nmb_command_buffer_commit(_handle))return; + (void)CFBridgingRelease(context); // refused registration/commit cannot call the registered handlers + [self refuse:[@"commit: " stringByAppendingString:lastError()]]; + } + self.failedLocally=YES;gCommitFailures++; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{[self finish];}); +} +- (void)waitUntilCompleted { + if(!self.committed)return; + const uint64_t began=nowUs(); + if(!self.failedLocally)nmb_command_buffer_wait(_handle,0); + dispatch_group_wait(_handlers,DISPATCH_TIME_FOREVER); + gTimeWaitCompleted.add(began); +} +- (void)waitUntilScheduled { + if(!self.committed)return; + const uint64_t began=nowUs(); + if(!self.failedLocally && !nmb_command_buffer_wait_scheduled(_handle)){ + NBLOG(@"native-metal: waitUntilScheduled refused: %@",lastError());return; + } + dispatch_group_wait(_schedulingHandlers,DISPATCH_TIME_FOREVER); + gTimeWaitScheduled.add(began); +} +@end +static void scheduled(void *context) {[(__bridge NBCommandBuffer *)context didSchedule];} +// The runtime's completion: the command buffer's own reference, taken at commit, is given back here. +static void completed(void *context) { + gCompleted++; + NBCommandBuffer *commandBuffer=(__bridge_transfer NBCommandBuffer *)context; + { + const uint64_t took=nowUs()-commandBuffer.committedUs; + gTimeEncodeToCommit.count++;gTimeEncodeToCommit.total+=took; + uint64_t seen=gTimeEncodeToCommit.slowest.load();while(took>seen && !gTimeEncodeToCommit.slowest.compare_exchange_weak(seen,took)){} + std::lock_guard lock(gCommitLock); + CommitRecord &r=gCommitRing[commandBuffer.commitIndex%32]; + if(r.index==commandBuffer.commitIndex)r.callbackUs=took; + } + [commandBuffer finish]; +} + +@implementation NBQueue +- (void)dealloc {nmb_queue_release(_handle);} +- (id)commandBuffer { + NmbCommandBuffer *handle=nmb_command_buffer_new(_handle); + if(!handle){NBLOG(@"native-metal: commandBuffer refused: %@",lastError());return nil;} + NBCommandBuffer *commandBuffer=[NBCommandBuffer new];commandBuffer.handle=handle;commandBuffer.queue=self;return (id)commandBuffer; +} +- (id)commandBufferWithUnretainedReferences {return [self commandBuffer];} +- (void)setSubmissionQueue:(dispatch_queue_t)queue {(void)queue;} // submission happens on the caller's thread here +// setCompletionQueue: is the property's setter. +// GPU priorities are not applied (one hardware queue): reported as not applied. +- (BOOL)setGPUPriority:(NSUInteger)priority {(void)priority;return NO;} +- (BOOL)setBackgroundGPUPriority:(NSUInteger)priority {(void)priority;return NO;} +- (id)commandBufferWithDescriptor:(MTLCommandBufferDescriptor *)d {(void)d;return [self commandBuffer];} +@end + +// ---- the device ---- +template static T failWith(NSError **error,NSString *what,T value) { + noteProblem([NSString stringWithFormat:@"%@: %@",what,lastError()]); + if(error)*error=nbError([NSString stringWithFormat:@"%@: %@",what,lastError()]); + return value; +} +// A render pipeline descriptor in the bridge's form; false, with the error, for a function of another device. +static bool renderDescriptor(MTLRenderPipelineDescriptor *d,NmbRenderPipelineDescriptor *p,NSError **error) { + if(![(id)d.vertexFunction isKindOfClass:[NBFunction class]] || (d.fragmentFunction && ![(id)d.fragmentFunction isKindOfClass:[NBFunction class]])) { + if(error)*error=nbError(@"foreign function");return false; + } + p->vertex=[(NBFunction *)d.vertexFunction handle];p->fragment=d.fragmentFunction?[(NBFunction *)d.fragmentFunction handle]:nullptr; + for(unsigned i=0;i<8;i++) { + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i]; + p->colors[i]={(uint64_t)a.pixelFormat,(uint64_t)a.writeMask,(uint64_t)a.blendingEnabled,(uint64_t)a.sourceRGBBlendFactor,(uint64_t)a.destinationRGBBlendFactor, + (uint64_t)a.rgbBlendOperation,(uint64_t)a.sourceAlphaBlendFactor,(uint64_t)a.destinationAlphaBlendFactor,(uint64_t)a.alphaBlendOperation}; + } + p->depth_format=d.depthAttachmentPixelFormat;p->stencil_format=d.stencilAttachmentPixelFormat;p->samples=d.rasterSampleCount; + p->topology=d.inputPrimitiveTopology;p->rasterization=d.rasterizationEnabled; + if(MTLVertexDescriptor *v=d.vertexDescriptor) { + for(unsigned i=0;i<31;i++) { + MTLVertexAttributeDescriptor *a=v.attributes[i];if(a.format==MTLVertexFormatInvalid)continue; + p->vertex_inputs=1;p->attributes[i]={(uint64_t)a.format,a.offset,a.bufferIndex}; + if(a.bufferIndex<31){MTLVertexBufferLayoutDescriptor *l=v.layouts[a.bufferIndex];p->layouts[a.bufferIndex]={l.stride,(uint64_t)l.stepFunction,l.stepRate};} + } + } + return true; +} + +// ---- heaps ---- +// Metal heaps over the runtime's: an automatic heap places each resource, a placement heap takes an offset. Resources made from one keep it +// alive (their `heap`); a heap's resources share its storage mode, CPU cache mode and (here) hazard tracking mode. +@interface NBHeap : NBObject +@property(nonatomic) NmbHeap *handle; +@property(nonatomic) MTLResourceOptions options; +@property(nonatomic) MTLHeapType heapType; +@end +@implementation NBHeap +- (void)dealloc {nmb_heap_release(_handle);} +- (NSUInteger)size {return nmb_heap_size(_handle);} +- (NSUInteger)usedSize {return nmb_heap_used_size(_handle);} +- (NSUInteger)currentAllocatedSize {return nmb_heap_allocated_size(_handle);} +- (NSUInteger)maxAvailableSizeWithAlignment:(NSUInteger)alignment {return nmb_heap_max_available_size(_handle,alignment);} +- (MTLResourceOptions)resourceOptions {return _options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (MTLHazardTrackingMode)hazardTrackingMode {return (MTLHazardTrackingMode)((_options&MTLResourceHazardTrackingModeMask)>>MTLResourceHazardTrackingModeShift);} +- (MTLHeapType)type {return _heapType;} +- (uint64_t)protectionOptions {return 0;} +- (MTLPurgeableState)setPurgeableState:(MTLPurgeableState)state { + const uint64_t answer=nmb_heap_set_purgeable_state(_handle,state); + if(!answer)NBLOG(@"native-metal: heap setPurgeableState refused: %@",lastError()); + return (MTLPurgeableState)answer; +} +- (id)bufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options offset:(uint64_t)offset { + NmbBuffer *handle=nmb_heap_new_buffer(_handle,length,options,offset); + if(!handle){NBLOG(@"native-metal: heap buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;buffer.parentHeap=(id)self; + buffer.heapOffset=offset==UINT64_MAX?0:offset;return (id)buffer; +} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options {return [self bufferWithLength:length options:options offset:UINT64_MAX];} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options offset:(NSUInteger)offset { + return [self bufferWithLength:length options:options offset:offset]; +} +- (id)textureWithDescriptor:(MTLTextureDescriptor *)d offset:(uint64_t)offset { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_heap_new_texture(_handle,&descriptor,offset); + if(!handle){NBLOG(@"native-metal: heap texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.parentHeap=(id)self;texture.heapOffset=offset==UINT64_MAX?0:offset;return (id)texture; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d {return [self textureWithDescriptor:d offset:UINT64_MAX];} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset {return [self textureWithDescriptor:d offset:offset];} +@end +// Apple's SPI resource groups name resources for cache hints (encodeCacheHintTag:/Finalize:); a group only keeps its resources alive. +@interface NBResourceGroup : NBObject +@property(nonatomic,strong) NSArray *resources; +@end +@implementation NBResourceGroup +@end + +// Creation and archive population must use the same complete supported descriptor. +static bool computeDescriptor(MTLComputePipelineDescriptor *d,NmbComputePipelineDescriptor *p,NSError **error) { + if(!d || ![(id)d.computeFunction isKindOfClass:[NBFunction class]]){if(error)*error=nbError(@"foreign compute function");return false;} + // Metal may materialize empty stage/linked descriptors on access. Preserve the + // default meaning while refusing state this frontend cannot yet represent. + MTLStageInputOutputDescriptor *stage=d.stageInputDescriptor; + bool empty=!stage || (stage.indexType==MTLIndexTypeUInt16 && stage.indexBufferIndex==0); + for(NSUInteger index=0;stage && index<31;++index) { + MTLAttributeDescriptor *a=stage.attributes[index];MTLBufferLayoutDescriptor *b=stage.layouts[index]; + empty=empty && a.format==MTLAttributeFormatInvalid && !a.offset && !a.bufferIndex && !b.stride && + b.stepFunction==MTLStepFunctionPerVertex && b.stepRate==1; + } + if(!empty){if(error)*error=nbError(@"compute stage inputs are not implemented");return false;} + MTLLinkedFunctions *linked=d.linkedFunctions; + if(linked.functions.count || linked.binaryFunctions.count || linked.privateFunctions.count || linked.groups.count || + d.preloadedLibraries.count || d.insertLibraries.count) { + if(error)*error=nbError(@"compute linked/dynamic libraries are not implemented");return false; + } + p->function=[(NBFunction *)d.computeFunction handle];p->max_threads=d.maxTotalThreadsPerThreadgroup; + p->multiple_of_execution_width=d.threadGroupSizeIsMultipleOfThreadExecutionWidth;p->indirect=d.supportIndirectCommandBuffers; + p->max_call_stack_depth=d.maxCallStackDepth;p->adding_binary_functions=d.supportAddingBinaryFunctions; + for(NSUInteger index=0;index<31;++index)p->mutability[index]=d.buffers[index].mutability; + return true; +} + +// ---- binary archives ---- +@interface NBBinaryArchive : NBObject +@property(nonatomic) NmbBinaryArchive *handle; +@end +@implementation NBBinaryArchive +- (void)dealloc {nmb_binary_archive_release(_handle);} +- (NSUInteger)options {return 0;} +- (BOOL)addRenderPipelineFunctionsWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + auto *p=new NmbRenderPipelineDescriptor(); + const bool ok=renderDescriptor(d,p,error) && nmb_binary_archive_add_render(_handle,p);delete p; + if(!ok)return failWith(error,@"archive render pipeline",NO); + if(error)*error=nil;return YES; +} +- (BOOL)addComputePipelineFunctionsWithDescriptor:(MTLComputePipelineDescriptor *)d error:(NSError **)error { + NmbComputePipelineDescriptor p={};if(!computeDescriptor(d,&p,error))return NO; + if(!nmb_binary_archive_add_compute_descriptor(_handle,&p))return failWith(error,@"archive compute pipeline",NO); + if(error)*error=nil;return YES; +} +- (BOOL)serializeToURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=nbError(@"a file URL is required");return NO;} + if(!nmb_binary_archive_serialize(_handle,url.fileSystemRepresentation))return failWith(error,@"archive serialize",NO); + if(error)*error=nil;return YES; +} +@end + +static bool pipelineArchives(NSArray *list,std::vector &archives,NSError **error) { + for(id archive in list) { + if(![archive isKindOfClass:[NBBinaryArchive class]]){if(error)*error=nbError(@"foreign binary archive");return false;} + archives.push_back([(NBBinaryArchive *)archive handle]); + } + return true; +} + +@interface NBDeviceMethods : NSObject +@end +@implementation NBDeviceMethods +- (NSString *)name {return gName;} +- (uint64_t)registryID {return gRegistryID;} +- (NSUInteger)maxBufferLength {return nmb_device_max_buffer_length(gDevice);} +- (BOOL)hasUnifiedMemory {return YES;} +- (BOOL)isLowPower {return NO;} +- (BOOL)isRemovable {return NO;} +- (BOOL)isHeadless {return NO;} +// supportsFamily: and the other queries the runtime answers are installed from it (installAnswers). No legacy feature set is claimed: the +// base implementation reads Apple tables this device does not have. +- (BOOL)supportsFeatureSet:(MTLFeatureSet)set {(void)set;return NO;} +// The original discovery (_MTLAddDevice) runs these four on every device it registers. The target's own versions fill the base class's limits +// and feature tables from a hardware profile (`featureProfile`, `indirectArgumentBufferCapabilities`, a table of Apple GPUs) that this device +// has not got: running them would claim an Apple GPU's capabilities. They are empty here, so a limit or feature the runtime measured is answered +// by this file's own answers (installAnswers) and any other stays the base class's neutral 0/NO. Nothing is claimed that nothing measured. +- (void)initLimits {} +- (void)initFeatureQueries {} +- (void)initGPUFamilySupport {} +- (void)initWorkarounds {} +- (NSString *)productName {return gName;} +- (NSString *)vendorName {char name[64]={};nmb_device_vendor_name(gDevice,name,sizeof name);return [NSString stringWithUTF8String:name];} +- (MTLSize)maxThreadsPerThreadgroup { + uint64_t s[3]={};if(!nmb_device_max_threads_per_threadgroup(gDevice,s))NBLOG(@"native-metal: maxThreadsPerThreadgroup: %@",lastError()); + return MTLSizeMake(s[0],s[1],s[2]); +} +- (void)doesNotRecognizeSelector:(SEL)selector { + if(gMissing)gMissing("NativeBridgeMetalDevice",sel_getName(selector)); + noteProblem([NSString stringWithFormat:@"missing device %s",sel_getName(selector)]); + [super doesNotRecognizeSelector:selector]; +} +- (id)newEvent { + NmbEvent *handle=nmb_event_new(gDevice,false);if(!handle)return nil; + NBEvent *event=[NBEvent new];event.handle=handle;return (id)event; +} +- (id)newSharedEvent { + NmbEvent *handle=nmb_event_new(gDevice,true);if(!handle)return nil; + NBSharedEvent *event=[NBSharedEvent new];event.handle=handle;return (id)event; +} +- (id)newFence { + NmbFence *handle=nmb_fence_new(gDevice);if(!handle)return nil; + NBFence *fence=[NBFence new];fence.handle=handle;return (id)fence; +} +- (id)newCommandQueue { + NmbQueue *handle=nmb_queue_new(gDevice);if(!handle)return nil; + NBQueue *queue=[NBQueue new];queue.handle=handle;return (id)queue; +} +- (id)newHeapWithDescriptor:(MTLHeapDescriptor *)d { + if(!d)return nil; + NmbHeap *handle=nmb_heap_new(gDevice,d.type,d.size,(uint64_t)d.resourceOptions); + if(!handle){NBLOG(@"native-metal: heap refused: %@",lastError());return nil;} + NBHeap *heap=[NBHeap new];heap.handle=handle;heap.options=d.resourceOptions;heap.heapType=d.type;return (id)heap; +} +- (id)newResourceGroupFromResources:(const id __unsafe_unretained [])resources count:(NSUInteger)count { + NBResourceGroup *group=[NBResourceGroup new]; + NSMutableArray *held=[NSMutableArray arrayWithCapacity:count]; + for(NSUInteger i=0;i)newTextureWithBytesNoCopy:(void *)bytes length:(NSUInteger)length descriptor:(MTLTextureDescriptor *)d deallocator:(void (^)(void *,NSUInteger))deallocator { + (void)bytes;(void)length;(void)d;(void)deallocator; + NBLOG(@"native-metal: newTextureWithBytesNoCopy:length:descriptor:deallocator: is not implemented (Apple's texture layout of raw memory is not known)"); + return nil; +} +- (id)newTextureLayoutWithDescriptor:(MTLTextureDescriptor *)d isHeapOrBufferBacked:(BOOL)backed { + (void)d;(void)backed; + NBLOG(@"native-metal: newTextureLayoutWithDescriptor:isHeapOrBufferBacked: is not implemented (Apple's layout object is not known)"); + return nil; +} +- (id)newCommandQueueWithMaxCommandBufferCount:(NSUInteger)count {(void)count;return [(id)self newCommandQueue];} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options { + NmbBuffer *handle=nmb_buffer_new(gDevice,length,options,nullptr); + if(!handle){NBLOG(@"native-metal: buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;return (id)buffer; +} +- (id)newBufferWithBytes:(const void *)bytes length:(NSUInteger)length options:(MTLResourceOptions)options { + if(!bytes)return nil; + NmbBuffer *handle=nmb_buffer_new(gDevice,length,options,bytes); + if(!handle){NBLOG(@"native-metal: buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;return (id)buffer; +} +- (id)newBufferWithBytesNoCopy:(void *)pointer length:(NSUInteger)length options:(MTLResourceOptions)options deallocator:(void (^)(void *,NSUInteger))deallocator { + NmbBuffer *handle=nmb_buffer_new_no_copy(gDevice,pointer,length,options); + if(!handle){NBLOG(@"native-metal: no-copy buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options; + buffer.wrapped=pointer;buffer.deallocator=deallocator; + return (id)buffer; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new(gDevice,&descriptor);noteFormat((uint32_t)descriptor.format,handle!=nullptr); + if(!handle){NBLOG(@"native-metal: texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor;return (id)texture; +} +// The surface's pages are the texels: its mapping in this process is imported without a copy (pinned by the kernel, mapped into the GPU's +// 4 KiB context by Turnip), so every process that imports the surface renders and samples the same memory. +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d iosurface:(IOSurfaceRef)surface plane:(NSUInteger)plane { + if(!surface || !d || plane || IOSurfaceGetPlaneCount(surface))return nil; + void *base=IOSurfaceGetBaseAddress(surface);const size_t page=(size_t)getpagesize(),allocation=IOSurfaceGetAllocSize(surface); + if(!page || (page&(page-1)) || !base || ((uintptr_t)base&(page-1)) || !allocation || allocation>SIZE_MAX-(page-1))return nil; + const size_t bytes=(allocation+page-1)&~(page-1); + NBBuffer *buffer=surfaceBuffer(surface,base,bytes);if(!buffer)return nil; + NBTexture *texture=textureOnBuffer(buffer,d,0,IOSurfaceGetBytesPerRow(surface),true);if(!texture)return nil; + texture.surface=(IOSurfaceRef)CFRetain(surface);return (id)texture; +} +- (id)newLibraryWithData:(dispatch_data_t)data error:(NSError **)error { + const void *bytes=nullptr;size_t size=0;dispatch_data_t mapped=dispatch_data_create_map(data,&bytes,&size); + const uint64_t libraryBegan=nowUs(); + NmbLibrary *handle=nmb_library_new(gDevice,bytes,size,false);(void)mapped;gTimeLibrary.add(libraryBegan); + if(!handle)return failWith>(error,@"library",nil); + NBLibrary *library=[NBLibrary new];library.handle=handle;if(error)*error=nil;return (id)library; +} +- (id)newLibraryWithFile:(NSString *)path error:(NSError **)error { + NSData *bytes=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:error]:nil;if(!bytes)return nil; + const uint64_t libraryBegan=nowUs(); + NmbLibrary *handle=nmb_library_new(gDevice,bytes.bytes,bytes.length,false);gTimeLibrary.add(libraryBegan); + if(!handle)return failWith>(error,@"library",nil); + NBLibrary *library=[NBLibrary new];library.handle=handle;if(error)*error=nil;return (id)library; +} +- (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=nbError(@"a file URL is required");return nil;} + return [(id)self newLibraryWithFile:url.path error:error]; +} +- (id)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(NSError **)error { + NSURL *url=[bundle URLForResource:@"default" withExtension:@"metallib"]; + if(!url){if(error)*error=nbError(@"the bundle has no default.metallib");return nil;} + return [(id)self newLibraryWithURL:url error:error]; +} +- (id)newDefaultLibrary {return [(id)self newDefaultLibraryWithBundle:[NSBundle mainBundle] error:nil];} +- (id)newComputePipelineStateWithFunction:(id)function error:(NSError **)error { + return [(id)self newComputePipelineStateWithFunction:function options:MTLPipelineOptionNone reflection:nil error:error]; +} +- (id)newComputePipelineStateWithFunction:(id)function options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedComputePipelineReflection *)reflection error:(NSError **)error { + if(reflection)*reflection=nil; + if(![(id)function isKindOfClass:[NBFunction class]]){if(error)*error=nbError(@"foreign function");return nil;} + const NmbPipelineOptions creation={(uint64_t)options,nullptr,0}; + const uint64_t pipelineBegan=nowUs(); + NmbComputePipeline *handle=nmb_compute_pipeline_new_with_options(gDevice,[(NBFunction *)function handle],&creation);gTimePipeline.add(pipelineBegan); + if(!handle)return failWith>(error,@"compute pipeline",nil); + NBComputePipeline *state=[NBComputePipeline new];state.handle=handle;gPipelines++;if(error)*error=nil;return (id)state; +} +- (id)newComputePipelineStateWithDescriptor:(MTLComputePipelineDescriptor *)d options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedComputePipelineReflection *)reflection error:(NSError **)error { + if(reflection)*reflection=nil; + NmbComputePipelineDescriptor p={};if(!computeDescriptor(d,&p,error))return nil; + std::vector archives; + if(!pipelineArchives(d.binaryArchives,archives,error))return nil; + const NmbPipelineOptions creation={(uint64_t)options,archives.data(),archives.size()}; + const uint64_t pipelineBegan=nowUs(); + NmbComputePipeline *handle=nmb_compute_pipeline_new_with_descriptor(gDevice,&p,&creation);gTimePipeline.add(pipelineBegan); + if(!handle)return failWith>(error,@"compute pipeline",nil); + NBComputePipeline *state=[NBComputePipeline new];state.handle=handle;state.label=d.label;gPipelines++; + if(error)*error=nil;return (id)state; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + return [(id)self newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionNone reflection:nil error:error]; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedRenderPipelineReflection *)reflection error:(NSError **)error { + if(reflection)*reflection=nil; + auto *p=new NmbRenderPipelineDescriptor(); + if(!renderDescriptor(d,p,error)){delete p;return nil;} + std::vector archives; + if(!pipelineArchives(d.binaryArchives,archives,error)){delete p;return nil;} + const NmbPipelineOptions creation={(uint64_t)options,archives.data(),archives.size()}; + const uint64_t pipelineBegan=nowUs(); + NmbRenderPipeline *handle=nmb_render_pipeline_new_with_options(gDevice,p,&creation);delete p;gTimePipeline.add(pipelineBegan); + if(!handle)return failWith>(error,@"render pipeline",nil); + NBRenderPipeline *state=[NBRenderPipeline new];state.handle=handle;gPipelines++;if(error)*error=nil;return (id)state; +} +- (id)newBinaryArchiveWithDescriptor:(MTLBinaryArchiveDescriptor *)d error:(NSError **)error { + if(d.url && !d.url.isFileURL){if(error)*error=nbError(@"a file URL is required");return nil;} + NmbBinaryArchive *handle=nmb_binary_archive_new(gDevice,d.url?d.url.fileSystemRepresentation:nullptr); + if(!handle)return failWith>(error,@"binary archive",nil); + NBBinaryArchive *archive=[NBBinaryArchive new];archive.handle=handle;if(error)*error=nil;return (id)archive; +} +- (NSUInteger)requiredLinearTextureBytesPerRowForDescriptor:(MTLTextureDescriptor *)d { + const uint64_t row=nmb_device_linear_bytes_per_row(gDevice,d.pixelFormat,d.width); + if(!row)NBLOG(@"native-metal: requiredLinearTextureBytesPerRow: %@",lastError()); + return row; +} +- (id)newSamplerStateWithDescriptor:(MTLSamplerDescriptor *)d { + NmbSamplerDescriptor s={(uint64_t)d.minFilter,(uint64_t)d.magFilter,(uint64_t)d.mipFilter,(uint64_t)d.sAddressMode,(uint64_t)d.tAddressMode,(uint64_t)d.rAddressMode, + (uint64_t)d.borderColor,(uint64_t)d.normalizedCoordinates,d.maxAnisotropy,(uint64_t)d.compareFunction,(uint64_t)d.lodAverage,d.lodMinClamp,d.lodMaxClamp}; + NmbSampler *handle=nmb_sampler_new(gDevice,&s); + if(!handle){NBLOG(@"native-metal: sampler refused: %@",lastError());return nil;} + NBSampler *sampler=[NBSampler new];sampler.handle=handle;return (id)sampler; +} +- (id)newDepthStencilStateWithDescriptor:(MTLDepthStencilDescriptor *)d { + auto face=[](MTLStencilDescriptor *s)->NmbStencil { + return {(uint64_t)s.stencilCompareFunction,(uint64_t)s.stencilFailureOperation,(uint64_t)s.depthFailureOperation,(uint64_t)s.depthStencilPassOperation,s.readMask,s.writeMask}; + }; + NmbDepthStencilDescriptor s={(uint64_t)d.depthCompareFunction,(uint64_t)d.depthWriteEnabled,face(d.frontFaceStencil),face(d.backFaceStencil)}; + NmbDepthStencil *handle=nmb_depth_stencil_new(gDevice,&s); + if(!handle){NBLOG(@"native-metal: depth/stencil state refused: %@",lastError());return nil;} + NBDepthStencil *state=[NBDepthStencil new];state.handle=handle;return (id)state; +} +@end + +// The device queries the runtime answers (nmb_device_answer_selector), as methods of the device class. One without an argument is read once +// and installed only where the device has an answer of its own; one with an argument asks the runtime each call, and a refusal answers 0/NO +// with the reason logged. +static std::unordered_map gAnswers; +static NSUInteger countAnswer(id,SEL cmd) {return (NSUInteger)gAnswers.at(cmd);} +static BOOL flagAnswer(id,SEL cmd) {return gAnswers.at(cmd)!=0;} +static uint64_t askAnswer(SEL cmd,NSUInteger arg) { + uint64_t value=0; + if(!nmb_device_answer(gDevice,sel_getName(cmd),arg,&value)){NBLOG(@"native-metal: %s(%lu): %@",sel_getName(cmd),(unsigned long)arg,lastError());return 0;} + return value; +} +static NSUInteger countArgAnswer(id,SEL cmd,NSUInteger arg) {return (NSUInteger)askAnswer(cmd,arg);} +static BOOL flagArgAnswer(id,SEL cmd,NSUInteger arg) {return askAnswer(cmd,arg)!=0;} +static void installAnswers(Class cls) { + uint32_t kind=0; + for(size_t i=0;const char *name=nmb_device_answer_selector(i,&kind);i++) { + SEL sel=sel_registerName(name);uint64_t value=0; + switch(kind) { + case NMB_ANSWER_COUNT: case NMB_ANSWER_FLAG: + if(!nmb_device_answer(gDevice,name,0,&value)){NBLOG(@"native-metal: no device answer: %@",lastError());continue;} + // Extended-range 10-bit (XR10) framebuffers are withheld: this display chain presents 8-bit BGRX scanout only. The render server picks its + // wide-gamut 64-bit (and Apple-compressed) framebuffer when a device offers them, and the display driver could not show it (int-089: the + // swap's surface was '&w4a', 8 bytes per pixel, and every swap was refused). + if(!strcmp(name,"supportsExtendedXR10Formats"))value=0; + gAnswers[sel]=value; + class_addMethod(cls,sel,kind==NMB_ANSWER_COUNT?(IMP)countAnswer:(IMP)flagAnswer,kind==NMB_ANSWER_COUNT?"Q16@0:8":"B16@0:8"); + break; + case NMB_ANSWER_COUNT_ARG: class_addMethod(cls,sel,(IMP)countArgAnswer,"Q24@0:8Q16");break; + case NMB_ANSWER_FLAG_ARG: class_addMethod(cls,sel,(IMP)flagArgAnswer,"B24@0:8Q16");break; + } + } +} + +extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint64_t registryID,void (*missing)(const char *,const char *)) { + if(gDeviceObject)return gDeviceObject; + gMissing=missing;gInFlight=[NSMutableSet set]; + gDevice=nmb_device_open(loader,cache); + if(!gDevice){NBLOG(@"native-metal: device refused: %s",nmb_last_error());return nil;} + char name[128]={};nmb_device_name(gDevice,name,sizeof name);gName=[NSString stringWithUTF8String:name];gRegistryID=registryID; + Class base=objc_getClass("_MTLDevice"); +#if !TARGET_OS_IPHONE + // Host instrument: on NSObject every selector a client sends that this file lacks reaches the missing hook, instead of the host's + // own _MTLDevice implementation. + if(getenv("NMB_DEVICE_BASE_NSOBJECT"))base=[NSObject class]; +#endif +#if TARGET_OS_IPHONE + // The target's base class ABI (iPadOS 17.7.11): the dynamic subclass adds no instance variables. + if(!base || class_getInstanceSize(base)!=536){NBLOG(@"native-metal: unexpected _MTLDevice ABI");return nil;} +#endif + if(!base)base=[NSObject class]; + Class cls=objc_allocateClassPair(base,"NativeBridgeMetalDevice",0);if(!cls)return nil; + unsigned count=0;Method *methods=class_copyMethodList([NBDeviceMethods class],&count); + for(unsigned i=0;i lock(gProblemLock); + return (size_t)snprintf(out,size,"c%llu/%llu d%llu p%llu x%llu R%llu D%llu F%llu M%llu C%llu B%llu %s | %s",(unsigned long long)gCommits.load(),(unsigned long long)gCommitFailures.load(),(unsigned long long)gCompleted.load(), + (unsigned long long)gPipelines.load(),(unsigned long long)gProblems.load(),(unsigned long long)gPasses.load(),(unsigned long long)gDraws.load(),(unsigned long long)gFullScreenPasses.load(),(unsigned long long)(gBiggestPass.load()>>10),(unsigned long long)gDispatches.load(),(unsigned long long)gBlits.load(),gLastProblems[0],gLastProblems[1]); +} + +static size_t native_bridge_metal_formats_locked(char *out,size_t size) { + int n=snprintf(out,size,"tex x%llu f",(unsigned long long)gTextureRefusals.load()); + for(unsigned i=0;i0 && (size_t)n0?(size_t)n:0; +} +// The benchmark's rows (native_bridge_metal_benchmark), kept for the telemetry table. +static std::mutex gBenchLock; +static char gBench[4][96]; +static std::atomic gBenchRuns; +extern "C" size_t native_bridge_metal_benchmark(id device,const char *metallib,unsigned pairs,char *out,size_t size) { + auto now=[]{return clock_gettime_nsec_np(CLOCK_UPTIME_RAW);}; + // the first run fills rows 0-1 (Y, Y1), a later one rows 2-3 (Z, Z1), so a busy and an idle run are both kept + const unsigned base=gBenchRuns++?2u:0u; + auto put=[&](unsigned row,const char *format,auto... args){std::lock_guard lock(gBenchLock);snprintf(gBench[base+row],sizeof gBench[base+row],format,args...); + if(base)gBench[base+row][0]='Z';}; + // compute: a dependent integer chain (no memory), 100M steps + uint64_t t=now(),x=0x9e3779b97f4a7c15ull; + for(uint32_t i=0;i<100000000u;i++){x^=x<<13;x^=x>>7;x^=x<<17;} + const uint64_t computeUs=(now()-t)/1000; + // first touch: 32 MiB of fresh anonymous memory (the tablet's free memory is small), one store per 4 KiB page (the kernel's fault and zero-fill), then four memsets of it + const size_t bytes=32u<<20; + uint64_t faultUs=0,fillUs=0,shaUs=0; + void *m=mmap(nullptr,bytes,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(m!=MAP_FAILED) { + volatile uint8_t *b=(volatile uint8_t *)m; + t=now();for(size_t at=0;at dev=(id)device;NSError *error=nil; + t=now(); + id qc=metallib?[dev newLibraryWithURL:[NSURL fileURLWithPath:@(metallib)] error:&error]:nil; + libraryUs=(now()-t)/1000; + if(qc) { + NSArray *names=qc.functionNames; + id queue=[dev newCommandQueue]; + MTLTextureDescriptor *td=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:64 height:64 mipmapped:NO]; + td.usage=MTLTextureUsageRenderTarget; + id target=[dev newTextureWithDescriptor:td]; + id zeros=[dev newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + unsigned n=0; + for(NSString *name in names) { + if(n>=pairs)break; + id f=[qc newFunctionWithName:name]; + if(f.functionType!=MTLFunctionTypeVertex)continue; + NSString *fragmentName=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if([fragmentName isEqualToString:name] || ![names containsObject:fragmentName])continue; + ++n; + MTLRenderPipelineDescriptor *d=[MTLRenderPipelineDescriptor new]; + d.vertexFunction=f;d.fragmentFunction=[qc newFunctionWithName:fragmentName]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm; + t=now(); + id state=[dev newRenderPipelineStateWithDescriptor:d error:&error]; + const uint64_t took=(now()-t)/1000; + if(!state){++refused;continue;} + ++made;createUs+=took;createMax=std::max(createMax,took); + t=now(); + MTLRenderPassDescriptor *pass=[MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture=target;pass.colorAttachments[0].loadAction=MTLLoadActionClear;pass.colorAttachments[0].storeAction=MTLStoreActionStore; + id commands=[queue commandBuffer]; + id encoder=[commands renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:state]; + for(NSUInteger i=0;i<8;++i){[encoder setVertexBuffer:zeros offset:0 atIndex:i];[encoder setFragmentBuffer:zeros offset:0 atIndex:i];} + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; + [encoder endEncoding];[commands commit];[commands waitUntilCompleted]; + const uint64_t drew=(now()-t)/1000; + drawUs+=drew;drawMax=std::max(drawMax,drew); + } + } + } + put(1,"Y1 qc lib=%llums made=%llu refused=%llu create=%llums max=%llums draw=%llums max=%llums",(unsigned long long)(libraryUs/1000), + (unsigned long long)made,(unsigned long long)refused,(unsigned long long)(createUs/1000),(unsigned long long)(createMax/1000), + (unsigned long long)(drawUs/1000),(unsigned long long)(drawMax/1000)); + size_t at=0; + if(out && size) { + std::lock_guard lock(gBenchLock); + out[0]=0;for(unsigned r=base;rat?size-at:0,"%s\n",gBench[r]); + } + return at; +} +// The telemetry table the plugin streams to the kernel's log: newline-separated rows of at most 90 characters. +extern "C" size_t native_bridge_metal_telemetry(char *out,size_t size) { + size_t at=0;out[0]=0; + auto row=[&](const char *format,auto... args){ + char text[200];int n=snprintf(text,sizeof text,format,args...);if(n<=0)return;if(n>90)n=90; + if(at+(size_t)n+2>=size)return;memcpy(out+at,text,(size_t)n);at+=(size_t)n;out[at++]='\n';out[at]=0;}; + auto ms=[](uint64_t us){return (unsigned long long)(us/1000);}; + {std::lock_guard lock(gBenchLock);for(unsigned r=0;r<4;r++)if(gBench[r][0])row("%.90s",gBench[r]);} + row("S c%llu/%llu d%llu p%llu x%llu R%llu D%llu F%llu M%lluk C%llu B%llu",(unsigned long long)gCommits.load(),(unsigned long long)gCommitFailures.load(), + (unsigned long long)gCompleted.load(),(unsigned long long)gPipelines.load(),(unsigned long long)gProblems.load(),(unsigned long long)gPasses.load(), + (unsigned long long)gDraws.load(),(unsigned long long)gFullScreenPasses.load(),(unsigned long long)(gBiggestPass.load()>>10), + (unsigned long long)gDispatches.load(),(unsigned long long)gBlits.load()); + row("T pipeline n=%llu total=%llums max=%llums",(unsigned long long)gTimePipeline.count.load(),ms(gTimePipeline.total.load()),ms(gTimePipeline.slowest.load())); + row("T library n=%llu total=%llums max=%llums",(unsigned long long)gTimeLibrary.count.load(),ms(gTimeLibrary.total.load()),ms(gTimeLibrary.slowest.load())); + row("T function n=%llu total=%llums max=%llums", + (unsigned long long)gTimeFunction.count.load(),ms(gTimeFunction.total.load()),ms(gTimeFunction.slowest.load())); + row("G commit-to-complete n=%llu total=%llums max=%llums mean=%lluus",(unsigned long long)gTimeEncodeToCommit.count.load(),ms(gTimeEncodeToCommit.total.load()), + ms(gTimeEncodeToCommit.slowest.load()),(unsigned long long)(gTimeEncodeToCommit.count.load()?gTimeEncodeToCommit.total.load()/gTimeEncodeToCommit.count.load():0)); + row("J scheduled wait n=%llu total=%llums max=%llums",(unsigned long long)gTimeWaitScheduled.count.load(),ms(gTimeWaitScheduled.total.load()),ms(gTimeWaitScheduled.slowest.load())); + row("J completed wait n=%llu total=%llums max=%llums",(unsigned long long)gTimeWaitCompleted.count.load(),ms(gTimeWaitCompleted.total.load()),ms(gTimeWaitCompleted.slowest.load())); + row("J resource wait n=%llu total=%llums max=%llums",(unsigned long long)gTimeResourceWait.count.load(),ms(gTimeResourceWait.total.load()),ms(gTimeResourceWait.slowest.load())); + row("J resource checks=%llu skipped=%llu",(unsigned long long)gResourceChecks.load(),(unsigned long long)gResourceSkipped.load()); + row("J surface cache n=%llu bytes=%llu dup=%llu us=%llu max=%llu",(unsigned long long)gTimeSurfaceFlush.count.load(), + (unsigned long long)gSurfaceFlushBytes.load(),(unsigned long long)gSurfaceFlushDuplicates.load(), + (unsigned long long)gTimeSurfaceFlush.total.load(),(unsigned long long)gTimeSurfaceFlush.slowest.load()); + // The runtime's VkPipeline makings (the creation log, METAL2VULKAN_LOG_PIPELINES; they happen at draws, not at the state's creation) and + // its shader cache: where the driver's compile time goes. + uint64_t work[10]={}; + if(gDevice && nmb_device_work_stats(gDevice,work)) { + row("V vkpipe n=%llu total=%llums max=%llums waited=%llums failed=%llu", + (unsigned long long)work[0],ms(work[1]),ms(work[2]),ms(work[3]),(unsigned long long)work[4]); + row("V cache hit%llu miss%llu wr%llu | msg i%llu w%llu",(unsigned long long)work[7],(unsigned long long)work[8],(unsigned long long)work[9],(unsigned long long)work[5], + (unsigned long long)work[6]); + char warning[200];size_t n=nmb_last_warning(warning,sizeof warning); + if(n){row("W %.86s",warning);if(n>86)row("Wb %.86s",warning+86);} + } + { + std::lock_guard lock(gProblemLock); + char formats[160];native_bridge_metal_formats_locked(formats,sizeof formats);row("F %.86s",formats); + row("P0 %.86s",gLastProblems[0]);row("P0b %.86s",strlen(gLastProblems[0])>86?gLastProblems[0]+86:""); + row("P1 %.86s",gLastProblems[1]);row("P1b %.86s",strlen(gLastProblems[1])>86?gLastProblems[1]+86:""); + } + { + std::lock_guard lock(gCommitLock); + const uint32_t newest=(uint32_t)gCommits.load(); + row("%s","Q timing v2: per-buffer API counts; ctc=commit-to-callback; age includes caller idle"); + for(uint32_t k=0;k<20 && k>10),r.work.dispatches,r.work.blits,(unsigned long long)r.callbackUs); + if(k<4)row("E%u age=%lluus scheduled=%lldus",r.index,(unsigned long long)r.ageUs,(long long)r.scheduledUs); + } + } + return at; +} + +extern "C" size_t native_bridge_metal_formats(char *out,size_t size) { + std::lock_guard lock(gProblemLock); + return native_bridge_metal_formats_locked(out,size); +} diff --git a/NativeIOGPUProviderProbe.cpp b/NativeIOGPUProviderProbe.cpp new file mode 100644 index 0000000..debc868 --- /dev/null +++ b/NativeIOGPUProviderProbe.cpp @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Unpublished original-provider ABI instrument. Required methods never return +// fabricated success. An explicit missing-resource control returns NULL from +// descriptor slot170; all other required calls remain non-returning captures. +// Original C2/D2 and every inherited operational address +// remain intact; registration is intercepted only on this own probe class. +#include "OriginalGPUProviderABI.h" +extern "C" __attribute__((noreturn)) void native_original_provider_required(unsigned,const unsigned long long *); +extern "C" IOMemoryDescriptor *native_original_provider_descriptor(const unsigned long long *); +class NativeIOGPUProviderProbe : public IOGPU { + OSDeclareDefaultStructors(NativeIOGPUProviderProbe) +public: + __attribute__((noreturn)) void registerService(IOOptionBits = 0) override; + IOMemoryDescriptor *abiSlot170(UInt32 *) override; + __attribute__((noreturn)) void abiSlot174() override; + __attribute__((noreturn)) void abiSlot189() override; + __attribute__((noreturn)) void abiSlot192(void *) override; + __attribute__((noreturn)) bool abiSlot195(IOService *) override; + __attribute__((noreturn)) void abiSlot196() override; + __attribute__((noreturn)) void abiSlot197() override; + __attribute__((noreturn)) void abiSlot200() override; + __attribute__((noreturn)) void abiSlot202() override; + __attribute__((noreturn)) void abiSlot205() override; +}; +OSDefineMetaClassAndStructors(NativeIOGPUProviderProbe,IOGPU) +// Save actual GPR arguments, caller LR, live CPU translation registers and SP. +// No FP instruction is executed in an unqualified EL1 FP context. These +// non-returning hooks do not reinterpret or mutate unknown pointer arguments. +#define SAVE_REQUIRED_FRAME \ + "sub sp, sp, #128\n" \ + "stp x0, x1, [sp, #0]\n" "stp x2, x3, [sp, #16]\n" \ + "stp x4, x5, [sp, #32]\n" "stp x6, x7, [sp, #48]\n" \ + "stp x8, x30, [sp, #64]\n" \ + "mrs x9, tcr_el1\n" "mrs x10, sctlr_el1\n" "stp x9, x10, [sp, #80]\n" \ + "mrs x9, ttbr0_el1\n" "mrs x10, ttbr1_el1\n" "stp x9, x10, [sp, #96]\n" \ + "add x9, sp, #128\n" "mrs x10, daif\n" "stp x9, x10, [sp, #112]\n" +#define CAPTURE_REQUIRED(number) __asm__ volatile(SAVE_REQUIRED_FRAME \ + "mov w0, #" #number "\n" "mov x1, sp\n" \ + "bl _native_original_provider_required\n" "brk #1\n") +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::registerService(IOOptionBits) {CAPTURE_REQUIRED(84);} +__attribute__((naked)) IOMemoryDescriptor *NativeIOGPUProviderProbe::abiSlot170(UInt32 *) { + __asm__ volatile(SAVE_REQUIRED_FRAME + "mov x0, sp\n" "bl _native_original_provider_descriptor\n" + "ldr x30, [sp, #72]\n" "add sp, sp, #128\n" "ret\n"); +} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot174() {CAPTURE_REQUIRED(174);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot189() {CAPTURE_REQUIRED(189);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot192(void *) {CAPTURE_REQUIRED(192);} +__attribute__((naked,noreturn)) bool NativeIOGPUProviderProbe::abiSlot195(IOService *) {CAPTURE_REQUIRED(195);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot196() {CAPTURE_REQUIRED(196);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot197() {CAPTURE_REQUIRED(197);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot200() {CAPTURE_REQUIRED(200);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot202() {CAPTURE_REQUIRED(202);} +__attribute__((naked,noreturn)) void NativeIOGPUProviderProbe::abiSlot205() {CAPTURE_REQUIRED(205);} +extern "C" IOService *native_original_provider_allocate(void) {return new NativeIOGPUProviderProbe;} +extern "C" unsigned native_original_provider_instances(void) {return NativeIOGPUProviderProbe::gMetaClass.getInstanceCount();} diff --git a/NativeMetalBridgePhase.h b/NativeMetalBridgePhase.h new file mode 100644 index 0000000..86c9439 --- /dev/null +++ b/NativeMetalBridgePhase.h @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: MIT +// Plugin-only phase after Turnip: the Metal bridge self-test (bridge/NativeMetalBridge.rs) on the same Turnip, on an 8 MiB worker under a +// watchdog. Markers 0x6501..0x650f come from the bridge itself; the result line goes to GPU7 ("M ..."), a stuck worker is reported there +// as "MW ". +// After a passing self-test the worker runs the frontend checks (bridge/FrontendChecks.h) through the public Metal API on the native frontend +// (NativeBridgeMetalFrontend.mm) over the same Turnip: markers 0x6600 (device), 0x6601..0x660f, then 0x66f0 pass / 0x66f1 fail. Its verdict +// ends the same GPU7 line: "F0" (with the first unimplemented selector, if any), "F ", "Fo"/"Fx" + reason +// when the device or the fixtures were refused. +#pragma once +#include "bridge/NativeMetalBridge.h" +#include "bridge/NativeMetalObjects.h" +#include "bridge/FrontendChecks.h" +#include "NativeBridgeMetalFrontend.h" +#include "GpuScanoutClient.h" +#include +#include +#include +#include +#include + +namespace bridgephase { +struct Job { + const char *turnip; + char cache[512]; + NmbSelftest result; + uint32_t code; + frontendchecks::Report frontend; + bool frontendRan, frontendOpened; + char missing[64], error[48]; + volatile uint32_t last; + volatile bool done; +}; +static Gpu *gGpu; +static Job *gJob; +static id gDevice; // the frontend device of the checks, kept for the scanout stage +static void mark(uint32_t value) { + if (gJob) gJob->last = value; + if (gGpu) gGpu->mark(value); +} +static void missingSelector(const char *cls, const char *selector) { + if (gJob && !gJob->missing[0]) snprintf(gJob->missing, sizeof gJob->missing, "FM %.20s %.36s", cls, selector); +} +// The frontend checks on the native frontend over Turnip (QuartzCore is already translated by the self-test, so it is not repeated). +static void frontend(Job *job) { + mark(0x6600); + id device = native_bridge_metal_open(job->turnip, job->cache, 0, missingSelector); + // The runtime's reasons are per thread: keep this worker's for the report. + if (!device) {strlcpy(job->error, nmb_last_error(), sizeof job->error); return;} + job->frontendOpened = true; + gDevice = device; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + air = nmb_function_constant_air(&bytes); + id fcLibrary = native_bridge_metal_library_with_air(air, bytes); + if (!library || !fcLibrary) {strlcpy(job->error, nmb_last_error(), sizeof job->error); return;} + job->frontend.mark = mark; + job->frontendRan = true; + // An exception escaping a worker thread aborts the whole host process (SpringBoard died that way, int-075/083): report it as a failed check. + @try { + frontendchecks::run(device, library, fcLibrary, nil, &job->frontend); + } @catch (NSException *exception) { + ++job->frontend.failed; + snprintf(job->frontend.first, sizeof job->frontend.first, "exception %.40s", exception.name.UTF8String ?: "?"); + } +} +static void *worker(void *arg) { + Job *job = static_cast(arg); + // The target's own compositor library, read in place (never copied anywhere). + const uint8_t *qc = nullptr; + size_t qcBytes = 0; + const int fd = open("/System/Library/Frameworks/QuartzCore.framework/default.metallib", O_RDONLY); + struct stat st = {}; + if (fd >= 0 && fstat(fd, &st) == 0 && st.st_size > 0) { + void *m = mmap(nullptr, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (m != MAP_FAILED) {qc = static_cast(m); qcBytes = (size_t)st.st_size;} + } + if (fd >= 0) close(fd); + job->code = nmb_selftest(job->turnip, job->cache, qc, qcBytes, mark, &job->result); + if (qc) munmap(const_cast(qc), qcBytes); + if (!job->code) frontend(job); + __atomic_store_n(&job->done, true, __ATOMIC_RELEASE); + return nullptr; +} +// Serves the display driver's frames with the GPU on `device` (the kernel offers them only with sheng-fb-gpu=1). The calls go through a hand-off-only +// connection to the accelerator (open type 0x4148: no GPU slot is taken): the original sandbox profiles admit that class and not the presenter's own +// service. Reports on GPU7 only when the state changes: "P e" (0 no connection, 1 no selector, 2 mode refused, 3 scanout not +// mappable/importable, 4 serving), then "PF / r v/ t" at the first frame, every fifth after +// and the first failure, "PX ..." when serving stops. Returns when serving stops. +struct ScanoutContext { + Gpu link; + Gpu *marker; +}; +static void serveScanout(Gpu &gpu, id device) { + ScanoutContext context; + context.marker = &gpu; + Gpu &link = context.link; + link.call = gpu.call; link.map = gpu.map; link.unmap = gpu.unmap; link.openFn = gpu.openFn; link.closeFn = gpu.closeFn; link.port = gpu.port; + gpuscanout::Stats st = {}; + char line[64]; + gpu.mark(0x6700); + const int openResult = link.openFn ? link.openFn(link.port, mach_task_self(), 0x4148, &link.conn) : -1; + if (openResult != 0 || !link.conn) { + snprintf(line, sizeof line, "P0 e%x", (unsigned)openResult); + } else if (gpu.mark(0x6701), !link.probe()) { + snprintf(line, sizeof line, "P1"); + } else { + gpu.mark(0x6702); + gpuscanout::Transport transport = {&context, + [](void *c, uint32_t method, const void *in, size_t is, void *out, size_t os) -> int { + return static_cast(c)->link.method(0x7000 + method, in, is, out, os); + }, + [](void *c, uint64_t *address, uint64_t *length) -> int { + Gpu &g = static_cast(c)->link; + return g.map(g.conn, 0x5343, mach_task_self(), address, length, 0x101); // anywhere + inhibit cache + }, + [](void *c, uint32_t value) {static_cast(c)->marker->mark(value);}}; + gpuscanout::Client client; + const bool opened = client.open((id)device, transport, &st); + snprintf(line, sizeof line, "P%d e%x", st.step, (unsigned)st.error); + streamText7(gpu, line); + if (opened) { + uint64_t reportedFrames = 0, reportedFailures = 0; + while (client.serveOne(500)) { + const bool frame = st.frames && (st.frames == 1 || st.frames - reportedFrames >= 5); + if (frame || (st.failures && !reportedFailures)) { + reportedFrames = st.frames ? st.frames : reportedFrames; + reportedFailures = st.failures ? st.failures : reportedFailures; + const uint64_t served = st.frames + st.failures; + snprintf(line, sizeof line, "PF %llu/%llu r%d v%llu/%llu t%u a%llu", (unsigned long long)st.frames, (unsigned long long)st.failures, + st.lastResult, (unsigned long long)st.verified, (unsigned long long)st.mismatchRows, st.lastMs, + (unsigned long long)(served ? st.totalMs / served : 0)); + streamText7(gpu, line); + } + } + snprintf(line, sizeof line, "PX %llu/%llu r%d e%x", (unsigned long long)st.frames, (unsigned long long)st.failures, st.lastResult, + (unsigned)st.error); + } + } + streamText7(gpu, line); +} +static void run(Gpu &gpu, const char *turnipPath) { + gpu.mark(0x6500); + Job *job = new Job{}; + job->turnip = turnipPath; + // A per-user cache directory of the host process (its sandbox allows it); the translated-shader cache goes in a subdirectory. + if (!confstr(_CS_DARWIN_USER_CACHE_DIR, job->cache, sizeof job->cache - 16)) strlcpy(job->cache, "/tmp/", sizeof job->cache); + strlcat(job->cache, "m2v", sizeof job->cache); + mkdir(job->cache, 0700); + gGpu = &gpu; + gJob = job; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8u << 20); + pthread_t thread; + const bool started = pthread_create(&thread, &attr, worker, job) == 0; + pthread_attr_destroy(&attr); + if (!started) {gpu.mark(0x65fd); return;} + static const unsigned checkpoints[] = {30, 90, 180}; + unsigned next = 0; + for (unsigned tenths = 0; !__atomic_load_n(&job->done, __ATOMIC_ACQUIRE); ++tenths) { + usleep(100000); + if (next < 3 && tenths == checkpoints[next] * 10) { + char text[64]; + snprintf(text, sizeof text, "MW %x %us", job->last, checkpoints[next]); + streamText7(gpu, text); + ++next; + } + } + pthread_join(thread, nullptr); + const NmbSelftest &r = job->result; + // One GPU7 line (the page keeps only the latest): the self-test's counts, then after a passing self-test the frontend's verdict. + char text[64]; + const int at = snprintf(text, sizeof text, "M s%x c%u r%u i%u q%u k%u/%u p%u/%u", r.stage, r.compute_bad, r.render_bad, r.import_bad, + r.qc_functions, r.qc_kernels_ok, r.qc_kernels_failed, r.qc_pairs_ok, r.qc_pairs_failed); + char *rest = text + (at > 0 && at < (int)sizeof text ? at : 0); + const size_t room = sizeof text - (size_t)(rest - text); + if (job->code) snprintf(rest, room, " %s", r.message); + else if (!job->frontendOpened) snprintf(rest, room, " Fo %s", job->error); + else if (!job->frontendRan) snprintf(rest, room, " Fx %s", job->error); + else if (job->frontend.failed) snprintf(rest, room, " F%u %s", job->frontend.failed, job->frontend.first); + else snprintf(rest, room, " F0%s%s", job->missing[0] ? " " : "", job->missing[0] ? job->missing + 3 : ""); + streamText7(gpu, text); + const bool ok = !job->code && !r.compute_bad && !r.render_bad && !r.import_bad; + gpu.mark(ok ? 0x65f0 : 0x65f1); + if (!job->code) gpu.mark(job->frontendRan && !job->frontend.failed ? 0x66f0 : 0x66f1); + const bool scanout = ok && job->frontendRan && !job->frontend.failed && gDevice; + gJob = nullptr; + delete job; + // After the checks' verdict has been on the page for a while: serve the display driver's frames with the GPU. + if (scanout) { + sleep(8); + serveScanout(gpu, gDevice); + } + gGpu = nullptr; +} +} // namespace bridgephase diff --git a/NativeOfflineAccelerator.cpp b/NativeOfflineAccelerator.cpp new file mode 100644 index 0000000..11f1b9d --- /dev/null +++ b/NativeOfflineAccelerator.cpp @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Unpublished original-base lifecycle qualification. No GPU service or device +// is attached/started/published, and no original client contract is claimed. +#include "OriginalAcceleratorABI.h" +class NativeOfflineAccelerator : public IOAcceleratorES { + OSDeclareDefaultStructors(NativeOfflineAccelerator) +}; +OSDefineMetaClassAndStructors(NativeOfflineAccelerator,IOAcceleratorES) +extern "C" IOService *native_offline_accelerator_allocate(void) {return new NativeOfflineAccelerator;} +extern "C" unsigned native_offline_accelerator_instance_size(void) {return sizeof(NativeOfflineAccelerator);} +extern "C" unsigned native_offline_accelerator_instances(void) {return NativeOfflineAccelerator::gMetaClass.getInstanceCount();} diff --git a/NativeOfflineGPU.cpp b/NativeOfflineGPU.cpp new file mode 100644 index 0000000..ae33bae --- /dev/null +++ b/NativeOfflineGPU.cpp @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: MIT +// Explicit offline GPU broker. No normal Metal/IOAccelerator publication. +// The kernel, not request bytes, selects each userclient's host namespace. +#include +#include +#include +#include +#include "NativeVirtioConsoleTransport.h" +#include "OfflineMetalSessionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalSharedEventWire.h" +#include +extern "C" { +volatile uint32_t native_gpu_broker_stage,native_gpu_broker_clients,native_gpu_broker_active,native_gpu_broker_closed; +volatile uint32_t native_gpu_broker_requests,native_gpu_broker_failures,native_gpu_broker_refusals; +volatile int native_gpu_broker_status=-1; +volatile uint64_t native_gpu_broker_reply_bytes; +volatile uint32_t native_gpu_broker_cpu_context_count; +volatile uint32_t native_gpu_shared_exports,native_gpu_shared_imports,native_gpu_shared_drops,native_gpu_shared_refusals,native_gpu_shared_tokens_live,native_gpu_shared_ports; +volatile uint64_t native_gpu_broker_cpu_contexts[8][8]; +} +class NativeOfflineGPUClient; +class NativeGPUSharedEventToken; +class NativeOfflineGPU : public IOService { + OSDeclareDefaultStructors(NativeOfflineGPU) + friend class NativeOfflineGPUClient; + friend class NativeGPUSharedEventToken; + IOService *transport=nullptr; + IOLock *gate=nullptr; + bool online=false; + uint32_t nextControlRequest=1; + IOReturn readExact(void *bytes,size_t size) { + auto p=static_cast(bytes); + while(size) { + size_t count=size>4096?4096:size; + IOReturn status=native_console_receive(transport,this,p,&count); + if(status!=kIOReturnSuccess)return status; + if(!count || count>size)return kIOReturnIOError; + p+=count;size-=count; + } + return kIOReturnSuccess; + } + // gate covers the complete actual request/reply, including every byte + // beyond a VirtIO descriptor. Userclients retain separate reply buffers. + IOReturn exchange(uint64_t session,const OMWireHeader &request,const void *input, + unsigned char **reply,size_t *length) { + *reply=nullptr;*length=0;if(!online)return kIOReturnNotReady; + OMSessionWireHeader header={request,session};header.wire.version=OFFLINE_METAL_SESSION_VERSION; + IOReturn status=native_console_send(transport,this,&header,sizeof(header)); + if(status==kIOReturnSuccess && request.bytes)status=native_console_send(transport,this,input,request.bytes); + OMSessionWireHeader response={};OMResult result={}; + if(status==kIOReturnSuccess)status=readExact(&response,sizeof(response)); + if(status==kIOReturnSuccess && (response.wire.magic!=OM_WIRE_MAGIC || response.wire.version!=OFFLINE_METAL_SESSION_VERSION || + response.wire.operation!=(request.operation|OM_WIRE_RESPONSE) || response.wire.request!=request.request || + response.session!=session || response.wire.bytesOFFLINE_METAL_MAX_REPLY))status=kIOReturnIOError; + if(status==kIOReturnSuccess)status=readExact(&result,sizeof(result)); + if(status==kIOReturnSuccess && result.reserved)status=kIOReturnIOError; + unsigned char *buffer=nullptr;size_t size=sizeof(OMWireHeader)+response.wire.bytes; + if(status==kIOReturnSuccess) { + buffer=static_cast(IOMallocData(size)); + if(!buffer)status=kIOReturnNoMemory; + } + if(status==kIOReturnSuccess) { + response.wire.version=OFFLINE_METAL_VERSION; + memcpy(buffer,&response.wire,sizeof(OMWireHeader));memcpy(buffer+sizeof(OMWireHeader),&result,sizeof(result)); + status=readExact(buffer+sizeof(OMWireHeader)+sizeof(result),response.wire.bytes-sizeof(result)); + } + native_gpu_broker_status=status; + if(status!=kIOReturnSuccess) { + if(buffer)IOFreeData(buffer,size); + // A partial stream cannot be reused for another client's reply. + online=false;native_gpu_broker_failures=native_gpu_broker_failures+1;return status; + } + native_gpu_broker_requests=native_gpu_broker_requests+1; + native_gpu_broker_reply_bytes=native_gpu_broker_reply_bytes+size; + if(result.error)native_gpu_broker_refusals=native_gpu_broker_refusals+1; + *reply=buffer;*length=size;return kIOReturnSuccess; + } + IOReturn createSession(task_t owner,uint64_t *session) { + *session=0;IOLockLock(gate); + uint64_t tcr=0,ttbr0=0,ttbr1=0,sctlr=0; + __asm__ volatile("mrs %0, tcr_el1" : "=r"(tcr)); + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(ttbr0)); + __asm__ volatile("mrs %0, ttbr1_el1" : "=r"(ttbr1)); + __asm__ volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + bool native4k=((tcr>>14)&3)==0 && ((tcr>>30)&3)==2 && (sctlr&1) && ttbr0 && ttbr1; + unsigned index=native_gpu_broker_cpu_context_count; + native_gpu_broker_cpu_context_count=index+1; + if(index<8) { + uint64_t values[8]={(uint64_t)proc_selfpid(),0,tcr,ttbr0,ttbr1,sctlr,(uintptr_t)owner,(uint64_t)native4k}; + for(unsigned n=0;n<8;n++)native_gpu_broker_cpu_contexts[index][n]=values[n]; + } + if(!native4k) { + native_gpu_broker_status=kIOReturnUnsupported;native_gpu_broker_failures=native_gpu_broker_failures+1; + IOLockUnlock(gate);return kIOReturnUnsupported; + } + OMWireHeader request={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SESSION_OPEN,0,1}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(0,request,nullptr,&reply,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result)); + if(result.error || !result.handle || length!=sizeof(OMWireHeader)+sizeof(result))status=kIOReturnNotOpen; + else { + *session=result.handle;native_gpu_broker_clients=native_gpu_broker_clients+1;native_gpu_broker_active=native_gpu_broker_active+1; + if(index<8)native_gpu_broker_cpu_contexts[index][1]=result.handle; + } + } + if(reply)IOFreeData(reply,length);IOLockUnlock(gate);return status; + } + void closeSession(uint64_t session,uint32_t number) { + IOLockLock(gate);OMWireHeader request={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SESSION_CLOSE,0,number}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(session,request,nullptr,&reply,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result)); + if(!result.error){native_gpu_broker_closed=native_gpu_broker_closed+1;native_gpu_broker_active=native_gpu_broker_active-1;} + } + if(reply)IOFreeData(reply,length);IOLockUnlock(gate); + } + void dropSharedExport(uint64_t exported) { + if(!exported)return;IOLockLock(gate); + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SHARED_EXPORT_DROP,sizeof(OMHandle),nextControlRequest};OMHandle input={exported}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(0,header,&input,&reply,&length); + if(status==kIOReturnSuccess){nextControlRequest++;OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result));if(!result.error)__atomic_fetch_add(&native_gpu_shared_drops,1,__ATOMIC_RELAXED);} + if(reply)IOFreeData(reply,length);IOLockUnlock(gate); + } +public: + bool start(IOService *provider) override { + if(!IOService::start(provider))return false; + gate=IOLockAlloc();if(!gate){IOService::stop(provider);return false;} + if(!provider->open(this)){IOLockFree(gate);gate=nullptr;IOService::stop(provider);return false;} + transport=provider;online=true;native_gpu_broker_stage=2;return true; + } + void stop(IOService *provider) override { + if(gate){IOLockLock(gate);online=false;if(transport)transport->close(this);transport=nullptr;IOLockUnlock(gate);} + IOService::stop(provider); + } + void free() override { + if(transport){transport->close(this);transport=nullptr;} + if(gate){IOLockFree(gate);gate=nullptr;}IOService::free(); + } + IOReturn newUserClient(task_t,void *,UInt32,OSDictionary *,IOUserClient **) override; +}; +// A real kernel OSObject carried exclusively by an IKOT_IOKIT_IDENT Mach +// right. No user-supplied integer can stand in for this object's authority. +class NativeGPUSharedEventToken : public OSObject { + OSDeclareDefaultStructors(NativeGPUSharedEventToken) + friend class NativeOfflineGPUClient; + NativeOfflineGPU *controller=nullptr; + uint64_t exported=0; + bool initWithExport(NativeOfflineGPU *owner,uint64_t identifier) { + if(!owner || !identifier || !OSObject::init())return false; + controller=owner;controller->retain();exported=identifier; + __atomic_fetch_add(&native_gpu_shared_tokens_live,1,__ATOMIC_RELAXED);return true; + } + void free() override { + if(controller){uint64_t identifier=exported;exported=0;controller->dropSharedExport(identifier);controller->release();controller=nullptr;__atomic_fetch_sub(&native_gpu_shared_tokens_live,1,__ATOMIC_RELAXED);} + OSObject::free(); + } +}; +class NativeOfflineGPUClient : public IOUserClient { + OSDeclareDefaultStructors(NativeOfflineGPUClient) + NativeOfflineGPU *controller=nullptr; + task_t owner=nullptr; // The native IOUserClient base owns this task reference. + IOLock *gate=nullptr; + uint64_t session=0; + uint32_t nextRequest=1,nextHostRequest=1; + unsigned char *reply=nullptr;size_t replyLength=0,replyOffset=0; + void clearReply() {if(reply)IOFreeData(reply,replyLength);reply=nullptr;replyLength=replyOffset=0;} + void closeContext() { + if(gate)IOLockLock(gate); + clearReply();if(session && controller)controller->closeSession(session,nextHostRequest); + session=0;controller=nullptr;if(gate)IOLockUnlock(gate); + } + static IOReturn sendAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(args->structureInputDescriptor)return kIOReturnUnsupported; + if(!args->structureInput || args->structureInputSizestructureInputSize>4096)return kIOReturnBadArgument; + OMWireHeader header={};memcpy(&header,args->structureInput,sizeof(header)); + IOLockLock(self->gate);IOReturn status=kIOReturnNotOpen; + if(self->session && self->controller) { + if(self->replyOffset!=self->replyLength)status=kIOReturnBusy; + else if(header.magic!=OM_WIRE_MAGIC || header.version!=OFFLINE_METAL_VERSION || header.request!=self->nextRequest || + header.bytes+sizeof(header)!=args->structureInputSize || header.bytes>4080 || + header.operationOM_SHARED_LABEL_SET || (header.operation==OM_FINISH && header.bytes))status=kIOReturnBadArgument; + else { + self->clearReply();uint16_t operation=header.operation;if(operation==OM_FINISH)header.operation=OM_SESSION_CLOSE; + IOLockLock(self->controller->gate); + OMWireHeader hostHeader=header;hostHeader.request=self->nextHostRequest; + status=self->controller->exchange(self->session,hostHeader,static_cast(args->structureInput)+sizeof(header),&self->reply,&self->replyLength); + if(status==kIOReturnSuccess) { + self->nextRequest++;self->nextHostRequest++; + // Preserve the real namespace-close result in the client's + // own framing; no host-wide Finish is forwarded. + header.operation=operation|OM_WIRE_RESPONSE;header.bytes=(uint32_t)(self->replyLength-sizeof(header)); + memcpy(self->reply,&header,sizeof(header));OMResult result={};memcpy(&result,self->reply+sizeof(header),sizeof(result)); + if(operation==OM_FINISH && !result.error) { + self->session=0;native_gpu_broker_closed=native_gpu_broker_closed+1;native_gpu_broker_active=native_gpu_broker_active-1; + } + } + IOLockUnlock(self->controller->gate); + } + } + IOLockUnlock(self->gate);return status; + } + static IOReturn receiveAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(args->structureOutputDescriptor)return kIOReturnUnsupported; + if(!args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + IOLockLock(self->gate);size_t available=self->replyLength-self->replyOffset; + size_t count=args->structureOutputSize;if(count>available)count=available; + if(count){memcpy(args->structureOutput,self->reply+self->replyOffset,count);self->replyOffset+=count;} + args->structureOutputSize=(uint32_t)count;IOLockUnlock(self->gate);return count?kIOReturnSuccess:kIOReturnNotReady; + } + IOReturn sharedExchange(uint16_t operation,uint64_t identifier,unsigned char **output,size_t *length) { + *output=nullptr;*length=0; + if(!session || !controller)return kIOReturnNotOpen; + if(replyOffset!=replyLength)return kIOReturnBusy; + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,operation,sizeof(OMHandle),nextHostRequest};OMHandle input={identifier}; + IOLockLock(controller->gate);IOReturn status=controller->exchange(session,header,&input,output,length); + if(status==kIOReturnSuccess)nextHostRequest++;IOLockUnlock(controller->gate);return status; + } + static IOReturn exportSharedAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(current_task()!=self->owner || args->structureOutputDescriptor || !args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + args->scalarOutput[0]=0;IOLockLock(self->gate);unsigned char *bytes=nullptr;size_t length=0; + IOReturn status=self->sharedExchange(OM_SHARED_EXPORT,args->scalarInput[0],&bytes,&length);OMResult result={}; + if(status==kIOReturnSuccess) { + memcpy(&result,bytes+sizeof(OMWireHeader),sizeof(result));size_t prefix=sizeof(OMWireHeader)+sizeof(result); + if(result.error || !result.handle)status=kIOReturnBadArgument; + else { + __atomic_fetch_add(&native_gpu_shared_exports,1,__ATOMIC_RELAXED); + size_t count=length-prefix; + if(!count || count>args->structureOutputSize){self->controller->dropSharedExport(result.handle);status=kIOReturnNoSpace;} + else { + auto token=new NativeGPUSharedEventToken; + if(!token || !token->initWithExport(self->controller,result.handle)) { + OSSafeReleaseNULL(token);self->controller->dropSharedExport(result.handle);status=kIOReturnNoMemory; + } else { + // Kernel MACH_PORT_NULL is an ipc_port pointer, while + // this API returns a name in the owner's namespace. + mach_port_name_t name=0;status=IOUserClient::copyPortNameForObjectInTask(self->owner,token,&name); + if(status==kIOReturnSuccess && name && name!=MACH_PORT_DEAD) { + args->scalarOutput[0]=name;memcpy(args->structureOutput,bytes+prefix,count);args->structureOutputSize=(uint32_t)count; + __atomic_fetch_add(&native_gpu_shared_ports,1,__ATOMIC_RELAXED); + } else if(status==kIOReturnSuccess)status=kIOReturnNoResources; + token->release(); + } + } + } + } + if(bytes)IOFreeData(bytes,length);IOLockUnlock(self->gate); + if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gpu_shared_refusals,1,__ATOMIC_RELAXED);return status; + } + static IOReturn importSharedAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(current_task()!=self->owner || args->scalarInput[0]>UINT32_MAX || args->structureOutputDescriptor || !args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + args->scalarOutput[0]=0;OSObject *object=nullptr;IOReturn status=IOUserClient::copyObjectForPortNameInTask(self->owner,(mach_port_name_t)args->scalarInput[0],&object); + auto token=OSDynamicCast(NativeGPUSharedEventToken,object);unsigned char *bytes=nullptr;size_t length=0;IOLockLock(self->gate); + if(status!=kIOReturnSuccess || !token || token->controller!=self->controller || !token->exported)status=kIOReturnBadArgument; + else { + status=self->sharedExchange(OM_SHARED_IMPORT,token->exported,&bytes,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,bytes+sizeof(OMWireHeader),sizeof(result));size_t prefix=sizeof(OMWireHeader)+sizeof(result),count=length-prefix; + if(result.error || !result.handle)status=kIOReturnBadArgument; + else if(!count || count>args->structureOutputSize) { + // Retire the just-created independent event handle through + // the same real session without consuming a user sequence. + unsigned char *released=nullptr;size_t releasedLength=0; + (void)self->sharedExchange(OM_RELEASE,result.handle,&released,&releasedLength);if(released)IOFreeData(released,releasedLength);status=kIOReturnNoSpace; + } else {args->scalarOutput[0]=result.handle;memcpy(args->structureOutput,bytes+prefix,count);args->structureOutputSize=(uint32_t)count;__atomic_fetch_add(&native_gpu_shared_imports,1,__ATOMIC_RELAXED);} + } + } + if(bytes)IOFreeData(bytes,length);OSSafeReleaseNULL(object);IOLockUnlock(self->gate); + if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gpu_shared_refusals,1,__ATOMIC_RELAXED);return status; + } +public: + bool initWithTask(task_t task,void *security,UInt32 type,OSDictionary *properties) override { + if(type || clientHasPrivilege(security,kIOClientPrivilegeAdministrator)!=kIOReturnSuccess || + !IOUserClient::initWithTask(task,security,type,properties))return false; + owner=task;gate=IOLockAlloc();return gate!=nullptr; + } + bool start(IOService *provider) override { + controller=OSDynamicCast(NativeOfflineGPU,provider); + if(!controller || !IOUserClient::start(provider))return false; + if(controller->createSession(owner,&session)!=kIOReturnSuccess){controller=nullptr;IOUserClient::stop(provider);return false;} + return true; + } + void stop(IOService *provider) override {closeContext();IOUserClient::stop(provider);} + void free() override {closeContext();if(gate){IOLockFree(gate);gate=nullptr;}IOUserClient::free();} + IOReturn clientClose() override {terminate();return kIOReturnSuccess;} + IOReturn externalMethod(uint32_t selector,IOExternalMethodArguments *args,IOExternalMethodDispatch *,OSObject *,void *) override { + static const IOExternalMethodDispatch methods[]={{sendAction,0,kIOUCVariableStructureSize,0,0},{receiveAction,0,0,0,kIOUCVariableStructureSize}, + {exportSharedAction,1,0,1,kIOUCVariableStructureSize},{importSharedAction,1,0,1,kIOUCVariableStructureSize}}; + if(selector>=4)return kIOReturnUnsupported; + return IOUserClient::externalMethod(selector,args,const_cast(methods+selector),this,nullptr); + } +}; +OSDefineMetaClassAndStructors(NativeOfflineGPU,IOService) +OSDefineMetaClassAndStructors(NativeOfflineGPUClient,IOUserClient) +OSDefineMetaClassAndStructors(NativeGPUSharedEventToken,OSObject) +IOReturn NativeOfflineGPU::newUserClient(task_t task,void *security,UInt32 type,OSDictionary *properties,IOUserClient **out) { + if(!out)return kIOReturnBadArgument;*out=nullptr; + auto client=new NativeOfflineGPUClient;if(!client)return kIOReturnNoMemory; + bool attached=false; + if(!client->initWithTask(task,security,type,properties) || !(attached=client->attach(this)) || !client->start(this)) { + if(attached)client->detach(this);client->release();return kIOReturnNotOpen; + } + *out=client;return kIOReturnSuccess; +} +extern "C" IOService *native_offline_gpu_create(void) {return new NativeOfflineGPU;} diff --git a/NativeTurnipSmoke.h b/NativeTurnipSmoke.h new file mode 100644 index 0000000..8450ee7 --- /dev/null +++ b/NativeTurnipSmoke.h @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: MIT +// Turnip smoke test, run by the Adreno Metal plugin from inside the system process that loaded it (under that process's sandbox) once the raw +// userspace test (NativeAdrenoUserTest.h) has passed. It loads the real Mesa Turnip driver (libvulkan_freedreno.dylib, built with the native +// Darwin kernel backend and shipped next to the plugin binary) and drives it through the Vulkan ICD entry points without a loader: +// instance, physical device, device, queue, a host-visible buffer, one vkCmdFillBuffer, a fence, and a read-back of the fill pattern. +// Every stage leaves a progress marker (IOServiceOpen type 0x4147, see NativeAdrenoUserTest.h): 6001 library loaded, 6002 entry points, +// 6003 instance, 6004 physical device, 6005 device, 6007 buffer memory mapped, 6006 command buffer recorded, 6008 submitted, 6009 fence +// signalled, 600a fill pattern verified; then the shader tests: 6100 compute begins, 6101 pipeline, 6102 recorded, 6103 fence, 6104 all words +// right (61fS = compute step S failed); 6200 render begins, 6201 pipeline, 6202 recorded, 6203 fence, 6204 image right (62fS = render step S +// failed); 600b everything destroyed. A setup failure leaves 6fSS (stage SS) followed by 6eRR (RR = -VkResult). +// The outcome is a short text for the kernel's report row: +// "T s r v c g/ ". +#pragma once +#define VK_NO_PROTOTYPES 1 +#include +#include "NativeTurnipShaders.h" // generated: scripts/gen-turnip-smoke-shaders.py +#include +#include +#include +#include +#include +#include +#include +#include + +// Vulkan create-info structs are written as {sType} with every other member zero. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmissing-field-initializers" +namespace turnip { + +static char gFirstError[96]; + +static VKAPI_ATTR VkBool32 VKAPI_CALL onMessage(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT, + const VkDebugUtilsMessengerCallbackDataEXT *data, void *) { + if ((severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) && !gFirstError[0] && data && data->pMessage) + snprintf(gFirstError, sizeof gFirstError, "%s", data->pMessage); + return VK_FALSE; +} + +struct Result { + int stage = 0; // first setup failure (0 = none): 1 library .. 9 fence wait + int vkResult = 0; + int verified = -1; // fill test: words of 1024 that hold the expected value (-1 = not run) + int computeWords = -1; // compute test: words of 1024 written by the shader with the expected value (-1 = not run) + int computeStage = 0; // first failing step of the compute test (0 = none) + int renderTriangle = -1, renderOther = -1; // render test: triangle-colour pixels, pixels that are neither colour (-1 = not run) + int renderStage = 0; + bool ok = false; +}; + +// Device-level entry points the tests use, resolved once through vkGetDeviceProcAddr. +#define TURNIP_DEVICE_FNS(X) \ + X(vkDestroyDevice) X(vkGetDeviceQueue) X(vkCreateCommandPool) X(vkDestroyCommandPool) X(vkAllocateCommandBuffers) \ + X(vkResetCommandBuffer) X(vkBeginCommandBuffer) X(vkEndCommandBuffer) X(vkCmdFillBuffer) X(vkCreateBuffer) X(vkDestroyBuffer) \ + X(vkGetBufferMemoryRequirements) X(vkAllocateMemory) X(vkFreeMemory) X(vkBindBufferMemory) X(vkMapMemory) X(vkUnmapMemory) \ + X(vkCreateFence) X(vkDestroyFence) X(vkQueueSubmit) X(vkWaitForFences) X(vkDeviceWaitIdle) X(vkCreateShaderModule) \ + X(vkDestroyShaderModule) X(vkCreateDescriptorSetLayout) X(vkDestroyDescriptorSetLayout) X(vkCreatePipelineLayout) \ + X(vkDestroyPipelineLayout) X(vkCreateComputePipelines) X(vkCreateGraphicsPipelines) X(vkDestroyPipeline) \ + X(vkCreateDescriptorPool) X(vkDestroyDescriptorPool) X(vkAllocateDescriptorSets) X(vkUpdateDescriptorSets) \ + X(vkCmdBindPipeline) X(vkCmdBindDescriptorSets) X(vkCmdPushConstants) X(vkCmdDispatch) X(vkCmdPipelineBarrier) \ + X(vkCreateImage) X(vkDestroyImage) X(vkGetImageMemoryRequirements) X(vkBindImageMemory) X(vkCreateImageView) \ + X(vkDestroyImageView) X(vkCreateRenderPass) X(vkDestroyRenderPass) X(vkCreateFramebuffer) X(vkDestroyFramebuffer) \ + X(vkCmdBeginRenderPass) X(vkCmdEndRenderPass) X(vkCmdDraw) X(vkCmdCopyImageToBuffer) + +struct Device { + VkDevice device = nullptr; + VkQueue queue = nullptr; + VkCommandPool pool = nullptr; + VkCommandBuffer cmd = nullptr; + VkPhysicalDeviceMemoryProperties memory = {}; +#define TURNIP_DECLARE(name) PFN_##name name = nullptr; + TURNIP_DEVICE_FNS(TURNIP_DECLARE) +#undef TURNIP_DECLARE + + uint32_t memoryType(uint32_t bits, VkMemoryPropertyFlags flags) const { + for (uint32_t i = 0; i < memory.memoryTypeCount; ++i) + if ((bits & (1u << i)) && (memory.memoryTypes[i].propertyFlags & flags) == flags) return i; + return UINT32_MAX; + } + // Records into the one command buffer, submits it with a fresh fence and waits up to five seconds. + VkResult submitAndWait() { + VkFenceCreateInfo fci = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence = nullptr; + VkResult r = vkCreateFence(device, &fci, nullptr, &fence); + if (r != VK_SUCCESS) return r; + VkSubmitInfo si = {VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.commandBufferCount = 1; + si.pCommandBuffers = &cmd; + r = vkQueueSubmit(queue, 1, &si, fence); + if (r == VK_SUCCESS) r = vkWaitForFences(device, 1, &fence, VK_TRUE, 5000000000ull); + vkDestroyFence(device, fence, nullptr); + return r; + } + VkResult begin() { + vkResetCommandBuffer(cmd, 0); + VkCommandBufferBeginInfo bi = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + return vkBeginCommandBuffer(cmd, &bi); + } +}; + +// A host-visible buffer with its memory, mapped for the CPU. +struct HostBuffer { + VkBuffer buffer = nullptr; + VkDeviceMemory memory = nullptr; + volatile uint32_t *words = nullptr; + VkResult create(Device &d, VkDeviceSize bytes, VkBufferUsageFlags usage) { + VkBufferCreateInfo bci = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bci.size = bytes; + bci.usage = usage; + bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VkResult r = d.vkCreateBuffer(d.device, &bci, nullptr, &buffer); + if (r != VK_SUCCESS) return r; + VkMemoryRequirements req = {}; + d.vkGetBufferMemoryRequirements(d.device, buffer, &req); + const uint32_t type = d.memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (type == UINT32_MAX) return VK_ERROR_FEATURE_NOT_PRESENT; + VkMemoryAllocateInfo mai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = req.size; + mai.memoryTypeIndex = type; + r = d.vkAllocateMemory(d.device, &mai, nullptr, &memory); + if (r == VK_SUCCESS) r = d.vkBindBufferMemory(d.device, buffer, memory, 0); + void *mapped = nullptr; + if (r == VK_SUCCESS) r = d.vkMapMemory(d.device, memory, 0, bytes, 0, &mapped); + words = static_cast(mapped); + return r != VK_SUCCESS ? r : (words ? VK_SUCCESS : VK_ERROR_MEMORY_MAP_FAILED); + } + void destroy(Device &d) { + if (words) d.vkUnmapMemory(d.device, memory); + if (buffer) d.vkDestroyBuffer(d.device, buffer, nullptr); + if (memory) d.vkFreeMemory(d.device, memory, nullptr); + *this = HostBuffer(); + } +}; + +static VkShaderModule shaderModule(Device &d, const uint32_t *words, size_t bytes) { + VkShaderModuleCreateInfo ci = {VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + ci.codeSize = bytes; + ci.pCode = words; + VkShaderModule module = nullptr; + return d.vkCreateShaderModule(d.device, &ci, nullptr, &module) == VK_SUCCESS ? module : nullptr; +} + +static void hostBarrier(Device &d, VkPipelineStageFlags from, VkAccessFlags access) { + VkMemoryBarrier barrier = {VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = access; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + d.vkCmdPipelineBarrier(d.cmd, from, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +// Fill: vkCmdFillBuffer of the first half of a 4 KiB buffer (Turnip implements it with its own internal blit). Returns verified words or < 0. +template +static int fillTest(Device &d, Marker &mark, int &failStage, VkResult &failResult) { + constexpr VkDeviceSize kBytes = 4096; + constexpr uint32_t kPattern = 0xc0ffee42u; + HostBuffer b; + VkResult r = b.create(d, kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (r != VK_SUCCESS) {b.destroy(d); failStage = 7; failResult = r; return -1;} + for (unsigned i = 0; i < kBytes / 4; ++i) b.words[i] = 0xdeadbeef; + mark(0x6007); + if ((r = d.begin()) != VK_SUCCESS) {b.destroy(d); failStage = 6; failResult = r; return -1;} + d.vkCmdFillBuffer(d.cmd, b.buffer, 0, kBytes / 2, kPattern); // the first half only: the second half must keep 0xdeadbeef + hostBarrier(d, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + if ((r = d.vkEndCommandBuffer(d.cmd)) != VK_SUCCESS) {b.destroy(d); failStage = 6; failResult = r; return -1;} + mark(0x6006); + mark(0x6008); + if ((r = d.submitAndWait()) != VK_SUCCESS) {b.destroy(d); failStage = 9; failResult = r; return -1;} + mark(0x6009); + int good = 0; + for (unsigned i = 0; i < kBytes / 8; ++i) good += b.words[i] == kPattern; + for (unsigned i = kBytes / 8; i < kBytes / 4; ++i) good += b.words[i] == 0xdeadbeefu; + if (good == int(kBytes / 4)) mark(0x600a); + b.destroy(d); + return good; +} + +// Compute: 16 workgroups of 64 invocations each write v[i] = i * 3 + 0x1000 (push constants) into a storage buffer. +template +static int computeTest(Device &d, Marker &mark, int &stage) { + constexpr uint32_t kWords = 1024, kMul = 3, kAdd = 0x1000; + mark(0x6100); + HostBuffer b; + VkShaderModule module = nullptr; + VkDescriptorSetLayout setLayout = nullptr; + VkPipelineLayout layout = nullptr; + VkPipeline pipeline = nullptr; + VkDescriptorPool pool = nullptr; + int result = -1; + auto fail = [&](int s) {stage = s; mark(0x61f0 | s);}; + do { + if (b.create(d, kWords * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != VK_SUCCESS) {fail(1); break;} + for (uint32_t i = 0; i < kWords; ++i) b.words[i] = 0xdeadbeef; + if (!(module = shaderModule(d, kSpv_smoke_fill_comp, sizeof kSpv_smoke_fill_comp))) {fail(2); break;} + VkDescriptorSetLayoutBinding binding = {0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + VkDescriptorSetLayoutCreateInfo dsl = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + dsl.bindingCount = 1; + dsl.pBindings = &binding; + if (d.vkCreateDescriptorSetLayout(d.device, &dsl, nullptr, &setLayout) != VK_SUCCESS) {fail(3); break;} + VkPushConstantRange push = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8}; + VkPipelineLayoutCreateInfo pl = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.setLayoutCount = 1; + pl.pSetLayouts = &setLayout; + pl.pushConstantRangeCount = 1; + pl.pPushConstantRanges = &push; + if (d.vkCreatePipelineLayout(d.device, &pl, nullptr, &layout) != VK_SUCCESS) {fail(4); break;} + VkComputePipelineCreateInfo cp = {VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + cp.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_COMPUTE_BIT, module, "main", nullptr}; + cp.layout = layout; + if (d.vkCreateComputePipelines(d.device, nullptr, 1, &cp, nullptr, &pipeline) != VK_SUCCESS) {fail(5); break;} + mark(0x6101); + VkDescriptorPoolSize size = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; + VkDescriptorPoolCreateInfo dp = {VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dp.maxSets = 1; + dp.poolSizeCount = 1; + dp.pPoolSizes = &size; + if (d.vkCreateDescriptorPool(d.device, &dp, nullptr, &pool) != VK_SUCCESS) {fail(6); break;} + VkDescriptorSetAllocateInfo da = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + da.descriptorPool = pool; + da.descriptorSetCount = 1; + da.pSetLayouts = &setLayout; + VkDescriptorSet set = nullptr; + if (d.vkAllocateDescriptorSets(d.device, &da, &set) != VK_SUCCESS) {fail(6); break;} + VkDescriptorBufferInfo info = {b.buffer, 0, kWords * 4}; + VkWriteDescriptorSet write = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = set; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &info; + d.vkUpdateDescriptorSets(d.device, 1, &write, 0, nullptr); + if (d.begin() != VK_SUCCESS) {fail(7); break;} + const uint32_t constants[2] = {kMul, kAdd}; + d.vkCmdBindPipeline(d.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + d.vkCmdBindDescriptorSets(d.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 1, &set, 0, nullptr); + d.vkCmdPushConstants(d.cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof constants, constants); + d.vkCmdDispatch(d.cmd, kWords / 64, 1, 1); + hostBarrier(d, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT); + if (d.vkEndCommandBuffer(d.cmd) != VK_SUCCESS) {fail(7); break;} + mark(0x6102); + if (d.submitAndWait() != VK_SUCCESS) {fail(8); break;} + mark(0x6103); + int good = 0; + for (uint32_t i = 0; i < kWords; ++i) good += b.words[i] == i * kMul + kAdd; + result = good; + if (good == int(kWords)) mark(0x6104); + } while (false); + if (pipeline) d.vkDestroyPipeline(d.device, pipeline, nullptr); + if (pool) d.vkDestroyDescriptorPool(d.device, pool, nullptr); + if (layout) d.vkDestroyPipelineLayout(d.device, layout, nullptr); + if (setLayout) d.vkDestroyDescriptorSetLayout(d.device, setLayout, nullptr); + if (module) d.vkDestroyShaderModule(d.device, module, nullptr); + b.destroy(d); + return result; +} + +// Render: a 64x64 RGBA8 colour attachment cleared to blue, one triangle drawn in orange (push constant), copied to a host buffer. +// The triangle covers 512 pixels in exact arithmetic; every other pixel must keep the clear colour. +template +static void renderTest(Device &d, Marker &mark, int &stage, int &triangle, int &other) { + constexpr uint32_t kSize = 64; + mark(0x6200); + HostBuffer b; + VkImage image = nullptr; + VkDeviceMemory imageMemory = nullptr; + VkImageView view = nullptr; + VkRenderPass pass = nullptr; + VkFramebuffer framebuffer = nullptr; + VkShaderModule vs = nullptr, fs = nullptr; + VkPipelineLayout layout = nullptr; + VkPipeline pipeline = nullptr; + auto fail = [&](int s) {stage = s; mark(0x62f0 | s);}; + do { + if (b.create(d, kSize * kSize * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT) != VK_SUCCESS) {fail(1); break;} + for (uint32_t i = 0; i < kSize * kSize; ++i) b.words[i] = 0x5a5a5a5a; + VkImageCreateInfo ic = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + ic.imageType = VK_IMAGE_TYPE_2D; + ic.format = VK_FORMAT_R8G8B8A8_UNORM; + ic.extent = {kSize, kSize, 1}; + ic.mipLevels = 1; + ic.arrayLayers = 1; + ic.samples = VK_SAMPLE_COUNT_1_BIT; + ic.tiling = VK_IMAGE_TILING_OPTIMAL; + ic.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + ic.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + if (d.vkCreateImage(d.device, &ic, nullptr, &image) != VK_SUCCESS) {fail(2); break;} + VkMemoryRequirements req = {}; + d.vkGetImageMemoryRequirements(d.device, image, &req); + uint32_t type = d.memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + if (type == UINT32_MAX) type = d.memoryType(req.memoryTypeBits, 0); + VkMemoryAllocateInfo mai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = req.size; + mai.memoryTypeIndex = type; + if (type == UINT32_MAX || d.vkAllocateMemory(d.device, &mai, nullptr, &imageMemory) != VK_SUCCESS || + d.vkBindImageMemory(d.device, image, imageMemory, 0) != VK_SUCCESS) {fail(2); break;} + VkImageViewCreateInfo vc = {VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + vc.image = image; + vc.viewType = VK_IMAGE_VIEW_TYPE_2D; + vc.format = ic.format; + vc.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + if (d.vkCreateImageView(d.device, &vc, nullptr, &view) != VK_SUCCESS) {fail(3); break;} + VkAttachmentDescription attachment = {0, ic.format, VK_SAMPLE_COUNT_1_BIT, VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE, + VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_DONT_CARE, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL}; + VkAttachmentReference ref = {0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass = {}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &ref; + VkSubpassDependency after = {0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, 0}; + VkRenderPassCreateInfo rp = {VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; + rp.attachmentCount = 1; + rp.pAttachments = &attachment; + rp.subpassCount = 1; + rp.pSubpasses = &subpass; + rp.dependencyCount = 1; + rp.pDependencies = &after; + if (d.vkCreateRenderPass(d.device, &rp, nullptr, &pass) != VK_SUCCESS) {fail(4); break;} + VkFramebufferCreateInfo fb = {VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + fb.renderPass = pass; + fb.attachmentCount = 1; + fb.pAttachments = &view; + fb.width = kSize; + fb.height = kSize; + fb.layers = 1; + if (d.vkCreateFramebuffer(d.device, &fb, nullptr, &framebuffer) != VK_SUCCESS) {fail(4); break;} + vs = shaderModule(d, kSpv_smoke_triangle_vert, sizeof kSpv_smoke_triangle_vert); + fs = shaderModule(d, kSpv_smoke_triangle_frag, sizeof kSpv_smoke_triangle_frag); + if (!vs || !fs) {fail(5); break;} + VkPushConstantRange push = {VK_SHADER_STAGE_FRAGMENT_BIT, 0, 16}; + VkPipelineLayoutCreateInfo pl = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.pushConstantRangeCount = 1; + pl.pPushConstantRanges = &push; + if (d.vkCreatePipelineLayout(d.device, &pl, nullptr, &layout) != VK_SUCCESS) {fail(6); break;} + VkPipelineShaderStageCreateInfo stages[2] = { + {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_VERTEX_BIT, vs, "main", nullptr}, + {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_FRAGMENT_BIT, fs, "main", nullptr}}; + VkPipelineVertexInputStateCreateInfo vi = {VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; + VkPipelineInputAssemblyStateCreateInfo ia = {VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; + ia.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + VkViewport viewport = {0, 0, float(kSize), float(kSize), 0, 1}; + VkRect2D scissor = {{0, 0}, {kSize, kSize}}; + VkPipelineViewportStateCreateInfo vp = {VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; + vp.viewportCount = 1; + vp.pViewports = &viewport; + vp.scissorCount = 1; + vp.pScissors = &scissor; + VkPipelineRasterizationStateCreateInfo rs = {VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; + rs.polygonMode = VK_POLYGON_MODE_FILL; + rs.cullMode = VK_CULL_MODE_NONE; + rs.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + rs.lineWidth = 1.0f; + VkPipelineMultisampleStateCreateInfo ms = {VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; + ms.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + VkPipelineColorBlendAttachmentState blend = {}; + blend.colorWriteMask = 0xf; + VkPipelineColorBlendStateCreateInfo cb = {VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; + cb.attachmentCount = 1; + cb.pAttachments = &blend; + VkGraphicsPipelineCreateInfo gp = {VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; + gp.stageCount = 2; + gp.pStages = stages; + gp.pVertexInputState = &vi; + gp.pInputAssemblyState = &ia; + gp.pViewportState = &vp; + gp.pRasterizationState = &rs; + gp.pMultisampleState = &ms; + gp.pColorBlendState = &cb; + gp.layout = layout; + gp.renderPass = pass; + if (d.vkCreateGraphicsPipelines(d.device, nullptr, 1, &gp, nullptr, &pipeline) != VK_SUCCESS) {fail(7); break;} + mark(0x6201); + if (d.begin() != VK_SUCCESS) {fail(8); break;} + VkClearValue clear = {}; + clear.color.float32[2] = 1.0f; + clear.color.float32[3] = 1.0f; + VkRenderPassBeginInfo rb = {VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; + rb.renderPass = pass; + rb.framebuffer = framebuffer; + rb.renderArea = scissor; + rb.clearValueCount = 1; + rb.pClearValues = &clear; + d.vkCmdBeginRenderPass(d.cmd, &rb, VK_SUBPASS_CONTENTS_INLINE); + d.vkCmdBindPipeline(d.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + const float color[4] = {1.0f, 0.5f, 0.0f, 1.0f}; + d.vkCmdPushConstants(d.cmd, layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof color, color); + d.vkCmdDraw(d.cmd, 3, 1, 0, 0); + d.vkCmdEndRenderPass(d.cmd); + VkBufferImageCopy copy = {}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {kSize, kSize, 1}; + d.vkCmdCopyImageToBuffer(d.cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, b.buffer, 1, ©); + hostBarrier(d, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + if (d.vkEndCommandBuffer(d.cmd) != VK_SUCCESS) {fail(8); break;} + mark(0x6202); + if (d.submitAndWait() != VK_SUCCESS) {fail(9); break;} + mark(0x6203); + // RGBA8 little-endian words: clear = (0, 0, 255, 255), triangle = (255, 127 or 128, 0, 255). + int tri = 0, rest = 0; + for (uint32_t i = 0; i < kSize * kSize; ++i) { + const uint32_t px = b.words[i]; + if (px == 0xffff0000u) continue; + if (px == 0xff0080ffu || px == 0xff007fffu) ++tri; + else ++rest; + } + triangle = tri; + other = rest; + if (rest == 0 && tri >= 480 && tri <= 544) mark(0x6204); + } while (false); + if (pipeline) d.vkDestroyPipeline(d.device, pipeline, nullptr); + if (layout) d.vkDestroyPipelineLayout(d.device, layout, nullptr); + if (vs) d.vkDestroyShaderModule(d.device, vs, nullptr); + if (fs) d.vkDestroyShaderModule(d.device, fs, nullptr); + if (framebuffer) d.vkDestroyFramebuffer(d.device, framebuffer, nullptr); + if (pass) d.vkDestroyRenderPass(d.device, pass, nullptr); + if (view) d.vkDestroyImageView(d.device, view, nullptr); + if (image) d.vkDestroyImage(d.device, image, nullptr); + if (imageMemory) d.vkFreeMemory(d.device, imageMemory, nullptr); + b.destroy(d); +} + +template +static Result run(Marker &&mark, const char *libraryPath) { + Result out; + auto fail = [&](int stage, int result) { + out.stage = stage; + out.vkResult = result; + mark(0x6f00 | stage); + mark(0x6e00 | ((-result) & 0xff)); + return out; + }; + // The Darwin backend reaches the driver the same way the plugin does: the AGX accelerator service with open type 0x4147, ABI methods through + // an admitted selector. Nothing to configure; set here only so the choice is visible. The shader disk cache is off: this runs inside a + // system process whose home directory is not ours to fill. + setenv("TU_DARWIN_SERVICE", "IOAcceleratorES", 0); + setenv("MESA_SHADER_CACHE_DISABLE", "true", 0); + // Concurrent binning (BV running ahead of BR) is a performance feature; its barriers make BR wait for BV timestamps. int-058's + // first fill hung with the ring stopped inside the IB (no fault), before the kernel framed IBs for both threads. Keep it off + // until a run with the framing has passed. + setenv("TU_DEBUG", "nocb", 0); + + void *lib = dlopen(libraryPath, RTLD_NOW | RTLD_LOCAL); + if (!lib) return fail(1, -1); + mark(0x6001); + auto gipa = reinterpret_cast(dlsym(lib, "vk_icdGetInstanceProcAddr")); + if (!gipa) return fail(2, -2); + auto createInstance = reinterpret_cast(gipa(nullptr, "vkCreateInstance")); + if (!createInstance) return fail(2, -3); + mark(0x6002); + + const char *extensions[] = {VK_EXT_DEBUG_UTILS_EXTENSION_NAME}; + VkDebugUtilsMessengerCreateInfoEXT messenger = {VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT}; + messenger.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; + messenger.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT; + messenger.pfnUserCallback = onMessage; + VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; + app.pApplicationName = "NativeAdrenoMetal"; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo ici = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + ici.pNext = &messenger; + ici.pApplicationInfo = &app; + ici.enabledExtensionCount = 1; + ici.ppEnabledExtensionNames = extensions; + VkInstance instance = nullptr; + VkResult r = createInstance(&ici, nullptr, &instance); + if (r != VK_SUCCESS) return fail(3, r); + mark(0x6003); + +#define INSTANCE_FN(name) auto name = reinterpret_cast(gipa(instance, #name)); if (!name) return fail(3, -4) + INSTANCE_FN(vkEnumeratePhysicalDevices); + INSTANCE_FN(vkGetPhysicalDeviceMemoryProperties); + INSTANCE_FN(vkGetPhysicalDeviceQueueFamilyProperties); + INSTANCE_FN(vkCreateDevice); + INSTANCE_FN(vkGetDeviceProcAddr); + INSTANCE_FN(vkDestroyInstance); +#undef INSTANCE_FN + + uint32_t count = 1; + VkPhysicalDevice physical = nullptr; + r = vkEnumeratePhysicalDevices(instance, &count, &physical); + if ((r != VK_SUCCESS && r != VK_INCOMPLETE) || !count || !physical) return fail(4, r == VK_SUCCESS ? -5 : r); + mark(0x6004); + + uint32_t families = 0; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, nullptr); + if (!families) return fail(4, -6); + VkQueueFamilyProperties family[8] = {}; + if (families > 8) families = 8; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, family); + uint32_t queueFamily = families; + for (uint32_t i = 0; i < families && queueFamily == families; ++i) + if (family[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) queueFamily = i; + if (queueFamily == families) return fail(4, -7); + + float priority = 1.0f; + VkDeviceQueueCreateInfo qci = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + qci.queueFamilyIndex = queueFamily; + qci.queueCount = 1; + qci.pQueuePriorities = &priority; + VkDeviceCreateInfo dci = {VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + dci.queueCreateInfoCount = 1; + dci.pQueueCreateInfos = &qci; + Device d; + r = vkCreateDevice(physical, &dci, nullptr, &d.device); + if (r != VK_SUCCESS) return fail(5, r); + mark(0x6005); +#define TURNIP_RESOLVE(name) d.name = reinterpret_cast(vkGetDeviceProcAddr(d.device, #name)); if (!d.name) return fail(5, -8); + TURNIP_DEVICE_FNS(TURNIP_RESOLVE) +#undef TURNIP_RESOLVE + vkGetPhysicalDeviceMemoryProperties(physical, &d.memory); + d.vkGetDeviceQueue(d.device, queueFamily, 0, &d.queue); + VkCommandPoolCreateInfo pci = {VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pci.queueFamilyIndex = queueFamily; + r = d.vkCreateCommandPool(d.device, &pci, nullptr, &d.pool); + if (r != VK_SUCCESS) return fail(6, r); + VkCommandBufferAllocateInfo cai = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + cai.commandPool = d.pool; + cai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cai.commandBufferCount = 1; + r = d.vkAllocateCommandBuffers(d.device, &cai, &d.cmd); + if (r != VK_SUCCESS) return fail(6, r); + + // The three tests are independent: a failed or unverified one does not stop the next, unless the GPU stopped retiring work (fence timeout). + int failStage = 0; + VkResult failResult = VK_SUCCESS; + out.verified = fillTest(d, mark, failStage, failResult); + if (failStage) { + fail(failStage, failResult); + } + if (failStage != 9) { + out.computeWords = computeTest(d, mark, out.computeStage); + if (out.computeStage != 8) renderTest(d, mark, out.renderStage, out.renderTriangle, out.renderOther); + } + + d.vkDeviceWaitIdle(d.device); + d.vkDestroyCommandPool(d.device, d.pool, nullptr); + d.vkDestroyDevice(d.device, nullptr); + vkDestroyInstance(instance, nullptr); + mark(0x600b); + out.ok = !failStage && out.verified == 1024 && out.computeWords == 1024 && out.renderOther == 0 && out.renderTriangle >= 480 && + out.renderTriangle <= 544; + return out; +} + +// Watchdog. run() executes on a worker thread; if it has not returned after 15, 45 and 90 seconds, the calling thread samples the worker and +// sends "W s u/k over 300 ms, then the stack": the PC, LR and frame-pointer return addresses, each +// as (v = libvulkan_freedreno.dylib; offsets resolve against the unstripped build). A worker that is +// running in the kernel (system time only, or no time at all: a core parked by the panic screen) is not suspended, because thread_suspend +// would wait for it forever; its line ends in "kern". +struct Watched { + const char *libraryPath; + void (*mark)(void *, uint32_t); + void *markContext; + volatile uint32_t lastMark; + volatile int done; + Result result; +}; + +static void *watchedWorker(void *argument) { + Watched *w = static_cast(argument); + w->result = run([w](uint32_t value) { w->lastMark = value; w->mark(w->markContext, value); }, w->libraryPath); + __atomic_store_n(&w->done, 1, __ATOMIC_RELEASE); + return nullptr; +} + +static size_t frameText(char *out, size_t size, uint64_t address) { + Dl_info info = {}; + if (!address || !dladdr(reinterpret_cast(address), &info) || !info.dli_fbase) + return (size_t)snprintf(out, size, " ?%llx", (unsigned long long)address); + const char *name = info.dli_fname ? strrchr(info.dli_fname, '/') : nullptr; + name = name ? name + 1 : (info.dli_fname ? info.dli_fname : "?"); + const char tag = !strncmp(name, "libvulkan", 9) ? 'v' : name[0]; + return (size_t)snprintf(out, size, " %c%llx", tag, (unsigned long long)(address - reinterpret_cast(info.dli_fbase))); +} + +static void sample(Watched &w, pthread_t worker, char *text, size_t size) { + const mach_port_t thread = pthread_mach_thread_np(worker); + thread_basic_info_data_t a = {}, b = {}; + mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; + thread_info(thread, THREAD_BASIC_INFO, reinterpret_cast(&a), &count); + usleep(300000); + count = THREAD_BASIC_INFO_COUNT; + thread_info(thread, THREAD_BASIC_INFO, reinterpret_cast(&b), &count); + auto us = [](time_value_t t) { return int64_t(t.seconds) * 1000000 + t.microseconds; }; + const int64_t du = us(b.user_time) - us(a.user_time), dk = us(b.system_time) - us(a.system_time); + size_t at = (size_t)snprintf(text, size, "W %x s%d u%lld/k%lld", unsigned(w.lastMark), b.run_state, (long long)du, (long long)dk); + if (b.run_state == TH_STATE_RUNNING && du <= 0) { + snprintf(text + at, size - at, " kern"); + return; + } + if (thread_suspend(thread) != KERN_SUCCESS) {snprintf(text + at, size - at, " nosusp"); return;} + arm_thread_state64_t state = {}; + count = ARM_THREAD_STATE64_COUNT; + if (thread_get_state(thread, ARM_THREAD_STATE64, reinterpret_cast(&state), &count) == KERN_SUCCESS) { + const uint64_t pc = arm_thread_state64_get_pc(state), lr = arm_thread_state64_get_lr(state); + uint64_t fp = arm_thread_state64_get_fp(state); + at += frameText(text + at, size - at, pc); + if (at < size) at += frameText(text + at, size - at, lr); + const uint64_t low = reinterpret_cast(pthread_get_stackaddr_np(worker)) - pthread_get_stacksize_np(worker); + const uint64_t high = reinterpret_cast(pthread_get_stackaddr_np(worker)); + for (unsigned i = 0; i < 8 && at + 4 < size && fp >= low && fp + 16 <= high && !(fp & 7); ++i) { + const uint64_t next = reinterpret_cast(fp)[0], ret = reinterpret_cast(fp)[1]; + if (ret != lr || i) at += frameText(text + at, size - at, ret); + if (next <= fp) break; + fp = next; + } + } + thread_resume(thread); +} + +template +static Result runWatched(Marker &mark, Reporter &&report, const char *libraryPath) { + Watched *w = new Watched{}; + w->libraryPath = libraryPath; + w->markContext = &mark; + w->mark = [](void *context, uint32_t value) { (*static_cast(context))(value); }; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8u << 20); // Mesa's NIR/ir3 passes recurse; the default 512 KiB secondary stack is small + pthread_t worker; + const bool started = pthread_create(&worker, &attr, watchedWorker, w) == 0; + pthread_attr_destroy(&attr); + if (!started) {Result r; r.stage = 0; r.vkResult = -12; return r;} + static const unsigned checkpoints[] = {15, 45, 90}; + unsigned next = 0; + for (unsigned tenths = 0; !__atomic_load_n(&w->done, __ATOMIC_ACQUIRE); ++tenths) { + usleep(100000); + if (next < 3 && tenths == checkpoints[next] * 10) { + char text[64] = {}; + sample(*w, worker, text, sizeof text); + report(text); + ++next; + } + } + pthread_join(worker, nullptr); + Result result = w->result; + delete w; + return result; +} + +// Path of libvulkan_freedreno.dylib: next to the image this code is part of (the plugin binary inside the bundle). +static bool libraryPath(char *path, size_t size) { + Dl_info info = {}; + if (!dladdr(reinterpret_cast(&libraryPath), &info) || !info.dli_fname) return false; + snprintf(path, size, "%s", info.dli_fname); + char *slash = strrchr(path, '/'); + if (!slash) return false; + snprintf(slash + 1, size - size_t(slash + 1 - path), "libvulkan_freedreno.dylib"); + return true; +} + +} // namespace turnip +#pragma clang diagnostic pop diff --git a/OfflineMetalCommandWire.h b/OfflineMetalCommandWire.h new file mode 100644 index 0000000..a0f4c12 --- /dev/null +++ b/OfflineMetalCommandWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Actual asynchronous host submission and status observation. Native integration +// must separately qualify resource visibility, lifetime and callback delivery. +#pragma once +#include "OfflineMetalFenceWire.h" +enum { OM_COMMAND_SUBMIT=0x12e,OM_COMMAND_STATUS=0x12f }; +static_assert(OM_COMMAND_SUBMIT==OM_TEXTURE_HAZARD+1,"Command extension ordering"); +// Reply is OMCommandResult: non-completed states have unavailable (zero) times. +// Successful completion requires actual status 4 and genuine nonzero GPU times. diff --git a/OfflineMetalDepthStencilWire.h b/OfflineMetalDepthStencilWire.h new file mode 100644 index 0000000..20ec4f4 --- /dev/null +++ b/OfflineMetalDepthStencilWire.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT +// Explicit offline depth/stencil resources, state and render attachments. +#pragma once +#include "OfflineMetalTextureWire.h" +enum { OM_DEPTH_STENCIL_STATE=0x123,OM_SET_DEPTH_STENCIL=0x124, + OM_STENCIL_REFERENCE=0x125,OM_PIPELINE_ATTACHMENTS=0x126,OM_RENDER_ATTACHMENTS=0x127 }; +static_assert(OM_DEPTH_STENCIL_STATE==OM_COPY_TEXTURE+1,"Depth/stencil extension follows texture copy"); +typedef struct {uint64_t enabled,compare,failure,depthFailure,pass,readMask,writeMask;} OMStencilFace; +typedef struct {uint64_t compare,write;OMStencilFace front,back;} OMDepthStencil; +typedef struct {uint64_t encoder,state;} OMSetDepthStencil; +typedef struct {uint64_t encoder,front,back;} OMStencilReference; +typedef struct {OMBlendPipeline colors;uint64_t depthFormat,stencilFormat;} OMAttachmentPipeline; +typedef struct {uint64_t texture,load,store;double clear;} OMDepthAttachment; +typedef struct {uint64_t texture,load,store,clear;} OMStencilAttachment; +typedef struct {OMRender colors;OMDepthAttachment depth;OMStencilAttachment stencil;} OMAttachmentRender; +static_assert(sizeof(OMDepthStencil)==128 && sizeof(OMAttachmentRender)==520,"Bounded attachment/state ABI"); diff --git a/OfflineMetalEventWire.h b/OfflineMetalEventWire.h new file mode 100644 index 0000000..3f2d277 --- /dev/null +++ b/OfflineMetalEventWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Actual GPU timeline events for cross-queue dependencies; CPU bookkeeping +// never substitutes for executing a GPU signal/wait or reports a signaled value. +#pragma once +#include "OfflineMetalCommandWire.h" +enum { OM_EVENT=0x130,OM_EVENT_SIGNAL,OM_EVENT_WAIT }; +static_assert(OM_EVENT==OM_COMMAND_STATUS+1,"Event extension ordering"); +typedef struct {uint64_t command,event,value;} OMEventDependency; +static_assert(sizeof(OMEventDependency)==24,"Bounded real GPU event dependency"); diff --git a/OfflineMetalFenceWire.h b/OfflineMetalFenceWire.h new file mode 100644 index 0000000..a533fcd --- /dev/null +++ b/OfflineMetalFenceWire.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Explicit offline actual GPU fence dependencies; no emulated completion. +#pragma once +#include "OfflineMetalDepthStencilWire.h" +enum { OM_FENCE=0x128,OM_RENDER_FENCE_UPDATE,OM_RENDER_FENCE_WAIT, + OM_BLIT_FENCE_UPDATE,OM_BLIT_FENCE_WAIT,OM_TEXTURE_HAZARD }; +static_assert(OM_FENCE==OM_RENDER_ATTACHMENTS+1,"Fence extension ordering"); +typedef struct {uint64_t encoder,fence,stages;} OMFenceDependency; +typedef struct {OMTexture texture;uint64_t hazard;} OMHazardTexture; +static_assert(sizeof(OMHazardTexture)==96,"Bounded actual texture hazard mode"); +static_assert(sizeof(OMFenceDependency)==24,"Bounded real fence dependency"); diff --git a/OfflineMetalFrontend.mm b/OfflineMetalFrontend.mm new file mode 100644 index 0000000..55fab6b --- /dev/null +++ b/OfflineMetalFrontend.mm @@ -0,0 +1,1001 @@ +// SPDX-License-Identifier: MIT +// Explicit offline native Metal API frontend. Limited supported operations; +// never installed or advertised as a complete system/hardware GPU driver. +#import +#import +#import +#import +#import +#import +#include +#include +#include "OfflineMetalWire.h" +#include "OfflineMetalFunctionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalVertexWire.h" +#include "OfflineMetalFenceWire.h" +#include "OfflineMetalSharedEventWire.h" +#include +#include +#include +#include +using IOMethod=int (*)(unsigned,unsigned,const uint64_t *,unsigned,const void *,size_t,uint64_t *,unsigned *,void *,size_t *); +static IOMethod ioMethod; +static unsigned connection; +static uint32_t sequence; +static NSRecursiveLock *transaction; +static NSDictionary *deviceInfo; +static id remoteDevice; +static std::atomic live{false},asyncEnabled{false},closing{false}; +static NSLock *pendingLock; +static NSMutableSet *pendingCommands; +static NSMutableSet *pendingNotifications; +static NSException *notificationFailure; +static uint64_t notificationsRegistered,notificationsDelivered,notificationsCancelled,notificationsFailed; +static void fail(NSString *reason) { @throw [NSException exceptionWithName:@"OfflineMetalTransport" reason:reason userInfo:nil]; } +static void receiveBytes(void *bytes,size_t size) { + unsigned char *p=(unsigned char *)bytes; + while(size){size_t count=MIN(size,(size_t)4096);int status=ioMethod(connection,1,nullptr,0,nullptr,0,nullptr,nullptr,p,&count); + if(status || !count || count>size)fail(@"Actual console receive failed");p+=count;size-=count;} +} +static NSData *call(uint16_t operation,const void *input,size_t size,OMResult *result) { + [transaction lock]; + @try { + if(!live || size>4080)fail(@"Invalid live transport request"); + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,operation,(uint32_t)size,sequence+1}; + unsigned char bytes[4096];memcpy(bytes,&header,sizeof(header));if(size)memcpy(bytes+sizeof(header),input,size); + int sendStatus=ioMethod(connection,0,nullptr,0,bytes,sizeof(header)+size,nullptr,nullptr,nullptr,nullptr); + if(sendStatus)fail([NSString stringWithFormat:@"Actual transport send failed: %08x",(unsigned)sendStatus]); + sequence++; + OMWireHeader reply={};receiveBytes(&reply,sizeof(reply)); + if(reply.magic!=OM_WIRE_MAGIC || reply.version!=OFFLINE_METAL_VERSION || reply.operation!=(operation|OM_WIRE_RESPONSE) + || reply.request!=sequence || reply.bytesOFFLINE_METAL_MAX_REPLY)fail(@"Invalid actual Metal reply"); + receiveBytes(result,sizeof(*result));if(result->reserved)fail(@"Unknown reply flags"); + NSMutableData *body=[NSMutableData dataWithLength:reply.bytes-sizeof(*result)];if(body.length)receiveBytes(body.mutableBytes,body.length); + return body; + } @finally {[transaction unlock];} +} +static NSData *checked(uint16_t operation,const void *input,size_t size,OMResult *result) { + NSData *body=call(operation,input,size,result);if(result->error)fail([[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding]?:@"Unsupported or failed actual Metal operation");return body; +} +static NSError *operationError(OMResult result,NSData *body) { + NSString *why=[[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding]?:@"Actual offline backend refused this operation"; + return [NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:result.error userInfo:@{NSLocalizedDescriptionKey:why}]; +} +@interface OMObject : NSObject +@property(readonly) id device; +@property(nonatomic) uint64_t handle; +@property(atomic,copy) NSString *label; +@end +@implementation OMObject +- (id)device {return remoteDevice;} +- (void)dealloc { + // Actual host Metal objects retain resources required by their recorded + // command buffers. Drop exactly this proxy's independent table reference. + // Session teardown releases any remaining references after real commits. + [transaction lock]; + @try {if(live && !closing && _handle){OMHandle request={_handle};OMResult result={};checked(OM_RELEASE,&request,sizeof(request),&result);}} + @finally {[transaction unlock];} +} +@end +@interface OMFenceObject : OMObject +@end +@implementation OMFenceObject +@end +@interface OMEventObject : OMObject +@end +@implementation OMEventObject +@end +// These selectors are read by the unchanged target handle constructor. The +// eventPort is a real Mach capability created by the native kernel broker. +@interface MTLSharedEventHandle (OMOriginalSharedHandle) +- (instancetype)initWithSharedEvent:(id)event; +- (mach_port_t)eventPort; +- (uint64_t)labelTraceID; +@end +static NSDictionary *sharedMetadata(NSData *body) { + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil]; + if(![info isKindOfClass:[NSDictionary class]] || ![info[@"device_is_nil"] isKindOfClass:[NSNumber class]] || + ![info[@"device_is_owner"] isKindOfClass:[NSNumber class]] || + ([info[@"device_is_nil"] boolValue]==[info[@"device_is_owner"] boolValue]) || + ![info[@"signaled_value"] isKindOfClass:[NSNumber class]] || + (info[@"label"]!=[NSNull null] && ![info[@"label"] isKindOfClass:[NSString class]]))fail(@"Invalid actual shared-event metadata"); + return info; +} +enum {OMNotificationPending,OMNotificationQueued,OMNotificationRunning,OMNotificationFinished,OMNotificationCancelled}; +@interface OMSharedNotificationState : NSObject +@property(nonatomic) uint64_t handle,value; +@property(nonatomic,strong) id event; +@property(nonatomic,strong) MTLSharedEventListener *listener; +@property(nonatomic,strong) dispatch_queue_t queue; +@property(nonatomic,strong) dispatch_group_t callbackGroup; +@property(nonatomic,copy) MTLSharedEventNotificationBlock block; +// All state transitions and real callback thread identities use pendingLock. +@property(nonatomic) unsigned phase; +@property(nonatomic) pthread_t callbackThread; +- (void)start; +@end +@implementation OMSharedNotificationState +- (void)start { + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED,0),^{@autoreleasepool { + @try { + struct timespec start={},now={};if(clock_gettime(CLOCK_MONOTONIC,&start))fail(@"Actual listener clock unavailable"); + for(;;) { + [pendingLock lock];BOOL cancelled=self.phase==OMNotificationCancelled;[pendingLock unlock];if(cancelled)return; + OMSharedNotification reply={}; + [transaction lock];@try { + if(!live || closing)return; + OMHandle input={self.handle};OMResult result={};NSData *body=checked(OM_SHARED_NOTIFICATION_POLL,&input,sizeof(input),&result); + if(body.length!=sizeof(reply))fail(@"Invalid actual listener reply");memcpy(&reply,body.bytes,sizeof(reply)); + } @finally {[transaction unlock];} + if(reply.ready) { + if(reply.ready!=1 || reply.event!=[(OMObject *)self.event handle] || reply.value=60)fail(@"Bounded actual listener observation timed out");usleep(1000); + } + } @catch(NSException *exception) { + [pendingLock lock];if(self.phase!=OMNotificationCancelled){if(!notificationFailure)notificationFailure=exception;notificationsFailed++;self.phase=OMNotificationFinished;[pendingNotifications removeObject:self];}[pendingLock unlock]; + } + }}); +} +@end +@interface OMSharedEventObject : OMEventObject { + mach_port_t _exportPort; + uint64_t _traceID; +} +@property(nonatomic) BOOL deviceIsNil; +- (mach_port_t)eventPort; +- (uint64_t)labelTraceID; +@end +@implementation OMSharedEventObject +- (id)device {return self.deviceIsNil?nil:remoteDevice;} +- (void)dealloc {if(_exportPort)mach_port_deallocate(mach_task_self(),_exportPort);} +- (uint64_t)signaledValue { + OMHandle input={self.handle};OMResult result={};NSData *body=checked(OM_SHARED_VALUE_GET,&input,sizeof(input),&result); + if(body.length!=8)fail(@"Invalid actual event counter");uint64_t value=0;memcpy(&value,body.bytes,8);return value; +} +- (void)setSignaledValue:(uint64_t)value { + OMSharedEventValue input={self.handle,value};OMResult result={};NSData *body=checked(OM_SHARED_VALUE_SET,&input,sizeof(input),&result); + if(body.length!=8)fail(@"Missing actual event counter write result"); +} +- (void)setLabel:(NSString *)label { + if(label && ![label isKindOfClass:[NSString class]])fail(@"Invalid shared-event label"); + NSData *text=[label dataUsingEncoding:NSUTF8StringEncoding];if(text.length>512)fail(@"Excessive shared-event label"); + OMSharedLabel input={self.handle,(uint64_t)(label!=nil),text.length};NSMutableData *data=[NSMutableData dataWithBytes:&input length:sizeof(input)];if(text)[data appendData:text]; + [transaction lock];@try { + OMResult result={};NSData *body=checked(OM_SHARED_LABEL_SET,data.bytes,data.length,&result);NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil]; + id actual=info[@"label"];if(actual!=[NSNull null] && ![actual isKindOfClass:[NSString class]])fail(@"Missing actual label result"); + [super setLabel:actual==[NSNull null]?nil:actual]; + if(_exportPort){mach_port_deallocate(mach_task_self(),_exportPort);_exportPort=0;_traceID=0;} + } @finally {[transaction unlock];} +} +- (mach_port_t)eventPort { + [transaction lock];@try { + if(!live || closing)fail(@"Shared export on a closing context");if(_exportPort)return _exportPort; + uint64_t output=0,input=self.handle;unsigned count=1;unsigned char bytes[4096];size_t size=sizeof(bytes); + int status=ioMethod(connection,2,&input,1,nullptr,0,&output,&count,bytes,&size); + if(status || count!=1 || !output || output>UINT32_MAX || output==MACH_PORT_DEAD)fail(@"Native shared-event capability export refused"); + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:[NSData dataWithBytes:bytes length:size] options:0 error:nil]; + if(![info[@"trace_id_supported"] boolValue] || ![info[@"trace_id"] isKindOfClass:[NSNumber class]]){mach_port_deallocate(mach_task_self(),(mach_port_t)output);fail(@"Unqualified actual shared-event trace ABI");} + _exportPort=(mach_port_t)output;_traceID=[info[@"trace_id"] unsignedLongLongValue];return _exportPort; + } @finally {[transaction unlock];} +} +- (uint64_t)labelTraceID {(void)self.eventPort;return _traceID;} +- (MTLSharedEventHandle *)newSharedEventHandle { + Class cls=NSClassFromString(@"MTLSharedEventHandle");if(!cls || class_getInstanceSize(cls)!=16)fail(@"Wrong original shared-handle ABI"); + return [[cls alloc] initWithSharedEvent:self]; +} +- (void)notifyListener:(MTLSharedEventListener *)listener atValue:(uint64_t)value block:(MTLSharedEventNotificationBlock)block { + Class cls=NSClassFromString(@"MTLSharedEventListenerInternal");Method getter=class_getInstanceMethod(cls,@selector(dispatchQueue)); + if(!listener || !block || object_getClass(listener)!=cls || class_getInstanceSize(cls)!=24 || !getter || strcmp(method_getTypeEncoding(getter),"@16@0:8"))fail(@"Unqualified original shared listener ABI"); + dispatch_queue_t queue=listener.dispatchQueue;if(!queue)fail(@"Missing actual original listener queue"); + [transaction lock];@try { + if(!live || closing)fail(@"Listener registration on a closing context");OMSharedEventValue input={self.handle,value};OMResult result={};checked(OM_SHARED_NOTIFY,&input,sizeof(input),&result);if(!result.handle)fail(@"Missing actual listener registration"); + OMSharedNotificationState *state=[OMSharedNotificationState new];state.handle=result.handle;state.value=value;state.event=self;state.listener=listener;state.queue=queue;state.block=block;state.callbackGroup=dispatch_group_create(); + [pendingLock lock];if(closing){[pendingLock unlock];fail(@"Listener registration raced context finish");}[pendingNotifications addObject:state];notificationsRegistered++;[pendingLock unlock];[state start]; + } @finally {[transaction unlock];} +} +- (BOOL)waitUntilSignaledValue:(uint64_t)value timeoutMS:(uint64_t)milliseconds { + struct timespec start={},now={};if(clock_gettime(CLOCK_MONOTONIC,&start))fail(@"Actual monotonic clock unavailable"); + for(;;){if(self.signaledValue>=value)return YES;if(clock_gettime(CLOCK_MONOTONIC,&now))fail(@"Actual monotonic clock unavailable"); + uint64_t elapsed=(uint64_t)(now.tv_sec-start.tv_sec)*1000; + if(now.tv_nsec>=start.tv_nsec)elapsed+=(now.tv_nsec-start.tv_nsec)/1000000; + else elapsed-=(start.tv_nsec-now.tv_nsec+999999)/1000000; + if(elapsed>=milliseconds)return NO;usleep(1000);} +} +@end +@interface OMFunctionObject : OMObject +@property(nonatomic,strong) NSString *name; +@property(nonatomic) MTLFunctionType functionType; +@end +@implementation OMFunctionObject +@end +@interface MTLFunctionConstantValues (OMOriginalConstantAccess) +- (NSArray *)newIndexedConstantArray; +- (NSArray *)newNamedConstantArray; +@end +@protocol OMOriginalConstantValue +- (NSUInteger)dataType; +- (void *)data; +- (NSUInteger)index; +- (NSString *)name; +@end +static bool methodABI(id object,SEL selector,const char *encoding) { + Method method=class_getInstanceMethod(object_getClass(object),selector); + return method && !strcmp(method_getTypeEncoding(method),encoding); +} +static NSError *constantError(NSString *reason) { + return [NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 userInfo:@{NSLocalizedDescriptionKey:reason}]; +} +@interface OMLibraryObject : OMObject +@property(nonatomic,strong) NSArray *functionNames; +- (id)newFunctionWithName:(NSString *)name; +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error; +@end +@implementation OMLibraryObject +- (id)newFunctionWithName:(NSString *)name { + if(!name || [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128)return nil; + OMFunction request={};request.library=self.handle;strcpy(request.name,name.UTF8String); + OMResult result={};NSData *body=call(OM_FUNCTION,&request,sizeof(request),&result);if(result.error)return nil; + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![info isKindOfClass:[NSDictionary class]])fail(@"Actual function metadata missing"); + OMFunctionObject *object=[OMFunctionObject new];object.handle=result.handle;object.name=info[@"name"];object.functionType=(MTLFunctionType)[info[@"type"] unsignedIntegerValue]; + return (id)object; +} +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error { + auto refuse=[&](NSString *why)->id {if(error)*error=constantError(why);return nil;}; + if(!name.length || [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128 || + object_getClass(values)!=NSClassFromString(@"MTLFunctionConstantValuesInternal") || + !methodABI(values,@selector(newIndexedConstantArray),"@16@0:8") || !methodABI(values,@selector(newNamedConstantArray),"@16@0:8")) + return refuse(@"Unqualified actual function-constant object or name"); + // Read through the original target's real accessors. Never replace setters, + // interpret opaque storage, or substitute unspecialized shader functions. + MTLFunctionConstantValues *snapshot=[values copy]; + NSArray *indexed=[snapshot newIndexedConstantArray],*named=[snapshot newNamedConstantArray]; + if(![indexed isKindOfClass:[NSArray class]] || ![named isKindOfClass:[NSArray class]] || indexed.count+named.count>OFFLINE_METAL_MAX_CONSTANTS) + return refuse(@"Unqualified or excessive original constants"); + OMSpecializedFunction request={};request.function.library=self.handle;strcpy(request.function.name,name.UTF8String);request.count=indexed.count+named.count; + NSMutableData *packet=[NSMutableData dataWithBytes:&request length:sizeof(request)]; + for(unsigned group=0;group<2;group++)for(id value in (group?named:indexed)) { + if(object_getClass(value)!=NSClassFromString(group?@"MTLNamedConstantValue":@"MTLIndexedConstantValue") || + !methodABI(value,@selector(dataType),"Q16@0:8") || !methodABI(value,@selector(data),"^v16@0:8") || + !methodABI(value,group?@selector(name):@selector(index),group?"@16@0:8":"Q16@0:8")) + return refuse(@"Unqualified actual constant descriptor ABI"); + id descriptor=value; + OMFunctionConstant row={};row.type=[descriptor dataType];row.bytes=OMFunctionConstantBytes(row.type);row.named=group; + const void *data=[descriptor data];if(!row.bytes || !data)return refuse(@"Unsupported original constant type or missing actual value"); + if(group) { + NSString *key=[descriptor name];if(![key isKindOfClass:[NSString class]] || !key.length || [key lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128){ + return refuse(@"Invalid actual named constant");}strcpy(row.name,key.UTF8String); + } else {row.index=[descriptor index];if(row.index>65535)return refuse(@"Original constant index exceeds qualified range");} + memcpy(row.data,data,row.bytes);[packet appendBytes:&row length:sizeof(row)]; + } + OMResult result={};NSData *body=call(OM_FUNCTION,packet.bytes,packet.length,&result); + if(result.error){if(error)*error=operationError(result,body);return nil;} + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![info isKindOfClass:[NSDictionary class]])fail(@"Actual specialized function metadata missing"); + OMFunctionObject *object=[OMFunctionObject new];object.handle=result.handle;object.name=info[@"name"];object.functionType=(MTLFunctionType)[info[@"type"] unsignedIntegerValue]; + if(error)*error=nil;return (id)object; +} +@end +@interface OMTextureObject : OMObject +@property(nonatomic) OMTexture descriptor; +@property(nonatomic) MTLHazardTrackingMode actualHazardMode; +@property(nonatomic) IOSurfaceRef surface; +@property(nonatomic) uint32_t surfaceSeed; +@property(nonatomic) BOOL surfaceSeedValid; +- (void)uploadSurface; +- (void)downloadSurface; +- (void)getBytes:(void *)pixels bytesPerRow:(NSUInteger)stride fromRegion:(MTLRegion)region mipmapLevel:(NSUInteger)level; +@end +@implementation OMTextureObject +- (void)dealloc {if(_surface)CFRelease(_surface);} +- (IOSurfaceRef)iosurface {return self.surface;} +- (NSUInteger)iosurfacePlane {return 0;} +- (void)uploadSurface { + @synchronized(self) { + if(!self.surface)return;uint32_t seed=0; + if(IOSurfaceLock(self.surface,kIOSurfaceLockReadOnly,&seed))fail(@"Actual IOSurface read lock failed"); + @try { + if(self.surfaceSeedValid && seed==self.surfaceSeed)return; + const unsigned char *base=(const unsigned char *)IOSurfaceGetBaseAddress(self.surface);size_t row=IOSurfaceGetBytesPerRow(self.surface); + if(!base)fail(@"Missing actual IOSurface mapping"); + for(uint64_t y=0;ySIZE_MAX/4 + || strideSIZE_MAX/region.size.height)fail(@"Unsupported texture readback layout"); + OMReadTexture request={self.handle,region.origin.x,region.origin.y,region.size.width,region.size.height,level};OMResult result={}; + NSData *body=checked(OM_READ_TEXTURE,&request,sizeof(request),&result); + if(region.size.width>SIZE_MAX/4/region.size.height || body.length!=region.size.width*region.size.height*4)fail(@"Actual pixel extent differs"); + for(NSUInteger row=0;rowself.data.length || range.length>self.data.length-range.location)fail(@"Invalid native buffer modification range");} +- (void)upload { + for(NSUInteger offset=0;offset +- (id)texture; +@end +@interface OMCommandObject : OMObject +@property(nonatomic,strong) id commandQueue; +@property(atomic) OMCommandResult actualResult; +@property(nonatomic) BOOL useAsync; +@property(atomic) BOOL submissionStarted; +@property(atomic) BOOL resourcesVisible; +@property(atomic) pthread_t callbackThread; +@property(atomic,strong) NSException *asynchronousFailure; +@property(atomic,strong) NSException *deliveryFailure; +- (void)commitAsync; +@property(nonatomic) MTLCommandBufferStatus actualEnqueueStatus; +@property(nonatomic,strong) NSMutableArray *completedHandlers; +@property(nonatomic,strong) NSMutableArray *scheduledHandlers; +@property(atomic,strong) dispatch_group_t callbackGroup; +@property(atomic,strong) dispatch_group_t scheduledGroup; +@property(atomic,strong) dispatch_group_t visibilityGroup; +@property(atomic) BOOL scheduledObserved; +@property(atomic) MTLCommandBufferStatus actualScheduledStatus; +@property(nonatomic,strong) NSMutableArray *surfaceTargets; +@property(nonatomic,strong) NSMutableArray *buffers; +@property(nonatomic,strong) NSMutableArray *textures; +@property(nonatomic,strong) NSMutableArray> *presentations; +@property(nonatomic,strong) NSMutableArray *fences; +@property(nonatomic,strong) NSMutableArray *events; +- (void)trackFence:(id)fence; +- (void)trackBuffer:(OMBufferObject *)buffer; +- (void)trackTexture:(OMTextureObject *)texture; +- (void)trackSurfaceTarget:(OMTextureObject *)texture; +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)descriptor; +- (id)blitCommandEncoder; +- (void)commit; +- (void)waitUntilCompleted; +@end +@interface OMEncoderObject : OMObject +@property(nonatomic,strong) OMCommandObject *command; +@property(nonatomic,strong) NSMutableDictionary *vertexBuffers; +@property(nonatomic,strong) NSMutableDictionary *fragmentBuffers; +@end +@implementation OMEncoderObject +- (void)updateFence:(id)fence afterStages:(MTLRenderStages)stages { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign render fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],(uint64_t)stages};OMResult result={};checked(OM_RENDER_FENCE_UPDATE,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)waitForFence:(id)fence beforeStages:(MTLRenderStages)stages { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign render fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],(uint64_t)stages};OMResult result={};checked(OM_RENDER_FENCE_WAIT,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)setDepthStencilState:(id)state { + if(state && ![(id)state isKindOfClass:[OMDepthStateObject class]])fail(@"Foreign depth/stencil state"); + OMSetDepthStencil request={self.handle,state?[(OMObject *)state handle]:0};OMResult result={};checked(OM_SET_DEPTH_STENCIL,&request,sizeof(request),&result); +} +- (void)setStencilFrontReferenceValue:(uint32_t)front backReferenceValue:(uint32_t)back { + OMStencilReference request={self.handle,front,back};OMResult result={};checked(OM_STENCIL_REFERENCE,&request,sizeof(request),&result); +} +- (void)setStencilReferenceValue:(uint32_t)value {[self setStencilFrontReferenceValue:value backReferenceValue:value];} +- (void)setRenderPipelineState:(id)pipeline { + if(![(id)pipeline isKindOfClass:[OMPipelineObject class]])fail(@"Foreign pipeline object"); + OMSetPipeline request={self.handle,[(OMObject *)pipeline handle]};OMResult result={};checked(OM_SET_PIPELINE,&request,sizeof(request),&result); +} +- (void)setCullMode:(MTLCullMode)mode {OMSetCull request={self.handle,(uint64_t)mode};OMResult result={};checked(OM_SET_CULL,&request,sizeof(request),&result);} +- (void)setViewport:(MTLViewport)viewport { + OMViewport request={self.handle,viewport.originX,viewport.originY,viewport.width,viewport.height,viewport.znear,viewport.zfar};OMResult result={};checked(OM_VIEWPORT,&request,sizeof(request),&result); +} +- (void)setScissorRect:(MTLScissorRect)rect {OMScissor request={self.handle,rect.x,rect.y,rect.width,rect.height};OMResult result={};checked(OM_SCISSOR,&request,sizeof(request),&result);} +- (void)setBlendColorRed:(float)red green:(float)green blue:(float)blue alpha:(float)alpha { + OMBlendColor request={self.handle,red,green,blue,alpha};OMResult result={};checked(OM_BLEND_COLOR,&request,sizeof(request),&result); +} +- (void)setTexture:(id)texture index:(NSUInteger)index vertex:(BOOL)vertex { + if(texture && ![(id)texture isKindOfClass:[OMTextureObject class]])fail(@"Foreign texture object"); + OMSetTexture request={self.handle,texture?[(OMObject *)texture handle]:0,index};OMResult result={}; + checked(vertex?OM_VERTEX_TEXTURE:OM_FRAGMENT_TEXTURE,&request,sizeof(request),&result); + if(texture)[self.command trackTexture:(OMTextureObject *)texture]; +} +- (void)setVertexTexture:(id)texture atIndex:(NSUInteger)index {[self setTexture:texture index:index vertex:YES];} +- (void)setFragmentTexture:(id)texture atIndex:(NSUInteger)index {[self setTexture:texture index:index vertex:NO];} +- (void)drawPrimitives:(MTLPrimitiveType)type vertexStart:(NSUInteger)start vertexCount:(NSUInteger)count { + OMDraw request={self.handle,(uint64_t)type,start,count,1};OMResult result={};checked(OM_DRAW,&request,sizeof(request),&result); +} +- (void)setBuffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index vertex:(BOOL)vertex { + if(index>=31 || (buffer && ![(id)buffer isKindOfClass:[OMBufferObject class]]))fail(@"Foreign buffer object or invalid binding index"); + OMSetBuffer request={self.handle,buffer?[(OMObject *)buffer handle]:0,offset,index};OMResult result={}; + checked(vertex?OM_VERTEX_BUFFER:OM_FRAGMENT_BUFFER,&request,sizeof(request),&result); + if(vertex && !self.vertexBuffers)self.vertexBuffers=[NSMutableDictionary dictionary]; + if(!vertex && !self.fragmentBuffers)self.fragmentBuffers=[NSMutableDictionary dictionary]; + NSMutableDictionary *bindings=vertex?self.vertexBuffers:self.fragmentBuffers; + if(buffer)bindings[@(index)]=buffer;else [bindings removeObjectForKey:@(index)]; + if(buffer)[self.command trackBuffer:(OMBufferObject *)buffer]; +} +- (void)setVertexBuffer:(id)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index {[self setBuffer:buffer offset:offset index:index vertex:YES];} +- (void)setFragmentBuffer:(id)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index {[self setBuffer:buffer offset:offset index:index vertex:NO];} +- (void)setVertexBufferOffset:(NSUInteger)offset atIndex:(NSUInteger)index { + OMBufferObject *buffer=self.vertexBuffers[@(index)];if(!buffer)fail(@"No actual vertex buffer bound at this index"); + [self setVertexBuffer:(id)buffer offset:offset atIndex:index]; +} +- (void)setFragmentBufferOffset:(NSUInteger)offset atIndex:(NSUInteger)index { + OMBufferObject *buffer=self.fragmentBuffers[@(index)];if(!buffer)fail(@"No actual fragment buffer bound at this index"); + [self setFragmentBuffer:(id)buffer offset:offset atIndex:index]; +} +- (void)setVertexBytes:(const void *)bytes length:(NSUInteger)length atIndex:(NSUInteger)index { + id buffer=[remoteDevice newBufferWithBytes:bytes length:length options:MTLResourceStorageModeShared]; + if(!buffer)fail(@"Invalid vertex-byte allocation");[self setVertexBuffer:buffer offset:0 atIndex:index]; +} +- (void)setFragmentBytes:(const void *)bytes length:(NSUInteger)length atIndex:(NSUInteger)index { + id buffer=[remoteDevice newBufferWithBytes:bytes length:length options:MTLResourceStorageModeShared]; + if(!buffer)fail(@"Invalid fragment-byte allocation");[self setFragmentBuffer:buffer offset:0 atIndex:index]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset instanceCount:(NSUInteger)instances baseVertex:(NSInteger)baseVertex baseInstance:(NSUInteger)baseInstance { + if(![(id)buffer isKindOfClass:[OMBufferObject class]])fail(@"Foreign index buffer"); + OMDrawIndexed request={self.handle,(uint64_t)type,count,(uint64_t)indexType,[(OMObject *)buffer handle],offset,instances,baseVertex,baseInstance};OMResult result={}; + checked(OM_DRAW_INDEXED,&request,sizeof(request),&result);[self.command trackBuffer:(OMBufferObject *)buffer]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset { + [self drawIndexedPrimitives:type indexCount:count indexType:indexType indexBuffer:buffer indexBufferOffset:offset instanceCount:1 baseVertex:0 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset instanceCount:(NSUInteger)instances { + [self drawIndexedPrimitives:type indexCount:count indexType:indexType indexBuffer:buffer indexBufferOffset:offset instanceCount:instances baseVertex:0 baseInstance:0]; +} +- (void)endEncoding {OMHandle request={self.handle};OMResult result={};checked(OM_END,&request,sizeof(request),&result);} +@end +@interface OMBlitEncoderObject : OMEncoderObject +@end +@implementation OMBlitEncoderObject +- (void)updateFence:(id)fence { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign blit fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],0};OMResult result={};checked(OM_BLIT_FENCE_UPDATE,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)waitForFence:(id)fence { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign blit fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],0};OMResult result={};checked(OM_BLIT_FENCE_WAIT,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)copyFromTexture:(id)source sourceSlice:(NSUInteger)sourceSlice sourceLevel:(NSUInteger)sourceLevel sourceOrigin:(MTLOrigin)sourceOrigin sourceSize:(MTLSize)sourceSize + toTexture:(id)destination destinationSlice:(NSUInteger)destinationSlice destinationLevel:(NSUInteger)destinationLevel destinationOrigin:(MTLOrigin)destinationOrigin { + if(![(id)source isKindOfClass:[OMTextureObject class]] || ![(id)destination isKindOfClass:[OMTextureObject class]])fail(@"Foreign blit texture"); + OMCopyTexture request={self.handle,[(OMObject *)source handle],[(OMObject *)destination handle],sourceSlice,sourceLevel, + {sourceOrigin.x,sourceOrigin.y,sourceOrigin.z},{sourceSize.width,sourceSize.height,sourceSize.depth},destinationSlice,destinationLevel, + {destinationOrigin.x,destinationOrigin.y,destinationOrigin.z}}; + OMResult result={};checked(OM_COPY_TEXTURE,&request,sizeof(request),&result); + [self.command trackTexture:(OMTextureObject *)source];[self.command trackSurfaceTarget:(OMTextureObject *)destination]; +} +- (void)copyFromTexture:(id)source toTexture:(id)destination { + [self copyFromTexture:source sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(source.width,source.height,source.depth) + toTexture:destination destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; +} +- (void)copyFromBuffer:(id)source sourceOffset:(NSUInteger)sourceOffset toBuffer:(id)destination destinationOffset:(NSUInteger)destinationOffset size:(NSUInteger)size { + if(![(id)source isKindOfClass:[OMBufferObject class]] || ![(id)destination isKindOfClass:[OMBufferObject class]])fail(@"Foreign blit buffer"); + OMCopyBuffer request={self.handle,[(OMObject *)source handle],sourceOffset,[(OMObject *)destination handle],destinationOffset,size};OMResult result={}; + checked(OM_COPY_BUFFER,&request,sizeof(request),&result); + [self.command trackBuffer:(OMBufferObject *)source];[self.command trackBuffer:(OMBufferObject *)destination]; +} +@end +@implementation OMCommandObject +- (void)encodeSignalEvent:(id)event value:(uint64_t)value { + if(self.submissionStarted || self.actualResult.status || ![(id)event isKindOfClass:[OMEventObject class]])fail(@"Unqualified event or submitted command"); + OMEventDependency request={self.handle,[(OMObject *)event handle],value};OMResult result={};checked(OM_EVENT_SIGNAL,&request,sizeof(request),&result); + if(!self.events)self.events=[NSMutableArray array];if([self.events indexOfObjectIdenticalTo:event]==NSNotFound)[self.events addObject:(OMEventObject *)event]; +} +- (void)encodeWaitForEvent:(id)event value:(uint64_t)value { + if(self.submissionStarted || self.actualResult.status || ![(id)event isKindOfClass:[OMEventObject class]])fail(@"Unqualified event or submitted command"); + OMEventDependency request={self.handle,[(OMObject *)event handle],value};OMResult result={};checked(OM_EVENT_WAIT,&request,sizeof(request),&result); + if(!self.events)self.events=[NSMutableArray array];if([self.events indexOfObjectIdenticalTo:event]==NSNotFound)[self.events addObject:(OMEventObject *)event]; +} +- (void)commitAsync { + if(self.submissionStarted || self.actualResult.status)fail(@"Command already submitted"); + dispatch_group_t group=dispatch_group_create();dispatch_group_enter(group);self.callbackGroup=group; + self.scheduledGroup=dispatch_group_create();self.visibilityGroup=dispatch_group_create();dispatch_group_enter(self.scheduledGroup);dispatch_group_enter(self.visibilityGroup); + [pendingLock lock]; + if(closing || !live){[pendingLock unlock];dispatch_group_leave(self.scheduledGroup);dispatch_group_leave(self.visibilityGroup);dispatch_group_leave(group);fail(@"Context is closing");} + self.submissionStarted=YES;[pendingCommands addObject:self];[pendingLock unlock]; + @try { + for(OMBufferObject *buffer in self.buffers)[buffer upload]; + for(OMTextureObject *texture in self.textures)[texture uploadSurface]; + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMAND_SUBMIT,&request,sizeof(request),&result); + if(body.length!=sizeof(OMCommandResult))fail(@"Missing actual asynchronous submission");OMCommandResult initial={};memcpy(&initial,body.bytes,sizeof(initial)); + if(initial.statusMTLCommandBufferStatusCompleted || + (initial.status=MTLCommandBufferStatusScheduled){self.actualScheduledStatus=(MTLCommandBufferStatus)initial.status;self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + NSArray *scheduled=[self.scheduledHandlers copy],*completed=[self.completedHandlers copy],*presentations=[self.presentations copy]; + self.scheduledHandlers=nil;self.completedHandlers=nil;self.presentations=nil; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{@autoreleasepool { + @try { + OMCommandResult value=initial;unsigned polls=0; + while(value.status!=MTLCommandBufferStatusCompleted) { + if(++polls>4096)fail(@"Actual GPU completion observation exceeded its bound"); + OMResult observed={};NSData *reply=checked(OM_COMMAND_STATUS,&request,sizeof(request),&observed); + if(reply.length!=sizeof(value))fail(@"Missing actual GPU status");memcpy(&value,reply.bytes,sizeof(value)); + if(value.statusMTLCommandBufferStatusCompleted || + (value.status=MTLCommandBufferStatusScheduled && !self.scheduledObserved){self.actualScheduledStatus=(MTLCommandBufferStatus)value.status;self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + if(value.status!=MTLCommandBufferStatusCompleted){self.actualResult=value;usleep(1000);} + } + if(!value.start_ns || value.end_ns<=value.start_ns)fail(@"Missing actual completed GPU timing"); + for(OMTextureObject *texture in self.surfaceTargets)[texture downloadSurface]; + for(OMBufferObject *buffer in self.buffers)[buffer download]; + self.actualResult=value;self.resourcesVisible=YES; + @try {for(id drawable in presentations)[drawable present];} + @catch(NSException *exception){self.deliveryFailure=exception;} + } @catch(NSException *exception){self.asynchronousFailure=exception;} + if(!self.scheduledObserved){self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + dispatch_group_leave(self.visibilityGroup); + // Callbacks are deliberately late: real GPU completion and CPU visibility + // precede success delivery. A transport/GPU failure has a real error and + // never delivers a fabricated successful command or display timestamp. + self.callbackThread=pthread_self(); + @try { + if(self.resourcesVisible)for(MTLCommandBufferHandler handler in scheduled)handler((id)self); + for(MTLCommandBufferHandler handler in completed)handler((id)self); + } @catch(NSException *exception){self.deliveryFailure=exception;} + @finally { + self.callbackThread=nullptr;[pendingLock lock];[pendingCommands removeObject:self];[pendingLock unlock];dispatch_group_leave(group); + } + }}); + } @catch(NSException *exception) { + self.asynchronousFailure=exception;if(!self.scheduledObserved){self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);}dispatch_group_leave(self.visibilityGroup);[pendingLock lock];[pendingCommands removeObject:self];[pendingLock unlock];dispatch_group_leave(group);@throw; + } +} +- (void)trackFence:(id)fence { + if(!self.fences)self.fences=[NSMutableArray array];if([self.fences indexOfObjectIdenticalTo:fence]==NSNotFound)[self.fences addObject:(OMFenceObject *)fence]; +} +- (void)presentDrawable:(id)drawable { + if(self.submissionStarted || self.actualResult.status || !drawable || ![(id)drawable conformsToProtocol:@protocol(MTLDrawable)] || + !methodABI(drawable,@selector(texture),"@16@0:8") || !methodABI(drawable,@selector(present),"v16@0:8"))fail(@"Unqualified drawable or already completed command"); + id texture=[(id)drawable texture]; + if(![(id)texture isKindOfClass:[OMTextureObject class]] || texture.device!=remoteDevice || !texture.iosurface) + fail(@"Drawable requires this device's actual IOSurface texture"); + if(!self.presentations)self.presentations=[NSMutableArray array]; + if(self.presentations.count>=8 || [self.presentations indexOfObjectIdenticalTo:drawable]!=NSNotFound)fail(@"Duplicate or excessive actual drawable presentation"); + [self trackSurfaceTarget:(OMTextureObject *)texture];[self.presentations addObject:drawable]; +} +- (void)addCompletedHandler:(MTLCommandBufferHandler)handler { + if(!handler || self.submissionStarted || self.actualResult.status)fail(@"Completion handler requires a valid uncommitted command"); + if(!self.completedHandlers)self.completedHandlers=[NSMutableArray array];[self.completedHandlers addObject:[handler copy]]; +} +- (void)addScheduledHandler:(MTLCommandBufferHandler)handler { + if(!handler || self.submissionStarted || self.actualResult.status)fail(@"Scheduled handler requires a valid uncommitted command"); + if(!self.scheduledHandlers)self.scheduledHandlers=[NSMutableArray array];[self.scheduledHandlers addObject:[handler copy]]; +} +- (void)enqueue { + if(self.submissionStarted || self.actualResult.status || self.actualEnqueueStatus)fail(@"Command already enqueued or submitted"); + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMAND_ENQUEUE,&request,sizeof(request),&result); + if(body.length!=sizeof(uint64_t))fail(@"Missing actual enqueue status");uint64_t status=0;memcpy(&status,body.bytes,sizeof(status)); + if(status!=MTLCommandBufferStatusEnqueued)fail(@"Actual GPU command was not enqueued");self.actualEnqueueStatus=(MTLCommandBufferStatus)status; +} +- (void)trackBuffer:(OMBufferObject *)buffer { + if(!self.buffers)self.buffers=[NSMutableArray array];if(![self.buffers containsObject:buffer])[self.buffers addObject:buffer]; +} +- (void)trackTexture:(OMTextureObject *)texture { + if(!self.textures)self.textures=[NSMutableArray array];if(![self.textures containsObject:texture])[self.textures addObject:texture]; +} +- (void)trackSurfaceTarget:(OMTextureObject *)texture { + [self trackTexture:texture]; + if(texture.surface){if(!self.surfaceTargets)self.surfaceTargets=[NSMutableArray array];if(![self.surfaceTargets containsObject:texture])[self.surfaceTargets addObject:texture];} +} +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)descriptor { + if(!descriptor)fail(@"Missing render pass descriptor"); + OMAttachmentRender extended={};OMRender &request=extended.colors;request.command=self.handle; + for(unsigned i=0;i<8;i++) { + MTLRenderPassColorAttachmentDescriptor *a=descriptor.colorAttachments[i];if(!a.texture)continue; + if(![(id)a.texture isKindOfClass:[OMTextureObject class]] || a.level || a.slice || a.depthPlane || a.resolveTexture)fail(@"Unsupported or foreign render attachment"); + request.colors[i].texture=[(OMObject *)a.texture handle];request.colors[i].load=a.loadAction;request.colors[i].store=a.storeAction; + OMTextureObject *texture=(OMTextureObject *)a.texture; + [self trackSurfaceTarget:texture]; + if(texture.surface)[texture uploadSurface]; + MTLClearColor color=a.clearColor;request.colors[i].clear[0]=color.red;request.colors[i].clear[1]=color.green;request.colors[i].clear[2]=color.blue;request.colors[i].clear[3]=color.alpha; + } + MTLRenderPassDepthAttachmentDescriptor *depth=descriptor.depthAttachment; + if(depth.texture) { + if(![(id)depth.texture isKindOfClass:[OMTextureObject class]] || depth.level || depth.slice || depth.depthPlane || depth.resolveTexture)fail(@"Unsupported or foreign depth attachment"); + extended.depth={[(OMObject *)depth.texture handle],(uint64_t)depth.loadAction,(uint64_t)depth.storeAction,depth.clearDepth};[self trackTexture:(OMTextureObject *)depth.texture]; + } + MTLRenderPassStencilAttachmentDescriptor *stencil=descriptor.stencilAttachment; + if(stencil.texture) { + if(![(id)stencil.texture isKindOfClass:[OMTextureObject class]] || stencil.level || stencil.slice || stencil.depthPlane || stencil.resolveTexture)fail(@"Unsupported or foreign stencil attachment"); + extended.stencil={[(OMObject *)stencil.texture handle],(uint64_t)stencil.loadAction,(uint64_t)stencil.storeAction,stencil.clearStencil};[self trackTexture:(OMTextureObject *)stencil.texture]; + } + bool attachments=depth.texture || stencil.texture; + OMResult result={};checked(attachments?OM_RENDER_ATTACHMENTS:OM_RENDER,attachments?(const void *)&extended:(const void *)&request,attachments?sizeof(extended):sizeof(request),&result); + OMEncoderObject *object=[OMEncoderObject new];object.handle=result.handle;object.command=self;return (id)object; +} +- (id)blitCommandEncoder { + OMHandle request={self.handle};OMResult result={};checked(OM_BLIT,&request,sizeof(request),&result); + OMBlitEncoderObject *object=[OMBlitEncoderObject new];object.handle=result.handle;object.command=self;return (id)object; +} +- (void)commit { + if(self.useAsync){[self commitAsync];return;} + if(self.actualResult.status)fail(@"Command already submitted"); + for(OMBufferObject *buffer in self.buffers)[buffer upload]; + for(OMTextureObject *texture in self.textures)[texture uploadSurface]; + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMIT,&request,sizeof(request),&result); + if(body.length!=sizeof(OMCommandResult))fail(@"Missing actual command result");OMCommandResult value={};memcpy(&value,body.bytes,sizeof(value)); + if(value.status!=MTLCommandBufferStatusCompleted || !value.start_ns || value.end_ns<=value.start_ns)fail(@"Actual GPU command did not complete"); + for(OMTextureObject *texture in self.surfaceTargets)[texture downloadSurface]; + for(OMBufferObject *buffer in self.buffers)[buffer download]; + self.actualResult=value;self.resourcesVisible=YES; + // Present the real original drawable only after actual GPU completion and + // visibility in its original IOSurface. Its real framework owns presentation + // scheduling and timestamps; GPU completion is never a fabricated display + // completion. Preserve actual GPU status/callbacks even if presentation fails. + NSArray> *presentations=self.presentations;self.presentations=nil; + NSException *presentationFailure=nil; + @try {for(id drawable in presentations)[drawable present];} + @catch(NSException *exception){presentationFailure=exception;} + // This transport is synchronous: submission, execution and surface/buffer + // readback have all actually finished. Deliver each registered block exactly + // once on a real dispatch worker, with the real completed command. Completion + // waits also wait for that delivery, without invoking callbacks under the + // original renderer's encoding/commit call stack. + // Scheduled callbacks are deliberately late here; no asynchronous timing or + // fabricated Scheduled/Completed state is advertised. + NSArray *scheduled=self.scheduledHandlers,*completed=self.completedHandlers; + self.scheduledHandlers=nil;self.completedHandlers=nil; + dispatch_group_t group=nil; + if(scheduled.count || completed.count){group=dispatch_group_create();dispatch_group_enter(group);self.callbackGroup=group;} + if(group)dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{ + self.callbackThread=pthread_self(); + @try { + for(MTLCommandBufferHandler handler in scheduled)handler((id)self); + for(MTLCommandBufferHandler handler in completed)handler((id)self); + } @finally {self.callbackThread=nullptr;dispatch_group_leave(group);} + }); + if(presentationFailure)@throw presentationFailure; +} +- (void)waitUntilScheduled { + if(self.useAsync) { + if(!self.submissionStarted || !self.scheduledGroup)fail(@"Command has not actually been submitted"); + if(dispatch_group_wait(self.scheduledGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual scheduling observation timed out"); + if(self.asynchronousFailure)@throw self.asynchronousFailure; + if(self.actualScheduledStatus=MTLCommandBufferStatusCommitted && self.actualResult.status<=MTLCommandBufferStatusCompleted; + [self waitUntilScheduled];return self.actualResult.status==MTLCommandBufferStatusCompleted; +} +- (void)waitUntilCompleted { + if(self.useAsync) { + if(!self.submissionStarted || !self.callbackGroup)fail(@"Command has not actually been submitted"); + bool ownCallback=self.callbackThread && pthread_equal(self.callbackThread,pthread_self()); + if(!ownCallback && dispatch_group_wait(self.callbackGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual asynchronous completion/visibility/callback wait timed out"); + if(self.asynchronousFailure)@throw self.asynchronousFailure;if(self.deliveryFailure)@throw self.deliveryFailure; + if(!self.resourcesVisible || self.actualResult.status!=MTLCommandBufferStatusCompleted)fail(@"Actual GPU resources are not completed and visible");return; + } + if(self.actualResult.status!=MTLCommandBufferStatusCompleted)fail(@"Command has not actually completed"); + if(self.callbackGroup && !(self.callbackThread && pthread_equal(self.callbackThread,pthread_self())) && dispatch_group_wait(self.callbackGroup,dispatch_time(DISPATCH_TIME_NOW,10*NSEC_PER_SEC)))fail(@"Actual completion-handler delivery timed out"); +} +- (MTLCommandBufferStatus)status { + if(self.asynchronousFailure)return MTLCommandBufferStatusError; + OMCommandResult value=self.actualResult; + if(self.useAsync && value.status==MTLCommandBufferStatusCompleted && !self.resourcesVisible){if(dispatch_group_wait(self.visibilityGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual resource visibility timed out");if(self.asynchronousFailure)return MTLCommandBufferStatusError;value=self.actualResult;} + return value.status?(MTLCommandBufferStatus)value.status:self.actualEnqueueStatus; +} +- (NSError *)error { + if(self.asynchronousFailure)return [NSError errorWithDomain:@"org.blu-sharky.offline-metal.async" code:-1 userInfo:@{NSLocalizedDescriptionKey:self.asynchronousFailure.reason?:@"Actual asynchronous transport/GPU failure"}]; + return nil; +} +- (CFTimeInterval)GPUStartTime {return self.actualResult.start_ns/1e9;} +- (CFTimeInterval)GPUEndTime {return self.actualResult.end_ns/1e9;} +@end +@interface OMQueueObject : OMObject +@end +@implementation OMQueueObject +- (NSUInteger)readPriority:(BOOL)background { + OMQueuePriority request={self.handle,(uint64_t)background,0};OMResult result={};NSData *body=checked(OM_QUEUE_PRIORITY_GET,&request,sizeof(request),&result); + if(body.length!=sizeof(uint64_t))fail(@"Missing actual queue priority");uint64_t value=0;memcpy(&value,body.bytes,sizeof(value));return value; +} +- (BOOL)writePriority:(NSUInteger)priority background:(BOOL)background { + // Exact original MTLIOAccelCommandQueue validation: 0..5, except 3 in foreground. + if(priority>5 || (!background && priority==3))return NO; + OMQueuePriority request={self.handle,(uint64_t)background,priority};OMResult result={};NSData *body=call(OM_QUEUE_PRIORITY_SET,&request,sizeof(request),&result); + if(result.error)return NO;if(body.length!=sizeof(uint64_t))fail(@"Missing actual queue priority-change result");uint64_t value=0;memcpy(&value,body.bytes,sizeof(value)); + if(value>1)fail(@"Invalid actual priority-change result");return value!=0; +} +- (NSUInteger)getGPUPriority {return [self readPriority:NO];} +- (NSUInteger)getBackgroundGPUPriority {return [self readPriority:YES];} +- (BOOL)setGPUPriority:(NSUInteger)priority {return [self writePriority:priority background:NO];} +- (BOOL)setBackgroundGPUPriority:(NSUInteger)priority {return [self writePriority:priority background:YES];} +- (id)commandBuffer { + OMHandle request={self.handle};OMResult result={};checked(OM_COMMAND,&request,sizeof(request),&result); + OMCommandObject *object=[OMCommandObject new];object.handle=result.handle;object.commandQueue=(id)self;object.useAsync=asyncEnabled;return (id)object; +} +@end +// This model contributes only own implementations to a dynamically allocated +// real _MTLDevice subclass. No original method implementation is overwritten. +@interface OMDeviceMethods : NSObject +@end +@implementation OMDeviceMethods +- (NSString *)name {return deviceInfo[@"name"];} +- (uint64_t)registryID {return [deviceInfo[@"registryID"] unsignedLongLongValue];} +- (NSUInteger)maxBufferLength {return [deviceInfo[@"maxBufferLength"] unsignedIntegerValue];} +- (BOOL)isLowPower {return [deviceInfo[@"lowPower"] boolValue];} +- (BOOL)isRemovable {return [deviceInfo[@"removable"] boolValue];} +- (BOOL)isHeadless {return [deviceInfo[@"headless"] boolValue];} +// This partial offline device does not implement any complete GPU family or +// feature set. The original base implementation needs Apple capability tables +// which this independent device neither owns nor populates. +- (BOOL)supportsFamily:(MTLGPUFamily)family {(void)family;return NO;} +- (BOOL)supportsFeatureSet:(MTLFeatureSet)featureSet {(void)featureSet;return NO;} +- (id)newEvent { + OMResult result={};call(OM_EVENT,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual GPU event identity");OMEventObject *object=[OMEventObject new];object.handle=result.handle;return (id)object; +} +- (id)newSharedEvent { + OMResult result={};NSData *body=call(OM_SHARED_EVENT,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual shared-event identity");NSDictionary *info=sharedMetadata(body); + OMSharedEventObject *object=[OMSharedEventObject new];object.handle=result.handle;object.deviceIsNil=[info[@"device_is_nil"] boolValue]; + // Initialize without writing metadata back to the actual event. + ((void (*)(id,SEL,id))class_getMethodImplementation([OMObject class],@selector(setLabel:)))(object,@selector(setLabel:),info[@"label"]==[NSNull null]?nil:info[@"label"]); + return object; +} +- (id)newSharedEventWithHandle:(MTLSharedEventHandle *)handle { + Class cls=NSClassFromString(@"MTLSharedEventHandle");if(!handle || object_getClass(handle)!=cls || class_getInstanceSize(cls)!=16)return nil; + mach_port_t port=handle.eventPort;if(!MACH_PORT_VALID(port))return nil; + [transaction lock];@try { + if(!live || closing)return nil;uint64_t input=port,output=0;unsigned count=1;unsigned char bytes[4096];size_t size=sizeof(bytes); + int status=ioMethod(connection,3,&input,1,nullptr,0,&output,&count,bytes,&size);if(status)return nil; + if(count!=1 || !output)fail(@"Missing actual imported shared event");NSDictionary *info=sharedMetadata([NSData dataWithBytes:bytes length:size]); + OMSharedEventObject *object=[OMSharedEventObject new];object.handle=output;object.deviceIsNil=[info[@"device_is_nil"] boolValue]; + ((void (*)(id,SEL,id))class_getMethodImplementation([OMObject class],@selector(setLabel:)))(object,@selector(setLabel:),info[@"label"]==[NSNull null]?nil:info[@"label"]); + return object; + } @finally {[transaction unlock];} +} +- (id)newFence { + OMResult result={};call(OM_FENCE,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual GPU fence identity");OMFenceObject *object=[OMFenceObject new];object.handle=result.handle;return (id)object; +} +- (id)newCommandQueue { + OMResult result={};checked(OM_QUEUE,nullptr,0,&result);OMQueueObject *object=[OMQueueObject new];object.handle=result.handle;return (id)object; +} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options { + if(!length || length>MIN((NSUInteger)OFFLINE_METAL_MAX_BUFFER,self.maxBufferLength) || options)return nil; + OMBuffer request={length,(uint64_t)options};OMResult result={};call(OM_BUFFER,&request,sizeof(request),&result);if(result.error)return nil; + OMBufferObject *object=[OMBufferObject new];object.handle=result.handle;object.data=[NSMutableData dataWithLength:length];return (id)object; +} +- (id)newBufferWithBytes:(const void *)bytes length:(NSUInteger)length options:(MTLResourceOptions)options { + if(!bytes)return nil;OMBufferObject *object=(OMBufferObject *)[self newBufferWithLength:length options:options];if(!object)return nil; + memcpy(object.data.mutableBytes,bytes,length);return (id)object; +} +- (id)newPipelineLibraryWithFilePath:(NSString *)path error:(NSError **)error { + // This optional original directory contains Apple-family pipeline entries. + // No backend qualification exists for that private archive format. Refuse + // explicitly so the unchanged caller can take its real AIR-library path, + // rather than invoking the Apple base implementation with absent internals. + (void)path; + if(error){*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 + userInfo:@{NSLocalizedDescriptionKey:@"Apple-family pipeline libraries are not qualified for this independent device; use portable AIR input"}];} + return nil; +} +- (id)newLibraryWithFile:(NSString *)path error:(NSError **)error { + NSError *failure=nil;NSData *bytes=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:&failure]:nil; + if(!bytes){if(error)*error=failure?:[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"A readable actual library file is required"}];return nil;} + dispatch_data_t data=dispatch_data_create(bytes.bytes,bytes.length,dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{(void)bytes.length;}); + return [(id)self newLibraryWithData:data error:error]; +} +- (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"An actual local library file URL is required"}];return nil;} + return [(id)self newLibraryWithFile:url.path error:error]; +} +- (id)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(NSError **)error { + NSURL *url=[bundle URLForResource:@"default" withExtension:@"metallib"]; + if(!url){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"The actual bundle has no default Metal library"}];return nil;} + return [(id)self newLibraryWithURL:url error:error]; +} +- (id)newLibraryWithData:(dispatch_data_t)data error:(NSError **)error { + const void *bytes=nullptr;size_t size=0;dispatch_data_t mapped=dispatch_data_create_map(data,&bytes,&size);(void)mapped; + if(!bytes || !size || size>64*1024*1024){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:nil];return nil;} + OMLibrary request={};CC_SHA256(bytes,(CC_LONG)size,request.sha256);OMResult result={};NSData *body=call(OM_LIBRARY,&request,sizeof(request),&result); + if(result.error){if(error)*error=operationError(result,body);return nil;} + NSArray *names=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![names isKindOfClass:[NSArray class]])fail(@"Missing actual library metadata"); + OMLibraryObject *object=[OMLibraryObject new];object.handle=result.handle;object.functionNames=names;if(error)*error=nil;return (id)object; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d { + if(!d || d.hazardTrackingMode>MTLHazardTrackingModeTracked || !d.allowGPUOptimizedContents)return nil; + OMTexture request={(uint64_t)d.textureType,(uint64_t)d.pixelFormat,d.width,d.height,d.depth,d.mipmapLevelCount,d.sampleCount,d.arrayLength,(uint64_t)d.usage,(uint64_t)d.storageMode,(uint64_t)d.cpuCacheMode}; + OMHazardTexture extended={request,(uint64_t)d.hazardTrackingMode};OMResult result={};NSData *body=call(OM_TEXTURE_HAZARD,&extended,sizeof(extended),&result);if(result.error)return nil; + if(body.length!=sizeof(uint64_t))fail(@"Missing actual texture hazard mode");uint64_t actual=0;memcpy(&actual,body.bytes,sizeof(actual)); + if(actual>MTLHazardTrackingModeTracked || (extended.hazard && actual!=extended.hazard))fail(@"Actual texture hazard mode differs"); + OMTextureObject *object=[OMTextureObject new];object.handle=result.handle;object.descriptor=request;object.actualHazardMode=(MTLHazardTrackingMode)actual;return (id)object; +} +- (id)newDepthStencilStateWithDescriptor:(MTLDepthStencilDescriptor *)d { + if(!d)return nil; + auto face=[](MTLStencilDescriptor *s)->OMStencilFace { + if(!s)return {};return {1,(uint64_t)s.stencilCompareFunction,(uint64_t)s.stencilFailureOperation,(uint64_t)s.depthFailureOperation, + (uint64_t)s.depthStencilPassOperation,s.readMask,s.writeMask}; + }; + OMDepthStencil request={(uint64_t)d.depthCompareFunction,(uint64_t)d.depthWriteEnabled,face(d.frontFaceStencil),face(d.backFaceStencil)}; + OMResult result={};call(OM_DEPTH_STENCIL_STATE,&request,sizeof(request),&result);if(result.error)return nil; + OMDepthStateObject *object=[OMDepthStateObject new];object.handle=result.handle;return (id)object; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d iosurface:(IOSurfaceRef)surface plane:(NSUInteger)plane { + if(!surface || CFGetTypeID(surface)!=IOSurfaceGetTypeID() || !d || plane || IOSurfaceGetPlaneCount(surface) || d.storageMode!=MTLStorageModeShared)return nil; + bool format=((d.pixelFormat==MTLPixelFormatR32Float && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_OneComponent32Float) + || (d.pixelFormat==MTLPixelFormatRGBA8Unorm && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_32RGBA) + || (d.pixelFormat==MTLPixelFormatBGRA8Unorm && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_32BGRA)); + if(!format + || !d.width || !d.height || d.width!=IOSurfaceGetWidth(surface) || d.height!=IOSurfaceGetHeight(surface) + || d.width>SIZE_MAX/4 || IOSurfaceGetBytesPerElement(surface)!=4 || IOSurfaceGetBytesPerRow(surface)SIZE_MAX/d.height || IOSurfaceGetAllocSize(surface))object; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + static_assert(MTLVertexFormatFloat4==31 && MTLVertexFormatHalf4==27 && MTLVertexStepFunctionPerInstance==2,"Actual target SDK vertex enum ABI"); + bool vertexInputs=false;OMVertexPipeline vertexPipeline={}; + // Read real original descriptors. Inactive attributes/layouts carry zeroes; + // static stepping is transported without modifying any original descriptor. + if(d.vertexDescriptor)for(unsigned i=0;i<31;i++) { + MTLVertexAttributeDescriptor *a=d.vertexDescriptor.attributes[i];if(a.format==MTLVertexFormatInvalid)continue; + vertexInputs=true;vertexPipeline.attributes[i]={(uint64_t)a.format,a.offset,a.bufferIndex}; + if(a.bufferIndex<31) { + MTLVertexBufferLayoutDescriptor *l=d.vertexDescriptor.layouts[a.bufferIndex]; + vertexPipeline.layouts[a.bufferIndex]={l.stride,(uint64_t)l.stepFunction,l.stepRate}; + } + } + if(![(id)d.vertexFunction isKindOfClass:[OMFunctionObject class]] || ![(id)d.fragmentFunction isKindOfClass:[OMFunctionObject class]] + || (vertexInputs && !OMVertexInputsValid(vertexPipeline)) || !d.rasterizationEnabled || d.alphaToCoverageEnabled || d.alphaToOneEnabled) { + if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 userInfo:nil];return nil; + } + OMAttachmentPipeline attachments={};OMBlendPipeline &extended=attachments.colors;OMPipeline &request=extended.pipeline;bool blending=false; + attachments.depthFormat=d.depthAttachmentPixelFormat;attachments.stencilFormat=d.stencilAttachmentPixelFormat; + request.vertex=[(OMObject *)d.vertexFunction handle];request.fragment=[(OMObject *)d.fragmentFunction handle];request.samples=d.rasterSampleCount; + for(unsigned i=0;i<8;i++) { + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];request.formats[i]=a.pixelFormat;request.masks[i]=a.writeMask; + blending|=a.blendingEnabled;extended.colors[i]={(uint64_t)a.blendingEnabled,(uint64_t)a.sourceRGBBlendFactor,(uint64_t)a.destinationRGBBlendFactor, + (uint64_t)a.rgbBlendOperation,(uint64_t)a.sourceAlphaBlendFactor,(uint64_t)a.destinationAlphaBlendFactor,(uint64_t)a.alphaBlendOperation}; + } + bool attached=attachments.depthFormat || attachments.stencilFormat; + vertexPipeline.pipeline=attachments; + OMResult result={};NSData *body=call((vertexInputs || attached)?OM_PIPELINE_ATTACHMENTS:blending?OM_PIPELINE_BLEND:OM_PIPELINE, + vertexInputs?(const void *)&vertexPipeline:attached?(const void *)&attachments:blending?(const void *)&extended:(const void *)&request, + vertexInputs?sizeof(vertexPipeline):attached?sizeof(attachments):blending?sizeof(extended):sizeof(request),&result);if(result.error){if(error)*error=operationError(result,body);return nil;} + OMPipelineObject *object=[OMPipelineObject new];object.handle=result.handle;return (id)object; +} +@end +extern "C" id offline_metal_frontend_open(void); +extern "C" id offline_metal_frontend_open_service(const char *); +extern "C" void offline_metal_frontend_finish(void); +id offline_metal_frontend_open(void) { + return offline_metal_frontend_open_service("NativeVirtioConsole"); +} +id offline_metal_frontend_open_service(const char *serviceName) { + if(!serviceName || (strcmp(serviceName,"NativeVirtioConsole") && strcmp(serviceName,"NativeOfflineGPU")))return nil; + if(live)return nil;void *iokit=dlopen("/System/Library/Frameworks/IOKit.framework/IOKit",RTLD_NOW|RTLD_LOCAL);if(!iokit)return nil; + auto matching=(void *(*)(const char *))dlsym(iokit,"IOServiceMatching");auto services=(int (*)(unsigned,void *,unsigned *))dlsym(iokit,"IOServiceGetMatchingServices"); + auto next=(unsigned (*)(unsigned))dlsym(iokit,"IOIteratorNext");auto release=(int (*)(unsigned))dlsym(iokit,"IOObjectRelease");auto open=(int (*)(unsigned,unsigned,unsigned,unsigned *))dlsym(iokit,"IOServiceOpen"); + ioMethod=(IOMethod)dlsym(iokit,"IOConnectCallMethod");if(!matching || !services || !next || !release || !open || !ioMethod)return nil; + unsigned iterator=0;if(services(0,matching(serviceName),&iterator) || !iterator)return nil;unsigned service=next(iterator); + unsigned extra=next(iterator);int openStatus=-1; + if(service && !extra)openStatus=open(service,mach_task_self(),0,&connection); + if(extra)release(extra);if(service)release(service);release(iterator);if(openStatus)return nil; + transaction=[NSRecursiveLock new];pendingLock=[NSLock new];pendingCommands=[NSMutableSet set];pendingNotifications=[NSMutableSet set];notificationFailure=nil;asyncEnabled=false;closing=false;live=true;OMResult result={};NSData *body=checked(OM_DEVICE,nullptr,0,&result); + deviceInfo=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![deviceInfo isKindOfClass:[NSDictionary class]])fail(@"Actual device metadata missing"); + Class base=objc_getClass("_MTLDevice");if(!base || class_getInstanceSize(base)!=536)fail(@"Wrong original Metal base ABI"); + Class cls=objc_allocateClassPair(base,"NativeOfflineMetalDevice",0);if(!cls)fail(@"Own device class allocation failed"); + unsigned count=0;Method *methods=class_copyMethodList([OMDeviceMethods class],&count); + for(unsigned i=0;i=1 && format<=12)return 2+(format-1)%3; + if(format>=13 && format<=24)return 2*(2+(format-13)%3); + if(format>=25 && format<=27)return 2*(format-23); + if(format>=28 && format<=39)return 4*(1+(format-28)%4); + if(format>=40 && format<=42)return 4; + if(format>=45 && format<=48)return 1; + if(format>=49 && format<=53)return 2; + if(format==54 || format==55)return 4; + return 0; +} +static inline bool OMVertexInputsValid(const OMVertexPipeline &p) { + bool used[31]={};unsigned count=0; + for(const auto &a:p.attributes) { + if(!a.format){if(a.offset || a.buffer)return false;continue;} + unsigned bytes=OMVertexFormatBytes(a.format); + if(!bytes || a.buffer>=31 || a.offset>4096-bytes)return false; + used[a.buffer]=true;count++; + } + if(!count)return false; + for(unsigned i=0;i<31;i++) { + const auto &l=p.layouts[i]; + if(!used[i]){if(l.stride || l.step || l.rate)return false;continue;} + if(l.stride>OFFLINE_METAL_MAX_VERTEX_STRIDE || l.step>2 || l.rate>65535 || (l.step && (!l.rate || !l.stride)))return false; + } + return true; +} diff --git a/OfflineMetalWire.h b/OfflineMetalWire.h new file mode 100644 index 0000000..90f644e --- /dev/null +++ b/OfflineMetalWire.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT +// Offline Metal command transport. No deployment GPU or page-table ABI. +#pragma once +#include +enum { OM_WIRE_MAGIC=0x5550474e, OM_WIRE_RESPONSE=0x8000 }; +typedef struct {uint32_t magic;uint16_t version,operation;uint32_t bytes,request;} OMWireHeader; +static_assert(sizeof(OMWireHeader)==16,"Native Metal wire header"); +enum { OFFLINE_METAL_VERSION=2, OFFLINE_METAL_MAX_REPLY=16*1024*1024, + OM_DEVICE=0x100,OM_QUEUE,OM_LIBRARY,OM_FUNCTION,OM_TEXTURE,OM_PIPELINE, + OM_COMMAND,OM_RENDER,OM_SET_PIPELINE,OM_SET_CULL,OM_DRAW,OM_END, + OM_COMMIT,OM_READ_TEXTURE,OM_RELEASE,OM_FINISH,OM_WRITE_TEXTURE, + OM_BUFFER,OM_WRITE_BUFFER,OM_READ_BUFFER,OM_VERTEX_BUFFER,OM_FRAGMENT_BUFFER, + OM_DRAW_INDEXED,OM_BLIT,OM_COPY_BUFFER,OM_PIPELINE_BLEND,OM_VERTEX_TEXTURE,OM_FRAGMENT_TEXTURE, + OM_SCISSOR,OM_VIEWPORT,OM_BLEND_COLOR }; +enum { OFFLINE_METAL_MAX_BUFFER=64*1024*1024 }; +typedef struct {int32_t error;uint32_t reserved;uint64_t handle;} OMResult; +typedef struct {uint64_t handle;} OMHandle; +typedef struct {uint8_t sha256[32];} OMLibrary; +typedef struct {uint64_t library;char name[128];} OMFunction; +typedef struct {uint64_t type,format,width,height,depth,levels,samples,array,usage,storage,cache;} OMTexture; +typedef struct {uint64_t vertex,fragment,samples,formats[8],masks[8];} OMPipeline; +typedef struct {uint64_t texture,load,store;double clear[4];} OMColor; +typedef struct {uint64_t command;OMColor colors[8];} OMRender; +typedef struct {uint64_t encoder,pipeline;} OMSetPipeline; +typedef struct {uint64_t encoder,mode;} OMSetCull; +typedef struct {uint64_t encoder,primitive,start,count,instances;} OMDraw; +typedef struct {uint64_t status,start_ns,end_ns;} OMCommandResult; +typedef struct {uint64_t texture,x,y,width,height,level;} OMReadTexture; +typedef struct {uint64_t texture,x,y,width,height,level,stride;} OMWriteTexture; +typedef struct {uint64_t bytes,options;} OMBuffer; +typedef struct {uint64_t buffer,offset,bytes;} OMBufferRegion; +typedef struct {uint64_t encoder,buffer,offset,index;} OMSetBuffer; +typedef struct {uint64_t encoder,primitive,count,type,buffer,offset,instances;int64_t baseVertex;uint64_t baseInstance;} OMDrawIndexed; +typedef struct {uint64_t encoder,source,sourceOffset,destination,destinationOffset,bytes;} OMCopyBuffer; +typedef struct {uint64_t enabled,sourceRGB,destinationRGB,rgbOperation,sourceAlpha,destinationAlpha,alphaOperation;} OMBlend; +typedef struct {OMPipeline pipeline;OMBlend colors[8];} OMBlendPipeline; +typedef struct {uint64_t encoder,texture,index;} OMSetTexture; +typedef struct {uint64_t encoder,x,y,width,height;} OMScissor; +typedef struct {uint64_t encoder;double x,y,width,height,nearDepth,farDepth;} OMViewport; +typedef struct {uint64_t encoder;float red,green,blue,alpha;} OMBlendColor; diff --git a/OriginalAcceleratorABI.h b/OriginalAcceleratorABI.h new file mode 100644 index 0000000..a052b09 --- /dev/null +++ b/OriginalAcceleratorABI.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +#pragma once +// Exact supplied iOS17 base, independently audited against its complete +// original vtable and protected constructor/destructor bodies. No extra fields +// or operational overrides beyond IOService. This declares a real ancestry; +// no original class, method, vtable or metaclass is replaced. +#include +class IOAcceleratorES : public IOService { + OSDeclareAbstractStructors(IOAcceleratorES) +}; +static_assert(sizeof(IOService)==136 && sizeof(IOAcceleratorES)==136,"Qualified original accelerator base extent"); diff --git a/OriginalGPUProviderABI.h b/OriginalGPUProviderABI.h new file mode 100644 index 0000000..78ecf5b --- /dev/null +++ b/OriginalGPUProviderABI.h @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +#pragma once +// Unpublished ABI instrument only. The native service prefix and opaque storage +// match the exact original abstract base. Non-pure tail entries are bound to +// their ORIGINAL entry addresses, with no C++ caller supplied for those opaque +// source signatures. Required tail probes capture their actual register frame; +// descriptor slot170 may return NULL only for an explicit missing-resource +// control. No unknown signature, capability or successful +// operational implementation is claimed. This header is not a system driver API. +#include "OriginalAcceleratorABI.h" +class IOMemoryDescriptor; +class IOGPU : public IOAcceleratorES { + OSDeclareAbstractStructors(IOGPU) + unsigned char originalPrivateState[600]; +protected: + void free() override; +public: + IOReturn configureReport(IOReportChannelList *,IOReportConfigureAction,void *,void *) override; + IOReturn updateReport(IOReportChannelList *,IOReportUpdateAction,void *,void *) override; + bool start(IOService *) override; + void stop(IOService *) override; + IOWorkLoop *getWorkLoop() const override; + using IOAcceleratorES::newUserClient; + IOReturn newUserClient(task_t,void *,UInt32,IOUserClient **) override; +protected: + virtual void abiSlot168(); + virtual void abiSlot169(); + virtual IOMemoryDescriptor *abiSlot170(UInt32 *) = 0; + virtual void abiSlot171(); + virtual void abiSlot172(); + virtual void abiSlot173(); + virtual void abiSlot174() = 0; + virtual void abiSlot175(); + virtual void abiSlot176(); + virtual void abiSlot177(); + virtual void abiSlot178(); + virtual void abiSlot179(); + virtual void abiSlot180(); + virtual void abiSlot181(); + virtual void abiSlot182(); + virtual void abiSlot183(); + virtual void abiSlot184(); + virtual void abiSlot185(); + virtual void abiSlot186(); + virtual void abiSlot187(); + virtual void abiSlot188(); + virtual void abiSlot189() = 0; + virtual void abiSlot190(); + virtual void abiSlot191(); + virtual void abiSlot192(void *) = 0; + virtual void abiSlot193(); + virtual void abiSlot194(); + virtual bool abiSlot195(IOService *) = 0; + virtual void abiSlot196() = 0; + virtual void abiSlot197() = 0; + virtual void abiSlot198(); + virtual void abiSlot199(); + virtual void abiSlot200() = 0; + virtual void abiSlot201(); + virtual void abiSlot202() = 0; + virtual void abiSlot203(); + virtual void abiSlot204(); + virtual void abiSlot205() = 0; + virtual void abiSlot206(); + virtual void abiSlot207(); + virtual void abiSlot208(); + virtual void abiSlot209(); + virtual void abiSlot210(); + virtual void abiSlot211(); + virtual void abiSlot212(); + virtual void abiSlot213(); + virtual void abiSlot214(); +}; +static_assert(sizeof(IOGPU)==736,"Exact original provider storage extent"); diff --git a/README.md b/README.md new file mode 100644 index 0000000..e41fd1d --- /dev/null +++ b/README.md @@ -0,0 +1,1074 @@ +# NativeMetal + +## Scheduling and completion separation (offline, 2026-10-20) + +The native frontend now forwards scheduled callbacks and `waitUntilScheduled` +to the runtime's actual scheduling transition. Previously it called scheduled +handlers from completion and implemented the scheduling wait as a completion +wait. That also serialized `commitAndWaitUntilSubmitted` with GPU completion. +Completed callbacks keep their existing completion-queue behavior; scheduled +callbacks finish before the runtime advances to completion. Frontend encoding +refusals and commits refused by the runtime unblock waits without announcing a +scheduling event. A refused commit releases its callback ownership reference. + +`bridge/scheduled_host.mm` uses a bounded semaphore to hold a completed handler: +the scheduling wait must return independently, and the scheduled handler must +observe `Scheduled`, before completion. It covers the runtime thread, an explicit +serial completion queue, a frontend encoding refusal, and an open-encoder commit +refusal. The old frontend fails the successful-command case. Telemetry adds +separate scheduling/completion wait counts and elapsed times (`J` rows). + +The host qualification and ARM64/iOS17 cross-build use metal2vulkan revision +`720ef1ce9ae2` with the umbrella's recorded patches. These checks +qualify API ordering over a host Vulkan driver, not Adreno timing, system UI +frame rate, panel presentation, or cold-start shader coverage. This is a +confirmed serialization defect, not proof of the sole cause of hardware lag. + +The render pipeline creation entry points also forward descriptor +`binaryArchives` and the raw `MTLPipelineOption` bits to the runtime. Previously +the archive could be populated/serialized but creation always used default +options, silently compiling even when a listed empty archive should fail under +`FailOnBinaryArchiveMiss`. Foreign archive objects are refused. The C ABI keeps +the original default entry point and adds a counted archive list for creation. +`bridge/archive_host.mm` proves empty/changed-descriptor strict misses, then +serializes and reloads an archive in a second process with an independent fresh +shader-cache directory, and checks the rendered pixels over the host GPU. + +These archives contain the translator's artifacts, not Adreno machine code. +The runtime does not consult them for later draw variants, and Vulkan driver +pipeline compilation still requires its separate, identity-matched cache. +This change repairs render archive consumption; it does not seed the system +image or qualify complete CoreAnimation shader/variant coverage or a cold-frame +latency bound. Compute descriptor archive consumption remains separate work. + +Timing schema v2 attributes encoder API counts to the command buffer that +received them, rather than subtracting global counts at commit. Commit indices +are assigned under the ring lock so simultaneous commits cannot reuse an index. +The historical `gpu=` field is now `ctc=` (CPU commit-to-completion-callback). +The latest four buffers also report creation-to-commit age, which includes +caller idle time, and commit-to-scheduled-callback time (`-1` if not observed). +Pipeline/library/function and Vulkan/cache totals occupy separate rows to fit +the 90-character transport limit. The worst-case table stays within 48 rows. + +`bridge/telemetry_host.mm` interleaves render/blit encoding on two queues and +commits in reverse order, then checks both the attributed work and actual host +GPU pixels. The previous global-delta implementation fails this test. Counts +describe API requests, not proof that the GPU executed every requested command. + +## Resource-specific completion waits (offline, 2026-10-20) + +The resource SPI `waitUntilComplete` no longer drains every in-flight command +buffer. It asks the runtime whether an already-committed buffer references the +resource's storage. Texture views resolve to their root or backing buffer; +heap resources conservatively share a dependency, and imported host ranges are +compared for overlap without reading their bytes. An unknown bridge result +preserves the conservative wait. The frontend has no live remap/rebind API; +this query is not a remapping lifetime protocol. Completion and actual GPU +results remain the runtime's. + +`bridge/patches/0002-resource-wait-storage-dependencies.patch` records the exact +runtime extension. The umbrella builder verifies the complete source diff for +both patches. This changes the runtime source digest and therefore the shader +cache build identity; artifacts must be produced for this exact build. + +The bounded `resource_wait_host.mm` test holds pending work behind an unsignaled +shared event. The previous frontend blocks even for a resource the work never +uses. The fixed frontend returns for that resource and keeps waiting for the +actual buffer, a buffer-backed texture view, nested shared/private texture +views, two imports of one IOSurface, heap storage and overlapping no-copy +buffers. It then signals the event and checks the actual host GPU bytes, +using GPU readback for private resources. J rows report resource wait time and +dependency checks/skips; the maximum table now occupies 47 of 48 rows. + +This removes a reproduced serialization defect. Static target-code callsites +do not establish its per-frame frequency, hardware speedup, stable 120 FPS or +the sole cause of lag. No hardware boot or timing qualification is implied. + +Native ARM64/iOS 17 Metal API frontend for the original iPadOS graphics stack. +The device is published through MTLAddDevice. The frontend translates AIR with +metal2vulkan and submits through Turnip and NativeAdreno. The desktop is +GPU-composited but slow; shader compilation is not fixed. + +`tools/MetalVulkanProbe.rs` is an explicit offline consumer of metal2vulkan's +public compiler API. The umbrella pins the upstream source and locked Cargo +dependencies. The probe selects `exact_answers()` and separately selects +exact sampling; it still exports all precision notes, resource requirements, +specialization and runtime dependencies. Those switches are not an assertion +of numerical equivalence. The executable links no Vulkan loader and never +publishes a Metal device. It emits SPIR-V for independent validation, not +Adreno instructions or completion events. + +The accompanying vertex layout is an authored diagnostic interleaving of +float4/float2/float4 attributes at locations 0/1/3, with a 48-byte stride. +It is supplied explicitly by the caller, not inferred as a system runtime +layout. Missing descriptors/constants and unsupported stages remain errors. +Fragment compilation requires an explicit attachment map. The coverage +attachment fixture enables R32Float at attachment 2; silently attaching only +target 0 would let these coverage shaders optimize into empty fragments. +The umbrella records actual SPIR-V Output-variable stores and can require +them for selected fragment tests. A store instruction is still not proof of +shader execution or correct pixel values. +Target parsing, pipeline translation and SPIR-V validation require separate +evidence from shader execution, native GPU4K translation, submission/fences, +IOSurface sharing and display presentation. + +The partial device explicitly returns false for every GPU-family and feature-set +query. Its implemented subset does not qualify a complete family. This also +avoids the original base class dereferencing Apple capability tables which +this independent device does not initialize. The original RenderBox RBDevice +initializer, device getter and internal dispatch-queue getter now execute in +both actual original-runtime diagnostic workers; their subsequent original +IOSurface sharing and four real GPU completions still succeed. This qualifies +construction only, not original RenderBox rendering or system device discovery. + +Queue foreground/background priority queries and setters forward to the actual +host queue after validating its real Objective-C method ABI. No priority or +successful change is synthesized. Host qualification changes and restores a +real foreground priority, preserves a peer queue, and refuses foreign/wrong +objects and invalid arguments. Original RenderBox executes both real +priority queries in each native diagnostic worker. No RenderBox frame is qualified. +The broker explicitly allows the two new bounded operation IDs; its +native-4K candidate retains all 29 original executable component hashes. + +An interrupted backend session now retains its actual operation/completion +ledger with transport_healthy=false and session_finished=false. The caller +must return failure for such a report. A real disconnect control verifies +the incomplete report contains no fabricated command completion. + +Explicit command-buffer enqueue now reserves the real host Metal queue and +returns its actual Enqueued status. The synchronous wire permits one pending +reservation per queue; duplicate/foreign requests and unsupported predecessor +ordering refuse before entering a blocking host wait. Independent queues can +both reserve and execute. Host qualification completes four GPU commands and +verifies all returned pixels, including retrying a second reservation after +its real predecessor finishes. Actual original RenderBox enqueues both +native workers' real buffers. Pending reservations are never reported as GPU completions. + +Registered completion and scheduled handlers execute once on a genuine +dispatch worker after verified GPU completion and original surface/buffer +synchronization. waitUntilCompleted also waits for those handlers with a +bounded timeout. Scheduled callbacks are late in this synchronous transport; +no early asynchronous scheduling timing is claimed. The original target's +BOOL-returning commitAndWaitUntilSubmitted interface derives success only +after a real successful commit. The two-process surface regression exercises +both public/private commit paths, real enqueue status, ordered once-only +callbacks with actual completion/timestamps, shared-surface seed coherence and +post-peer-exit GPU work. All five native broker/permission controls also pass +against the same new candidate. Original RenderBox's convenience image path +can return an image with only empty command buffers; that image is explicitly +not qualified as GPU rendering. Actual renderer resource use/output must still +be established before normal device discovery or any desktop claim. + +The frontend dynamically derives from the supplied target's actual 536-byte +_MTLDevice class and adds only its own method implementations. Original +implementations are not overwritten. Actual original descriptor objects +are marshalled through bounded typed requests over NativeVirtioConsole's +administrator userclient. Host replies carry real object handles, GPU +completion/timing and texture bytes. Unsupported operations refuse. + +Implemented operations create queues, known original libraries, named +functions, simple render pipelines, shared R32Float/BGRA8Unorm/RGBA8Unorm 2D textures, command +buffers and render encoders. Pipeline/cull/draw/end/commit operations call +real host Metal APIs. Commit waits for actual GPU completion; texture reads +copy actual getBytes output. Proxy deallocation drops its real host table +reference, while actual Metal command buffers retain their own resources. +Session finish drops remaining references. There is no CPU pixel renderer. + +RGBA8Unorm now pairs with real original 32RGBA IOSurfaces as requested by +original RenderBox. Upload/readback retain actual channel bytes without CPU +swizzling. The native two-process sharing/callback regression uses RGBA: +four real GPU completions, five full readbacks, original seed coherence, +Mach authorization/refusal and post-peer-exit work pass with correct channel +order. The separate BGRA two-process/permission regression also passes on the +same source/candidate. Original RenderBox allocates both actual RGBA surface +textures before its next real private RG16Float resource refusal; no original +renderer frame is qualified by those allocations. + +Private RG16Float textures and actual bounded texture blits are now implemented. +Private textures never use CPU getBytes/replaceRegion; shared staging is filled +by a genuine Metal blit. The native two-worker resource instrument clears each +private texture and a contrasting shared destination on the GPU, copies between +them, and checks every half-float result. Invalid bounds and direct private CPU +reads refuse in both workers. Together with the original RGBA surface sequence, +six actual commands, twelve ordered callbacks and seven complete readbacks pass, +with distinct live native-4K worker CPU contexts and no pending reservations. +All five broker/permission controls pass on the fresh texture-copy candidate. +Blit destinations backed by original IOSurfaces join the real post-completion +readback/seed path; no CPU pixel conversion or private-memory mapping is used. +Private Stencil8 is now supported by the attachment/state extension below. + +Private Stencil8/Depth32Float resources, real depth/stencil states, reference +values and pipeline/render attachments are implemented. Both actual native +workers execute the unchanged original shader pair through stencil mismatch/ +match and depth reject/pass controls, checking every returned pixel. Together +with the RGBA surface sequence, twelve real commands, twenty-four ordered +callbacks and thirteen complete readbacks pass; private stencil CPU reads and +wrong-type state bindings refuse. All five current-candidate broker controls +pass. Combined Depth32Float_Stencil8 also passes the corresponding real host +GPU tests, but has no separate native combined-attachment qualification yet. +Original RenderBox now reaches its inherited optional Apple pipeline-library +loader and faults on a null internal object. No original renderer frame is +qualified. An interrupted active encoder also exposes a host teardown assertion; +both are retained explicit failures, to fix before progressing the renderer. + +Shared/default-cache buffers now have real host allocations and stable native +contents mirrors. Bound buffers are retained by their command; their actual +current bytes upload immediately before GPU commit, and actual buffer bytes +download after completion. Thus CPU writes after encoding remain visible to +the GPU. This is explicit copying, not shared physical GPU memory. Allocation +is bounded at 64 MiB and each diagnostic session at 8192 operations; arbitrary +large workloads/session combinations are not qualified. Other storage/options +refuse. Vertex/fragment bindings, inline byte bindings, indexed draws and +aligned nonoverlapping GPU buffer blits use actual corresponding Metal calls. + +The native resource/presentation instrument qualifies 84-byte original +RenderBox uniforms and 12-byte indices. A real GPU blit replaces a distinct +destination pattern, and that actual destination becomes the original +accumulator_gradient_vertex input. Indexed triangles produce the right-half +image, then a uniform update after encoding produces a full image. All pixels +reach actual GOP and are read back. Exit 163 and absent/default controls pass; +this does not establish complete Metal capabilities, system UI or Adreno. + +Original texture-read shaders now execute through real vertex/fragment +texture bindings. Commands retain referenced textures and recheck real +IOSurface generation seeds before commit. GPU blend pipelines carry actual +per-attachment factors/operations; viewport, bounded scissor and constant +blend color use real Metal state. Unsupported multisampling and invalid +clipping refuse before issuing the unsupported GPU operation. + +The native composition instrument creates its two source images on the GPU, +then uses unchanged read_surf_vert/read_surf_frag to copy and blend them. It +qualifies full blue, clipped premultiplied composition and constant-factor +composition through actual IOSurface and GOP, checking every pixel. Exit 167 +and both device/default controls pass. This is actual GPU composition, not a +CPU-generated image or the original system desktop. Filtered samplers, +function constants, complete resource/format/capability contracts, system +discovery and original framebuffer swap integration remain incomplete. + +Basic nonplanar R32Float guest IOSurfaces now synchronize with host textures. +BGRA8Unorm also supports the original framework's actual 32BGRA surfaces; +descriptor/surface format mismatches and unsupported planes refuse. A genuine +GPU clear and the unchanged original attachment_clear_frag_lpf shader operate +on this format, with full native surface/readback hashes and actual ordered +GPU timing. The BGRA instrument exits 151. The bounded host session allows +8192 operations so full display-size surface uploads can fit; individual +request/reply and texture extent limits remain unchanged. This does not add +system device discovery, display presentation or physical Adreno execution. +The frontend uses the original IOSurface framework's actual locks, mapped +base/pitch/allocation and generation seed. Bounded byte uploads preserve +existing native surface contents; actual completed GPU output is copied back +before the frontend reports completion. This is explicit offline transfer, +not shared guest/host GPU physical memory or deployment zero-copy. Unsupported +planes/formats/extents refuse. Command buffers retain their surface targets. + +The parent native IOSurface instrument actually creates a 64x64 surface, +exports/imports its real Mach port and verifies both references see the GPU +output. Initial float-two pixels upload unchanged; a real GPU clear returns +float-one to the surface, which becomes the next texture's actual upload. +An original shader draw then returns zeros through that same original surface. +The actual native exit is 149, with active CPU 4K and original AMFI/root +retained. Full IOSurface cross-process/presentation semantics remain open. + +Libraries use a bounded content-addressed cache of the exact original +QuartzCore and RenderBox files supplied to the host instrument. The guest +hashes complete actual newLibraryWithData input. Unknown identities refuse; +known input selects the already hash-qualified unchanged AIR slice of that +same original file. Arbitrary named functions are obtained from the real +host library. No function body or target metadata is rewritten. General +library uploads, specialization constants and source compilation remain open. + +The parent native instrument performs an actual clear and an original +QuartzCore shader draw using normal Metal API calls, checks every returned +pixel and GPU timing, and exits 119. Unknown library/function controls refuse. +Automatic original MTLCreateSystemDefaultDevice still returns nil: this +explicit partial object is not advertised as a completed system driver. +Active CPU translations remain native 4 KiB; original AMFI/root checks +remain active. No active GPU SMMU geometry is inferred from host Metal. + +Not yet: full Metal feature/device capabilities, general texture formats, +general render descriptors, cross-process resource sharing, normal +bundle/service packaging, events/fences, heaps, argument buffers, indirect +commands, drawables and presentation, or a native Adreno backend. +Keep binary inputs, extracted shader files and observation artifacts outside +this source repository. + +The separate version-3 offline session protocol adds independent real host +object tables and per-session request ordering. Host resource identifiers +are unique across sessions, but lookup/release/bind operations resolve only +inside the selected table. Closing one session drops only its independent +references; another session can still submit real GPU work and read its own +output. Existing version-2 framing remains available without multiplexing. +The bounded host instrument opens exactly two sessions and checks seven +actual cross-session, stale/unknown-session and replay refusals, three real +GPU completions and complete texture readbacks. Run the umbrella +check-host-metal-originals.py followed by check-host-metal-sessions.py. + +Session selection belongs to the owned kernel broker, never caller-supplied +Metal request bytes. The host-only checkpoint remains distinct from the +native runtime evidence below. It establishes no guest display, active +Adreno SMMU mapping, hardware execution or original desktop. + +NativeOfflineGPU.cpp is the explicit private kernel component of this +partial offline stack. It holds a real exclusive byte-transport open, +serializes each complete host request/reply and buffers replies separately +for each actual native IOUserClient. The kernel selects an independent host +namespace; public request bytes contain no selectable session identity. +Framing, ordering, reply bounds, unread-reply ownership and teardown have +real failure paths. A transport failure prevents reuse of a partial stream +by another client. Closing a namespace releases only its own actual object +references. This component registers only its private test class, not an +IOAcceleratorES service or a system Metal device. + +The umbrella native instrument actually spawns two original-runtime child +processes. Each uses the real Metal object/command frontend and a distinct +original IOSurface. Three actual M1 GPU submissions complete, all surface +pixels are checked, a foreign texture handle is refused, and one process +continues GPU execution after the peer exits. At each actual userclient +creation the broker records live TCR/TTBR0/TTBR1/SCTLR and the actual task/PID, +and refuses nonnative-4K CPU contexts. Both workers report a 4096-byte ABI; +their actual TTBR0/task identities differ. These are CPU observations, not +active native-4K Adreno SMMU qualification. + +Only the own signed test process has the exact service/IOKit permissions +required by the original Sandbox: no-container and an allowlist containing +NativeOfflineGPUClient and IOSurfaceRootUserClient. Removing either +permission genuinely refuses spawn or device access. Original Sandbox, +AMFI, code signatures and both original trust modules remain unchanged. +Run the umbrella build-offline-gpu-broker.py, Metal build/stage/probe helpers +with --broker-processes, and check-offline-metal-processes.py. Authorized +general concurrent IOSurface sharing/coherence, normal system discovery, +complete Metal/presentation contracts, the original desktop and native Adreno +execution remain open. Scoped synchronous sharing and explicit own bundle +packaging are separately qualified below. + +## Explicit own bundle factory and normal-path offline view + +`OfflineMetalPlugin.mm` packages this partial frontend as an own signed bundle. +Its real principal class derives from the target's original `_MTLDevice`. +`initWithAcceleratorPort:` rejects null and unrelated services, requires the +actual unique private NativeOfflineGPU registry entry, and opens that same +service through the real frontend. The returned device reports the actual +provided guest service registry ID; host metadata retains the separate backend +identity. No original class implementation is replaced. This factory still +uses the private diagnostic broker, not an IOAcceleratorES implementation. + +The umbrella addon instrument mounts an existing source-built XNU bindfs +read-only union view at `/System/Library/Extensions`. Only own signed code is +added; actual lower-volume AGX metadata remains accessible. Its backing is +mutable own RAM Data, so it is not an immutable authenticated production addon +volume. Original NSBundle loads the own bundle in two real original-runtime +children, accepts its principal class and invokes the factory with real ports. +Both children then perform genuine Mach-authorized original IOSurface sharing, +seed-driven refresh, four actual GPU completions and peer execution after the +creator exits. The own helper has only its scoped no-container/IOKit permissions +and the original bindfs-allow permission; original applications retain theirs. + +`python3 scripts/check-offline-metal-addon.py` in the umbrella checks this +checkpoint and two real controls. Identical own machine instructions lose +mount access without bindfs-allow. A one-byte change to the own factory's code, +retaining its original signature and explicitly trusted hash, triggers an +actual bad code-page count and cs_invalid_page before any GPU access. Both +original trust modules and every original executable remain unchanged. + +This is explicit offline factory/packaging qualification. It does not publish +a normal Metal device, invoke normal Metal registry discovery, qualify ordinary +application access, asynchronous shared fences or a desktop. Original default +Metal remains nil. The complete Adreno backend and its active native-4K SMMU +remain unqualified. + + +## Actual file libraries and interrupted encoders + +The explicit device loads actual local file/URL/default-bundle libraries through +its existing exact-input hash qualification. Successful loads clear stale errors; +unreadable files, non-file URLs and readable non-library inputs fail honestly. +The optional Apple-family pipeline-library directory is explicitly unsupported. +Unchanged original RenderBox handles that error and genuinely loads its original +portable AIR library, then requests actual function-constant specialization. That +next interface remains absent at this checkpoint; no renderer frame is claimed. + +The host tracks actual open render/blit encoders. Ending an encoder invalidates +further encoder calls; duplicate end, post-end draw/state/copy, concurrent encoder +creation and commit with active encoding refuse before host Metal APIs. On actual +session closure or transport loss, cleanup only ends abandoned encoding, without +submitting the interrupted command or manufacturing GPU completion. The complete +partial ledger survives with explicit failure. Host unfinished/ended/disconnect +controls and native two-worker library/depth/stencil regressions pass. These are +explicit offline results, not normal system discovery, desktop or Adreno proof. + + +## Original function constants and RenderBox GPU drawing + +The explicit frontend reads a copy of the actual iOS17 constant-values object +through its original typed array/data/type/index/name methods, with exact runtime +class and ABI checks. It transfers bounded original bytes to actual host Metal +specialization; it never substitutes an unspecialized function for that request. +Scalar and two/four-component types have explicit bounded sizes; matrices, +three-component alignment profiles and unsupported values refuse. The extension +uses a distinct bounded payload on the existing function operation. The host +checks exact lengths, counts, identities, types, sizes and duplicate keys before +calling real Metal, and records actual values and compiler outcomes. + +Both actual native workers now run the unchanged original RenderBox renderer, +which specializes original plane__vertex/plane_color_fragment for both shapes, +binds real original buffer offsets and executes GPU indexed draws. Every pixel +of each returned original image matches left-red/right-blue, independently of +the complete host GPU readback hashes. Original surface uploads are all zero, +so the shapes do not originate in CPU-painted texture data. Six real commands +complete including the separate four-command surface-sharing/lifetime sequence. +This qualifies the explicit original renderer path, not the system desktop. + +The native depth/stencil fixture additionally qualifies both original named and +indexed constant accessors, source-object preservation, stale-error clearing, +unknown-function refusal and unsupported-type refusal. An empty constants set +and its pipeline are genuinely accepted by this exact host backend; this does +not qualify unspecified shader-state semantics. Earlier incorrect negative-test +expectations and actual observations remain retained separately. The host-only +function-constants control compares indexed/named red output with white output +after one real UInt4 bit changes, verifies every pixel and checks nine genuine +argument/isolation/compiler refusals. Native buffer-offset methods retain the +actual previously bound resource and rebind its real offset through the existing +checked path. Complete normal discovery, asynchronous contracts, presentation, +the original desktop, Adreno execution and active native-4K GPU SMMU remain open. + + +## Unpublished original accelerator-base lifecycle + +OriginalAcceleratorABI.h declares only the exact supplied IOAcceleratorES base: +136 bytes and the existing 168-entry IOService table, with no new operational +methods. The umbrella audit derives its real protected-compatible constructor +aliases, destructor and metaclass from the unchanged original IOGPU component. +NativeOfflineAccelerator uses those genuine bodies and its own compiler-produced +type metadata. No original class/vtable/method is replaced. This object is never +attached, started or registered as a system service. + +The opt-in kernel lifecycle fixture actually allocates/initializes the own object, +confirms its actual original parent and rejects unrelated IOGPU/private-broker +ancestry, then releases it. Real instance counts go 0 -> 1 -> 0. Omitting original +IOGPUFamily load genuinely refuses before allocation; default-off creates no +object. The complete native table and exact base imports are independently audited. +Both controls share the same immutable candidate. This is early CPU-4K/kernel ABI +evidence before root acceptance or userland; no GPU client, backend response, +normal Metal publication, original desktop or Adreno execution is qualified. +Run the umbrella build-offline-gpu-broker.py --accelerator-abi and the dedicated +probe-accelerator-abi.py / check-accelerator-abi.py tools. The build option defaults +off. Complete normal device/client/resource/queue/surface/presentation contracts +remain required before original system discovery can publish this partial stack. + +## Original CAMetalLayer drawable present requests + +The explicit frontend now retains each actual drawable through standard +presentDrawable:, validates its protocol, method ABI, own-device IOSurface +texture and bounded identity, and refuses duplicate or completed submissions. +After real GPU completion and actual IOSurface downloads, commit invokes the +original drawable's present method. It preserves real GPU completion callbacks +if that original method raises. It creates no display timestamp or fake fence. + +The target's unchanged CAMetalLayer creates an actual CAMetalDrawable on this +explicit device. Its original present implementation remains at its original +cache offset. Two actual GPU commands produce the original RenderBox RGBA image +and convert it to the drawable's BGRA texture. Every one of 4096 pixels and both +independent full GPU readback hashes agree; all initial texture uploads are zero. +Invalid-object, duplicate and post-completion requests refuse, and both actual +command callbacks execute once. The actual first drawable ID and presentedTime +are zero. This detached layer performs no framebuffer swap or retirement and +qualifies a genuine present request only, not display completion. Time/duration +presentation variants, normal system discovery, original desktop, Adreno and +active native-4K GPU SMMU remain open. The umbrella --metal-layer fixture and +check-original-metal-layer.py retain exact source, signed-input and CPU-4K evidence. + +## Genuine GPU fences and explicit texture hazard modes + +OfflineMetalFenceWire.h extends the bounded transport with real fence creation, +render stage updates/waits and blit updates/waits. The frontend implements the +complete MTLFence object protocol, owns the actual device identity and retains +fence proxies through commands. Object labels use atomic copy semantics. The +backend invokes real Metal fence APIs on actual per-session objects; it never +marks a fence signaled in CPU state. It refuses foreign/released handles, wrong +or ended encoders, unsupported stages, an update-then-wait cycle in one pass, +missing producers and unsent cross-command dependencies. This synchronous subset +accepts an earlier ended producer pass in the same command or a genuinely +completed producer on another queue; asynchronous pending producers remain open. +Producer bookkeeping releases with the independent fence handle or namespace. + +The texture extension preserves old request layouts and explicitly accepts +Default/Tracked/Untracked descriptors. It queries and returns the actual backend +mode and refuses an explicit mode mismatch. Texture resourceOptions reflect the +actual storage/cache/hazard attributes. Default really resolves to Tracked on +the observed backend. Neither resource attributes nor host Metal fences qualify +Adreno residency, interrupts, fences or an active native-4K GPU SMMU context. + +The host-only check-host-metal-fences.py executes four explicitly Untracked +resources through real render -> blit -> original QuartzCore shader rendering, +then a real completed-producer dependency on another queue. Both full readbacks +match all 8192 pixels, with no texture uploads. Eight actual fence calls and two +GPU completions pass; twelve argument, namespace, lifetime and dependency controls +refuse. An encoded but unsubmitted producer/control is never submitted by cleanup. +The existing original function-specialization and ended-encoder regressions pass. + +The native --fences fixture uses an actual private Untracked GPU target, a real +render-update -> blit-wait/update -> render-wait chain and unchanged original +QuartzCore shaders. The private result is GPU-copied to the actual original +CAMetalDrawable, then GPU-sampled in its final render pass. Both full readback +hashes and all 4096 drawable pixels agree. Five native fence refusals, real +protocol/copy-label behavior and existing drawable/callback controls pass. +Normal present invokes the original implementation only after real completion +and IOSurface visibility. The detached layer still has presentedTime zero and +no framebuffer swap. This is explicit offline synchronization qualification, +not a system desktop, asynchronous general queue contract or native Adreno proof. + +## Asynchronous host submission and real cross-queue GPU events + +OfflineMetalCommandWire.h adds separate submit/status operations. Submit invokes +actual Metal commit and returns without waitUntilCompleted. Status reads the real +command state; completion times remain unavailable until genuine completion. +Completion records are unique even across repeated queries. Namespace close and +transport-loss cleanup wait for already submitted work and record its real GPU +result before releasing references. They never submit merely encoded controls. +Transport loss remains unhealthy/unfinished even when its actual GPU work completes. + +An initial pending-producer cross-queue MTLFence experiment returned an all-zero +copy while the original shader output was red; both buffers reported Completed. +The exact failed source, executable, GPU readback and timing ledger are retained. +This is a failed qualification, not successful asynchronous resource visibility. +Cross-queue fence waits therefore still require a genuinely completed producer. +Same-queue waits can consume actually committed producers. The supplied host +profile uses actual MTLEvent timeline signals/waits for overlapping queues. + +OfflineMetalEventWire.h creates genuine per-session GPU events and encodes real +command-buffer signals/waits. CPU state records only encoded producers; it never +reports a signaled value. Missing, foreign, unsubmitted, same-command, zero/future +and nonmonotonic dependencies refuse. No CPU fence or precomputed completion is +substituted. The current bounded profile requires the producer already committed +before encoding its consumer wait; future unsubmitted producers remain unsupported. + +check-host-metal-async.py --api-validation and its --disconnect-control execute +three genuinely nonblocking submissions, six fence calls and five GPU timeline +event calls on two actual queues. Initial real states are Committed; queries also +observe Scheduled and Completed with actual GPU times. Every one of 8192 pixels +matches after the original shader and cross-queue event dependency. Thirty-one +argument, namespace, state and lifetime controls refuse. An independently released +submitted command finishes during real cleanup; unsubmitted controls never gain +GPU submissions. The disconnect case exits with failure and keeps its truthful +incomplete transport ledger. Eight additional bounded API-validation runs pass +24 GPU commands and 65536 pixels, including eight waits on actually in-flight +producers and four disconnect cases, with no unaccounted validation messages. +The original synchronous fence regression also passes. + +That initial checkpoint was host-only. The separate opt-in native integration +below now qualifies its own visibility/callback/lifetime subset. It does not +qualify system discovery, original desktop, native Adreno events/fences or active +native-4K GPU SMMU state. + +## Opt-in original-runtime asynchronous commands + +The private broker now accepts the exact bounded command/event extensions. The +frontend explicitly opts in through offline_metal_frontend_enable_async; default +contexts retain synchronous commit. Each new command captures that mode. Actual +submission returns independently of a dispatch worker that observes real GPU +status, synchronizes actual IOSurface/buffer data, invokes the original drawable +present and delivers callbacks. Event/fence/resource proxies stay retained through +completion. Proxy seed/data access is synchronized across workers. + +Scheduling, resource visibility and callback completion have independent groups. +waitUntilScheduled only waits for actually observed scheduling; it cannot wait +for a callback while an original caller holds its lock. The original private +commitAndWaitUntilSubmitted B16 ABI returns actual accepted submission in async +mode. Successful completion waits include resource visibility and callback +execution; callback reentrant waits use their actual worker identity. Scheduled +callbacks remain deliberately late. Real transport/GPU failures retain errors; +presentation/callback exceptions remain separate from real GPU completion. + +Context finish refuses an active callback caller, waits pending work and all its +callbacks, then closes the actual namespace/userclient. Atomic lifecycle flags, +a recursive transport lock and release/close handoff keep a proxy destructor +from sending a stale release after namespace close. Finish never submits merely +encoded work. It permits the real completion callback's readback while closing. + +The --async-commands native fixture enables the own device before unchanged +original RenderBox initialization/drawing. All four actual GPU commands use +nonblocking submit, including that original renderer. Native workers deliver +five own callbacks away from the caller thread, two reentrant completion waits +and one genuine callback-close refusal. An explicit caller-held lock blocks its +scheduled callback while waitUntilScheduled still returns, qualifying the +scheduling/callback separation. A final GPU clear has its caller reference +released and no explicit wait; context finish waits its callback, which checks +every one of 4096 real green pixels before close. Original drawable/peer images +also match all pixels and independent GPU hashes. Both GPU event values execute +on genuine commands; no CPU signal is substituted. + +The same kernel passes the existing default synchronous drawable/fence path and +real default-off original launchd main, preserving actual root, original AMFI/ +signatures/trust and active native CPU 4K. Use the umbrella dedicated async probe +and checker, including --default-off and --synchronous-control. This is explicit +offline original-runtime integration, not normal device/client authorization, +window-server composition, display timing, original desktop or native Adreno. +The actual detached drawable presentedTime remains zero and no framebuffer swap +occurs. Shared-event Mach export/import and complete concurrent/system contracts +remain separate requirements before normal discovery publication. + +## Actual host shared events, listeners and exported Metal handles + +OfflineMetalSharedEventWire.h adds actual newSharedEvent, value access, labels, +real listener registrations/polls and internal export/import/drop operations. +Only original Metal callbacks mark a notification ready. Pending registrations +are cancelled at namespace teardown without fabricating delivery. A separate CPU +setter control is explicitly recorded and never drives the rendered dependencies. +Shared-event aliases retain actual Metal handles and common producer metadata; +closing an owner retires its producer references. Imported waits can use the real +hardware counter or an actually committed producer, never a CPU signaled flag. + +The backend imports with actual newSharedEventWithHandle after the original +namespace closes. Its imported object continues GPU work after the actual export +handle is released. Export references keep the private transport alive until +released. Internal control framing is distinct from client sequence accounting; +a forbidden client control request refuses without desynchronizing its sequence. +Public native raw submission must exclude these internal integer export IDs and +mediate them through actual native Mach capabilities before this feature connects. +At that host-only checkpoint the native broker still ended at ordinary GPU event operations. + +check-host-metal-shared-events.py --api-validation executes three GPU tasks, +five shared GPU signals/waits, all 12288 original-shader/peer pixels and eighteen +refusals. Three actual listener callbacks and two genuinely unfired cancellations +pass. The hardware timeline reaches 1/2/3 from GPU commands; only an unrelated +control event uses a real CPU setter to 7. Actual shared-event device metadata +reports this owner device on the tested host; nil is never assumed from a type +name. Its actual exported handle copies the label and has a real supported trace +getter returning zero. Previous async and disconnect regressions still pass. + +The umbrella audit derives the unchanged target's original 16-byte secure handle, +24-byte private data, typed eventPort/label/labelTraceID getters, label copy and +real mach_port_mod_refs send-right increment. Its original listener construction +ABI is also pinned. That is source analysis only: native Mach transfer, original +handle construction/coding, recipient import and process-exit lifetime must be +qualified separately. No raw host export ID is a native permission or completion. +System desktop, native Adreno and active native-4K GPU SMMU remain unqualified. + +The native private broker now mediates shared-event export/import through real +IKOT_IOKIT_IDENT OSObject Mach capabilities, with independent internal host and +public request sequences. Raw host export identifiers are never accepted. Token +no-senders destruction drops the actual retained host Metal handle exactly once, +independently of creator userclient lifetime. Both custom selectors verify the +actual owning task, genuine token class/controller and bounded metadata buffers. + +The frontend calls the unchanged original MTLSharedEventHandle constructor, whose +real send-right reference count rises and falls 2 -> 3 -> 2. Host-derived event +metadata preserves a genuine nil or owner device (which varies with actual host +API validation). Counter reads, bounded waits, copied labels and GPU wait/signal +encoding are backed by actual Metal operations. At this capability-only checkpoint native listeners explicitly +refuse until independently qualified. No original code/methods are replaced. + +Run the umbrella build/stage --shared-events, probe-native-metal-shared-events.py +and check-native-metal-shared-events.py. Two real original-runtime native-4K +workers prove recipient import after creator waitpid/exit, three actual GPU tasks, +all 12288 pixels with no texture uploads, and continued imported GPU execution +after the final original handle/Mach right is destroyed. Wrong/stale Mach rights +and guessed raw host imports refuse without breaking subsequent real requests. +One export/import/drop and zero live tokens remain at finish. The own received- +right adapter uses the actual original constructor; it does not qualify original +NSXPC/NSSecureCoding serialization. System Metal/desktop and Adreno remain open. + +The native shared-notification extension now validates the unchanged original +MTLSharedEventListenerInternal 24-byte class and real typed dispatchQueue getter. +Only genuine host Metal listener ready records can schedule a native user block, +on that original listener's actual queue, with exact event identity/counter. +Bounded workers record failures separately. Pending/queued/running callbacks have +distinct states under the lifecycle lock. Context finish cancels unfired or queued +registrations, waits genuinely running callbacks and rejects reentrant finish from +one of those callbacks. It never waits a queued block behind a caller-held queue. +After close a cancelled queued block skips delivery; no new transport call escapes +past namespace close. A cancelled or timed-out registration is never successful. + +Build/stage --shared-notifications, probe-native-metal-shared-events.py +--notifications --api-validation, then the matching checker flags. Two callbacks +run on distinct original listener queues away from the caller, reenter real +counter access and refuse callback-local finish. Two value-99 registrations never +fire. A third real host callback reaches a deliberately blocked original queue: +finish cancels it without deadlock, and draining the queue confirms no user block. +The host independently records three actual GPU-triggered notifications/two +unfired ones; native counters record two delivered/three cancelled/zero failed. +The existing creator-exit/Mach rights/12288-pixel GPU controls still pass. The new +frontend and shared-event kernel also pass the full original RenderBox async +primary path (four actual GPU tasks, original images, event/fence/callback controls). +Original secure coding, normal system publication/authorization, desktop and +native Adreno/SMMU remain independent unqualified work. + +The umbrella --shared-coding instrument independently qualifies genuine original +same-process NSXPCListener/NSXPCConnection shared-handle coding and original +non-NSXPC-coder refusal. --shared-cross-process additionally transports a genuine +anonymous endpoint via original libxpc serialization and the original endpoint +accessors, with real peer PID checks. It uses the supplied target's actual third +receive flags argument, not an obsolete two-argument declaration. The peer's +original NSXPCCoder decodes a real Mach send right; the native broker imports its +actual event. After creator waitpid/exit, that NSXPC event executes the real blue +GPU wait/signal task. A distinct genuine Mach import sees its hardware counter +and continues with green GPU work after final handle destruction. All 12288 real +GPU pixels, the original handle reference controls, wrong/stale/raw refusals, +three real imports, exact final cleanup, native CPU 4K/root/AMFI and GPU API +Validation pass. No new bootstrap or deployment authorization is implied. +Normal system discovery/client policy, full compositor UI/desktop and actual +Adreno/SMMU qualification remain open. + +The unpublished NativeIOGPUProviderProbe now derives from the actual abstract +736-byte/215-entry IOGPU. Its real original protected C2/D2 and all inherited +operational entry addresses remain intact. The ten required private hooks have +non-returning register captures; a separate publication guard also never returns. +Opaque tail declarations bind original entries directly without introducing C++ +callers or claiming recovered source signatures. It is an introspection object, +not a system driver. + +Native QEMU observes original construction/init/free/D2 restoring instance count +0 -> 1 -> 0. In the independent start observation, actual original IOGPU::start +reaches required slot 195 with genuine this/provider and the exact original +caller LR. The hook records live CPU translation registers, reports Unsupported +and deliberately stops; startup does not return, and this start object remains +live at the checkpoint. Missing original registration refuses before allocation; +default-off executes original launchd with no probe object. No normal publication, +successful provider startup, system Metal/UI, Adreno execution or active native +GPU SMMU is qualified. + +## Original QuartzCore vertex attributes and real buffer geometry + +OfflineMetalVertexWire.h adds a 2104-byte, size-discriminated extension of the +existing attachment-pipeline operation. No new kernel selector or permission is +needed. The actual original vertex descriptor is serialized into 31 attributes +and 31 static layouts; the host constructs real Metal descriptors and pipelines. +Known scalar/vector/packed formats are bounded explicitly; unused entries must +be zero. Dynamic strides, patch stepping and out-of-range formats/indices/extents +refuse. No unsupported vertex layout is silently replaced with an empty one. + +The original unchanged QuartzCore std_vert0_lpf and attachment_clear_frag_lpf +now execute through actual float4/float2 attributes, a 40-byte vertex stride, +real MVP buffer and updated vertex bytes. Host API Validation qualifies two +GPU draws/all 8192 pixels plus ten framing/layout/foreign-namespace refusals. +Two actual native iPadOS workers then qualify four original-shader GPU draws, +all 16384 geometry-dependent pixels, descriptor mutation independence and real +shared-surface/foreign-resource lifetime controls. All 137 actual requests and +144974 reply bytes reconcile independently; seven GPU submissions complete, +and both workers exit zero with distinct native CPU 4K task/TTBR0 contexts. +New geometry targets never receive uploaded CPU pixels. + +The original RenderBox asynchronous regression also passes on the same source +kernel/frontend/backend: original GPU output, genuine events/callback visibility +and cleanup remain intact. Nonempty static vertex input is newly supported; +ordinary system discovery, full display-server composition, original desktop, +Adreno execution and active native-4K GPU SMMU are still unqualified. + +## Original client absent-descriptor refusal and cleanup + +The unpublished original-provider instrument now declares slot170's actual +descriptor return and uint32 output pointer. The supplied original concrete +method writes a uint32 through x1 and returns its real descriptor through x0; +the actual original client retains it and invokes native createMappingInTask. +This register contract is derived from this target's unchanged code. Other +opaque tail source signatures remain unqualified. + +The naked slot170 entry saves the same actual register frame before C++ code. +Its normal observation remains non-returning. Only the explicit own +client-refusal instrument returns NULL for the genuinely absent backend +descriptor, restoring the original LR/SP. All other required callbacks and +publication remain non-returning captures. No descriptor or device success is +supplied, and no original method or class identity is replaced. + +The original client factory, custom owning-task/subtype initializer and client +start now execute together. An absent descriptor returns through original error +handling, cleanup, detach and release. QEMU checks actual type1 and high16 +subtype controls: the original client count rises by one at the real callback +and returns to its baseline; the own provider count also returns to baseline. +The original factory returns its actual failure with NULL output, no live +attachment/publication and zero GPU protocol traffic. The read-only observer +never dereferences freed client storage. Separate non-returning observations +still inspect the real live original 304-byte client and its stored subtype. + +This is genuine refusal/lifetime qualification for the unpublished instrument. +Real shared descriptors, device/resource/queue implementations, successful +original startup, normal system discovery, desktop and native Adreno/SMMU +remain required. + +## Native frontend over the bridge + +`NativeBridgeMetalFrontend.mm` is the Metal API frontend of the native path: +the metal2vulkan runtime (`bridge/NativeMetalObjects.rs`, a C ABI of the +runtime's device, resources, states, queues, command buffers and encoders) +on Turnip and NativeAdreno. The device is a dynamic subclass of the target's +_MTLDevice adding only its own methods, as in the offline frontend. Implemented: +buffers (also no-copy), textures (also over buffers and over an IOSurface's +own pages, without a copy), metallib and AIR libraries, function-constant +specialization through the target's own accessors, compute and render +pipelines with vertex descriptors and blending, samplers, depth/stencil states, +queues, command buffers with scheduled/completed handlers and GPU times, and +compute, render and blit encoders. An encoder call the runtime refuses fails +its command buffer (status Error, handlers still run); an unimplemented +selector is reported through a hook before the usual exception. Not yet: +events/fences, heaps, argument buffers, indirect commands, drawables and +presentation. No GPU family or feature set is claimed. + +`scripts/check-native-metal-frontend-host.py` (umbrella) qualifies it on the +Mac through the public Metal API only, over MoltenVK as the runtime's Vulkan +driver: compute, render with blit readback, rendering into a real IOSurface +read by the CPU, sampling that surface into a second surface, texture compute, +handler order, a refusal, function constants, and translation of the target's +QuartzCore library. That is a host observation of the frontend and runtime; +it does not qualify Turnip, the Adreno 740, its SMMU context or presentation. + +Compute descriptor creation and archive population use the same supported +descriptor fields: function constants, thread limit/promise, indirect/binary +function support, call stack depth and buffer mutability. Explicit archive +lists and pipeline option bits reach the runtime for both compute and render +creation. Nonempty compute stage inputs and linked/dynamic libraries are +refused here; empty descriptors materialized by Metal retain their default +meaning. A separate-process, empty-cache archive regression checks strict +hits/misses, descriptor/function-constant identity and actual host GPU output. +The archives contain translator artifacts, not Adreno machine code; later +draw/dispatch variants and a complete cold UI shader pack remain unqualified. + +The default compute-function path is separately checked over Turnip and the +software kernel peer in `bridge/driver_cache_host.cpp`. A cold process writes +translation artifacts and a real driver-pipeline record. Another process with +the same directory must report artifact/driver hits and no new misses. The C +bridge exposes the two layers' separate counters for this check. Mesa disk +shader caching is disabled. Commands retire through the software peer, and +their output is unchecked; actual host GPU pixels have separate regressions. +Matching these host caches does not qualify them for a different iOS driver +build or cover every target UI function/constant/dispatch variant. + +Bound buffer/texture owners are retained separately from the runtime's GPU +storage handles. The former owns no-copy deallocators, IOSurface references +and buffer/view/heap ancestry. Compute/render bindings, blits, index buffers, +pass attachments and visibility buffers retain those owners until real +completion (or a local refusal which submitted no work). Completion also drops +the IOSurface visibility set, so keeping a completed command does not keep its +old frame storage pinned. Discarding an uncommitted command releases its owners. + +`bridge/resource_lifetime_host.mm` drops all caller-owned references while +actual host GPU work is gated on an unsignaled event. It checks raw-buffer +copies/fills, nested buffer-backed views, compute input and a clear-only render +attachment, plus local refusal and uncommitted discard. The no-copy callback +must remain pending and then run exactly once; successful work checks actual +GPU output. The old frontend invokes the callback before submission in all +seven cases. The test owns its mapping separately so that the old failure +never submits a dangling pointer. This qualifies host ownership, not safe +native GPU kernel resource leases, scanout-map reuse or 120 FPS. + +IOSurface cache maintenance deduplicates exact live CPU address/length pairs +within each command phase. Independent imports and nested texture views of +one surface previously repeated the same full-range operation before commit +and after completion. Range sorting removes those duplicates, keeping every +distinct range's maintenance and carrying no address identity across phases +or frames. The command's retained owners keep all mappings live during each +phase. No hardware coherence assumption or cross-frame mapping cache is added. + +`bridge/surface_flush_host.mm` forwards every intercepted maintenance call to +libSystem and checks real host GPU writes, independent alias reads and shared +surface bytes. For 128 aliases of one allocation the old code makes 256 calls +covering 4 MiB; the new code makes two covering 32 KiB. Two distinct allocations +require four calls. A single alias still requires both phases. Telemetry reports +actual maintenance calls/bytes/time and removed duplicates; the plugin streams +all 48 supported rows. Actual native workload frequency and hardware frame-time +benefit remain unqualified. + +IOSurface texture creation now reuses a still-owned no-copy buffer for an exact +surface ID, CPU base and rounded allocation size. The buffer retains the +surface through its own release. A mutex covers lookup and first import; the +cache has weak values and at most 256 associations, so it cannot keep an old +frame's GPU allocation alive. An evicted association may be recreated while +its original resources remain live. General caller-owned no-copy buffers keep +their own objects and deallocators; no remap or rebind API is added. + +`bridge/surface_import_host.mm` counts actual frontend import/release calls +while forwarding every call to the unmodified bridge. The separate host-test +`import_count_loader.cpp` forwards all Vulkan requests to the real driver and +counts successful external-host-memory allocations. These are distinct counts: +the runtime already shares sequential live imports. For 128 aliases of one +surface, frontend calls fall from 128 to one while actual Vulkan imports stay +one. A 64-way concurrent first creation instead reproduced seven actual imports +and stale host GPU texels on the previous frontend; the fixed path makes one +and reads the correct GPU output. This also avoids the runtime's documented +concurrent first-import window for this frontend's IOSurface path. The common +native bridge also covers raw no-copy callers, as described below; direct +runtime clients outside that C ABI still need their own synchronization. + +Distinct surfaces retain distinct imports. Parallel creation, owner expiry, +work gated behind a real unsignaled GPU event and dropping the caller's last +surface reference all pass. Completed commands release their owners even while +retained. A 514-alias/257-surface case crosses the metadata bound: 258 frontend +objects and 257 Vulkan imports, all GPU bytes correct and all owners released. +The tests do not qualify Adreno timing, target UI concurrency, repeated-blur +frame deadlines, or active native-4K GPU translation. + +The common `nmb_buffer_new_no_copy` entry now gates its runtime construction per +device. This covers both `newBufferWithBytesNoCopy:` and IOSurface textures, +including concurrent mixtures, while returning a separate buffer object for +every raw call. The gate covers the live-import lookup, allocation and join; +it ends before returning an object/error, and resource release, GPU work, +completion waits and other APIs do not take it. The C ABI supplies neither a +dma-buf source nor a user runtime deallocator. Frontend deallocators still run +outside the bridge. No upstream runtime patch or shader cache identity change +is introduced by this boundary fix. Distinct no-copy constructors on the same +device share the CPU construction gate; this is not a claim about their timing. + +`bridge/no_copy_import_host.mm` starts sixteen simultaneous public Metal calls +in each fresh process. The forwarding loader inserts a one-shot 25 ms delay +before the first actual host-memory allocation to expose the lookup window, +then calls the real driver unchanged. Old code makes sixteen imports for +sixteen raw aliases and nine for eight raw buffers mixed with eight surface +textures. Both now make one. The GPU producer completes before alias readback, +so the check assumes no implicit synchronization for arbitrary raw Metal +aliases. All GPU bytes match, every raw object remains distinct, and each +deallocator runs exactly once. A separate callback creates a buffer from +another thread and waits for it, qualifying release-time reentry without the +construction gate held. These are host allocation/ownership controls, not +Adreno timing or stable UI frame evidence. + +`bridge/quartzcore_blur_host.mm` separates bare-descriptor failures from valid +pipeline coverage in the unchanged target library. Its 167 entries comprise +41 vertex, 111 fragment and 15 kernel functions. Four ordinary compute entries +compile; eleven report a tile-attachment requirement when incorrectly requested +as ordinary compute. Of 22 name-matched render pairs, the bare probe's eight +failures are six missing vertex descriptions and two missing MRT attachments. +Compatible authored layouts/formats make all 22 pipelines, without changing +the source library or advertising tile support. These are selected descriptor +recipes, not the target UI's complete function-constant/format/state inventory. + +The same control executes the original float/half narrow-blur functions on the +host GPU. An authored seven-tap binomial stencil provides an independent +reference; horizontal and vertical passes run six times for each precision, +checking every output component. No further pipeline making or cache miss is +allowed during the repeated draws. The native frontend exposes structured +work/cache counters for this qualification; cumulative creation diagnostics +must also be positive so disabled logging cannot masquerade as reuse. + +The umbrella's `check-quartzcore-blur-host.py` runs cold producer, archive-only +and provisioned-cache processes, with strict empty/changed-descriptor refusals. +The archive contains 22 compatible pairs and the two blur recipes. Archive-only +reload avoids translation but still reports 20 driver misses in the observed +host run. Copying the matching producer cache reports 24 driver feedback hits +and no translation or driver miss. All 24 pipeline objects still have to be +created, and MoltenVK's feedback may describe cached stages; this is not proof +of a native Adreno first-frame deadline. Archives, driver records and raw logs +remain ignored. A different iOS/Adreno profile needs its own artifacts; the +complete initial-image UI pack and stable 120 FPS remain unqualified. + +`bridge/quartzcore_cache.mm` runs the same 24 authored descriptor recipes over +actual Turnip/IR3 and the explicitly required software peer. Its offline-only +`graphics_cache_loader.cpp` forwards real calls, enables the queried Vulkan +cache-control feature and can forbid compilation after device initialization. +It independently records real compiler results and feedback. The negative +UUID control mutates a private copy of loaded data; producer files stay intact. +No draw is encoded. Device entry-point priming submits barriers/timestamps +to the software peer, which executes no shader. This is compiler evidence, +separate from the original-blur host GPU pixel control. + +The creating API can return a state even when a Vulkan prewarming attempt +failed. Therefore nonnil states cannot qualify driver-cache readiness. In the +observed Turnip graphics-library path, 24 states use 65 actual graphics +creations, 33 with shader application caches. The strict consumer succeeds in +all 65 and reports valid application hits in all 33; empty and UUID-mismatched +caches produce actual compile-required results. The runtime's aggregate +shader-cache hit statistics do not report this graphics-library branch, so the +independent call counters are required. The umbrella records a separate Mesa +feedback fix: its graphics hit flag's AND reduction previously began false, +concealing successful cache hits. + +Device initialization itself makes a graphics pipeline before the strict UI +scope and submits initialization work. It is explicitly outside the consumer's +compile-forbidden claim. These checks do not qualify every actual draw variant, +initial-image completeness, native iOS/Adreno shader execution, active GPU 4K +translation or any frame deadline. The control loader is never deployed. + +`bridge/quartzcore_cache_ios.cpp` is an own QEMU-only PID 1 harness for this +compiler instrument. It finds the exact own RAM Data UUID and NativeRamBlock +ancestry, establishes the existing readonly union extension view, and spawns +four separate normally linked workers. Actual original dyld/AMFI and 4K +CodeDirectories remain in force. Parent and workers require the genuine 4096 +byte ABI; no page-size shim is used. The parent hashes the source library, +archive and every sorted cache file, calls fsync/sync and stops at its own EL0 +checkpoint. Its small manifest and complete worker log fit the existing bounded +saved-register captures. Existing bounded own-Data export supplies the full +cache bytes without widening a user-memory read. + +The iOS Turnip build passes the same 24-recipe strict compiler controls: 65 +successful graphics creations and 33 real application shader-cache hits, with +UUID/empty-cache refusals in separate workers. The actual selected pack has +153 files; readonly extraction matches every byte against the paused iOS +parent's independent manifest. The backing is mutable test Data, and these +workers produce the pack during that boot. A fresh boot provisioned from these +bytes, complete UI/draw variants, authenticated production packaging, Adreno +execution/active GPU 4K translation and display deadlines remain unqualified. + +`bridge/quartzcore_cache_provisioned_ios.cpp` reuses the exact volume, manifest +and checkpoint helpers with a consumer-only main. The staged image already +holds the exported iOS archive/cache at the normal consumer paths. Before any +worker starts, the parent hashes that initial pack. Only consumer, wrong-UUID +and empty-driver workers run, in distinct processes; their final pack manifest +must equal the initial one byte for byte. Actual iOS first-consumer checks pass +all 65 compile-forbidden UI graphics creations and all 33 shader cache hits, +with zero translation misses/writes or driver record saves and no boot producer. +Original signing/AMFI/trust, native CPU4K and real worker page ABI checks pass. +This qualifies the selected recipes in an owned test image only. + +The separately observed device initialization is exit-guard priming in the +runtime's `device/prime.rs`: it calls the driver entry points before registering +the teardown hook, makes an initially empty pipeline cache, and discards that +cache's bytes. It includes compute and graphics pipeline creation and a +barrier/timestamp submission. The render/compute recording is not submitted; +this is not a semantic GPU probe or a source of measured shader output. +Caching that startup work must preserve its complete priming and teardown +ordering. Device initialization remains outside the UI consumer's strict scope; +complete variant coverage, production authenticated packaging and native +Adreno frame deadlines remain unqualified. + +`bridge/patches/0003-cache-device-entry-point-priming.patch` now keeps that +startup cache after all priming succeeds and reloads it in later devices. +Its raw disk/cache path creates no worker and enters no exit-guard section +before priming. Every primed call, the barrier/timestamp batch, waits, failure +cleanup and post-priming hook registration remain. Raw loader words are restored +before each added cache entry point. The optional generated helper also uses +the shared priming cache; generated-command execution is not qualified by the +current controls. + +Records reuse the existing bounded, checksummed, atomic driver-cache format. +Their keys include the driver/build UUID, compile profile, instance settings, +priming implementation bytes and relevant feature/queue choices. Invalid or +refused data starts empty; failed priming never persists partial cache data. +An unchanged loaded record is not rewritten. This startup store has separate +counters from the ordinary shader-cache store, so independent driver calls are +used to qualify startup hits. + +`bridge/priming_cache_loader.cpp` extends the offline graphics-cache forwarding +loader without changing its previously qualified source. It adds actual compute +feedback and a compiler-forbidden device-open policy. `priming_cache_host.cpp` +opens/closes the device without authored shader work. Both compute and graphics +helpers now report real application-cache hits with compilation forbidden. +Cold, wrong-UUID, truncated and bad-checksum controls refuse and leave zero live +software-peer objects. An exact preserved pre-fix executable still fails during +device open with the same populated cache. Every case retains the one real +priming barrier/timestamp submission; no shader execution is claimed. + +The full host GPU/frontend and Turnip controls pass. The instrument and complete +plugin cross-build for ordinary ARM64/iOS17 with own 4K signatures. New priming +cache execution under iOS and a fresh initial image still need qualification. +Earlier iOS UI packs remain evidence for their earlier runtime; regenerate and +verify artifacts for this build rather than silently treating them as current. +These results do not qualify all UI variants, native Adreno execution/4K SMMU, +physical display deadlines or stable 120 FPS. The test loader is never deployed. + +`bridge/priming_cache_ios.cpp` now runs the exact priming instrument in actual +iOS/QEMU workers. Four cold processes qualify empty-cache refusal, production, +strict import and wrong-UUID refusal. A second fresh image prepopulates the +actual native record and selects a consumer-only main path: its first worker +hits both compute and graphics helpers while compilation is forbidden; no +producer runs in that boot. The same bounded manifest helpers independently +hash the initial/final record and the readonly exported Data snapshot. The +8,977-byte native record has its actual iOS driver identity, not a relabelled +host header. Original AMFI/signatures/trust/root, active CPU4K and real worker +page ABI checks pass. Both positive and refused workers retain priming and +release all software-peer resources. The peer executes no shader. + +This proves the runtime's public priming helper path only. A separate source +audit finds Turnip device creation initializes fifteen clear/blit shader +variants and its empty shaders before these public pipeline calls. Its IR3 +compiler is explicitly created with `disable_cache=true`, independent of the +runtime's new cache. That path needs compiler-level measurement and separate +optimization; public Vulkan compile-forbidden flags cannot qualify it. No +complete startup-compilation, UI-variant, native GPU4K or frame-rate claim follows. diff --git a/bridge/FrontendChecks.h b/bridge/FrontendChecks.h new file mode 100644 index 0000000..fdc1014 --- /dev/null +++ b/bridge/FrontendChecks.h @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: MIT +// The native Metal frontend's qualification through the public Metal API only, shared by the host test (frontend_host.mm, over MoltenVK) and +// the plugin (NativeMetalBridgePhase.h, over Turnip on the Adreno 740). Every check reads back what the GPU wrote. Steps (markers 0x6601..): +// 1 compute; 2 render to a private texture with a blit readback; 3 render into a real IOSurface read by the CPU, 4 sampling it into a +// second surface (the composition shape); 5 a compute kernel reading and writing textures; 6 completion handler order; 7 an encoder refusal +// failing its command buffer; 8 the target's QuartzCore library (when a path is given); 9 function constants (when that fixture is given); +// 10 a shared event ordering two queues (GPU wait, GPU signal, CPU signal, listener notification) and a fence between two encoders; +// 11 heaps (automatic and placement), a visibility result buffer and setDepthBias. +#pragma once +#import +#import +#import +#include +#include +#include +#include + +namespace frontendchecks { +struct Report { + unsigned failed, last; + char first[96]; + unsigned qcFunctions, qcKernels, qcKernelsOk, qcPairs, qcPairsOk; + void (*mark)(uint32_t); + bool hostSimulation; // the host simulation's IOKit shim hides IOSurface (steps 3-4 skipped) and Metal's listener (none is made) +}; +static Report *gReport; +__attribute__((format(printf, 2, 3))) static void fail(Report *r, const char *format, ...) { + char text[160]; + va_list args; + va_start(args, format); + vsnprintf(text, sizeof text, format, args); + va_end(args); + std::printf("FAIL %s\n", text); + if (!r->failed++) snprintf(r->first, sizeof r->first, "%x %s", r->last, text); +} +static void step(Report *r, uint32_t marker) { + r->last = marker; + if (r->mark) r->mark(marker); +} +#define CHECK(c, ...) do { if (!(c)) fail(gReport, __VA_ARGS__); } while (0) +static bool completed(id cb, const char *what) { + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusCompleted) { + fail(gReport, "%s: status %lu %s", what, (unsigned long)cb.status, cb.error.localizedDescription.UTF8String ?: ""); + return false; + } + return true; +} +static bool finish(id cb, const char *what) { + [cb commit]; + return completed(cb, what); +} +// The fixture's picture: top-left quarter (255, 51, 0, 255), the rest cleared blue; `bgra` swaps red and blue in memory. +static unsigned wrongTexels(const uint8_t *bytes, size_t row, unsigned side, bool bgra = false) { + unsigned bad = 0; + for (unsigned y = 0; y < side; ++y) + for (unsigned x = 0; x < side; ++x) { + const bool inside = x < side / 2 && y < side / 2; + uint8_t e[4] = {inside ? (uint8_t)255 : (uint8_t)0, inside ? (uint8_t)51 : (uint8_t)0, inside ? (uint8_t)0 : (uint8_t)255, 255}; + if (bgra) std::swap(e[0], e[2]); + if (memcmp(bytes + y * row + x * 4, e, 4)) ++bad; + } + return bad; +} +static IOSurfaceRef surface(unsigned side, OSType format) { + NSDictionary *p = @{(id)kIOSurfaceWidth: @(side), (id)kIOSurfaceHeight: @(side), (id)kIOSurfaceBytesPerElement: @4, + (id)kIOSurfacePixelFormat: @(format)}; + return IOSurfaceCreate((__bridge CFDictionaryRef)p); +} +static id pipeline(id device, id library, NSString *fragment, MTLPixelFormat format) { + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = [library newFunctionWithName:@"colored_vertex"]; + d.fragmentFunction = [library newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat = format; + NSError *error = nil; + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + CHECK(state, "render pipeline %s: %s", fragment.UTF8String, error.localizedDescription.UTF8String); + return state; +} +// Draws the fixture quad (vertex positions/colors at buffers 0/1) into `target`, cleared blue first. +static void drawQuad(id cb, id state, id target, id sampled, + const float (*positions)[2], unsigned count) { + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 1, 1); + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:state]; + [e setVertexBytes:positions length:sizeof(float) * 2 * count atIndex:0]; + std::vector colors; + for (unsigned i = 0; i < count; ++i) colors.insert(colors.end(), {1.0f, 0.2f, 0.0f, 1.0f}); + [e setVertexBytes:colors.data() length:colors.size() * sizeof(float) atIndex:1]; + if (sampled) [e setFragmentTexture:sampled atIndex:0]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:count]; + [e endEncoding]; +} +static const float kQuarter[6][2] = {{-1, 1}, {0, 1}, {-1, 0}, {0, 1}, {0, 0}, {-1, 0}}; +static const float kFull[6][2] = {{-1, 1}, {1, 1}, {-1, -1}, {1, 1}, {1, -1}, {-1, -1}}; + +// Runs every step on `device` (the frontend's); `library` is the bridge's runtime_examples fixture, `fcLibrary` (may be nil) the +// function-constant fixture, `qcPath` (may be nil) the target's QuartzCore default.metallib. +static void run(id device, id library, id fcLibrary, NSString *qcPath, Report *r) { + gReport = r; + @autoreleasepool { + CHECK([library.functionNames containsObject:@"sampled_fragment"], "function names"); + CHECK(![library newFunctionWithName:@"no_such_function"], "a missing function must be nil"); + id queue = [device newCommandQueue]; + CHECK(queue, "command queue"); + if (!queue) return; + + step(r, 0x6601); + // 1. compute: out[i] = 3 * in[i] (the fixture's scale_and_offset with scale 2) + { + NSError *error = nil; + id f = [library newFunctionWithName:@"scale_and_offset"]; + CHECK(f.functionType == MTLFunctionTypeKernel, "kernel type %lu", (unsigned long)f.functionType); + id state = [device newComputePipelineStateWithFunction:f error:&error]; + CHECK(state, "compute pipeline: %s", error.localizedDescription.UTF8String); + std::vector in(1000); + for (unsigned i = 0; i < in.size(); ++i) in[i] = (float)i; + id input = [device newBufferWithBytes:in.data() length:in.size() * 4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:in.size() * 4 options:MTLResourceStorageModeShared]; + const float scale = 2; + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setBuffer:input offset:0 atIndex:0]; + [e setBuffer:output offset:0 atIndex:1]; + [e setBytes:&scale length:4 atIndex:2]; + [e dispatchThreads:MTLSizeMake(in.size(), 1, 1) threadsPerThreadgroup:MTLSizeMake(64, 1, 1)]; + [e endEncoding]; + if (finish(cb, "compute")) { + const float *v = (const float *)output.contents; + unsigned bad = 0; + for (unsigned i = 0; i < in.size(); ++i) bad += v[i] != 3.0f * i; + CHECK(!bad, "compute: %u wrong outputs", bad); + } + } + id colored = pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + step(r, 0x6602); + // 2. render into a private texture, blit to a buffer + { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget; + d.storageMode = MTLStorageModePrivate; + id target = [device newTextureWithDescriptor:d]; + CHECK(target && target.width == 8 && target.pixelFormat == MTLPixelFormatRGBA8Unorm, "private texture"); + id readback = [device newBufferWithLength:8 * 8 * 4 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + drawQuad(cb, colored, target, nil, kQuarter, 6); + id b = [cb blitCommandEncoder]; + [b copyFromTexture:target sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(8, 8, 1) + toBuffer:readback destinationOffset:0 destinationBytesPerRow:32 destinationBytesPerImage:256]; + [b endEncoding]; + if (finish(cb, "render")) CHECK(!wrongTexels((const uint8_t *)readback.contents, 32, 8), "render: wrong texels"); + } + step(r, 0x6603); + // 3. render into an IOSurface (its pages imported, no copy), then 4. sample it into a second surface + IOSurfaceRef layer = r->hostSimulation ? nullptr : surface(64, 'RGBA'), screen = r->hostSimulation ? nullptr : surface(64, 'RGBA'); + CHECK(r->hostSimulation || (layer && screen), "IOSurfaceCreate"); + if (layer && screen) { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:64 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + d.storageMode = MTLStorageModeShared; + id layerTexture = [device newTextureWithDescriptor:d iosurface:layer plane:0]; + id screenTexture = [device newTextureWithDescriptor:d iosurface:screen plane:0]; + CHECK(layerTexture && screenTexture && layerTexture.iosurface == layer, "IOSurface textures"); + id cb = [queue commandBuffer]; + drawQuad(cb, colored, layerTexture, nil, kQuarter, 6); + if (finish(cb, "surface render")) { + IOSurfaceLock(layer, kIOSurfaceLockReadOnly, nullptr); + CHECK(!wrongTexels((const uint8_t *)IOSurfaceGetBaseAddress(layer), IOSurfaceGetBytesPerRow(layer), 64), "surface render: wrong texels"); + IOSurfaceUnlock(layer, kIOSurfaceLockReadOnly, nullptr); + } + step(r, 0x6604); + // sampled_fragment samples its texture at position * 1/8 (normalized), so an 8x8 viewport of the screen reads the layer whole + id sampled = pipeline(device, library, @"sampled_fragment", MTLPixelFormatRGBA8Unorm); + cb = [queue commandBuffer]; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = screenTexture; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 1, 0, 1); + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:sampled]; + [e setViewport:(MTLViewport){0, 0, 8, 8, 0, 1}]; + [e setVertexBytes:kFull length:sizeof kFull atIndex:0]; + float colors[6][4] = {}; + [e setVertexBytes:colors length:sizeof colors atIndex:1]; + [e setFragmentTexture:layerTexture atIndex:0]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [e endEncoding]; + if (finish(cb, "composition")) { + IOSurfaceLock(screen, kIOSurfaceLockReadOnly, nullptr); + const uint8_t *p = (const uint8_t *)IOSurfaceGetBaseAddress(screen); + std::vector corner(8 * 8 * 4); + for (unsigned y = 0; y < 8; ++y) memcpy(&corner[y * 32], p + y * IOSurfaceGetBytesPerRow(screen), 32); + const unsigned bad = wrongTexels(corner.data(), 32, 8); + const uint8_t *outside = p + 20 * IOSurfaceGetBytesPerRow(screen) + 20 * 4; + CHECK(!bad, "composition: %u wrong texels in the sampled corner", bad); + CHECK(outside[0] == 0 && outside[1] == 255 && outside[2] == 0, "composition: the rest keeps its clear color"); + IOSurfaceUnlock(screen, kIOSurfaceLockReadOnly, nullptr); + } + } + step(r, 0x6605); + // 5. compute over textures: invert_texels writes 1 - rgb of its input texel + { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + d.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite; + id src = [device newTextureWithDescriptor:d], dst = [device newTextureWithDescriptor:d]; + std::vector texels(16 * 16 * 4); + for (size_t i = 0; i < texels.size(); ++i) texels[i] = (uint8_t)(i * 7); + [src replaceRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0 withBytes:texels.data() bytesPerRow:64]; + NSError *error = nil; + id state = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"invert_texels"] error:&error]; + CHECK(state, "invert pipeline: %s", error.localizedDescription.UTF8String); + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setTexture:src atIndex:0]; + [e setTexture:dst atIndex:1]; + [e dispatchThreads:MTLSizeMake(16, 16, 1) threadsPerThreadgroup:MTLSizeMake(8, 8, 1)]; + [e endEncoding]; + if (finish(cb, "texture compute")) { + std::vector out(texels.size()); + [dst getBytes:out.data() bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + unsigned bad = 0; + for (size_t i = 0; i < out.size(); ++i) { + const int expected = i % 4 == 3 ? texels[i] : 255 - texels[i]; + bad += abs(out[i] - expected) > 1; + } + CHECK(!bad, "texture compute: %u wrong channels", bad); + } + } + step(r, 0x6606); + // 6. handlers: scheduled before completed, each once, after the GPU finished + { + __block int order = 0, scheduled = -1, done = -1; + id cb = [queue commandBuffer]; + id b = [cb blitCommandEncoder]; + id buffer = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + [b fillBuffer:buffer range:NSMakeRange(0, 4096) value:0xa5]; + [b endEncoding]; + [cb addScheduledHandler:^(id) {scheduled = order++;}]; + __block MTLCommandBufferStatus seen = MTLCommandBufferStatusNotEnqueued; + [cb addCompletedHandler:^(id c) {seen = c.status; done = order++;}]; + if (finish(cb, "handlers")) { + CHECK(scheduled == 0 && done == 1 && order == 2, "handler order %d %d", scheduled, done); + CHECK(seen == MTLCommandBufferStatusCompleted, "status inside the completed handler"); + const uint8_t *p = (const uint8_t *)buffer.contents; + CHECK(p[0] == 0xa5 && p[4095] == 0xa5, "fill"); + } + } + step(r, 0x6607); + // 7. a refused encoder call fails its command buffer (no silent drop), and the handlers still run + { + id cb = [queue commandBuffer]; + id e = nil; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = [device newTextureWithDescriptor:d]; + e = [cb renderCommandEncoderWithDescriptor:pass]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; // no pipeline set + [e endEncoding]; + __block bool ran = false; + [cb addCompletedHandler:^(id) {ran = true;}]; + [cb commit]; + [cb waitUntilCompleted]; + CHECK(cb.status == MTLCommandBufferStatusError && cb.error, "a draw without a pipeline must fail the command buffer"); + CHECK(ran, "the completed handler runs for a failed command buffer"); + } + step(r, 0x6609); + // 9. function constants (newFunctionWithName:constantValues:error:): the runtime's fixture kernel_function_constant_gate_fold, whose + // out[24] is in * 1000 * mode for mode 1..3 and in otherwise. + if (fcLibrary) { + for (int mode : {0, 2, 3}) { + MTLFunctionConstantValues *values = [MTLFunctionConstantValues new]; + [values setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + NSError *error = nil; + id f = [fcLibrary newFunctionWithName:@"fcfold" constantValues:values error:&error]; + CHECK(f, "specialized function: %s", error.localizedDescription.UTF8String); + id state = f ? [device newComputePipelineStateWithFunction:f error:&error] : nil; + CHECK(state, "specialized pipeline: %s", error.localizedDescription.UTF8String); + if (!state) continue; + const float in = 2; + id input = [device newBufferWithBytes:&in length:4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:32 * 4 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setBuffer:input offset:0 atIndex:0]; + [e setBuffer:output offset:0 atIndex:1]; + [e dispatchThreads:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [e endEncoding]; + if (finish(cb, "specialized compute")) { + const float got = ((const float *)output.contents)[24], expected = mode ? in * 1000 * mode : in; + CHECK(got == expected, "mode %d: out[24] %g, expected %g", mode, got, expected); + } + } + } + step(r, 0x6608); + // 8. the target's own QuartzCore library through newLibraryWithURL: (a universal metallib; its AIR64 slice): every kernel into a + // compute pipeline and every _vert_

/ _frag_

pair into a BGRA8 render pipeline. Refusals are counted, not hidden. + if (qcPath) { + NSError *error = nil; + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:qcPath] error:&error]; + CHECK(qc, "QuartzCore library: %s", error.localizedDescription.UTF8String); + unsigned kernels = 0, kernelsOk = 0, pairs = 0, pairsOk = 0; + NSArray *names = qc.functionNames; + for (NSString *name in names) { + id f = [qc newFunctionWithName:name]; + if (f.functionType == MTLFunctionTypeKernel) { + ++kernels; + kernelsOk += [device newComputePipelineStateWithFunction:f error:nil] != nil; + } else if (f.functionType == MTLFunctionTypeVertex) { + NSString *fragment = [name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if ([fragment isEqualToString:name] || ![names containsObject:fragment]) continue; + ++pairs; + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = f; + d.fragmentFunction = [qc newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + pairsOk += [device newRenderPipelineStateWithDescriptor:d error:nil] != nil; + } + } + CHECK(names.count && kernelsOk + pairsOk, "QuartzCore: nothing translated"); + r->qcFunctions = (unsigned)names.count; r->qcKernels = kernels; r->qcKernelsOk = kernelsOk; r->qcPairs = pairs; r->qcPairsOk = pairsOk; + } + step(r, 0x660a); + // 10. A second queue's command buffer waits for value 1, then copies; the first queue fills and signals 1. The waiting one is committed + // first, so its copy only sees the fill if the GPU wait held. A third waits for 3, which only the CPU signals. Then a fence orders two + // blit encoders of one command buffer, and the listener is notified at 3. + { + id event = [device newSharedEvent]; + id fence = [device newFence]; + id other = [device newCommandQueue]; + CHECK(event && fence && other, "event, fence, second queue"); + id a = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id b = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id c = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + if (event && fence && other) { + id waiting = [other commandBuffer]; + [waiting encodeWaitForEvent:event value:1]; + id copy = [waiting blitCommandEncoder]; + [copy copyFromBuffer:a sourceOffset:0 toBuffer:b destinationOffset:0 size:4096]; + [copy endEncoding]; + [waiting commit]; + id signalling = [queue commandBuffer]; + id fill = [signalling blitCommandEncoder]; + [fill fillBuffer:a range:NSMakeRange(0, 4096) value:0x3c]; + [fill endEncoding]; + [signalling encodeSignalEvent:event value:1]; + id cpu = [other commandBuffer]; + [cpu encodeWaitForEvent:event value:3]; + id second = [cpu blitCommandEncoder]; + [second fillBuffer:c range:NSMakeRange(0, 64) value:0x77]; + [second updateFence:fence]; + [second endEncoding]; + id third = [cpu blitCommandEncoder]; + [third waitForFence:fence]; + [third copyFromBuffer:c sourceOffset:0 toBuffer:c destinationOffset:2048 size:64]; + [third endEncoding]; + [cpu commit]; + if (finish(signalling, "event signal") && completed(waiting, "event wait")) { + CHECK(((const uint8_t *)b.contents)[4095] == 0x3c, "the GPU wait did not order the copy after the fill"); + CHECK(event.signaledValue == 1, "signaled value %llu", (unsigned long long)event.signaledValue); + } + CHECK(cpu.status != MTLCommandBufferStatusCompleted, "a wait for 3 completed before 3 was signaled"); + __block uint64_t heard = 0; + dispatch_semaphore_t notified = dispatch_semaphore_create(0); + MTLSharedEventListener *listener = r->hostSimulation ? nil : [[MTLSharedEventListener alloc] init]; + [event notifyListener:listener atValue:3 block:^(id, uint64_t value) {heard = value; dispatch_semaphore_signal(notified);}]; + event.signaledValue = 3; + if (completed(cpu, "CPU-signaled wait")) { + const uint8_t *p = (const uint8_t *)c.contents; + CHECK(p[2048] == 0x77 && p[2111] == 0x77, "the fence did not order the copy after the fill"); + } + CHECK(!dispatch_semaphore_wait(notified, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)) && heard >= 3, + "listener notification %llu", (unsigned long long)heard); + CHECK([event waitUntilSignaledValue:3 timeoutMS:10], "waitUntilSignaledValue"); + } + } + step(r, 0x660b); + // 11. heaps, a visibility result and a depth bias. An automatic heap's buffer is a buffer of that heap (`heap`, `usedSize`); a placement + // heap's two buffers are distinct bytes at their offsets; a pass counts the visible quad into its visibility result buffer (Boolean: 1); + // setDepthBias reaches the runtime without failing the command buffer. + { + MTLHeapDescriptor *hd = [MTLHeapDescriptor new]; + hd.size = 65536; + hd.storageMode = MTLStorageModeShared; + id heap = [device newHeapWithDescriptor:hd]; + CHECK(heap && heap.size >= 65536, "automatic heap"); + if (heap) { + id hb = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + CHECK(hb && hb.heap == heap, "a heap buffer names its heap"); + CHECK(heap.usedSize >= 4096, "heap used size %lu", (unsigned long)heap.usedSize); + if (hb) { + id cb = [queue commandBuffer]; + id fill = [cb blitCommandEncoder]; + [fill fillBuffer:hb range:NSMakeRange(0, 4096) value:0x5a]; + [fill endEncoding]; + if (finish(cb, "heap fill")) CHECK(((const uint8_t *)hb.contents)[0] == 0x5a && ((const uint8_t *)hb.contents)[4095] == 0x5a, "heap buffer contents"); + } + } + MTLHeapDescriptor *pd = [MTLHeapDescriptor new]; + pd.size = 65536; + pd.type = MTLHeapTypePlacement; + pd.storageMode = MTLStorageModeShared; + id placed = [device newHeapWithDescriptor:pd]; + CHECK(placed && placed.type == MTLHeapTypePlacement, "placement heap"); + if (placed) { + id first = [placed newBufferWithLength:4096 options:MTLResourceStorageModeShared offset:0]; + id second = [placed newBufferWithLength:4096 options:MTLResourceStorageModeShared offset:4096]; + CHECK(first && second && second.heapOffset == 4096, "placed buffers"); + if (first && second) { + id cb = [queue commandBuffer]; + id fill = [cb blitCommandEncoder]; + [fill fillBuffer:first range:NSMakeRange(0, 4096) value:0x11]; + [fill fillBuffer:second range:NSMakeRange(0, 4096) value:0x22]; + [fill endEncoding]; + if (finish(cb, "placement fill")) + CHECK(((const uint8_t *)first.contents)[4095] == 0x11 && ((const uint8_t *)second.contents)[0] == 0x22, "placed buffers overlap"); + } + } + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + td.storageMode = MTLStorageModePrivate; + id target = [device newTextureWithDescriptor:td]; + id visible = [device newBufferWithLength:8 options:MTLResourceStorageModeShared]; + if (target && visible) { + memset(visible.contents, 0, 8); + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.visibilityResultBuffer = visible; + id cb = [queue commandBuffer]; + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:colored]; + [e setVisibilityResultMode:MTLVisibilityResultModeBoolean offset:0]; + [e setDepthBias:0 slopeScale:0 clamp:0]; + std::vector colors; + for (unsigned i = 0; i < 6; ++i) colors.insert(colors.end(), {1.0f, 0.2f, 0.0f, 1.0f}); + [e setVertexBytes:kQuarter length:sizeof(float) * 2 * 6 atIndex:0]; + [e setVertexBytes:colors.data() length:colors.size() * sizeof(float) atIndex:1]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [e endEncoding]; + if (finish(cb, "visibility result")) CHECK(*(const uint64_t *)visible.contents != 0, "visibility result stayed 0 for a visible quad"); + } + } + if (layer) CFRelease(layer); + if (screen) CFRelease(screen); + } + step(r, 0x660f); + gReport = nullptr; +} +#undef CHECK +} // namespace frontendchecks diff --git a/bridge/NativeMetalBridge.h b/bridge/NativeMetalBridge.h new file mode 100644 index 0000000..7d6bbf0 --- /dev/null +++ b/bridge/NativeMetalBridge.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* C ABI of the native Metal bridge (NativeMetalBridge.rs): the metal2vulkan runtime over Turnip. */ +#ifndef NATIVE_METAL_BRIDGE_H +#define NATIVE_METAL_BRIDGE_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +struct NmbSelftest { + uint32_t stage; /* 0 when every step ran, else the failing step's marker (0x6501..0x6506) */ + uint32_t compute_bad; /* wrong outputs of 1000 */ + uint32_t render_bad; /* wrong texels of 64 (8x8) */ + uint32_t import_bad; /* wrong texels of 4096 (64x64 over imported host pages) */ + uint32_t qc_functions; + uint32_t qc_kernels_ok, qc_kernels_failed; + uint32_t qc_pairs_ok, qc_pairs_failed; + uint32_t reserved; + char message[96]; +}; +/* Markers: 0x6501 device, 0x6502 fixture library, 0x6503 compute, 0x6504 render, 0x6505 import, 0x6506 QuartzCore, 0x650f done. */ +uint32_t nmb_selftest(const char *loader, const char *cache_dir, const uint8_t *quartzcore, size_t quartzcore_bytes, + void (*mark)(uint32_t), struct NmbSelftest *out); +#ifdef __cplusplus +} +#endif +#endif diff --git a/bridge/NativeMetalBridge.rs b/bridge/NativeMetalBridge.rs new file mode 100644 index 0000000..40a753f --- /dev/null +++ b/bridge/NativeMetalBridge.rs @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: MIT +//! Native Metal bridge: the metal2vulkan runtime (the Metal API on Vulkan) over Turnip, behind a C ABI the Objective-C +//! Metal frontend and the plugin's self-test call. Turnip is opened as the runtime's Vulkan "loader" (it exports +//! vkGetInstanceProcAddr; iPadOS has no Khronos loader). The pinned runtime is GPL-2.0-or-later OR LGPL-3.0-or-later and is +//! linked unmodified from its own source; nothing of it is copied here. +//! +//! The self-test runs real Metal-level work and checks the GPU's results: a compute kernel, a render pass, a render into +//! host pages imported without a copy (the IOSurface path), and translation of the target's own QuartzCore library into +//! A740 pipelines. Each step reports a marker before it starts so a hang names the step. + +#[path = "NativeMetalObjects.rs"] +pub mod objects; + +use metal2vulkan::metal::{PixelFormat, PrimitiveType, TextureUsage}; +use metal2vulkan::pipeline::{ColorAttachmentDescriptor, RenderPipelineDescriptor, StageSource}; +use metal2vulkan_runtime::{ + BlitOption, ClearColor, CommandBuffer, CommandBufferStatus, Device, DeviceOptions, HostMemory, LoadAction, MetalLevel, + Origin, PipelineOptions, RenderPassDescriptor, ResourceOptions, Size, StoreAction, TextureDescriptor, +}; +use std::ffi::{c_char, c_void, CStr}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::time::Duration; + +/// The runtime's own example library (validation/fixtures/public/examples/runtime_examples.ll), assembled to AIR at build time. +static FIXTURE_AIR: &[u8] = include_bytes!(env!("NMB_FIXTURE_AIR")); + +unsafe extern "C" { + /// libkern (libSystem): clean and invalidate the data cache over a range, from user space. + fn sys_dcache_flush(start: *mut c_void, len: usize); +} + +/// Result of nmb_selftest; mirrors NativeMetalBridge.h. +#[repr(C)] +pub struct NmbSelftest { + /// 0 when every step ran; otherwise the marker of the step that failed. + pub stage: u32, + pub compute_bad: u32, + pub render_bad: u32, + pub import_bad: u32, + pub qc_functions: u32, + pub qc_kernels_ok: u32, + pub qc_kernels_failed: u32, + pub qc_pairs_ok: u32, + pub qc_pairs_failed: u32, + pub reserved: u32, + pub message: [u8; 96], +} + +type Mark = Option; + +struct Run<'a> { + out: &'a mut NmbSelftest, + mark: Mark, +} + +impl Run<'_> { + fn step(&mut self, marker: u32) { + if let Some(mark) = self.mark { + // SAFETY: the caller's callback, valid for the call. + unsafe { mark(marker) }; + } + } + fn fail(&mut self, marker: u32, why: impl std::fmt::Display) -> u32 { + self.out.stage = marker; + let text = format!("{why}"); + let n = text.len().min(self.out.message.len() - 1); + self.out.message[..n].copy_from_slice(&text.as_bytes()[..n]); + self.out.message[n] = 0; + marker + } +} + +const WAIT: Duration = Duration::from_secs(10); + +fn finish(command_buffer: &CommandBuffer) -> Result<(), String> { + command_buffer.commit().map_err(|e| format!("commit: {e:?}"))?; + command_buffer.wait_until_completed_for(WAIT).map_err(|_| "no completion in 10 s".to_string())?; + match command_buffer.status() { + CommandBufferStatus::Completed => Ok(()), + status => Err(format!("{status:?} {:?}", command_buffer.error())), + } +} + +fn open(loader: &CStr, cache: &CStr) -> Result { + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + // No macOS release names this host; answer as the newest measured Metal. + options.metal_level = Some(MetalLevel::NEWEST); + options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + Device::system_default_with(&options).map_err(|e| format!("device: {e:?}")) +} + +fn compute(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const COUNT: usize = 1000; + let function = library.new_function_with_name("scale_and_offset").map_err(|e| format!("{e:?}"))?; + let state = device + .new_compute_pipeline_state_with_function(&function, &PipelineOptions::default()) + .map_err(|e| format!("pipeline: {e:?}"))?; + let input: Vec = (0..COUNT).flat_map(|i| (i as f32).to_le_bytes()).collect(); + let input = device.new_buffer_with_bytes(&input, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let output = device.new_buffer((COUNT * 4) as u64, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let scale = device.new_buffer_with_bytes(&2.0f32.to_le_bytes(), ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let queue = device.new_command_queue().map_err(|e| format!("{e:?}"))?; + let command_buffer = queue.command_buffer().map_err(|e| format!("{e:?}"))?; + let mut encoder = command_buffer.compute_command_encoder().map_err(|e| format!("{e:?}"))?; + encoder.set_compute_pipeline_state(&state).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&input), 0, 0).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&output), 0, 1).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&scale), 0, 2).map_err(|e| format!("{e:?}"))?; + encoder.dispatch_threads(Size::new(COUNT as u64, 1, 1), Size::new(64, 1, 1)).map_err(|e| format!("{e:?}"))?; + encoder.end_encoding().map_err(|e| format!("{e:?}"))?; + finish(&command_buffer)?; + let contents = output.contents().ok_or("no contents")?; + // SAFETY: the buffer holds COUNT floats and no command buffer writes it now. + let values = unsafe { std::slice::from_raw_parts(contents.as_ptr() as *const f32, COUNT) }; + Ok(values.iter().enumerate().filter(|(i, v)| **v != 3.0 * *i as f32).count() as u32) +} + +/// Draws the fixture's top-left-quarter quad into `target` (cleared blue) and returns the encoded command buffer's result. +fn draw_quad(device: &Device, library: &metal2vulkan_runtime::Library, target: &metal2vulkan_runtime::Texture, + readback: Option<(&metal2vulkan_runtime::Buffer, u64, u64)>) -> Result<(), String> { + let vertex = library.new_function_with_name("colored_vertex").map_err(|e| format!("{e:?}"))?; + let fragment = library.new_function_with_name("colored_fragment").map_err(|e| format!("{e:?}"))?; + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function(vertex.function_descriptor())); + descriptor.fragment = Some(StageSource::Function(fragment.function_descriptor())); + descriptor.color_attachments[0] = Some(ColorAttachmentDescriptor::new(PixelFormat::RGBA8Unorm)); + let state = device.new_render_pipeline_state(&descriptor, &PipelineOptions::default()).map_err(|e| format!("pipeline: {e:?}"))?; + let positions: [[f32; 2]; 6] = [[-1.0, 1.0], [0.0, 1.0], [-1.0, 0.0], [0.0, 1.0], [0.0, 0.0], [-1.0, 0.0]]; + let positions: Vec = positions.iter().flatten().flat_map(|v| v.to_le_bytes()).collect(); + let colors: Vec = (0..6).flat_map(|_| [1.0f32, 0.2, 0.0, 1.0]).flat_map(|v| v.to_le_bytes()).collect(); + let positions = device.new_buffer_with_bytes(&positions, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let colors = device.new_buffer_with_bytes(&colors, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let mut pass = RenderPassDescriptor::new(); + let color = &mut pass.color_attachments[0]; + color.texture = Some(target.clone()); + color.load_action = LoadAction::Clear; + color.store_action = StoreAction::Store; + color.clear_color = ClearColor::new(0.0, 0.0, 1.0, 1.0); + let queue = device.new_command_queue().map_err(|e| format!("{e:?}"))?; + let command_buffer = queue.command_buffer().map_err(|e| format!("{e:?}"))?; + let mut encoder = command_buffer.render_command_encoder(&pass).map_err(|e| format!("{e:?}"))?; + encoder.set_render_pipeline_state(&state).map_err(|e| format!("{e:?}"))?; + encoder.set_vertex_buffer(Some(&positions), 0, 0).map_err(|e| format!("{e:?}"))?; + encoder.set_vertex_buffer(Some(&colors), 0, 1).map_err(|e| format!("{e:?}"))?; + encoder.draw_primitives(PrimitiveType::Triangle, 0, 6, 1, 0).map_err(|e| format!("{e:?}"))?; + encoder.end_encoding().map_err(|e| format!("{e:?}"))?; + if let Some((buffer, row, side)) = readback { + let mut blit = command_buffer.blit_command_encoder().map_err(|e| format!("{e:?}"))?; + blit.copy_from_texture_to_buffer(target, 0, 0, Origin::new(0, 0, 0), Size::new(side, side, 1), buffer, 0, row, + row * side, BlitOption::None).map_err(|e| format!("{e:?}"))?; + blit.end_encoding().map_err(|e| format!("{e:?}"))?; + } + finish(&command_buffer) +} + +/// Texels off the expected picture: the top-left quarter drawn (255, 51, 0, 255), the rest cleared blue. +fn wrong_texels(bytes: &[u8], row: usize, side: usize) -> u32 { + let mut bad = 0; + for y in 0..side { + for x in 0..side { + let at = y * row + x * 4; + let inside = x < side / 2 && y < side / 2; + let expected: [u8; 4] = if inside { [255, 51, 0, 255] } else { [0, 0, 255, 255] }; + if bytes[at..at + 4] != expected { + bad += 1; + } + } + } + bad +} + +fn render(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const SIDE: u64 = 8; + let mut texture = TextureDescriptor::texture_2d(PixelFormat::RGBA8Unorm, SIDE, SIDE, false); + texture.usage = TextureUsage::RENDER_TARGET; + texture.resource_options = ResourceOptions::private(); + let target = device.new_texture(&texture).map_err(|e| format!("{e:?}"))?; + let readback = device.new_buffer(SIDE * 4 * SIDE, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + draw_quad(device, library, &target, Some((&readback, SIDE * 4, SIDE)))?; + let contents = readback.contents().ok_or("no contents")?; + // SAFETY: SIDE rows of SIDE * 4 bytes, no longer written. + let bytes = unsafe { std::slice::from_raw_parts(contents.as_ptr(), (SIDE * 4 * SIDE) as usize) }; + Ok(wrong_texels(bytes, (SIDE * 4) as usize, SIDE as usize)) +} + +/// The IOSurface path without an IOSurface: page-aligned host pages imported without a copy (the kernel pins them, Turnip maps +/// them into the GPU), a linear render target over them, CPU caches cleaned before and invalidated after the GPU's use. +fn import(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const SIDE: u64 = 64; + const ROW: u64 = 256; + let bytes = (ROW * SIDE) as usize; + // SAFETY: an anonymous private mapping of whole pages, unmapped below after every GPU use completed. + let pages = unsafe { + libc::mmap(std::ptr::null_mut(), bytes, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_PRIVATE | libc::MAP_ANON, -1, 0) + }; + if pages == libc::MAP_FAILED { + return Err("mmap".into()); + } + let result = (|| { + // SAFETY: the pages were just mapped; nothing else uses them. + unsafe { std::ptr::write_bytes(pages as *mut u8, 0x5a, bytes) }; + // SAFETY: the mapping is writable, page aligned and outlives the buffer (dropped before the unmap). + let memory = unsafe { HostMemory::new(NonNull::new(pages as *mut u8).ok_or("null")?) }; + // SAFETY: as above, for `bytes` bytes. + let buffer = unsafe { device.new_buffer_with_bytes_no_copy(memory, bytes as u64, ResourceOptions::shared(), None) } + .map_err(|e| format!("no-copy buffer: {e:?}"))?; + let mut texture = TextureDescriptor::texture_2d(PixelFormat::RGBA8Unorm, SIDE, SIDE, false); + texture.usage = TextureUsage::RENDER_TARGET | TextureUsage::SHADER_READ; + texture.resource_options = ResourceOptions::shared(); + let target = buffer.new_texture(&texture, 0, ROW).map_err(|e| format!("texture over pages: {e:?}"))?; + // SAFETY: the range is this mapping. + unsafe { sys_dcache_flush(pages, bytes) }; + draw_quad(device, library, &target, None)?; + // SAFETY: the GPU finished writing; drop any line the CPU fetched meanwhile. + unsafe { sys_dcache_flush(pages, bytes) }; + // SAFETY: the pages hold `bytes` bytes and no command buffer writes them now. + let view = unsafe { std::slice::from_raw_parts(pages as *const u8, bytes) }; + let bad = wrong_texels(view, ROW as usize, SIDE as usize); + drop(target); + drop(buffer); + Ok(bad) + })(); + // SAFETY: the buffer and texture over the pages are gone. + unsafe { libc::munmap(pages, bytes) }; + result +} + +/// The AIR slice of a universal metallib (a fat container, big endian, of an AIR64 metallib and an Apple-GPU binary, as the target's +/// frameworks ship them), as Metal itself picks it; a thin metallib is returned as it is. +pub fn air_slice(bytes: &[u8]) -> Option<&[u8]> { + const AIR64: u32 = 0x0100_0017; + let word = |at: usize| bytes.get(at..at + 4).map(|w| u32::from_be_bytes([w[0], w[1], w[2], w[3]])); + if word(0)? != 0xcafe_babe { + return Some(bytes); + } + let count = word(4)? as usize; + for i in 0..count.min(16) { + let at = 8 + 20 * i; + if word(at)? == AIR64 { + let (offset, size) = (word(at + 8)? as usize, word(at + 12)? as usize); + return bytes.get(offset..offset.checked_add(size)?); + } + } + None +} + +/// Translates the target's own QuartzCore library: every kernel into a compute pipeline, and the vertex/fragment pairs named +/// alike into render pipelines over a BGRA8 attachment. Counts only; refusals (function constants, tile functions) are expected. +fn quartzcore(run: &mut Run, device: &Device, bytes: &[u8]) -> Result<(), String> { + let bytes = air_slice(bytes).ok_or("QC library: no AIR64 slice")?; + let library = device.new_library_with_data(bytes).map_err(|e| format!("QC library: {e:?}"))?; + let names = library.function_names(); + let list = std::env::var_os("NMB_LIST").is_some(); + run.out.qc_functions = names.len() as u32; + let mut vertices = Vec::new(); + let mut fragments = Vec::new(); + for name in &names { + let Ok(function) = library.new_function_with_name(name) else { continue }; + if list { + eprintln!("fn {name} {:?}", function.function_type()); + } + match function.function_type() { + Ok(metal2vulkan::metal::FunctionType::Kernel) => { + match device.new_compute_pipeline_state_with_function(&function, &PipelineOptions::default()) { + Ok(_) => run.out.qc_kernels_ok += 1, + Err(e) => { + run.out.qc_kernels_failed += 1; + if list { + eprintln!("kernel {name}: {e:?}"); + } + } + } + } + Ok(metal2vulkan::metal::FunctionType::Vertex) => vertices.push((function, name.clone())), + Ok(metal2vulkan::metal::FunctionType::Fragment) => fragments.push((function, name.clone())), + _ => {} + } + } + // The library's own pairs: `_vert_` with `_frag_`. + for (vertex, vname) in &vertices { + let Some(fname) = [vname.replace("_vert_", "_frag_"), vname.replacen("_vert", "_frag", 1)] + .into_iter() + .find(|candidate| fragments.iter().any(|(_, n)| n == candidate)) + else { + continue; + }; + let fragment = &fragments.iter().find(|(_, n)| *n == fname).unwrap().0; + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function(vertex.function_descriptor())); + descriptor.fragment = Some(StageSource::Function(fragment.function_descriptor())); + descriptor.color_attachments[0] = Some(ColorAttachmentDescriptor::new(PixelFormat::BGRA8Unorm)); + match device.new_render_pipeline_state(&descriptor, &PipelineOptions::default()) { + Ok(_) => run.out.qc_pairs_ok += 1, + Err(e) => { + run.out.qc_pairs_failed += 1; + if list { + eprintln!("pair {vname} {fname}: {e:?}"); + } + } + } + } + Ok(()) +} + +fn selftest(run: &mut Run, loader: &CStr, cache: &CStr, qc: &[u8]) -> u32 { + run.step(0x6501); + let device = match open(loader, cache) { + Ok(device) => device, + Err(why) => return run.fail(0x6501, why), + }; + run.step(0x6502); + let library = match metal2vulkan::library::Library::from_air(FIXTURE_AIR) { + Ok(library) => device.new_library_with_compile_library(library), + Err(why) => return run.fail(0x6502, format!("{why:?}")), + }; + run.step(0x6503); + match compute(&device, &library) { + Ok(bad) => run.out.compute_bad = bad, + Err(why) => return run.fail(0x6503, why), + } + run.step(0x6504); + match render(&device, &library) { + Ok(bad) => run.out.render_bad = bad, + Err(why) => return run.fail(0x6504, why), + } + run.step(0x6505); + match import(&device, &library) { + Ok(bad) => run.out.import_bad = bad, + Err(why) => return run.fail(0x6505, why), + } + if !qc.is_empty() { + run.step(0x6506); + if let Err(why) = quartzcore(run, &device, qc) { + return run.fail(0x6506, why); + } + } + run.step(0x650f); + 0 +} + +/// Runs the self-test; returns 0 or the marker of the failed step (also in out->stage, with out->message). +/// +/// # Safety +/// `loader` and `cache` are C strings; `qc` points at `qc_len` bytes (or is null with 0); `out` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_selftest(loader: *const c_char, cache: *const c_char, qc: *const u8, qc_len: usize, mark: Mark, + out: *mut NmbSelftest) -> u32 { + if loader.is_null() || cache.is_null() || out.is_null() { + return 0x65ff; + } + // SAFETY: the caller's contract. + let out = unsafe { &mut *out }; + // SAFETY: as above. + let (loader, cache) = unsafe { (CStr::from_ptr(loader), CStr::from_ptr(cache)) }; + let qc: &[u8] = if qc.is_null() || qc_len == 0 { &[] } else { unsafe { std::slice::from_raw_parts(qc, qc_len) } }; + // SAFETY: plain integers and bytes. + *out = unsafe { std::mem::zeroed() }; + let mut run = Run { out, mark }; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| selftest(&mut run, loader, cache, qc))) { + Ok(code) => code, + Err(_) => run.fail(0x65fe, "panic"), + } +} diff --git a/bridge/NativeMetalObjects.h b/bridge/NativeMetalObjects.h new file mode 100644 index 0000000..2ba32ff --- /dev/null +++ b/bridge/NativeMetalObjects.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: MIT */ +/* C ABI of the bridge's Metal objects (NativeMetalObjects.rs): the metal2vulkan runtime's device, resources, states, queues, command buffers + * and encoders as opaque handles, for the Objective-C frontend (NativeBridgeMetalFrontend.mm). Enumerations and option words are Metal's raw + * values. A call that fails returns 0/NULL and leaves its reason in nmb_last_error() (per thread). Every handle is released once with its + * release call; an encoder is released by its end call (or by nmb_encoder_discard when it is never ended). */ +#ifndef NATIVE_METAL_OBJECTS_H +#define NATIVE_METAL_OBJECTS_H +#include +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +typedef struct NmbDevice NmbDevice; +typedef struct NmbBuffer NmbBuffer; +typedef struct NmbHeap NmbHeap; +typedef struct NmbTexture NmbTexture; +typedef struct NmbLibrary NmbLibrary; +typedef struct NmbFunction NmbFunction; +typedef struct NmbComputePipeline NmbComputePipeline; +typedef struct NmbRenderPipeline NmbRenderPipeline; +typedef struct NmbSampler NmbSampler; +typedef struct NmbDepthStencil NmbDepthStencil; +typedef struct NmbQueue NmbQueue; +typedef struct NmbCommandBuffer NmbCommandBuffer; +typedef struct NmbEncoder NmbEncoder; +typedef struct NmbEvent NmbEvent; +typedef struct NmbFence NmbFence; + +const char *nmb_last_error(void); +/* The bridge's embedded AIR fixtures: the runtime's own runtime_examples and kernel_function_constant_gate_fold. */ +const void *nmb_fixture_air(size_t *length); +const void *nmb_function_constant_air(size_t *length); + +/* loader: a library exporting vkGetInstanceProcAddr (Turnip on the tablet). cache: a writable directory for translated shaders. */ +NmbDevice *nmb_device_open(const char *loader, const char *cache); +void nmb_device_release(NmbDevice *); +size_t nmb_device_name(const NmbDevice *, char *out, size_t size); +uint64_t nmb_device_max_buffer_length(const NmbDevice *); +// Pipeline makings (count, total us, longest us, waited us, failed), sink messages (info, warning+), shader cache hits/misses/writes. +bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]); +/* Artifact lookups and driver pipeline records are separate cache layers. */ +struct NmbShaderCacheStats { + uint64_t hits, misses, rejected, read_failures, writes, write_failures, evictions; + uint64_t driver_loads, driver_hits, driver_misses, driver_saves, driver_truncated, driver_mismatched, driver_unchanged; +}; +bool nmb_device_shader_cache_stats(const NmbDevice *, struct NmbShaderCacheStats *); +size_t nmb_last_warning(char *out, size_t size); +bool nmb_device_max_threads_per_threadgroup(const NmbDevice *, uint64_t out[3]); +/* Device queries the runtime answers by Metal selector: Apple's private counts and flags where the device has an answer of its own, + * and the public queries. A selector's kind says its result and argument. */ +enum {NMB_ANSWER_COUNT = 0, NMB_ANSWER_FLAG = 1, NMB_ANSWER_COUNT_ARG = 2, NMB_ANSWER_FLAG_ARG = 3}; +const char *nmb_device_answer_selector(size_t index, uint32_t *kind /* NMB_ANSWER_* */); /* NULL past the last */ +bool nmb_device_answer(const NmbDevice *, const char *selector, uint64_t arg, uint64_t *out); /* false: no answer, nmb_last_error says why */ +size_t nmb_device_vendor_name(const NmbDevice *, char *out, size_t size); +uint64_t nmb_device_linear_bytes_per_row(const NmbDevice *, uint64_t format, uint64_t width); /* 0: refused */ + +typedef struct NmbBinaryArchive NmbBinaryArchive; +NmbBinaryArchive *nmb_binary_archive_new(NmbDevice *, const char *path /* NULL: empty */); +void nmb_binary_archive_release(NmbBinaryArchive *); +bool nmb_binary_archive_serialize(NmbBinaryArchive *, const char *path); + +NmbBuffer *nmb_buffer_new(NmbDevice *, uint64_t length, uint64_t options, const void *bytes /* NULL: zeroed */); +/* Wraps page-aligned memory the caller keeps mapped until the buffer is released (an IOSurface's base, host pages). */ +NmbBuffer *nmb_buffer_new_no_copy(NmbDevice *, void *pointer, uint64_t length, uint64_t options); +void nmb_buffer_release(NmbBuffer *); +void *nmb_buffer_contents(const NmbBuffer *); +uint64_t nmb_buffer_length(const NmbBuffer *); +uint64_t nmb_buffer_gpu_address(const NmbBuffer *); + +struct NmbTextureDescriptor { + uint64_t type, format, width, height, depth, levels, samples, array_length, usage, options; +}; +/* Heaps: type 0 automatic / 1 placement; offset UINT64_MAX asks an automatic heap to place the resource. */ +NmbHeap *nmb_heap_new(NmbDevice *, uint64_t type, uint64_t size, uint64_t options); +void nmb_heap_release(NmbHeap *); +uint64_t nmb_heap_size(const NmbHeap *); +uint64_t nmb_heap_used_size(const NmbHeap *); +uint64_t nmb_heap_allocated_size(const NmbHeap *); +uint64_t nmb_heap_max_available_size(const NmbHeap *, uint64_t alignment); +uint64_t nmb_heap_set_purgeable_state(const NmbHeap *, uint64_t state); +NmbBuffer *nmb_heap_new_buffer(const NmbHeap *, uint64_t length, uint64_t options, uint64_t offset); +NmbTexture *nmb_heap_new_texture(const NmbHeap *, const struct NmbTextureDescriptor *, uint64_t offset); +NmbTexture *nmb_texture_new(NmbDevice *, const struct NmbTextureDescriptor *); +/* A linear texture over a buffer's bytes; iosurface marks the buffer as standing for a surface without planes (Metal's IOSurface rules). */ +NmbTexture *nmb_texture_new_on_buffer(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, bool iosurface); +NmbTexture *nmb_texture_new_linear(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image); +/* whole: a view of every level and slice as format; else levels and slices are {start, count} and kind the view's texture type. */ +NmbTexture *nmb_texture_new_view(NmbTexture *, uint64_t format, uint64_t kind, const uint64_t levels[2], const uint64_t slices[2], bool whole); +/* setPurgeableState: with Metal's raw state; the answer's raw state, 0 where it fails. */ +uint64_t nmb_texture_set_purgeable_state(const NmbTexture *, uint64_t state); +uint64_t nmb_buffer_set_purgeable_state(const NmbBuffer *, uint64_t state); +void nmb_texture_release(NmbTexture *); +void nmb_texture_describe(const NmbTexture *, struct NmbTextureDescriptor *out); +bool nmb_texture_replace(NmbTexture *, const uint64_t region[6], uint64_t level, uint64_t slice, const void *bytes, uint64_t bytes_per_row, + uint64_t bytes_per_image, uint64_t length); +bool nmb_texture_get_bytes(const NmbTexture *, void *bytes, uint64_t bytes_per_row, uint64_t bytes_per_image, uint64_t length, + const uint64_t region[6], uint64_t level, uint64_t slice); + +/* A metallib (a universal one gives its AIR64 slice), or with air set bare AIR bitcode. */ +NmbLibrary *nmb_library_new(NmbDevice *, const void *bytes, size_t length, bool air); +void nmb_library_release(NmbLibrary *); +uint32_t nmb_library_function_count(const NmbLibrary *); +size_t nmb_library_function_name(const NmbLibrary *, uint32_t index, char *out, size_t size); +NmbFunction *nmb_function_new(NmbLibrary *, const char *name); +/* A function specialized with constant values (newFunctionWithName:constantValues:error:): type is Metal's MTLDataType, name is null for an + * indexed value; bytes holds the value's bytes. */ +struct NmbConstant {uint64_t type, index; const char *name; const void *bytes; uint64_t length;}; +NmbFunction *nmb_function_new_specialized(NmbLibrary *, const char *name, const struct NmbConstant *constants, uint32_t count); +void nmb_function_release(NmbFunction *); +/* Metal's MTLFunctionType (1 vertex, 2 fragment, 3 kernel, ...). */ +uint64_t nmb_function_type(const NmbFunction *); + +NmbComputePipeline *nmb_compute_pipeline_new(NmbDevice *, NmbFunction *); +void nmb_compute_pipeline_release(NmbComputePipeline *); +uint64_t nmb_compute_pipeline_max_threads(const NmbComputePipeline *); +uint64_t nmb_compute_pipeline_execution_width(const NmbComputePipeline *); + +struct NmbColorAttachment { + uint64_t format, write_mask, blending, source_rgb, destination_rgb, rgb_operation, source_alpha, destination_alpha, alpha_operation; +}; +struct NmbVertexAttribute {uint64_t format, offset, buffer;}; +struct NmbVertexLayout {uint64_t stride, step_function, step_rate;}; +struct NmbRenderPipelineDescriptor { + NmbFunction *vertex, *fragment; + struct NmbColorAttachment colors[8]; + uint64_t depth_format, stencil_format, samples, topology, rasterization; + uint64_t vertex_inputs; /* 0: no vertex descriptor */ + struct NmbVertexAttribute attributes[31]; + struct NmbVertexLayout layouts[31]; +}; +NmbRenderPipeline *nmb_render_pipeline_new(NmbDevice *, const struct NmbRenderPipelineDescriptor *); +/* Descriptor binaryArchives and MTLPipelineOption bits. NULL options means defaults. + * The archive array and its handles must remain valid for the duration of this call. */ +struct NmbPipelineOptions {uint64_t flags; NmbBinaryArchive *const *archives; size_t archive_count;}; +struct NmbComputePipelineDescriptor { + NmbFunction *function; + uint64_t max_threads, multiple_of_execution_width, indirect, max_call_stack_depth, adding_binary_functions; + uint64_t mutability[31]; +}; +NmbComputePipeline *nmb_compute_pipeline_new_with_options(NmbDevice *, NmbFunction *, const struct NmbPipelineOptions *); +NmbComputePipeline *nmb_compute_pipeline_new_with_descriptor(NmbDevice *, const struct NmbComputePipelineDescriptor *, const struct NmbPipelineOptions *); +NmbRenderPipeline *nmb_render_pipeline_new_with_options(NmbDevice *, const struct NmbRenderPipelineDescriptor *, const struct NmbPipelineOptions *); +/* The archive makes the state as the matching new call does and keeps what it used (addRender/ComputePipelineFunctionsWithDescriptor:). */ +bool nmb_binary_archive_add_render(NmbBinaryArchive *, const struct NmbRenderPipelineDescriptor *); +bool nmb_binary_archive_add_compute(NmbBinaryArchive *, NmbFunction *); +bool nmb_binary_archive_add_compute_descriptor(NmbBinaryArchive *, const struct NmbComputePipelineDescriptor *); +void nmb_render_pipeline_release(NmbRenderPipeline *); + +struct NmbSamplerDescriptor { + uint64_t min_filter, mag_filter, mip_filter, s_address, t_address, r_address, border_color, normalized, max_anisotropy, compare, lod_average; + float lod_min, lod_max; +}; +NmbSampler *nmb_sampler_new(NmbDevice *, const struct NmbSamplerDescriptor *); +void nmb_sampler_release(NmbSampler *); + +struct NmbStencil {uint64_t compare, failure, depth_failure, pass, read_mask, write_mask;}; +struct NmbDepthStencilDescriptor {uint64_t compare, write; struct NmbStencil front, back;}; +NmbDepthStencil *nmb_depth_stencil_new(NmbDevice *, const struct NmbDepthStencilDescriptor *); +void nmb_depth_stencil_release(NmbDepthStencil *); + +NmbQueue *nmb_queue_new(NmbDevice *); +void nmb_queue_release(NmbQueue *); +NmbCommandBuffer *nmb_command_buffer_new(NmbQueue *); +void nmb_command_buffer_release(NmbCommandBuffer *); +/* 1 storage dependency, 0 unrelated/already complete, -1 unknown (caller must wait). + * Includes texture roots, buffer-backed textures, heaps and overlapping imported host ranges. + * This frontend does not rebind/remap live backing ranges. */ +int nmb_command_buffer_uses_buffer(const NmbCommandBuffer *, const NmbBuffer *); +int nmb_command_buffer_uses_texture(const NmbCommandBuffer *, const NmbTexture *); +/* The handler runs once, on a runtime thread, after the command buffer completed or failed. */ +bool nmb_command_buffer_on_completed(NmbCommandBuffer *, void (*handler)(void *context), void *context); +/* Runs on actual scheduling, before completion. Not called if submission fails before scheduling. + * The caller keeps context alive until completion (or a refused commit). */ +bool nmb_command_buffer_on_scheduled(NmbCommandBuffer *, void (*handler)(void *context), void *context); +bool nmb_command_buffer_commit(NmbCommandBuffer *); +/* Wait for scheduling and scheduled handlers only; false on a refused wait. */ +bool nmb_command_buffer_wait_scheduled(NmbCommandBuffer *); +/* 0 completed in time, 1 timed out (timeout_ms 0 waits without a bound). */ +int nmb_command_buffer_wait(NmbCommandBuffer *, uint64_t timeout_ms); +/* Metal's MTLCommandBufferStatus (0 not enqueued .. 4 completed, 5 error). */ +uint64_t nmb_command_buffer_status(const NmbCommandBuffer *); +size_t nmb_command_buffer_error(const NmbCommandBuffer *, char *out, size_t size); +double nmb_command_buffer_gpu_time(const NmbCommandBuffer *, bool end); + +/* Events: an MTLEvent (shared false) or an MTLSharedEvent within this process. */ +NmbEvent *nmb_event_new(NmbDevice *, bool shared); +void nmb_event_release(NmbEvent *); +uint64_t nmb_shared_event_value(const NmbEvent *); +bool nmb_shared_event_set_value(NmbEvent *, uint64_t value); +/* true once the value is reached within timeout_ms */ +bool nmb_shared_event_wait(NmbEvent *, uint64_t value, uint64_t timeout_ms); +/* handler(context, value) runs once on the bridge's listener thread when the value is reached (at once if it already is). */ +bool nmb_shared_event_notify(NmbEvent *, uint64_t value, void (*handler)(void *context, uint64_t value), void *context); +bool nmb_command_buffer_signal_event(NmbCommandBuffer *, NmbEvent *, uint64_t value); +bool nmb_command_buffer_wait_event(NmbCommandBuffer *, NmbEvent *, uint64_t value); +NmbFence *nmb_fence_new(NmbDevice *); +void nmb_fence_release(NmbFence *); +/* stages: Metal's MTLRenderStages for a render encoder, ignored otherwise */ +bool nmb_encoder_update_fence(NmbEncoder *, NmbFence *, uint64_t stages); +bool nmb_encoder_wait_fence(NmbEncoder *, NmbFence *, uint64_t stages); + +NmbEncoder *nmb_compute_encoder_new(NmbCommandBuffer *); +NmbEncoder *nmb_blit_encoder_new(NmbCommandBuffer *); +struct NmbPassAttachment { + NmbTexture *texture; + uint64_t level, slice, depth_plane, load, store; + double clear[4]; /* color, or clear[0] depth / stencil */ +}; +struct NmbRenderPassDescriptor { + struct NmbPassAttachment colors[8], depth, stencil; + uint64_t width, height, array_length, samples; + NmbBuffer *visibility; /* the pass's visibility result buffer, or null */ +}; +NmbEncoder *nmb_render_encoder_new(NmbCommandBuffer *, const struct NmbRenderPassDescriptor *); +bool nmb_encoder_end(NmbEncoder *); +void nmb_encoder_discard(NmbEncoder *); + +/* stage: 0 compute, 1 vertex, 2 fragment */ +bool nmb_encoder_set_buffer(NmbEncoder *, uint32_t stage, NmbBuffer *, uint64_t offset, uint32_t index); +bool nmb_encoder_set_buffer_offset(NmbEncoder *, uint32_t stage, uint64_t offset, uint32_t index); +bool nmb_encoder_set_bytes(NmbEncoder *, uint32_t stage, const void *bytes, size_t length, uint32_t index); +bool nmb_encoder_set_texture(NmbEncoder *, uint32_t stage, NmbTexture *, uint32_t index); +bool nmb_encoder_set_sampler(NmbEncoder *, uint32_t stage, NmbSampler *, uint32_t index); + +bool nmb_compute_set_pipeline(NmbEncoder *, NmbComputePipeline *); +bool nmb_compute_set_threadgroup_memory(NmbEncoder *, uint32_t length, uint32_t index); +bool nmb_compute_dispatch(NmbEncoder *, const uint64_t size[3], const uint64_t per_group[3], bool threads); + +bool nmb_render_set_pipeline(NmbEncoder *, NmbRenderPipeline *); +bool nmb_render_set_depth_stencil(NmbEncoder *, NmbDepthStencil *); +bool nmb_render_set_viewport(NmbEncoder *, const double viewport[6]); +bool nmb_render_set_scissor(NmbEncoder *, const uint64_t rect[4]); +bool nmb_render_set_raster(NmbEncoder *, uint64_t cull_mode, uint64_t winding, uint64_t fill_mode); +bool nmb_render_set_stencil_reference(NmbEncoder *, uint32_t front, uint32_t back); +bool nmb_render_set_blend_color(NmbEncoder *, const float color[4]); +bool nmb_render_set_depth_bias(NmbEncoder *, float bias, float slope_scale, float clamp); +bool nmb_render_set_line_width(NmbEncoder *, float width); +bool nmb_render_set_visibility_mode(NmbEncoder *, uint64_t mode, uint64_t offset); +bool nmb_render_draw(NmbEncoder *, uint64_t primitive, uint32_t start, uint32_t count, uint32_t instances, uint32_t base_instance); +bool nmb_render_draw_indexed(NmbEncoder *, uint64_t primitive, uint32_t count, uint64_t index_type, NmbBuffer *indices, uint64_t offset, + uint32_t instances, int32_t base_vertex, uint32_t base_instance); + +bool nmb_blit_copy_buffer(NmbEncoder *, NmbBuffer *source, uint64_t source_offset, NmbBuffer *destination, uint64_t destination_offset, + uint64_t size); +bool nmb_blit_fill_buffer(NmbEncoder *, NmbBuffer *, uint64_t start, uint64_t end, uint8_t value); +/* origins and sizes are x,y,z / width,height,depth */ +bool nmb_blit_copy_texture(NmbEncoder *, NmbTexture *source, uint64_t source_slice, uint64_t source_level, const uint64_t source_origin[3], + const uint64_t size[3], NmbTexture *destination, uint64_t destination_slice, uint64_t destination_level, + const uint64_t destination_origin[3]); +bool nmb_blit_buffer_to_texture(NmbEncoder *, NmbBuffer *source, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image, + const uint64_t size[3], NmbTexture *destination, uint64_t slice, uint64_t level, const uint64_t origin[3]); +bool nmb_blit_texture_to_buffer(NmbEncoder *, NmbTexture *source, uint64_t slice, uint64_t level, const uint64_t origin[3], + const uint64_t size[3], NmbBuffer *destination, uint64_t offset, uint64_t bytes_per_row, + uint64_t bytes_per_image); +bool nmb_blit_generate_mipmaps(NmbEncoder *, NmbTexture *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/bridge/NativeMetalObjects.rs b/bridge/NativeMetalObjects.rs new file mode 100644 index 0000000..1ac4904 --- /dev/null +++ b/bridge/NativeMetalObjects.rs @@ -0,0 +1,1987 @@ +// SPDX-License-Identifier: MIT +//! The bridge's Metal objects behind a C ABI (NativeMetalObjects.h): the metal2vulkan runtime's device, resources, states, queues, +//! command buffers and encoders as boxed handles for the Objective-C frontend. Raw Metal enumerations go through the runtime's own +//! checked conversions; a value it does not know fails the call (never a substitute). Failures leave their reason in a per-thread +//! string, and no call unwinds into C. + +use metal2vulkan::bind::SamplerDescriptor; +use metal2vulkan::metal::state::{DepthStencilDescriptor, StencilDescriptor}; +use metal2vulkan::library::{FunctionConstantValue, FunctionConstantValues}; +use metal2vulkan::metal::{ + BlendFactor, BlendOperation, ColorWriteMask, CompareFunction, CullMode, DataType, IndexType, PixelFormat, PrimitiveTopologyClass, + PrimitiveType, RenderStages, SamplerAddressMode, SamplerBorderColor, SamplerMinMagFilter, SamplerMipFilter, StencilOperation, TextureType, + TextureUsage, TriangleFillMode, VertexFormat, VertexStepFunction, Winding, +}; +use metal2vulkan::pipeline::{ + ColorAttachmentDescriptor, ComputePipelineDescriptor, RenderPipelineDescriptor, StageSource, VertexAttributeDescriptor, VertexBufferLayoutDescriptor, + VertexDescriptor, +}; +use metal2vulkan_runtime::{ + BinaryArchive, BinaryArchiveDescriptor, BlitCommandEncoder, FamilyRow, FeatureRow, GpuFamily, RowState, BlitOption, Buffer, ClearColor, CommandBuffer, CommandQueue, ComputeCommandEncoder, + ComputePipelineState, DebugMessage, DepthStencilState, Device, DeviceOptions, MessageSeverity, MessageSink, Event, EventRef, Fence, Function, Heap, HeapDescriptor, HeapType, HostMemory, IoSurfacePlane, Library, LoadAction, + MetalLevel, Origin, PipelineOption, PipelineOptions, PurgeableState, PrivateCount, PrivateFlag, PrivateProperties, MAX_COLOR_ATTACHMENTS, Region, RenderCommandEncoder, RenderPassDescriptor, RenderPipelineState, ResourceOptions, + SamplerState, ScissorRect, SharedEvent, SharedEventListener, Size, StoreAction, Texture, TextureDescriptor, Viewport, VisibilityResultMode, +}; +use std::cell::RefCell; +use std::ffi::{c_char, c_void, CStr}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::time::Duration; + +thread_local! { + static LAST: RefCell> = const { RefCell::new(Vec::new()) }; +} + +fn set_error(why: String) { + LAST.with(|last| { + let mut last = last.borrow_mut(); + last.clear(); + last.extend(why.bytes().filter(|b| *b != 0)); + last.push(0); + }); +} + +/// Runs `body`, turning an error or a panic into the per-thread reason and `fallback`. +fn guard(fallback: T, body: impl FnOnce() -> Result) -> T { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)) { + Ok(Ok(value)) => value, + Ok(Err(why)) => { + set_error(why); + fallback + } + Err(_) => { + set_error("panic in the Metal runtime".into()); + fallback + } + } +} + +fn boxed(value: T) -> *mut T { + Box::into_raw(Box::new(value)) +} + +/// The object a handle points at. +fn get<'a, T>(handle: *const T) -> Result<&'a T, String> { + // SAFETY: handles come from `boxed` and live until their release call (the header's contract). + unsafe { handle.as_ref() }.ok_or_else(|| "null handle".to_string()) +} + +fn get_mut<'a, T>(handle: *mut T) -> Result<&'a mut T, String> { + // SAFETY: as in `get`; an encoder is used by one thread at a time (Metal's rule). + unsafe { handle.as_mut() }.ok_or_else(|| "null handle".to_string()) +} + +fn release(handle: *mut T) { + if !handle.is_null() { + // SAFETY: from `boxed`, released once. + drop(unsafe { Box::from_raw(handle) }); + } +} + +fn copy_out(text: &str, out: *mut c_char, size: usize) -> usize { + if !out.is_null() && size > 0 { + let n = text.len().min(size - 1); + // SAFETY: the caller's buffer holds `size` bytes. + unsafe { + std::ptr::copy_nonoverlapping(text.as_ptr(), out as *mut u8, n); + *out.add(n) = 0; + } + } + text.len() +} + +fn raw>(value: u64, what: &str) -> Result +where + T::Error: std::fmt::Debug, +{ + u32::try_from(value).ok().and_then(|v| T::try_from(v).ok()).ok_or_else(|| format!("{what} {value} is not a Metal value")) +} + +fn e(what: &'static str) -> impl FnOnce(E) -> String { + move |error| format!("{what}: {error:?}") +} + +fn options(value: u64) -> Result { + ResourceOptions::from_raw(value).map_err(e("resource options")) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_last_error() -> *const c_char { + LAST.with(|last| { + let last = last.borrow(); + if last.is_empty() { c"".as_ptr() } else { last.as_ptr() as *const c_char } + }) +} + +/// The runtime's kernel_function_constant_gate_fold fixture, assembled to AIR at build time. +static FUNCTION_CONSTANT_AIR: &[u8] = include_bytes!(env!("NMB_FC_AIR")); + +/// # Safety +/// `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_fixture_air(length: *mut usize) -> *const c_void { + // SAFETY: the caller's contract. + if let Some(length) = unsafe { length.as_mut() } { + *length = crate::FIXTURE_AIR.len(); + } + crate::FIXTURE_AIR.as_ptr() as *const c_void +} + +/// # Safety +/// `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_constant_air(length: *mut usize) -> *const c_void { + // SAFETY: the caller's contract. + if let Some(length) = unsafe { length.as_mut() } { + *length = FUNCTION_CONSTANT_AIR.len(); + } + FUNCTION_CONSTANT_AIR.as_ptr() as *const c_void +} + +// ---- device ---- + +pub struct NmbDevice(Device, std::sync::OnceLock, std::sync::Mutex<()>); + +impl NmbDevice { + /// The runtime judges the private properties by the family rows once. + fn private(&self) -> &PrivateProperties { + self.1.get_or_init(|| self.0.private_properties()) + } +} + +/// # Safety +/// `loader` and `cache` are C strings. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_char) -> *mut NmbDevice { + guard(std::ptr::null_mut(), || { + if loader.is_null() || cache.is_null() { + return Err("loader and cache are required".into()); + } + // SAFETY: the caller's contract. + let (loader, cache) = unsafe { (CStr::from_ptr(loader), CStr::from_ptr(cache)) }; + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + options.metal_level = Some(MetalLevel::NEWEST); + options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + options.messages = Some(MessageSink::new(observe_message)); + Ok(boxed(NmbDevice(Device::system_default_with(&options).map_err(e("device"))?, std::sync::OnceLock::new(), std::sync::Mutex::new(())))) + }) +} + +// What the runtime says through the device's sink, kept as numbers for the plugin's telemetry: each render VkPipeline making the +// creation log (METAL2VULKAN_LOG_PIPELINES) reports ("... made in ms[ after waiting ms for its turn]"), and the other +// messages by severity with the last warning's text. +static PIPELINE_MAKINGS: [std::sync::atomic::AtomicU64; 5] = [const { std::sync::atomic::AtomicU64::new(0) }; 5]; // n, us, max us, waited us, failed +static OTHER_MESSAGES: [std::sync::atomic::AtomicU64; 2] = [const { std::sync::atomic::AtomicU64::new(0) }; 2]; // info, warning or worse +static LAST_WARNING: std::sync::Mutex = std::sync::Mutex::new(String::new()); + +fn millis_after(text: &str, marker: &str) -> Option { + let at = text.find(marker)? + marker.len(); + let rest = &text[at..]; + let end = rest.find(' ').unwrap_or(rest.len()); + rest[..end].parse().ok() +} + +fn observe_message(message: &DebugMessage) { + use std::sync::atomic::Ordering::Relaxed; + if message.id_name == "metal2vulkan-pipeline-creation" { + let made = millis_after(&message.text, " made in "); + let us = made.or_else(|| millis_after(&message.text, " failed in ")).map_or(0, |ms| (ms * 1000.0) as u64); + PIPELINE_MAKINGS[0].fetch_add(1, Relaxed); + PIPELINE_MAKINGS[1].fetch_add(us, Relaxed); + PIPELINE_MAKINGS[2].fetch_max(us, Relaxed); + PIPELINE_MAKINGS[3].fetch_add(millis_after(&message.text, " after waiting ").map_or(0, |ms| (ms * 1000.0) as u64), Relaxed); + if made.is_none() { + PIPELINE_MAKINGS[4].fetch_add(1, Relaxed); + } + return; + } + if message.severity >= MessageSeverity::Warning { + OTHER_MESSAGES[1].fetch_add(1, Relaxed); + if let Ok(mut last) = LAST_WARNING.lock() { + *last = message.text.chars().take(160).collect(); + } + } else { + OTHER_MESSAGES[0].fetch_add(1, Relaxed); + } +} + +/// The sink's counts into the caller's ten words: pipeline makings (count, total us, longest us, waited us, failed), messages (info, +/// warning or worse), and the device's shader cache hits, misses and writes. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_work_stats(device: *const NmbDevice, out: *mut u64) -> bool { + use std::sync::atomic::Ordering::Relaxed; + guard(false, || { + if out.is_null() { + return Err("null stats".into()); + } + // SAFETY: the header's ten words. + let out = unsafe { std::slice::from_raw_parts_mut(out, 10) }; + for (i, v) in PIPELINE_MAKINGS.iter().enumerate() { + out[i] = v.load(Relaxed); + } + out[5] = OTHER_MESSAGES[0].load(Relaxed); + out[6] = OTHER_MESSAGES[1].load(Relaxed); + let cache = get(device)?.0.shader_cache_statistics(); + out[7] = cache.hits; + out[8] = cache.misses; + out[9] = cache.writes; + Ok(true) + }) +} + +/// Separate artifact and driver-record counters, mirroring the C ABI. +#[repr(C)] +pub struct NmbShaderCacheStats { + pub hits: u64, + pub misses: u64, + pub rejected: u64, + pub read_failures: u64, + pub writes: u64, + pub write_failures: u64, + pub evictions: u64, + pub driver_loads: u64, + pub driver_hits: u64, + pub driver_misses: u64, + pub driver_saves: u64, + pub driver_truncated: u64, + pub driver_mismatched: u64, + pub driver_unchanged: u64, +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_shader_cache_stats(device: *const NmbDevice, out: *mut NmbShaderCacheStats) -> bool { + guard(false, || { + let out = get_mut(out)?; + let stats = get(device)?.0.shader_cache_statistics(); + *out = NmbShaderCacheStats { + hits: stats.hits, misses: stats.misses, rejected: stats.rejected, + read_failures: stats.read_failures, writes: stats.writes, write_failures: stats.write_failures, evictions: stats.evictions, + driver_loads: stats.driver_loads, driver_hits: stats.driver_pipeline_hits, driver_misses: stats.driver_pipeline_misses, + driver_saves: stats.driver_saves, driver_truncated: stats.driver_data_truncated, + driver_mismatched: stats.driver_data_mismatched, driver_unchanged: stats.driver_data_unchanged, + }; + Ok(true) + }) +} + +/// The last warning the device's sink received, as a C string; returns its length. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_last_warning(out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(copy_out(&LAST_WARNING.lock().map(|s| s.clone()).unwrap_or_default(), out, size))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_release(device: *mut NmbDevice) { + release(device) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_name(device: *const NmbDevice, out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(copy_out(get(device)?.0.name(), out, size))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_max_buffer_length(device: *const NmbDevice) -> u64 { + guard(0, || Ok(get(device)?.0.properties().max_buffer_length)) +} + +/// `maxThreadsPerThreadgroup` into the caller's three words. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_max_threads_per_threadgroup(device: *const NmbDevice, out: *mut u64) -> bool { + guard(false, || { + if out.is_null() { + return Err("null size".into()); + } + let s = get(device)?.0.properties().max_threads_per_threadgroup; + // SAFETY: the header's three words. + unsafe { std::slice::from_raw_parts_mut(out, 3) }.copy_from_slice(&[s.width, s.height, s.depth]); + Ok(true) + }) +} + +/// The device selectors `nmb_device_answer` takes beyond Apple's private counts and flags: the selector and its kind +/// (the header's NMB_ANSWER_* values). +const PUBLIC_ANSWERS: &[(&str, u32)] = &[ + ("maxThreadgroupMemoryLength", 0), + ("argumentBuffersSupport", 0), + ("readWriteTextureSupport", 0), + ("maxColorAttachments", 0), + ("minConstantBufferAlignmentBytes", 0), + ("deviceLinearReadOnlyTextureAlignmentBytes", 0), + ("areRasterOrderGroupsSupported", 1), + ("supportsRasterOrderGroups", 1), + ("isDepth24Stencil8PixelFormatSupported", 1), + ("areProgrammableSamplePositionsSupported", 1), + ("supports32BitFloatFiltering", 1), + ("supportsMemorylessRenderTargets", 1), + ("supportsSamplerReductionMode", 1), + ("supportsYCBCRFormats12", 1), + ("supportsYCBCRPackedFormats12", 1), + ("supportsSIMDGroup", 1), + ("minimumLinearTextureAlignmentForPixelFormat:", 2), + ("supportsTextureSampleCount:", 3), + ("supportsVertexAmplificationCount:", 3), + ("supportsRasterizationRateMapWithLayerCount:", 3), + ("supportsFamily:", 3), + ("supportsPrimitiveType:", 3), + ("supportsTextureWriteRoundingMode:", 3), +]; + +/// Every selector `nmb_device_answer` takes, with its kind, as C strings. +fn answer_selectors() -> &'static [(std::ffi::CString, u32)] { + static ALL: std::sync::OnceLock> = std::sync::OnceLock::new(); + ALL.get_or_init(|| { + let counts = PrivateCount::ALL.into_iter().map(|c| (c.selector(), 0)); + let flags = PrivateFlag::ALL.into_iter().map(|f| (f.selector(), 1)); + counts + .chain(flags) + .chain(PUBLIC_ANSWERS.iter().copied()) + .map(|(s, kind)| (std::ffi::CString::new(s).expect("selector"), kind)) + .collect() + }) +} + +/// The selector at `index` of the ones `nmb_device_answer` takes and its kind; null past the last. +/// +/// # Safety +/// `kind` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_answer_selector(index: usize, kind: *mut u32) -> *const c_char { + let Some((selector, k)) = answer_selectors().get(index) else { return std::ptr::null() }; + // SAFETY: the caller's contract. + if let Some(kind) = unsafe { kind.as_mut() } { + *kind = *k; + } + selector.as_ptr() +} + +fn device_answer(device: &NmbDevice, selector: &str, arg: u64) -> Result { + let d = &device.0; + let p = d.properties(); + let none = || format!("the device has no answer for {selector}"); + if let Some(c) = PrivateCount::ALL.into_iter().find(|c| c.selector() == selector) { + return device.private().count(c).value().ok_or_else(none); + } + if let Some(f) = PrivateFlag::ALL.into_iter().find(|f| f.selector() == selector) { + return device.private().flag(f).value().map(u64::from).ok_or_else(none); + } + let gap = |r: Result| r.map(u64::from).map_err(|g| format!("{selector}: {g:?}")); + let format_supported = |f: PixelFormat| d.pixel_format_capabilities(f) != metal2vulkan_runtime::FormatCapabilities::NONE; + Ok(match selector { + "maxThreadgroupMemoryLength" => p.max_threadgroup_memory_length, + "argumentBuffersSupport" => p.argument_buffers_support.as_raw() as u64, + "readWriteTextureSupport" => p.read_write_texture_support.as_raw() as u64, + // The runtime's pass takes MAX_COLOR_ATTACHMENTS; the device's Vulkan limits bound what a pipeline links. + "maxColorAttachments" => { + let l = &p.description.limits; + (MAX_COLOR_ATTACHMENTS as u64).min(l.max_color_attachments.min(l.max_fragment_output_attachments) as u64) + } + // The runtime binds a buffer at any offset within it (render.rs, setVertexBuffer); 4 is a scalar's alignment, which the shader's + // own loads need. + "minConstantBufferAlignmentBytes" => 4, + "deviceLinearReadOnlyTextureAlignmentBytes" => { + device.private().count(PrivateCount::DeviceLinearTextureAlignmentBytes).value().ok_or_else(none)? + } + "areRasterOrderGroupsSupported" | "supportsRasterOrderGroups" => gap(p.are_raster_order_groups_supported())?, + "isDepth24Stencil8PixelFormatSupported" => gap(p.is_depth24_stencil8_pixel_format_supported())?, + "areProgrammableSamplePositionsSupported" => gap(p.are_programmable_sample_positions_supported())?, + "supports32BitFloatFiltering" => p.supports_32bit_float_filtering as u64, + // A Memoryless texture is a transient render attachment (texture.rs), which every device makes. + "supportsMemorylessRenderTargets" => 1, + "supportsSamplerReductionMode" => gap(p.supports_sampler_reduction_mode())?, + // Apple's SIMD-group functions: the family's SIMD-scoped barrier and permute rows are exact on the device. + "supportsSIMDGroup" => { + let rows = d.family_rows(GpuFamily::Apple9); + let exact = |r: FeatureRow| rows.iter().any(|a| a.row == FamilyRow::Feature(r) && a.state == RowState::Exact); + (exact(FeatureRow::SimdBarrier) && exact(FeatureRow::SimdPermute)) as u64 + } + "supportsYCBCRFormats12" => format_supported(PixelFormat::YCBCR124202P) as u64, + "supportsYCBCRPackedFormats12" => format_supported(PixelFormat::YCBCR124202PPACKED) as u64, + "minimumLinearTextureAlignmentForPixelFormat:" => p + .minimum_linear_texture_alignment_for_pixel_format(raw::(arg, "pixel format")?) + .map_err(e("minimum linear texture alignment"))?, + "supportsTextureSampleCount:" => gap(p.supports_texture_sample_count(arg))?, + "supportsVertexAmplificationCount:" => gap(p.supports_vertex_amplification_count(arg))?, + "supportsRasterizationRateMapWithLayerCount:" => gap(p.supports_rasterization_rate_map_with_layer_count(arg))?, + "supportsFamily:" => d.supports_family(arg as i64) as u64, + "supportsPrimitiveType:" => p.supports_primitive_type(arg) as u64, + "supportsTextureWriteRoundingMode:" => p.supports_texture_write_rounding_mode(arg) as u64, + _ => return Err(format!("{selector} is not a device query of the bridge")), + }) +} + +/// The runtime's answer on the device to `selector` (one `nmb_device_answer_selector` lists), with `arg` for a selector that takes one. +/// False, with the reason, where the device has no answer of its own. +/// +/// # Safety +/// `selector` is a C string; `out` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_answer(device: *const NmbDevice, selector: *const c_char, arg: u64, out: *mut u64) -> bool { + guard(false, || { + if selector.is_null() || out.is_null() { + return Err("null selector or answer".into()); + } + // SAFETY: the caller's contract. + let selector = unsafe { CStr::from_ptr(selector) }.to_str().map_err(e("selector"))?; + let answer = device_answer(get(device)?, selector, arg)?; + // SAFETY: as above. + unsafe { *out = answer }; + Ok(true) + }) +} + +/// `requiredLinearTextureBytesPerRowForDescriptor:` (Apple SPI): a row of `width` texels of `format` rounded up to the format's +/// linear alignment, from the runtime's format description; 0, with the reason, for a format no buffer can back. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_linear_bytes_per_row(device: *const NmbDevice, format: u64, width: u64) -> u64 { + guard(0, || { + let d = &get(device)?.0; + let alignment = d + .properties() + .minimum_linear_texture_alignment_for_pixel_format(raw::(format, "pixel format")?) + .map_err(e("linear texture"))?; + let info = d.pixel_format_info(format); + // Flags bit 0x400: a format of blocks, which the alignment query has refused already. + if info.flags & 0x400 != 0 || info.words[0] == 0 { + return Err(format!("pixel format {format} has no texel size")); + } + Ok((width * info.words[0]).div_ceil(alignment) * alignment) + }) +} + +/// `vendorName`: the Vulkan device's PCI vendor, named where the ID is a known GPU vendor's, else the ID itself. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_vendor_name(device: *const NmbDevice, out: *mut c_char, size: usize) -> usize { + guard(0, || { + let id = get(device)?.0.properties().vendor_id; + let name = match id { + 0x5143 => "Qualcomm".to_string(), + 0x106b => "Apple".to_string(), + 0x10de => "NVIDIA".to_string(), + 0x1002 => "AMD".to_string(), + 0x8086 => "Intel".to_string(), + 0x13b5 => "ARM".to_string(), + _ => format!("0x{id:04x}"), + }; + Ok(copy_out(&name, out, size)) + }) +} + +// ---- buffers ---- + +pub struct NmbBuffer(Buffer); + +/// # Safety +/// `bytes` is null or holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_buffer_new(device: *mut NmbDevice, length: u64, options_word: u64, bytes: *const c_void) -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let options = options(options_word)?; + let buffer = if bytes.is_null() { + device.new_buffer(length, options) + } else { + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; + device.new_buffer_with_bytes(bytes, options) + }; + Ok(boxed(NmbBuffer(buffer.map_err(e("buffer"))?))) + }) +} + +/// # Safety +/// `pointer` is page aligned, holds `length` bytes and stays mapped until the buffer is released. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_buffer_new_no_copy(device: *mut NmbDevice, pointer: *mut c_void, length: u64, options_word: u64) + -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let device = get(device)?; + // SAFETY: the caller's contract. + let memory = unsafe { HostMemory::new(NonNull::new(pointer as *mut u8).ok_or("null memory")?) }; + let modes = options(options_word)?; + // The runtime's live-import lookup and subsequent join are separate: + // simultaneous first constructors can allocate distinct imports of the + // same pages. Cover every native ABI no-copy call, including mixtures + // of raw buffers and IOSurface textures. Each call still gets its own + // buffer object. This ABI passes neither a dma-buf source nor a runtime + // deallocator; no caller callback runs under this gate. Drop it before + // returning/formatting a failure; GPU work and resource release never + // take it. A device's unrelated APIs and other devices remain separate. + let buffer = { + let _gate = device.2.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + // SAFETY: as above. + unsafe { device.0.new_buffer_with_bytes_no_copy(memory, length, modes, None) } + }; + Ok(boxed(NmbBuffer(buffer.map_err(e("no-copy buffer"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_release(buffer: *mut NmbBuffer) { + release(buffer) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_contents(buffer: *const NmbBuffer) -> *mut c_void { + guard(std::ptr::null_mut(), || Ok(get(buffer)?.0.contents().map_or(std::ptr::null_mut(), |p| p.as_ptr() as *mut c_void))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_length(buffer: *const NmbBuffer) -> u64 { + guard(0, || Ok(get(buffer)?.0.length())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_gpu_address(buffer: *const NmbBuffer) -> u64 { + guard(0, || Ok(get(buffer)?.0.gpu_address().0)) +} + +// ---- heaps ---- + +pub struct NmbHeap(Heap); + +/// `newHeapWithDescriptor:`: type (0 automatic, 1 placement), size, resource options. Sparse heaps are not made here. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_new(device: *mut NmbDevice, heap_type: u64, size: u64, options_word: u64) -> *mut NmbHeap { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let heap_type = HeapType::from_raw(heap_type).map_err(e("heap type"))?; + if heap_type == HeapType::Sparse { + return Err("sparse heaps are not implemented".into()); + } + let descriptor = HeapDescriptor::new(heap_type, size, options(options_word)?); + Ok(boxed(NmbHeap(device.new_heap(&descriptor).map_err(e("heap"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_release(heap: *mut NmbHeap) { + release(heap) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_size(heap: *const NmbHeap) -> u64 { + guard(0, || Ok(get(heap)?.0.size())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_used_size(heap: *const NmbHeap) -> u64 { + guard(0, || Ok(get(heap)?.0.used_size())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_allocated_size(heap: *const NmbHeap) -> u64 { + guard(0, || Ok(get(heap)?.0.current_allocated_size())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_max_available_size(heap: *const NmbHeap, alignment: u64) -> u64 { + guard(0, || Ok(get(heap)?.0.max_available_size(alignment).map_err(e("heap available size"))?)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_set_purgeable_state(heap: *const NmbHeap, state: u64) -> u64 { + guard(0, || { + let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; + Ok(get(heap)?.0.set_purgeable_state(state).map_err(e("heap purgeable state"))?.raw()) + }) +} + +/// A buffer in the heap: automatic heaps place it, a placement heap takes `offset` (`u64::MAX` asks for automatic placement). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_new_buffer(heap: *const NmbHeap, length: u64, options_word: u64, offset: u64) -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let heap = &get(heap)?.0; + let options = options(options_word)?; + let buffer = if offset == u64::MAX { heap.new_buffer(length, options) } else { heap.new_buffer_at_offset(length, options, offset) }; + Ok(boxed(NmbBuffer(buffer.map_err(e("heap buffer"))?))) + }) +} + +/// A texture in the heap, placed as `nmb_heap_new_buffer` places a buffer. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_heap_new_texture(heap: *const NmbHeap, descriptor: *const NmbTextureDescriptor, offset: u64) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let heap = &get(heap)?.0; + let descriptor = texture_descriptor(descriptor)?; + let texture = if offset == u64::MAX { heap.new_texture(&descriptor) } else { heap.new_texture_at_offset(&descriptor, offset) }; + Ok(boxed(NmbTexture(texture.map_err(e("heap texture"))?))) + }) +} + +// ---- textures ---- + +#[repr(C)] +pub struct NmbTextureDescriptor { + pub kind: u64, + pub format: u64, + pub width: u64, + pub height: u64, + pub depth: u64, + pub levels: u64, + pub samples: u64, + pub array_length: u64, + pub usage: u64, + pub options: u64, +} + +pub struct NmbTexture(Texture); + +fn texture_descriptor(d: *const NmbTextureDescriptor) -> Result { + let d = get(d)?; + let mut out = TextureDescriptor::default(); + out.texture_type = raw::(d.kind, "texture type")?; + out.pixel_format = raw::(d.format, "pixel format")?; + out.width = d.width; + out.height = d.height; + out.depth = d.depth; + out.mipmap_level_count = d.levels; + out.sample_count = d.samples; + out.array_length = d.array_length; + out.usage = raw::(d.usage, "texture usage")?; + out.resource_options = options(d.options)?; + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new(device: *mut NmbDevice, descriptor: *const NmbTextureDescriptor) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + Ok(boxed(NmbTexture(get(device)?.0.new_texture(&descriptor).map_err(e("texture"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_on_buffer(buffer: *mut NmbBuffer, descriptor: *const NmbTextureDescriptor, offset: u64, + bytes_per_row: u64, iosurface: bool) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + let buffer = &get(buffer)?.0; + let texture = if iosurface { + buffer.new_texture_with_iosurface_plane(&descriptor, IoSurfacePlane::new(offset, bytes_per_row, false)) + } else { + buffer.new_texture(&descriptor, offset, bytes_per_row) + }; + Ok(boxed(NmbTexture(texture.map_err(e("texture over buffer"))?))) + }) +} + +/// `newLinearTextureWithDescriptor:offset:bytesPerRow:bytesPerImage:` (Apple SPI). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_linear(buffer: *mut NmbBuffer, descriptor: *const NmbTextureDescriptor, offset: u64, bytes_per_row: u64, + bytes_per_image: u64) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + let texture = get(buffer)?.0.new_linear_texture(&descriptor, offset, bytes_per_row, bytes_per_image); + Ok(boxed(NmbTexture(texture.map_err(e("linear texture"))?))) + }) +} + +/// A view of `texture`: of the whole texture as `format` where `whole`, else of `levels` and `slices` (start, count) as `format` and `kind`. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_view(texture: *mut NmbTexture, format: u64, kind: u64, levels: *const u64, slices: *const u64, whole: bool) + -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let t = &get(texture)?.0; + let format = raw::(format, "pixel format")?; + let view = if whole { + t.new_texture_view(format) + } else { + if levels.is_null() || slices.is_null() { + return Err("null levels or slices".into()); + } + // SAFETY: the header's two words each. + let (l, s) = unsafe { (std::slice::from_raw_parts(levels, 2), std::slice::from_raw_parts(slices, 2)) }; + let ranges = (l[0]..l[0].checked_add(l[1]).ok_or("levels overflow")?, s[0]..s[0].checked_add(s[1]).ok_or("slices overflow")?); + t.new_texture_view_with_levels_slices(format, raw::(kind, "texture type")?, ranges.0, ranges.1) + }; + Ok(boxed(NmbTexture(view.map_err(e("texture view"))?))) + }) +} + +/// `setPurgeableState:` on a texture: the runtime's answer as Metal's raw state, 0 where it fails. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_set_purgeable_state(texture: *const NmbTexture, state: u64) -> u64 { + guard(0, || { + let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; + Ok(get(texture)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw()) + }) +} + +/// `setPurgeableState:` on a buffer, as on a texture. +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_set_purgeable_state(buffer: *const NmbBuffer, state: u64) -> u64 { + guard(0, || { + let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?; + Ok(get(buffer)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_release(texture: *mut NmbTexture) { + release(texture) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_describe(texture: *const NmbTexture, out: *mut NmbTextureDescriptor) { + guard((), || { + let t = &get(texture)?.0; + let out = get_mut(out)?; + *out = NmbTextureDescriptor { + kind: t.texture_type().as_raw() as u64, + format: t.pixel_format().as_raw() as u64, + width: t.width(), + height: t.height(), + depth: t.depth(), + levels: t.mipmap_level_count(), + samples: t.sample_count(), + array_length: t.array_length(), + usage: t.usage().as_raw() as u64, + options: t.resource_options().raw(), + }; + Ok(()) + }) +} + +fn region(r: *const u64) -> Result { + if r.is_null() { + return Err("null region".into()); + } + // SAFETY: the header's six words. + let r = unsafe { std::slice::from_raw_parts(r, 6) }; + Ok(Region::new_3d(r[0], r[1], r[2], r[3], r[4], r[5])) +} + +fn triple(v: *const u64) -> Result<[u64; 3], String> { + if v.is_null() { + return Err("null origin or size".into()); + } + // SAFETY: the header's three words. + let v = unsafe { std::slice::from_raw_parts(v, 3) }; + Ok([v[0], v[1], v[2]]) +} + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_texture_replace(texture: *mut NmbTexture, region_words: *const u64, level: u64, slice: u64, + bytes: *const c_void, bytes_per_row: u64, bytes_per_image: u64, length: u64) -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; + get(texture)?.0.replace_region(region(region_words)?, level, slice, bytes, bytes_per_row, bytes_per_image) + .map_err(e("replace region"))?; + Ok(true) + }) +} + +/// # Safety +/// `bytes` holds `length` writable bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_texture_get_bytes(texture: *const NmbTexture, bytes: *mut c_void, bytes_per_row: u64, bytes_per_image: u64, + length: u64, region_words: *const u64, level: u64, slice: u64) -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts_mut(bytes as *mut u8, length as usize) }; + get(texture)?.0.get_bytes(bytes, bytes_per_row, bytes_per_image, region(region_words)?, level, slice) + .map_err(e("get bytes"))?; + Ok(true) + }) +} + +// ---- libraries and functions ---- + +pub struct NmbLibrary(Library, Vec); +pub struct NmbFunction(Function); + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_library_new(device: *mut NmbDevice, bytes: *const c_void, length: usize, air: bool) -> *mut NmbLibrary { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + if bytes.is_null() || length == 0 { + return Err("no library bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length) }; + let library = if air { + device.new_library_with_compile_library(metal2vulkan::library::Library::from_air(bytes).map_err(e("AIR"))?) + } else { + device.new_library_with_data(crate::air_slice(bytes).ok_or("metallib without an AIR64 slice")?).map_err(e("metallib"))? + }; + let names = library.function_names(); + Ok(boxed(NmbLibrary(library, names))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_release(library: *mut NmbLibrary) { + release(library) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_function_count(library: *const NmbLibrary) -> u32 { + guard(0, || Ok(get(library)?.1.len() as u32)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_function_name(library: *const NmbLibrary, index: u32, out: *mut c_char, size: usize) -> usize { + guard(0, || { + let names = &get(library)?.1; + Ok(copy_out(names.get(index as usize).ok_or("function index")?, out, size)) + }) +} + +/// # Safety +/// `name` is a C string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_new(library: *mut NmbLibrary, name: *const c_char) -> *mut NmbFunction { + guard(std::ptr::null_mut(), || { + if name.is_null() { + return Err("null function name".into()); + } + // SAFETY: the caller's contract. + let name = unsafe { CStr::from_ptr(name) }.to_str().map_err(e("function name"))?; + Ok(boxed(NmbFunction(get(library)?.0.new_function_with_name(name).map_err(e("function"))?))) + }) +} + +#[repr(C)] +pub struct NmbConstant { + pub kind: u64, + pub index: u64, + pub name: *const c_char, + pub bytes: *const c_void, + pub length: u64, +} + +/// # Safety +/// `name` is a C string; `constants` holds `count` entries whose names are C strings (or null) and whose bytes hold `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_new_specialized(library: *mut NmbLibrary, name: *const c_char, constants: *const NmbConstant, + count: u32) -> *mut NmbFunction { + guard(std::ptr::null_mut(), || { + if name.is_null() || (constants.is_null() && count > 0) { + return Err("null function name or constants".into()); + } + // SAFETY: the caller's contract. + let name = unsafe { CStr::from_ptr(name) }.to_str().map_err(e("function name"))?; + let constants = if count == 0 { &[][..] } else { unsafe { std::slice::from_raw_parts(constants, count as usize) } }; + let mut values = FunctionConstantValues::defaults(); + for c in constants { + if c.bytes.is_null() { + return Err("constant without bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(c.bytes as *const u8, c.length as usize) }; + let value = FunctionConstantValue::new(raw::(c.kind, "constant type")?, bytes.to_vec()); + if c.name.is_null() { + values.by_index.insert(u32::try_from(c.index).map_err(e("constant index"))?, value); + } else { + // SAFETY: the caller's contract. + let key = unsafe { CStr::from_ptr(c.name) }.to_str().map_err(e("constant name"))?; + values.by_name.insert(key.to_string(), value); + } + } + let function = get(library)?.0.new_function_with_name_constant_values(name, &values).map_err(e("specialized function"))?; + Ok(boxed(NmbFunction(function))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_release(function: *mut NmbFunction) { + release(function) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_type(function: *const NmbFunction) -> u64 { + guard(0, || Ok(get(function)?.0.function_type().map_err(e("function type"))?.as_raw() as u64)) +} + +// ---- pipelines and states ---- + +pub struct NmbComputePipeline(ComputePipelineState); +pub struct NmbRenderPipeline(RenderPipelineState); +pub struct NmbSampler(SamplerState); +pub struct NmbDepthStencil(DepthStencilState); + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_new(device: *mut NmbDevice, function: *mut NmbFunction) -> *mut NmbComputePipeline { + // SAFETY: null options names no array, the existing handles follow this ABI. + unsafe { nmb_compute_pipeline_new_with_options(device, function, std::ptr::null()) } +} + +/// # Safety +/// `options` is null or a valid record with a counted array of live archive handles. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_compute_pipeline_new_with_options(device: *mut NmbDevice, function: *mut NmbFunction, + options: *const NmbPipelineOptions) -> *mut NmbComputePipeline { + guard(std::ptr::null_mut(), || { + let resolved = unsafe { pipeline_options(options)? }; + let state = get(device)?.0.new_compute_pipeline_state_with_function(&get(function)?.0, &resolved); + Ok(boxed(NmbComputePipeline(state.map_err(e("compute pipeline"))?))) + }) +} + +#[repr(C)] +pub struct NmbComputePipelineDescriptor { + pub function: *mut NmbFunction, + pub max_threads: u64, + pub multiple_of_execution_width: u64, + pub indirect: u64, + pub max_call_stack_depth: u64, + pub adding_binary_functions: u64, + pub mutability: [u64; 31], +} + +fn compute_descriptor(descriptor: *const NmbComputePipelineDescriptor) -> Result { + let d = get(descriptor)?; + let mut out = ComputePipelineDescriptor::new(StageSource::Function(get(d.function)?.0.function_descriptor())); + out.max_total_threads_per_threadgroup = if d.max_threads == 0 { None } else { Some(u32::try_from(d.max_threads).map_err(e("compute max threads"))?) }; + out.threadgroup_size_is_multiple_of_execution_width = d.multiple_of_execution_width != 0; + out.support_indirect_command_buffers = d.indirect != 0; + out.max_call_stack_depth = u32::try_from(d.max_call_stack_depth).map_err(e("compute stack depth"))?; + out.support_adding_binary_functions = d.adding_binary_functions != 0; + for (index, &mutability) in d.mutability.iter().enumerate() { + if mutability != 0 { + out.buffers.insert(index as u32, raw::(mutability, "buffer mutability")?); + } + } + Ok(out) +} + +/// # Safety +/// `options` is null or a valid record with a counted array of live archive handles. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_compute_pipeline_new_with_descriptor(device: *mut NmbDevice, descriptor: *const NmbComputePipelineDescriptor, + options: *const NmbPipelineOptions) -> *mut NmbComputePipeline { + guard(std::ptr::null_mut(), || { + let resolved = unsafe { pipeline_options(options)? }; + let state = get(device)?.0.new_compute_pipeline_state(&compute_descriptor(descriptor)?, &resolved).map_err(e("compute pipeline"))?; + Ok(boxed(NmbComputePipeline(state))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_release(pipeline: *mut NmbComputePipeline) { + release(pipeline) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_max_threads(pipeline: *const NmbComputePipeline) -> u64 { + guard(0, || Ok(get(pipeline)?.0.max_total_threads_per_threadgroup() as u64)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_execution_width(pipeline: *const NmbComputePipeline) -> u64 { + guard(0, || Ok(get(pipeline)?.0.thread_execution_width() as u64)) +} + +#[repr(C)] +pub struct NmbColorAttachment { + pub format: u64, + pub write_mask: u64, + pub blending: u64, + pub source_rgb: u64, + pub destination_rgb: u64, + pub rgb_operation: u64, + pub source_alpha: u64, + pub destination_alpha: u64, + pub alpha_operation: u64, +} + +#[repr(C)] +pub struct NmbVertexAttribute { + pub format: u64, + pub offset: u64, + pub buffer: u64, +} + +#[repr(C)] +pub struct NmbVertexLayout { + pub stride: u64, + pub step_function: u64, + pub step_rate: u64, +} + +#[repr(C)] +pub struct NmbRenderPipelineDescriptor { + pub vertex: *mut NmbFunction, + pub fragment: *mut NmbFunction, + pub colors: [NmbColorAttachment; 8], + pub depth_format: u64, + pub stencil_format: u64, + pub samples: u64, + pub topology: u64, + pub rasterization: u64, + pub vertex_inputs: u64, + pub attributes: [NmbVertexAttribute; 31], + pub layouts: [NmbVertexLayout; 31], +} + +fn optional_format(value: u64) -> Result, String> { + if value == 0 { Ok(None) } else { raw::(value, "pixel format").map(Some) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_pipeline_new(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor) -> *mut NmbRenderPipeline { + // SAFETY: null options names no array; the device/descriptor handles follow the existing ABI. + unsafe { nmb_render_pipeline_new_with_options(device, descriptor, std::ptr::null()) } +} + +#[repr(C)] +pub struct NmbPipelineOptions { + pub flags: u64, + pub archives: *const *mut NmbBinaryArchive, + pub archive_count: usize, +} + +/// # Safety +/// The options record and its counted archive array are valid for this call. +unsafe fn pipeline_options(options: *const NmbPipelineOptions) -> Result { + let mut resolved = PipelineOptions::default(); + if let Some(options) = unsafe { options.as_ref() } { + resolved.options = PipelineOption::from_raw(options.flags); + if options.archive_count > 0 { + if options.archives.is_null() || options.archive_count > isize::MAX as usize / std::mem::size_of::<*mut NmbBinaryArchive>() { + return Err("invalid binary archive array".into()); + } + let archives = unsafe { std::slice::from_raw_parts(options.archives, options.archive_count) }; + for &archive in archives { + resolved.binary_archives.push(get(archive)?.0.clone()); + } + } + } + Ok(resolved) +} + +/// # Safety +/// `options` is null or a valid record; its array holds `archive_count` live archive handles. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_render_pipeline_new_with_options(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor, + options: *const NmbPipelineOptions) -> *mut NmbRenderPipeline { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let resolved = unsafe { pipeline_options(options)? }; + let state = device.new_render_pipeline_state(&render_descriptor(descriptor)?, &resolved).map_err(e("render pipeline"))?; + Ok(boxed(NmbRenderPipeline(state))) + }) +} + +fn render_descriptor(descriptor: *const NmbRenderPipelineDescriptor) -> Result { + let d = get(descriptor)?; + let mut out = RenderPipelineDescriptor::new(StageSource::Function(get(d.vertex)?.0.function_descriptor())); + if !d.fragment.is_null() { + out.fragment = Some(StageSource::Function(get(d.fragment)?.0.function_descriptor())); + } + for (slot, c) in out.color_attachments.iter_mut().zip(d.colors.iter()) { + if c.format == 0 { + continue; + } + let mut a = ColorAttachmentDescriptor::new(raw::(c.format, "color format")?); + a.write_mask = raw::(c.write_mask, "write mask")?; + a.blending_enabled = c.blending != 0; + a.source_rgb_blend_factor = raw::(c.source_rgb, "blend factor")?; + a.destination_rgb_blend_factor = raw::(c.destination_rgb, "blend factor")?; + a.rgb_blend_operation = raw::(c.rgb_operation, "blend operation")?; + a.source_alpha_blend_factor = raw::(c.source_alpha, "blend factor")?; + a.destination_alpha_blend_factor = raw::(c.destination_alpha, "blend factor")?; + a.alpha_blend_operation = raw::(c.alpha_operation, "blend operation")?; + *slot = Some(a); + } + out.depth_format = optional_format(d.depth_format)?; + out.stencil_format = optional_format(d.stencil_format)?; + out.raster_sample_count = d.samples.max(1) as u32; + out.input_primitive_topology = raw::(d.topology, "topology")?; + out.rasterization_enabled = d.rasterization != 0; + if d.vertex_inputs != 0 { + let mut inputs = VertexDescriptor::new(); + for (i, a) in d.attributes.iter().enumerate() { + if a.format == 0 { + continue; + } + inputs.attributes.insert(i as u32, VertexAttributeDescriptor::new(raw::(a.format, "vertex format")?, + a.offset as u32, a.buffer as u32)); + if let Some(l) = d.layouts.get(a.buffer as usize) { + let mut layout = VertexBufferLayoutDescriptor::new(Some(l.stride as u32)); + layout.step_function = raw::(l.step_function, "step function")?; + layout.step_rate = l.step_rate as u32; + inputs.layouts.insert(a.buffer as u32, layout); + } + } + out.vertex_descriptor = Some(inputs); + } + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_pipeline_release(pipeline: *mut NmbRenderPipeline) { + release(pipeline) +} + +// ---- binary archives ---- + +pub struct NmbBinaryArchive(BinaryArchive); + +/// # Safety +/// `path` is null (an empty archive) or a C string naming the archive's file. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_binary_archive_new(device: *mut NmbDevice, path: *const c_char) -> *mut NmbBinaryArchive { + guard(std::ptr::null_mut(), || { + // SAFETY: the caller's contract. + let url = (!path.is_null()).then(|| PathBuf::from(unsafe { CStr::from_ptr(path) }.to_string_lossy().into_owned())); + let archive = get(device)?.0.new_binary_archive(&BinaryArchiveDescriptor::new(url)).map_err(e("binary archive"))?; + Ok(boxed(NmbBinaryArchive(archive))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_release(archive: *mut NmbBinaryArchive) { + release(archive) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_add_render(archive: *mut NmbBinaryArchive, descriptor: *const NmbRenderPipelineDescriptor) -> bool { + guard(false, || { + get(archive)?.0.add_render_pipeline_functions(&render_descriptor(descriptor)?).map_err(e("archive render pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_add_compute(archive: *mut NmbBinaryArchive, function: *mut NmbFunction) -> bool { + guard(false, || { + let descriptor = ComputePipelineDescriptor::new(StageSource::Function(get(function)?.0.function_descriptor())); + get(archive)?.0.add_compute_pipeline_functions(&descriptor).map_err(e("archive compute pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_binary_archive_add_compute_descriptor(archive: *mut NmbBinaryArchive, descriptor: *const NmbComputePipelineDescriptor) -> bool { + guard(false, || { + get(archive)?.0.add_compute_pipeline_functions(&compute_descriptor(descriptor)?).map_err(e("archive compute pipeline"))?; + Ok(true) + }) +} + +/// # Safety +/// `path` is a C string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_binary_archive_serialize(archive: *mut NmbBinaryArchive, path: *const c_char) -> bool { + guard(false, || { + if path.is_null() { + return Err("null path".into()); + } + // SAFETY: the caller's contract. + let path = PathBuf::from(unsafe { CStr::from_ptr(path) }.to_string_lossy().into_owned()); + get(archive)?.0.serialize_to_url(&path).map_err(e("archive serialize"))?; + Ok(true) + }) +} + +#[repr(C)] +pub struct NmbSamplerDescriptor { + pub min_filter: u64, + pub mag_filter: u64, + pub mip_filter: u64, + pub s_address: u64, + pub t_address: u64, + pub r_address: u64, + pub border_color: u64, + pub normalized: u64, + pub max_anisotropy: u64, + pub compare: u64, + pub lod_average: u64, + pub lod_min: f32, + pub lod_max: f32, +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_sampler_new(device: *mut NmbDevice, descriptor: *const NmbSamplerDescriptor) -> *mut NmbSampler { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut out = SamplerDescriptor::new(); + out.min_filter = raw::(d.min_filter, "min filter")?; + out.mag_filter = raw::(d.mag_filter, "mag filter")?; + out.mip_filter = raw::(d.mip_filter, "mip filter")?; + out.s_address_mode = raw::(d.s_address, "address mode")?; + out.t_address_mode = raw::(d.t_address, "address mode")?; + out.r_address_mode = raw::(d.r_address, "address mode")?; + out.border_color = raw::(d.border_color, "border color")?; + out.normalized_coordinates = d.normalized != 0; + out.max_anisotropy = d.max_anisotropy.max(1) as u32; + out.compare_function = raw::(d.compare, "compare function")?; + out.lod_average = d.lod_average != 0; + out.lod_min_clamp = d.lod_min; + out.lod_max_clamp = d.lod_max; + Ok(boxed(NmbSampler(get(device)?.0.new_sampler_state(&out).map_err(e("sampler"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_sampler_release(sampler: *mut NmbSampler) { + release(sampler) +} + +#[repr(C)] +pub struct NmbStencil { + pub compare: u64, + pub failure: u64, + pub depth_failure: u64, + pub pass: u64, + pub read_mask: u64, + pub write_mask: u64, +} + +#[repr(C)] +pub struct NmbDepthStencilDescriptor { + pub compare: u64, + pub write: u64, + pub front: NmbStencil, + pub back: NmbStencil, +} + +fn stencil(s: &NmbStencil) -> Result { + let mut out = StencilDescriptor::default(); + out.stencil_compare_function = raw::(s.compare, "stencil compare")?; + out.stencil_failure_operation = raw::(s.failure, "stencil operation")?; + out.depth_failure_operation = raw::(s.depth_failure, "stencil operation")?; + out.depth_stencil_pass_operation = raw::(s.pass, "stencil operation")?; + out.read_mask = s.read_mask as u32; + out.write_mask = s.write_mask as u32; + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_depth_stencil_new(device: *mut NmbDevice, descriptor: *const NmbDepthStencilDescriptor) -> *mut NmbDepthStencil { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut out = DepthStencilDescriptor::default(); + out.depth_compare_function = raw::(d.compare, "depth compare")?; + out.depth_write_enabled = d.write != 0; + out.front_face_stencil = stencil(&d.front)?; + out.back_face_stencil = stencil(&d.back)?; + Ok(boxed(NmbDepthStencil(get(device)?.0.new_depth_stencil_state(&out).map_err(e("depth/stencil state"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_depth_stencil_release(state: *mut NmbDepthStencil) { + release(state) +} + +// ---- queues and command buffers ---- + +pub struct NmbQueue(CommandQueue); +pub struct NmbCommandBuffer(CommandBuffer); + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_queue_new(device: *mut NmbDevice) -> *mut NmbQueue { + guard(std::ptr::null_mut(), || Ok(boxed(NmbQueue(get(device)?.0.new_command_queue().map_err(e("queue"))?)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_queue_release(queue: *mut NmbQueue) { + release(queue) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_new(queue: *mut NmbQueue) -> *mut NmbCommandBuffer { + guard(std::ptr::null_mut(), || Ok(boxed(NmbCommandBuffer(get(queue)?.0.command_buffer().map_err(e("command buffer"))?)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_release(command_buffer: *mut NmbCommandBuffer) { + release(command_buffer) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_uses_buffer(command_buffer: *const NmbCommandBuffer, buffer: *const NmbBuffer) -> i32 { + guard(-1, || { + let used = get(command_buffer)?.0.used_resources(); + Ok(i32::from(used.references_resource(get(buffer)?.0.as_resource()))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_uses_texture(command_buffer: *const NmbCommandBuffer, texture: *const NmbTexture) -> i32 { + guard(-1, || { + let used = get(command_buffer)?.0.used_resources(); + Ok(i32::from(used.references_resource(get(texture)?.0.as_resource()))) + }) +} + +struct Context(*mut c_void); +// SAFETY: the context is the caller's, handed back on the runtime's thread as the header states. +unsafe impl Send for Context {} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_on_completed(command_buffer: *mut NmbCommandBuffer, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + get(command_buffer)?.0.add_completed_handler(Box::new(move |_| { + let context = context; + handler(context.0) + })).map_err(e("completed handler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_on_scheduled(command_buffer: *mut NmbCommandBuffer, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + get(command_buffer)?.0.add_scheduled_handler(Box::new(move |_| { + let context = context; + handler(context.0) + })).map_err(e("scheduled handler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait_scheduled(command_buffer: *mut NmbCommandBuffer) -> bool { + guard(false, || { + get(command_buffer)?.0.wait_until_scheduled().map_err(e("wait scheduled"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffer) -> bool { + guard(false, || { + get(command_buffer)?.0.commit().map_err(e("commit"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait(command_buffer: *mut NmbCommandBuffer, timeout_ms: u64) -> i32 { + guard(-1, || { + let command_buffer = &get(command_buffer)?.0; + if timeout_ms == 0 { + command_buffer.wait_until_completed().map_err(e("wait"))?; + return Ok(0); + } + Ok(if command_buffer.wait_until_completed_for(Duration::from_millis(timeout_ms)).is_ok() { 0 } else { 1 }) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_status(command_buffer: *const NmbCommandBuffer) -> u64 { + guard(5, || { + Ok(get(command_buffer)?.0.status().raw()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_error(command_buffer: *const NmbCommandBuffer, out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(get(command_buffer)?.0.error_description().map_or(0, |text| copy_out(&text, out, size)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_gpu_time(command_buffer: *const NmbCommandBuffer, end: bool) -> f64 { + guard(0.0, || { + let command_buffer = &get(command_buffer)?.0; + let time = if end { command_buffer.gpu_end_time() } else { command_buffer.gpu_start_time() }; + time.map_err(e("GPU time")) + }) +} + +// ---- events and fences ---- + +pub enum NmbEvent { + Event(Event), + Shared(SharedEvent), +} +pub struct NmbFence(Fence); + +impl NmbEvent { + fn as_ref(&self) -> EventRef<'_> { + match self { + NmbEvent::Event(e) => EventRef::Event(e), + NmbEvent::Shared(s) => EventRef::Shared(s), + } + } +} + +fn shared<'a>(event: *const NmbEvent) -> Result<&'a SharedEvent, String> { + match get(event)? { + NmbEvent::Shared(s) => Ok(s), + NmbEvent::Event(_) => Err("not a shared event".into()), + } +} + +/// One listener thread for every notification of this process (Metal's listener only names the queue a block runs on; the frontend +/// forwards to it). +fn listener() -> Result<&'static SharedEventListener, String> { + static LISTENER: std::sync::OnceLock> = std::sync::OnceLock::new(); + LISTENER.get_or_init(|| SharedEventListener::new().map_err(e("listener"))).as_ref().map_err(|why| why.clone()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_event_new(device: *mut NmbDevice, is_shared: bool) -> *mut NmbEvent { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + Ok(boxed(if is_shared { + NmbEvent::Shared(device.new_shared_event().map_err(e("shared event"))?) + } else { + NmbEvent::Event(device.new_event().map_err(e("event"))?) + })) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_event_release(event: *mut NmbEvent) { + release(event) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_value(event: *const NmbEvent) -> u64 { + guard(0, || Ok(shared(event)?.signaled_value())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_set_value(event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + shared(event)?.set_signaled_value(value); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_wait(event: *mut NmbEvent, value: u64, timeout_ms: u64) -> bool { + guard(false, || Ok(shared(event)?.wait_until_signaled_value(value, Duration::from_millis(timeout_ms)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_notify(event: *mut NmbEvent, value: u64, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + shared(event)?.notify_listener(listener()?, value, Box::new(move |_, reached| { + let context = context; + handler(context.0, reached) + })); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_signal_event(command_buffer: *mut NmbCommandBuffer, event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + get(command_buffer)?.0.encode_signal_event(get(event)?.as_ref(), value).map_err(e("signal event"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait_event(command_buffer: *mut NmbCommandBuffer, event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + get(command_buffer)?.0.encode_wait_for_event(get(event)?.as_ref(), value).map_err(e("wait for event"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_fence_new(device: *mut NmbDevice) -> *mut NmbFence { + guard(std::ptr::null_mut(), || Ok(boxed(NmbFence(get(device)?.0.new_fence())))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_fence_release(fence: *mut NmbFence) { + release(fence) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_update_fence(encoder: *mut NmbEncoder, fence: *mut NmbFence, stages: u64) -> bool { + guard(false, || { + let fence = &get(fence)?.0; + match get_mut(encoder)? { + NmbEncoder::Compute(c) => c.update_fence(fence), + NmbEncoder::Blit(b) => b.update_fence(fence), + NmbEncoder::Render(r) => r.update_fence(fence, raw::(stages, "render stages")?), + } + .map_err(e("update fence"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_wait_fence(encoder: *mut NmbEncoder, fence: *mut NmbFence, stages: u64) -> bool { + guard(false, || { + let fence = &get(fence)?.0; + match get_mut(encoder)? { + NmbEncoder::Compute(c) => c.wait_for_fence(fence), + NmbEncoder::Blit(b) => b.wait_for_fence(fence), + NmbEncoder::Render(r) => r.wait_for_fence(fence, raw::(stages, "render stages")?), + } + .map_err(e("wait for fence"))?; + Ok(true) + }) +} + +// ---- encoders ---- + +pub enum NmbEncoder { + Compute(ComputeCommandEncoder), + Render(RenderCommandEncoder), + Blit(BlitCommandEncoder), +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_encoder_new(command_buffer: *mut NmbCommandBuffer) -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || { + Ok(boxed(NmbEncoder::Compute(get(command_buffer)?.0.compute_command_encoder().map_err(e("compute encoder"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_encoder_new(command_buffer: *mut NmbCommandBuffer) -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || Ok(boxed(NmbEncoder::Blit(get(command_buffer)?.0.blit_command_encoder().map_err(e("blit encoder"))?)))) +} + +#[repr(C)] +pub struct NmbPassAttachment { + pub texture: *mut NmbTexture, + pub level: u64, + pub slice: u64, + pub depth_plane: u64, + pub load: u64, + pub store: u64, + pub clear: [f64; 4], +} + +#[repr(C)] +pub struct NmbRenderPassDescriptor { + pub colors: [NmbPassAttachment; 8], + pub depth: NmbPassAttachment, + pub stencil: NmbPassAttachment, + pub width: u64, + pub height: u64, + pub array_length: u64, + pub samples: u64, + pub visibility: *const NmbBuffer, +} + +fn texture_of(a: &NmbPassAttachment) -> Result, String> { + if a.texture.is_null() { Ok(None) } else { Ok(Some(get(a.texture)?.0.clone())) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, descriptor: *const NmbRenderPassDescriptor) + -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut pass = RenderPassDescriptor::new(); + for (slot, a) in pass.color_attachments.iter_mut().zip(d.colors.iter()) { + slot.texture = texture_of(a)?; + slot.level = a.level; + slot.slice = a.slice; + slot.depth_plane = a.depth_plane; + slot.load_action = LoadAction::from_raw(a.load).map_err(e("load action"))?; + slot.store_action = StoreAction::from_raw(a.store).map_err(e("store action"))?; + slot.clear_color = ClearColor::new(a.clear[0], a.clear[1], a.clear[2], a.clear[3]); + } + let depth = &mut pass.depth_attachment; + depth.texture = texture_of(&d.depth)?; + depth.level = d.depth.level; + depth.slice = d.depth.slice; + depth.depth_plane = d.depth.depth_plane; + depth.load_action = LoadAction::from_raw(d.depth.load).map_err(e("load action"))?; + depth.store_action = StoreAction::from_raw(d.depth.store).map_err(e("store action"))?; + depth.clear_depth = d.depth.clear[0]; + let stencil = &mut pass.stencil_attachment; + stencil.texture = texture_of(&d.stencil)?; + stencil.level = d.stencil.level; + stencil.slice = d.stencil.slice; + stencil.depth_plane = d.stencil.depth_plane; + stencil.load_action = LoadAction::from_raw(d.stencil.load).map_err(e("load action"))?; + stencil.store_action = StoreAction::from_raw(d.stencil.store).map_err(e("store action"))?; + stencil.clear_stencil = d.stencil.clear[0] as u32; + pass.render_target_width = d.width; + pass.render_target_height = d.height; + pass.render_target_array_length = d.array_length; + pass.default_raster_sample_count = d.samples; + if !d.visibility.is_null() { + pass.visibility_result_buffer = Some(get(d.visibility)?.0.clone()); + } + let encoder = get(command_buffer)?.0.render_command_encoder(&pass).map_err(e("render encoder"))?; + Ok(boxed(NmbEncoder::Render(encoder))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_end(encoder: *mut NmbEncoder) -> bool { + guard(false, || { + if encoder.is_null() { + return Err("null encoder".into()); + } + // SAFETY: from `boxed`; ending releases it. + let encoder = unsafe { Box::from_raw(encoder) }; + match *encoder { + NmbEncoder::Compute(c) => c.end_encoding(), + NmbEncoder::Render(r) => r.end_encoding(), + NmbEncoder::Blit(b) => b.end_encoding(), + } + .map_err(e("end encoding"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_discard(encoder: *mut NmbEncoder) { + release(encoder) +} + +fn compute<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut ComputeCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Compute(c) => Ok(c), + _ => Err("not a compute encoder".into()), + } +} + +fn render<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut RenderCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Render(r) => Ok(r), + _ => Err("not a render encoder".into()), + } +} + +fn blit<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut BlitCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Blit(b) => Ok(b), + _ => Err("not a blit encoder".into()), + } +} + +fn optional<'a, T>(handle: *const T) -> Result, String> { + if handle.is_null() { Ok(None) } else { get(handle).map(Some) } +} + +const COMPUTE: u32 = 0; +const VERTEX: u32 = 1; +const FRAGMENT: u32 = 2; + +fn bad_stage(stage: u32) -> String { + format!("stage {stage} does not fit this encoder") +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_buffer(encoder: *mut NmbEncoder, stage: u32, buffer: *mut NmbBuffer, offset: u64, index: u32) -> bool { + guard(false, || { + let buffer = optional(buffer)?.map(|b| &b.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer(buffer, offset, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer(buffer, offset, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_buffer(buffer, offset, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set buffer"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_buffer_offset(encoder: *mut NmbEncoder, stage: u32, offset: u64, index: u32) -> bool { + guard(false, || { + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer_offset(offset, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer_offset(offset, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_buffer_offset(offset, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set buffer offset"))?; + Ok(true) + }) +} + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_encoder_set_bytes(encoder: *mut NmbEncoder, stage: u32, bytes: *const c_void, length: usize, index: u32) + -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length) }; + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_bytes(bytes, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_bytes(bytes, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_bytes(bytes, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set bytes"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_texture(encoder: *mut NmbEncoder, stage: u32, texture: *mut NmbTexture, index: u32) -> bool { + guard(false, || { + let texture = optional(texture)?.map(|t| &t.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_texture(texture, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_texture(texture, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_texture(texture, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set texture"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_sampler(encoder: *mut NmbEncoder, stage: u32, sampler: *mut NmbSampler, index: u32) -> bool { + guard(false, || { + let sampler = optional(sampler)?.map(|s| &s.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_sampler_state(sampler, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_sampler_state(sampler, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_sampler_state(sampler, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set sampler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_set_pipeline(encoder: *mut NmbEncoder, pipeline: *mut NmbComputePipeline) -> bool { + guard(false, || { + compute(encoder)?.set_compute_pipeline_state(&get(pipeline)?.0).map_err(e("compute pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_set_threadgroup_memory(encoder: *mut NmbEncoder, length: u32, index: u32) -> bool { + guard(false, || { + compute(encoder)?.set_threadgroup_memory_length(length, index).map_err(e("threadgroup memory"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_dispatch(encoder: *mut NmbEncoder, size: *const u64, per_group: *const u64, threads: bool) -> bool { + guard(false, || { + let (s, g) = (triple(size)?, triple(per_group)?); + let (s, g) = (Size::new(s[0], s[1], s[2]), Size::new(g[0], g[1], g[2])); + let c = compute(encoder)?; + if threads { c.dispatch_threads(s, g) } else { c.dispatch_threadgroups(s, g) }.map_err(e("dispatch"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_pipeline(encoder: *mut NmbEncoder, pipeline: *mut NmbRenderPipeline) -> bool { + guard(false, || { + render(encoder)?.set_render_pipeline_state(&get(pipeline)?.0).map_err(e("render pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_depth_stencil(encoder: *mut NmbEncoder, state: *mut NmbDepthStencil) -> bool { + guard(false, || { + render(encoder)?.set_depth_stencil_state(optional(state)?.map(|s| &s.0)).map_err(e("depth/stencil state"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_viewport(encoder: *mut NmbEncoder, viewport: *const f64) -> bool { + guard(false, || { + if viewport.is_null() { + return Err("null viewport".into()); + } + // SAFETY: the header's six values. + let v = unsafe { std::slice::from_raw_parts(viewport, 6) }; + render(encoder)?.set_viewport(Viewport::new(v[0], v[1], v[2], v[3], v[4], v[5])).map_err(e("viewport"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_scissor(encoder: *mut NmbEncoder, rect: *const u64) -> bool { + guard(false, || { + if rect.is_null() { + return Err("null scissor".into()); + } + // SAFETY: the header's four values. + let r = unsafe { std::slice::from_raw_parts(rect, 4) }; + render(encoder)?.set_scissor_rect(ScissorRect::new(r[0], r[1], r[2], r[3])).map_err(e("scissor"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_raster(encoder: *mut NmbEncoder, cull_mode: u64, winding: u64, fill_mode: u64) -> bool { + guard(false, || { + let r = render(encoder)?; + r.set_cull_mode(raw::(cull_mode, "cull mode")?); + r.set_front_facing_winding(raw::(winding, "winding")?); + r.set_triangle_fill_mode(raw::(fill_mode, "fill mode")?).map_err(e("fill mode"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_stencil_reference(encoder: *mut NmbEncoder, front: u32, back: u32) -> bool { + guard(false, || { + render(encoder)?.set_stencil_reference_values(front, back); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_depth_bias(encoder: *mut NmbEncoder, bias: f32, slope_scale: f32, clamp: f32) -> bool { + guard(false, || { + render(encoder)?.set_depth_bias(bias, slope_scale, clamp).map_err(e("depth bias"))?; + Ok(true) + }) +} + +/// SPI `setLineWidth:`: the runtime accepts Metal's default width of 1 and refuses any other (Apple's wide-line rule is unmeasured). +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_line_width(encoder: *mut NmbEncoder, width: f32) -> bool { + guard(false, || { + render(encoder)?.set_line_width(width).map_err(e("line width"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_visibility_mode(encoder: *mut NmbEncoder, mode: u64, offset: u64) -> bool { + guard(false, || { + let mode = VisibilityResultMode::from_raw(mode).map_err(e("visibility result mode"))?; + render(encoder)?.set_visibility_result_mode(mode, offset).map_err(e("visibility result mode"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_blend_color(encoder: *mut NmbEncoder, color: *const f32) -> bool { + guard(false, || { + if color.is_null() { + return Err("null color".into()); + } + // SAFETY: the header's four values. + let c = unsafe { std::slice::from_raw_parts(color, 4) }; + render(encoder)?.set_blend_color(c[0], c[1], c[2], c[3]); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_draw(encoder: *mut NmbEncoder, primitive: u64, start: u32, count: u32, instances: u32, base_instance: u32) + -> bool { + guard(false, || { + render(encoder)?.draw_primitives(raw::(primitive, "primitive")?, start, count, instances, base_instance) + .map_err(e("draw"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_render_draw_indexed(encoder: *mut NmbEncoder, primitive: u64, count: u32, index_type: u64, indices: *mut NmbBuffer, + offset: u64, instances: u32, base_vertex: i32, base_instance: u32) -> bool { + guard(false, || { + render(encoder)?.draw_indexed_primitives(raw::(primitive, "primitive")?, count, + raw::(index_type, "index type")?, &get(indices)?.0, offset, instances, + base_vertex, base_instance).map_err(e("indexed draw"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_copy_buffer(encoder: *mut NmbEncoder, source: *mut NmbBuffer, source_offset: u64, destination: *mut NmbBuffer, + destination_offset: u64, size: u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_buffer_to_buffer(&get(source)?.0, source_offset, &get(destination)?.0, destination_offset, size) + .map_err(e("buffer copy"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_fill_buffer(encoder: *mut NmbEncoder, buffer: *mut NmbBuffer, start: u64, end: u64, value: u8) -> bool { + guard(false, || { + blit(encoder)?.fill_buffer(&get(buffer)?.0, start..end, value).map_err(e("fill"))?; + Ok(true) + }) +} + +fn origin(v: *const u64) -> Result { + let v = triple(v)?; + Ok(Origin::new(v[0], v[1], v[2])) +} + +fn size(v: *const u64) -> Result { + let v = triple(v)?; + Ok(Size::new(v[0], v[1], v[2])) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_copy_texture(encoder: *mut NmbEncoder, source: *mut NmbTexture, source_slice: u64, source_level: u64, + source_origin: *const u64, extent: *const u64, destination: *mut NmbTexture, + destination_slice: u64, destination_level: u64, destination_origin: *const u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_texture_to_texture(&get(source)?.0, source_slice, source_level, origin(source_origin)?, size(extent)?, + &get(destination)?.0, destination_slice, destination_level, + origin(destination_origin)?).map_err(e("texture copy"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_buffer_to_texture(encoder: *mut NmbEncoder, source: *mut NmbBuffer, offset: u64, bytes_per_row: u64, + bytes_per_image: u64, extent: *const u64, destination: *mut NmbTexture, slice: u64, + level: u64, destination_origin: *const u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_buffer_to_texture(&get(source)?.0, offset, bytes_per_row, bytes_per_image, size(extent)?, + &get(destination)?.0, slice, level, origin(destination_origin)?, BlitOption::None) + .map_err(e("buffer to texture"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_texture_to_buffer(encoder: *mut NmbEncoder, source: *mut NmbTexture, slice: u64, level: u64, + source_origin: *const u64, extent: *const u64, destination: *mut NmbBuffer, offset: u64, + bytes_per_row: u64, bytes_per_image: u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_texture_to_buffer(&get(source)?.0, slice, level, origin(source_origin)?, size(extent)?, + &get(destination)?.0, offset, bytes_per_row, bytes_per_image, BlitOption::None) + .map_err(e("texture to buffer"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_generate_mipmaps(encoder: *mut NmbEncoder, texture: *mut NmbTexture) -> bool { + guard(false, || { + blit(encoder)?.generate_mipmaps(&get(texture)?.0).map_err(e("mipmaps"))?; + Ok(true) + }) +} diff --git a/bridge/archive_host.mm b/bridge/archive_host.mm new file mode 100644 index 0000000..f77e973 --- /dev/null +++ b/bridge/archive_host.mm @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: MIT +// Two processes, distinct initially empty caches: archive selection cannot be satisfied by +// the producer's live pipeline memo. A strict empty/changed-descriptor miss must refuse. +#include "FrontendChecks.h" +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" + +int main(int argc, char **argv) { + if (argc != 5) return 2; + @autoreleasepool { + const bool produce = !strcmp(argv[4], "produce"); + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + if (!library) return 2; + auto descriptor = [MTLRenderPipelineDescriptor new]; + descriptor.vertexFunction = [library newFunctionWithName:@"colored_vertex"]; + descriptor.fragmentFunction = [library newFunctionWithName:@"colored_fragment"]; + descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatRGBA8Unorm; + auto archiveDescriptor = [MTLBinaryArchiveDescriptor new]; + NSURL *url = [NSURL fileURLWithPath:@(argv[3])]; + if (!produce) archiveDescriptor.url = url; + NSError *error = nil; + id archive = [device newBinaryArchiveWithDescriptor:archiveDescriptor error:&error]; + if (!archive) {std::printf("FAIL open archive: %s\n", error.localizedDescription.UTF8String);return 1;} + descriptor.binaryArchives = @[archive]; + if (produce) { + id state = [device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if (state || !error) {std::puts("FAIL empty archive miss was silently compiled");return 1;} + if (![archive addRenderPipelineFunctionsWithDescriptor:descriptor error:&error] || ![archive serializeToURL:url error:&error]) { + std::printf("FAIL archive add/save: %s\n", error.localizedDescription.UTF8String);return 1; + } + } + error = nil; + id state = [device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if (!state || error) {std::printf("FAIL strict archive hit: %s\n", error.localizedDescription.UTF8String);return 1;} + // A format not added to the archive must still miss, even after a valid pipeline is live. + descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + if ([device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error) { + std::puts("FAIL changed-descriptor archive miss");return 1; + } + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id target = [device newTextureWithDescriptor:td]; + id cb = [[device newCommandQueue] commandBuffer]; + if (!target || !cb) return 2; + frontendchecks::Report report = {}; + frontendchecks::gReport = &report; + frontendchecks::drawQuad(cb, state, target, nil, frontendchecks::kQuarter, 6); + if (!frontendchecks::finish(cb, "archived draw")) return 1; + uint8_t pixels[16 * 16 * 4] = {}; + [target getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + if (frontendchecks::wrongTexels(pixels, 64, 16)) {std::puts("FAIL archived render pixels");return 1;} + std::printf("PASS: %s render archive, strict hit/miss and GPU pixel check (host only)\n", produce ? "produced" : "reloaded"); + } +} diff --git a/bridge/bridge_smoke.cpp b/bridge/bridge_smoke.cpp new file mode 100644 index 0000000..b89220c --- /dev/null +++ b/bridge/bridge_smoke.cpp @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MIT +// Host run of the bridge self-test over the host Turnip and the host kernel stack (dyld-interposed IOKit shim). The software CP executes +// nothing, so result counts are not expected to be zero here; the run proves every step builds, submits and retires. +#include "NativeMetalBridge.h" +#include +#include +#include +static void mark(uint32_t value) {std::printf("mark %x\n",value);std::fflush(stdout);} +int main(int argc,char **argv) { + if(argc<3)return 2; + std::vector qc; + if(argc>3) { + FILE *f=std::fopen(argv[3],"rb");if(!f)return 3; + std::fseek(f,0,SEEK_END);qc.resize(size_t(std::ftell(f)));std::fseek(f,0,SEEK_SET); + if(std::fread(qc.data(),1,qc.size(),f)!=qc.size())return 4;std::fclose(f); + } + NmbSelftest r{}; + const uint32_t code=nmb_selftest(argv[1],argv[2],qc.data(),qc.size(),mark,&r); + std::printf("code %x stage %x compute_bad %u render_bad %u import_bad %u qc %u kernels %u/%u pairs %u/%u msg '%s'\n",code,r.stage, + r.compute_bad,r.render_bad,r.import_bad,r.qc_functions,r.qc_kernels_ok,r.qc_kernels_failed,r.qc_pairs_ok,r.qc_pairs_failed,r.message); + return code?1:0; +} diff --git a/bridge/cache_export_host.cpp b/bridge/cache_export_host.cpp new file mode 100644 index 0000000..30b4f9f --- /dev/null +++ b/bridge/cache_export_host.cpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: MIT +// Host check of the pipeline-cache export/import format (CacheExport.h): a directory tree becomes the export blob, the blob is written where +// the image would carry it, and importPack rebuilds the tree in a second directory; every file must come back byte for byte, an existing +// file of equal size is kept, and damaged packs import nothing. argv: source dir, destination dir, blob path. No kernel call is made. +#define CACHEEXPORT_FORMAT_ONLY +#include +#include "../CacheExport.h" +#include +#include + +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) + +static std::vector slurp(const std::string &path) { + std::vector out;FILE *f=fopen(path.c_str(),"rb");if(!f)return out; + uint8_t b[4096];for(size_t n;(n=fread(b,1,sizeof b,f))>0;)out.insert(out.end(),b,b+n);fclose(f);return out; +} +static void spill(const std::string &path,const std::vector &data) {FILE *f=fopen(path.c_str(),"wb");CHECK(f);CHECK(fwrite(data.data(),1,data.size(),f)==data.size());fclose(f);} + +int main(int argc,char **argv) { + CHECK(argc==4); + const std::string source=argv[1],dest=argv[2],blobPath=argv[3]; + std::vector files;cacheexport::scan(source,"",files,0); + CHECK(files.size()>=3); + const uint64_t sig=cacheexport::signature(files); + std::vector again;cacheexport::scan(source,"",again,0); + CHECK(cacheexport::signature(again)==sig); + std::vector blob;CHECK(cacheexport::build(source,files,blob)); + /* the export's stream: as long as announced, and written out as the uncompressed export blob for the host's readers */ + std::vector raw={'S','H','E','X','R','0','0','1',0,0,0,0,0,0,0,0}; + CHECK(cacheexport::stream(source,files,[&](const uint8_t *p,size_t n){raw.insert(raw.end(),p,p+n);return true;})); + CHECK(raw.size()-16==cacheexport::streamBytes(source,files)); + {const uint32_t n=(uint32_t)(raw.size()-16);std::memcpy(&raw[8],&n,4);} + spill(blobPath+".raw",raw); + mkdir(dest.c_str(),0700);mkdir((dest+"-damaged").c_str(),0700); /* the plugin makes its directory before importing */ + spill(blobPath,blob); + CHECK(cacheexport::baked().empty()); + CHECK(cacheexport::importPack(blobPath.c_str(),dest)==(int)files.size()); + CHECK(cacheexport::baked().size()==files.size()); /* every entry of a pack read to its end is known as baked */ + for(const auto &f:files)CHECK(slurp(source+"/"+f.path)==slurp(dest+"/"+f.path)); + CHECK(cacheexport::importPack(blobPath.c_str(),dest)==0); /* all present with their sizes: nothing rewritten */ + /* damaged packs: truncated, bad magic, flipped compressed byte */ + std::vector bad=blob;bad.resize(bad.size()/2);spill(blobPath+".t",bad); + CHECK(cacheexport::importPack((blobPath+".t").c_str(),dest+"-damaged")==-1); + bad=blob;bad[0]='X';spill(blobPath+".m",bad);CHECK(cacheexport::importPack((blobPath+".m").c_str(),dest+"-damaged")==-1); + bad=blob;bad[bad.size()-5]^=0x55;spill(blobPath+".f",bad);CHECK(cacheexport::importPack((blobPath+".f").c_str(),dest+"-damaged")==-1); + CHECK(cacheexport::importPack((blobPath+".missing").c_str(),dest)==0); + std::vector left;cacheexport::scan(dest+"-damaged","",left,0);CHECK(left.empty()); + CHECK(cacheexport::baked().size()==files.size()); /* damaged packs add nothing */ + /* the export leaves baked files out: only a new file, or one whose size changed, goes out */ + std::vector now;cacheexport::scan(dest,"",now,0);cacheexport::dropBaked(now);CHECK(now.empty()); + spill(dest+"/new-entry",{1,2,3}); + { std::vector grown=slurp(dest+"/"+files[0].path);grown.push_back(7);spill(dest+"/"+files[0].path,grown); } + now.clear();cacheexport::scan(dest,"",now,0);cacheexport::dropBaked(now); + CHECK(now.size()==2); + std::printf("PASS: cache export %zu files, %zu blob bytes; import exact, idempotent, 3 damaged packs refused, baked files left out of the export\n",files.size(),blob.size()); + return 0; +} diff --git a/bridge/carenderer_host.mm b/bridge/carenderer_host.mm new file mode 100644 index 0000000..c7d6348 --- /dev/null +++ b/bridge/carenderer_host.mm @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: MIT +// Host instrument: the Mac's own QuartzCore renders a layer tree through CARenderer into a texture of the native frontend's device (bridge +// over a host Vulkan driver, MoltenVK on the Mac). CoreAnimation's Metal renderer then drives the frontend as the target's render server +// will, and the pixels it produced are checked. With NMB_DEVICE_BASE_NSOBJECT=1 every device selector the renderer sends that the frontend +// lacks is reported as missing. The Mac's QuartzCore is not the target's: this names gaps and checks results, it qualifies nothing on iPadOS. +#import +#import +#import +#import +#import +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include +#include +#import +extern "C" size_t native_bridge_metal_telemetry(char *, size_t); + +static std::set missing; +static void onMissing(const char *cls, const char *sel) { + if (missing.insert(std::string(cls) + " " + sel).second) std::fprintf(stderr, "missing selector: %s %s\n", cls, sel); +} + +// NMB_DISCOVER=1 (instrument only): an unimplemented selector is logged and answers zero instead of raising, so one run lists every gap on +// the renderer's path. Results after a zero answer mean nothing; the frontend itself never answers this way. +static const char *protocolTypes(SEL selector) { + Protocol *protocols[] = {@protocol(MTLDevice), @protocol(MTLTexture), @protocol(MTLBuffer), @protocol(MTLResource), @protocol(MTLCommandQueue), + @protocol(MTLCommandBuffer), @protocol(MTLRenderCommandEncoder), @protocol(MTLBlitCommandEncoder), + @protocol(MTLComputeCommandEncoder), @protocol(MTLCommandEncoder), @protocol(MTLLibrary), @protocol(MTLFunction), + @protocol(MTLRenderPipelineState), @protocol(MTLComputePipelineState), @protocol(MTLSamplerState), + @protocol(MTLDepthStencilState), @protocol(MTLHeap), @protocol(MTLFence), @protocol(MTLEvent), @protocol(MTLSharedEvent)}; + for (Protocol *p : protocols) + for (BOOL required : {YES, NO}) { + objc_method_description m = protocol_getMethodDescription(p, selector, required, YES); + if (m.types) return m.types; + } + return nullptr; +} +static NSMethodSignature *discoverSignature(id self, SEL _cmd, SEL selector) { + const char *types = protocolTypes(selector); + if (!types) {onMissing(class_getName(object_getClass(self)), (std::string(sel_getName(selector)) + " (no protocol signature)").c_str()); types = "@@:";} + return [NSMethodSignature signatureWithObjCTypes:types]; +} +static void discoverForward(id self, SEL _cmd, NSInvocation *invocation) { + onMissing(class_getName(object_getClass(self)), sel_getName(invocation.selector)); + const NSUInteger length = invocation.methodSignature.methodReturnLength; + if (length) {std::vector zero(length); [invocation setReturnValue:zero.data()];} +} +static void discover(Class cls) { + class_addMethod(cls, @selector(methodSignatureForSelector:), (IMP)discoverSignature, "@@::"); + class_addMethod(cls, @selector(forwardInvocation:), (IMP)discoverForward, "v@:@"); +} + +static CGColorRef rgba(double r, double g, double b, double a) { + CGColorSpaceRef space = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); + const CGFloat c[4] = {r, g, b, a}; + CGColorRef color = CGColorCreate(space, c); + CGColorSpaceRelease(space); + return color; +} + +// argv: loader, shader cache. Prints PASS or the first wrong pixel. +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + // NMB_SYSTEM_DEVICE=1: the Mac's own Metal device, the instrument's baseline. + id device = getenv("NMB_SYSTEM_DEVICE") ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, onMissing); + if (!device) {std::printf("FAIL device\n"); return 1;} + if (getenv("NMB_DISCOVER")) {discover(object_getClass(device)); discover(objc_getClass("NBObject"));} + const NSUInteger side = 64; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:side height:side mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + d.storageMode = MTLStorageModeShared; + id target = [device newTextureWithDescriptor:d]; + id queue = [device newCommandQueue]; + if (!target || !queue) {std::printf("FAIL target or queue\n"); return 1;} + + // Blue background; a red square in the top-left quarter; a half-transparent green square in the bottom-right quarter. + CALayer *root = [CALayer layer]; + root.frame = CGRectMake(0, 0, side, side); + root.backgroundColor = rgba(0, 0, 1, 1); + CALayer *red = [CALayer layer]; + red.frame = CGRectMake(0, 0, side / 2, side / 2); + red.backgroundColor = rgba(1, 0, 0, 1); + CALayer *green = [CALayer layer]; + green.frame = CGRectMake(side / 2, side / 2, side / 2, side / 2); + green.backgroundColor = rgba(0, 1, 0, 1); + green.opacity = 0.5; + [root addSublayer:red]; + [root addSublayer:green]; + + CARenderer *renderer = [CARenderer rendererWithMTLTexture:target options:@{kCARendererMetalCommandQueue: queue}]; + if (!renderer) {std::printf("FAIL CARenderer refused the frontend's texture\n"); return 1;} + renderer.layer = root; + renderer.bounds = CGRectMake(0, 0, side, side); + // The layers' implicit transaction would commit at the run loop's end; the renderer sees only committed trees. + // NMB_BLUR_TIMING=1: also render a blurred, shadowed layer for several frames and print the frontend's creation telemetry (pipeline + // counts and times, VkPipeline makings), so the cost of the pipelines CoreAnimation's own renderer asks for can be measured. + if (getenv("NMB_BLUR_TIMING")) { + CALayer *blurred = [CALayer layer]; + blurred.frame = CGRectMake(4, 4, side - 8, side - 8); + blurred.backgroundColor = rgba(1, 1, 0, 0.7); + blurred.cornerRadius = 9; + blurred.shadowOpacity = 0.8; + blurred.shadowRadius = 5; + CIFilter *blur = [CIFilter filterWithName:@"CIGaussianBlur"]; + [blur setValue:@6 forKey:kCIInputRadiusKey]; + blurred.filters = @[blur]; + [root addSublayer:blurred]; + [CATransaction flush]; + for (int frame = 0; frame < 3; ++frame) { + const uint64_t t0 = clock_gettime_nsec_np(CLOCK_UPTIME_RAW); + [renderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [renderer addUpdateRect:renderer.bounds]; + [renderer render]; + [renderer endFrame]; + id fence = [queue commandBuffer]; + [fence commit]; + [fence waitUntilCompleted]; + std::printf("BLUR frame %d %.1f ms\n", frame, (clock_gettime_nsec_np(CLOCK_UPTIME_RAW) - t0) / 1e6); + } + static char table[16384]; + native_bridge_metal_telemetry(table, sizeof table); + for (char *line = std::strtok(table, "\n"); line; line = std::strtok(nullptr, "\n")) + if (line[0] == 'T' || line[0] == 'V' || line[0] == 'S' || line[0] == 'W') std::printf("TELEMETRY %s\n", line); + [blurred removeFromSuperlayer]; + [CATransaction flush]; + } + [renderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL]; + [renderer addUpdateRect:renderer.bounds]; + [renderer render]; + [renderer endFrame]; + // CARenderer commits its own command buffers; one more on the same queue completes after them. + id last = [queue commandBuffer]; + [last commit]; + [last waitUntilCompleted]; + + std::vector pixels(side * side * 4); + [target getBytes:pixels.data() bytesPerRow:side * 4 fromRegion:MTLRegionMake2D(0, 0, side, side) mipmapLevel:0]; + // BGRA. Row 0 of the texture is the layer tree's y 0 (the Mac's Metal device renders it so, NMB_SYSTEM_DEVICE=1): red top-left, + // green bottom-right. + auto at = [&](NSUInteger x, NSUInteger y) {return &pixels[(y * side + x) * 4];}; + struct Probe {NSUInteger x, y; int b, g, r;} probes[] = { + {8, 8, 0, 0, 255}, // red quarter + {side - 8, side - 8, 128, 128, 0}, // green at 0.5 over blue: (0, 0.5, 0.5) + {side - 8, 8, 255, 0, 0}, // background + {8, side - 8, 255, 0, 0}, // background + }; + bool ok = true; + for (auto &p : probes) { + const uint8_t *px = at(p.x, p.y); + if (std::abs(px[0] - p.b) > 3 || std::abs(px[1] - p.g) > 3 || std::abs(px[2] - p.r) > 3 || px[3] != 255) { + std::printf("FAIL pixel (%lu,%lu) = b%u g%u r%u a%u, want b%d g%d r%d\n", (unsigned long)p.x, (unsigned long)p.y, px[0], px[1], px[2], px[3], + p.b, p.g, p.r); + ok = false; + } + } + std::printf("missing selectors: %zu\n", missing.size()); + if (!ok) return 1; + std::printf("PASS: the Mac's CARenderer rendered a layer tree on %s\n", device.name.UTF8String); + return 0; + } +} diff --git a/bridge/compute_archive_host.mm b/bridge/compute_archive_host.mm new file mode 100644 index 0000000..3a0ce49 --- /dev/null +++ b/bridge/compute_archive_host.mm @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: MIT +// Separate processes/cache directories distinguish archive consumption from live memo hits. +#import +#import +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" +#include +#include + +#define REQUIRE(condition) do {if (!(condition)) {std::printf("FAIL line %d: %s; %s\n", __LINE__, #condition, error.localizedDescription.UTF8String ?: "");return 1;}} while (0) + +int main(int argc, char **argv) { + if (argc != 5) return 2; + @autoreleasepool { + NSError *error = nil; + const bool produce = !strcmp(argv[4], "produce"); + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + REQUIRE(device); + size_t bytes = 0; + const void *air = nmb_function_constant_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + REQUIRE(library); + auto constants = [MTLFunctionConstantValues new]; + int mode = 2; + [constants setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + id function = [library newFunctionWithName:@"fcfold" constantValues:constants error:&error]; + REQUIRE(function && !error); + REQUIRE([device respondsToSelector:@selector(newComputePipelineStateWithDescriptor:options:reflection:error:)]); + auto descriptor = [MTLComputePipelineDescriptor new]; + descriptor.computeFunction = function; + descriptor.maxTotalThreadsPerThreadgroup = 32; + descriptor.buffers[1].mutability = MTLMutabilityMutable; + descriptor.label = @"archived compute"; + auto archiveDescriptor = [MTLBinaryArchiveDescriptor new]; + NSURL *url = [NSURL fileURLWithPath:@(argv[3])]; + if (!produce) archiveDescriptor.url = url; + id archive = [device newBinaryArchiveWithDescriptor:archiveDescriptor error:&error]; + REQUIRE(archive && !error); + descriptor.binaryArchives = @[archive]; + if (produce) { + id miss = [device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if (miss || !error) { + std::printf("FAIL strict empty archive: state=%s error=%s\n", miss ? "created" : "nil", error.localizedDescription.UTF8String ?: "nil"); + return 1; + } + REQUIRE([archive addComputePipelineFunctionsWithDescriptor:descriptor error:&error] && !error); + REQUIRE([archive serializeToURL:url error:&error] && !error); + } + id state = [device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + REQUIRE(state && !error && state.maxTotalThreadsPerThreadgroup == 32); + REQUIRE([state.label isEqualToString:descriptor.label]); + // Both descriptor fields and function constants must retain their archive identity. + descriptor.maxTotalThreadsPerThreadgroup = 16; + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error); + descriptor.maxTotalThreadsPerThreadgroup = 32; + descriptor.buffers[1].mutability = MTLMutabilityDefault; + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error); + descriptor.buffers[1].mutability = MTLMutabilityMutable; + mode = 3; + [constants setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + descriptor.computeFunction = [library newFunctionWithName:@"fcfold" constantValues:constants error:&error]; + REQUIRE(descriptor.computeFunction && !error); + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error); + descriptor.computeFunction = function; + descriptor.stageInputDescriptor = [MTLStageInputOutputDescriptor stageInputOutputDescriptor]; + descriptor.stageInputDescriptor.attributes[0].format = MTLAttributeFormatFloat; + // Unsupported descriptor state must be refused by both population and creation. + REQUIRE(![archive addComputePipelineFunctionsWithDescriptor:descriptor error:&error] && error); + REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionNone reflection:nil error:&error] && error); + descriptor.stageInputDescriptor = nil; + const float inputValue = 2; + id input = [device newBufferWithBytes:&inputValue length:4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:128 options:MTLResourceStorageModeShared]; + id command = [[device newCommandQueue] commandBuffer]; + id encoder = [command computeCommandEncoder]; + REQUIRE(input && output && encoder); + [encoder setComputePipelineState:state]; + [encoder setBuffer:input offset:0 atIndex:0]; + [encoder setBuffer:output offset:0 atIndex:1]; + [encoder dispatchThreads:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [encoder endEncoding]; + [command commit]; + [command waitUntilCompleted]; + REQUIRE(command.status == MTLCommandBufferStatusCompleted); + REQUIRE(((const float *)output.contents)[24] == 4000); + // Check the no-list behavior only after the cold archive consumer and GPU verdict. + REQUIRE([device newComputePipelineStateWithFunction:function options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && !error); + std::printf("PASS: %s compute archive, strict descriptor misses and host GPU output\n", produce ? "produced" : "reloaded"); + } +} diff --git a/bridge/driver_cache_host.cpp b/bridge/driver_cache_host.cpp new file mode 100644 index 0000000..9eac391 --- /dev/null +++ b/bridge/driver_cache_host.cpp @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT +// The default compute-function path used by the target; no explicit binary archive. +// Runs over Turnip and the software kernel peer: no shader output is claimed. +#include "NativeMetalObjects.h" +#include +#include +#include + +#define REQUIRE(x) do {if (!(x)) {std::fprintf(stderr,"FAIL line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while (0) + +int main(int argc,char **argv) { + if(argc!=4)return 2; + const bool consume=!std::strcmp(argv[3],"consume"); + REQUIRE(consume||!std::strcmp(argv[3],"produce")); + auto submissions=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_submissions");REQUIRE(submissions); + NmbDevice *device=nmb_device_open(argv[1],argv[2]);REQUIRE(device); + NmbShaderCacheStats before={},after={};REQUIRE(nmb_device_shader_cache_stats(device,&before)); + size_t size=0;const void *air=nmb_fixture_air(&size); + NmbLibrary *library=nmb_library_new(device,air,size,true);REQUIRE(library); + NmbFunction *function=nmb_function_new(library,"scale_and_offset");REQUIRE(function); + NmbComputePipeline *pipeline=nmb_compute_pipeline_new(device,function);REQUIRE(pipeline); + float input[64]={}; + NmbBuffer *source=nmb_buffer_new(device,sizeof input,0,input); + NmbBuffer *destination=nmb_buffer_new(device,sizeof input,0,nullptr);REQUIRE(source&&destination); + NmbQueue *queue=nmb_queue_new(device);REQUIRE(queue); + NmbCommandBuffer *command=nmb_command_buffer_new(queue);REQUIRE(command); + NmbEncoder *encoder=nmb_compute_encoder_new(command);REQUIRE(encoder); + REQUIRE(nmb_compute_set_pipeline(encoder,pipeline)); + REQUIRE(nmb_encoder_set_buffer(encoder,0,source,0,0)); + REQUIRE(nmb_encoder_set_buffer(encoder,0,destination,0,1)); + const float scale=2;REQUIRE(nmb_encoder_set_bytes(encoder,0,&scale,sizeof scale,2)); + const uint64_t threads[]={64,1,1}; + REQUIRE(nmb_compute_dispatch(encoder,threads,threads,true)); + REQUIRE(nmb_encoder_end(encoder)); + const unsigned submitted=submissions(); + REQUIRE(nmb_command_buffer_commit(command)); + REQUIRE(nmb_command_buffer_wait(command,10000)==0); + REQUIRE(nmb_command_buffer_status(command)==4&&submissions()>submitted); + REQUIRE(nmb_device_shader_cache_stats(device,&after)); + std::printf("cache %s: artifacts hit=%llu miss=%llu write=%llu; driver load=%llu hit=%llu miss=%llu save=%llu same=%llu; rejected=%llu readfail=%llu writefail=%llu truncated=%llu mismatched=%llu\n",argv[3], + (unsigned long long)(after.hits-before.hits),(unsigned long long)(after.misses-before.misses),(unsigned long long)(after.writes-before.writes), + (unsigned long long)(after.driver_loads-before.driver_loads),(unsigned long long)(after.driver_hits-before.driver_hits), + (unsigned long long)(after.driver_misses-before.driver_misses),(unsigned long long)(after.driver_saves-before.driver_saves), + (unsigned long long)(after.driver_unchanged-before.driver_unchanged),(unsigned long long)after.rejected,(unsigned long long)after.read_failures, + (unsigned long long)after.write_failures,(unsigned long long)after.driver_truncated,(unsigned long long)after.driver_mismatched); + if(consume){ + REQUIRE(after.hits>before.hits&&after.misses==before.misses); + REQUIRE(after.driver_loads>before.driver_loads&&after.driver_hits>before.driver_hits); + REQUIRE(after.driver_misses==before.driver_misses); + }else{ + REQUIRE(after.misses>before.misses&&after.writes>before.writes); + REQUIRE(after.driver_misses>before.driver_misses&&after.driver_saves>before.driver_saves); + } + REQUIRE(!after.read_failures&&!after.write_failures&&!after.rejected&&!after.driver_truncated&&!after.driver_mismatched); + nmb_command_buffer_release(command);nmb_queue_release(queue); + nmb_buffer_release(destination);nmb_buffer_release(source);nmb_compute_pipeline_release(pipeline); + nmb_function_release(function);nmb_library_release(library);nmb_device_release(device); + std::printf("PASS: default compute %s uses separate artifact/Turnip caches; software transport, results unchecked\n",argv[3]); + return 0; +} diff --git a/bridge/frontend_host.mm b/bridge/frontend_host.mm new file mode 100644 index 0000000..39f0414 --- /dev/null +++ b/bridge/frontend_host.mm @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +// Host run of the frontend checks (FrontendChecks.h) over the bridge and a host Vulkan driver as the loader (MoltenVK on the Mac: an offline +// instrument; on the tablet the plugin runs the same checks over Turnip). Prints one PASS line or the failures. +#include "FrontendChecks.h" +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +static std::vector missing; +static void onMissing(const char *cls, const char *sel) {missing.push_back(std::string(cls) + " " + sel);} + +// argv: loader, shader cache, [QuartzCore default.metallib or ""] +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, onMissing); + if (!device) {std::printf("FAIL device\n"); return 1;} + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + air = nmb_function_constant_air(&bytes); + id fcLibrary = native_bridge_metal_library_with_air(air, bytes); + if (!library || !fcLibrary) {std::printf("FAIL fixture libraries\n"); return 1;} + frontendchecks::Report r = {}; + frontendchecks::run(device, library, fcLibrary, argc > 3 && argv[3][0] ? @(argv[3]) : nil, &r); + if (r.qcFunctions) std::printf("QuartzCore bare-descriptor probe: %u functions, kernels %u/%u, pairs %u/%u (not complete shader coverage)\n", r.qcFunctions, r.qcKernelsOk, r.qcKernels, r.qcPairsOk, r.qcPairs); + for (auto &m : missing) std::printf("missing selector: %s\n", m.c_str()); + if (r.failed) {std::printf("FAILED %u checks, first: %s\n", r.failed, r.first); return 1;} + std::printf("PASS: native Metal frontend over the bridge on %s: compute, render, IOSurface render, surface sampling, texture compute, " + "handlers, refusal, function constants, events and fences\n", device.name.UTF8String); + return 0; + } +} diff --git a/bridge/graphics_cache_loader.cpp b/bridge/graphics_cache_loader.cpp new file mode 100644 index 0000000..4d71bcc --- /dev/null +++ b/bridge/graphics_cache_loader.cpp @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: MIT */ +/* Offline qualification only: forward to the actual Vulkan compiler, enabling + * its reported cache-control feature and optionally forbidding compilation. + * A negative control corrupts only a private copy of a cache UUID. Never deploy. */ +#include +#include +#include +#include +#include +#include +#include + +static PFN_vkGetInstanceProcAddr realInstance; +static std::atomic realDevice; +static std::once_flag initialized; +static std::atomic activeInstance; +static std::atomic strict, badUuid; +// calls, success, compile-required, valid feedback, whole-cache hit, UUID mutations, devices, +// calls with an application cache, their valid feedback and their cache hits +static std::atomic counts[10]; +static void initialize() { + std::call_once(initialized,[]{ + const char *path=std::getenv("NMB_TEST_REAL_VULKAN_LOADER"); + void *library=path?dlopen(path,RTLD_NOW|RTLD_LOCAL):nullptr; + if(!library)std::abort(); + realInstance=reinterpret_cast(dlsym(library,"vkGetInstanceProcAddr")); + if(!realInstance)realInstance=reinterpret_cast(dlsym(library,"vk_icdGetInstanceProcAddr")); + if(!realInstance)std::abort(); + }); +} +static VKAPI_ATTR VkResult VKAPI_CALL createDevice(VkPhysicalDevice physical,const VkDeviceCreateInfo *info, + const VkAllocationCallbacks *allocator,VkDevice *device) { + initialize(); + // The qualified runtime requests Vulkan 1.2. Refuse an unexpected chain + // rather than duplicate a promoted feature or alter caller-owned structures. + for(auto p=static_cast(info->pNext);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES || + p->sType==VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES)return VK_ERROR_FEATURE_NOT_PRESENT; + auto query=reinterpret_cast(realInstance(activeInstance.load(),"vkGetPhysicalDeviceFeatures2")); + if(!query)return VK_ERROR_FEATURE_NOT_PRESENT; + VkPhysicalDevicePipelineCreationCacheControlFeatures supported={};supported.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES; + VkPhysicalDeviceFeatures2 features={};features.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;features.pNext=&supported; + query(physical,&features);if(!supported.pipelineCreationCacheControl)return VK_ERROR_FEATURE_NOT_PRESENT; + std::vector extensions; + if(info->enabledExtensionCount)extensions.assign(info->ppEnabledExtensionNames,info->ppEnabledExtensionNames+info->enabledExtensionCount); + bool listed=false;for(auto name:extensions)listed|=!std::strcmp(name,VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME); + if(!listed)extensions.push_back(VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME); + supported.pNext=const_cast(info->pNext); + VkDeviceCreateInfo copy=*info;copy.pNext=&supported;copy.enabledExtensionCount=(uint32_t)extensions.size();copy.ppEnabledExtensionNames=extensions.data(); + auto real=reinterpret_cast(realInstance(activeInstance.load(),"vkCreateDevice")); + if(!real)return VK_ERROR_INITIALIZATION_FAILED; + const auto result=real(physical,©,allocator,device);if(result==VK_SUCCESS)counts[6]++; + return result; +} +static VKAPI_ATTR VkResult VKAPI_CALL createCache(VkDevice device,const VkPipelineCacheCreateInfo *info, + const VkAllocationCallbacks *allocator,VkPipelineCache *cache) { + VkPipelineCacheCreateInfo copy=*info;std::vector bytes; + if(badUuid.load() && info->initialDataSize) { + if(info->initialDataSize<32)return VK_ERROR_INITIALIZATION_FAILED; + const auto *source=static_cast(info->pInitialData); + bytes.assign(source,source+info->initialDataSize);bytes[16]^=1;copy.pInitialData=bytes.data();counts[5]++; + } + auto real=reinterpret_cast(realDevice.load()(device,"vkCreatePipelineCache")); + return real(device,©,allocator,cache); +} +static VKAPI_ATTR VkResult VKAPI_CALL createGraphics(VkDevice device,VkPipelineCache cache,uint32_t count, + const VkGraphicsPipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) { + if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED; + std::vector copies(infos,infos+count); + std::vector feedback(count); + std::vector additions(count); + std::vector observed(count); + for(uint32_t i=0;i(copies[i].pNext);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO) + observed[i]=reinterpret_cast(p)->pPipelineCreationFeedback; + if(!observed[i]) { + auto &a=additions[i];a.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO; + a.pNext=copies[i].pNext;a.pPipelineCreationFeedback=&feedback[i];copies[i].pNext=&a;observed[i]=&feedback[i]; + } + } + auto real=reinterpret_cast(realDevice.load()(device,"vkCreateGraphicsPipelines")); + counts[0]+=count;if(cache)counts[7]+=count;const auto result=real(device,cache,count,copies.data(),allocator,pipelines); + if(result==VK_PIPELINE_COMPILE_REQUIRED)counts[2]++; + for(uint32_t i=0;iflags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) { + counts[3]++;if(cache)counts[8]++; + if(observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT){counts[4]++;if(cache)counts[9]++;} + } + } + return result; +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) { + initialize(); + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(createGraphics); + if(!std::strcmp(name,"vkCreatePipelineCache"))return reinterpret_cast(createCache); + return realDevice.load()(device,name); +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) { + initialize(); + if(instance){ + activeInstance=instance; + realDevice=reinterpret_cast(realInstance(instance,"vkGetDeviceProcAddr")); + if(!realDevice.load())std::abort(); + } + if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast(vkGetDeviceProcAddr); + if(!std::strcmp(name,"vkCreateDevice"))return reinterpret_cast(createDevice); + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(createGraphics); + if(!std::strcmp(name,"vkCreatePipelineCache"))return reinterpret_cast(createCache); + return realInstance(instance,name); +} +extern "C" void nmb_test_graphics_cache_control(bool forbidCompilation,bool corruptUuid) {strict=forbidCompilation;badUuid=corruptUuid;} +extern "C" void nmb_test_graphics_cache_counts(uint64_t out[10]) {for(unsigned i=0;i<10;++i)out[i]=counts[i].load();} diff --git a/bridge/import_count_loader.cpp b/bridge/import_count_loader.cpp new file mode 100644 index 0000000..a98c4a1 --- /dev/null +++ b/bridge/import_count_loader.cpp @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: MIT */ +/* Host-test Vulkan forwarding loader. Every request reaches the real driver; + * count successful external-host-memory imports without changing any request, + * code, driver identity, memory contents, fence or GPU result. Never deployed. */ +#include +#include +#include +#include +#include +#include +#include +#include + +static PFN_vkGetInstanceProcAddr realInstance; +static PFN_vkGetDeviceProcAddr realDevice; +static std::once_flag initialized; +static std::atomic imported; +static std::atomic importDelayMs; +static void initialize() { + std::call_once(initialized,[]{ + const char *path=std::getenv("NMB_TEST_REAL_VULKAN_LOADER"); + void *library=path?dlopen(path,RTLD_NOW|RTLD_LOCAL):nullptr; + if(!library)std::abort(); + realInstance=reinterpret_cast(dlsym(library,"vkGetInstanceProcAddr")); + realDevice=reinterpret_cast(dlsym(library,"vkGetDeviceProcAddr")); + if(!realInstance || !realDevice)std::abort(); + }); +} +static VKAPI_ATTR VkResult VKAPI_CALL allocate(VkDevice device,const VkMemoryAllocateInfo *info, + const VkAllocationCallbacks *allocator,VkDeviceMemory *memory) { + initialize(); + bool hostImport=false; + for(auto node=static_cast(info->pNext);node;node=node->pNext) + if(node->sType==VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT){hostImport=true;break;} + // A bounded, one-shot delay makes a concurrent first-lookup miss + // reproducible. The actual allocation and its result are unchanged. + if(hostImport) { + const unsigned delay=importDelayMs.exchange(0); + if(delay)std::this_thread::sleep_for(std::chrono::milliseconds(delay)); + } + auto real=reinterpret_cast(realDevice(device,"vkAllocateMemory")); + const VkResult result=real(device,info,allocator,memory); + if(result==VK_SUCCESS && hostImport)imported++; + return result; +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) { + initialize(); + if(!std::strcmp(name,"vkAllocateMemory"))return reinterpret_cast(allocate); + return realDevice(device,name); +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) { + initialize(); + if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast(vkGetDeviceProcAddr); + if(!std::strcmp(name,"vkAllocateMemory"))return reinterpret_cast(allocate); + return realInstance(instance,name); +} +extern "C" uint64_t nmb_test_vk_host_import_count() {return imported.load();} +extern "C" void nmb_test_vk_host_import_delay(unsigned milliseconds) { + importDelayMs.store(milliseconds<=100?milliseconds:100); +} diff --git a/bridge/nil_args_host.mm b/bridge/nil_args_host.mm new file mode 100644 index 0000000..09dd2be --- /dev/null +++ b/bridge/nil_args_host.mm @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +// Host check: a blit encoder call with a nil resource (what a refused allocation hands the caller) fails its command buffer instead of raising an +// exception. An exception escaping a worker thread of the host process aborts the process; on the tablet that killed SpringBoard (int-075/083). +#import +#import +#include +#include "../NativeBridgeMetalFrontend.h" + +__attribute__((noinline)) static id nothing() {return nil;} // not a literal: the compiler rejects nil for these nonnull parameters + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + id queue = [device newCommandQueue]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + id texture = [device newTextureWithDescriptor:d]; + int failed = 0; + for (int variant = 0; variant < 4; ++variant) { + @try { + id cb = [queue commandBuffer]; + id blit = [cb blitCommandEncoder]; + switch (variant) { + case 0: [blit copyFromBuffer:(id)nothing() sourceOffset:0 toBuffer:buffer destinationOffset:0 size:16]; break; + case 1: [blit fillBuffer:(id)nothing() range:NSMakeRange(0, 16) value:0]; break; + case 2: [blit copyFromTexture:(id)nothing() sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(8, 8, 1) + toBuffer:buffer destinationOffset:0 destinationBytesPerRow:32 destinationBytesPerImage:256]; break; + default: [blit copyFromTexture:texture toTexture:(id)nothing()]; break; + } + [blit endEncoding]; + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusError) {std::printf("FAIL variant %d: status %ld, not an error\n", variant, (long)cb.status); ++failed;} + } @catch (NSException *e) { + std::printf("FAIL variant %d raised %s\n", variant, e.name.UTF8String); ++failed; + } + } + if (failed) return 1; + std::printf("PASS: a nil resource fails the command buffer in every blit variant, no exception\n"); + } + return 0; +} diff --git a/bridge/no_copy_import_host.mm b/bridge/no_copy_import_host.mm new file mode 100644 index 0000000..c00441c --- /dev/null +++ b/bridge/no_copy_import_host.mm @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +// Actual public Metal no-copy calls through the native bridge. Every buffer +// remains a distinct owner; count real Vulkan imports and check real GPU bytes. +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +constexpr unsigned workers=16; +struct State {std::array,workers> released{};std::atomic errors{0};}; + +static bool creation(id device,bool mixed,uint64_t (*imports)(),void (*delay)(unsigned)) { + auto state=std::make_shared();bool gpu=true; + void *mapping=nullptr;size_t bytes=0;IOSurfaceRef surface=nullptr; + uint64_t allocated=0; + @autoreleasepool { + if(mixed) { + surface=IOSurfaceCreate((__bridge CFDictionaryRef)@{(id)kIOSurfaceWidth:@64,(id)kIOSurfaceHeight:@16, + (id)kIOSurfaceBytesPerElement:@4,(id)kIOSurfaceBytesPerRow:@256,(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}); + if(!surface)return false; + mapping=IOSurfaceGetBaseAddress(surface);bytes=IOSurfaceGetAllocSize(surface); + } else { + bytes=(size_t)getpagesize();mapping=mmap(nullptr,bytes,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(mapping==MAP_FAILED)return false; + } + if(!mapping || bytes<4096)return false; + memset(mapping,0xcc,bytes); + auto buffers=[NSMutableArray array],textures=[NSMutableArray array]; + std::mutex startLock;std::condition_variable ready;unsigned waiting=0;bool start=false; + std::vector threads; + const uint64_t before=imports();delay(25); + for(unsigned i=0;i lock(startLock);++waiting;ready.notify_all();ready.wait(lock,[&]{return start;});} + @autoreleasepool { + if(mixed && (i&1)) { + auto d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:16 mipmapped:NO]; + d.storageMode=MTLStorageModeShared;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite; + id texture=[device newTextureWithDescriptor:d iosurface:surface plane:0]; + if(!texture){state->errors++;return;} + @synchronized(textures){[textures addObject:texture];} + } else { + auto owner=state;const unsigned ordinal=i; + id buffer=[device newBufferWithBytesNoCopy:mapping length:bytes options:MTLResourceStorageModeShared + deallocator:^(void *,NSUInteger){owner->released[ordinal]++;}]; + if(!buffer){state->errors++;return;} + @synchronized(buffers){[buffers addObject:buffer];} + } + } + }); + {std::unique_lock lock(startLock);ready.wait(lock,[&]{return waiting==workers;});start=true;ready.notify_all();} + for(auto &thread:threads)thread.join(); + allocated=imports()-before; + const unsigned expected=mixed?workers/2:workers; + if(state->errors || buffers.count!=expected || textures.count!=(mixed?workers/2:0))return false; + @autoreleasepool { + // The API still returns distinct buffer objects and independent + // callbacks; only the driver's import is shared. + if([NSSet setWithArray:buffers].count!=expected)return false; + } + id queue=[device newCommandQueue]; + id write=[queue commandBuffer];id fill=[write blitCommandEncoder]; + [fill fillBuffer:buffers[0] range:NSMakeRange(0,4096) value:0x5a];[fill endEncoding];[write commit];[write waitUntilCompleted]; + // Use an explicit completed producer, so correctness here does not + // assume implicit synchronization of arbitrary no-copy Metal aliases. + id output=[device newBufferWithLength:workers*4096 options:MTLResourceStorageModeShared]; + id read=[queue commandBuffer];id copy=[read blitCommandEncoder]; + if(!queue || !write || !fill || !output || !read || !copy)return false; + NSUInteger at=0; + for(id buffer in buffers){[copy copyFromBuffer:buffer sourceOffset:0 toBuffer:output destinationOffset:at size:4096];at+=4096;} + for(id texture in textures){ + [copy copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1) + toBuffer:output destinationOffset:at destinationBytesPerRow:256 destinationBytesPerImage:4096];at+=4096; + } + [copy endEncoding];[read commit];[read waitUntilCompleted]; + gpu=write.status==MTLCommandBufferStatusCompleted && read.status==MTLCommandBufferStatusCompleted && at==workers*4096; + for(unsigned i=0;ireleased[i].load()!=(mixed&&(i&1)?0u:1u))owners=false; + if(owners)break;usleep(1000); + } + if(surface)CFRelease(surface);else munmap(mapping,bytes); + std::printf("mode=%s buffers=%u vk_imports=%llu gpu=%s independent_owners=%s\n",mixed?"mixed":"raw",mixed?workers/2:workers, + (unsigned long long)allocated,gpu?"pass":"FAIL",owners?"pass":"FAIL"); + return allocated==1 && gpu && owners; +} + +static bool reentry(id device) { + const size_t page=(size_t)getpagesize();void *mapping=mmap(nullptr,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(mapping==MAP_FAILED)return false; + auto finished=dispatch_semaphore_create(0);auto state=std::make_shared(); + @autoreleasepool { + id outer=[device newBufferWithBytesNoCopy:mapping length:page options:MTLResourceStorageModeShared + deallocator:^(void *,NSUInteger){ + state->released[0]++; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{ + @autoreleasepool { + id nested=[device newBufferWithBytesNoCopy:(char *)mapping+page length:page options:MTLResourceStorageModeShared + deallocator:^(void *,NSUInteger){state->released[1]++;}]; + if(!nested)state->errors++; + } + dispatch_semaphore_signal(finished); + }); + if(dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,3*NSEC_PER_SEC)))state->errors++; + }]; + if(!outer)return false; + } + const bool ok=!state->errors && state->released[0]==1 && state->released[1]==1; + // On a timeout keep the test-owned mapping alive until process exit; + // never turn a lifetime failure into a submitted dangling pointer. + if(ok)munmap(mapping,page*2); + std::printf("deallocator cross-thread reentry=%s\n",ok?"pass":"FAIL");return ok; +} + +int main(int argc,char **argv) { + if(argc!=4)return 2; + @autoreleasepool { + id device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2; + void *loader=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!loader)return 2; + auto imports=(uint64_t (*)())dlsym(loader,"nmb_test_vk_host_import_count"); + auto delay=(void (*)(unsigned))dlsym(loader,"nmb_test_vk_host_import_delay"); + if(!imports || !delay)return 2; + bool ok=false; + if(!strcmp(argv[3],"raw"))ok=creation(device,false,imports,delay); + else if(!strcmp(argv[3],"mixed"))ok=creation(device,true,imports,delay); + else if(!strcmp(argv[3],"reentry"))ok=reentry(device); + if(!ok)return 1; + std::puts("PASS: native no-copy construction coalesces real imports while retaining independent objects, GPU bytes and callbacks"); + } +} diff --git a/bridge/patches/0001-optional-constant-output-members.patch b/bridge/patches/0001-optional-constant-output-members.patch new file mode 100644 index 0000000..8042565 --- /dev/null +++ b/bridge/patches/0001-optional-constant-output-members.patch @@ -0,0 +1,59 @@ +diff --git a/crates/metal2vulkan-air/src/canon/sccp.rs b/crates/metal2vulkan-air/src/canon/sccp.rs +index e47e683..f89e2fc 100644 +--- a/crates/metal2vulkan-air/src/canon/sccp.rs ++++ b/crates/metal2vulkan-air/src/canon/sccp.rs +@@ -995,6 +995,30 @@ impl<'b> Solver<'b> { + self.set(i, new, m, held) + } + ++ /// Whether `v` is an aggregate built by `insertvalue` chains whose only unsupplied-constant ++ /// taint sits in individual inserted members, so those members alone can become `undef` ++ /// (Metal leaves an unsupplied optional constant's value indeterminate) while the rest of ++ /// the aggregate keeps its computed members. ++ fn leafwise(&self, v: Value) -> bool { ++ let mut cur = v; ++ loop { ++ if !matches!(self.lat(cur), Lat::Fc(_)) { ++ return true; ++ } ++ let Value::Inst(i) = cur else { return false }; ++ let Some(inst) = self.body.insts.get(i) else { ++ return false; ++ }; ++ let InstKind::InsertValue { agg, val, .. } = &inst.kind else { ++ return false; ++ }; ++ if matches!(self.lat(*val), Lat::Fc(_)) && !matches!(val, Value::Inst(_)) { ++ return false; ++ } ++ cur = *agg; ++ } ++ } ++ + /// After the fixpoint: every reachable non-pure use of an unsupplied constant needs the + /// caller's value, and a reachable branch on a literal `undef`/`poison` is refused. + fn check(&self, cx: &Context<'_>) -> Result<(), Error> { +@@ -1031,6 +1055,11 @@ impl<'b> Solver<'b> { + ))); + } + } ++ if let InstKind::Ret(Some(v)) = &inst.kind { ++ if self.leafwise(*v) { ++ continue; ++ } ++ } + let mut needs = None; + inst.kind.for_each_value(|v| { + if let Lat::Fc(index) = self.lat(v) { +@@ -1070,7 +1099,11 @@ impl<'b> Solver<'b> { + let deletable = is_pure(&inst.kind) + || is_normalize(&inst.kind) + || is_defined_arg(&inst.kind, cx.folder.consts, cx.fcs).is_some(); ++ let leaf_chain = matches!(inst.kind, InstKind::InsertValue { .. }) ++ && matches!(lat, Lat::Fc(_)) ++ && self.leafwise(Value::Inst(i)); + match (&inst.kind, lat) { ++ _ if leaf_chain => {} + (_, Lat::Const(c)) if deletable => edit.replace(i, Value::Const(c)), + (_, Lat::Fc(_)) if deletable => { + let undef = cx.folder.consts.intern( diff --git a/bridge/patches/0002-resource-wait-storage-dependencies.patch b/bridge/patches/0002-resource-wait-storage-dependencies.patch new file mode 100644 index 0000000..6177488 --- /dev/null +++ b/bridge/patches/0002-resource-wait-storage-dependencies.patch @@ -0,0 +1,67 @@ +diff --git a/crates/metal2vulkan-runtime/src/used_resources.rs b/crates/metal2vulkan-runtime/src/used_resources.rs +index 7521345..03001a8 100644 +--- a/crates/metal2vulkan-runtime/src/used_resources.rs ++++ b/crates/metal2vulkan-runtime/src/used_resources.rs +@@ -58,6 +58,62 @@ pub struct UsedResources { + pub heaps: Vec, + } + ++// Compare host address ranges without touching their bytes. Object identity covers private ++// memory; host ranges additionally cover distinct imports of an IOSurface or overlapping ++// no-copy buffers. Unknown arithmetic is conservative. Rebinding is not performed by this ++// consumer, and callers must not remap these ranges while the resources are in use. ++fn buffers_overlap(a: &Buffer, b: &Buffer) -> bool { ++ if a == b { ++ return true; ++ } ++ let (Some(a_start), Some(b_start)) = (a.contents(), b.contents()) else { ++ return false; ++ }; ++ let (Ok(a_len), Ok(b_len)) = (usize::try_from(a.length()), usize::try_from(b.length())) else { ++ return true; ++ }; ++ let a_start = a_start.as_ptr() as usize; ++ let b_start = b_start.as_ptr() as usize; ++ let (Some(a_end), Some(b_end)) = (a_start.checked_add(a_len), b_start.checked_add(b_len)) else { ++ return true; ++ }; ++ a_start < b_end && b_start < a_end ++} ++ ++impl UsedResources { ++ fn references_buffer(&self, buffer: &Buffer) -> bool { ++ if buffer.heap().is_some_and(|heap| self.heaps.contains(heap)) { ++ return true; ++ } ++ self.buffers.iter().any(|other| buffers_overlap(buffer, other)) ++ || self.heaps.iter().any(|heap| heap.bytes().is_some_and(|bytes| buffers_overlap(buffer, bytes))) ++ } ++ ++ /// Whether this list names a resource's storage, including a view's root, a buffer ++ /// texture's backing buffer, a heap's resources and overlapping imported host ranges. ++ /// A caller may skip waiting only while this is false. This consumer does not rebind ++ /// or remap live backing ranges. Indirect command buffers remain conservative. ++ pub fn references_resource(&self, resource: super::resource::ResourceRef<'_>) -> bool { ++ use super::resource::ResourceRef; ++ match resource { ++ ResourceRef::Buffer(buffer) => self.references_buffer(buffer), ++ ResourceRef::Texture(texture) => match texture.memory() { ++ TextureMemory::Heap(heap) => self.references_heap(heap), ++ TextureMemory::Buffer(buffer) => self.references_buffer(buffer), ++ TextureMemory::Own(root) => self.textures.contains(root), ++ }, ++ ResourceRef::IndirectCommandBuffer(_) => true, ++ } ++ } ++ ++ /// Heap storage is treated as one dependency, conservatively including its disjoint ++ /// members. Imports over its host range are dependencies too. ++ pub fn references_heap(&self, heap: &Heap) -> bool { ++ self.heaps.contains(heap) ++ || heap.bytes().is_some_and(|bytes| self.references_buffer(bytes)) ++ } ++} ++ + /// The kind of a command encoder ([`EndedEncoder::kind`]), by the Metal protocol it is. + #[non_exhaustive] + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] diff --git a/bridge/patches/0003-cache-device-entry-point-priming.patch b/bridge/patches/0003-cache-device-entry-point-priming.patch new file mode 100644 index 0000000..47bd1ea --- /dev/null +++ b/bridge/patches/0003-cache-device-entry-point-priming.patch @@ -0,0 +1,253 @@ +diff --git a/crates/metal2vulkan-runtime/src/device/mod.rs b/crates/metal2vulkan-runtime/src/device/mod.rs +index 990c892..00a6e55 100644 +--- a/crates/metal2vulkan-runtime/src/device/mod.rs ++++ b/crates/metal2vulkan-runtime/src/device/mod.rs +@@ -962,6 +962,21 @@ impl Device { + // SAFETY: the device is live, of `physical`, and enables timeline semaphores, and + // `bufferDeviceAddress` where `device_address` says so; `queues` holds at least one of + // its queues, which nothing uses yet. ++ let priming_has = prime::Has { ++ timestamps: timestamp_bits > 0, ++ device_addresses: device_address, ++ generated_commands: generated_commands.is_some(), ++ sparse_binding: sparse.binds(), ++ }; ++ // Raw disk/cache I/O only: starting the ordinary guarded cache or its ++ // worker before priming would change the destructor ordering it protects. ++ let priming_cache = cache.priming( ++ &unsafe { instance.raw.get_physical_device_properties(physical) }, ++ compile.profile_fingerprint(), ++ instance.settings, ++ prime::cache_key(priming_has, queue_family), ++ instance.sink().cloned(), ++ ); + if let Err(error) = unsafe { + prime::prime( + &instance.raw, +@@ -971,13 +986,9 @@ impl Device { + queue_family, + &queues, + Sharing::new(&queue_families), +- prime::Has { +- timestamps: timestamp_bits > 0, +- device_addresses: device_address, +- generated_commands: generated_commands.is_some(), +- sparse_binding: sparse.binds(), +- }, ++ priming_has, + generated_commands.as_ref(), ++ priming_cache, + ) + } { + scheduler.stop(); +diff --git a/crates/metal2vulkan-runtime/src/device/prime.rs b/crates/metal2vulkan-runtime/src/device/prime.rs +index 91791ba..50856aa 100644 +--- a/crates/metal2vulkan-runtime/src/device/prime.rs ++++ b/crates/metal2vulkan-runtime/src/device/prime.rs +@@ -52,9 +52,11 @@ use crate::generated_commands::{ + use crate::loader_word::LoaderWord; + use crate::memory; + use crate::release::{self, Generated, Released}; ++use crate::shader_cache::PrimingCache; + use crate::sharing::Sharing; + use crate::tracker::Barrier; + use ash::vk; ++use sha2::{Digest, Sha256}; + use std::time::{Duration, Instant}; + + use super::QueueSlot; +@@ -184,7 +186,7 @@ const VERTEX_SHADER: [u32; 29] = [ + + /// What the device primed has: a queue family that writes timestamps, device addresses, + /// `VK_EXT_device_generated_commands`, and sparse binding on its queues. +-#[derive(Clone, Copy)] ++#[derive(Clone, Copy, Debug)] + pub(crate) struct Has { + pub(crate) timestamps: bool, + pub(crate) device_addresses: bool, +@@ -192,6 +194,16 @@ pub(crate) struct Has { + pub(crate) sparse_binding: bool, + } + ++/// Identity includes every byte of the helper implementation, enabled compile ++/// profile (in Records), instance settings, driver identity, and these choices. ++/// No pointer/handle or host path enters the persistent key. ++pub(super) fn cache_key(has: Has, queue_family: u32) -> metal2vulkan::cache::Hash256 { ++ crate::shader_cache::driver::Making::new("entry-point-priming-v1") ++ .digest(&Sha256::digest(include_bytes!("prime.rs")).into()) ++ .text(&(has, queue_family)) ++ .finish() ++} ++ + /// Prime `device` of `physical`, made from `instance`, whose queues are `queues`, the first of + /// its primary family `queue_family`, its buffers and images shared as `sharing` says, on this + /// thread; `has` says what the device has, and `generated` holds the commands of +@@ -213,6 +225,7 @@ pub(super) unsafe fn prime( + sharing: Sharing<'_>, + has: Has, + generated: Option<&Commands>, ++ mut disk: PrimingCache, + ) -> Result<(), Error> { + let mut primer = Primer { + device, +@@ -222,6 +235,8 @@ pub(super) unsafe fn prime( + memory: unsafe { instance.get_physical_device_memory_properties(physical) }, + calls: 0, + held: Vec::new(), ++ disk: &mut disk, ++ pipeline_cache: vk::PipelineCache::null(), + }; + // SAFETY: the caller's. + let primed = unsafe { +@@ -239,6 +254,10 @@ pub(super) unsafe fn prime( + unsafe { + let _ = device.device_wait_idle(); + primer.made(Call::DeviceWaitIdle); ++ if primed.is_ok() && primer.pipeline_cache != vk::PipelineCache::null() { ++ primer.disk.keep(device, loader_word, primer.pipeline_cache); ++ primer.made(Call::GetPipelineCacheData); ++ } + while let Some(release) = primer.held.pop() { + let call = release(device); + primer.made(call); +@@ -303,6 +322,8 @@ struct Primer<'a> { + calls: u128, + /// What was made, released last first. + held: Vec, ++ disk: &'a mut PrimingCache, ++ pipeline_cache: vk::PipelineCache, + } + + impl Primer<'_> { +@@ -575,7 +596,7 @@ impl Primer<'_> { + .layout(layout) + .push_next(&mut flags); + let pipeline = device +- .create_compute_pipelines(vk::PipelineCache::null(), &[info], None) ++ .create_compute_pipelines(self.pipeline_cache, &[info], None) + .map_err(|(_, error)| vulkan(error))?[0]; + self.hold(pipeline); + let pipelines = ExecutionSetPipelineInfo::new(pipeline, 1); +@@ -848,9 +869,8 @@ impl Primer<'_> { + let module = device.create_shader_module(&info, None).map_err(vulkan)?; + self.made(Call::CreateShaderModule); + self.hold(module); +- let cache = device +- .create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) +- .map_err(vulkan)?; ++ let cache = self.disk.open(device, self.loader_word).map_err(vulkan)?; ++ self.pipeline_cache = cache; + self.made(Call::CreatePipelineCache); + self.hold(cache); + let stage = vk::PipelineShaderStageCreateInfo::default() +diff --git a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +index 88e0fad..513fe37 100644 +--- a/crates/metal2vulkan-runtime/src/shader_cache/mod.rs ++++ b/crates/metal2vulkan-runtime/src/shader_cache/mod.rs +@@ -282,6 +282,106 @@ impl CacheSetup { + } + } + ++/// Disk access for entry-point priming, before any exit hook is registered. ++/// Deliberately no worker, exit-guard section, or ordinary driver::make call. ++/// Priming keeps its raw call/cleanup ordering and records every entry point. ++pub(super) struct PrimingCache { ++ store: Store, ++ identity: driver::Identity, ++ name: String, ++ loaded: Option>, ++} ++ ++impl CacheSetup { ++ pub(super) fn priming( ++ &self, ++ properties: &vk::PhysicalDeviceProperties, ++ profile: Hash256, ++ settings: Hash256, ++ key: Hash256, ++ sink: Option, ++ ) -> PrimingCache { ++ let identity = driver::Identity::of(properties); ++ let name = Records { ++ identity, ++ settings, ++ profile, ++ } ++ .name(Making::new("device-entry-point-priming").digest(&key)); ++ PrimingCache { ++ store: Store::new(self.directory.clone(), sink), ++ identity, ++ name, ++ loaded: None, ++ } ++ } ++} ++ ++impl PrimingCache { ++ /// The caller owns the raw cache and primes/releases its entry points. ++ /// No exit_guard::called/enter may happen here, before priming is finished. ++ pub(super) unsafe fn open( ++ &mut self, ++ raw: &ash::Device, ++ word: &crate::loader_word::LoaderWord, ++ ) -> Result { ++ let seed = match self.store.load_content(&self.name, |bytes| { ++ let data = driver::open(&self.identity, &bytes)?; ++ Some(bytes[data].to_vec()) ++ }) { ++ Lookup::Hit(bytes) => Some(bytes), ++ _ => None, ++ }; ++ if let Some(bytes) = seed { ++ let info = vk::PipelineCacheCreateInfo::default().initial_data(&bytes); ++ // SAFETY: the caller owns the live device and the info's bytes outlive the call. ++ if let Ok(cache) = unsafe { ++ word.restore(); ++ raw.create_pipeline_cache(&info, None) ++ } { ++ self.loaded = Some(bytes); ++ return Ok(cache); ++ } ++ } ++ // Invalid/absent/refused data is never treated as a successful load. ++ // SAFETY: as above; the empty create info outlives the call. ++ unsafe { ++ word.restore(); ++ raw.create_pipeline_cache(&vk::PipelineCacheCreateInfo::default(), None) ++ } ++ } ++ ++ /// Called after all priming (including graphics/generated helpers) succeeds, ++ /// before its raw cache is destroyed and before exit-hook registration. ++ pub(super) unsafe fn keep( ++ &self, ++ raw: &ash::Device, ++ word: &crate::loader_word::LoaderWord, ++ cache: vk::PipelineCache, ++ ) { ++ // SAFETY: priming owns the live device/cache and every making has returned. ++ let Ok(data) = (unsafe { ++ word.restore(); ++ raw.get_pipeline_cache_data(cache) ++ }) else { ++ return; ++ }; ++ if self.loaded.as_deref() == Some(data.as_slice()) { ++ return; ++ } ++ if self.identity.check_vulkan_header(&data).is_err() { ++ return; ++ } ++ if self ++ .store ++ .write_record(&self.name, &driver::seal(&self.identity, &data)) ++ .is_err() ++ { ++ self.store.write_failed(); ++ } ++ } ++} ++ + /// One device's shader cache: its memo, its view of the directory, what names its makings' + /// records, and the cache thread. + pub(super) struct ShaderCache { diff --git a/bridge/patches/0004-fold-constant-global-field-loads.patch b/bridge/patches/0004-fold-constant-global-field-loads.patch new file mode 100644 index 0000000..5a79314 --- /dev/null +++ b/bridge/patches/0004-fold-constant-global-field-loads.patch @@ -0,0 +1,205 @@ +diff --git a/crates/metal2vulkan-air/src/canon/const_loads.rs b/crates/metal2vulkan-air/src/canon/const_loads.rs +index 7b3dfd7..98976df 100644 +--- a/crates/metal2vulkan-air/src/canon/const_loads.rs ++++ b/crates/metal2vulkan-air/src/canon/const_loads.rs +@@ -1,4 +1,4 @@ +-//! Pointers loaded from fields of constant globals. ++//! Values loaded from fields of constant globals. + //! + //! A constant global whose initializer is known once function constants are bound and the static + //! initializers have run may hold pointers: a program-scope `constexpr sampler` taken by value is +@@ -9,18 +9,28 @@ + //! constant and threadgroup spaces, so such a module was refused, under the values that made the + //! select one global. + //! ++//! The same holds for every other first-class field. An uber-shader takes its whole configuration ++//! as one struct of function constants that a static initializer copies into an internal ++//! `constant` global (QuartzCore's `fc_fragment_shader_config`, read field by field after a ++//! function-constant `select` between it and a buffer). SCCP folds that select to the global but ++//! not the field loads, so every branch the configuration decides survived: QuartzCore's ++//! `fixed_frag` became about 46000 SPIR-V instructions where its specialized form is a few ++//! hundred, and the driver compiled each such pipeline for about a second. Metal's compiler ++//! specializes these reads away, so a module that reads them is translated as Metal compiles it. ++//! + //! This pass reads such a load as the initializer's constant at the field: a non-volatile, +-//! non-atomic load of a pointer whose address is a field of a constant global at constant indices +-//! (a `getelementptr` instruction or constant expression over the global's own value type, its +-//! first index 0), where the field's type is the load's. A copy LLVM made through an integer (a +-//! load of the field as an integer of the pointer's size, then `inttoptr` to the field's type) +-//! reads the same constant at its `inttoptr`. The global is never written (it is `constant`, and +-//! initialized by no one else), so every run reads that constant: the address of the sampler +-//! state, which the sampler family reads as the sampler it names. Only pointers are read this +-//! way: a load of any other type from a constant global is lowered from the global's form (its +-//! typed private copy, or the constant blob), and reading it here would change those modules +-//! without changing what they compute. SCCP then runs again over a body this pass changed, so +-//! what the constants decide (an aggregate of them, a phi of them, a member of either) folds too. ++//! non-atomic load of a scalar, vector or pointer whose address is a field of a constant global at ++//! constant indices (a `getelementptr` instruction or constant expression over the global's own ++//! value type, its first index 0), where the field's type is the load's. A copy LLVM made through ++//! an integer (a load of a pointer field as an integer of the pointer's size, then `inttoptr` to ++//! the field's type) reads the same constant at its `inttoptr`. The global is never written (it ++//! is `constant`, and initialized by no one else), so every run reads that constant: for a ++//! pointer, the address of the sampler state, which the sampler family reads as the sampler it ++//! names. An aggregate load, a load of another type than the field's, and a field the initializer ++//! leaves `undef` (an unsupplied constant's) are left to the global's lowered form. SCCP then runs ++//! again over a body this pass changed, so what the constants decide (a branch, an aggregate of ++//! them, a phi of them, a member of either) folds too, and the two alternate until a round folds ++//! nothing: a decided branch can expose a further select of the global. + + use crate::budget::Meter; + use crate::error::Error; +@@ -29,9 +39,9 @@ use crate::ir::{InstId, InstKind, ModuleParts, Type, TypeId, Value}; + + use super::rewrite::{apply, scoped, BodyEdit}; + +-/// Read every load of a pointer from a field of a constant global in `body`, and every +-/// `inttoptr` of such a field's integer copy, as the field's constant (module docs), and say +-/// whether there was one. Returns `body` itself when there is none. ++/// Read every first-class load from a field of a constant global in `body`, and every `inttoptr` ++/// of a pointer field's integer copy, as the field's constant (module docs), and say whether there ++/// was one. Returns `body` itself when there is none. + pub(super) fn fold( + body: Body, + parts: &mut ModuleParts, +@@ -61,8 +71,8 @@ pub(super) fn fold( + } + } + +-/// Each instruction of `body` that reads a pointer field of a constant global (module docs), with +-/// the field's constant. ++/// Each instruction of `body` that reads a first-class field of a constant global (module docs), ++/// with the field's constant. + fn find( + body: &Body, + parts: &mut ModuleParts, +@@ -71,24 +81,27 @@ fn find( + ) -> Result<(), Error> { + for (id, inst) in body.insts.iter() { + m.tick(1)?; +- let Some(space) = parts.types.ptr_space(inst.ty) else { +- continue; +- }; +- if !matches!(parts.types.get(inst.ty), Some(Type::Ptr { .. })) { +- continue; +- } ++ let first_class = parts ++ .types ++ .get(inst.ty) ++ .is_some_and(|t| t.is_sized() && !t.is_aggregate()); ++ let pointer = matches!(parts.types.get(inst.ty), Some(Type::Ptr { .. })); + let ptr = match inst.kind { + InstKind::Load { + ptr, + volatile: false, + atomic: None, + .. +- } => ptr, ++ } if first_class => ptr, + // `inttoptr` of an integer load of the pointer's size: a copy through an integer. + InstKind::Cast { + op: CastOp::IntToPtr, + v: Value::Inst(l), +- } => match body.insts.get(l).map(|l| (&l.kind, l.ty)) { ++ } if pointer => { ++ let Some(space) = parts.types.ptr_space(inst.ty) else { ++ continue; ++ }; ++ match body.insts.get(l).map(|l| (&l.kind, l.ty)) { + Some(( + &InstKind::Load { + ptr, +@@ -104,7 +117,8 @@ fn find( + ptr + } + _ => continue, +- }, ++ } ++ } + _ => continue, + }; + let Some((g, path)) = field_address(body, parts, ptr) else { +diff --git a/crates/metal2vulkan-air/src/canon/mod.rs b/crates/metal2vulkan-air/src/canon/mod.rs +index bd05d94..e3bd590 100644 +--- a/crates/metal2vulkan-air/src/canon/mod.rs ++++ b/crates/metal2vulkan-air/src/canon/mod.rs +@@ -567,11 +567,17 @@ pub fn specialize_dynamic( + }; + body.release(m); + let body = solved; +- // A pointer loaded from a field of a constant global is the field's constant: a `constexpr +- // sampler` that SCCP's folded select picked is its sampler state. SCCP runs again over a body +- // that changed, to fold what those constants decide. +- let (body, folded) = const_loads::fold(body, &mut parts, m)?; +- let body = if folded { ++ // A value loaded from a field of a constant global is the field's constant: a `constexpr ++ // sampler` that SCCP's folded select picked is its sampler state, and an uber-shader's ++ // configuration struct of function constants is its members. SCCP runs again over a body that ++ // changed, to fold what those constants decide; the two alternate until a round folds nothing. ++ let mut body = body; ++ loop { ++ let (next, folded) = const_loads::fold(body, &mut parts, m)?; ++ if !folded { ++ body = next; ++ break; ++ } + let solved = { + let mut cx = sccp::Context { + folder: fold::Folder { +@@ -583,13 +589,11 @@ pub fn specialize_dynamic( + fcs: &fcs, + name: name.clone(), + }; +- sccp::run(&body, &mut cx, m)? ++ sccp::run(&next, &mut cx, m)? + }; +- body.release(m); +- solved +- } else { +- body +- }; ++ next.release(m); ++ body = solved; ++ } + matmul2d::refuse_left(&body, &parts, &ep.stage, &name, m)?; + post_check(&body, &parts, &stubs, &name, &control_points(ep), m)?; + // Rasterization-rate maps: each `float2` map call becomes Apple's GPU decoder of the parameter +diff --git a/crates/metal2vulkan-air/src/canon/tests.rs b/crates/metal2vulkan-air/src/canon/tests.rs +index fb0dad5..93edc0a 100644 +--- a/crates/metal2vulkan-air/src/canon/tests.rs ++++ b/crates/metal2vulkan-air/src/canon/tests.rs +@@ -1658,7 +1658,18 @@ fn a_deeply_nested_written_global_is_rebuilt_on_a_small_stack() { + static_init(&mut m, "init", |f| store(f, five, gaddr)); + let fty = m.fn_ty(void, &[dev]); + let mut f = m.define("k", fty); +- let v = load(&mut f, i32t, gaddr); ++ // A volatile read keeps the global: a plain one now reads the rebuilt initializer's leaf ++ // as its constant, and the unused global is pruned. ++ let v = f.push( ++ i32t, ++ InstKind::Load { ++ ptr: gaddr, ++ align: Align::from_bytes(4), ++ volatile: true, ++ atomic: None, ++ range: None, ++ }, ++ ); + store(&mut f, v, arg(0)); + f.ret(None); + let k = f.finish(); +@@ -2359,9 +2370,12 @@ fn a_pointer_loaded_from_a_constant_globals_field_is_the_fields_constant() { + assert!(text.contains(kept) && !text.contains(gone), "{text}"); + } + } +- // A field of any other type is left to the memory plan: its load stays. +- let text = body_text(&picked_field(Some(1), 1).unwrap()); +- assert!(text.contains("getelementptr"), "{text}"); ++ // A field of any other first-class type reads the same way: the picked struct's own `i64`. ++ for (flag, kept, gone) in [(1, "333", "444"), (0, "444", "333")] { ++ let text = body_text(&picked_field(Some(flag), 1).unwrap()); ++ assert!(!text.contains("getelementptr") && !text.contains("load"), "{text}"); ++ assert!(text.contains(kept) && !text.contains(gone), "{text}"); ++ } + // Unsupplied, the select stays undecided, as before. + assert_eq!( + picked_field(None, 0).unwrap_err(), diff --git a/bridge/pipeline_timing_host.mm b/bridge/pipeline_timing_host.mm new file mode 100644 index 0000000..f010fd4 --- /dev/null +++ b/bridge/pipeline_timing_host.mm @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: MIT +// Host instrument: how long the target's own QuartzCore shaders take through the native frontend (metal2vulkan, then the Vulkan driver given as +// loader: the Adreno Turnip over the host simulation, or MoltenVK). Every _vert_

/ _frag_

pair becomes a BGRA8 render pipeline +// and every kernel a compute pipeline, each timed; with METAL2VULKAN_LOG_PIPELINES=1 the runtime also prints the VkPipeline creation time of each, +// which separates the driver's compile from the translation. The host CPU is not the tablet's: compare the split, not the absolute numbers. +// argv: loader, shader cache dir, QuartzCore default.metallib, [limit], [draw]. With draw=1 each render state made is drawn once (three +// vertices into a 64x64 BGRA8 target, a zeroed buffer at every vertex and fragment index 0..7), which makes its VkPipeline: the runtime +// makes it at the draw, so without a draw the creation time is the translation alone. +#import +#import +#include +#include +#include +#include +#include +#include +#include "NativeBridgeMetalFrontend.h" + +extern "C" size_t native_bridge_metal_telemetry(char *, size_t); + +static double nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1e6;} +static void missing(const char *c, const char *s) {std::fprintf(stderr, "missing %s %s\n", c, s);} + +int main(int argc, char **argv) { + if (argc < 4) {std::fprintf(stderr, "usage: loader cache metallib [limit]\n"); return 2;} + @autoreleasepool { + const unsigned limit = argc > 4 ? (unsigned)std::atoi(argv[4]) : 0; + const bool draw = argc > 5 && std::atoi(argv[5]) == 1; + double t0 = nowMs(); + id device = native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed\n"); return 1;} + std::printf("open %.1f ms\n", nowMs() - t0); + // draw=2: the plugin's on-device benchmark (native_bridge_metal_benchmark) with the same pair count, for the host side of the comparison. + if (argc > 5 && std::atoi(argv[5]) == 2) { + static char rows[512]; + native_bridge_metal_benchmark(device, argv[3], limit ? limit : 8, rows, sizeof rows); + std::printf("%s", rows); + return 0; + } + NSError *error = nil; + t0 = nowMs(); + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:@(argv[3])] error:&error]; + if (!qc) {std::printf("library failed: %s\n", error.localizedDescription.UTF8String); return 1;} + NSArray *names = qc.functionNames; + std::printf("library %.1f ms, %lu functions\n", nowMs() - t0, (unsigned long)names.count); + std::vector function, pipeline, drawn; + id queue = [device newCommandQueue]; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:64 height:64 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id target = [device newTextureWithDescriptor:td]; + id zeros = [device newBufferWithLength:65536 options:MTLResourceStorageModeShared]; + unsigned made = 0, refused = 0, n = 0; + for (NSString *name in names) { + if (limit && n >= limit) break; + double f0 = nowMs(); + id f = [qc newFunctionWithName:name]; + double f1 = nowMs(); + function.push_back(f1 - f0); + if (f.functionType == MTLFunctionTypeKernel) { + ++n; + double p0 = nowMs(); + id state = [device newComputePipelineStateWithFunction:f error:&error]; + double p1 = nowMs(); + pipeline.push_back(p1 - p0); + std::printf("K %-60s fn=%7.1f pipe=%8.1f %s\n", name.UTF8String, f1 - f0, p1 - p0, state ? "ok" : error.localizedDescription.UTF8String); + state ? ++made : ++refused; + } else if (f.functionType == MTLFunctionTypeVertex) { + NSString *fragmentName = [name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if ([fragmentName isEqualToString:name] || ![names containsObject:fragmentName]) continue; + ++n; + double g0 = nowMs(); + id fragment = [qc newFunctionWithName:fragmentName]; + double g1 = nowMs(); + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = f; + d.fragmentFunction = fragment; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + double p0 = nowMs(); + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + double p1 = nowMs(); + pipeline.push_back(p1 - p0); + double took = 0; + if (state && draw) { + double d0 = nowMs(); + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + id commands = [queue commandBuffer]; + id encoder = [commands renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:state]; + for (NSUInteger i = 0; i < 8; ++i) { + [encoder setVertexBuffer:zeros offset:0 atIndex:i]; + [encoder setFragmentBuffer:zeros offset:0 atIndex:i]; + } + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; + [encoder endEncoding]; + [commands commit]; + [commands waitUntilCompleted]; + took = nowMs() - d0; + drawn.push_back(took); + } + std::printf("R %-60s fn=%7.1f+%7.1f pipe=%8.1f draw=%8.1f %s\n", name.UTF8String, f1 - f0, g1 - g0, p1 - p0, took, + state ? "ok" : error.localizedDescription.UTF8String); + state ? ++made : ++refused; + } + std::fflush(stdout); + } + std::sort(pipeline.begin(), pipeline.end()); + double total = 0; + for (double v : pipeline) total += v; + double ftotal = 0; + for (double v : function) ftotal += v; + if (!pipeline.empty()) + std::printf("SUMMARY pipelines=%zu made=%u refused=%u total=%.0f ms median=%.1f ms p90=%.1f ms max=%.1f ms | functions=%zu total=%.0f ms\n", + pipeline.size(), made, refused, total, pipeline[pipeline.size() / 2], pipeline[pipeline.size() * 9 / 10], pipeline.back(), + function.size(), ftotal); + if (!drawn.empty()) { + std::sort(drawn.begin(), drawn.end()); + double dt = 0; + for (double v : drawn) dt += v; + static char table[8192]; + native_bridge_metal_telemetry(table, sizeof table); + for (char *line = std::strtok(table, "\n"); line; line = std::strtok(nullptr, "\n")) + if (line[0] == 'V' || line[0] == 'W' || line[0] == 'S') std::printf("TELEMETRY %s\n", line); + std::printf("DRAWS n=%zu total=%.0f ms median=%.1f ms max=%.1f ms\n", drawn.size(), dt, drawn[drawn.size() / 2], drawn.back()); + } + } + return 0; +} diff --git a/bridge/plugin_sequence_host.mm b/bridge/plugin_sequence_host.mm new file mode 100644 index 0000000..6dafabe --- /dev/null +++ b/bridge/plugin_sequence_host.mm @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +// The plugin's bridge phase (NativeMetalBridgePhase.h) in order over the host simulation: the bridge self-test, then the native frontend on a +// second device over the same Turnip and the frontend checks. The software CP executes nothing, so results are wrong by design; this +// proves every step builds, submits and retires over the host kernel stack and that the second device opens after the first is gone. +// IOSurface steps are skipped (the simulation's IOKit shim hides that service). Prints "frontend failed first <..> last ". +#include "bridge/FrontendChecks.h" +#include "bridge/NativeMetalBridge.h" +#include "bridge/NativeMetalObjects.h" +#include "NativeBridgeMetalFrontend.h" +static void mark(uint32_t v) {std::printf("mark %x\n", v); + std::fflush(stdout);} +static void missing(const char *c, const char *s) {std::printf("missing %s %s\n", c, s);} +int main(int, char **argv) { + NmbSelftest r{}; + uint32_t code = nmb_selftest(argv[1], argv[2], nullptr, 0, mark, &r); + std::printf("selftest code %x stage %x msg %s\n", code, r.stage, r.message); + id device = native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed: %s\n", nmb_last_error()); return 1;} + size_t n = 0; + const void *a = nmb_fixture_air(&n); + id lib = native_bridge_metal_library_with_air(a, n); + a = nmb_function_constant_air(&n); + id fc = native_bridge_metal_library_with_air(a, n); + frontendchecks::Report rep = {}; + rep.mark = mark; + rep.hostSimulation = true; + frontendchecks::run(device, lib, fc, nil, &rep); + std::printf("frontend failed %u first %s last %x\n", rep.failed, rep.first, rep.last); + return 0; +} diff --git a/bridge/present_host.mm b/bridge/present_host.mm new file mode 100644 index 0000000..23a91ef --- /dev/null +++ b/bridge/present_host.mm @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +// Host instrument for drivers/NativeMetal/GpuScanoutPresenter.h: anonymous pages stand for the scanout and for the compositor's surface, +// the GPU copy runs on the native frontend's device (bridge over MoltenVK) and every pixel of the scanout is compared with the surface. +// With NMB_SYSTEM_DEVICE=1 the Mac's own device runs the same code as the baseline. This is a host check of the copy's contract, not an +// Adreno run and not the kernel presenter. +#import +#import +#include +#include +#include +#include "../GpuScanoutPresenter.h" +#include "../NativeBridgeMetalFrontend.h" + +static void *pages(size_t bytes) { + void *p = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + return p == MAP_FAILED ? nullptr : p; +} +static size_t roundUp(size_t n) {const size_t page = (size_t)getpagesize(); return (n + page - 1) & ~(page - 1);} + +// One frame at a given surface pitch and scanout pitch; every pixel of the scanout must equal the surface's pixel, padding must stay untouched. +static bool run(id device, uint32_t w, uint32_t h, size_t srcStride, size_t dstStride, uint32_t seed, const char *name) { + const size_t srcBytes = roundUp(srcStride * h), dstBytes = roundUp(dstStride * h); + uint8_t *src = (uint8_t *)pages(srcBytes), *dst = (uint8_t *)pages(dstBytes); + if (!src || !dst) {std::printf("FAIL %s pages\n", name); return false;} + for (size_t i = 0; i < srcBytes; ++i) src[i] = (uint8_t)((i * 131 + seed) ^ (i >> 7)); + for (size_t i = 0; i < dstBytes; ++i) dst[i] = 0xa5; + gpupresent::Presenter presenter(device, {dst, dstBytes, dstStride, w, h}); + if (!presenter.ready()) {std::printf("FAIL %s scanout import\n", name); return false;} + const gpupresent::Result r = presenter.present({src, srcBytes, srcStride, w, h}); + if (r != gpupresent::kOk) {std::printf("FAIL %s present %d\n", name, (int)r); return false;} + for (uint32_t y = 0; y < h; ++y) { + if (memcmp(dst + y * dstStride, src + y * srcStride, (size_t)w * 4)) {std::printf("FAIL %s row %u differs\n", name, y); return false;} + for (size_t x = (size_t)w * 4; x < dstStride; ++x) + if (dst[y * dstStride + x] != 0xa5) {std::printf("FAIL %s row %u padding written at %zu\n", name, y, x); return false;} + } + std::printf("ok %s %ux%u pitch %zu->%zu\n", name, w, h, srcStride, dstStride); + munmap(src, srcBytes); munmap(dst, dstBytes); + return true; +} + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = getenv("NMB_SYSTEM_DEVICE") ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + bool ok = true; + ok &= run(device, 256, 64, 256 * 4, 256 * 4, 1, "equal pitch"); + ok &= run(device, 250, 61, 250 * 4 + 8, 256 * 4, 2, "surface pitch smaller"); + ok &= run(device, 250, 61, 256 * 4, 250 * 4 + 40, 3, "scanout pitch smaller"); + ok &= run(device, 254, 33, 254 * 4 + 8, 254 * 4 + 64, 4, "unaligned pitches"); + if (!ok) return 1; + std::printf("PASS: the GPU copied every pixel of four frames into the scanout pages, padding untouched\n"); + } + return 0; +} diff --git a/bridge/priming_cache_host.cpp b/bridge/priming_cache_host.cpp new file mode 100644 index 0000000..cbf9332 --- /dev/null +++ b/bridge/priming_cache_host.cpp @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* Device open/close only. Actual Turnip compiler; mandatory software peer. + * No authored command, shader execution, timing or hardware claim. */ +#include "NativeMetalObjects.h" +#include +#include +#include +#include +#include +#include +#include +#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL priming line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while(0) +int main(int argc,char **argv) { + if(argc!=4)return 2; + const bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),uuid=!std::strcmp(argv[3],"uuid-miss"),cold=!std::strcmp(argv[3],"cold-miss"); + REQUIRE(produce||consume||uuid||cold); +#if TARGET_OS_IPHONE + REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE")); +#endif + REQUIRE(!setenv("NMB_TEST_REAL_VULKAN_LOADER",argv[1],1) && !setenv("MESA_SHADER_CACHE_DISABLE","true",1) && !setenv("NMB_TEST_FORCE_PRIME_STRICT",produce?"0":"1",1)); + const char *path=getenv("NMB_TEST_PRIMING_CACHE_LOADER");REQUIRE(path); + void *loader=dlopen(path,RTLD_NOW|RTLD_LOCAL);REQUIRE(loader); + auto control=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_control")); + auto graphics=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_counts")); + auto compute=reinterpret_cast(dlsym(loader,"nmb_test_prime_compute_counts")); + auto live=reinterpret_cast(dlsym(RTLD_DEFAULT,"host_gpu_live_objects")); + auto submitted=reinterpret_cast(dlsym(RTLD_DEFAULT,"host_gpu_submissions")); + REQUIRE(control&&graphics&&compute&&live&&submitted);control(false,uuid); + NmbDevice *device=nmb_device_open(path,argv[2]);uint64_t c[5]={},g[10]={};compute(c);graphics(g); + if(cold||uuid){REQUIRE(!device && c[2]>0);if(uuid)REQUIRE(g[5]>0);} + else { + REQUIRE(device && c[0] && c[1]==c[0] && !c[2] && c[3]==c[0] && g[0] && g[1]==g[0] && !g[2] && g[3]==g[0]); + if(consume)REQUIRE(c[4]==c[0] && g[4]==g[0]); + nmb_device_release(device); + } + REQUIRE(live()==0); + std::printf("PASS: device priming %s pid=%d compute=%llu/%llu/%llu/%llu/%llu graphics=%llu/%llu/%llu/%llu/%llu uuid_mutations=%llu submissions=%u live=0; software peer, no shader execution\n",argv[3],getpid(), + (unsigned long long)c[0],(unsigned long long)c[1],(unsigned long long)c[2],(unsigned long long)c[3],(unsigned long long)c[4], + (unsigned long long)g[0],(unsigned long long)g[1],(unsigned long long)g[2],(unsigned long long)g[3],(unsigned long long)g[4],(unsigned long long)g[5],submitted()); +} diff --git a/bridge/priming_cache_ios.cpp b/bridge/priming_cache_ios.cpp new file mode 100644 index 0000000..ef857af --- /dev/null +++ b/bridge/priming_cache_ios.cpp @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: MIT */ +/* Owned offline QEMU harness for the real iOS device-priming cache. Shared + * helpers retain exact RAM-volume identity, readonly view and bounded capture. + * The optional first-consumer marker is part of the hashed initial test image. */ +#define main unused_ui_cold_suite_main +#include "quartzcore_cache_ios.cpp" +#undef main + +int main(int argc,char **argv) { + (void)argv;prefix="/private/var/tmp/priming";workerLog="/private/var/tmp/priming-worker.log"; + bool first=false; + if(FILE *file=fopen("/private/var/tmp/priming-first-consumer","rb")) { + char marker[32]={};const auto n=fread(marker,1,sizeof marker,file);fclose(file); + if(n!=strlen("priming-first-v1\n") || memcmp(marker,"priming-first-v1\n",n))stop(96); + first=true; + } else if(errno!=ENOENT)stop(97); + result[0]=first?0x5052494d494f5332ull:0x5052494d494f5331ull; + result[2]=getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE); + if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1); + mountCompilerView(); + if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") || + setenv("NMB_TEST_PRIMING_CACHE_LOADER","/System/Library/Extensions/OfflineTurnipCompiler/libOfflinePrimingCacheControl.dylib",1))stop(2); + const std::string cache=std::string(prefix)+".cache",empty=std::string(prefix)+".empty",copy=std::string(prefix)+".archive"; + std::array initial={}; + if(first){summarize();memcpy(initial.data(),snapshot,initial.size());} + else {struct stat old={};if(!lstat(copy.c_str(),&old) || errno!=ENOENT)stop(3);} + if(unlink(workerLog) && errno!=ENOENT)stop(5); + posix_spawn_file_actions_t actions; + if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) || + posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4); + const char *coldModes[]={"cold-miss","produce","consume","uuid-miss"},*firstModes[]={"consume","uuid-miss","cold-miss"}; + const unsigned cases=first?3:4; + for(unsigned i=0;i("compiler-worker"),const_cast(driverPath),const_cast(directory.c_str()),const_cast(mode),nullptr}; + pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ); + result[12+3*i]=child;result[13+3*i]=error;if(error || child<=1)stop(10+i); + int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i); + result[14+3*i]=status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1; + } + posix_spawn_file_actions_destroy(&actions); + // A diagnostic duplicate of the sole actual driver record, not a Metal + // binary archive. The full directory is independently hashed by summarize. + if(!first) { + std::error_code ec;std::filesystem::path record;unsigned files=0; + std::filesystem::recursive_directory_iterator iterator(cache,ec),end;if(ec)stop(98); + for(;iterator!=end;iterator.increment(ec)) { + if(ec)stop(99);const auto status=iterator->symlink_status(ec); + if(ec || std::filesystem::is_symlink(status))stop(100); + if(std::filesystem::is_directory(status))continue; + if(!std::filesystem::is_regular_file(status))stop(101); + record=iterator->path();++files; + } + if(ec || files!=1 || !std::filesystem::copy_file(record,copy,std::filesystem::copy_options::none,ec) || ec)stop(102); + } + summarize(); + if(first) { + if(memcmp(initial.data(),snapshot,initial.size()))stop(103); + memcpy(snapshot+256,initial.data(),initial.size());result[6]=384;result[45]=2;result[46]=1; + } + stop(0); +} diff --git a/bridge/priming_cache_loader.cpp b/bridge/priming_cache_loader.cpp new file mode 100644 index 0000000..405990c --- /dev/null +++ b/bridge/priming_cache_loader.cpp @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +/* Offline only. Preserve the existing graphics/UUID counters and forward real + * calls, additionally forbid/observe compute and graphics during device open. */ +#define vkGetInstanceProcAddr baselineGetInstanceProcAddr +#define vkGetDeviceProcAddr baselineGetDeviceProcAddr +#include "graphics_cache_loader.cpp" +#undef vkGetInstanceProcAddr +#undef vkGetDeviceProcAddr + +static std::atomic computeCounts[5]; // calls, successes, required, valid, app hits +static bool forceCompilationForbidden() { + const char *value=std::getenv("NMB_TEST_FORCE_PRIME_STRICT");return value && !std::strcmp(value,"1"); +} +static bool hasFlags2(const void *next) { + for(auto p=static_cast(next);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR)return true; + return false; +} +static VKAPI_ATTR VkResult VKAPI_CALL primeGraphics(VkDevice device,VkPipelineCache cache,uint32_t count, + const VkGraphicsPipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) { + if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED; + std::vector copies(infos,infos+count); + if(forceCompilationForbidden())for(auto &info:copies) { + // Explicit refusal rather than silently missing flags promoted to pNext. + if(hasFlags2(info.pNext))return VK_ERROR_FEATURE_NOT_PRESENT; + info.flags|=VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT; + } + return createGraphics(device,cache,count,copies.data(),allocator,pipelines); +} +static VKAPI_ATTR VkResult VKAPI_CALL primeCompute(VkDevice device,VkPipelineCache cache,uint32_t count, + const VkComputePipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) { + if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED; + std::vector copies(infos,infos+count); + std::vector feedback(count); + std::vector additions(count); + std::vector observed(count); + for(uint32_t i=0;i(copies[i].pNext);p;p=p->pNext) + if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO) + observed[i]=reinterpret_cast(p)->pPipelineCreationFeedback; + if(!observed[i]) { + auto &a=additions[i];a.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO; + a.pNext=copies[i].pNext;a.pPipelineCreationFeedback=&feedback[i];copies[i].pNext=&a;observed[i]=&feedback[i]; + } + } + auto real=reinterpret_cast(realDevice.load()(device,"vkCreateComputePipelines")); + computeCounts[0]+=count;const auto result=real(device,cache,count,copies.data(),allocator,pipelines); + if(result==VK_PIPELINE_COMPILE_REQUIRED)computeCounts[2]++; + for(uint32_t i=0;iflags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) { + computeCounts[3]++; + if(observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT)computeCounts[4]++; + } + } + return result; +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) { + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(primeGraphics); + if(!std::strcmp(name,"vkCreateComputePipelines"))return reinterpret_cast(primeCompute); + return baselineGetDeviceProcAddr(device,name); +} +extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) { + const auto result=baselineGetInstanceProcAddr(instance,name); + if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast(vkGetDeviceProcAddr); + if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast(primeGraphics); + if(!std::strcmp(name,"vkCreateComputePipelines"))return reinterpret_cast(primeCompute); + return result; +} +extern "C" void nmb_test_prime_compute_counts(uint64_t out[5]) {for(unsigned i=0;i<5;++i)out[i]=computeCounts[i].load();} diff --git a/bridge/quartzcore_blur_host.mm b/bridge/quartzcore_blur_host.mm new file mode 100644 index 0000000..3049353 --- /dev/null +++ b/bridge/quartzcore_blur_host.mm @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT +// Execute unchanged target blur functions on a real host GPU. The reference +// is an authored seven-tap binomial convolution, not a deployment renderer. +#import +#import +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +constexpr unsigned width=32,height=24; +struct Vertex {float position[4],uv[2],unusedColor[4];}; +static const Vertex vertices[]={ + {{-1,-1,0,1},{0,1},{}},{{1,-1,0,1},{1,1},{}},{{-1,1,0,1},{0,0},{}}, + {{-1,1,0,1},{0,0},{}},{{1,-1,0,1},{1,1},{}},{{1,1,0,1},{1,0},{}}}; +struct alignas(16) UniformF {float offset[8][2],weight[8],divide;unsigned char padding[12];}; +struct alignas(8) UniformH {float offset[8][2];_Float16 weight[8],divide;unsigned char padding[6];}; +static_assert(sizeof(UniformF)==112 && sizeof(UniformH)==88); + +static MTLRenderPipelineDescriptor *blurDescriptor(id library,bool half) { + NSString *suffix=half?@"lph":@"lpf";auto d=[MTLRenderPipelineDescriptor new]; + d.vertexFunction=[library newFunctionWithName:[@"downsample_blur_vert_" stringByAppendingString:suffix]]; + d.fragmentFunction=[library newFunctionWithName:[@"narrow_blur_7_frag_" stringByAppendingString:suffix]]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatRGBA16Float; + auto vd=[MTLVertexDescriptor vertexDescriptor]; + const unsigned indices[]={0,1,3},offsets[]={0,16,24}; + const MTLVertexFormat formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat4}; + for(unsigned i=0;i<3;++i){vd.attributes[indices[i]].format=formats[i];vd.attributes[indices[i]].offset=offsets[i];vd.attributes[indices[i]].bufferIndex=0;} + vd.layouts[0].stride=sizeof(Vertex);d.vertexDescriptor=vd;return d; +} +static MTLRenderPipelineDescriptor *namedDescriptor(id library,NSString *vertex,NSString *fragment) { + auto d=[MTLRenderPipelineDescriptor new];d.vertexFunction=[library newFunctionWithName:vertex];d.fragmentFunction=[library newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm; + auto vd=[MTLVertexDescriptor vertexDescriptor]; + const MTLVertexFormat formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat2,MTLVertexFormatFloat4,MTLVertexFormatFloat}; + const unsigned offsets[]={0,16,24,32,48}; + for(unsigned i=0;i<5;++i){vd.attributes[i].format=formats[i];vd.attributes[i].bufferIndex=0;vd.attributes[i].offset=offsets[i];} + vd.layouts[0].stride=64;d.vertexDescriptor=vd; + if([vertex hasPrefix:@"path_blit_"]){d.colorAttachments[1].pixelFormat=MTLPixelFormatBGRA8Unorm;d.colorAttachments[2].pixelFormat=MTLPixelFormatR32Float;} + return d; +} +static bool cacheState(const char *phase) { + NmbShaderCacheStats s={};uint64_t work[10]={}; + if(!native_bridge_metal_shader_cache_stats(&s) || !native_bridge_metal_work_stats(work))return false; + std::printf("cache %s artifact_hit=%llu miss=%llu write=%llu driver_load=%llu hit=%llu miss=%llu save=%llu same=%llu vk_makings=%llu\n",phase, + (unsigned long long)s.hits,(unsigned long long)s.misses,(unsigned long long)s.writes, + (unsigned long long)s.driver_loads,(unsigned long long)s.driver_hits,(unsigned long long)s.driver_misses, + (unsigned long long)s.driver_saves,(unsigned long long)s.driver_unchanged,(unsigned long long)work[0]); + return !s.rejected && !s.read_failures && !s.write_failures && !s.driver_truncated && !s.driver_mismatched && !work[4]; +} +static bool inventory(id device,id library) { + unsigned vertex=0,fragment=0,kernels=0,ordinary=0,tile=0,pairs=0,bare=0,attributes=0,attachments=0; + for(NSString *name in library.functionNames) { + id f=[library newFunctionWithName:name];if(!f)return false; + NSError *error=nil; + if(f.functionType==MTLFunctionTypeKernel) { + ++kernels; + if([device newComputePipelineStateWithFunction:f error:&error])++ordinary; + else if([error.localizedDescription containsString:@"NeedsInput(TileAttachments)"])++tile; + else {std::printf("FAIL unclassified compute %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return false;} + } else if(f.functionType==MTLFunctionTypeFragment)++fragment; + else if(f.functionType==MTLFunctionTypeVertex) { + ++vertex;NSString *other=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if([other isEqualToString:name] || ![library.functionNames containsObject:other])continue; + ++pairs;auto d=[MTLRenderPipelineDescriptor new];d.vertexFunction=f;d.fragmentFunction=[library newFunctionWithName:other]; + d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm; + if([device newRenderPipelineStateWithDescriptor:d error:&error])++bare; + else if([error.localizedDescription containsString:@"InvalidDescriptor(VertexAttribute"])++attributes; + else if([error.localizedDescription containsString:@"InvalidDescriptor(ColorAttachment"])++attachments; + else {std::printf("FAIL unclassified render %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return false;} + } else return false; + } + std::printf("inventory functions=167 vertex=%u fragment=%u kernels=%u ordinary=%u tile_requires=%u pairs=%u bare_ok=%u missing_vertex=%u missing_attachment=%u\n", + vertex,fragment,kernels,ordinary,tile,pairs,bare,attributes,attachments); + return vertex==41 && fragment==111 && kernels==15 && ordinary==4 && tile==11 && pairs==22 && bare==14 && attributes==6 && attachments==2; +} +static std::vector<_Float16> convolution(const std::vector<_Float16> &input,bool vertical) { + // Independent integer stencil: coefficients sum to 64, clamp at the edge. + constexpr unsigned weight[]={1,6,15,20,15,6,1};std::vector<_Float16> result(input.size()); + for(unsigned y=0;y command,id state,id source, + id destination,id sampler,bool half,bool vertical) { + auto pass=[MTLRenderPassDescriptor renderPassDescriptor];pass.colorAttachments[0].texture=destination; + pass.colorAttachments[0].loadAction=MTLLoadActionDontCare;pass.colorAttachments[0].storeAction=MTLStoreActionStore; + id encoder=[command renderCommandEncoderWithDescriptor:pass];if(!encoder)return false; + [encoder setRenderPipelineState:state];[encoder setVertexBytes:vertices length:sizeof vertices atIndex:0]; + const float identity[]={1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1},texmat[]={1,1,0,0}; + [encoder setVertexBytes:identity length:sizeof identity atIndex:2];[encoder setVertexBytes:texmat length:sizeof texmat atIndex:3]; + [encoder setFragmentTexture:source atIndex:3];[encoder setFragmentSamplerState:sampler atIndex:0]; + // Two bilinear pairs exactly represent the seven authored stencil taps. + UniformF f={};UniformH h={}; + f.offset[0][vertical?1:0]=0.6f/(vertical?height:width); + f.offset[1][vertical?1:0]=(15.0f/7)/(vertical?height:width); + f.weight[0]=25.0f/64;f.weight[1]=7.0f/64; + if(half) { + memcpy(h.offset,f.offset,sizeof h.offset);h.weight[0]=(_Float16)f.weight[0];h.weight[1]=(_Float16)f.weight[1]; + const _Float16 scale=1;[encoder setFragmentBytes:&h length:sizeof h atIndex:1];[encoder setFragmentBytes:&scale length:sizeof scale atIndex:6]; + } else { + const float scale=1;[encoder setFragmentBytes:&f length:sizeof f atIndex:1];[encoder setFragmentBytes:&scale length:sizeof scale atIndex:6]; + } + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6];[encoder endEncoding];return true; +} +static bool runBlur(id device,id state,bool half) { + std::vector<_Float16> data(width*height*4); + for(unsigned y=0;y input=[device newTextureWithDescriptor:td],middle=[device newTextureWithDescriptor:td],output=[device newTextureWithDescriptor:td]; + auto sd=[MTLSamplerDescriptor new];sd.minFilter=MTLSamplerMinMagFilterLinear;sd.magFilter=MTLSamplerMinMagFilterLinear; + sd.sAddressMode=MTLSamplerAddressModeClampToEdge;sd.tAddressMode=MTLSamplerAddressModeClampToEdge; + id sampler=[device newSamplerStateWithDescriptor:sd];id queue=[device newCommandQueue]; + if(!input || !middle || !output || !sampler || !queue)return false; + [input replaceRegion:MTLRegionMake2D(0,0,width,height) mipmapLevel:0 withBytes:data.data() bytesPerRow:width*8]; + float worst=0; + for(unsigned round=0;round<6;++round){@autoreleasepool { + NmbShaderCacheStats before={},after={};uint64_t workBefore[10]={},workAfter[10]={}; + if(!native_bridge_metal_shader_cache_stats(&before) || !native_bridge_metal_work_stats(workBefore))return false; + id command=[queue commandBuffer]; + if(!encode(command,state,input,middle,sampler,half,false) || !encode(command,state,middle,output,sampler,half,true))return false; + [command commit];[command waitUntilCompleted]; + if(command.status!=MTLCommandBufferStatusCompleted){std::printf("FAIL blur command: %s\n",command.error.localizedDescription.UTF8String);return false;} + std::vector<_Float16> got(data.size());[output getBytes:got.data() bytesPerRow:width*8 fromRegion:MTLRegionMake2D(0,0,width,height) mipmapLevel:0]; + for(size_t i=0;i0.003f){std::printf("FAIL blur %s round=%u element=%zu got=%g expected=%g\n",half?"half":"float",round,i,(double)got[i],(double)expected[i]);return false;} + worst=std::max(worst,error); + } + if(!native_bridge_metal_shader_cache_stats(&after) || !native_bridge_metal_work_stats(workAfter))return false; + std::printf("round %s %u vk_makings=%llu artifact_hit=%llu miss=%llu driver_load=%llu hit=%llu miss=%llu\n",half?"half":"float",round, + (unsigned long long)(workAfter[0]-workBefore[0]),(unsigned long long)(after.hits-before.hits),(unsigned long long)(after.misses-before.misses), + (unsigned long long)(after.driver_loads-before.driver_loads),(unsigned long long)(after.driver_hits-before.driver_hits),(unsigned long long)(after.driver_misses-before.driver_misses)); + if(round && (workAfter[0]!=workBefore[0] || after.misses!=before.misses || after.driver_misses!=before.driver_misses)) { + std::puts("FAIL repeated identical blur performs new pipeline/translation work");return false; + } + }} + std::printf("blur %s rounds=6 passes=12 texels=%u max_error=%g GPU=pass\n",half?"half":"float",width*height,worst);return true; +} + +int main(int argc,char **argv) { + if(argc!=6 || (strcmp(argv[5],"produce") && strcmp(argv[5],"consume") && strcmp(argv[5],"inventory")))return 2; + if(setenv("METAL2VULKAN_LOG_PIPELINES","1",1))return 2; + const bool produce=!strcmp(argv[5],"produce"); + @autoreleasepool { + id device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2; + NSError *error=nil;id library=[device newLibraryWithFile:@(argv[3]) error:&error];if(!library)return 2; + if(library.functionNames.count!=167)return 1; + if(!strcmp(argv[5],"inventory"))return inventory(device,library)?0:1; + auto ad=[MTLBinaryArchiveDescriptor new];NSURL *url=[NSURL fileURLWithPath:@(argv[4])];if(!produce)ad.url=url; + id archive=[device newBinaryArchiveWithDescriptor:ad error:&error];if(!archive)return 1; + if(!cacheState("opened"))return 1; + if(produce) { + auto empty=blurDescriptor(library,false);empty.binaryArchives=@[archive];error=nil; + if([device newRenderPipelineStateWithDescriptor:empty options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error)return 1; + } + unsigned pairs=0; + for(NSString *name in library.functionNames) { + id function=[library newFunctionWithName:name];if(function.functionType!=MTLFunctionTypeVertex)continue; + NSString *fragment=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if([fragment isEqualToString:name] || ![library.functionNames containsObject:fragment])continue; + auto d=namedDescriptor(library,name,fragment);d.binaryArchives=@[archive]; + if(produce && ![archive addRenderPipelineFunctionsWithDescriptor:d error:&error]){ + std::printf("FAIL archive add %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return 1;} + error=nil; + if(![device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || error){ + std::printf("FAIL strict pair %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return 1;} + ++pairs; + } + if(pairs!=22)return 1; + std::printf("valid descriptor pairs=%u strict archive=pass\n",pairs); + for(bool half:{false,true}) { + auto d=blurDescriptor(library,half);d.binaryArchives=@[archive]; + if(produce && ![archive addRenderPipelineFunctionsWithDescriptor:d error:&error])return 1; + error=nil;id state=[device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error]; + if(!state){std::printf("FAIL blur pipeline: %s\n",error.localizedDescription.UTF8String);return 1;} + const auto format=d.colorAttachments[0].pixelFormat;d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm;error=nil; + if([device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error)return 1; + d.colorAttachments[0].pixelFormat=format; + if(!runBlur(device,state,half))return 1; + } + if(produce && ![archive serializeToURL:url error:&error])return 1; + if(!cacheState("finished"))return 1; + uint64_t work[10]={};if(!native_bridge_metal_work_stats(work) || !work[0])return 1; // logging must be active, never a zero-counter shortcut + std::puts("PASS: unchanged target blur functions execute repeated separable Gaussian reference cases on the host GPU"); + } +} diff --git a/bridge/quartzcore_cache.mm b/bridge/quartzcore_cache.mm new file mode 100644 index 0000000..5b24ca8 --- /dev/null +++ b/bridge/quartzcore_cache.mm @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: MIT +// Actual compiler/cache instrument. No draw is encoded. The runtime may initialize +// neutral resources over the mandatory software peer; it executes no shader. +#import +#include +#include +#include "NativeMetalObjects.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL UI cache line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while(0) +static NmbRenderPipelineDescriptor descriptor(NmbLibrary *library,const std::string &vertex,const std::string &fragment,bool blur) { + NmbRenderPipelineDescriptor d={};d.vertex=nmb_function_new(library,vertex.c_str());d.fragment=nmb_function_new(library,fragment.c_str()); + d.colors[0].format=blur?MTLPixelFormatRGBA16Float:MTLPixelFormatBGRA8Unorm; + if(vertex.rfind("path_blit_",0)==0){d.colors[1].format=MTLPixelFormatBGRA8Unorm;d.colors[2].format=MTLPixelFormatR32Float;} + for(auto &c:d.colors)if(c.format){c.write_mask=MTLColorWriteMaskAll;c.source_rgb=MTLBlendFactorOne;c.source_alpha=MTLBlendFactorOne;} + d.samples=1;d.rasterization=1;d.vertex_inputs=1; + if(blur) { + d.attributes[0]={MTLVertexFormatFloat4,0,0};d.attributes[1]={MTLVertexFormatFloat2,16,0};d.attributes[3]={MTLVertexFormatFloat4,24,0}; + d.layouts[0]={40,MTLVertexStepFunctionPerVertex,1}; + } else { + const uint64_t formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat2,MTLVertexFormatFloat4,MTLVertexFormatFloat}; + const uint64_t offsets[]={0,16,24,32,48};for(unsigned i=0;i<5;++i)d.attributes[i]={formats[i],offsets[i],0}; + d.layouts[0]={64,MTLVertexStepFunctionPerVertex,1}; + } + return d; +} +int main(int argc,char **argv) { + if(argc!=4)return 2; + const bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),uuid=!std::strcmp(argv[3],"uuid-miss"),empty=!std::strcmp(argv[3],"empty-driver"); + REQUIRE(produce||consume||uuid||empty); +#if TARGET_OS_IPHONE + REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE")); +#endif + REQUIRE(!setenv("MESA_SHADER_CACHE_DISABLE","true",1) && !setenv("METAL2VULKAN_LOG_PIPELINES","1",1) && !setenv("NMB_TEST_REAL_VULKAN_LOADER",argv[1],1)); + std::string path=argv[1];path=path.substr(0,path.find_last_of('/')+1)+"libOfflineGraphicsCacheControl.dylib"; + if(const char *override=getenv("NMB_TEST_GRAPHICS_CACHE_LOADER"))path=override; + void *loader=dlopen(path.c_str(),RTLD_NOW|RTLD_LOCAL);REQUIRE(loader); + auto control=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_control")); + auto counters=reinterpret_cast(dlsym(loader,"nmb_test_graphics_cache_counts")); + auto submitted=reinterpret_cast(dlsym(RTLD_DEFAULT,"host_gpu_submissions"));REQUIRE(control&&counters&&submitted); + const unsigned submissions=submitted();control(false,false); + const std::string prefix=argv[2],archivePath=prefix+".archive",cachePath=prefix+(empty?".empty":".cache"); + NmbDevice *device=nmb_device_open(path.c_str(),cachePath.c_str());REQUIRE(device); + uint64_t deviceCounts[10]={};counters(deviceCounts);control(!produce,uuid); + const char *libraryPath="/System/Library/Frameworks/QuartzCore.framework/default.metallib"; + if(const char *override=getenv("NMB_TEST_QUARTZCORE_LIBRARY"))libraryPath=override; + std::ifstream input(libraryPath,std::ios::binary);REQUIRE(input); + std::vector bytes(9670832);input.read(bytes.data(),bytes.size());REQUIRE(input.good() && input.peek()==EOF); + unsigned char digest[CC_SHA256_DIGEST_LENGTH];CC_SHA256(bytes.data(),(CC_LONG)bytes.size(),digest); + char digestHex[65]={};for(unsigned i=0;i<32;++i)std::snprintf(digestHex+2*i,3,"%02x",digest[i]); + REQUIRE(!std::strcmp(digestHex,"606f913adf520420f9b4bb666efb782a68c830f082f1e559e17b20d0d13adbc9")); + NmbLibrary *library=nmb_library_new(device,bytes.data(),bytes.size(),false);REQUIRE(library&&nmb_library_function_count(library)==167); + std::vector names;for(unsigned i=0;i<167;++i){char name[256]={};REQUIRE(nmb_library_function_name(library,i,name,sizeof name) recipes; + for(const auto &name:names) { + const auto marker=name.find("_vert_");if(marker==std::string::npos)continue; + auto fragment=name;fragment.replace(marker,6,"_frag_");if(std::find(names.begin(),names.end(),fragment)==names.end())continue; + recipes.push_back(descriptor(library,name,fragment,false)); + } + REQUIRE(recipes.size()==22); + for(const auto *suffix:{"lpf","lph"})recipes.push_back(descriptor(library,std::string("downsample_blur_vert_")+suffix,std::string("narrow_blur_7_frag_")+suffix,true)); + for(const auto &d:recipes)REQUIRE(d.vertex&&d.fragment&&nmb_function_type(d.vertex)==1&&nmb_function_type(d.fragment)==2); + NmbBinaryArchive *archive=nmb_binary_archive_new(device,produce?nullptr:archivePath.c_str());REQUIRE(archive); + NmbPipelineOptions options={MTLPipelineOptionFailOnBinaryArchiveMiss,&archive,1}; + uint64_t coldRequired=0;bool coldObject=false; + if(produce) { + // Exercise the real driver's compile-forbidden cold refusal, separately + // from strict binary-archive misses. Clear only the test policy afterwards. + control(true,false);auto cold=nmb_render_pipeline_new(device,&recipes.back()); + coldObject=cold!=nullptr;if(cold)nmb_render_pipeline_release(cold); + uint64_t counts[10]={};counters(counts);coldRequired=counts[2];REQUIRE(coldRequired>0);control(false,false); + } + uint64_t before[10]={};counters(before);unsigned made=0; + for(const auto &d:recipes) { + if(produce)REQUIRE(nmb_binary_archive_add_render(archive,&d)); + NmbRenderPipeline *state=nmb_render_pipeline_new_with_options(device,&d,&options); + if(uuid||empty){if(state)nmb_render_pipeline_release(state);break;} + REQUIRE(state);nmb_render_pipeline_release(state);++made; + } + uint64_t counts[10]={};counters(counts); + std::fprintf(stderr,"graphics results %s: calls=%llu success=%llu required=%llu valid=%llu hits=%llu cached=%llu/%llu/%llu\n",argv[3], + (unsigned long long)(counts[0]-before[0]),(unsigned long long)(counts[1]-before[1]),(unsigned long long)(counts[2]-before[2]), + (unsigned long long)(counts[3]-before[3]),(unsigned long long)(counts[4]-before[4]),(unsigned long long)(counts[7]-before[7]), + (unsigned long long)(counts[8]-before[8]),(unsigned long long)(counts[9]-before[9])); + if(uuid||empty) {REQUIRE(counts[2]>before[2]);if(uuid)REQUIRE(counts[5]>before[5]);} + else REQUIRE(made==24 && counts[1]>before[1]); + if(consume)REQUIRE(counts[2]==before[2] && counts[7]>before[7] && counts[8]-before[8]==counts[7]-before[7] && counts[9]-before[9]==counts[7]-before[7]); + if(produce)REQUIRE(nmb_binary_archive_serialize(archive,archivePath.c_str())); + NmbShaderCacheStats stats={};REQUIRE(nmb_device_shader_cache_stats(device,&stats)); + REQUIRE(!stats.rejected&&!stats.read_failures&&!stats.write_failures&&!stats.driver_truncated&&!stats.driver_mismatched); + if(consume)REQUIRE(!stats.misses&&!stats.writes&&!stats.driver_misses&&stats.driver_loads); + const unsigned initializationSubmissions=submitted()-submissions; + std::printf("PASS: QuartzCore compiler %s pid=%d recipes=%u calls=%llu success=%llu required=%llu valid=%llu hits=%llu uuid_mutations=%llu devices=%llu artifacts=%llu/%llu/%llu driver=%llu/%llu/%llu/%llu cached_calls=%llu/%llu/%llu device_graphics=%llu cold_required=%llu cold_object=%u init_submissions=%u; no shader execution\n",argv[3],getpid(),made, + (unsigned long long)(counts[0]-before[0]),(unsigned long long)(counts[1]-before[1]),(unsigned long long)(counts[2]-before[2]), + (unsigned long long)(counts[3]-before[3]),(unsigned long long)(counts[4]-before[4]),(unsigned long long)(counts[5]-before[5]),(unsigned long long)counts[6], + (unsigned long long)stats.hits,(unsigned long long)stats.misses,(unsigned long long)stats.writes, + (unsigned long long)stats.driver_loads,(unsigned long long)stats.driver_hits,(unsigned long long)stats.driver_misses,(unsigned long long)stats.driver_saves,(unsigned long long)(counts[7]-before[7]), + (unsigned long long)(counts[8]-before[8]),(unsigned long long)(counts[9]-before[9]),(unsigned long long)deviceCounts[0],(unsigned long long)coldRequired,(unsigned)coldObject,initializationSubmissions); + nmb_binary_archive_release(archive);for(const auto &d:recipes){nmb_function_release(d.vertex);nmb_function_release(d.fragment);} + nmb_library_release(library);nmb_device_release(device); + return 0; +} diff --git a/bridge/quartzcore_cache_ios.cpp b/bridge/quartzcore_cache_ios.cpp new file mode 100644 index 0000000..fb1f62f --- /dev/null +++ b/bridge/quartzcore_cache_ios.cpp @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: MIT */ +/* Owned QEMU-only PID 1 harness. Run the actual iOS UI compiler in four cold + * children, flush own mutable Data, then expose a bounded manifest checkpoint. + * Original dyld/AMFI, signatures, read-only OS and exact RAM-volume selection + * remain in force. The linked software peer executes no shader. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; +static uint64_t result[48]; +alignas(4096) static unsigned char snapshot[4096]; +alignas(4096) static unsigned char workerOutput[4096]; +static const char *workerLog="/private/var/tmp/quartzcore-compiler-worker.log"; +static const char *driverPath="/System/Library/Extensions/OfflineTurnipCompiler/libvulkan_freedreno.dylib"; +static const char *prefix="/private/var/tmp/quartzcore-ui"; + +[[noreturn]] static void stop(unsigned failure) { + result[1]=failure; + if(FILE *file=fopen(workerLog,"rb")) { + result[11]=fread(workerOutput,1,sizeof workerOutput-1,file); + if(fgetc(file)!=EOF || ferror(file))result[11]=sizeof workerOutput; + fclose(file); + } + __asm__ volatile("mov x13, %0\nmov x14, %1\nmov x15, %2\nbrk #0"::"r"(result),"r"(snapshot),"r"(workerOutput):"x13","x14","x15","memory"); + _exit(99); +} +static void mountCompilerView() { + char uuid[37]={};FILE *file=fopen("/private/var/tmp/compiler-data-uuid","rb"); + if(!file)stop(60); + const size_t count=fread(uuid,1,37,file);fclose(file); + if(count!=36 || uuid[36])stop(61); + CFStringRef identity=CFStringCreateWithCString(nullptr,uuid,kCFStringEncodingASCII); + CFMutableDictionaryRef match=IOServiceMatching("IOMedia");if(!identity || !match)stop(62); + const void *keys[]={CFSTR("UUID")},*values[]={identity}; + CFDictionaryRef properties=CFDictionaryCreate(nullptr,keys,values,1,&kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks); + if(!properties)stop(63); + CFDictionarySetValue(match,CFSTR("IOPropertyMatch"),properties);CFRelease(properties); + io_iterator_t iterator=0;if(IOServiceGetMatchingServices(0,match,&iterator) || !iterator)stop(64); + char device[64]={};unsigned found=0; + while(io_service_t media=IOIteratorNext(iterator)) { + ++found; + CFTypeRef observed=IORegistryEntryCreateCFProperty(media,CFSTR("UUID"),nullptr,0); + CFTypeRef name=IORegistryEntryCreateCFProperty(media,CFSTR("BSD Name"),nullptr,0); + if(!observed || !CFEqual(observed,identity) || !name || CFGetTypeID(name)!=CFStringGetTypeID())stop(65); + char bsd[56]={};if(!CFStringGetCString((CFStringRef)name,bsd,sizeof bsd,kCFStringEncodingASCII) || strncmp(bsd,"disk",4))stop(66); + for(const char *p=bsd;*p;++p)if(!((*p>='a'&&*p<='z')||(*p>='0'&&*p<='9')))stop(67); + bool backed=false;io_registry_entry_t node=media; + for(unsigned depth=0;node && depth<64;++depth) { + if(IOObjectConformsTo(node,"NativeRamBlock")){backed=true;break;} + io_registry_entry_t above=0;auto error=IORegistryEntryGetParentEntry(node,"IOService",&above); + if(node!=media)IOObjectRelease(node);node=error?0:above; + } + if(node && node!=media)IOObjectRelease(node);if(!backed)stop(68); + snprintf(device,sizeof device,"/dev/%s",bsd);CFRelease(observed);CFRelease(name);IOObjectRelease(media); + } + IOObjectRelease(iterator);CFRelease(identity);if(found!=1 || !device[0])stop(69); + const char *point="/System/Library/Extensions",*directory="/private/var/tmp/native-compiler-addons"; + struct stat block={},source={},before={},after={},original={},code={};struct statfs fs={}; + if(stat(device,&block) || !S_ISBLK(block.st_mode) || stat(point,&before) || stat(directory,&source) || source.st_dev!=block.st_rdev)stop(70); + if(mount("bindfs",point,MNT_RDONLY|MNT_UNION,(void *)directory))stop(71); + if(statfs(point,&fs) || (fs.f_flags&(MNT_RDONLY|MNT_UNION))!=(MNT_RDONLY|MNT_UNION) || + strcmp(fs.f_mntfromname,directory) || strcmp(fs.f_fstypename,"bindfs") || stat(point,&after) || + after.st_dev==before.st_dev || stat(driverPath,&code) || code.st_dev!=after.st_dev || + stat("/System/Library/Extensions/AGXMetalA10.bundle/Info.plist",&original) || original.st_dev!=before.st_dev)stop(72); + result[38]=fs.f_flags;result[39]=(uint32_t)before.st_dev;result[40]=(uint32_t)after.st_dev; + result[41]=(uint32_t)original.st_dev;result[42]=(uint32_t)code.st_dev;result[43]=found;result[44]=1; +} +static std::array digestFile(const std::filesystem::path &path,uint64_t *length) { + struct stat st={};if(lstat(path.c_str(),&st) || !S_ISREG(st.st_mode) || st.st_size<0 || st.st_size>16*1024*1024)stop(80); + FILE *file=fopen(path.c_str(),"rb");if(!file)stop(81); + CC_SHA256_CTX hash;CC_SHA256_Init(&hash);unsigned char buffer[16384];uint64_t total=0; + for(;;){const size_t n=fread(buffer,1,sizeof buffer,file);if(n){CC_SHA256_Update(&hash,buffer,(CC_LONG)n);total+=n;}if(n digest={};CC_SHA256_Final(digest.data(),&hash);*length=total;return digest; +} +static void summarize() { + const std::filesystem::path directory=std::string(prefix)+".cache",archive=std::string(prefix)+".archive"; + std::error_code ec;std::vector names; + std::filesystem::recursive_directory_iterator iterator(directory,std::filesystem::directory_options::none,ec),end; + if(ec)stop(83); + for(;iterator!=end;iterator.increment(ec)) { + if(ec)stop(84); + const auto status=iterator->symlink_status(ec);if(ec || std::filesystem::is_symlink(status))stop(85); + if(std::filesystem::is_directory(status))continue; + if(!std::filesystem::is_regular_file(status) || names.size()>=2048)stop(86); + auto name=iterator->path().lexically_relative(directory).generic_string(); + if(name.empty() || name.size()>192 || name.find("..")!=std::string::npos)stop(87); + for(unsigned char c:name)if(c<33 || c>126)stop(88); + names.push_back(name); + } + if(ec || names.empty())stop(89);std::sort(names.begin(),names.end()); + CC_SHA256_CTX manifest;CC_SHA256_Init(&manifest);uint64_t total=0; + for(const auto &name:names) { + uint64_t size=0;const auto digest=digestFile(directory/name,&size);total+=size;if(total>64*1024*1024)stop(90); + const uint32_t length=(uint32_t)name.size();CC_SHA256_Update(&manifest,&length,sizeof length); + CC_SHA256_Update(&manifest,name.data(),length);CC_SHA256_Update(&manifest,&size,sizeof size);CC_SHA256_Update(&manifest,digest.data(),digest.size()); + } + uint64_t archiveBytes=0,sourceBytes=0;const auto archiveDigest=digestFile(archive,&archiveBytes); + const auto sourceDigest=digestFile("/System/Library/Frameworks/QuartzCore.framework/default.metallib",&sourceBytes); + auto *words=reinterpret_cast(snapshot);words[0]=0x515455495041434bull;words[1]=archiveBytes; + memcpy(snapshot+16,archiveDigest.data(),32);words[6]=names.size();words[7]=total; + CC_SHA256_Final(snapshot+64,&manifest);memcpy(snapshot+96,sourceDigest.data(),32); + result[6]=128;result[7]=sourceBytes;result[8]=names.size();result[9]=total;result[10]=archiveBytes; + int fd=open(archive.c_str(),O_RDONLY);if(fd<0 || fsync(fd))stop(91);close(fd); + sync();result[45]=1;result[47]=1; +} +int main(int argc,char **argv) { + (void)argv;result[0]=0x515443504b494f31ull;result[2]=(uint64_t)getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE); + if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1); + mountCompilerView(); + if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") || + unsetenv("NMB_TEST_GRAPHICS_CACHE_LOADER") || unsetenv("NMB_TEST_QUARTZCORE_LIBRARY"))stop(2); + // A fresh fixture must never reuse the earlier run's products. + struct stat old={};if(!lstat((std::string(prefix)+".archive").c_str(),&old) || errno!=ENOENT)stop(3); + if(unlink(workerLog) && errno!=ENOENT)stop(5); + posix_spawn_file_actions_t actions; + if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) || + posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4); + const char *modes[]={"produce","consume","uuid-miss","empty-driver"}; + for(unsigned i=0;i<4;++i) { + char *arguments[]={const_cast("compiler-worker"),const_cast(driverPath),const_cast(prefix),const_cast(modes[i]),nullptr}; + pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ); + result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;if(error || child<=1)stop(10+i); + int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i); + result[14+3*i]=(uint64_t)status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1; + } + posix_spawn_file_actions_destroy(&actions);summarize();stop(0); +} diff --git a/bridge/quartzcore_cache_provisioned_ios.cpp b/bridge/quartzcore_cache_provisioned_ios.cpp new file mode 100644 index 0000000..dffb162 --- /dev/null +++ b/bridge/quartzcore_cache_provisioned_ios.cpp @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* QEMU-only first-consumer harness. Reuse the exact volume/manifest/checkpoint + * helpers, but never call the cold producer suite. Initial Data already holds + * the actual iOS-generated archive and caches at the consumer's normal paths. */ +#define main unused_cold_suite_main +#include "quartzcore_cache_ios.cpp" +#undef main + +int main(int argc,char **argv) { + (void)argv;result[0]=0x515443504b494f32ull;result[2]=(uint64_t)getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE); + if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1); + mountCompilerView(); + if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") || + unsetenv("NMB_TEST_GRAPHICS_CACHE_LOADER") || unsetenv("NMB_TEST_QUARTZCORE_LIBRARY"))stop(2); + // Hash the actual initial-image bytes before any compiler worker is spawned. + summarize();std::array initial={};memcpy(initial.data(),snapshot,initial.size()); + if(unlink(workerLog) && errno!=ENOENT)stop(5); + posix_spawn_file_actions_t actions; + if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) || + posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4); + const char *modes[]={"consume","uuid-miss","empty-driver"}; + for(unsigned i=0;i<3;++i) { + char *arguments[]={const_cast("compiler-worker"),const_cast(driverPath),const_cast(prefix),const_cast(modes[i]),nullptr}; + pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ); + result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;if(error || child<=1)stop(10+i); + int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i); + result[14+3*i]=(uint64_t)status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1; + } + posix_spawn_file_actions_destroy(&actions);summarize(); + if(memcmp(initial.data(),snapshot,initial.size()))stop(95); + memcpy(snapshot+256,initial.data(),initial.size());result[6]=384;result[45]=2;result[46]=1;stop(0); +} diff --git a/bridge/register_host.mm b/bridge/register_host.mm new file mode 100644 index 0000000..2a2a333 --- /dev/null +++ b/bridge/register_host.mm @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +// Host check: the system's own device registration (MTLAddDevice, the call the target's discovery makes with a plugin's device) accepts the frontend's +// device. It asserts that the device conforms to Metal's device protocols and runs the base class's initLimits/initFeatureQueries/initWorkarounds on it; +// the Mac's version differs from the target's, but it caught the missing MTLDeviceSPI conformance that the target's _MTLAddDevice also asserts. +#import +#import +#include +#import +#include +#include "../NativeBridgeMetalFrontend.h" + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + void *add = dlsym(dlopen("/System/Library/Frameworks/Metal.framework/Metal", RTLD_NOW), "MTLAddDevice"); + if (!add) {std::printf("SKIP: this system has no MTLAddDevice\n"); return 0;} + id device = native_bridge_metal_open(argv[1], argv[2], 1, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + if (![(id)device conformsToProtocol:objc_getProtocol("MTLDeviceSPI")]) {std::printf("FAIL MTLDeviceSPI conformance\n"); return 1;} + ((void (*)(id))add)(device); + bool listed = false; + for (id d in MTLCopyAllDevices()) listed |= (d == device); + if (!listed) {std::printf("FAIL the registered device is not among the system's devices\n"); return 1;} + std::printf("PASS: the system's device registration accepted the frontend's device and lists it\n"); + } + return 0; +} diff --git a/bridge/resource_lifetime_host.mm b/bridge/resource_lifetime_host.mm new file mode 100644 index 0000000..b530625 --- /dev/null +++ b/bridge/resource_lifetime_host.mm @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +// Actual host GPU work waits on an unsignaled event while every caller-owned +// no-copy buffer/view reference is dropped. Its deallocator must not run yet. +#import +#import +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +struct State {std::atomic released{0};}; + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + unsigned failures = 0; + size_t airBytes = 0; + const void *air = nmb_fixture_air(&airBytes); + id library = native_bridge_metal_library_with_air(air, airBytes); + id pipeline = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"scale_and_offset"] error:nil]; + if (!pipeline) return 2; + for (unsigned variant = 0; variant < 7; ++variant) { + auto state = std::make_shared(); + const size_t page = (size_t)getpagesize(); + void *pages = mmap(nullptr, page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (pages == MAP_FAILED) return 2; + memset(pages, variant == 6 ? 0xcc : 0x5a, page); + if (variant == 5) for (unsigned i = 0; i < 256; ++i) ((float *)pages)[i] = (float)i; + id output = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id event = [device newSharedEvent]; + id commands = [[device newCommandQueue] commandBuffer]; + __weak id weakBuffer; + if (!output || !event || !commands) return 2; + @autoreleasepool { + // Deliberately keep the mapping owned by the test, so the old + // early-deallocator failure cannot make the pending GPU read + // unmapped memory. No pixel or GPU completion is synthesized. + id buffer = [device newBufferWithBytesNoCopy:pages length:page options:MTLResourceStorageModeShared + deallocator:^(void *, NSUInteger){state->released.fetch_add(1);}]; + if (!buffer) return 2; + weakBuffer = buffer; + [commands encodeWaitForEvent:event value:1]; + if (variant == 5) { + id compute = [commands computeCommandEncoder]; + [compute setComputePipelineState:pipeline]; + [compute setBuffer:buffer offset:0 atIndex:0]; + [compute setBuffer:output offset:0 atIndex:1]; + const float scale = 2; + [compute setBytes:&scale length:sizeof scale atIndex:2]; + [compute dispatchThreads:MTLSizeMake(256,1,1) threadsPerThreadgroup:MTLSizeMake(64,1,1)]; + [compute endEncoding]; + } else if (variant == 1 || variant == 6) { + auto d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + d.storageMode = MTLStorageModeShared; + d.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView | MTLTextureUsageRenderTarget; + id root = [buffer newTextureWithDescriptor:d offset:0 bytesPerRow:256]; + id view = [[root newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] + newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + if (!view) return 2; + if (variant == 6) { + auto pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = view; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(90.0/255,90.0/255,90.0/255,90.0/255); + [[commands renderCommandEncoderWithDescriptor:pass] endEncoding]; + } + if (variant == 1) { + id blit = [commands blitCommandEncoder]; + [blit copyFromTexture:view sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(16,16,1) + toBuffer:output destinationOffset:0 destinationBytesPerRow:64 destinationBytesPerImage:1024]; + [blit endEncoding]; + } + } else { + id blit = [commands blitCommandEncoder]; + if (variant == 2) [blit fillBuffer:buffer range:NSMakeRange(0, 1024) value:0x37]; + [blit copyFromBuffer:buffer sourceOffset:0 toBuffer:output destinationOffset:0 size:1024]; + id missing = nil; + if (variant == 3) [blit fillBuffer:missing range:NSMakeRange(0, 16) value:0]; // local refusal, never submit + [blit endEncoding]; + } + } + const unsigned before = state->released.load(); + if (before || !weakBuffer) ++failures; + unsigned pending = before; + if (variant != 4) { + [commands commit]; + // Completion must remain pending until the actual event is + // signaled, except a locally refused command which submitted + // no GPU work. Check with bounded dispatch waits. + auto finished = dispatch_semaphore_create(0); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + @autoreleasepool {[commands waitUntilCompleted];} + dispatch_semaphore_signal(finished); + }); + const bool early = dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, 100 * NSEC_PER_MSEC)) == 0; + pending = state->released.load(); + if (variant != 3 && (early || pending || !weakBuffer)) ++failures; + event.signaledValue = 1; + if (!early && dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC))) return 1; + if (commands.status != (variant == 3 ? MTLCommandBufferStatusError : MTLCommandBufferStatusCompleted)) ++failures; + if (variant != 3) for (unsigned i = 0; i < (variant == 5 ? 256 : 1024); ++i) { + const bool bad = variant == 5 ? ((const float *)output.contents)[i] != 3.0f*i + : variant == 6 ? ((const unsigned char *)pages)[(i / 64) * 256 + i % 64] != 0x5a + : ((const unsigned char *)output.contents)[i] != (variant == 2 ? 0x37 : 0x5a); + if (bad) {++failures;break;} + } + if (state->released.load() != 1 || weakBuffer) ++failures; // even while commands remains retained + } + commands = nil; + // The real completion callback can release its own reference just + // after waitUntilCompleted returns. Bound that scheduling margin. + for (unsigned i = 0; i < 300 && state->released.load() == 0; ++i) usleep(1000); + if (state->released.load() != 1 || weakBuffer) ++failures; + std::printf("case %u before=%u pending=%u released=%u\n", variant, before, pending, state->released.load()); + munmap(pages, page); + } + if (failures) {std::printf("FAIL: resource owner lifetime (%u checks)\n", failures);return 1;} + std::puts("PASS: no-copy owners/views survive encoding and gated GPU work, release once after completion/refusal/discard; actual GPU bytes match"); + } +} diff --git a/bridge/resource_wait_host.mm b/bridge/resource_wait_host.mm new file mode 100644 index 0000000..bd6790b --- /dev/null +++ b/bridge/resource_wait_host.mm @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: MIT +// Keep real work pending behind an unsignaled shared event. Waiting for an unrelated +// resource must return; waiting for its storage, views or IOSurface alias must not. +#import +#import +#import +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +@protocol ResourceWait +- (void)waitUntilComplete; +@end + +static dispatch_time_t seconds(unsigned n) {return dispatch_time(DISPATCH_TIME_NOW, n * NSEC_PER_SEC);} + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + for (unsigned variant = 0; variant < 9; ++variant) { + @autoreleasepool { + id output = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id input = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id query = output; + id target = nil; + IOSurfaceRef surface = nullptr; + void *pages = nullptr; + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.storageMode = MTLStorageModeShared; + td.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView; + if (variant == 0) query = input; // never used by the pending fill + if (variant == 2) { + id texture = [output newTextureWithDescriptor:td offset:0 bytesPerRow:256]; + query = [texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + } + if (variant == 3 || variant == 7) { + if (variant == 7) td.storageMode = MTLStorageModePrivate; + target = [device newTextureWithDescriptor:td]; + query = [[target newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + } + if (variant == 4) { + NSDictionary *p = @{(id)kIOSurfaceWidth:@16, (id)kIOSurfaceHeight:@16, + (id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}; + surface = IOSurfaceCreate((__bridge CFDictionaryRef)p); + target = [device newTextureWithDescriptor:td iosurface:surface plane:0]; + query = [device newTextureWithDescriptor:td iosurface:surface plane:0]; + } + if (variant == 5) { + auto hd = [MTLHeapDescriptor new];hd.size = 65536;hd.storageMode = MTLStorageModeShared; + id heap = [device newHeapWithDescriptor:hd]; + output = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + query = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + } + if (variant == 6) { + const size_t page = (size_t)getpagesize(); + pages = mmap(nullptr, page * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (pages == MAP_FAILED) return 2; + auto keep = std::shared_ptr(pages, [page](void *p){munmap(p, page * 2);}); + auto release = ^(void *, NSUInteger){(void)keep;}; + output = [device newBufferWithBytesNoCopy:pages length:page * 2 options:MTLResourceStorageModeShared deallocator:release]; + query = [device newBufferWithBytesNoCopy:(char *)pages + page length:page options:MTLResourceStorageModeShared deallocator:release]; + } + if (variant == 8) { + output = [device newBufferWithLength:4096 options:MTLResourceStorageModePrivate]; + query = output; + } + if (!output || !input || !query || ((variant == 3 || variant == 4 || variant == 7) && !target)) { + std::printf("FAIL resource allocation %u\n", variant);return 2; + } + memset(input.contents, 0x5a, 4096); + id event = [device newSharedEvent]; + id commands = [[device newCommandQueue] commandBuffer]; + [commands encodeWaitForEvent:event value:1]; + id blit = [commands blitCommandEncoder]; + if (target) [blit copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:64 sourceBytesPerImage:1024 sourceSize:MTLSizeMake(16,16,1) + toTexture:target destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; + else [blit fillBuffer:output range:NSMakeRange(0, output.length) value:0x5a]; + [blit endEncoding];[commands commit]; + auto entered = dispatch_semaphore_create(0), returned = dispatch_semaphore_create(0); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + dispatch_semaphore_signal(entered); + [(id)query waitUntilComplete]; + dispatch_semaphore_signal(returned); + }); + if (dispatch_semaphore_wait(entered, seconds(3))) return 1; + const bool early = dispatch_semaphore_wait(returned, dispatch_time(DISPATCH_TIME_NOW, 250 * NSEC_PER_MSEC)) == 0; + event.signaledValue = 1; + if (!early && dispatch_semaphore_wait(returned, seconds(3))) {std::puts("FAIL resource wait stuck");return 1;} + [commands waitUntilCompleted]; + if (commands.status != MTLCommandBufferStatusCompleted || early != (variant == 0)) { + std::printf("FAIL variant=%u early=%d status=%lu\n", variant, early, (unsigned long)commands.status);return 1; + } + if (target) { + unsigned char pixels[1024] = {}; + if (variant == 7) { + memset(input.contents, 0, 4096); + id readback = [[device newCommandQueue] commandBuffer]; + id copy = [readback blitCommandEncoder]; + [copy copyFromTexture:target sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(16,16,1) + toBuffer:input destinationOffset:0 destinationBytesPerRow:64 destinationBytesPerImage:1024]; + [copy endEncoding];[readback commit];[readback waitUntilCompleted]; + if (readback.status != MTLCommandBufferStatusCompleted) return 1; + memcpy(pixels, input.contents, sizeof pixels); + } else [target getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0,0,16,16) mipmapLevel:0]; + for (unsigned char pixel : pixels) if (pixel != 0x5a) return 1; + } else if (variant == 8) { + memset(input.contents, 0, 4096); + id readback = [[device newCommandQueue] commandBuffer]; + id copy = [readback blitCommandEncoder]; + [copy copyFromBuffer:output sourceOffset:0 toBuffer:input destinationOffset:0 size:4096]; + [copy endEncoding];[readback commit];[readback waitUntilCompleted]; + if (readback.status != MTLCommandBufferStatusCompleted) return 1; + for (NSUInteger i = 0; i < 4096; ++i) if (((const unsigned char *)input.contents)[i] != 0x5a) return 1; + } else for (NSUInteger i = 0; i < output.length; ++i) if (((const unsigned char *)output.contents)[i] != 0x5a) return 1; + // The shared deallocator owns no-copy pages until both imports release them. + query = nil;output = nil;target = nil; + if (surface) CFRelease(surface); + } + } + std::puts("PASS: unrelated resource wait returns while work is gated; storage, views, heap and IOSurface/no-copy aliases wait for real completion"); + } +} diff --git a/bridge/scheduled_host.mm b/bridge/scheduled_host.mm new file mode 100644 index 0000000..3c53a26 --- /dev/null +++ b/bridge/scheduled_host.mm @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT +// A blocked completed handler must not hold waitUntilScheduled. No hardware timing claim: +// the semaphores create a deterministic distinction between scheduling and completion. +#import +#import +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +static dispatch_time_t deadline() {return dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC);} +__attribute__((noinline)) static id nothing() {return nil;} +@protocol CompletionQueue +- (void)setCompletionQueue:(dispatch_queue_t)queue; +@end + +int main(int argc, char **argv) { + if (argc < 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + id queue = [device newCommandQueue]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + if (!queue || !buffer) return 2; + for (unsigned variant : {0u, 1u, 2u, 3u}) { + const bool refused = variant == 1 || variant == 3; + [(id)queue setCompletionQueue:variant == 2 ? dispatch_queue_create("scheduled-test", DISPATCH_QUEUE_SERIAL) : nil]; + id cb = [queue commandBuffer]; + auto entered = dispatch_semaphore_create(0), release = dispatch_semaphore_create(0); + auto returned = dispatch_semaphore_create(0); + std::atomic scheduledCount{0}, completedCount{0}; + auto scheduled = &scheduledCount, completed = &completedCount; + [cb addScheduledHandler:^(id command) { + // This also catches callbacks deferred to Completed by the frontend. + if (command.status == MTLCommandBufferStatusScheduled) ++*scheduled; + }]; + [cb addCompletedHandler:^(id) { + ++*completed; + dispatch_semaphore_signal(entered); + dispatch_semaphore_wait(release, deadline()); + }]; + id blit = [cb blitCommandEncoder]; + if (variant == 1) { + // A frontend encoding failure has no scheduling event to announce. + [blit fillBuffer:(id)nothing() range:NSMakeRange(0, 16) value:0]; + } else { + [blit fillBuffer:buffer range:NSMakeRange(0, 256) value:0x5a]; + } + if (variant != 3) [blit endEncoding]; // variant 3: runtime refuses commit with an encoder open + [cb commit]; + if (dispatch_semaphore_wait(entered, deadline())) {std::puts("FAIL completion never arrived");return 1;} + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{ + [cb waitUntilScheduled]; + dispatch_semaphore_signal(returned); + }); + const bool independent = dispatch_semaphore_wait(returned, dispatch_time(DISPATCH_TIME_NOW, 300 * NSEC_PER_MSEC)) == 0; + dispatch_semaphore_signal(release); + if (!independent && dispatch_semaphore_wait(returned, deadline())) {std::puts("FAIL scheduled wait stuck");return 1;} + [cb waitUntilCompleted]; + if (variant == 3) [blit endEncoding]; + if (!independent || completedCount != 1 || scheduledCount != (refused ? 0u : 1u) || + cb.status != (refused ? MTLCommandBufferStatusError : MTLCommandBufferStatusCompleted)) { + std::printf("FAIL variant=%u independent=%d scheduled=%u completed=%u status=%lu\n", + variant, independent, scheduledCount.load(), completedCount.load(), (unsigned long)cb.status); + return 1; + } + } + std::puts("PASS: scheduling precedes completion on runtime/serial queues; scheduled waits ignore blocked completed handlers; frontend/runtime refusals announce no scheduling"); + } +} diff --git a/bridge/selector_coverage_host.mm b/bridge/selector_coverage_host.mm new file mode 100644 index 0000000..78f6317 --- /dev/null +++ b/bridge/selector_coverage_host.mm @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +// Host instrument: which of the Metal methods the target's QuartzCore references (scripts/audit-quartzcore-metal-calls.py, JSON path as the third +// argument) the native frontend's live objects respond to. One object of each kind is made on the frontend's device (bridge over MoltenVK); a +// kind that cannot be made is reported as such. Prints every selector an object lacks. A reference proves QuartzCore can send it; it does not +// show that the system UI does. +#import +#import +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "FrontendChecks.h" +#include "../bridge/NativeMetalObjects.h" + +static id N(id object) {return object ?: (id)[NSNull null];} + +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 4) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) {std::printf("FAIL device\n"); return 1;} + NSData *json = [NSData dataWithContentsOfFile:[NSString stringWithUTF8String:argv[3]]]; + NSDictionary *all = [NSJSONSerialization JSONObjectWithData:json options:0 error:nil][@"kinds"]; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + id queue = [device newCommandQueue]; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:8 height:8 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + id texture = [device newTextureWithDescriptor:td]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = texture; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + id render = [cb renderCommandEncoderWithDescriptor:pass]; + [render endEncoding]; + id blit = [cb blitCommandEncoder]; + [blit endEncoding]; + id compute = [cb computeCommandEncoder]; + [compute endEncoding]; + id function = [library newFunctionWithName:@"scale_and_offset"]; + id computePipeline = [device newComputePipelineStateWithFunction:function error:nil]; + id renderPipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + id sampler = [device newSamplerStateWithDescriptor:[MTLSamplerDescriptor new]]; + id depth = [device newDepthStencilStateWithDescriptor:[MTLDepthStencilDescriptor new]]; + id archive = [device newBinaryArchiveWithDescriptor:[MTLBinaryArchiveDescriptor new] error:nil]; + MTLHeapDescriptor *hd = [MTLHeapDescriptor new]; + hd.size = 65536; + id heap = [device respondsToSelector:@selector(newHeapWithDescriptor:)] ? [device newHeapWithDescriptor:hd] : nil; + NSDictionary *objects = @{@"device": N(device), @"texture": N(texture), @"buffer": N(buffer), @"heap": N(heap), @"command queue": N(queue), + @"command buffer": N(cb), @"render encoder": N(render), @"blit encoder": N(blit), @"compute encoder": N(compute), + @"render pipeline": N(renderPipeline), @"compute pipeline": N(computePipeline), @"sampler": N(sampler), + @"depth stencil": N(depth), @"library": N(library), @"function": N(function), @"binary archive": N(archive)}; + unsigned total = 0; + for (NSString *kind in [all.allKeys sortedArrayUsingSelector:@selector(compare:)]) { + id object = objects[kind]; + NSArray *selectors = [all[kind][@"selectors"] allKeys]; + if (!object || object == [NSNull null]) {std::printf("%-18s cannot be created (%lu selectors referenced)\n", kind.UTF8String, (unsigned long)selectors.count); ++total; continue;} + NSMutableArray *missing = [NSMutableArray array]; + for (id entry in selectors) { + NSString *name = entry; + if ([name hasPrefix:@"."]) continue; + if (![object respondsToSelector:NSSelectorFromString(name)]) [missing addObject:name]; + } + total += missing.count; + std::printf("%-18s %lu referenced, %lu missing%s%s\n", kind.UTF8String, (unsigned long)selectors.count, (unsigned long)missing.count, + missing.count ? ": " : "", [missing componentsJoinedByString:@" "].UTF8String); + } + std::printf("%s: %u selectors the target's QuartzCore references are not answered by the frontend's objects\n", total ? "GAPS" : "PASS", total); + } + return 0; +} diff --git a/bridge/surface_flush_host.mm b/bridge/surface_flush_host.mm new file mode 100644 index 0000000..e88c932 --- /dev/null +++ b/bridge/surface_flush_host.mm @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT +// Count actual frontend cache-maintenance calls while forwarding every call +// to libSystem. GPU writes and alias reads must still reach the real pages. +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" + +static std::atomic calls, bytes; +extern "C" void sys_dcache_flush(void *address, size_t length) { + using Flush = void (*)(void *, size_t); + static Flush real = reinterpret_cast(dlsym(RTLD_NEXT, "sys_dcache_flush")); + if (!real || real == sys_dcache_flush) std::abort(); + calls++;bytes += length; + real(address, length); +} + +static bool run(id device, unsigned aliases, unsigned allocations) { + @autoreleasepool { + constexpr unsigned width = 64, height = 16, row = width * 4, frameBytes = row * height; + std::vector surfaces; + auto textures = [NSMutableArray array]; + auto descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm + width:width height:height mipmapped:NO]; + descriptor.storageMode = MTLStorageModeShared; + descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView; + for (unsigned i = 0; i < allocations; ++i) { + NSDictionary *properties = @{(id)kIOSurfaceWidth:@(width), (id)kIOSurfaceHeight:@(height), + (id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfaceBytesPerRow:@(row), + (id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}; + IOSurfaceRef surface = IOSurfaceCreate((__bridge CFDictionaryRef)properties); + if (!surface || IOSurfaceLock(surface, 0, nullptr)) return false; + memset(IOSurfaceGetBaseAddress(surface), i ? 0xa7 : 0xcc, IOSurfaceGetAllocSize(surface)); + if (IOSurfaceUnlock(surface, 0, nullptr)) return false; + surfaces.push_back(surface); + } + for (unsigned i = 0; i < aliases; ++i) { + // Include independent imports as well as nested views of an import. + id texture = [device newTextureWithDescriptor:descriptor iosurface:surfaces[i % allocations] plane:0]; + if (i & 1) texture = [[texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] + newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + if (!texture) return false; + [textures addObject:texture]; + } + std::vector upload(frameBytes, 0x5a); + id input = [device newBufferWithBytes:upload.data() length:frameBytes options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:aliases * frameBytes options:MTLResourceStorageModeShared]; + id command = [[device newCommandQueue] commandBuffer]; + id blit = [command blitCommandEncoder]; + if (!input || !output || !blit) return false; + [blit copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:row sourceBytesPerImage:frameBytes + sourceSize:MTLSizeMake(width,height,1) toTexture:textures[0] destinationSlice:0 destinationLevel:0 + destinationOrigin:MTLOriginMake(0,0,0)]; + for (unsigned i = 0; i < aliases; ++i) + [blit copyFromTexture:textures[i] sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) + sourceSize:MTLSizeMake(width,height,1) toBuffer:output destinationOffset:i * frameBytes + destinationBytesPerRow:row destinationBytesPerImage:frameBytes]; + [blit endEncoding]; + const uint64_t beforeCalls = calls.load(), beforeBytes = bytes.load(); + [command commit];[command waitUntilCompleted]; + uint64_t expectedBytes = 0; + for (IOSurfaceRef surface : surfaces) expectedBytes += 2 * IOSurfaceGetAllocSize(surface); + bool ok = command.status == MTLCommandBufferStatusCompleted; + for (unsigned i = 0; i < aliases; ++i) for (unsigned j = 0; j < frameBytes; ++j) + if (((const unsigned char *)output.contents)[i * frameBytes + j] != (i % allocations ? 0xa7 : 0x5a)) ok = false; + // The GPU also changed the first surface; check its actual shared bytes. + for (unsigned y = 0; y < height; ++y) for (unsigned x = 0; x < row; ++x) + if (((const unsigned char *)IOSurfaceGetBaseAddress(surfaces[0]))[y * IOSurfaceGetBytesPerRow(surfaces[0]) + x] != 0x5a) ok = false; + const uint64_t actualCalls = calls.load() - beforeCalls, actualBytes = bytes.load() - beforeBytes; + std::printf("aliases=%u allocations=%u flushes=%llu bytes=%llu expected=%u/%llu pixels=%s\n", aliases, allocations, + (unsigned long long)actualCalls, (unsigned long long)actualBytes, 2 * allocations, + (unsigned long long)expectedBytes, ok ? "pass" : "FAIL"); + ok &= actualCalls == 2 * allocations && actualBytes == expectedBytes; + for (IOSurfaceRef surface : surfaces) CFRelease(surface); + return ok; + } +} + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + bool ok = run(device, 1, 1); + ok &= run(device, 128, 1); + ok &= run(device, 128, 2); + char telemetry[6144] = {}; + native_bridge_metal_telemetry(telemetry, sizeof telemetry); + const char *line = strstr(telemetry, "J surface cache "); + unsigned long long count = 0, maintained = 0, duplicates = 0; + if (!line || sscanf(line, "J surface cache n=%llu bytes=%llu dup=%llu", &count, &maintained, &duplicates) != 3 || + count != calls.load() || maintained != bytes.load() || duplicates != 506) ok = false; + if (!ok) {std::puts("FAIL: duplicate IOSurface cache maintenance");return 1;} + std::puts("PASS: each live surface range flushed once per phase, independent imports/views deduplicated, distinct surfaces and GPU bytes preserved"); + } +} diff --git a/bridge/surface_import_host.mm b/bridge/surface_import_host.mm new file mode 100644 index 0000000..820a828 --- /dev/null +++ b/bridge/surface_import_host.mm @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: MIT +// Count real frontend no-copy imports/releases, forwarding every call to the +// unmodified bridge. Use actual GPU pixels and bounded pending-work lifetimes. +#import +#import +#import +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +static std::atomic imports, releases; +static std::mutex handlesLock; +static std::set handles; +static uint64_t (*vkImports)(); +extern "C" NmbBuffer *tracked_surface_import(NmbDevice *d, void *p, uint64_t n, uint64_t options) { + NmbBuffer *buffer = nmb_buffer_new_no_copy(d,p,n,options); + if (buffer) {std::lock_guard lock(handlesLock);handles.insert(buffer);imports++;} + return buffer; +} +extern "C" void tracked_buffer_release(NmbBuffer *buffer) { + {std::lock_guard lock(handlesLock);if(handles.erase(buffer))releases++;} + nmb_buffer_release(buffer); +} +static bool drained(unsigned expected) { + for(unsigned i=0;i<300 && releases.load()!=expected;++i)usleep(1000); + return releases.load()==expected; +} +static IOSurfaceRef makeSurface(unsigned byte) { + IOSurfaceRef surface=IOSurfaceCreate((__bridge CFDictionaryRef)@{ + (id)kIOSurfaceWidth:@64,(id)kIOSurfaceHeight:@16,(id)kIOSurfaceBytesPerElement:@4, + (id)kIOSurfaceBytesPerRow:@256,(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)}); + if(!surface || IOSurfaceLock(surface,0,nullptr))return nullptr; + memset(IOSurfaceGetBaseAddress(surface),byte,IOSurfaceGetAllocSize(surface)); + if(IOSurfaceUnlock(surface,0,nullptr)){CFRelease(surface);return nullptr;} + return surface; +} +static MTLTextureDescriptor *descriptor() { + auto d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:16 mipmapped:NO]; + d.storageMode=MTLStorageModeShared; + d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite|MTLTextureUsagePixelFormatView; + return d; +} + +static bool aliases(id device,unsigned count,unsigned allocations,bool parallel) { + const unsigned before=imports.load(),released=releases.load();bool ok=true; + const uint64_t vkBefore=vkImports?vkImports():0; + @autoreleasepool { + std::vector surfaces; + for(unsigned i=0;i failures{0};auto *failed=&failures; + const IOSurfaceRef surface=surfaces[0]; + dispatch_apply(count,dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^(size_t i){ + (void)i; + @autoreleasepool { + id texture=[device newTextureWithDescriptor:d iosurface:surface plane:0]; + if(!texture){(*failed)++;return;} + @synchronized(textures){[textures addObject:texture];} + } + }); + ok=failures.load()==0; + } else for(unsigned i=0;i texture=[device newTextureWithDescriptor:d iosurface:surfaces[i%allocations] plane:0]; + if(i&1)texture=[[texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]; + if(!texture)return false;[textures addObject:texture]; + } + if(textures.count!=count)return false; + std::vector data(4096,0x5a); + id input=[device newBufferWithBytes:data.data() length:data.size() options:MTLResourceStorageModeShared]; + id output=[device newBufferWithLength:count*4096 options:MTLResourceStorageModeShared]; + id command=[[device newCommandQueue] commandBuffer];id copy=[command blitCommandEncoder]; + if(!input || !output || !copy)return false; + [copy copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:256 sourceBytesPerImage:4096 sourceSize:MTLSizeMake(64,16,1) + toTexture:textures[0] destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; + for(unsigned i=0;i256?(made>=allocations && made<=count):made==allocations; + const uint64_t actualVk=vkImports?vkImports()-vkBefore:0; + std::printf("aliases=%u surfaces=%u parallel=%d imports=%u releases=%u vk_imports=%llu gpu=%s retired=%s reuse=%s\n", + count,allocations,parallel,made,releases.load()-released,(unsigned long long)actualVk,ok?"pass":"FAIL",retired?"pass":"FAIL",reuse?"pass":"FAIL"); + return ok && retired && reuse && (!vkImports || actualVk==allocations); +} + +static bool expires(id device) { + const unsigned before=imports.load(),released=releases.load();IOSurfaceRef surface=makeSurface(0x35);if(!surface)return false; + bool ok=true; + for(unsigned i=0;i<2;++i) { + @autoreleasepool {id texture=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];if(!texture)ok=false;} + ok &= drained(released+i+1) && imports.load()==before+i+1; + } + CFRelease(surface); + std::printf("expired owners imports=%u releases=%u result=%s\n",imports.load()-before,releases.load()-released,ok?"pass":"FAIL"); + return ok; +} + +static bool pending(id device) { + const unsigned before=imports.load(),released=releases.load();bool ok=true; + @autoreleasepool { + IOSurfaceRef surface=makeSurface(0x79);if(!surface)return false; + id event=[device newSharedEvent];id command=[[device newCommandQueue] commandBuffer]; + id output=[device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + if(!event || !command || !output)return false; + @autoreleasepool { + id texture=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];if(!texture)return false; + [command encodeWaitForEvent:event value:1];id copy=[command blitCommandEncoder]; + [copy copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1) + toBuffer:output destinationOffset:0 destinationBytesPerRow:256 destinationBytesPerImage:4096]; + [copy endEncoding];[command commit]; + } + auto finished=dispatch_semaphore_create(0); + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{@autoreleasepool{[command waitUntilCompleted];}dispatch_semaphore_signal(finished);}); + if(!dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,100*NSEC_PER_MSEC)))ok=false; + @autoreleasepool { + id alias=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0]; + if(!alias || imports.load()!=before+1 || releases.load()!=released)ok=false; + // Drop the caller's last surface reference before the GPU reads it. + CFRelease(surface);surface=nullptr; + } + event.signaledValue=1; + if(dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,3*NSEC_PER_SEC)))return false; + if(command.status!=MTLCommandBufferStatusCompleted)ok=false; + for(unsigned i=0;i<4096;++i)if(((const unsigned char *)output.contents)[i]!=0x79)ok=false; + // Completed commands must release the import even if still retained. + ok &= drained(released+1); + } + std::printf("pending owners imports=%u releases=%u result=%s\n",imports.load()-before,releases.load()-released,ok?"pass":"FAIL"); + return ok; +} + +int main(int argc,char **argv) { + if(argc!=3 && !(argc==4 && !strcmp(argv[3],"parallel")))return 2; + @autoreleasepool { + id device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2; + void *loader=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!loader)return 2; + vkImports=reinterpret_cast(dlsym(loader,"nmb_test_vk_host_import_count")); + if(getenv("NMB_TEST_REAL_VULKAN_LOADER") && !vkImports)return 2; + if(argc==4)return aliases(device,64,1,true)?0:1; + bool ok=aliases(device,1,1,false); + ok &= aliases(device,128,1,false); + ok &= aliases(device,128,2,false); + ok &= aliases(device,64,1,true); + ok &= expires(device); + ok &= pending(device); + ok &= aliases(device,514,257,false); + if(!ok || imports.load()!=releases.load()){std::puts("FAIL: live IOSurface import reuse");return 1;} + std::puts("PASS: live IOSurface aliases reuse imports with real GPU bytes; distinct, parallel, expired and pending owners preserve lifetimes"); + } +} diff --git a/bridge/telemetry_host.mm b/bridge/telemetry_host.mm new file mode 100644 index 0000000..1368bb1 --- /dev/null +++ b/bridge/telemetry_host.mm @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Interleave encoding on two queues, then reverse commit order. Global deltas assign +// the first buffer's render work to the second buffer's blits; per-buffer counts must not. +#include "FrontendChecks.h" +#include "NativeMetalObjects.h" +#include "../NativeBridgeMetalFrontend.h" +#include +#include + +int main(int argc, char **argv) { + if (argc != 3) return 2; + @autoreleasepool { + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr); + if (!device) return 2; + size_t length = 0; + const void *air = nmb_fixture_air(&length); + id library = native_bridge_metal_library_with_air(air, length); + frontendchecks::Report report = {}; + frontendchecks::gReport = &report; + id pipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + id texture = [device newTextureWithDescriptor:td]; + id buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared]; + id render = [[device newCommandQueue] commandBuffer]; + id copy = [[device newCommandQueue] commandBuffer]; + if (!library || !pipeline || !texture || !buffer || !render || !copy) return 2; + frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6); + for (unsigned i = 0; i < 3; ++i) { + id blit = [copy blitCommandEncoder]; + [blit fillBuffer:buffer range:NSMakeRange(0, 256) value:0x5a]; + [blit endEncoding]; + } + frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6); + [copy commit];[render commit]; + if (!frontendchecks::completed(copy, "blit") || !frontendchecks::completed(render, "render")) return 1; + char text[4608] = {}; + native_bridge_metal_telemetry(text, sizeof text); + std::map> counts; + for (char *line = strtok(text, "\n"); line; line = strtok(nullptr, "\n")) { + unsigned index, sec, ms, passes, draws, kpx, dispatches, blits; + if (sscanf(line, "K%u t=%u.%us rp%u dr%u kpx%u cs%u bl%u", &index, &sec, &ms, &passes, &draws, &kpx, &dispatches, &blits) != 8) continue; + std::printf("%s\n", line); + counts[index] = {passes, blits}; + if (draws != (index == 2 ? 2u : 0u)) {std::puts("FAIL cross-buffer draw attribution");return 1;} + } + if (counts.size() != 2 || counts[1] != std::make_pair(0u, 3u) || counts[2] != std::make_pair(2u, 0u)) { + std::puts("FAIL interleaved render/blit attribution");return 1; + } + uint8_t pixels[16 * 16 * 4] = {}; + [texture getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + if (frontendchecks::wrongTexels(pixels, 64, 16)) return 1; + for (unsigned i = 0; i < 256; ++i) if (((const uint8_t *)buffer.contents)[i] != 0x5a) return 1; + std::puts("PASS: opposite encode/commit orders retain per-buffer render/blit counts and correct host GPU pixels"); + } +} diff --git a/bridge/uber_diff_host.mm b/bridge/uber_diff_host.mm new file mode 100644 index 0000000..359ee0f --- /dev/null +++ b/bridge/uber_diff_host.mm @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT +// Host instrument: QuartzCore's own uber-shader (fixed_vert_lph_gen with fixed_frag_lph_cph / fixed_frag_lph_cpf) under recipes of its +// function constants, through the native frontend. Each recipe makes one render pipeline, draws it once over fixed pseudo-random inputs +// (every buffer and texture the stages may read, all values finite) into a 64x64 BGRA8 target, and prints the pipeline time, the draw time +// (the runtime makes the VkPipeline at the draw) and a hash of the pixels. Two runs of the same recipes over two builds of the translator, +// on the same loader, must print the same hashes: the instrument behind a change to what the translator emits for these modules. Over the +// Adreno Turnip host simulation the CP executes nothing, so there only the times mean anything. +// argv: loader (or apple), shader cache dir, QuartzCore AIR metallib, recipe file, [pixel dump dir]. A recipe line: fragment name, then name:type:value +// constants (bool, uchar, uchar4 as a,b,c,d); fixed_vert_lph_gen takes the ones it declares. +#import +#import +#include +#include +#include +#include +#include +#include +#include "NativeBridgeMetalFrontend.h" + +extern "C" size_t native_bridge_metal_telemetry(char *, size_t); + +static double nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1e6;} +static void missing(const char *c, const char *s) {std::fprintf(stderr, "missing %s %s\n", c, s);} +static uint32_t lcg(uint32_t &s) {s = s * 1664525u + 1013904223u; return s >> 8;} +// A half in [0, 1): sign 0, exponent 0..14, so any pair of them read as a float is finite too. +static uint16_t unitHalf(uint32_t &s) {return uint16_t((lcg(s) % 15) << 10 | (lcg(s) & 0x3ff));} +static uint64_t fnv(const uint8_t *p, size_t n) {uint64_t h = 1469598103934665603ull; while (n--) h = (h ^ *p++) * 1099511628211ull; return h;} + +static MTLFunctionConstantValues *constants(const std::vector &specs, MTLFunctionConstantValues *v) { + for (const std::string &spec : specs) { + size_t a = spec.find(':'), b = spec.find(':', a + 1); + NSString *name = @(spec.substr(0, a).c_str()); + std::string type = spec.substr(a + 1, b - a - 1), value = spec.substr(b + 1); + if (type == "bool") {bool x = value == "true" || value == "1"; [v setConstantValue:&x type:MTLDataTypeBool withName:name];} + else if (type == "uchar") {uint8_t x = uint8_t(std::strtoul(value.c_str(), nullptr, 0)); [v setConstantValue:&x type:MTLDataTypeUChar withName:name];} + else if (type == "uchar4") { + uint8_t x[4] = {}; + const char *p = value.c_str(); + for (int i = 0; i < 4 && *p; ++i) {char *e; x[i] = uint8_t(std::strtoul(p, &e, 0)); p = *e ? e + 1 : e;} + [v setConstantValue:x type:MTLDataTypeUChar4 withName:name]; + } + } + return v; +} + +int main(int argc, char **argv) { + if (argc < 5) {std::fprintf(stderr, "usage: loader cache metallib recipes [dump dir]\n"); return 2;} + @autoreleasepool { + // Loader "apple": the host's own Metal device compiles the same AIR, the reference the translator's results are held to. + const bool apple = !std::strcmp(argv[1], "apple"); + id device = apple ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed\n"); return 1;} + NSError *error = nil; + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:@(argv[3])] error:&error]; + if (!qc) {std::printf("library failed: %s\n", error.localizedDescription.UTF8String); return 1;} + FILE *recipes = std::fopen(argv[4], "r"); + if (!recipes) {std::printf("no recipes\n"); return 1;} + const char *dump = argc > 5 ? argv[5] : nullptr; + id queue = [device newCommandQueue]; + constexpr unsigned W = 64, H = 64; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:W height:H mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; + td.storageMode = MTLStorageModeShared; + id target = [device newTextureWithDescriptor:td]; + // Inputs, the same in every run: a full-screen pair of triangles (SurfaceVertexXC: float4 position, two float2 texcoords, half4 + // color, padding; 48 bytes), the identity MVP and texture matrices, a zero vertex config (layout 0), and finite pseudo-random + // words for every other buffer and texel. + uint32_t seed = 0x5eed1234u; + struct Vertex {float position[4], tc0[2], tc1[2]; uint16_t color[4]; uint8_t pad[8];}; + static_assert(sizeof(Vertex) == 48); + const float corners[6][2] = {{-1, -1}, {1, -1}, {-1, 1}, {-1, 1}, {1, -1}, {1, 1}}; + Vertex vertices[6] = {}; + for (int i = 0; i < 6; ++i) { + vertices[i].position[0] = corners[i][0]; vertices[i].position[1] = corners[i][1]; vertices[i].position[3] = 1; + vertices[i].tc0[0] = vertices[i].tc1[1] = (corners[i][0] + 1) / 2; + vertices[i].tc0[1] = vertices[i].tc1[0] = (1 - corners[i][1]) / 2; + for (int c = 0; c < 4; ++c) vertices[i].color[c] = unitHalf(seed); + } + id vertexBuffer = [device newBufferWithBytes:vertices length:sizeof vertices options:MTLResourceStorageModeShared]; + const float identity[16] = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + id matrix = [device newBufferWithBytes:identity length:sizeof identity options:MTLResourceStorageModeShared]; + id zeros = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + std::vector> words; + for (int i = 0; i < 9; ++i) { + std::vector w(131072 / 2 + 512); + for (uint16_t &x : w) x = unitHalf(seed); + if (i == 1) { + // CA::OGL::Metal::Uniform at its own layout: floats in [0, 1) at [6 x float4] (0..96), [2 x float4] and float4 + // (128..176); halves in [0, 1) elsewhere; its two flag bytes (120, 121) and the trailing [8 x i8] zero. + auto *bytes = reinterpret_cast(w.data()); + auto unitFloat = [&](size_t at) {float f = float(lcg(seed) % 65536) / 65536.0f; std::memcpy(bytes + at, &f, 4);}; + for (size_t at = 0; at < 96; at += 4) unitFloat(at); + for (size_t at = 128; at < 176; at += 4) unitFloat(at); + bytes[120] = bytes[121] = 0; + std::memset(bytes + 216, 0, 8); + } + words.push_back([device newBufferWithBytes:w.data() length:w.size() * 2 options:MTLResourceStorageModeShared]); + } + MTLTextureDescriptor *xd = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA16Float width:16 height:16 mipmapped:NO]; + xd.usage = MTLTextureUsageShaderRead; + std::vector> textures; + for (int i = 0; i < 12; ++i) { + id t = [device newTextureWithDescriptor:xd]; + std::vector texels(16 * 16 * 4); + for (uint16_t &x : texels) x = unitHalf(seed); + [t replaceRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0 withBytes:texels.data() bytesPerRow:16 * 8]; + textures.push_back(t); + } + MTLSamplerDescriptor *sd = [MTLSamplerDescriptor new]; + sd.minFilter = sd.magFilter = MTLSamplerMinMagFilterLinear; + id sampler = [device newSamplerStateWithDescriptor:sd]; + char line[4096]; + unsigned n = 0, made = 0; + double pipeTotal = 0, drawTotal = 0; + while (std::fgets(line, sizeof line, recipes)) { + std::vector tokens; + for (char *t = std::strtok(line, " \t\n"); t; t = std::strtok(nullptr, " \t\n")) tokens.emplace_back(t); + if (tokens.empty() || tokens[0][0] == '#') continue; + ++n; + std::vector specs(tokens.begin() + 1, tokens.end()); + double f0 = nowMs(); + id vertex = [qc newFunctionWithName:@"fixed_vert_lph_gen" constantValues:constants(specs, [MTLFunctionConstantValues new]) error:&error]; + id fragment = vertex ? [qc newFunctionWithName:@(tokens[0].c_str()) constantValues:constants(specs, [MTLFunctionConstantValues new]) error:&error] : nil; + double f1 = nowMs(); + if (!fragment) {std::printf("R%03u function refused: %s\n", n, error.localizedDescription.UTF8String); continue;} + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = vertex; + d.fragmentFunction = fragment; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + double p0 = nowMs(); + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + double p1 = nowMs(); + if (!state) {std::printf("R%03u pipeline refused: %s\n", n, error.localizedDescription.UTF8String); continue;} + ++made; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0); + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + double d0 = nowMs(); + id commands = [queue commandBuffer]; + id encoder = [commands renderCommandEncoderWithDescriptor:pass]; + [encoder setRenderPipelineState:state]; + [encoder setVertexBuffer:zeros offset:0 atIndex:0]; + [encoder setVertexBuffer:vertexBuffer offset:0 atIndex:1]; + [encoder setVertexBuffer:matrix offset:0 atIndex:2]; + [encoder setVertexBuffer:matrix offset:0 atIndex:3]; + [encoder setVertexBuffer:matrix offset:0 atIndex:4]; + for (NSUInteger i = 0; i < 9; ++i) [encoder setFragmentBuffer:words[i] offset:0 atIndex:i]; + for (NSUInteger i = 0; i < 12; ++i) [encoder setFragmentTexture:textures[i] atIndex:i]; + [encoder setFragmentSamplerState:sampler atIndex:0]; + [encoder setFragmentSamplerState:sampler atIndex:1]; + [encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [encoder endEncoding]; + [commands commit]; + [commands waitUntilCompleted]; + double d1 = nowMs(); + std::vector pixels(W * H * 4); + [target getBytes:pixels.data() bytesPerRow:W * 4 fromRegion:MTLRegionMake2D(0, 0, W, H) mipmapLevel:0]; + if (dump) { + char path[1024]; + std::snprintf(path, sizeof path, "%s/R%03u.bgra", dump, n); + if (FILE *out = std::fopen(path, "wb")) {std::fwrite(pixels.data(), 1, pixels.size(), out); std::fclose(out);} + } + pipeTotal += p1 - p0; + drawTotal += d1 - d0; + std::printf("R%03u %s fn=%.1f pipe=%.1f draw=%.1f status=%ld hash=%016llx\n", n, tokens[0].c_str(), f1 - f0, p1 - p0, d1 - d0, + (long)commands.status, (unsigned long long)fnv(pixels.data(), pixels.size())); + std::fflush(stdout); + } + static char table[8192]; + if (!apple) native_bridge_metal_telemetry(table, sizeof table); + for (char *l = std::strtok(table, "\n"); l; l = std::strtok(nullptr, "\n")) + if (l[0] == 'V' || l[0] == 'T') std::printf("TELEMETRY %s\n", l); + std::printf("SUMMARY recipes=%u made=%u pipe=%.0f ms draw=%.0f ms\n", n, made, pipeTotal, drawTotal); + } + return 0; +} diff --git a/shaders/smoke_fill.comp b/shaders/smoke_fill.comp new file mode 100644 index 0000000..42f392a --- /dev/null +++ b/shaders/smoke_fill.comp @@ -0,0 +1,9 @@ +#version 450 +// Shader-execution check: every invocation writes a value derived from its global index and two push constants. +layout(local_size_x = 64) in; +layout(std430, set = 0, binding = 0) buffer Out { uint v[]; } o; +layout(push_constant) uniform P { uint mul; uint add; } p; +void main() { + uint i = gl_GlobalInvocationID.x; + o.v[i] = i * p.mul + p.add; +} diff --git a/shaders/smoke_triangle.frag b/shaders/smoke_triangle.frag new file mode 100644 index 0000000..908bb17 --- /dev/null +++ b/shaders/smoke_triangle.frag @@ -0,0 +1,4 @@ +#version 450 +layout(location = 0) out vec4 color; +layout(push_constant) uniform P { vec4 color; } p; +void main() { color = p.color; } diff --git a/shaders/smoke_triangle.vert b/shaders/smoke_triangle.vert new file mode 100644 index 0000000..9aa6974 --- /dev/null +++ b/shaders/smoke_triangle.vert @@ -0,0 +1,6 @@ +#version 450 +// Rasterisation check: one triangle from the vertex index, no vertex buffers. +void main() { + const vec2 p[3] = vec2[](vec2(-0.5, -0.5), vec2(0.5, -0.5), vec2(0.0, 0.5)); + gl_Position = vec4(p[gl_VertexIndex], 0.0, 1.0); +} diff --git a/tools/MetalVulkanProbe.rs b/tools/MetalVulkanProbe.rs new file mode 100644 index 0000000..11321fa --- /dev/null +++ b/tools/MetalVulkanProbe.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Offline consumer of a pinned translator's public API; never opens a GPU. +use metal2vulkan::device::{Device, DeviceDescription, DevicePolicy, SamplingPolicy}; +use metal2vulkan::library::{FunctionStage, Library}; +use metal2vulkan::metal::PixelFormat; +use metal2vulkan::pipeline::{ + ColorAttachmentDescriptor, ComputePipelineDescriptor, FunctionDescriptor, + RenderPipelineDescriptor, StageModule, StageSource, VertexDescriptor, +}; +use metal2vulkan::Budget; +use serde_json::{json, Value}; +use std::{collections::BTreeMap, env, fs, process}; + +fn stage(module: &StageModule) -> Value { + json!({ + "stage": format!("{:?}", module.stage), + "entry_point": module.entry_point, + "spirv": &*module.spirv, + "precision_notes": module.precision_notes.iter().map(|n| format!("{n:?}")).collect::>(), + "required_subgroup_size": module.required_subgroup_size, + "specialization": format!("{:?}", module.specialization), + "constant_blob_bytes": module.constant_blob.as_ref().map_or(0, |b| b.len()), + }) +} + +fn run(args: &[String]) -> Result> { + if args.len() == 2 && args[0] == "--list" { + let library = Library::parse(&fs::read(&args[1])?)?; + return Ok(json!({"functions": library.functions().map(|f| json!({ + "name": f.name, "stage": format!("{:?}", f.stage) + })).collect::>(), "gpu_execution": false})); + } + let mut args = args; + let mut vertex_descriptor = None; + let mut color_attachments = None; + while args.first().is_some_and(|v| v.starts_with("--")) { + let path = args.get(1).ok_or("missing descriptor path")?; + match args[0].as_str() { + "--vertex-descriptor" if vertex_descriptor.is_none() => { + vertex_descriptor = Some(serde_json::from_slice::(&fs::read( + path, + )?)?); + } + "--color-attachments" if color_attachments.is_none() => { + let colors = + serde_json::from_slice::>(&fs::read(path)?)?; + if colors.is_empty() || colors.keys().any(|index| *index >= 8) { + return Err("color attachments require indices in 0..8".into()); + } + color_attachments = Some(colors); + } + _ => return Err("unknown or duplicate descriptor option".into()), + } + args = &args[2..]; + } + if args.len() != 3 && args.len() != 5 { + return Err( + "expected --list INPUT, or PROFILE INPUT FUNCTION [FRAGMENT_INPUT FUNCTION]".into(), + ); + } + let description = DeviceDescription::from_khronos_json(&fs::read_to_string(&args[0])?, None)?; + let mut policy = DevicePolicy::default().exact_answers(); + // exact_answers deliberately keeps sampling unchanged in this upstream revision. + policy.sampling = SamplingPolicy::Exact; + let device = Device::new(description, policy)?; + let library = Library::parse(&fs::read(&args[1])?)?; + let function = library.function(&args[2])?; + let budget = Budget::standard(); + let (stages, layout, interface, dependencies, requirements) = match function.info().stage { + FunctionStage::Kernel if args.len() == 3 => { + if vertex_descriptor.is_some() || color_attachments.is_some() { + return Err("compute pipeline does not take render descriptors".into()); + } + let descriptor = ComputePipelineDescriptor::new(StageSource::Function( + FunctionDescriptor::new(function), + )); + let p = device.compile_compute_pipeline(&descriptor, &budget)?; + ( + vec![stage(&p.stage)], + format!("{:?}", p.layout), + format!("{:?}", p.interface), + format!("{:?}", p.dependencies), + format!("{:?}", p.requirements), + ) + } + FunctionStage::Vertex => { + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function( + FunctionDescriptor::new(function), + )); + descriptor.vertex_descriptor = vertex_descriptor; + if args.len() == 5 { + let fragment_library = Library::parse(&fs::read(&args[3])?)?; + let fragment = fragment_library.function(&args[4])?; + if fragment.info().stage != FunctionStage::Fragment { + return Err("fragment input is not a fragment function".into()); + } + descriptor.fragment = + Some(StageSource::Function(FunctionDescriptor::new(fragment))); + let colors = color_attachments + .as_ref() + .ok_or("fragment translation requires explicit color attachments")?; + for (&index, &format) in colors { + descriptor.color_attachments[index] = + Some(ColorAttachmentDescriptor::new(format)); + } + } else if color_attachments.is_some() { + return Err("color attachments require a fragment function in this probe".into()); + } + let p = device.compile_render_pipeline(&descriptor, &budget)?; + ( + p.stages.iter().map(stage).collect(), + format!("{:?}", p.layout), + format!("{:?}", p.interface), + format!("{:?}", p.dependencies), + format!("{:?}", p.requirements), + ) + } + _ => { + return Err( + "stage needs a matching pipeline descriptor; standalone fragment refused".into(), + ) + } + }; + Ok(json!({ + "stages": stages, "layout": layout, "interface": interface, + "dependencies": dependencies, "requirements": requirements, + "policy": "exact_answers+exact_sampling", + "validation_envs": device.profile().spirv_validation().target_envs, + "validation_relaxations": device.profile().spirv_validation().relaxations, + "profile_is_hardware_observation": false, + "gpu_execution": false, "metal_device_published": false, + })) +} + +fn main() { + match run(&env::args().skip(1).collect::>()) { + Ok(value) => println!("{value}"), + Err(error) => { + eprintln!("REFUSED: {error}"); + process::exit(2); + } + } +} diff --git a/tools/metal-vulkan-probe.lock b/tools/metal-vulkan-probe.lock new file mode 100644 index 0000000..3800b8d --- /dev/null +++ b/tools/metal-vulkan-probe.lock @@ -0,0 +1,208 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "metal2vulkan" +version = "0.2.0" +dependencies = [ + "metal2vulkan-air", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "metal2vulkan-air" +version = "0.2.0" +dependencies = [ + "serde", + "sha2", +] + +[[package]] +name = "native-metal-vulkan-probe" +version = "0.1.0" +dependencies = [ + "metal2vulkan", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/probe-coverage-attachment.json b/tools/probe-coverage-attachment.json new file mode 100644 index 0000000..d75ed15 --- /dev/null +++ b/tools/probe-coverage-attachment.json @@ -0,0 +1 @@ +{"2": "R32Float"} diff --git a/tools/probe-vertex-layout.json b/tools/probe-vertex-layout.json new file mode 100644 index 0000000..72deaf8 --- /dev/null +++ b/tools/probe-vertex-layout.json @@ -0,0 +1,10 @@ +{ + "layouts": { + "0": {"stride": 48, "step_function": "PerVertex", "step_rate": 1} + }, + "attributes": { + "0": {"format": "Float4", "offset": 0, "buffer_index": 0}, + "1": {"format": "Float2", "offset": 16, "buffer_index": 0}, + "3": {"format": "Float4", "offset": 32, "buffer_index": 0} + } +}