Files
UniXNU/scripts/prepare-xnu-source.py

131 lines
6.1 KiB
Python
Executable File

#!/usr/bin/env python3
"""Restore omitted DriverKit headers from Apple's earlier Darwin 23 source."""
import concurrent.futures
import hashlib
import http.client
import json
import os
from pathlib import Path
import tarfile
import time
import urllib.error
import urllib.parse
import urllib.request
# Tags of the Apple sources that still carry the headers newer xnu omits; feeds.toml sets them for the unixnu driver.
TAG = os.environ.get('UNIXNU_DRIVERKIT_TAG', 'xnu-10063.121.3')
TRUST_TAG = os.environ.get('UNIXNU_TRUSTCACHE_TAG', 'xnu-8796.101.5')
DESTINATION = Path('.cache/xnu-work/iokit/DriverKit')
CACHE = Path(f'.cache/sources/xnu-driverkit-{TAG.removeprefix("xnu-")}.json')
# Anonymous GitHub API calls share a small per-address quota that hosted CI runners exhaust (HTTP 403);
# a token in GITHUB_TOKEN or GH_TOKEN is sent to GitHub hosts only.
TOKEN = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN')
def fetch(url):
request = urllib.request.Request(url, headers={'User-Agent': 'unixnu-prepare-xnu-source'})
if TOKEN and urllib.parse.urlsplit(url).hostname in ('api.github.com', 'raw.githubusercontent.com'):
request.add_header('Authorization', f'Bearer {TOKEN}')
for attempt in range(3):
try:
with urllib.request.urlopen(request, timeout=45) as response:
return response.read()
except (OSError, EOFError, http.client.IncompleteRead) as error:
if isinstance(error, urllib.error.HTTPError) and error.code not in (429, 500, 502, 503, 504):
raise
if attempt == 2:
raise
time.sleep(attempt + 1)
def archive_entries(directory, tag):
"""Use the same Apple source tag when GitHub's directory API is rate limited."""
archive = CACHE.parent / f'xnu-public-headers-{tag}.tar.gz'
if not archive.exists():
print(f'Fetching public headers - Apple {tag} archive', flush=True)
data = fetch(f'https://codeload.github.com/apple-oss-distributions/xnu/tar.gz/refs/tags/{urllib.parse.quote(tag, safe="")}')
archive.parent.mkdir(parents=True, exist_ok=True)
temporary = archive.with_suffix('.part')
temporary.write_bytes(data)
with tarfile.open(temporary, 'r:gz') as source:
source.getmembers()
temporary.replace(archive)
entries = []
with tarfile.open(archive, 'r:gz') as source:
for member in source:
relative = member.name.partition('/')[2]
if not member.isfile() or not relative.startswith(directory + '/'):
continue
name = relative[len(directory) + 1:]
if '/' in name or not name.endswith('.h'):
continue
data = source.extractfile(member).read()
entries.append({'type': 'file', 'name': name,
'sha': hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest(),
'download_url': f'https://raw.githubusercontent.com/apple-oss-distributions/xnu/{tag}/{relative}'})
if not entries:
raise ValueError(f'No public headers in Apple {tag}: {directory}')
return entries
def directory_entries(cache, directory, tag):
if cache.exists():
return json.loads(cache.read_text())
url = ('https://api.github.com/repos/apple-oss-distributions/xnu/contents/'
f'{directory}?ref={urllib.parse.quote(tag, safe="")}')
try:
entries = json.loads(fetch(url))
except urllib.error.HTTPError as error:
if error.code not in (403, 429):
raise
print(f'GitHub directory API limited - using Apple {tag} archive for {directory}', flush=True)
entries = archive_entries(directory, tag)
if not isinstance(entries, list):
raise ValueError(f'Unexpected GitHub directory response: {directory}')
cache.parent.mkdir(parents=True, exist_ok=True)
temporary = cache.with_suffix('.tmp')
temporary.write_text(json.dumps(entries) + '\n')
temporary.replace(cache)
return entries
def restore(entry, destination=DESTINATION, tag=TAG):
if entry['type'] != 'file' or not entry['name'].endswith('.h'):
return None
name = entry.get('relative', entry['name'])
target = destination / name
if target.exists():
return None
data = fetch(entry['download_url'])
digest = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
if digest != entry['sha']:
raise ValueError(f'Git blob hash mismatch: {entry["name"]}')
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
return {'name': name, 'source_tag': tag, 'git_blob': digest,
'sha256': hashlib.sha256(data).hexdigest()}
if __name__ == '__main__':
if not DESTINATION.is_dir():
raise SystemExit('Create the local XNU work source before preparing headers')
entries = directory_entries(CACHE, 'iokit/DriverKit', TAG)
crypto_cache = CACHE.with_name(f'xnu-driverkit-crypto-{TAG.removeprefix("xnu-")}.json')
for entry in directory_entries(crypto_cache, 'iokit/DriverKit/crypto', TAG):
entry['relative'] = 'crypto/' + entry['name']
entries.append(entry)
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
restored = [entry for entry in executor.map(restore, entries) if entry]
trust_tag = TRUST_TAG
trust_cache = CACHE.with_name(f'xnu-trustcache-{TRUST_TAG.removeprefix("xnu-")}.json')
trust_destination = Path('.cache/xnu-work/EXTERNAL_HEADERS/TrustCache')
def restore_trust(entry):
return restore(entry, trust_destination, trust_tag)
restored.extend(entry for entry in executor.map(
restore_trust, directory_entries(trust_cache, 'EXTERNAL_HEADERS/TrustCache', trust_tag)) if entry)
previous = Path('.cache/sources/xnu-restored-headers.json')
existing = json.loads(previous.read_text()) if previous.exists() else []
by_name = {entry['name']: entry for entry in existing + restored}
previous.write_text(json.dumps(list(by_name.values()), indent=2) + '\n')
print(f'Restored {len(restored)} omitted public headers from Apple; blob hashes verified')