mirror of
https://github.com/UniXNU/UniXNU.git
synced 2026-11-18 10:01:39 +03:00
131 lines
6.1 KiB
Python
Executable File
131 lines
6.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Restore omitted DriverKit headers from Apple's earlier Darwin 23 source."""
|
|
import concurrent.futures
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tarfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
# Tags of the Apple sources that still carry the headers newer xnu omits; feeds.toml sets them for the unixnu driver.
|
|
TAG = os.environ.get('UNIXNU_DRIVERKIT_TAG', 'xnu-10063.121.3')
|
|
TRUST_TAG = os.environ.get('UNIXNU_TRUSTCACHE_TAG', 'xnu-8796.101.5')
|
|
DESTINATION = Path('.cache/xnu-work/iokit/DriverKit')
|
|
CACHE = Path(f'.cache/sources/xnu-driverkit-{TAG.removeprefix("xnu-")}.json')
|
|
# Anonymous GitHub API calls share a small per-address quota that hosted CI runners exhaust (HTTP 403);
|
|
# a token in GITHUB_TOKEN or GH_TOKEN is sent to GitHub hosts only.
|
|
TOKEN = os.environ.get('GITHUB_TOKEN') or os.environ.get('GH_TOKEN')
|
|
|
|
|
|
def fetch(url):
|
|
request = urllib.request.Request(url, headers={'User-Agent': 'unixnu-prepare-xnu-source'})
|
|
if TOKEN and urllib.parse.urlsplit(url).hostname in ('api.github.com', 'raw.githubusercontent.com'):
|
|
request.add_header('Authorization', f'Bearer {TOKEN}')
|
|
for attempt in range(3):
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=45) as response:
|
|
return response.read()
|
|
except (OSError, EOFError, http.client.IncompleteRead) as error:
|
|
if isinstance(error, urllib.error.HTTPError) and error.code not in (429, 500, 502, 503, 504):
|
|
raise
|
|
if attempt == 2:
|
|
raise
|
|
time.sleep(attempt + 1)
|
|
|
|
|
|
def archive_entries(directory, tag):
|
|
"""Use the same Apple source tag when GitHub's directory API is rate limited."""
|
|
archive = CACHE.parent / f'xnu-public-headers-{tag}.tar.gz'
|
|
if not archive.exists():
|
|
print(f'Fetching public headers - Apple {tag} archive', flush=True)
|
|
data = fetch(f'https://codeload.github.com/apple-oss-distributions/xnu/tar.gz/refs/tags/{urllib.parse.quote(tag, safe="")}')
|
|
archive.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = archive.with_suffix('.part')
|
|
temporary.write_bytes(data)
|
|
with tarfile.open(temporary, 'r:gz') as source:
|
|
source.getmembers()
|
|
temporary.replace(archive)
|
|
entries = []
|
|
with tarfile.open(archive, 'r:gz') as source:
|
|
for member in source:
|
|
relative = member.name.partition('/')[2]
|
|
if not member.isfile() or not relative.startswith(directory + '/'):
|
|
continue
|
|
name = relative[len(directory) + 1:]
|
|
if '/' in name or not name.endswith('.h'):
|
|
continue
|
|
data = source.extractfile(member).read()
|
|
entries.append({'type': 'file', 'name': name,
|
|
'sha': hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest(),
|
|
'download_url': f'https://raw.githubusercontent.com/apple-oss-distributions/xnu/{tag}/{relative}'})
|
|
if not entries:
|
|
raise ValueError(f'No public headers in Apple {tag}: {directory}')
|
|
return entries
|
|
|
|
|
|
def directory_entries(cache, directory, tag):
|
|
if cache.exists():
|
|
return json.loads(cache.read_text())
|
|
url = ('https://api.github.com/repos/apple-oss-distributions/xnu/contents/'
|
|
f'{directory}?ref={urllib.parse.quote(tag, safe="")}')
|
|
try:
|
|
entries = json.loads(fetch(url))
|
|
except urllib.error.HTTPError as error:
|
|
if error.code not in (403, 429):
|
|
raise
|
|
print(f'GitHub directory API limited - using Apple {tag} archive for {directory}', flush=True)
|
|
entries = archive_entries(directory, tag)
|
|
if not isinstance(entries, list):
|
|
raise ValueError(f'Unexpected GitHub directory response: {directory}')
|
|
cache.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = cache.with_suffix('.tmp')
|
|
temporary.write_text(json.dumps(entries) + '\n')
|
|
temporary.replace(cache)
|
|
return entries
|
|
|
|
|
|
def restore(entry, destination=DESTINATION, tag=TAG):
|
|
if entry['type'] != 'file' or not entry['name'].endswith('.h'):
|
|
return None
|
|
name = entry.get('relative', entry['name'])
|
|
target = destination / name
|
|
if target.exists():
|
|
return None
|
|
data = fetch(entry['download_url'])
|
|
digest = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
|
|
if digest != entry['sha']:
|
|
raise ValueError(f'Git blob hash mismatch: {entry["name"]}')
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(data)
|
|
return {'name': name, 'source_tag': tag, 'git_blob': digest,
|
|
'sha256': hashlib.sha256(data).hexdigest()}
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if not DESTINATION.is_dir():
|
|
raise SystemExit('Create the local XNU work source before preparing headers')
|
|
entries = directory_entries(CACHE, 'iokit/DriverKit', TAG)
|
|
crypto_cache = CACHE.with_name(f'xnu-driverkit-crypto-{TAG.removeprefix("xnu-")}.json')
|
|
for entry in directory_entries(crypto_cache, 'iokit/DriverKit/crypto', TAG):
|
|
entry['relative'] = 'crypto/' + entry['name']
|
|
entries.append(entry)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
|
restored = [entry for entry in executor.map(restore, entries) if entry]
|
|
trust_tag = TRUST_TAG
|
|
trust_cache = CACHE.with_name(f'xnu-trustcache-{TRUST_TAG.removeprefix("xnu-")}.json')
|
|
trust_destination = Path('.cache/xnu-work/EXTERNAL_HEADERS/TrustCache')
|
|
def restore_trust(entry):
|
|
return restore(entry, trust_destination, trust_tag)
|
|
restored.extend(entry for entry in executor.map(
|
|
restore_trust, directory_entries(trust_cache, 'EXTERNAL_HEADERS/TrustCache', trust_tag)) if entry)
|
|
previous = Path('.cache/sources/xnu-restored-headers.json')
|
|
existing = json.loads(previous.read_text()) if previous.exists() else []
|
|
by_name = {entry['name']: entry for entry in existing + restored}
|
|
previous.write_text(json.dumps(list(by_name.values()), indent=2) + '\n')
|
|
print(f'Restored {len(restored)} omitted public headers from Apple; blob hashes verified')
|