from pathlib import Path """Command line of the UniXNU build driver.""" import argparse import json import platform import shutil import sys from . import bootimg, components, feeds, firmware, ipsw, kernel, package, patches, payload, pipeline, profiles, runtime, toolchain, tree, userdata, vendor_boot from .config import LOCAL_KINDS, Context, parse_config from .util import Error, expand, git, log STEPS = ('kernel', 'package', 'firmware', 'payload') OPTIONAL_STEPS = ('userdata', 'runtime') def common(): p = argparse.ArgumentParser(add_help=False) p.add_argument('--config', metavar='FILE', help='local build inputs (default: unixnu.local.toml in the UniXNU checkout)') p.add_argument('-D', '--device', metavar='DIR', help='device repository (remembered; or UNIXNU_DEVICE)') p.add_argument('--os', metavar='VERSION', help='OS profile for this run: 17, 26, 27 or a profiles/os directory name (default: the board\'s os_profile)') p.add_argument('--out', metavar='DIR', help='output root (default: out/ here, or UNIXNU_OUT)') p.add_argument('-j', '--jobs', type=int, help='parallel jobs') p.add_argument('--local', metavar='DIR', help='directory laid out like this repository; its drivers, kits, patches, scripts and package are used') for kind in LOCAL_KINDS: p.add_argument(f'--local-{kind}', metavar='DIR', help=f'use DIR as the {kind} directory (beats --local)') p.add_argument('--local-module', action='append', metavar='[KIND:]NAME=PATH', help='use PATH for one module (beats everything)') p.add_argument('--uefi', metavar='DIR', help='Mu-Silicium checkout for the device (uefi-sheng); or UNIXNU_UEFI') p.add_argument('--clang-bin', metavar='DIR', help='EDK II clang, lld-link, llvm-lib and llvm-rc directory; or CLANG_BIN') p.add_argument('--edk2', metavar='DIR', help='Mu_Basecore checkout; or UNIXNU_EDK2') p.add_argument('--kernel-sdk', metavar='DIR', help='exported iPhoneOS SDK for kernel and prelink; or UNIXNU_KERNEL_SDK') p.add_argument('--kernel-tools', metavar='DIR', help='exported toolchain bin directory for kernel and prelink; or UNIXNU_KERNEL_TOOLS') p.add_argument('--kernelcache', metavar='FILE', help='original decompressed kernelcache; or UNIXNU_KERNELCACHE') p.add_argument('--skip-prelink', action='store_true', help='kernel/prelink: copy the built native kernel to kernel.macho without adding kernelcache kexts') p.add_argument('--skip-preflight', action='store_true', help='skip the board preflight scripts of the firmware step') p.add_argument('--kernel', metavar='FILE', help='prelinked kernel Mach-O for the payload (default: the kernel step output)') p.add_argument('--kernel-arg', action='append', metavar='NAME[=VALUE]', help='extra kernel boot argument (repeatable)') p.add_argument('--trust-cache', metavar='FILE', help='trust cache bundle for the payload') p.add_argument('--owner-trust-cache', metavar='FILE', help='owner-signed trust cache bundle that carries the original caches and extra modules; replaces --trust-cache and skips the board allow-list') p.add_argument('--skip-setup', action='store_true', help='mark Setup as done in the Data seed, so the Hello pages are skipped (needs --data-ramdisk; optional)') p.add_argument('--data-ramdisk', metavar='FILE', help='preserved Data ramdisk seed for this installation') p.add_argument('--init-data', action='store_true', help='create and retain a new Data seed from the IPSW for a FIRST installation; requires kernel build') p.add_argument('--gpu-addon', metavar='FILE', help='GPU addon file (scripts/make-gpu-addon.py) the kernel writes into the Data volume at boot') p.add_argument('--unique-chip-id', metavar='HEX', help='platform seed for the device tree; or UNIXNU_UNIQUE_CHIP_ID') p.add_argument('--ipsw', metavar='PATH_OR_URL', help='the target IPSW: a local file or an https URL (default: url of board.toml [ipsw]); or UNIXNU_IPSW') p.add_argument('--dry-run', action='store_true', help='userdata: check sizes and pins, write nothing') p.add_argument('--object', action='append', default=[], metavar='NAME=FILE[:OFFSET]', help='userdata: add a data object the kernel reads itself (policy table, firmware), ' 'the bytes of FILE from OFFSET on (repeatable)') p.add_argument('--target', choices=('DEBUG', 'RELEASE'), help='EDK II build target') return p def parser(): top = argparse.ArgumentParser(prog='unixnu', description='Build UniXNU, the XNU kernel and the boot image for a device repository.') sub = top.add_subparsers(dest='command', required=True) c = common() d = sub.add_parser('defconfig', parents=[c], help='select configs/NAME in the device repository') d.add_argument('name', nargs='?', help='defconfig name; omit to list') f = sub.add_parser('fetch', parents=[c], help='update module submodules, shallow-fetch git sources and write unixnu.lock') f.add_argument('names', nargs='*', help='sources to fetch (default: all)') f.add_argument('--update', action='store_true', help='ignore the lock and move to the newest ref') sub.add_parser('tree', parents=[c], help='assemble the build tree') b = sub.add_parser('build', parents=[c], help='build kernel, package and firmware') b.add_argument('steps', nargs='*', choices=STEPS + OPTIONAL_STEPS + ('all', 'prelink'), default=['all']) for step in STEPS + OPTIONAL_STEPS + ('prelink',): shortcut = sub.add_parser(step, parents=[c], help=f'alias for build {step}') shortcut.set_defaults(steps=[step]) i = sub.add_parser('bootimg', parents=[c], help='pack the boot image from the firmware volume and a device tree') i.add_argument('--fd', metavar='FILE', help='firmware volume with UniXNU, DarwinBoot and payload filesystem support (default: build firmware output under --uefi)') i.add_argument('--dtb', metavar='FILE', help='compiled device tree; or UNIXNU_DTB') i.add_argument('--ramdisk', metavar='FILE', help='ramdisk image, or none (default: the payload step output when present, else empty)') i.add_argument('-o', '--output', metavar='FILE', help='output (default: out//boot/boot.img)') v = sub.add_parser('vendor_boot', parents=[c], help='replace the vendor ramdisk of a stock vendor_boot image') v.add_argument('--stock', metavar='FILE', help='stock vendor_boot image to rebuild from') v.add_argument('--ramdisk', metavar='FILE', help='vendor ramdisk to install, or none for an empty one') v.add_argument('--ramdisk-addr', metavar='ADDR', help='ramdisk load address (default: the stock address)') v.add_argument('--verify-against', action='store_true', help='repack the stock ramdisk first and require a byte-identical result') v.add_argument('-o', '--output', metavar='FILE', help='output (default: vendor_boot.img next to --stock)') w = sub.add_parser('ipsw', parents=[c], help='show or extract the components of the target IPSW') w.add_argument('action', choices=('info', 'fetch')) w.add_argument('components', nargs='*', help='components to extract (default: the small ones the board lists)') status = sub.add_parser('status', parents=[c], help='show the device, configuration and where each module comes from') status.add_argument('--json', action='store_true', help='print resolved configuration as JSON without fetching sources') sub.add_parser('clean', parents=[c], help='remove the build output of the device (keeps fetched sources)') k = sub.add_parser('release', parents=[c], help='dependencies, patches, build steps and boot image of the device in one run, with a manifest') k.add_argument('--no-deps', action='store_true', help='skip fetching sources and the IPSW members') k.add_argument('--check', action='store_true', help='fetch, resolve and list the patch series, then stop before any build step') return top def need_uefi(ctx): uefi = ctx.option('uefi', 'UNIXNU_UEFI') if not uefi: raise Error('the firmware needs a Mu-Silicium checkout: --uefi DIR (or UNIXNU_UEFI)') if not (uefi / 'Resources/Scripts/mkbootimg.py').is_file(): raise Error(f'{uefi} is not a Mu-Silicium checkout') return uefi REFERENCES = {'kernel': ('xnu', 'libplatform', 'libdispatch', 'AvailabilityVersions', 'libpthread', 'corecrypto-2023', 'IOStorageFamily', 'IOHIDFamily'), 'package': ('edk2',), 'firmware': ('edk2',)} def prepare(ctx, steps=()): """Update module submodules, resolve modules, fetch the git references and assemble the tree.""" feeds.update_submodules(ctx) modules = feeds.resolve(ctx) wanted = {name for step in steps for name in REFERENCES.get(step, ())} feeds.fetch(ctx, modules, only={m.name for m in modules.values() if m.origin == 'git' and (m.kind != 'refer' or m.name in wanted)}) tree.assemble(ctx, modules) return modules def cmd_defconfig(ctx): configs = ctx.device / 'configs' names = sorted(p.name for p in configs.glob('*_defconfig')) if configs.is_dir() else [] if not ctx.args.name: print('\n'.join(names) or 'no defconfigs') return source = configs / ctx.args.name if not source.is_file(): raise Error(f'{source} not found; available: {", ".join(names) or "none"}') inputs = [] if ctx.schema_version == 2: inputs = [profiles.resolve_path(ctx, entry) for entry in ctx.board.get('kernel', {}).get('config_fragments', [])] inputs.append(source) chunks = [] for path in inputs: if not path.is_file(): raise Error(f'configuration fragment missing: {path}') contents = path.read_text() parse_config(contents) chunks.append(contents.rstrip() + '\n') ctx.work.mkdir(parents=True, exist_ok=True) ctx.config_path.write_text('\n'.join(chunks)) ctx.remember_config(ctx.args.name) log(f'configuration {ctx.args.name} -> {ctx.config_path}') def cmd_fetch(ctx): wanted = set(ctx.args.names) or None soc_commits = {} if ctx.soc and (wanted is None or ctx.soc.name in wanted): previous = ctx.soc.commit soc_commits = feeds.fetch(ctx, {('soc', ctx.soc.name): ctx.soc}, update=ctx.args.update) if soc_commits and ctx.soc.commit != previous: feeds.lock_write(ctx, {**feeds.lock_read(ctx), **soc_commits}) ctx.board = profiles.load(ctx, allow_fetch=True) feeds.update_submodules(ctx) modules = feeds.resolve(ctx) commits = {**soc_commits, **feeds.fetch(ctx, modules, update=ctx.args.update, only=wanted)} if commits: feeds.lock_write(ctx, {**feeds.lock_read(ctx), **commits}) log(f'wrote {ctx.lock_path}') def cmd_build(ctx): ctx.refuse_example() steps = ctx.args.steps if 'all' in steps: steps = list(STEPS) if 'userdata' in steps: userdata.build(ctx, ctx.args.dry_run) steps = [step for step in steps if step != 'userdata'] if not steps: return if 'kernel' in steps: ctx.need_config() if 'firmware' in steps: need_uefi(ctx) # fail before fetching anything modules = prepare(ctx, steps + (['kernel'] if 'prelink' in steps else [])) if 'payload' in steps or 'runtime' in steps: runtime.prepare(ctx, kernel_build='kernel' in steps) elif 'kernel' in steps and ctx.args.data_ramdisk: runtime.bind_seed(ctx, Path(ctx.args.data_ramdisk).expanduser().resolve(), kernel_build=True) if 'kernel' in steps or 'prelink' in steps: with toolchain.environment(ctx): if 'kernel' in steps: kernel.build(ctx, modules) kernel.prelink(ctx) if 'package' in steps: package.build(ctx, modules, ctx.args.target) if 'payload' in steps: ctx.uefi = ctx.option('uefi', 'UNIXNU_UEFI') payload.build(ctx) if 'firmware' in steps: firmware.build(ctx, modules, need_uefi(ctx), package.edk2_dir(ctx, modules), ctx.args.target) def cmd_bootimg(ctx): ctx.refuse_example() uefi = need_uefi(ctx) fd = expand(ctx.args.fd) if ctx.args.fd else firmware.fd_path(ctx, uefi, ctx.args.target) dtb = ctx.option('dtb', 'UNIXNU_DTB') if dtb is None: configured_dtb = ctx.board.get('bootimg', {}).get('dtb') dtb = ctx.tree / configured_dtb if configured_dtb else None ramdisk = None if ctx.args.ramdisk and ctx.args.ramdisk != 'none': ramdisk = expand(ctx.args.ramdisk) elif not ctx.args.ramdisk and payload.paths(ctx)[1].is_file(): ramdisk = payload.paths(ctx)[1] log(f'ramdisk: {ramdisk} (use --ramdisk none for an empty one)') if ramdisk is None: log('ramdisk: empty; build payload or pass --ramdisk FILE to include configured boot entries') bootimg.build(ctx, uefi, fd, dtb, ramdisk, expand(ctx.args.output) if ctx.args.output else None) def cmd_vendor_boot(ctx): stock = expand(ctx.args.stock) ramdisk = None if ctx.args.ramdisk in (None, 'none') else expand(ctx.args.ramdisk) vendor_boot.build(stock, ramdisk, expand(ctx.args.output) if ctx.args.output else None, int(ctx.args.ramdisk_addr, 0) if ctx.args.ramdisk_addr else None, ctx.args.verify_against) def cmd_ipsw(ctx): source = ipsw.Ipsw(ctx) components = ctx.args.components or ctx.board['ipsw'].get('fetch', []) if ctx.args.action == 'info': print(f'{source.manifest["ProductVersion"]} ({source.manifest["ProductBuildVersion"]}) ' f'{source.config["device_class"]} {"remote" if source.remote else "local"}') for name in sorted(source.identity['Manifest']): if source.identity['Manifest'][name].get('Info', {}).get('Path'): path, size, _ = source.info(name) print(f'{size:>12} {name:30} {path}') return for component in components: if component == 'KernelCache': print(ipsw.kernelcache(ctx, source)) else: print(source.extract(component)) def cmd_status(ctx): modules = feeds.resolve(ctx) if ctx.soc: modules[('soc', ctx.soc.name)] = ctx.soc locked = feeds.lock_read(ctx) resolved = [] for (kind, name), module in sorted(modules.items()): revision = module.commit if module.path and not revision: try: revision = git(module.path, 'rev-parse', 'HEAD').strip() except Error: pass spec = getattr(module, 'spec', {}) identity = components.source_identity(ctx, module) resolved.append({'kind': kind, 'name': name, 'path': str(module.path) if module.path else None, 'origin': module.origin, 'local': module.local, 'revision': revision, 'pin': None if module.local else feeds.pinned_revision(ctx, module, locked) or spec.get('revision'), 'scope': spec.get('scope'), 'repository': (module.source or {}).get('git') or ('https://' + identity if identity else spec.get('repository'))}) patch_order = patches.plan(ctx, modules) if ctx.schema_version == 2 else [] if ctx.args.json: print(json.dumps({'schema_version': ctx.schema_version, 'device': str(ctx.device), 'name': ctx.name, 'output': str(ctx.work), 'configuration': ctx.config_name, 'build_config': ctx.args.build_config, 'config_path': str(ctx.config_path), 'configured': ctx.config_path.is_file(), 'profiles': {key: str(value) for key, value in ctx.profile_roots.items()}, 'configuration_sources': [str(path) for path in ctx.profile_files], 'board': ctx.board, 'components': ctx.component_specs, 'modules': resolved, 'patch_order': [str(path) for path in patch_order]}, indent=2)) return print(f'device {ctx.device} ({ctx.name})' + (' [EXAMPLE]' if ctx.example else '')) print(f'output {ctx.work}') if ctx.args.build_config: print(f'inputs {ctx.args.build_config}') print(f'config {ctx.config_path if ctx.config_path.is_file() else "none (run defconfig)"}') if ctx.schema_version == 2: for name, path in ctx.profile_roots.items(): print(f'profile {name:24} {path}') for module in resolved: if module['scope']: print(f'component {module["name"]:23} {module["scope"]}') for index, path in enumerate(patch_order, 1): print(f'patch {index:02} {path}') unpinned = [] for (kind, name), module in sorted(modules.items()): extra = '' if module.origin == 'git': pinned = feeds.pinned_revision(ctx, module, locked) extra = f' pinned {pinned[:12]}' if pinned else ' NOT LOCKED' elif module.local: try: extra = ' (dirty)' if git(module.path, 'status', '--porcelain', '--', '.').strip() else ' (clean)' except Error: extra = ' (not a git work tree)' unpinned.append(f'{kind}:{name}') print(f'{kind:8} {name:24} {module.origin}{extra}') if unpinned: print(f'\nnot reproducible: {len(unpinned)} local source(s) are not pinned by unixnu.lock') def cmd_release(ctx): """deps, patches, build, pack, verify: the fixed steps of the device packaging script (docs/porting/packaging.md).""" ctx.refuse_example() args = ctx.args log('step deps') if args.no_deps: log('deps skipped (--no-deps)') else: args.names, args.update = [], False cmd_fetch(ctx) if ctx.board.get('ipsw', {}).get('url') or getattr(args, 'ipsw', None): args.action, args.components = 'fetch', [] cmd_ipsw(ctx) log('step preflight') args.name = ctx.config_name cmd_defconfig(ctx) modules = feeds.resolve(ctx) if ctx.soc: modules[('soc', ctx.soc.name)] = ctx.soc log('step patches') order = patches.plan(ctx, modules) if ctx.schema_version == 2 else [] for number, path in enumerate(order, 1): log(f'patch {number:02} {path}') if args.check: log(f'check done: {len(order)} patches, configuration {ctx.config_name}, OS {ctx.work.parent.name}') return log('step build') args.steps = list(STEPS) cmd_build(ctx) log('step pack') args.fd = args.ramdisk = args.output = None args.dtb = None cmd_bootimg(ctx) log('step verify') files = pipeline.outputs(ctx) manifest = pipeline.write_manifest(ctx, modules, order, files) pipeline.verify(ctx, files, manifest) log(f'manifest {manifest}') def cmd_clean(ctx): for name in ('tree', 'logs', 'kernel', 'package', 'boot', 'edk2-conf'): shutil.rmtree(ctx.work / name, ignore_errors=True) log(f'cleaned {ctx.work}') def main(argv=None): if sys.version_info < (3, 11): sys.exit('unixnu needs Python 3.11 or newer') if platform.system() == 'Linux': print('unixnu: on Linux the kernel step uses the toolchain of scripts/linux/setup-toolchain.py; ' '--skip-setup and seed-data-grow need macOS hdiutil', file=sys.stderr) elif platform.system() != 'Darwin': print('unixnu: developed and tested on macOS and Linux only; the kernel step needs Xcode or the Linux toolchain, ' 'and --skip-setup needs hdiutil', file=sys.stderr) args = parser().parse_args(argv) if args.command in STEPS + OPTIONAL_STEPS + ('prelink',): args.command = 'build' try: ctx = Context(args) {'defconfig': cmd_defconfig, 'fetch': cmd_fetch, 'tree': lambda c: prepare(c), 'build': cmd_build, 'bootimg': cmd_bootimg, 'vendor_boot': cmd_vendor_boot, 'ipsw': cmd_ipsw, 'status': cmd_status, 'clean': cmd_clean, 'release': cmd_release}[args.command](ctx) except Error as error: sys.exit(f'unixnu: {error}') except KeyboardInterrupt: sys.exit(130)