diff --git a/Dockerfile b/Dockerfile index 046b74a..b3f51e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Note: We use 20.04 since AppImage recommends building on the # oldest configuration that you support -FROM swift:6.0-focal AS build-base +FROM swift:6.1-focal AS build-base RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -17,6 +17,7 @@ RUN apt-get update \ libxml2 \ curl libcurl4-openssl-dev \ zip unzip \ + liblzma-dev zlib1g-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/Package.resolved b/Package.resolved index 8654e97..a70ee8d 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "360bc3ae9f3e8613a4fb4b6583a90d4819977ed68df921d928f49717dad97299", + "originHash" : "7d8342a181303ec948ba0a80e3bdadfdefbde5ca30e750282a9caf834410fae7", "pins" : [ { "identity" : "aexml", @@ -334,6 +334,15 @@ "version" : "1.3.1" } }, + { + "identity" : "unxip", + "kind" : "remoteSourceControl", + "location" : "https://github.com/saagarjha/unxip", + "state" : { + "revision" : "d6f0009109f7502f8f5b3f23dcd5b43cf5b29b32", + "version" : "3.1.0" + } + }, { "identity" : "version", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 1e3c459..3fe0c05 100644 --- a/Package.swift +++ b/Package.swift @@ -51,6 +51,7 @@ let package = Package( .package(url: "https://github.com/attaswift/BigInt", from: "5.5.0"), .package(url: "https://github.com/mxcl/Version", from: "2.1.0"), .package(url: "https://github.com/jpsim/Yams", from: "5.1.3"), + .package(url: "https://github.com/saagarjha/unxip", from: "3.1.0"), // TODO: just depend on tuist/XcodeProj instead .package(url: "https://github.com/yonaskolb/XcodeGen", from: "2.43.0"), @@ -109,6 +110,12 @@ let package = Package( ], cSettings: cSettings ), + .testTarget( + name: "SupersignCLITests", + dependencies: [ + "SupersignCLISupport", + ] + ), .testTarget( name: "SupersignTests", dependencies: [ @@ -133,6 +140,7 @@ let package = Package( .product(name: "SystemPackage", package: "swift-system"), .product(name: "NIOPosix", package: "swift-nio"), .product(name: "Version", package: "Version"), + .product(name: "libunxip", package: "unxip"), ], cSettings: cSettings ), diff --git a/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift b/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift index 78a51d3..4e59a87 100644 --- a/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift +++ b/Sources/SupersignCLISupport/DevCommands/DevSDKCommand.swift @@ -12,11 +12,69 @@ struct DevSDKCommand: AsyncParsableCommand { subcommands: [ DevSDKInstallCommand.self, DevSDKRemoveCommand.self, + DevSDKBuildCommand.self, ], defaultSubcommand: DevSDKInstallCommand.self ) } +struct DevSDKBuildCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "build", + abstract: "Build the Darwin SDK from Xcode.xip" + ) + + @Argument( + help: "Path to Xcode.xip or Xcode.app", + completion: .file(extensions: ["xip", "app"]) + ) + var path: String + + @Argument( + help: "Output directory" + ) + var outputDir: String + + @Option( + help: ArgumentHelp( + "The architecture of the Linux host the SDK is being built for.", + discussion: "Defaults to 'auto', which attempts to match the current host architecture." + ) + ) var arch: ArchSelection = .auto + + func run() async throws { + let builderArch: SDKBuilder.Arch = switch arch { + case .auto: + #if arch(arm64) + .aarch64 + #elseif arch(x86_64) + .x86_64 + #else + throw Console.Error("Could not auto-detect target architecture. Please specify one with '--arch'.") + #endif + case .arm64: .aarch64 + case .x86_64: .x86_64 + } + + let input: SDKBuilder.Input = if path.hasSuffix(".xip") { + .xip(path) + } else if path.hasSuffix(".app") || path.hasSuffix(".app/") { + .app(path) + } else { + throw Console.Error("Expected input path to end in .xip or .app") + } + + let builder = SDKBuilder(input: input, outputPath: outputDir, arch: builderArch) + try await builder.buildSDK() + } +} + +enum ArchSelection: String, ExpressibleByArgument { + case auto + case x86_64 + case arm64 +} + struct DevSDKInstallCommand: AsyncParsableCommand { static let configuration = CommandConfiguration( commandName: "install", diff --git a/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift b/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift new file mode 100644 index 0000000..3ad57ef --- /dev/null +++ b/Sources/SupersignCLISupport/DevCommands/SDKBuilder.swift @@ -0,0 +1,441 @@ +import Foundation +import Dependencies +import SystemPackage +import libunxip +import Supersign // HTTPClient, stdoutSafe +import PackLib // ToolRegistry + +struct SDKBuilder { + enum Arch: String { + case x86_64 + case aarch64 + } + + enum Input { + case xip(String) + case app(String) + } + + let input: Input + let outputPath: String + let arch: Arch + + func buildSDK() async throws { + // tag from https://github.com/kabiroberai/darwin-tools-linux-llvm + let darwinToolsVersion = "1.0.1" + + // TODO: store relevant info for staleness check + let sdkVersion = "develop" + + print("[Cleaning stage]") + + let output = URL(fileURLWithPath: outputPath, isDirectory: true) + .appendingPathComponent("darwin.artifactbundle") + + try? FileManager.default.removeItem(at: output) + try FileManager.default.createDirectory( + at: output, + withIntermediateDirectories: true + ) + + let toolsetDir = output.appendingPathComponent("toolset") + + try FileManager.default.createDirectory( + at: toolsetDir, + withIntermediateDirectories: false + ) + + let pipe = Pipe() + let untar = Process() + untar.currentDirectoryURL = toolsetDir + untar.executableURL = try await ToolRegistry.locate("tar") + untar.arguments = ["xzf", "-"] + untar.standardInput = pipe.fileHandleForReading + async let tarExit: Void = untar.runUntilExit() + + @Dependency(\.httpClient) var httpClient + let url = URL(string: """ + https://github.com/kabiroberai/darwin-tools-linux-llvm/releases/download/\ + v\(darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz + """)! + let (response, body) = try await httpClient.send(HTTPRequest(url: url)) + guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") } + let length: Int64? = switch body.length { + case .known(let known): known + case .unknown: nil + } + let writer = pipe.fileHandleForWriting + var written: Int64 = 0 + for try await chunk in body { + try writer.write(contentsOf: chunk) + written += Int64(chunk.count) + if let length { + let progress = Int(Double(written) / Double(length) * 100) + print("\r[Installing toolchain] \(progress)%", terminator: "") + fflush(stdoutSafe) + } + } + print() + try writer.close() + try await tarExit + + let dev = output.appendingPathComponent("Developer") + + let appDir: URL + let cleanupStageDir: URL? + let wanted: Int? + + switch input { + case .xip(let inputPath): + let devStage = output.appendingPathComponent("DeveloperStage") + try FileManager.default.createDirectory(at: devStage, withIntermediateDirectories: false) + wanted = try await extractXIP(inputPath: inputPath, outDir: devStage.path) + appDir = devStage.appendingPathComponent("Xcode.app") + cleanupStageDir = devStage + case .app(let appPath): + wanted = nil + appDir = URL(fileURLWithPath: appPath) + cleanupStageDir = nil + } + + try FileManager.default.createDirectory(at: dev, withIntermediateDirectories: false) + + var toDoDirs: [String] = ["Contents/Developer"] + var count = 0 + while let next = toDoDirs.popLast() { + try Task.checkCancellation() + + let url = appDir.appendingPathComponent(next) + let contents = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: [.isDirectoryKey]) + for child in contents { + let path = "\(next)/\(child.lastPathComponent)" + guard Self.isWanted(path[...]) else { continue } + + count += 1 + if let wanted { + let progress = Int(Double(count) / Double(wanted) * 100) + print("\r[Installing SDKs] \(progress)%", terminator: "") + fflush(stdoutSafe) + } + if count % 100 == 0 { + if wanted == nil { + print("\r[Installing SDKs] Copied \(count) files", terminator: "") + fflush(stdoutSafe) + } + await Task.yield() + } + + let insideDeveloper = path.dropFirst("Contents/Developer/".count) + guard !insideDeveloper.isEmpty else { continue } + let dest = dev.appendingPathComponent(String(insideDeveloper)) + + if try child.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true { + toDoDirs.append(path) + try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: false) + } else { + try FileManager.default.copyItem(at: child, to: dest) + } + } + } + + if wanted != nil { + // otherwise the last log might say 99% + print("\r[Installing SDKs] 100%", terminator: "") + } + print() + + print("[Cleaning XIP]") + if let cleanupStageDir { + try? FileManager.default.removeItem(at: cleanupStageDir) + } + + func sdk(platform: String, prefix: String) throws -> String { + let regex = try NSRegularExpression(pattern: #"^\#(prefix)\d+\.\d+\.sdk$"#) + let dir = dev.appendingPathComponent("Platforms/\(platform).platform/Developer/SDKs") + let names = try dir.contents().map(\.lastPathComponent) + guard let name = names.first(where: { + regex.firstMatch(in: $0, range: NSRange($0.startIndex..., in: $0)) != nil + }) else { + throw Console.Error("Could not find SDK for \(platform)/\(prefix)") + } + return name + } + + let iPhoneOSSDK = try sdk(platform: "iPhoneOS", prefix: "iPhoneOS") + let iPhoneSimSDK = try sdk(platform: "iPhoneSimulator", prefix: "iPhoneSimulator") + let macOSSDK = try sdk(platform: "MacOSX", prefix: "MacOSX") + + print(""" + [Finalizing SDKs] + - \(iPhoneOSSDK) + - \(iPhoneSimSDK) + - \(macOSSDK) + """) + + /* + XCTest and Testing.framework are located in *.platform/Developer/{Library/Frameworks,usr/lib} rather than inside + the SDK. These search paths are explicitly included when building tests, which presumably ensures that normal + applications don't accidentally link against them in production. + + SwiftPM makes no such affordances outside of macOS, so we add the usr/lib path as include/library search paths + in the SDK config, and symlink the frameworks into the SDKs (since there's no frameworkSearchPaths option). + While this drops a safeguard it's better than not having the testing libs at all. + */ + + for platform in ["iPhoneOS", "MacOSX", "iPhoneSimulator"] { + let lib = "../../../../../Library" + let dest = dev.appendingPathComponent(""" + Platforms/\(platform).platform/Developer/SDKs/\(platform).sdk\ + /System/Library/Frameworks + """).path + + try FileManager.default.createSymbolicLink( + atPath: "\(dest)/Testing.framework", + withDestinationPath: "\(lib)/Frameworks/Testing.framework" + ) + + try FileManager.default.createSymbolicLink( + atPath: "\(dest)/XCTest.framework", + withDestinationPath: "\(lib)/Frameworks/XCTest.framework" + ) + + try FileManager.default.createSymbolicLink( + atPath: "\(dest)/XCUIAutomation.framework", + withDestinationPath: "\(lib)/Frameworks/XCUIAutomation.framework" + ) + + try FileManager.default.createSymbolicLink( + atPath: "\(dest)/XCTestCore.framework", + withDestinationPath: "\(lib)/PrivateFrameworks/XCTestCore.framework" + ) + } + + print("[Writing metadata]") + try """ + { + "schemaVersion": "1.0", + "artifacts": { + "darwin": { + "type": "swiftSDK", + "version": "0.0.1", + "variants": [ + { + "path": ".", + "supportedTriples": ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"] + } + ] + } + } + } + """.write( + to: output.appendingPathComponent("info.json"), + atomically: false, + encoding: .utf8 + ) + + try """ + { + "schemaVersion": "1.0", + "rootPath": "toolset/bin", + "linker": { + "path": "ld64.lld" + }, + "swiftCompiler": { + "extraCLIOptions": [ + "-use-ld=lld" + ] + } + } + """.write( + to: output.appendingPathComponent("toolset.json"), + atomically: false, + encoding: .utf8 + ) + + func triple(platform: String, sdk: String) -> SDKDefinition.Triple { + SDKDefinition.Triple( + sdkRootPath: "Developer/Platforms/\(platform).platform/Developer/SDKs/\(sdk)", + includeSearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"], + librarySearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"], + swiftResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift", + swiftStaticResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static", + toolsetPaths: ["toolset.json"] + ) + } + + let sdkDefinition = SDKDefinition( + schemaVersion: "4.0", + targetTriples: [ + "arm64-apple-ios": triple(platform: "iPhoneOS", sdk: iPhoneOSSDK), + "arm64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK), + "x86_64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK), + "arm64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK), + "x86_64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK), + ] + ) + + let encoder = JSONEncoder() + try encoder + .encode(sdkDefinition) + .write(to: output.appendingPathComponent("swift-sdk.json")) + + try Data("\(sdkVersion)\n".utf8) + .write(to: output.appendingPathComponent("darwin-sdk-version.txt")) + + print("Built SDK at \(output.path)") + } + + // returns the number of files we actually want to keep, + // useful for computing progress % during fs traversal + private func extractXIP(inputPath: String, outDir: String) async throws -> Int { + let fd = try FileDescriptor.open(inputPath, .readOnly) + defer { try? fd.close() } + + let length = try fd.seek(offset: 0, from: .end) + try fd.seek(offset: 0, from: .start) + + // global state, ah well + let oldDirectory = FileManager.default.currentDirectoryPath + guard FileManager.default.changeCurrentDirectoryPath(outDir) else { + throw Console.Error("Could not change directory to '\(outDir)'") + } + defer { FileManager.default.changeCurrentDirectoryPath(oldDirectory) } + + let inputStream = DataReader.data(readingFrom: fd.rawValue) + + let (observer, source) = inputStream.lockstepSplit() + + async let readTask: Void = { + var read = 0 + for try await chunk in observer { + read += chunk.count + let progress = Int(Double(read) / Double(length) * 100) + print("\r[Extracting XIP] \(progress)%", terminator: "") + fflush(stdoutSafe) + if read == length { break } + } + }() + + let xipToChunks = XIP.transform( + DataReader(data: source), + options: nil + ) + + // ideally we would filter out the files we don't want at this stage, + // speeding up extraction AND entirely avoiding the "Installing SDKs" + // post-processing step. However, files in xip archives may be hardlinks + // to one another and so we need to handle the case where a file inside + // our filter() points to a file outside of it. We might be able to do this + // with a double-pass system. + + let chunksToFiles = Chunks.transform( + xipToChunks, + options: nil + ) + + let filesToDisk = Files.transform( + chunksToFiles, + options: .init( + compress: false, + dryRun: false + ) + ) + + var wanted = 0 + for try await file in filesToDisk { + wanted += Self.isWanted(file.name[...]) ? 1 : 0 + } + _ = try await readTask + + print() + + return wanted + } + + private static func isWanted(_ path: Substring) -> Bool { + var components = path.split(separator: "/")[...] + if components.first == "." { + components.removeFirst() + } + if components.first?.hasSuffix(".app") == true { + components.removeFirst() + } + guard SDKEntry.wanted.matches(components) else { return false } + + // TODO: see if we can exclude most dylibs. Seems we need the XCTest ones though. + + if components.count >= 10 && components[components.startIndex + 9] == "prebuilt-modules" && components.starts( + with: "Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift" + .split(separator: "/") + ) { + return false + } + + return true + } +} + +struct SDKDefinition: Encodable { + struct Triple: Encodable { + var sdkRootPath: String + var includeSearchPaths: [String] + var librarySearchPaths: [String] + var swiftResourcesPath: String + var swiftStaticResourcesPath: String + var toolsetPaths: [String] + } + var schemaVersion: String + var targetTriples: [String: Triple] +} + +struct SDKEntry { + var names: Set + var values: [SDKEntry] = [] + + // empty = wildcard + init(_ names: Set, _ values: [SDKEntry] = []) { + self.names = names + self.values = values + } + + init(_ name: Substring, _ values: [SDKEntry] = []) { + self.init([name], values) + } + + func matches(_ path: ArraySlice) -> Bool { + guard let first = path.first else { return true } + guard names.isEmpty || names.contains(first) else { return false } + if values.isEmpty { return true } // leaf, everything after is good + let afterName = path.dropFirst() + for value in values { + if value.matches(afterName) { + return true + } + } + return false + } + + static func E(_ name: Substring?, _ values: [SDKEntry] = []) -> SDKEntry { + guard let name else { return SDKEntry([], values) } + let parts = name.split(separator: "/").reversed() + return parts.dropFirst().reduce(SDKEntry(parts.first!, values)) { SDKEntry($1, [$0]) } + } + + static let wanted = E("Contents/Developer", [ + E("Toolchains/XcodeDefault.xctoolchain/usr/lib", [ + E("swift"), + E("swift_static"), + E("clang"), + ]), + E("Platforms", ["iPhoneOS", "MacOSX", "iPhoneSimulator"].map { + E("\($0).platform/Developer", [ + E("SDKs"), + E("Library", [ + E("Frameworks"), + E("PrivateFrameworks"), + ]), + E("usr/lib"), + ]) + }), + ]) +} diff --git a/Tests/SupersignCLITests/MatcherTests.swift b/Tests/SupersignCLITests/MatcherTests.swift new file mode 100644 index 0000000..903e9a8 --- /dev/null +++ b/Tests/SupersignCLITests/MatcherTests.swift @@ -0,0 +1,52 @@ +import Testing +@testable import SupersignCLISupport + +extension SDKEntry { + fileprivate static let test = E("Root", [ + E("Intermediate", [ + E("Leaf"), + ]), + E("AnotherLeaf"), + E("BeforeWildcard", [ + E(nil, [ + E("AfterWildcard"), + ]), + ]), + ]) +} + +@Test func testMatcherSupportsSlices() { + let s = SDKEntry.test + + let slice0 = ["Root"].map { $0[...] }[0...] + #expect(s.matches(slice0)) + + let slice1 = ["foo", "bar", "Root"].map { $0[...] }[2...] + #expect(s.matches(slice1)) + + let slice2 = ["foo", "bar", "NotRoot"].map { $0[...] }[2...] + #expect(!s.matches(slice2)) +} + +@Test func testMatcherLeaves() { + let s = SDKEntry.test + + #expect(s.matches(["Root", "Intermediate"])) + #expect(!s.matches(["Root", "NotIntermediate"])) + + #expect(s.matches(["Root", "Intermediate", "Leaf"])) + #expect(!s.matches(["Root", "Intermediate", "NotLeaf"])) + #expect(s.matches(["Root", "Intermediate", "Leaf", "AfterLeaf"])) + + #expect(s.matches(["Root", "AnotherLeaf"])) + #expect(s.matches(["Root", "AnotherLeaf", "AfterAnotherLeaf"])) + + #expect(!s.matches(["Junk", "Root"])) + + #expect(s.matches(["Root", "BeforeWildcard"])) + #expect(s.matches(["Root", "BeforeWildcard", "Blah"])) + #expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard"])) + #expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard", "Suffix"])) + #expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard"])) + #expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard", "Suffix"])) +}