SDK builder initial pass

This commit is contained in:
Kabir Oberai
2025-04-24 18:58:14 -04:00
Unverified
parent 3c5fe13244
commit baf71fa1da
6 changed files with 571 additions and 2 deletions
+2 -1
View File
@@ -1,7 +1,7 @@
# Note: We use 20.04 since AppImage recommends building on the
# oldest configuration that you support
FROM swift:6.0-focal AS build-base
FROM swift:6.1-focal AS build-base
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
@@ -17,6 +17,7 @@ RUN apt-get update \
libxml2 \
curl libcurl4-openssl-dev \
zip unzip \
liblzma-dev zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
+10 -1
View File
@@ -1,5 +1,5 @@
{
"originHash" : "360bc3ae9f3e8613a4fb4b6583a90d4819977ed68df921d928f49717dad97299",
"originHash" : "7d8342a181303ec948ba0a80e3bdadfdefbde5ca30e750282a9caf834410fae7",
"pins" : [
{
"identity" : "aexml",
@@ -334,6 +334,15 @@
"version" : "1.3.1"
}
},
{
"identity" : "unxip",
"kind" : "remoteSourceControl",
"location" : "https://github.com/saagarjha/unxip",
"state" : {
"revision" : "d6f0009109f7502f8f5b3f23dcd5b43cf5b29b32",
"version" : "3.1.0"
}
},
{
"identity" : "version",
"kind" : "remoteSourceControl",
+8
View File
@@ -51,6 +51,7 @@ let package = Package(
.package(url: "https://github.com/attaswift/BigInt", from: "5.5.0"),
.package(url: "https://github.com/mxcl/Version", from: "2.1.0"),
.package(url: "https://github.com/jpsim/Yams", from: "5.1.3"),
.package(url: "https://github.com/saagarjha/unxip", from: "3.1.0"),
// TODO: just depend on tuist/XcodeProj instead
.package(url: "https://github.com/yonaskolb/XcodeGen", from: "2.43.0"),
@@ -109,6 +110,12 @@ let package = Package(
],
cSettings: cSettings
),
.testTarget(
name: "SupersignCLITests",
dependencies: [
"SupersignCLISupport",
]
),
.testTarget(
name: "SupersignTests",
dependencies: [
@@ -133,6 +140,7 @@ let package = Package(
.product(name: "SystemPackage", package: "swift-system"),
.product(name: "NIOPosix", package: "swift-nio"),
.product(name: "Version", package: "Version"),
.product(name: "libunxip", package: "unxip"),
],
cSettings: cSettings
),
@@ -12,11 +12,69 @@ struct DevSDKCommand: AsyncParsableCommand {
subcommands: [
DevSDKInstallCommand.self,
DevSDKRemoveCommand.self,
DevSDKBuildCommand.self,
],
defaultSubcommand: DevSDKInstallCommand.self
)
}
struct DevSDKBuildCommand: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "build",
abstract: "Build the Darwin SDK from Xcode.xip"
)
@Argument(
help: "Path to Xcode.xip or Xcode.app",
completion: .file(extensions: ["xip", "app"])
)
var path: String
@Argument(
help: "Output directory"
)
var outputDir: String
@Option(
help: ArgumentHelp(
"The architecture of the Linux host the SDK is being built for.",
discussion: "Defaults to 'auto', which attempts to match the current host architecture."
)
) var arch: ArchSelection = .auto
func run() async throws {
let builderArch: SDKBuilder.Arch = switch arch {
case .auto:
#if arch(arm64)
.aarch64
#elseif arch(x86_64)
.x86_64
#else
throw Console.Error("Could not auto-detect target architecture. Please specify one with '--arch'.")
#endif
case .arm64: .aarch64
case .x86_64: .x86_64
}
let input: SDKBuilder.Input = if path.hasSuffix(".xip") {
.xip(path)
} else if path.hasSuffix(".app") || path.hasSuffix(".app/") {
.app(path)
} else {
throw Console.Error("Expected input path to end in .xip or .app")
}
let builder = SDKBuilder(input: input, outputPath: outputDir, arch: builderArch)
try await builder.buildSDK()
}
}
enum ArchSelection: String, ExpressibleByArgument {
case auto
case x86_64
case arm64
}
struct DevSDKInstallCommand: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "install",
@@ -0,0 +1,441 @@
import Foundation
import Dependencies
import SystemPackage
import libunxip
import Supersign // HTTPClient, stdoutSafe
import PackLib // ToolRegistry
struct SDKBuilder {
enum Arch: String {
case x86_64
case aarch64
}
enum Input {
case xip(String)
case app(String)
}
let input: Input
let outputPath: String
let arch: Arch
func buildSDK() async throws {
// tag from https://github.com/kabiroberai/darwin-tools-linux-llvm
let darwinToolsVersion = "1.0.1"
// TODO: store relevant info for staleness check
let sdkVersion = "develop"
print("[Cleaning stage]")
let output = URL(fileURLWithPath: outputPath, isDirectory: true)
.appendingPathComponent("darwin.artifactbundle")
try? FileManager.default.removeItem(at: output)
try FileManager.default.createDirectory(
at: output,
withIntermediateDirectories: true
)
let toolsetDir = output.appendingPathComponent("toolset")
try FileManager.default.createDirectory(
at: toolsetDir,
withIntermediateDirectories: false
)
let pipe = Pipe()
let untar = Process()
untar.currentDirectoryURL = toolsetDir
untar.executableURL = try await ToolRegistry.locate("tar")
untar.arguments = ["xzf", "-"]
untar.standardInput = pipe.fileHandleForReading
async let tarExit: Void = untar.runUntilExit()
@Dependency(\.httpClient) var httpClient
let url = URL(string: """
https://github.com/kabiroberai/darwin-tools-linux-llvm/releases/download/\
v\(darwinToolsVersion)/toolset-\(arch.rawValue).tar.gz
""")!
let (response, body) = try await httpClient.send(HTTPRequest(url: url))
guard response.status == 200, let body else { throw Console.Error("Could not fetch toolset") }
let length: Int64? = switch body.length {
case .known(let known): known
case .unknown: nil
}
let writer = pipe.fileHandleForWriting
var written: Int64 = 0
for try await chunk in body {
try writer.write(contentsOf: chunk)
written += Int64(chunk.count)
if let length {
let progress = Int(Double(written) / Double(length) * 100)
print("\r[Installing toolchain] \(progress)%", terminator: "")
fflush(stdoutSafe)
}
}
print()
try writer.close()
try await tarExit
let dev = output.appendingPathComponent("Developer")
let appDir: URL
let cleanupStageDir: URL?
let wanted: Int?
switch input {
case .xip(let inputPath):
let devStage = output.appendingPathComponent("DeveloperStage")
try FileManager.default.createDirectory(at: devStage, withIntermediateDirectories: false)
wanted = try await extractXIP(inputPath: inputPath, outDir: devStage.path)
appDir = devStage.appendingPathComponent("Xcode.app")
cleanupStageDir = devStage
case .app(let appPath):
wanted = nil
appDir = URL(fileURLWithPath: appPath)
cleanupStageDir = nil
}
try FileManager.default.createDirectory(at: dev, withIntermediateDirectories: false)
var toDoDirs: [String] = ["Contents/Developer"]
var count = 0
while let next = toDoDirs.popLast() {
try Task.checkCancellation()
let url = appDir.appendingPathComponent(next)
let contents = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: [.isDirectoryKey])
for child in contents {
let path = "\(next)/\(child.lastPathComponent)"
guard Self.isWanted(path[...]) else { continue }
count += 1
if let wanted {
let progress = Int(Double(count) / Double(wanted) * 100)
print("\r[Installing SDKs] \(progress)%", terminator: "")
fflush(stdoutSafe)
}
if count % 100 == 0 {
if wanted == nil {
print("\r[Installing SDKs] Copied \(count) files", terminator: "")
fflush(stdoutSafe)
}
await Task.yield()
}
let insideDeveloper = path.dropFirst("Contents/Developer/".count)
guard !insideDeveloper.isEmpty else { continue }
let dest = dev.appendingPathComponent(String(insideDeveloper))
if try child.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true {
toDoDirs.append(path)
try FileManager.default.createDirectory(at: dest, withIntermediateDirectories: false)
} else {
try FileManager.default.copyItem(at: child, to: dest)
}
}
}
if wanted != nil {
// otherwise the last log might say 99%
print("\r[Installing SDKs] 100%", terminator: "")
}
print()
print("[Cleaning XIP]")
if let cleanupStageDir {
try? FileManager.default.removeItem(at: cleanupStageDir)
}
func sdk(platform: String, prefix: String) throws -> String {
let regex = try NSRegularExpression(pattern: #"^\#(prefix)\d+\.\d+\.sdk$"#)
let dir = dev.appendingPathComponent("Platforms/\(platform).platform/Developer/SDKs")
let names = try dir.contents().map(\.lastPathComponent)
guard let name = names.first(where: {
regex.firstMatch(in: $0, range: NSRange($0.startIndex..., in: $0)) != nil
}) else {
throw Console.Error("Could not find SDK for \(platform)/\(prefix)")
}
return name
}
let iPhoneOSSDK = try sdk(platform: "iPhoneOS", prefix: "iPhoneOS")
let iPhoneSimSDK = try sdk(platform: "iPhoneSimulator", prefix: "iPhoneSimulator")
let macOSSDK = try sdk(platform: "MacOSX", prefix: "MacOSX")
print("""
[Finalizing SDKs]
- \(iPhoneOSSDK)
- \(iPhoneSimSDK)
- \(macOSSDK)
""")
/*
XCTest and Testing.framework are located in *.platform/Developer/{Library/Frameworks,usr/lib} rather than inside
the SDK. These search paths are explicitly included when building tests, which presumably ensures that normal
applications don't accidentally link against them in production.
SwiftPM makes no such affordances outside of macOS, so we add the usr/lib path as include/library search paths
in the SDK config, and symlink the frameworks into the SDKs (since there's no frameworkSearchPaths option).
While this drops a safeguard it's better than not having the testing libs at all.
*/
for platform in ["iPhoneOS", "MacOSX", "iPhoneSimulator"] {
let lib = "../../../../../Library"
let dest = dev.appendingPathComponent("""
Platforms/\(platform).platform/Developer/SDKs/\(platform).sdk\
/System/Library/Frameworks
""").path
try FileManager.default.createSymbolicLink(
atPath: "\(dest)/Testing.framework",
withDestinationPath: "\(lib)/Frameworks/Testing.framework"
)
try FileManager.default.createSymbolicLink(
atPath: "\(dest)/XCTest.framework",
withDestinationPath: "\(lib)/Frameworks/XCTest.framework"
)
try FileManager.default.createSymbolicLink(
atPath: "\(dest)/XCUIAutomation.framework",
withDestinationPath: "\(lib)/Frameworks/XCUIAutomation.framework"
)
try FileManager.default.createSymbolicLink(
atPath: "\(dest)/XCTestCore.framework",
withDestinationPath: "\(lib)/PrivateFrameworks/XCTestCore.framework"
)
}
print("[Writing metadata]")
try """
{
"schemaVersion": "1.0",
"artifacts": {
"darwin": {
"type": "swiftSDK",
"version": "0.0.1",
"variants": [
{
"path": ".",
"supportedTriples": ["aarch64-unknown-linux-gnu", "x86_64-unknown-linux-gnu"]
}
]
}
}
}
""".write(
to: output.appendingPathComponent("info.json"),
atomically: false,
encoding: .utf8
)
try """
{
"schemaVersion": "1.0",
"rootPath": "toolset/bin",
"linker": {
"path": "ld64.lld"
},
"swiftCompiler": {
"extraCLIOptions": [
"-use-ld=lld"
]
}
}
""".write(
to: output.appendingPathComponent("toolset.json"),
atomically: false,
encoding: .utf8
)
func triple(platform: String, sdk: String) -> SDKDefinition.Triple {
SDKDefinition.Triple(
sdkRootPath: "Developer/Platforms/\(platform).platform/Developer/SDKs/\(sdk)",
includeSearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"],
librarySearchPaths: ["Developer/Platforms/\(platform).platform/Developer/usr/lib"],
swiftResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift",
swiftStaticResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static",
toolsetPaths: ["toolset.json"]
)
}
let sdkDefinition = SDKDefinition(
schemaVersion: "4.0",
targetTriples: [
"arm64-apple-ios": triple(platform: "iPhoneOS", sdk: iPhoneOSSDK),
"arm64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK),
"x86_64-apple-ios-simulator": triple(platform: "iPhoneSimulator", sdk: iPhoneSimSDK),
"arm64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK),
"x86_64-apple-macosx": triple(platform: "MacOSX", sdk: macOSSDK),
]
)
let encoder = JSONEncoder()
try encoder
.encode(sdkDefinition)
.write(to: output.appendingPathComponent("swift-sdk.json"))
try Data("\(sdkVersion)\n".utf8)
.write(to: output.appendingPathComponent("darwin-sdk-version.txt"))
print("Built SDK at \(output.path)")
}
// returns the number of files we actually want to keep,
// useful for computing progress % during fs traversal
private func extractXIP(inputPath: String, outDir: String) async throws -> Int {
let fd = try FileDescriptor.open(inputPath, .readOnly)
defer { try? fd.close() }
let length = try fd.seek(offset: 0, from: .end)
try fd.seek(offset: 0, from: .start)
// global state, ah well
let oldDirectory = FileManager.default.currentDirectoryPath
guard FileManager.default.changeCurrentDirectoryPath(outDir) else {
throw Console.Error("Could not change directory to '\(outDir)'")
}
defer { FileManager.default.changeCurrentDirectoryPath(oldDirectory) }
let inputStream = DataReader.data(readingFrom: fd.rawValue)
let (observer, source) = inputStream.lockstepSplit()
async let readTask: Void = {
var read = 0
for try await chunk in observer {
read += chunk.count
let progress = Int(Double(read) / Double(length) * 100)
print("\r[Extracting XIP] \(progress)%", terminator: "")
fflush(stdoutSafe)
if read == length { break }
}
}()
let xipToChunks = XIP.transform(
DataReader(data: source),
options: nil
)
// ideally we would filter out the files we don't want at this stage,
// speeding up extraction AND entirely avoiding the "Installing SDKs"
// post-processing step. However, files in xip archives may be hardlinks
// to one another and so we need to handle the case where a file inside
// our filter() points to a file outside of it. We might be able to do this
// with a double-pass system.
let chunksToFiles = Chunks.transform(
xipToChunks,
options: nil
)
let filesToDisk = Files.transform(
chunksToFiles,
options: .init(
compress: false,
dryRun: false
)
)
var wanted = 0
for try await file in filesToDisk {
wanted += Self.isWanted(file.name[...]) ? 1 : 0
}
_ = try await readTask
print()
return wanted
}
private static func isWanted(_ path: Substring) -> Bool {
var components = path.split(separator: "/")[...]
if components.first == "." {
components.removeFirst()
}
if components.first?.hasSuffix(".app") == true {
components.removeFirst()
}
guard SDKEntry.wanted.matches(components) else { return false }
// TODO: see if we can exclude most dylibs. Seems we need the XCTest ones though.
if components.count >= 10 && components[components.startIndex + 9] == "prebuilt-modules" && components.starts(
with: "Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift"
.split(separator: "/")
) {
return false
}
return true
}
}
struct SDKDefinition: Encodable {
struct Triple: Encodable {
var sdkRootPath: String
var includeSearchPaths: [String]
var librarySearchPaths: [String]
var swiftResourcesPath: String
var swiftStaticResourcesPath: String
var toolsetPaths: [String]
}
var schemaVersion: String
var targetTriples: [String: Triple]
}
struct SDKEntry {
var names: Set<Substring>
var values: [SDKEntry] = []
// empty = wildcard
init(_ names: Set<Substring>, _ values: [SDKEntry] = []) {
self.names = names
self.values = values
}
init(_ name: Substring, _ values: [SDKEntry] = []) {
self.init([name], values)
}
func matches(_ path: ArraySlice<Substring>) -> Bool {
guard let first = path.first else { return true }
guard names.isEmpty || names.contains(first) else { return false }
if values.isEmpty { return true } // leaf, everything after is good
let afterName = path.dropFirst()
for value in values {
if value.matches(afterName) {
return true
}
}
return false
}
static func E(_ name: Substring?, _ values: [SDKEntry] = []) -> SDKEntry {
guard let name else { return SDKEntry([], values) }
let parts = name.split(separator: "/").reversed()
return parts.dropFirst().reduce(SDKEntry(parts.first!, values)) { SDKEntry($1, [$0]) }
}
static let wanted = E("Contents/Developer", [
E("Toolchains/XcodeDefault.xctoolchain/usr/lib", [
E("swift"),
E("swift_static"),
E("clang"),
]),
E("Platforms", ["iPhoneOS", "MacOSX", "iPhoneSimulator"].map {
E("\($0).platform/Developer", [
E("SDKs"),
E("Library", [
E("Frameworks"),
E("PrivateFrameworks"),
]),
E("usr/lib"),
])
}),
])
}
@@ -0,0 +1,52 @@
import Testing
@testable import SupersignCLISupport
extension SDKEntry {
fileprivate static let test = E("Root", [
E("Intermediate", [
E("Leaf"),
]),
E("AnotherLeaf"),
E("BeforeWildcard", [
E(nil, [
E("AfterWildcard"),
]),
]),
])
}
@Test func testMatcherSupportsSlices() {
let s = SDKEntry.test
let slice0 = ["Root"].map { $0[...] }[0...]
#expect(s.matches(slice0))
let slice1 = ["foo", "bar", "Root"].map { $0[...] }[2...]
#expect(s.matches(slice1))
let slice2 = ["foo", "bar", "NotRoot"].map { $0[...] }[2...]
#expect(!s.matches(slice2))
}
@Test func testMatcherLeaves() {
let s = SDKEntry.test
#expect(s.matches(["Root", "Intermediate"]))
#expect(!s.matches(["Root", "NotIntermediate"]))
#expect(s.matches(["Root", "Intermediate", "Leaf"]))
#expect(!s.matches(["Root", "Intermediate", "NotLeaf"]))
#expect(s.matches(["Root", "Intermediate", "Leaf", "AfterLeaf"]))
#expect(s.matches(["Root", "AnotherLeaf"]))
#expect(s.matches(["Root", "AnotherLeaf", "AfterAnotherLeaf"]))
#expect(!s.matches(["Junk", "Root"]))
#expect(s.matches(["Root", "BeforeWildcard"]))
#expect(s.matches(["Root", "BeforeWildcard", "Blah"]))
#expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard"]))
#expect(s.matches(["Root", "BeforeWildcard", "Blah", "AfterWildcard", "Suffix"]))
#expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard"]))
#expect(!s.matches(["Root", "BeforeWildcard", "Blah", "NotAfterWildcard", "Suffix"]))
}