SWBBuild SDK support

This commit is contained in:
Kabir Oberai
2026-05-25 05:55:37 -04:00
Unverified
parent c3bb2e5428
commit 4815e855b9
10 changed files with 333 additions and 150 deletions
+1
View File
@@ -166,6 +166,7 @@ let package = Package(
name: "PackLib",
dependencies: [
"XUtils",
.product(name: "Superutils", package: "xtool-core"),
.product(name: "Yams", package: "Yams"),
.product(name: "XcodeGenKit", package: "XcodeGen", condition: .when(platforms: [.macOS])),
]
+115 -29
View File
@@ -1,37 +1,73 @@
import Foundation
import Subprocess
import XUtils
import Superutils
public struct BuildSettings: Sendable {
private static let customBinDir =
// this is the same option used by SwiftPM itself for dev builds
ProcessInfo.processInfo.environment["SWIFTPM_CUSTOM_BIN_DIR"].map { FilePath($0) }
private static let envURL = URL(fileURLWithPath: "/usr/bin/env")
public let packagePath: String
public var packagePath: String
public let configuration: BuildConfiguration
public let triple: String
public let sdkOptions: [String]
public let options: [String]
public let buildSystem: BuildSystem
public let customOptions: [String]
public var sdkOptions: [String]
public var sdkEnvironment: [Environment.Key: String?]
private var configOptions: [String] {
return [
"--configuration", configuration.rawValue,
"--build-system", buildSystem.pmName,
"--package-path", packagePath,
]
}
private var resolvedBaseOptions: [String] {
configOptions + sdkOptions + customOptions
}
public init(
configuration: BuildConfiguration,
triple: String,
buildSystem: BuildSystem = .default,
packagePath: String = ".",
options: [String] = []
) async throws {
self.packagePath = packagePath
self.configuration = configuration
self.options = options
self.customOptions = options
self.triple = triple
self.buildSystem = buildSystem
// on macOS we don't explicitly install a Swift SDK but
// SwiftPM vends "implicit" Darwin SDKs as of Swift 6.1,
// i.e. we can pass `--swift-sdk arm64-apple-ios` and it
// just works. See:
// https://github.com/swiftlang/swift-package-manager/pull/6828
self.sdkOptions = ["--swift-sdk", triple]
self.sdkEnvironment = [
// xcrun passes an SDKROOT that messes with our sdk configuration
"SDKROOT": nil,
]
switch buildSystem {
case .swiftPM:
// on macOS we don't explicitly install a Swift SDK but
// SwiftPM vends "implicit" Darwin SDKs as of Swift 6.1,
// i.e. we can pass `--swift-sdk arm64-apple-ios` and it
// just works. See:
// https://github.com/swiftlang/swift-package-manager/pull/6828
self.sdkOptions = ["--swift-sdk", triple]
case .swiftBuild:
self.sdkOptions = ["--triple", triple]
#if !os(macOS)
let darwinSDK = try await DarwinSDK.current()
.orThrow(StringError("No Darwin SDK configured. Please run `xtool setup`."))
self.sdkOptions += [
"--toolset", "\(darwinSDK.bundle.path)/toolset-swb.json",
]
self.sdkEnvironment.merge([
"XCODE_EXTRA_PLATFORM_FOLDERS": "\(darwinSDK.bundle.path)/Developer/Platforms",
]) { $1 }
#endif
}
}
#if os(macOS)
@@ -44,15 +80,40 @@ public struct BuildSettings: Sendable {
return result.standardOutput?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
}
private static let swiftURL = Task {
private static let _swiftURL = Task {
try await FilePath(xcrun(["-f", "swift"]))
}
public static func swiftURL() async throws -> FilePath {
try await _swiftURL.value
}
private static let _swiftcURL = Task {
try await FilePath(xcrun(["-f", "swiftc"]))
}
public static func swiftcURL() async throws -> FilePath {
try await _swiftcURL.value
}
#else
public static func swiftURL() async throws -> FilePath {
try await FilePath(ToolRegistry.locate("swift")).orThrow(StringError("Got bad path for swift executable"))
}
public static func swiftcURL() async throws -> FilePath {
try await FilePath(ToolRegistry.locate("swiftc")).orThrow(StringError("Got bad path for swiftc executable"))
}
#endif
public func withPackagePath(_ path: String) -> Self {
var copy = self
copy.packagePath = path
return copy
}
public func swiftPMInvocation(
forTool tool: String,
arguments: [String],
packagePathOverride: String? = nil
) async throws -> Subprocess.Configuration {
let executable: Executable
let baseArguments: [String]
@@ -65,35 +126,60 @@ public struct BuildSettings: Sendable {
// to add SDKROOT=.../MacOSX.sdk to our invocations. We avoid this by
// 1) invoking the real swift executable (located with `xcrun -f`) and
// 2) explicitly removing SDKROOT from the env, as it may be inherited
// through the `swift run pack` invocation.
executable = .path(try await Self.swiftURL.value)
// through a parent process (e.g. `swift run xtool`).
executable = .path(try await Self.swiftURL())
#else
executable = .name("swift")
#endif
baseArguments = [tool]
}
let extraArguments: [String] = [
"--package-path", packagePathOverride ?? packagePath,
"--configuration", configuration.rawValue,
]
var rawEnv = ProcessInfo.processInfo.environment
rawEnv.removeValue(forKey: "SDKROOT")
let env = Dictionary(uniqueKeysWithValues: rawEnv.map {
(Environment.Key(rawValue: $0)!, $1)
})
return Configuration(
executable,
arguments: .init(baseArguments + extraArguments + sdkOptions + options + arguments),
environment: .custom(env),
arguments: .init(baseArguments + resolvedBaseOptions + arguments),
environment: .inherit.updating(sdkEnvironment),
platformOptions: .withGracefulShutDown,
)
}
public var buildServerArguments: [String] {
return [
"package", "experimental-build-server",
"--disable-automatic-resolution",
// TODO: once https://github.com/swiftlang/swift-package-manager/pull/9819 makes it into a release
// (Swift 6.4), pass --experimental-skip-acquiring-lock
] + resolvedBaseOptions
}
}
public enum BuildConfiguration: String, CaseIterable, Sendable {
case debug
case release
var swiftBuildValue: String {
switch self {
case .debug: "Debug"
case .release: "Release"
}
}
}
public enum BuildSystem: Sendable {
case swiftPM
case swiftBuild
public static var `default`: Self {
#if os(macOS)
return .swiftBuild
#else
return .swiftBuild
#endif
}
var pmName: String {
switch self {
case .swiftPM: "native"
case .swiftBuild: "swiftbuild"
}
}
}
+95
View File
@@ -0,0 +1,95 @@
import Foundation
import Subprocess
import XUtils
public struct DarwinSDK {
public let bundle: URL
public let version: String
public init?(bundle: URL) {
self.bundle = bundle
if let version = try? Data(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) {
self.version = String(decoding: version, as: UTF8.self)
.trimmingCharacters(in: .whitespacesAndNewlines)
} else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) {
self.version = "unknown"
} else {
return nil
}
}
public static func install(from path: String) async throws {
// we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory
// location depends on factors like $XDG_CONFIG_HOME. Rather than replicating
// SwiftPM's logic, which may change, it's more reliable to directly invoke
// `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40
let url = URL(fileURLWithPath: path)
guard DarwinSDK(bundle: url) != nil else { throw StringError("Invalid Darwin SDK at '\(path)'")}
try await addHostClangResourceDir(to: url)
try await Subprocess.run(
.name("swift"),
arguments: ["sdk", "install", url.path],
output: .discarded
)
.checkSuccess()
}
private static func addHostClangResourceDir(to sdk: URL) async throws {
let clangURL = try await ToolRegistry.locate("clang")
let process = try await Subprocess.run(
.path(FilePath(clangURL.path)),
arguments: ["-print-resource-dir"],
output: .string(limit: .max)
).checkSuccess()
let output = process.standardOutput ?? ""
let hostClangResources = URL(filePath: output.trimmingCharacters(in: .whitespacesAndNewlines))
let hostInclude = hostClangResources.appending(path: "include")
let sdkInclude = sdk.appending(path: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/clang/include")
try FileManager.default.copyItem(at: hostInclude, to: sdkInclude)
}
public static func current() async throws -> DarwinSDK? {
let outputString: String
do {
outputString = try await Subprocess.run(
.name("swift"),
arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"],
output: .string(limit: .max)
)
.checkSuccess()
.standardOutput
?? ""
} catch SubprocessFailure.exited {
return nil
}
// should be something like
// swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift
// swiftlint:disable:previous line_length
let resourcesPathPrefix = "swiftResourcesPath: "
guard let resourcesPath = outputString
.split(separator: "\n")
.first(where: { $0.hasPrefix(resourcesPathPrefix) })?
.dropFirst(resourcesPathPrefix.count)
else { return nil }
var resourcesURL = URL(fileURLWithPath: String(resourcesPath))
for _ in 0..<6 {
resourcesURL = resourcesURL.deletingLastPathComponent()
}
return DarwinSDK(bundle: resourcesURL)
}
public func isUpToDate() -> Bool {
true
}
public func remove() throws {
try FileManager.default.removeItem(at: bundle)
}
}
+24 -15
View File
@@ -55,20 +55,21 @@ public struct Packer: Sendable {
try Data().write(to: sources.appendingPathComponent("stub.c", isDirectory: false))
}
let buildConfig = try await buildSettings.swiftPMInvocation(
forTool: "build",
arguments: [
"--package-path", packageDir.path,
"--scratch-path", ".build",
// resolving can cause SwiftPM to overwrite the root package deps
// with just the deps needed for the builder package (which is to
// say, any "dev dependencies" of the root package may be removed.)
// fortunately we've already resolved the root package by this point
// in order to dump the plan, so we can skip resolution here to skirt
// the issue.
"--disable-automatic-resolution",
]
)
let buildConfig = try await buildSettings
.withPackagePath(packageDir.path)
.swiftPMInvocation(
forTool: "build",
arguments: [
"--scratch-path", ".build",
// resolving can cause SwiftPM to overwrite the root package deps
// with just the deps needed for the builder package (which is to
// say, any "dev dependencies" of the root package may be removed.)
// fortunately we've already resolved the root package by this point
// in order to dump the plan, so we can skip resolution here to skirt
// the issue.
"--disable-automatic-resolution",
],
)
try await Subprocess.run(
buildConfig,
output: .standardError,
@@ -84,8 +85,16 @@ public struct Packer: Sendable {
let outputURL = output.url
let binPath: String
switch buildSettings.buildSystem {
case .swiftPM:
binPath = "\(buildSettings.triple)/\(buildSettings.configuration.rawValue)"
case .swiftBuild:
let platformName = buildSettings.triple.contains("simulator") ? "iphonesimulator" : "iphoneos"
binPath = "out/Products/\(buildSettings.configuration.swiftBuildValue)-\(platformName)"
}
let binDir = URL(
fileURLWithPath: ".build/\(buildSettings.triple)/\(buildSettings.configuration.rawValue)",
fileURLWithPath: ".build/\(binPath)",
isDirectory: true
)
+1 -2
View File
@@ -244,10 +244,9 @@ public struct Planner: Sendable {
}
private func _dumpAction(arguments: [String], path: String) async throws -> Data {
let dumpConfig = try await buildSettings.swiftPMInvocation(
let dumpConfig = try await buildSettings.withPackagePath(path).swiftPMInvocation(
forTool: "package",
arguments: arguments,
packagePathOverride: path
)
return try await Subprocess.run(
dumpConfig,
+30
View File
@@ -0,0 +1,30 @@
import ArgumentParser
import Foundation
import XKit
import PackLib
import Subprocess
struct DevBSPCommand: AsyncParsableCommand {
static let configuration = CommandConfiguration(
commandName: "build-server",
abstract: "Run build server",
)
@Option
var triple: String?
func run() async throws {
let settings = try await BuildSettings(
configuration: .debug,
triple: triple ?? PackOperation.defaultTriple
)
try await Subprocess.run(
.path(BuildSettings.swiftURL()),
arguments: .init(settings.buildServerArguments),
input: .standardInput,
output: .standardOutput,
error: .standardError,
)
.checkSuccess()
}
}
+1
View File
@@ -274,6 +274,7 @@ struct DevCommand: AsyncParsableCommand {
DevXcodeCommand.self,
DevBuildCommand.self,
DevRunCommand.self,
DevBSPCommand.self,
],
defaultSubcommand: DevRunCommand.self
)
+65 -12
View File
@@ -56,6 +56,7 @@ struct SDKBuilder {
let output: URL
let arch: Arch
// swiftlint:disable:next function_body_length
func buildSDK() async throws {
// TODO: store relevant info for staleness check
let sdkVersion = "develop"
@@ -136,6 +137,9 @@ struct SDKBuilder {
"linker": {
"path": "ld64.lld"
},
"librarian": {
"path": "llvm-lib"
},
"swiftCompiler": {
"extraCLIOptions": [
"-Xfrontend", "-enable-cross-import-overlays",
@@ -149,6 +153,26 @@ struct SDKBuilder {
encoding: .utf8
)
// this toolset works with the swiftbuild system on Linux
// note that we need Swift 6.4+ because 6.3 has bugs in
// resolving the librarian and linker paths.
try """
{
"schemaVersion": "1.0",
"rootPath": "toolset/bin",
"linker": {
"path": "ld"
},
"librarian": {
"path": "llvm-lib"
}
}
""".write(
to: output.appendingPathComponent("toolset-swb.json"),
atomically: false,
encoding: .utf8
)
let sdkDefinition = SDKDefinition(
schemaVersion: "4.0",
targetTriples: [
@@ -215,9 +239,19 @@ struct SDKBuilder {
try await input.finish()
}
.checkSuccess()
// swift-build assumes we have an Apple-flavored librarian for Apple platforms
// (https://github.com/swiftlang/swift-build/blob/b2433e74e/Sources/SWBCore/SpecImplementations/Tools/LinkerTools.swift#L1735)
// but allows us to override this assumption if we explicitly provide the name llvm-lib (look a few lines above in that file)
try FileManager.default.moveItem(
at: toolsetDir.appending(path: "bin/libtool"),
to: toolsetDir.appending(path: "bin/llvm-lib")
)
// TODO: install cctools ld as well
}
// swiftlint:disable:next cyclomatic_complexity function_body_length
// swiftlint:disable:next cyclomatic_complexity
private func installDeveloper(in output: URL) async throws -> URL {
let dev = output.appendingPathComponent("Developer")
@@ -321,10 +355,11 @@ struct SDKBuilder {
for platform in ["iPhoneOS", "MacOSX", "iPhoneSimulator"] {
let lib = "../../../../../Library"
let dest = dev.appendingPathComponent("""
Platforms/\(platform).platform/Developer/SDKs/\(platform).sdk\
/System/Library/Frameworks
""").path
let platformDir = dev.appending(path: "Platforms/\(platform).platform")
try patchPlatformManifest(path: platformDir.appending(path: "Info.plist"))
let dest = platformDir.appending(path: "Developer/SDKs/\(platform).sdk/System/Library/Frameworks").path
try FileManager.default.createSymbolicLink(
atPath: "\(dest)/Testing.framework",
@@ -354,6 +389,21 @@ struct SDKBuilder {
return dev
}
private func patchPlatformManifest(path: URL) throws {
let data = try Data(contentsOf: path)
guard var plist = try PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any] else {
throw Console.Error("Could not read '\(path.path)'")
}
var defaultProperties = (plist["DefaultProperties"] as? [String: Any]) ?? [:]
defaultProperties["SWIFT_RESOURCE_DIR"] = "$(PLATFORM_DIR)/../../Toolchains/XcodeDefault.xctoolchain/usr/lib/swift"
defaultProperties["CLANG_RESOURCE_DIR"] = "$(SWIFT_RESOURCE_DIR)/clang"
plist["DefaultProperties"] = defaultProperties
let newData = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0)
try newData.write(to: path)
}
// returns the number of files we actually want to keep,
// useful for computing progress % during fs traversal
private func extractXIP(inputPath: String, outDir: String) async throws -> Int {
@@ -497,13 +547,16 @@ struct SDKEntry {
E("clang"),
]),
E("Platforms", ["iPhoneOS", "MacOSX", "iPhoneSimulator"].map {
E("\($0).platform/Developer", [
E("SDKs"),
E("Library", [
E("Frameworks"),
E("PrivateFrameworks"),
]),
E("usr/lib"),
E("\($0).platform", [
E("Info.plist"),
E("Developer", [
E("SDKs"),
E("Library", [
E("Frameworks"),
E("PrivateFrameworks"),
]),
E("usr/lib"),
])
])
}),
])
-92
View File
@@ -125,98 +125,6 @@ struct DevSDKStatusCommand: AsyncParsableCommand {
}
}
struct DarwinSDK {
let bundle: URL
let version: String
init?(bundle: URL) {
self.bundle = bundle
if let version = try? Data(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) {
self.version = String(decoding: version, as: UTF8.self)
.trimmingCharacters(in: .whitespacesAndNewlines)
} else if ["darwin.xtoolsdk", "darwin.artifactbundle"].contains(bundle.lastPathComponent) {
self.version = "unknown"
} else {
return nil
}
}
static func install(from path: String) async throws {
// we can't just move into ~/.swiftpm/swift-sdks because the swiftpm directory
// location depends on factors like $XDG_CONFIG_HOME. Rather than replicating
// SwiftPM's logic, which may change, it's more reliable to directly invoke
// `swift sdk install`. See: https://github.com/xtool-org/xtool/pull/40
let url = URL(fileURLWithPath: path)
guard DarwinSDK(bundle: url) != nil else { throw Console.Error("Invalid Darwin SDK at '\(path)'")}
try await addHostClangResourceDir(to: url)
try await Subprocess.run(
.name("swift"),
arguments: ["sdk", "install", url.path],
output: .discarded
)
.checkSuccess()
}
private static func addHostClangResourceDir(to sdk: URL) async throws {
let clangURL = try await ToolRegistry.locate("clang")
let process = try await Subprocess.run(
.path(FilePath(clangURL.path)),
arguments: ["-print-resource-dir"],
output: .string(limit: .max)
).checkSuccess()
let output = process.standardOutput ?? ""
let hostClangResources = URL(filePath: output.trimmingCharacters(in: .whitespacesAndNewlines))
let hostInclude = hostClangResources.appending(path: "include")
let sdkInclude = sdk.appending(path: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/clang/include")
try FileManager.default.copyItem(at: hostInclude, to: sdkInclude)
}
static func current() async throws -> DarwinSDK? {
let outputString: String
do {
outputString = try await Subprocess.run(
.name("swift"),
arguments: ["sdk", "configure", "darwin", "arm64-apple-ios", "--show-configuration"],
output: .string(limit: .max)
)
.checkSuccess()
.standardOutput
?? ""
} catch SubprocessFailure.exited {
return nil
}
// should be something like
// swiftResourcesPath: /home/user/.swiftpm/swift-sdks/darwin.artifactbundle/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift
// swiftlint:disable:previous line_length
let resourcesPathPrefix = "swiftResourcesPath: "
guard let resourcesPath = outputString
.split(separator: "\n")
.first(where: { $0.hasPrefix(resourcesPathPrefix) })?
.dropFirst(resourcesPathPrefix.count)
else { return nil }
var resourcesURL = URL(fileURLWithPath: String(resourcesPath))
for _ in 0..<6 {
resourcesURL = resourcesURL.deletingLastPathComponent()
}
return DarwinSDK(bundle: resourcesURL)
}
func isUpToDate() -> Bool {
true
}
func remove() throws {
try FileManager.default.removeItem(at: bundle)
}
}
private enum SwiftVersion {}
extension SwiftVersion {
static func current() async throws -> Version {
+1
View File
@@ -1,6 +1,7 @@
import ArgumentParser
import Foundation
import XKit
import PackLib
struct SetupCommand: AsyncParsableCommand {
static let configuration = CommandConfiguration(