-
Notifications
You must be signed in to change notification settings - Fork 35
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Adopt the static linux sdk #196
Changes from all commits
e37a6cd
01a9af2
ab2354c
95cd4b6
04d881c
ca7aeb7
999f0e2
134b46e
e41fd13
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,16 @@ | ||
import ArgumentParser | ||
import Foundation | ||
|
||
public struct SwiftPlatform: Codable { | ||
public var name: String? | ||
public var checksum: String? | ||
} | ||
|
||
public struct SwiftRelease: Codable { | ||
public var name: String? | ||
public var platforms: [SwiftPlatform]? | ||
} | ||
|
||
// These functions are cloned and adapted from SwiftlyCore until we can do better bootstrapping | ||
public struct Error: LocalizedError { | ||
public let message: String | ||
|
@@ -13,6 +23,8 @@ public struct Error: LocalizedError { | |
} | ||
|
||
public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: String]?) throws { | ||
if !quiet { print("\(args.joined(separator: " "))") } | ||
|
||
let process = Process() | ||
process.executableURL = URL(fileURLWithPath: "/usr/bin/env") | ||
process.arguments = args | ||
|
@@ -40,6 +52,8 @@ public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: | |
} | ||
|
||
public func runProgram(_ args: String..., quiet: Bool = false) throws { | ||
if !quiet { print("\(args.joined(separator: " "))") } | ||
|
||
let process = Process() | ||
process.executableURL = URL(fileURLWithPath: "/usr/bin/env") | ||
process.arguments = args | ||
|
@@ -126,58 +140,6 @@ public func getShell() async throws -> String { | |
} | ||
#endif | ||
|
||
public func isSupportedLinux(useRhelUbi9: Bool) -> Bool { | ||
let osReleaseFiles = ["/etc/os-release", "/usr/lib/os-release"] | ||
var releaseFile: String? | ||
for file in osReleaseFiles { | ||
if FileManager.default.fileExists(atPath: file) { | ||
releaseFile = file | ||
break | ||
} | ||
} | ||
|
||
guard let releaseFile = releaseFile else { | ||
return false | ||
} | ||
|
||
guard let data = FileManager.default.contents(atPath: releaseFile) else { | ||
return false | ||
} | ||
|
||
guard let releaseInfo = String(data: data, encoding: .utf8) else { | ||
return false | ||
} | ||
|
||
var id: String? | ||
var idlike: String? | ||
var versionID: String? | ||
for info in releaseInfo.split(separator: "\n").map(String.init) { | ||
if info.hasPrefix("ID=") { | ||
id = String(info.dropFirst("ID=".count)).replacingOccurrences(of: "\"", with: "") | ||
} else if info.hasPrefix("ID_LIKE=") { | ||
idlike = String(info.dropFirst("ID_LIKE=".count)).replacingOccurrences(of: "\"", with: "") | ||
} else if info.hasPrefix("VERSION_ID=") { | ||
versionID = String(info.dropFirst("VERSION_ID=".count)).replacingOccurrences(of: "\"", with: "") | ||
} | ||
} | ||
|
||
guard let id = id, let idlike = idlike else { | ||
return false | ||
} | ||
|
||
if useRhelUbi9 { | ||
guard let versionID, versionID.hasPrefix("9"), (id + idlike).contains("rhel") else { | ||
return false | ||
} | ||
} else { | ||
guard let versionID = versionID, versionID == "2", (id + idlike).contains("amzn") else { | ||
return false | ||
} | ||
} | ||
|
||
return true | ||
} | ||
|
||
@main | ||
struct BuildSwiftlyRelease: AsyncParsableCommand { | ||
static let configuration = CommandConfiguration( | ||
|
@@ -195,7 +157,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { | |
@Option(help: "Package identifier of macOS package") | ||
var identifier: String = "org.swift.swiftly" | ||
#elseif os(Linux) | ||
@Flag(name: .long, help: "Use RHEL UBI9 as the supported Linux to build a release instead of Amazon Linux 2") | ||
@Flag(name: .long, help: "Deprecated option since releases can be built on any swift supported Linux distribution.") | ||
var useRhelUbi9: Bool = false | ||
#endif | ||
|
||
|
@@ -295,13 +257,6 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { | |
} | ||
|
||
func buildLinuxRelease() async throws { | ||
#if os(Linux) | ||
// Check system requirements | ||
guard isSupportedLinux(useRhelUbi9: self.useRhelUbi9) else { | ||
throw Error(message: "Linux releases must be made from specific distributions so that the binary can be used everyone else because it has the oldest version of glibc for maximum compatibility with other versions of Linux. Please try again with \(!self.useRhelUbi9 ? "Amazon Linux 2" : "RedHat UBI 9").") | ||
} | ||
#endif | ||
|
||
// TODO: turn these into checks that the system meets the criteria for being capable of using the toolchain + checking for packages, not tools | ||
let curl = try await self.assertTool("curl", message: "Please install curl with `yum install curl`") | ||
let tar = try await self.assertTool("tar", message: "Please install tar with `yum install tar`") | ||
|
@@ -340,8 +295,38 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { | |
let cwd = FileManager.default.currentDirectoryPath | ||
FileManager.default.changeCurrentDirectoryPath(libArchivePath) | ||
|
||
let swiftVerRegex: Regex<(Substring, Substring)> = try! Regex("Swift version (\\d+\\.\\d+\\.\\d+) ") | ||
let swiftVerOutput = (try await runProgramOutput(swift, "--version")) ?? "" | ||
guard let swiftVerMatch = try swiftVerRegex.firstMatch(in: swiftVerOutput) else { | ||
throw Error(message: "Unable to detect swift version") | ||
} | ||
|
||
let swiftVersion = swiftVerMatch.output.1 | ||
|
||
let sdkName = "swift-\(swiftVersion)-RELEASE_static-linux-0.0.1" | ||
|
||
#if arch(arm64) | ||
let arch = "aarch64" | ||
#else | ||
let arch = "x86_64" | ||
#endif | ||
|
||
let swiftReleasesJson = (try await runProgramOutput(curl, "https://www.swift.org/api/v1/install/releases.json")) ?? "[]" | ||
let swiftReleases = try JSONDecoder().decode([SwiftRelease].self, from: swiftReleasesJson.data(using: .utf8)!) | ||
|
||
guard let swiftRelease = swiftReleases.first(where: { ($0.name ?? "") == swiftVersion }) else { | ||
throw Error(message: "Unable to find swift release using swift.org API: \(swiftVersion)") | ||
} | ||
|
||
guard let sdkPlatform = (swiftRelease.platforms ?? [SwiftPlatform]()).first(where: { ($0.name ?? "") == "Static SDK" }) else { | ||
throw Error(message: "Swift release \(swiftVersion) has no Static SDK offering") | ||
} | ||
|
||
try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-\(swiftVersion)-release/static-sdk/swift-\(swiftVersion)-RELEASE/swift-\(swiftVersion)-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", sdkPlatform.checksum ?? "deadbeef") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the Static Linux SDK will always have this There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a good point. I'll ask around about the version compatibility story. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It appears that this version is expected to change only very rarely, if ever. |
||
|
||
var customEnv = ProcessInfo.processInfo.environment | ||
customEnv["CC"] = "clang" | ||
customEnv["CC"] = "\(cwd)/Tools/build-swiftly-release/musl-clang" | ||
customEnv["MUSL_PREFIX"] = "\(FileManager.default.homeDirectoryForCurrentUser.path)/.swiftpm/swift-sdks/\(sdkName).artifactbundle/\(sdkName)/swift-linux-musl/musl-1.2.5.sdk/\(arch)/usr" | ||
|
||
try runProgramEnv( | ||
"./configure", | ||
|
@@ -371,8 +356,8 @@ struct BuildSwiftlyRelease: AsyncParsableCommand { | |
|
||
FileManager.default.changeCurrentDirectoryPath(cwd) | ||
|
||
// Statically link standard libraries for maximum portability of the swiftly binary | ||
try runProgram(swift, "build", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") | ||
try runProgram(swift, "build", "--swift-sdk", "\(arch)-swift-linux-musl", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release") | ||
try runProgram(swift, "sdk", "remove", sdkName) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could this inadvertently remove the linux SDK from a users machine even if they didn't want it to? As in they already had the static sdk installed, built swiftly, and now their SDK is gone and maybe they didn't expect that? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't expect that users should run the release script on their own machine. It should be some kind of builder where keeping the SDK around could cause problems for future builds. It's unfortunate that SDK's can't just be referenced with a file path. |
||
|
||
let releaseDir = cwd + "/.build/release" | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
#!/bin/sh | ||
|
||
PREFIX=${MUSL_PREFIX:-"/usr/local/musl"} | ||
if [ ! -d "${PREFIX}" ]; then | ||
echo "invalid prefix: ${PREFIX}" | ||
return 1 | ||
fi | ||
|
||
CLANG=${REALCLANG:-"clang"} | ||
|
||
hasNo() { | ||
pat="$1" | ||
shift 1 | ||
|
||
for e in "$@"; do | ||
if [ "$e" = "${pat}" ]; then | ||
return 1 | ||
fi | ||
done | ||
return 0 | ||
} | ||
|
||
ARGS="-nostdinc" | ||
TAIL="" | ||
|
||
if hasNo '-nostdinc' "$@"; then | ||
ARGS="${ARGS} -isystem ${PREFIX}/include" | ||
fi | ||
|
||
if \ | ||
hasNo '-c' "$@" && \ | ||
hasNo '-S' "$@" && \ | ||
hasNo '-E' "$@" | ||
then | ||
ARGS="${ARGS} -nostdlib" | ||
ARGS="${ARGS} -Wl,-dynamic-linker=${PREFIX}/lib/libc.so" | ||
ARGS="${ARGS} -L${PREFIX}/lib -L${PREFIX}/lib/swift/clang/lib/linux" | ||
|
||
if hasNo '-nostartfiles' "$@" && \ | ||
hasNo '-nostdlib' "$@" && \ | ||
hasNo '-nodefaultlibs' "$@" | ||
then | ||
ARGS="${ARGS} ${PREFIX}/lib/crt1.o" | ||
ARGS="${ARGS} ${PREFIX}/lib/crti.o" | ||
|
||
TAIL="${TAIL} ${PREFIX}/lib/crtn.o" | ||
fi | ||
|
||
if hasNo '-nostdlib' "$@" && \ | ||
hasNo '-nodefaultlibs' "$@" | ||
then | ||
TAIL="${TAIL} -lc -lclang_rt.builtins-$(uname -m)" | ||
fi | ||
fi | ||
|
||
exec ${CLANG} ${ARGS} "$@" ${TAIL} -static | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So does the user input then get output on a new line? So the stdout would look like:
?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That's correct.
Something that I noticed with the switch to musl instead of libc is that the prompt doesn't get printed to stdout before the user types their answer, which is super confusing. Adding the line terminator seems to force the buffer to get flushed and printed to the screen. This is a mitigation for that behaviour. I'm not really sure why it happens and whether there's some kind of a bug in the stack, or if the bug was that it worked before.