Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adopt the static linux sdk #196

Merged
merged 9 commits into from
Jan 8, 2025
6 changes: 3 additions & 3 deletions Package.resolved
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"originHash" : "9b0d92b6fbb59080a05ce00f87dc9c6277b32d78e56905abba4c40947edf6d7d",
"originHash" : "531e10b955219c0de91ada74260f59bff8033189f1a9f9f78b199480c61f466a",
"pins" : [
{
"identity" : "async-http-client",
Expand Down Expand Up @@ -150,8 +150,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-tools-support-core.git",
"state" : {
"revision" : "5b130e04cc939373c4713b91704b0c47ceb36170",
"version" : "0.7.1"
"revision" : "b464fcd8d884e599e3202d9bd1eee29a9e504069",
"version" : "0.7.2"
}
},
{
Expand Down
2 changes: 1 addition & 1 deletion Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ let package = Package(
.package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"),
.package(url: "https://github.com/swift-server/async-http-client", from: "1.21.2"),
.package(url: "https://github.com/apple/swift-nio.git", from: "2.64.0"),
.package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.1"),
.package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.7.2"),
.package(url: "https://github.com/apple/swift-docc-plugin", from: "1.3.0"),
// This dependency provides the correct version of the formatter so that you can run `swift run swiftformat Package.swift Plugins/ Sources/ Tests/`
.package(url: "https://github.com/nicklockwood/SwiftFormat", exact: "0.49.18"),
Expand Down
2 changes: 1 addition & 1 deletion Sources/SwiftlyCore/SwiftlyCore.swift
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public protocol InputProvider {
public var inputProvider: (any InputProvider)?

public func readLine(prompt: String) -> String? {
print(prompt, terminator: ": ")
print(prompt, terminator: ": \n")

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So does the user input then get output on a new line? So the stdout would look like:

Yes or No: 
No   <-- this is my user input

?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's correct.

Something that I noticed with the switch to musl instead of libc is that the prompt doesn't get printed to stdout before the user types their answer, which is super confusing. Adding the line terminator seems to force the buffer to get flushed and printed to the screen. This is a mitigation for that behaviour. I'm not really sure why it happens and whether there's some kind of a bug in the stack, or if the bug was that it worked before.

guard let provider = SwiftlyCore.inputProvider else {
return Swift.readLine(strippingNewline: true)
}
Expand Down
111 changes: 48 additions & 63 deletions Tools/build-swiftly-release/BuildSwiftlyRelease.swift
Original file line number Diff line number Diff line change
@@ -1,6 +1,16 @@
import ArgumentParser
import Foundation

public struct SwiftPlatform: Codable {
public var name: String?
public var checksum: String?
}

public struct SwiftRelease: Codable {
public var name: String?
public var platforms: [SwiftPlatform]?
}

// These functions are cloned and adapted from SwiftlyCore until we can do better bootstrapping
public struct Error: LocalizedError {
public let message: String
Expand All @@ -13,6 +23,8 @@ public struct Error: LocalizedError {
}

public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String: String]?) throws {
if !quiet { print("\(args.joined(separator: " "))") }

let process = Process()
process.executableURL = URL(fileURLWithPath: "/usr/bin/env")
process.arguments = args
Expand Down Expand Up @@ -40,6 +52,8 @@ public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String:
}

public func runProgram(_ args: String..., quiet: Bool = false) throws {
if !quiet { print("\(args.joined(separator: " "))") }

let process = Process()
process.executableURL = URL(fileURLWithPath: "/usr/bin/env")
process.arguments = args
Expand Down Expand Up @@ -126,58 +140,6 @@ public func getShell() async throws -> String {
}
#endif

public func isSupportedLinux(useRhelUbi9: Bool) -> Bool {
let osReleaseFiles = ["/etc/os-release", "/usr/lib/os-release"]
var releaseFile: String?
for file in osReleaseFiles {
if FileManager.default.fileExists(atPath: file) {
releaseFile = file
break
}
}

guard let releaseFile = releaseFile else {
return false
}

guard let data = FileManager.default.contents(atPath: releaseFile) else {
return false
}

guard let releaseInfo = String(data: data, encoding: .utf8) else {
return false
}

var id: String?
var idlike: String?
var versionID: String?
for info in releaseInfo.split(separator: "\n").map(String.init) {
if info.hasPrefix("ID=") {
id = String(info.dropFirst("ID=".count)).replacingOccurrences(of: "\"", with: "")
} else if info.hasPrefix("ID_LIKE=") {
idlike = String(info.dropFirst("ID_LIKE=".count)).replacingOccurrences(of: "\"", with: "")
} else if info.hasPrefix("VERSION_ID=") {
versionID = String(info.dropFirst("VERSION_ID=".count)).replacingOccurrences(of: "\"", with: "")
}
}

guard let id = id, let idlike = idlike else {
return false
}

if useRhelUbi9 {
guard let versionID, versionID.hasPrefix("9"), (id + idlike).contains("rhel") else {
return false
}
} else {
guard let versionID = versionID, versionID == "2", (id + idlike).contains("amzn") else {
return false
}
}

return true
}

@main
struct BuildSwiftlyRelease: AsyncParsableCommand {
static let configuration = CommandConfiguration(
Expand All @@ -195,7 +157,7 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
@Option(help: "Package identifier of macOS package")
var identifier: String = "org.swift.swiftly"
#elseif os(Linux)
@Flag(name: .long, help: "Use RHEL UBI9 as the supported Linux to build a release instead of Amazon Linux 2")
@Flag(name: .long, help: "Deprecated option since releases can be built on any swift supported Linux distribution.")
var useRhelUbi9: Bool = false
#endif

Expand Down Expand Up @@ -295,13 +257,6 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
}

func buildLinuxRelease() async throws {
#if os(Linux)
// Check system requirements
guard isSupportedLinux(useRhelUbi9: self.useRhelUbi9) else {
throw Error(message: "Linux releases must be made from specific distributions so that the binary can be used everyone else because it has the oldest version of glibc for maximum compatibility with other versions of Linux. Please try again with \(!self.useRhelUbi9 ? "Amazon Linux 2" : "RedHat UBI 9").")
}
#endif

// TODO: turn these into checks that the system meets the criteria for being capable of using the toolchain + checking for packages, not tools
let curl = try await self.assertTool("curl", message: "Please install curl with `yum install curl`")
let tar = try await self.assertTool("tar", message: "Please install tar with `yum install tar`")
Expand Down Expand Up @@ -340,8 +295,38 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
let cwd = FileManager.default.currentDirectoryPath
FileManager.default.changeCurrentDirectoryPath(libArchivePath)

let swiftVerRegex: Regex<(Substring, Substring)> = try! Regex("Swift version (\\d+\\.\\d+\\.\\d+) ")
let swiftVerOutput = (try await runProgramOutput(swift, "--version")) ?? ""
guard let swiftVerMatch = try swiftVerRegex.firstMatch(in: swiftVerOutput) else {
throw Error(message: "Unable to detect swift version")
}

let swiftVersion = swiftVerMatch.output.1

let sdkName = "swift-\(swiftVersion)-RELEASE_static-linux-0.0.1"

#if arch(arm64)
let arch = "aarch64"
#else
let arch = "x86_64"
#endif

let swiftReleasesJson = (try await runProgramOutput(curl, "https://www.swift.org/api/v1/install/releases.json")) ?? "[]"
let swiftReleases = try JSONDecoder().decode([SwiftRelease].self, from: swiftReleasesJson.data(using: .utf8)!)

guard let swiftRelease = swiftReleases.first(where: { ($0.name ?? "") == swiftVersion }) else {
throw Error(message: "Unable to find swift release using swift.org API: \(swiftVersion)")
}

guard let sdkPlatform = (swiftRelease.platforms ?? [SwiftPlatform]()).first(where: { ($0.name ?? "") == "Static SDK" }) else {
throw Error(message: "Swift release \(swiftVersion) has no Static SDK offering")
}

try runProgram(swift, "sdk", "install", "https://download.swift.org/swift-\(swiftVersion)-release/static-sdk/swift-\(swiftVersion)-RELEASE/swift-\(swiftVersion)-RELEASE_static-linux-0.0.1.artifactbundle.tar.gz", "--checksum", sdkPlatform.checksum ?? "deadbeef")

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the Static Linux SDK will always have this .0.0.1 version identifier? Just want to make sure this doesn't break if that format changes

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good point. I'll ask around about the version compatibility story.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It appears that this version is expected to change only very rarely, if ever.


var customEnv = ProcessInfo.processInfo.environment
customEnv["CC"] = "clang"
customEnv["CC"] = "\(cwd)/Tools/build-swiftly-release/musl-clang"
customEnv["MUSL_PREFIX"] = "\(FileManager.default.homeDirectoryForCurrentUser.path)/.swiftpm/swift-sdks/\(sdkName).artifactbundle/\(sdkName)/swift-linux-musl/musl-1.2.5.sdk/\(arch)/usr"

try runProgramEnv(
"./configure",
Expand Down Expand Up @@ -371,8 +356,8 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {

FileManager.default.changeCurrentDirectoryPath(cwd)

// Statically link standard libraries for maximum portability of the swiftly binary
try runProgram(swift, "build", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release")
try runProgram(swift, "build", "--swift-sdk", "\(arch)-swift-linux-musl", "--product=swiftly", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig", "--static-swift-stdlib", "--configuration=release")
try runProgram(swift, "sdk", "remove", sdkName)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this inadvertently remove the linux SDK from a users machine even if they didn't want it to? As in they already had the static sdk installed, built swiftly, and now their SDK is gone and maybe they didn't expect that?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't expect that users should run the release script on their own machine. It should be some kind of builder where keeping the SDK around could cause problems for future builds. It's unfortunate that SDK's can't just be referenced with a file path.


let releaseDir = cwd + "/.build/release"

Expand Down
57 changes: 57 additions & 0 deletions Tools/build-swiftly-release/musl-clang
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/sh

PREFIX=${MUSL_PREFIX:-"/usr/local/musl"}
if [ ! -d "${PREFIX}" ]; then
echo "invalid prefix: ${PREFIX}"
return 1
fi

CLANG=${REALCLANG:-"clang"}

hasNo() {
pat="$1"
shift 1

for e in "$@"; do
if [ "$e" = "${pat}" ]; then
return 1
fi
done
return 0
}

ARGS="-nostdinc"
TAIL=""

if hasNo '-nostdinc' "$@"; then
ARGS="${ARGS} -isystem ${PREFIX}/include"
fi

if \
hasNo '-c' "$@" && \
hasNo '-S' "$@" && \
hasNo '-E' "$@"
then
ARGS="${ARGS} -nostdlib"
ARGS="${ARGS} -Wl,-dynamic-linker=${PREFIX}/lib/libc.so"
ARGS="${ARGS} -L${PREFIX}/lib -L${PREFIX}/lib/swift/clang/lib/linux"

if hasNo '-nostartfiles' "$@" && \
hasNo '-nostdlib' "$@" && \
hasNo '-nodefaultlibs' "$@"
then
ARGS="${ARGS} ${PREFIX}/lib/crt1.o"
ARGS="${ARGS} ${PREFIX}/lib/crti.o"

TAIL="${TAIL} ${PREFIX}/lib/crtn.o"
fi

if hasNo '-nostdlib' "$@" && \
hasNo '-nodefaultlibs' "$@"
then
TAIL="${TAIL} -lc -lclang_rt.builtins-$(uname -m)"
fi
fi

exec ${CLANG} ${ARGS} "$@" ${TAIL} -static

Loading