From 3010dfe038dce73a4d5b809b6d0949e723c76835 Mon Sep 17 00:00:00 2001 From: Peter Tripp Date: Mon, 30 Sep 2024 17:46:21 -0400 Subject: [PATCH] Support More Linux (#18480) - Add `script/build-docker` - Add `script/install-cmake` - Add `script/install-mold` - Improve `script/linux` - Add missing dependencies: `jq`, `git`, `tar`, `gzip` as required. - Add check for mold - Fix Redhat 8.x derivatives (RHEL, Centos, Almalinux, Rocky, Oracle, Amazon) - Fix perl libs to be Fedora only - Install the best `libstdc++` available on apt distros - ArchLinux: run `pacman -Syu` to update repos before installing. - Should work on Raspbian (untested) This make it possible to test builds on other distros using docker: ``` ./script/build-docker amazonlinux:2023 ``` --- Dockerfile-distros | 26 +++++++ Dockerfile-distros.dockerignore | 2 + docs/src/development/macos.md | 6 ++ script/build-docker | 25 +++++++ script/install-cmake | 77 +++++++++++++++++++++ script/install-mold | 2 +- script/linux | 116 +++++++++++++++++++++++--------- 7 files changed, 220 insertions(+), 34 deletions(-) create mode 100644 Dockerfile-distros create mode 100644 Dockerfile-distros.dockerignore create mode 100755 script/build-docker create mode 100755 script/install-cmake diff --git a/Dockerfile-distros b/Dockerfile-distros new file mode 100644 index 000000000..c8a98d2f7 --- /dev/null +++ b/Dockerfile-distros @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +WORKDIR /app +ARG TZ=Etc/UTC \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + DEBIAN_FRONTEND=noninteractive +ENV CARGO_TERM_COLOR=always + +COPY script/linux script/ +RUN ./script/linux +COPY script/install-mold script/install-cmake script/ +RUN ./script/install-mold "2.34.0" +RUN ./script/install-cmake "3.30.4" + +COPY . . + +# When debugging, make these into individual RUN statements. +# Cleanup to avoid saving big layers we aren't going to use. +RUN . "$HOME/.cargo/env" \ + && cargo fetch \ + && cargo build \ + && cargo run -- --help \ + && cargo clean --quiet diff --git a/Dockerfile-distros.dockerignore b/Dockerfile-distros.dockerignore new file mode 100644 index 000000000..de70e0d16 --- /dev/null +++ b/Dockerfile-distros.dockerignore @@ -0,0 +1,2 @@ +**/target +**/node_modules diff --git a/docs/src/development/macos.md b/docs/src/development/macos.md index 1407b0f61..c5cbf56e1 100644 --- a/docs/src/development/macos.md +++ b/docs/src/development/macos.md @@ -35,6 +35,12 @@ Clone down the [Zed repository](https://github.com/zed-industries/zed). brew install cmake ``` +- (Optional) Install `mold` to speed up link times + + ```sh + brew install mold + ``` + ## Backend Dependencies If you are developing collaborative features of Zed, you'll need to install the dependencies of zed's `collab` server: diff --git a/script/build-docker b/script/build-docker new file mode 100755 index 000000000..c5ea294c7 --- /dev/null +++ b/script/build-docker @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Use a docker BASE_IMAGE to test building Zed. +# e.g: ./script/bundle-docker ubuntu:20.04 +# +# Increasing resources available to podman may speed this up: +# podman machine stop +# podman machine set --memory 16384 --cpus 8 --disk-size 200 +# podman machine start + +set -euo pipefail + +BASE_IMAGE=${BASE_IMAGE:-${1:-}} +if [ -z "$BASE_IMAGE" ]; then + echo "Usage: $0 BASE_IMAGE" >&2 + exit 1 +fi + +export DOCKER_BUILDKIT=1 +cd "$(dirname "$0")/.." + +podman build . \ + -f Dockerfile-distros \ + -t many \ + --build-arg BASE_IMAGE="$BASE_IMAGE" diff --git a/script/install-cmake b/script/install-cmake new file mode 100755 index 000000000..71b5aaeee --- /dev/null +++ b/script/install-cmake @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# This script installs an up-to-date version of CMake. +# +# For MacOS use Homebrew to install the latest version. +# +# For Ubuntu use the official KitWare Apt repository with backports. +# See: https://apt.kitware.com/ +# +# For other systems (RHEL 8.x, 9.x, AmazonLinux, SUSE, Fedora, Arch, etc) +# use the official CMake installer script from KitWare. +# +# Note this is similar to how GitHub Actions runners install cmake: +# https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-cmake.sh +# +# Upstream: 3.30.4 (2024-09-27) + +set -euo pipefail + + +if [[ "$(uname -s)" == "darwin" ]]; then + brew --version >/dev/null \ + || echo "Error: Homebrew is required to install cmake on MacOS." && exit 1 + echo "Installing cmake via Homebrew (can't pin to old versions)." + brew install cmake + exit 0 +elif [ "$(uname -s)" != "Linux" ]; then + echo "Error: This script is intended for MacOS/Linux systems only." + exit 1 +elif [ -z "${1:-}" ]; then + echo "Usage: $0 [3.30.4]" + exit 1 +fi +CMAKE_VERSION="${CMAKE_VERSION:-${1:-3.30.4}}" + +if [ "$(whoami)" = root ]; then SUDO=; else SUDO="$(command -v sudo || command -v doas || true)"; fi + +if cmake --version | grep -q "$CMAKE_VERSION"; then + echo "CMake $CMAKE_VERSION is already installed." + exit 0 +elif [ -e /usr/local/bin/cmake ]; then + echo "Warning: existing cmake found at /usr/local/bin/cmake. Skipping installation." + exit 0 +elif [ -e /etc/apt/sources.list.d/kitware.list ]; then + echo "Warning: existing KitWare repository found. Skipping installation." + exit 0 +elif [ -e /etc/lsb-release ] && grep -qP 'DISTRIB_ID=Ubuntu' /etc/lsb-release; then + curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc \ + | $SUDO gpg --dearmor - \ + | $SUDO tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null + echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \ + | $SUDO tee /etc/apt/sources.list.d/kitware.list >/dev/null + $SUDO apt-get update + $SUDO apt-get install -y kitware-archive-keyring cmake==$CMAKE_VERSION +else + arch="$(uname -m)" + if [ "$arch" != "x86_64" ] && [ "$arch" != "aarch64" ]; then + echo "Error. Only x86_64 and aarch64 are supported." + exit 1 + fi + tempdir=$(mktemp -d) + pushd "$tempdir" + CMAKE_REPO="https://github.com/Kitware/CMake" + CMAKE_INSTALLER="cmake-$CMAKE_VERSION-linux-$arch.sh" + curl -fsSL --output cmake-$CMAKE_VERSION-SHA-256.txt \ + "$CMAKE_REPO/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt" + curl -fsSL --output $CMAKE_INSTALLER \ + "$CMAKE_REPO/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-linux-$arch.sh" + # workaround for old versions of sha256sum not having --ignore-missing + grep -F "cmake-$CMAKE_VERSION-linux-$arch.sh" "cmake-$CMAKE_VERSION-SHA-256.txt" \ + | sha256sum -c \ + | grep -qP "^${CMAKE_INSTALLER}: OK" + chmod +x cmake-$CMAKE_VERSION-linux-$arch.sh + $SUDO ./cmake-$CMAKE_VERSION-linux-$arch.sh --prefix=/usr/local --skip-license + popd + rm -rf "$tempdir" +fi diff --git a/script/install-mold b/script/install-mold index 1c451b726..fca8d73f2 100755 --- a/script/install-mold +++ b/script/install-mold @@ -30,7 +30,7 @@ MOLD_REPO="${MOLD_REPO:-https://github.com/rui314/mold}" MOLD_URL="${MOLD_URL:-$MOLD_REPO}/releases/download/v$MOLD_VERSION/mold-$MOLD_VERSION-$(uname -m)-linux.tar.gz" echo "Downloading from $MOLD_URL" -curl --location --show-error --output - --retry 3 --retry-delay 5 "$MOLD_URL" \ +curl -fsSL --output - "$MOLD_URL" \ | $SUDO tar -C /usr/local --strip-components=1 --no-overwrite-dir -xzf - # Note this binary depends on the system libatomic.so.1 which is usually diff --git a/script/linux b/script/linux index 189407a51..540bf063e 100755 --- a/script/linux +++ b/script/linux @@ -1,15 +1,25 @@ #!/usr/bin/env bash -set -ex +set -xeuo pipefail -# install the wasm toolchain -which rustup > /dev/null 2>&1 || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +# if root or if sudo/unavailable, define an empty variable +if [ "$(id -u)" -eq 0 ] +then maysudo='' +else maysudo="$(command -v sudo || command -v doas || true)" +fi -# if sudo is not installed, define an empty alias -maysudo=$(command -v sudo || command -v doas || true) +function finalize { + # after packages install (curl, etc), get the rust toolchain + which rustup > /dev/null 2>&1 || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + # verify the mold situation + if ! command -v mold >/dev/null 2>&1; then + echo "Warning: Mold binaries are unavailable on your system." >&2 + echo " Builds will be slower without mold. Try: scripts/install-mold" >&2 + fi + echo "Finished installing Linux dependencies with script/linux" +} -# Ubuntu, Debian, etc. -# https://packages.ubuntu.com/ +# Ubuntu, Debian, Mint, Kali, Pop!_OS, Raspbian, etc. apt=$(command -v apt-get || true) if [[ -n $apt ]]; then deps=( @@ -27,58 +37,88 @@ if [[ -n $apt ]]; then cmake clang jq + git + curl gettext-base elfutils libsqlite3-dev ) - # Ubuntu 20.04 / Debian Bullseye (including CI for release) - if grep -q "bullseye" /etc/debian_version; then - deps+=( - libstdc++-10-dev - ) - else - deps+=( - libstdc++-12-dev - mold - ) + if (grep -qP 'PRETTY_NAME="(.+24\.04)' /etc/os-release); then + deps+=( mold libstdc++-14-dev ) + elif (grep -qP 'PRETTY_NAME="((Debian|Raspbian).+12|.+22\.04)' /etc/os-release); then + deps+=( mold libstdc++-12-dev ) + elif (grep -qP 'PRETTY_NAME="((Debian|Raspbian).+11|.+20\.04)' /etc/os-release); then + deps+=( libstdc++-10-dev ) fi $maysudo "$apt" update $maysudo "$apt" install -y "${deps[@]}" + finalize exit 0 fi -# Fedora, CentOS, RHEL, etc. -# https://packages.fedoraproject.org/ +# Fedora, CentOS, RHEL, Alma, Amazon 2023, Oracle, etc. dnf=$(command -v dnf || true) -if [[ -n $dnf ]]; then +# Old Redhat (yum only): Amazon Linux 2, Oracle Linux 7, etc. +yum=$(command -v yum || true) + +if [[ -n $dnf ]] || [[ -n $yum ]]; then + pkg_cmd="${dnf:-${yum}}" deps=( gcc - g++ clang cmake - mold alsa-lib-devel fontconfig-devel wayland-devel libxkbcommon-x11-devel openssl-devel libzstd-devel - # Perl dependencies are needed for openssl-sys crate see https://docs.rs/openssl/latest/openssl/ - perl-FindBin - perl-IPC-Cmd - perl-File-Compare - perl-File-Copy vulkan-loader sqlite-devel + jq + git + tar ) + # perl used for building openssl-sys crate. See: https://docs.rs/openssl/latest/openssl/ + if grep -qP '^ID="(fedora)' /etc/os-release; then + deps+=( + perl-FindBin + perl-IPC-Cmd + perl-File-Compare + perl-File-Copy + mold + ) + elif grep grep -qP '^ID="(rhel|rocky|alma|centos|ol)' /etc/os-release; then + deps+=( perl-interpreter ) + fi - # libxkbcommon-x11-devel is in the crb repo on RHEL and CentOS, not needed for Fedora - if ! grep -q "Fedora" /etc/redhat-release; then - $maysudo "$dnf" config-manager --set-enabled crb + # gcc-c++ is g++ on RHEL8 and 8.x clones + if grep -qP '^ID="(rhel|rocky|alma|centos|ol)' /etc/os-release \ + && grep -qP '^VERSION_ID="8' /etc/os-release; then + deps+=( gcc-c++ ) + else + deps+=( g++ ) + fi + + # libxkbcommon-x11-devel is in a non-default repo on RHEL 8.x/9.x (except on AmazonLinux) + if grep -qP '^VERSION_ID="(8|9)' && grep -qP '^ID="(rhel|rocky|centos|alma|ol)' /etc/os-release; then + $maysudo dnf install -y 'dnf-command(config-manager)' + if grep -qP '^PRETTY_NAME="(AlmaLinux 8|Rocky Linux 8)' /etc/os-release; then + $maysudo dnf config-manager --set-enabled powertools + elif grep -qP '^PRETTY_NAME="((AlmaLinux|Rocky|CentOS Stream) 9|Red Hat.+(8|9))' /etc/os-release; then + $maysudo dnf config-manager --set-enabled crb + elif grep -qP '^PRETTY_NAME="Oracle Linux Server 8' /etc/os-release; then + $maysudo dnf config-manager --set-enabled ol8_codeready_builder + elif grep -qP '^PRETTY_NAME="Oracle Linux Server 9' /etc/os-release; then + $maysudo dnf config-manager --set-enabled ol9_codeready_builder + else + echo "Unexpected distro" && grep 'PRETTY_NAME' /etc/os-release && exit 1 + fi fi - $maysudo "$dnf" install -y "${deps[@]}" + $maysudo $pkg_cmd install -y "${deps[@]}" + finalize exit 0 fi @@ -99,10 +139,14 @@ if [[ -n $zyp ]]; then openssl-devel libzstd-devel libvulkan1 - mold sqlite3-devel + jq + git + tar + gzip ) $maysudo "$zyp" install -y "${deps[@]}" + finalize exit 0 fi @@ -125,8 +169,10 @@ if [[ -n $pacman ]]; then mold sqlite jq + git ) - $maysudo "$pacman" -S --needed --noconfirm "${deps[@]}" + $maysudo "$pacman" -Syu --needed --noconfirm "${deps[@]}" + finalize exit 0 fi @@ -153,6 +199,7 @@ if [[ -n $xbps ]]; then sqlite-devel ) $maysudo "$xbps" -Syu "${deps[@]}" + finalize exit 0 fi @@ -162,6 +209,7 @@ emerge=$(command -v emerge || true) if [[ -n $emerge ]]; then deps=( app-arch/zstd + app-misc/jq dev-libs/openssl dev-libs/wayland dev-util/cmake @@ -174,7 +222,9 @@ if [[ -n $emerge ]]; then dev-db/sqlite ) $maysudo "$emerge" -u "${deps[@]}" + finalize exit 0 fi echo "Unsupported Linux distribution in script/linux" +exit 1