merge: #2988

2988: feat/add cyclone rootfs target r=fnichol a=johnrwatson

Adds a new buck2 target, toolchain and macro which builds the rootfs required for firecracker to launch micro-vms.

Currently it only supports Cyclone due to some hardcoded values in the build chain but should be easily otherwise extensible to any of the services we which to run on firecracker or build this way.

The new buck2 target can be invoked via:
`buck2 build bin/cyclone:rootfs` 

The target is dependent on the cyclone `binary` not the docker image, so it should be nice and fast compared to if we added it to the tail end of the docker build.

I've added some placeholder content for Promoting the rootfs artifacts through Buck2 too, which should be helpful in the future for on-main merges and similar.

There is some minor work to include the nix flake content within the rootfs, which can come in a separate PR.

Example run:
```
john@Threadripper:~/working-folder/2-repos/si$ buck2 build bin/cyclone:rootfs --show-simple-output
File changed: prelude-si//rootfs/rootfs_build.sh
File changed: prelude-si//rootfs/z_TODO
File changed: prelude-si//rootfs/create-root-fs.sh
6 additional file change events
Build ID: 81af087a-6e28-4408-91cf-575ea3ee87a2
Network: Up: 0B  Down: 0B
Jobs completed: 47. Time elapsed: 3.7s.
Cache hits: 0%. Commands: 1 (cached: 0, remote: 0, local: 1)
BUILD SUCCEEDED
buck-out/v2/gen/root/524f8da68ea2a374/bin/cyclone/__rootfs__/johns-rootfs.tar
```

It's noteworthy this particular buck target `is not` buildable on mac devices, due to the mkfs.ext4 variant that's required. It's possible to do some VMM shenanigans to make this work however.

Co-authored-by: John Watson <[email protected]>

bin/cyclone/BUCK

@@ -1,6 +1,7 @@
 load(
     "@prelude-si//:macros.bzl",
     "docker_image",
+    "rootfs",
     "export_file",
     "rust_binary",
     "shellcheck",
@@ -25,7 +26,6 @@ test_suite(
     ],
 )
 
-
 rust_binary(
     name = "cyclone",
     deps = [
@@ -62,3 +62,11 @@ docker_image(
         "//bin/lang-js:bin",
     ]
 )
+
+rootfs(
+    name = "rootfs",
+    build_deps = [
+        ":cyclone",
+        #":nix_store",
+    ]
+)

prelude-si/git/git_info.py

@@ -11,12 +11,12 @@
 
 ABBREVIATED_COMMIT_HASH = "abbreviated_commit_hash"
 CAL_VER = "cal_ver"
+ARTIFACT_VER = "artifact_ver"
 COMMITER_DATE_STRICT = "committer_date_strict_iso8601"
 COMMITER_DATE_TIMESTAMP = "committer_date_timestamp"
 COMMIT_HASH = "commit_hash"
 IS_DIRTY = "is_dirty"
 
-
 def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
@@ -97,6 +97,10 @@ def finalize(data: Dict[str, Any]):
             COMMITER_DATE_TIMESTAMP: round(dt_utc.timestamp()),
         })
 
+        data.update({
+            ARTIFACT_VER: data["cal_ver"] + "-sha" + data["abbreviated_commit_hash"]
+        })
+
 
 if __name__ == "__main__":
     sys.exit(main())

prelude-si/macros.bzl

@@ -18,6 +18,12 @@ load(
 )
 docker_image = _docker_image
 
+load(
+    "@prelude-si//macros:rootfs.bzl",
+    _rootfs = "rootfs",
+)
+rootfs = _rootfs
+
 load(
     "@prelude-si//macros:nix.bzl",
     _nix_flake_lock = "nix_flake_lock",

prelude-si/macros/rootfs.bzl

@@ -0,0 +1,14 @@
+load(
+    "@prelude-si//:rootfs.bzl",
+    _rootfs = "rootfs",
+    #_rootfs_promote = "rootfs_promote",
+    _build_rootfs = "build_rootfs",
+)
+
+def rootfs(
+        name,
+        **kwargs):
+    _rootfs(
+        name = name,
+        **kwargs,
+    )

prelude-si/rootfs.bzl

@@ -0,0 +1,64 @@
+load("//rootfs:toolchain.bzl", "RootfsToolchainInfo")
+load("//git:toolchain.bzl", "GitToolchainInfo")
+
+RootfsInfo = provider(fields = {
+    "tar_archive": provider_field(typing.Any, default = None),  # [Artifact]
+})
+
+def rootfs_impl(ctx: AnalysisContext) -> list[[DefaultInfo, RunInfo, RootfsInfo]]:
+
+    rootfs_info = build_rootfs(ctx)
+
+    return [
+        DefaultInfo(
+            default_output = rootfs_info.tar_archive,
+        ),
+    ]
+
+rootfs = rule(
+    impl = rootfs_impl,
+    attrs = {
+        "rootfs_name": attrs.option(
+            attrs.string(),
+            default = None,
+            doc = """The rootfs output filename.""",
+        ),
+        "build_deps": attrs.list(
+            attrs.source(),
+            default = [],
+            doc = """Buck2 targets that could be built into a rootfs.""",
+        ),
+        "_rootfs_toolchain": attrs.toolchain_dep(
+            default = "toolchains//:rootfs",
+            providers = [RootfsToolchainInfo],
+        ),
+        "_git_toolchain": attrs.toolchain_dep(
+            default = "toolchains//:git",
+            providers = [GitToolchainInfo],
+        ),
+    },
+)
+
+def build_rootfs(ctx: AnalysisContext) -> RootfsInfo:
+
+    tar_archive = ctx.actions.declare_output("{}.tar".format("johns-rootfs"))
+
+    rootfs_toolchain = ctx.attrs._rootfs_toolchain[RootfsToolchainInfo]
+    git_toolchain = ctx.attrs._git_toolchain[GitToolchainInfo]
+
+    cmd = cmd_args(
+        "/bin/bash",
+        rootfs_toolchain.rootfs_build[DefaultInfo].default_outputs,
+        git_toolchain.git_info[DefaultInfo].default_outputs,
+        tar_archive.as_output()
+    )
+
+    # Add the dependent binary(s) to the build process
+    for dep in ctx.attrs.build_deps or []:
+        cmd.add(dep)
+
+    ctx.actions.run(cmd, category = "rootfs_build")
+
+    return RootfsInfo(
+        tar_archive = tar_archive,
+    )

prelude-si/rootfs/BUCK

@@ -0,0 +1,12 @@
+load(
+    "@prelude-si//:macros.bzl",
+    "export_file",
+)
+
+export_file(
+    name = "rootfs_build.sh",
+)
+
+export_file(
+    name = "rootfs_promote.sh",
+)

prelude-si/rootfs/rootfs_build.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# TODO(johnrwatson): In theory we should be able to run this task for any of the components we need rootfs' for 
+# but there are some cyclone-specifics bits that will cause us problems
+
+set -eo pipefail
+
+# TODO(johnrwatson): We need to port this to python or similar, and check for OS-dependencies that are required. i.e.
+# docker and basic priviledged escalation for the mounts
+
+git_helper=$1   # i.e. ./${git_helper} | jq -r '.abbreviated_commit_hash' (it returns a json blob output via python)
+output_file=$2  # i.e. output the file ./johns_rootfs.tar
+
+# Shift the parsed arguments off after assignment
+shift 2
+
+# The rest of the inputs are a list of input files or directories, to also include in the build
+# i.e. consume a binary ./johns_binary.bin for use within this script
+binary_inputs=("$@")
+
+echo "-------------------------------------"
+echo "Info: Initiating rootfs build"
+echo "Artifact Version: $(./${git_helper} | jq -r '.artifact_ver')"
+echo "Output File: $output_file"
+echo "Input Binaries (list):"
+for binary_input in "${binary_inputs[@]}"; do
+  echo "$(echo $binary_input | awk -F "/" '{print $NF}') Full Path: $binary_input"
+done
+echo "-------------------------------------"
+
+GITROOT="$(pwd)"
+BUCKROOT="$BUCK_SCRATCH_PATH" # This is provided by buck2
+
+PACKAGEDIR="$BUCKROOT/cyclone-pkg"
+ROOTFS="$PACKAGEDIR/cyclone-rootfs.ext4"
+ROOTFSMOUNT="$PACKAGEDIR/rootfs"
+GUESTDISK="/rootfs"
+INITSCRIPT="$PACKAGEDIR/init.sh"
+
+# create disk and mount to a known location
+mkdir -p $ROOTFSMOUNT
+dd if=/dev/zero of=$ROOTFS bs=1M count=1024
+mkfs.ext4 $ROOTFS
+sudo mount $ROOTFS $ROOTFSMOUNT
+
+# create our script to add an init system to our container image
+cat << EOL > $INITSCRIPT
+apk update
+apk add openrc openssh
+
+apk add --no-cache runuser; \
+adduser -D app; \
+for dir in /run /etc /usr/local/etc /home/app/.config; do \
+    mkdir -pv "$dir/$BIN"; \
+done;
+
+ssh-keygen -A
+
+# Make sure special file systems are mounted on boot:
+rc-update add devfs boot
+rc-update add procfs boot
+rc-update add sysfs boot
+rc-update add local default
+rc-update add networking boot
+rc-update add sshd
+
+# Then, copy the newly configured system to the rootfs image:
+for d in bin dev etc lib root sbin usr nix; do tar c "/\${d}" | tar x -C ${GUESTDISK}; done
+for dir in proc run sys var; do mkdir ${GUESTDISK}/\${dir}; done
+
+# autostart cyclone
+cat << EOF > ${GUESTDISK}/etc/init.d/cyclone
+#!/sbin/openrc-run
+
+name="cyclone"
+description="Cyclone"
+supervisor="supervise-daemon"
+command="cyclone"
+command_args="--bind-vsock 3:52 --decryption-key /dev.decryption.key --lang-server /usr/local/bin/lang-js --enable-watch --limit-requests 1 --watch-timeout 10 --enable-ping --enable-resolver --enable-action-run"
+pidfile="/run/agent.pid"
+EOF
+
+chmod +x ${GUESTDISK}/usr/local/bin/cyclone
+chmod +x ${GUESTDISK}/usr/local/bin/lang-js
+chmod +x ${GUESTDISK}/etc/init.d/cyclone
+
+chroot ${GUESTDISK} rc-update add cyclone boot
+
+# networking bits
+echo "nameserver 8.8.8.8" > ${GUESTDISK}/etc/resolv.conf
+cat << EOZ >${GUESTDISK}/etc/network/interfaces
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet static
+        address 10.0.0.1/30
+        gateway 10.0.0.2
+EOZ
+EOL
+
+# run the script, mounting the disk so we can create a rootfs
+docker run \
+  -v $(pwd)/$ROOTFSMOUNT:$GUESTDISK \
+  -v $(pwd)/$INITSCRIPT:/init.sh \
+  --rm \
+  --entrypoint sh \
+  alpine:3.1  \
+  /init.sh
+
+# TODO(johnrwatson): This can never make it into Production
+# We need to figure out how to pass these decryption keys at all for the services
+# That need them, maybe we need another sub-service specifically for fetching these from
+# a secret provider or similar. Only for cyclone pull the dev decryption key
+[[$(echo $binary_input | awk -F "/" '{print $NF}') == "cyclone" ]] && cp $GITROOT/lib/cyclone-server/src/dev.decryption.key $ROOTFSMOUNT
+
+# cleanup the PACKAGEDIR
+sudo umount $ROOTFSMOUNT
+rm -rf $ROOTFSMOUNT $INITSCRIPT
+
+mv $PACKAGEDIR/cyclone-rootfs.ext4 $output_file

prelude-si/rootfs/rootfs_promote.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "Promted the rootfs ${1:-null} [psuedo]!"