om ci: Copy results back from remote Nix store (#360)

During `om ci run --on ssh://.. -o om.json`

- Copy built paths back to local store but only if `-o is specified
- Also, make sure `-o om.json` creates that JSON file locally

Resolves #358 - and this can also be used as an alternative to remote
builder protocol with its own benefits (like custom CI steps being run
natively).

crates/nix_rs/CHANGELOG.md

@@ -11,8 +11,6 @@
   - Don't hardcode flake schema types
 - **`config`**
   - Don't enable flakes during `NixConfig::get`
-- **`env`**:
-  - use `whoami` crate to find the current user instead of depending on environment variable `USER`
 - Support Nix 2.20
 - **`flake::url`**
   - Add `without_attr`, `get_attr`
@@ -26,7 +24,10 @@
   - Add module (upstreamed from nixci)
   - Add `StoreURI`
   - Avoid running `nix-store` multiple times.
+- **`copy`**:
+  - Takes `NixCopyOptions` now.
 - **`env`**:
+  - use `whoami` crate to find the current user instead of depending on environment variable `USER`
   - `NixEnv::detect`'s logging uses DEBUG level now (formerly INFO)
   - Add Nix installer to `NixEnv`
 - **`command`

crates/nix_rs/src/copy.rs

@@ -3,7 +3,18 @@ use crate::{
     command::{CommandError, NixCmd},
     store::uri::StoreURI,
 };
-use std::path::Path;
+use std::{ffi::OsStr, path::Path};
+
+/// Options for `nix copy`.
+#[derive(Debug, Clone, Default)]
+pub struct NixCopyOptions {
+    /// The URI of the store to copy from.
+    pub from: Option<StoreURI>,
+    /// The URI of the store to copy to.
+    pub to: Option<StoreURI>,
+    /// Do not check signatures.
+    pub no_check_sigs: bool,
+}
 
 /// Copy store paths to a remote Nix store using `nix copy`.
 ///
@@ -12,21 +23,27 @@ use std::path::Path;
 /// * `cmd` - The `nix` command
 /// * `host` - The remote host to copy to
 /// * `paths` - The (locally available) store paths to copy
-pub async fn nix_copy(
+pub async fn nix_copy<I, P>(
     cmd: &NixCmd,
-    store_uri: &StoreURI,
-    paths: &[&Path],
-) -> Result<(), CommandError> {
-    let mut args = vec![
-        "copy".to_string(),
-        "--to".to_string(),
-        store_uri.to_string(),
-    ];
-    for path in paths {
-        args.push(path.to_string_lossy().into_owned());
-    }
+    options: NixCopyOptions,
+    paths: I,
+) -> Result<(), CommandError>
+where
+    I: IntoIterator<Item = P>,
+    P: AsRef<Path> + AsRef<OsStr>,
+{
     cmd.run_with(|cmd| {
-        cmd.args(args);
+        cmd.arg("copy");
+        if let Some(uri) = options.from {
+            cmd.arg("--from").arg(uri.to_string());
+        }
+        if let Some(uri) = options.to {
+            cmd.arg("--to").arg(uri.to_string());
+        }
+        if options.no_check_sigs {
+            cmd.arg("--no-check-sigs");
+        }
+        cmd.args(paths);
     })
     .await?;
     Ok(())

crates/nix_rs/src/store/path.rs

@@ -1,5 +1,10 @@
 //! Store path management
-use std::{convert::Infallible, fmt, path::PathBuf, str::FromStr};
+use std::{
+    convert::Infallible,
+    fmt,
+    path::{Path, PathBuf},
+    str::FromStr,
+};
 
 use serde_with::{DeserializeFromStr, SerializeDisplay};
 
@@ -23,9 +28,15 @@ impl FromStr for StorePath {
     }
 }
 
-impl From<&StorePath> for PathBuf {
-    fn from(sp: &StorePath) -> Self {
-        sp.as_path().clone()
+impl AsRef<Path> for StorePath {
+    fn as_ref(&self) -> &Path {
+        self.as_path().as_ref()
+    }
+}
+
+impl AsRef<std::ffi::OsStr> for StorePath {
+    fn as_ref(&self) -> &std::ffi::OsStr {
+        self.as_path().as_os_str()
     }
 }
 

crates/omnix-ci/src/command/run.rs

@@ -9,12 +9,12 @@ use nix_rs::{
     config::NixConfig,
     flake::{system::System, url::FlakeUrl},
     info::NixInfo,
-    store::uri::StoreURI,
+    store::{path::StorePath, uri::StoreURI},
     system_list::{SystemsList, SystemsListFlakeRef},
 };
 use omnix_common::config::OmConfig;
 use omnix_health::{traits::Checkable, NixHealth};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 use crate::{config::subflakes::SubflakesConfig, flake_ref::FlakeRef, step::core::StepsResult};
 
@@ -136,6 +136,11 @@ impl RunCommand {
             args.push(systems.0 .0.clone());
         }
 
+        if let Some(results_file) = self.results.as_ref() {
+            args.push("-o".to_string());
+            args.push(results_file.to_string_lossy().to_string());
+        }
+
         args.push(self.flake_ref.to_string());
 
         args.extend(self.steps_args.to_cli_args());
@@ -212,7 +217,7 @@ pub async fn ci_run(
 }
 
 /// Results of the 'ci run' command
-#[derive(Debug, Serialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct RunResult {
     /// The systems we are building for
     systems: Vec<System>,
@@ -221,3 +226,16 @@ pub struct RunResult {
     /// CI result for each subflake
     result: HashMap<String, StepsResult>,
 }
+
+impl RunResult {
+    /// Get all store paths mentioned in this type.
+    pub fn all_out_paths(&self) -> Vec<StorePath> {
+        let mut res = vec![];
+        for steps_res in self.result.values() {
+            if let Some(build) = steps_res.build_step.as_ref() {
+                res.extend(build.devour_flake_output.out_paths.clone());
+            }
+        }
+        res
+    }
+}

crates/omnix-ci/src/command/run_remote.rs

@@ -7,9 +7,11 @@ use nix_rs::{
     store::uri::StoreURI,
 };
 use omnix_common::config::OmConfig;
-use std::path::PathBuf;
+use std::{ffi::OsString, os::unix::ffi::OsStringExt, path::PathBuf};
 use tokio::process::Command;
 
+use crate::command::run::RunResult;
+
 use super::run::RunCommand;
 
 /// Path to Rust source corresponding to this (running) instance of Omnix
@@ -29,20 +31,97 @@ pub async fn run_on_remote_store(
 
     let (local_flake_path, local_flake_url) = cache_flake(nixcmd, cfg).await?;
     let omnix_source = PathBuf::from(OMNIX_SOURCE);
+    let StoreURI::SSH(ssh_uri) = store_uri;
 
     // First, copy the flake and omnix source to the remote store, because we will be needing them when running over ssh.
-    nix_rs::copy::nix_copy(nixcmd, store_uri, &[&omnix_source, &local_flake_path]).await?;
+    nix_rs::copy::nix_copy(
+        nixcmd,
+        nix_rs::copy::NixCopyOptions {
+            to: Some(store_uri.clone()),
+            no_check_sigs: true,
+            ..Default::default()
+        },
+        &[&omnix_source, &local_flake_path],
+    )
+    .await?;
+
+    // If the user requested creation of `om.json`, we copy all built store paths back, so that the resultant om.json available locally contains valid paths. `-o` can thus be used to trick omnix into copying build results back to local store.
+    if let Some(results_file) = run_cmd.results.as_ref() {
+        // Create a temp file to hold om.json
+        let om_json_path = path_from_bytes(
+            &run_ssh_with_output(
+                &ssh_uri.to_string(),
+                &[
+                    "nix",
+                    "shell",
+                    "nixpkgs#coreutils",
+                    "-c",
+                    "mktemp",
+                    "-t",
+                    "om.json.XXXXXX",
+                ],
+            )
+            .await?,
+        );
+
+        // Then, SSH and run the same `om ci run` CLI but without the `--on` argument.
+        run_ssh(
+            &ssh_uri.to_string(),
+            &om_cli_with(&RunCommand {
+                on: None,
+                flake_ref: local_flake_url.clone().into(),
+                results: Some(om_json_path.clone()),
+                ..run_cmd.clone()
+            }),
+        )
+        .await?;
 
-    // Then, SSH and run the same `om ci run` CLI but without the `--on` argument.
-    match store_uri {
-        StoreURI::SSH(ssh_uri) => {
-            run_ssh(
+        // Get om.json
+        let om_result: RunResult = serde_json::from_slice(
+            &run_ssh_with_output(
                 &ssh_uri.to_string(),
-                &om_cli_with(run_cmd, &local_flake_url),
+                &["cat", om_json_path.to_string_lossy().as_ref()],
             )
-            .await
-        }
+            .await?,
+        )?;
+
+        // Copy the results back to local store
+        tracing::info!("{}", "📦 Copying built paths back to local store".bold());
+        nix_rs::copy::nix_copy(
+            nixcmd,
+            nix_rs::copy::NixCopyOptions {
+                from: Some(store_uri.clone()),
+                no_check_sigs: true,
+                ..Default::default()
+            },
+            om_result.all_out_paths(),
+        )
+        .await?;
+
+        // Write the om.json to the requested file
+        serde_json::to_writer(std::fs::File::create(results_file)?, &om_result)?;
+        tracing::info!(
+            "Results written to {}",
+            results_file.to_string_lossy().bold()
+        );
+    } else {
+        // Then, SSH and run the same `om ci run` CLI but without the `--on` argument.
+        run_ssh(
+            &ssh_uri.to_string(),
+            &om_cli_with(&RunCommand {
+                on: None,
+                flake_ref: local_flake_url.clone().into(),
+                results: None,
+                ..run_cmd.clone()
+            }),
+        )
+        .await?;
     }
+    Ok(())
+}
+
+fn path_from_bytes(bytes: &[u8]) -> PathBuf {
+    PathBuf::from(OsString::from_vec(bytes.to_vec()))
 }
 
 /// Return the locally cached [FlakeUrl] for the given flake url that points to same selected [ConfigRef].
@@ -61,7 +140,7 @@ async fn cache_flake(nixcmd: &NixCmd, cfg: &OmConfig) -> anyhow::Result<(PathBuf
 /// Construct a `nix run ...` based CLI that runs Omnix using given arguments.
 ///
 /// Omnix itself will be compiled from source ([OMNIX_SOURCE]) if necessary. Thus, this invocation is totally independent and can be run on remote machines, as long as the paths exista on the nix store.
-fn om_cli_with(run_cmd: &RunCommand, flake_url: &FlakeUrl) -> Vec<String> {
+fn om_cli_with(run_cmd: &RunCommand) -> Vec<String> {
     let mut args: Vec<String> = vec![];
 
     let omnix_flake = format!("{}#default", OMNIX_SOURCE);
@@ -77,18 +156,7 @@ fn om_cli_with(run_cmd: &RunCommand, flake_url: &FlakeUrl) -> Vec<String> {
         ]
         .map(&str::to_owned),
     );
-
-    // Re-add current CLI arguments, with a couple of tweaks:
-    args.extend(
-        RunCommand {
-            // Remove --on
-            on: None,
-            // Substitute with flake path from the nix store
-            flake_ref: flake_url.clone().into(),
-            ..run_cmd.clone()
-        }
-        .to_cli_args(),
-    );
+    args.extend(run_cmd.to_cli_args());
     args
 }
 
@@ -104,3 +172,25 @@ async fn run_ssh(host: &str, args: &[String]) -> anyhow::Result<()> {
         .exit_ok()
         .map_err(|e| anyhow::anyhow!("SSH command failed: {}", e))
 }
+
+/// Run SSH command with given arguments and return the stdout.
+async fn run_ssh_with_output<I, S>(host: &str, args: I) -> anyhow::Result<Vec<u8>>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<str>,
+{
+    let mut cmd = Command::new("ssh");
+    cmd.args([host, &shell_words::join(args)]);
+
+    nix_rs::command::trace_cmd_with("🐌", &cmd);
+
+    let output = cmd.output().await?;
+    if output.status.success() {
+        Ok(output.stdout)
+    } else {
+        Err(anyhow::anyhow!(
+            "SSH command failed: {}",
+            String::from_utf8_lossy(&output.stderr)
+        ))
+    }
+}

crates/omnix-ci/src/nix/devour_flake.rs

@@ -35,19 +35,13 @@ pub struct DevourFlakeOutput {
     /// Output paths indexed by name (or pname) of the path if any
     #[serde(rename = "byName")]
     pub by_name: HashMap<String, StorePath>,
-
-    /// The devour-flake output store path from which Self is derived.
-    #[serde(skip_deserializing, rename = "devourOutput")]
-    pub devour_output: PathBuf,
 }
 
 impl DevourFlakeOutput {
     fn from_drv(drv_out: &Path) -> anyhow::Result<Self> {
         // Read drv_out file as JSON, decoding it into DevourFlakeOutput
         let mut out: DevourFlakeOutput = serde_json::from_reader(std::fs::File::open(drv_out)?)
             .context("Failed to parse devour-flake output")?;
-        // Provide the original devour-output store path itself.
-        out.devour_output = drv_out.to_owned();
         // Remove duplicates, which is possible in user's flake
         // e.g., when doing `packages.foo = self'.packages.default`
         out.out_paths.sort();

crates/omnix-ci/src/step/core.rs

@@ -46,7 +46,7 @@ pub struct StepsArgs {
 }
 
 /// Results of [Steps]
-#[derive(Debug, Serialize, Clone, Default)]
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
 pub struct StepsResult {
     /// [BuildStepResult]
     #[serde(rename = "build")]