Merge pull request #362 from juspay/omresult

- `-o` (`--out-link`) will now put the results in the Nix store and
create a symlink with the given name. By default, `--out-link=result` ..
ergo, `om ci run` behaves just like `nix build` in that it will create a
`result` link locally which is then symlinked to from the "indirect
roots" (auto directory).
- Note that this doesn't transitively pin the built paths (due to the
results file being created [without string
context](https://shealevy.com/blog/2018/08/05/understanding-nixs-string-context/),
something we will address in latter PR.
  - out-link creation can be disabled with `--no-out-link`
- `--on ssh://` (unless `--no-out-link` is passed) will automatically
copy the built paths -- both the out-link and the built paths -- back to
the local store.

crates/nix_rs/Cargo.toml

@@ -27,6 +27,7 @@ colored = { workspace = true }
 shell-words = { workspace = true }
 is_proc_translated = { workspace = true }
 sysinfo = { workspace = true }
+tempfile = { workspace = true }
 bytesize = { workspace = true }
 clap = { workspace = true, optional = true }
 nonempty = { workspace = true }

crates/nix_rs/src/store/command.rs

@@ -1,8 +1,9 @@
 //! Rust wrapper for `nix-store`
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use crate::command::{CommandError, NixCmdError};
 use serde::{Deserialize, Serialize};
+use tempfile::TempDir;
 use thiserror::Error;
 use tokio::process::Command;
 
@@ -51,25 +52,15 @@ impl NixStoreCmd {
         cmd.args(["--query", "--valid-derivers"])
             .args(out_paths.iter().map(StorePath::as_path));
 
-        crate::command::trace_cmd(&cmd);
-
-        let out = cmd.output().await?;
-        if out.status.success() {
-            let drv_paths: Vec<PathBuf> = String::from_utf8(out.stdout)?
-                .lines()
-                .map(PathBuf::from)
-                .collect();
-            if drv_paths.contains(&PathBuf::from("unknown-deriver")) {
-                return Err(NixStoreCmdError::UnknownDeriver);
-            }
-            Ok(drv_paths)
-        } else {
-            // TODO(refactor): When upstreaming this module to nix-rs, create a
-            // nicer and unified way to create `ProcessFailed`
-            let stderr = Some(String::from_utf8_lossy(&out.stderr).to_string());
-            let exit_code = out.status.code();
-            Err(CommandError::ProcessFailed { stderr, exit_code }.into())
+        let stdout = run_awaiting_stdout(&mut cmd).await?;
+        let drv_paths: Vec<PathBuf> = String::from_utf8(stdout)?
+            .lines()
+            .map(PathBuf::from)
+            .collect();
+        if drv_paths.contains(&PathBuf::from("unknown-deriver")) {
+            return Err(NixStoreCmdError::UnknownDeriver);
         }
+        Ok(drv_paths)
     }
 
     /// Given the derivation paths, this function recursively queries and return all
@@ -82,20 +73,67 @@ impl NixStoreCmd {
         cmd.args(["--query", "--requisites", "--include-outputs"])
             .args(drv_paths);
 
-        crate::command::trace_cmd(&cmd);
-
-        let out = cmd.output().await?;
-        if out.status.success() {
-            Ok(String::from_utf8(out.stdout)?
-                .lines()
-                .map(|line| StorePath::new(PathBuf::from(line)))
-                .collect())
-        } else {
-            // TODO(refactor): see above
-            let stderr = Some(String::from_utf8_lossy(&out.stderr).to_string());
-            let exit_code = out.status.code();
-            Err(CommandError::ProcessFailed { stderr, exit_code }.into())
-        }
+        let stdout = run_awaiting_stdout(&mut cmd).await?;
+        Ok(String::from_utf8(stdout)?
+            .lines()
+            .map(|line| StorePath::new(PathBuf::from(line)))
+            .collect())
+    }
+
+    /// Create a file in the Nix store such that it escapes garbage collection.
+    ///
+    /// Return the nix store path added.
+    pub async fn add_file_permanently(
+        &self,
+        symlink: &Path,
+        contents: &str,
+    ) -> Result<StorePath, NixStoreCmdError> {
+        let temp_dir = TempDir::with_prefix("omnix-ci-")?;
+        let temp_file = temp_dir.path().join("om.json");
+        std::fs::write(&temp_file, contents)?;
+
+        let path = self.nix_store_add(&temp_file).await?;
+        self.nix_store_add_root(symlink, &[&path]).await?;
+        Ok(path)
+    }
+
+    /// Run `nix-store --add` on the give path and return the store path added.
+    pub async fn nix_store_add(&self, path: &Path) -> Result<StorePath, NixStoreCmdError> {
+        let mut cmd = self.command();
+        cmd.arg("--add").arg(path);
+
+        let stdout = run_awaiting_stdout(&mut cmd).await?;
+        Ok(StorePath::new(PathBuf::from(
+            String::from_utf8(stdout)?.trim_end(),
+        )))
+    }
+
+    /// Run `nix-store --add-root` on the given paths and return the store path added.
+    pub async fn nix_store_add_root(
+        &self,
+        symlink: &Path,
+        paths: &[&StorePath],
+    ) -> Result<(), NixStoreCmdError> {
+        let mut cmd = self.command();
+        cmd.arg("--add-root")
+            .arg(symlink)
+            .arg("--realise")
+            .args(paths);
+
+        run_awaiting_stdout(&mut cmd).await?;
+        Ok(())
+    }
+}
+
+async fn run_awaiting_stdout(cmd: &mut Command) -> Result<Vec<u8>, NixStoreCmdError> {
+    crate::command::trace_cmd(cmd);
+    let out = cmd.output().await?;
+    if out.status.success() {
+        Ok(out.stdout)
+    } else {
+        let stderr = Some(String::from_utf8_lossy(&out.stderr).to_string());
+        let exit_code = out.status.code();
+        Err(CommandError::ProcessFailed { stderr, exit_code }.into())
     }
 }
 

crates/omnix-ci/src/command/run.rs

@@ -9,7 +9,7 @@ use nix_rs::{
     config::NixConfig,
     flake::{system::System, url::FlakeUrl},
     info::NixInfo,
-    store::{path::StorePath, uri::StoreURI},
+    store::{command::NixStoreCmd, path::StorePath, uri::StoreURI},
     system_list::{SystemsList, SystemsListFlakeRef},
 };
 use omnix_common::config::OmConfig;
@@ -38,9 +38,19 @@ pub struct RunCommand {
     #[arg(long)]
     pub systems: Option<SystemsListFlakeRef>,
 
-    /// Path to write the results of the CI run (in JSON) to
-    #[arg(long, short = 'o')]
-    pub results: Option<PathBuf>,
+    /// Symlink to create to build results JSON. Defaults to `result`
+    #[arg(
+        long,
+        short = 'o',
+        default_value = "result",
+        conflicts_with = "no_out_link",
+        alias = "results" // For backwards compat
+    )]
+    out_link: Option<PathBuf>,
+
+    /// Do not create a symlink to build results JSON
+    #[arg(long)]
+    no_out_link: bool,
 
     /// Flake URL or github URL
     ///
@@ -66,6 +76,25 @@ impl RunCommand {
         self.steps_args.build_step_args.preprocess();
     }
 
+    /// Get the out-link path
+    pub fn get_out_link(&self) -> Option<&PathBuf> {
+        if self.no_out_link {
+            None
+        } else {
+            self.out_link.as_ref()
+        }
+    }
+
+    /// Override the flake_ref and out_link for building locally.
+    pub fn local_with(&self, flake_ref: FlakeRef, out_link: Option<PathBuf>) -> Self {
+        let mut new = self.clone();
+        new.on = None; // Disable remote building
+        new.flake_ref = flake_ref;
+        new.no_out_link = out_link.is_none();
+        new.out_link = out_link;
+        new
+    }
+
     /// Run the build command which decides whether to do ci run on current machine or a remote machine
     pub async fn run(&self, nixcmd: &NixCmd, verbose: bool, cfg: OmConfig) -> anyhow::Result<()> {
         match &self.on {
@@ -96,11 +125,14 @@ impl RunCommand {
         );
         let res = ci_run(nixcmd, verbose, self, &cfg, &nix_info.nix_config).await?;
 
-        if let Some(results_file) = self.results.as_ref() {
-            serde_json::to_writer(std::fs::File::create(results_file)?, &res)?;
+        if let Some(out_link) = self.get_out_link() {
+            let s = serde_json::to_string(&res)?;
+            let nix_store = NixStoreCmd {};
+            let results_path = nix_store.add_file_permanently(out_link, &s).await?;
             tracing::info!(
-                "Results written to {}",
-                results_file.to_string_lossy().bold()
+                "Result available at {:?} and symlinked at {:?}",
+                results_path.as_path(),
+                out_link
             );
         }
 
@@ -136,9 +168,13 @@ impl RunCommand {
             args.push(systems.0 .0.clone());
         }
 
-        if let Some(results_file) = self.results.as_ref() {
-            args.push("-o".to_string());
-            args.push(results_file.to_string_lossy().to_string());
+        if let Some(out_link) = self.out_link.as_ref() {
+            args.push("--out-link".to_string());
+            args.push(out_link.to_string_lossy().to_string());
+        }
+
+        if self.no_out_link {
+            args.push("--no-out-link".to_string());
         }
 
         args.push(self.flake_ref.to_string());