WIP

crates/nix_rs/src/flake/functions/README.md

@@ -0,0 +1,7 @@
+## Rust + Nix FFI
+
+https://github.com/srid/devour-flake introduced the idea of defining "functions" in Nix flake, that can be called from any external process. The flakes `inputs` acts as "arguments" taken by this function, with the flake's package output acting as its `output`. 
+
+This package, `nix_rs::flake::functions`, provides the Rust FFI adapter to work with such Nix functions in Rust, using simpler API.
+
+In effect, this generalizes devour-flake to be able to define such functions. See [`devour_flake.rs`] for an example.

crates/nix_rs/src/flake/functions/mod.rs

@@ -0,0 +1,147 @@
+//! Calling Nix functions (defined in a flake) from Rust, as if to provide FFI.
+//
+// This model provides a simpler alternative to Flake Schemas, but it can also do more than Flake Schemas can (such as building derivations).
+
+use super::url::FlakeUrl;
+use crate::command::NixCmd;
+use serde::{Deserialize, Serialize};
+use std::{ffi::OsString, os::unix::ffi::OsStringExt, path::PathBuf, process::Stdio};
+use tokio::io::{AsyncBufReadExt, BufReader};
+
+/// Trait for flake functions
+pub trait FlakeFn {
+    /// Input type, corresponding to flake inputs
+    ///
+    /// A field named `flake` will be treated special (extra args' --override-inputs operates on this flake)
+    type Input;
+    /// Output generated by building the flake fn
+    type Output;
+
+    /// Get the flake URL referencing this function
+    fn flake() -> &'static FlakeUrl;
+
+    /// Initialize the type after reading from Nix build
+    fn init(out: &mut Self::Output);
+
+    /// Call the flake function, taking `Self::Input`, returning `Self::Output`
+    fn call(
+        nixcmd: &NixCmd,
+        // Whther to avoid the --override-input noise suppression.
+        verbose: bool,
+        // Extra arguments to pass to `nix build`
+        //
+        // --override-input is treated specially, to account for the flake input named `flake` (as defined in `Self::Input`)
+        extra_args: Vec<String>,
+        // The input arguments to the flake function.
+        input: Self::Input,
+    ) -> impl std::future::Future<Output = Result<Self::Output, Error>> + Send
+    where
+        Self::Input: Serialize + Send + Sync,
+        Self::Output: Sync + for<'de> Deserialize<'de>,
+    {
+        async move {
+            let mut cmd = nixcmd.command();
+            cmd.args([
+                "build",
+                Self::flake(),
+                "-L",
+                "--no-link",
+                "--print-out-paths",
+            ]);
+
+            let input_vec = to_vec(&input);
+            for (k, v) in input_vec {
+                cmd.arg("--override-input");
+                cmd.arg(k);
+                cmd.arg(v);
+            }
+
+            cmd.args(transform_override_inputs(&extra_args));
+
+            crate::command::trace_cmd(&cmd);
+
+            let mut output_fut = cmd.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?;
+            let stderr_handle = output_fut.stderr.take().unwrap();
+            tokio::spawn(async move {
+                // Suppress --override-input noise, since we expect these to be present.
+                let mut reader = BufReader::new(stderr_handle).lines();
+                while let Some(line) = reader.next_line().await.expect("read stderr") {
+                    if !verbose {
+                        if line.starts_with("• Added input") {
+                            // Consume the input logging itself
+                            reader.next_line().await.expect("read stderr");
+                            continue;
+                        } else if line
+                            .starts_with("warning: not writing modified lock file of flake")
+                        {
+                            continue;
+                        }
+                    }
+                    eprintln!("{}", line);
+                }
+            });
+            let output = output_fut.wait_with_output().await?;
+            if output.status.success() {
+                let drv_out =
+                    PathBuf::from(OsString::from_vec(output.stdout.trim_ascii_end().into()));
+                let mut v: Self::Output = serde_json::from_reader(std::fs::File::open(drv_out)?)?;
+                Self::init(&mut v);
+                Ok(v)
+            } else {
+                Err(Error::NixBuildFailed(output.status.code()))
+            }
+        }
+    }
+}
+
+/// Transform `--override-input` arguments to use `flake/` prefix, which
+/// devour_flake expects.
+///
+/// NOTE: This assumes that Input struct contains a field named exactly "flake" referring to the flake. We should probably be smart about this.
+fn transform_override_inputs(args: &[String]) -> Vec<String> {
+    let mut new_args = Vec::with_capacity(args.len());
+    let mut iter = args.iter().peekable();
+
+    while let Some(arg) = iter.next() {
+        new_args.push(arg.clone());
+        if arg == "--override-input" {
+            if let Some(next_arg) = iter.next() {
+                new_args.push(format!("flake/{}", next_arg));
+            }
+        }
+    }
+
+    new_args
+}
+
+/// Convert a struct of uniform value types (Option allowed, however) into a vector of fields. The value should be of String kind.
+fn to_vec<T>(value: &T) -> Vec<(String, String)>
+where
+    T: Serialize,
+{
+    let map = serde_json::to_value(value)
+        .unwrap()
+        .as_object()
+        .unwrap_or_else(|| panic!("Bad struct for FlakeFn"))
+        .clone();
+
+    map.into_iter()
+        .filter_map(|(k, v)| v.as_str().map(|v| (k, v.to_string())))
+        .collect()
+}
+
+/// Errors associated with `FlakeFn::call`
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    /// IO error
+    #[error("IO error: {0}")]
+    IOError(#[from] std::io::Error),
+
+    /// Non-zero exit code
+    #[error("`nix build` failed; exit code: {0:?}")]
+    NixBuildFailed(Option<i32>),
+
+    /// JSON error
+    #[error("JSON error: {0}")]
+    JSONError(#[from] serde_json::Error),
+}

crates/nix_rs/src/flake/mod.rs

@@ -2,6 +2,7 @@
 
 pub mod command;
 pub mod eval;
+pub mod functions;
 pub mod metadata;
 pub mod outputs;
 pub mod schema;

crates/omnix-ci/src/command/core.rs

@@ -27,13 +27,6 @@ impl Default for Command {
 }
 
 impl Command {
-    /// Pre-process `Command`
-    pub fn preprocess(&mut self) {
-        if let Command::Run(cmd) = self {
-            cmd.preprocess()
-        }
-    }
-
     /// Run the command
     #[instrument(name = "run", skip(self))]
     pub async fn run(self, nixcmd: &NixCmd, verbose: bool) -> anyhow::Result<()> {

crates/omnix-ci/src/command/run.rs

@@ -71,18 +71,6 @@ impl Default for RunCommand {
 }
 
 impl RunCommand {
-    /// Preprocess this command
-    pub fn preprocess(&mut self) {
-        if !self.is_remote() {
-            self.steps_args.build_step_args.preprocess();
-        }
-    }
-
-    /// Whether this command is to be run remotely
-    pub fn is_remote(&self) -> bool {
-        self.on.is_some()
-    }
-
     /// Get the out-link path
     pub fn get_out_link(&self) -> Option<&PathBuf> {
         if self.no_out_link {

crates/omnix-ci/src/nix/devour_flake.rs

@@ -1,25 +1,44 @@
 //! Rust support for invoking <https://github.com/srid/devour-flake>
 
-// TODO: Create a more general version of this module, where a function body is defined in Nix, but FFI invoked (as it were) from Rust.
-
-use anyhow::{bail, Context, Result};
-use nix_rs::{command::NixCmd, flake::url::FlakeUrl, store::path::StorePath};
-use serde::{Deserialize, Serialize};
-use std::{
-    collections::HashMap,
-    ffi::OsString,
-    os::unix::ffi::OsStringExt,
-    path::{Path, PathBuf},
-    process::Stdio,
+use lazy_static::lazy_static;
+use nix_rs::{
+    flake::{functions::FlakeFn, url::FlakeUrl},
+    store::path::StorePath,
 };
-use tokio::io::{AsyncBufReadExt, BufReader};
+use serde::{Deserialize, Serialize};
+use std::{collections::HashMap, path::Path};
+
+/// Devour all outputs of a flake producing their store paths
+pub struct DevourFlake;
+
+lazy_static! {
+    /// devour flake URL
+    static ref DEVOUR_FLAKE: FlakeUrl = {
+        let path = env!("DEVOUR_FLAKE");
+        Into::<FlakeUrl>::into(Path::new(path)).with_attr("json")
+    };
+}
+
+impl FlakeFn for DevourFlake {
+    type Input = DevourFlakeInput;
+    type Output = DevourFlakeOutput;
+
+    fn flake() -> &'static FlakeUrl {
+        &DEVOUR_FLAKE
+    }
 
-/// Absolute path to the devour-flake flake source
-pub const DEVOUR_FLAKE: &str = env!("DEVOUR_FLAKE");
+    fn init(out: &mut DevourFlakeOutput) {
+        // Remove duplicates, which is possible in user's flake
+        // e.g., when doing `packages.foo = self'.packages.default`
+        out.out_paths.sort();
+        out.out_paths.dedup();
+    }
+}
 
 /// Input arguments to devour-flake
+#[derive(Serialize)]
 pub struct DevourFlakeInput {
-    /// The flake devour-flake will build
+    /// The flake whose outputs will be built
     pub flake: FlakeUrl,
     /// The systems it will build for. An empty list means all allowed systems.
     pub systems: Option<FlakeUrl>,
@@ -36,91 +55,3 @@ pub struct DevourFlakeOutput {
     #[serde(rename = "byName")]
     pub by_name: HashMap<String, StorePath>,
 }
-
-impl DevourFlakeOutput {
-    fn from_drv(drv_out: &Path) -> anyhow::Result<Self> {
-        // Read drv_out file as JSON, decoding it into DevourFlakeOutput
-        let mut out: DevourFlakeOutput = serde_json::from_reader(std::fs::File::open(drv_out)?)
-            .context("Failed to parse devour-flake output")?;
-        // Remove duplicates, which is possible in user's flake
-        // e.g., when doing `packages.foo = self'.packages.default`
-        out.out_paths.sort();
-        out.out_paths.dedup();
-        Ok(out)
-    }
-}
-
-/// Run `devour-flake`
-pub async fn devour_flake(
-    nixcmd: &NixCmd,
-    verbose: bool,
-    input: DevourFlakeInput,
-    extra_args: Vec<String>,
-) -> Result<DevourFlakeOutput> {
-    // TODO: Use nix_rs here as well
-    // In the context of doing https://github.com/srid/nixci/issues/15
-    let devour_flake_url = format!("{}#json", env!("DEVOUR_FLAKE"));
-    let mut cmd = nixcmd.command();
-
-    let mut args = vec![
-        "build",
-        &devour_flake_url,
-        "-L",
-        "--no-link",
-        "--print-out-paths",
-        "--override-input",
-        "flake",
-        &input.flake,
-    ];
-    // Specify only if the systems is not the default
-    if let Some(systems) = input.systems.as_ref() {
-        args.extend(&["--override-input", "systems", &systems.0]);
-    }
-    args.extend(extra_args.iter().map(|s| s.as_str()));
-    cmd.args(args);
-
-    nix_rs::command::trace_cmd(&cmd);
-    let mut output_fut = cmd.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?;
-    let stderr_handle = output_fut.stderr.take().unwrap();
-    tokio::spawn(async move {
-        let mut reader = BufReader::new(stderr_handle).lines();
-        while let Some(line) = reader.next_line().await.expect("read stderr") {
-            if !verbose {
-                if line.starts_with("• Added input") {
-                    // Consume the input logging itself
-                    reader.next_line().await.expect("read stderr");
-                    continue;
-                } else if line.starts_with("warning: not writing modified lock file of flake") {
-                    continue;
-                }
-            }
-            eprintln!("{}", line);
-        }
-    });
-    let output = output_fut
-        .wait_with_output()
-        .await
-        .context("Unable to spawn devour-flake process")?;
-    if output.status.success() {
-        let drv_out = PathBuf::from(OsString::from_vec(output.stdout.trim_ascii_end().into()));
-        let v = DevourFlakeOutput::from_drv(&drv_out)?;
-        Ok(v)
-    } else {
-        let exit_code = output.status.code().unwrap_or(1);
-        bail!("devour-flake failed to run (exited: {})", exit_code);
-    }
-}
-
-/// Transform `--override-input` arguments to use `flake/` prefix, which
-/// devour_flake expects.
-pub fn transform_override_inputs(args: &mut [String]) {
-    let mut iter = args.iter_mut().peekable();
-
-    while let Some(arg) = iter.next() {
-        if *arg == "--override-input" {
-            if let Some(next_arg) = iter.next() {
-                *next_arg = format!("flake/{}", next_arg);
-            }
-        }
-    }
-}