diff --git a/crates/nix_rs/src/flake/functions/README.md b/crates/nix_rs/src/flake/functions/README.md new file mode 100644 index 00000000..ddeece00 --- /dev/null +++ b/crates/nix_rs/src/flake/functions/README.md @@ -0,0 +1,7 @@ +## Rust + Nix FFI + +https://github.com/srid/devour-flake introduced the idea of defining "functions" in Nix flake, that can be called from any external process. The flakes `inputs` acts as "arguments" taken by this function, with the flake's package output acting as its `output`. + +This package, `nix_rs::flake::functions`, provides the Rust FFI adapter to work with such Nix functions in Rust, using simpler API. + +In effect, this generalizes devour-flake to be able to define such functions. See [`devour_flake.rs`] for an example. diff --git a/crates/nix_rs/src/flake/functions/mod.rs b/crates/nix_rs/src/flake/functions/mod.rs new file mode 100644 index 00000000..6d25909c --- /dev/null +++ b/crates/nix_rs/src/flake/functions/mod.rs @@ -0,0 +1,147 @@ +//! Calling Nix functions (defined in a flake) from Rust, as if to provide FFI. +// +// This model provides a simpler alternative to Flake Schemas, but it can also do more than Flake Schemas can (such as building derivations). + +use super::url::FlakeUrl; +use crate::command::NixCmd; +use serde::{Deserialize, Serialize}; +use std::{ffi::OsString, os::unix::ffi::OsStringExt, path::PathBuf, process::Stdio}; +use tokio::io::{AsyncBufReadExt, BufReader}; + +/// Trait for flake functions +pub trait FlakeFn { + /// Input type, corresponding to flake inputs + /// + /// A field named `flake` will be treated special (extra args' --override-inputs operates on this flake) + type Input; + /// Output generated by building the flake fn + type Output; + + /// Get the flake URL referencing this function + fn flake() -> &'static FlakeUrl; + + /// Initialize the type after reading from Nix build + fn init(out: &mut Self::Output); + + /// Call the flake function, taking `Self::Input`, returning `Self::Output` + fn call( + nixcmd: &NixCmd, + // Whther to avoid the --override-input noise suppression. + verbose: bool, + // Extra arguments to pass to `nix build` + // + // --override-input is treated specially, to account for the flake input named `flake` (as defined in `Self::Input`) + extra_args: Vec, + // The input arguments to the flake function. + input: Self::Input, + ) -> impl std::future::Future> + Send + where + Self::Input: Serialize + Send + Sync, + Self::Output: Sync + for<'de> Deserialize<'de>, + { + async move { + let mut cmd = nixcmd.command(); + cmd.args([ + "build", + Self::flake(), + "-L", + "--no-link", + "--print-out-paths", + ]); + + let input_vec = to_vec(&input); + for (k, v) in input_vec { + cmd.arg("--override-input"); + cmd.arg(k); + cmd.arg(v); + } + + cmd.args(transform_override_inputs(&extra_args)); + + crate::command::trace_cmd(&cmd); + + let mut output_fut = cmd.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; + let stderr_handle = output_fut.stderr.take().unwrap(); + tokio::spawn(async move { + // Suppress --override-input noise, since we expect these to be present. + let mut reader = BufReader::new(stderr_handle).lines(); + while let Some(line) = reader.next_line().await.expect("read stderr") { + if !verbose { + if line.starts_with("• Added input") { + // Consume the input logging itself + reader.next_line().await.expect("read stderr"); + continue; + } else if line + .starts_with("warning: not writing modified lock file of flake") + { + continue; + } + } + eprintln!("{}", line); + } + }); + let output = output_fut.wait_with_output().await?; + if output.status.success() { + let drv_out = + PathBuf::from(OsString::from_vec(output.stdout.trim_ascii_end().into())); + let mut v: Self::Output = serde_json::from_reader(std::fs::File::open(drv_out)?)?; + Self::init(&mut v); + Ok(v) + } else { + Err(Error::NixBuildFailed(output.status.code())) + } + } + } +} + +/// Transform `--override-input` arguments to use `flake/` prefix, which +/// devour_flake expects. +/// +/// NOTE: This assumes that Input struct contains a field named exactly "flake" referring to the flake. We should probably be smart about this. +fn transform_override_inputs(args: &[String]) -> Vec { + let mut new_args = Vec::with_capacity(args.len()); + let mut iter = args.iter().peekable(); + + while let Some(arg) = iter.next() { + new_args.push(arg.clone()); + if arg == "--override-input" { + if let Some(next_arg) = iter.next() { + new_args.push(format!("flake/{}", next_arg)); + } + } + } + + new_args +} + +/// Convert a struct of uniform value types (Option allowed, however) into a vector of fields. The value should be of String kind. +fn to_vec(value: &T) -> Vec<(String, String)> +where + T: Serialize, +{ + let map = serde_json::to_value(value) + .unwrap() + .as_object() + .unwrap_or_else(|| panic!("Bad struct for FlakeFn")) + .clone(); + + map.into_iter() + .filter_map(|(k, v)| v.as_str().map(|v| (k, v.to_string()))) + .collect() +} + +/// Errors associated with `FlakeFn::call` +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// IO error + #[error("IO error: {0}")] + IOError(#[from] std::io::Error), + + /// Non-zero exit code + #[error("`nix build` failed; exit code: {0:?}")] + NixBuildFailed(Option), + + /// JSON error + #[error("JSON error: {0}")] + JSONError(#[from] serde_json::Error), +} diff --git a/crates/nix_rs/src/flake/mod.rs b/crates/nix_rs/src/flake/mod.rs index daec3397..63fa9782 100644 --- a/crates/nix_rs/src/flake/mod.rs +++ b/crates/nix_rs/src/flake/mod.rs @@ -2,6 +2,7 @@ pub mod command; pub mod eval; +pub mod functions; pub mod metadata; pub mod outputs; pub mod schema; diff --git a/crates/omnix-ci/src/command/core.rs b/crates/omnix-ci/src/command/core.rs index 72dbcc7f..dfec6f0d 100644 --- a/crates/omnix-ci/src/command/core.rs +++ b/crates/omnix-ci/src/command/core.rs @@ -27,13 +27,6 @@ impl Default for Command { } impl Command { - /// Pre-process `Command` - pub fn preprocess(&mut self) { - if let Command::Run(cmd) = self { - cmd.preprocess() - } - } - /// Run the command #[instrument(name = "run", skip(self))] pub async fn run(self, nixcmd: &NixCmd, verbose: bool) -> anyhow::Result<()> { diff --git a/crates/omnix-ci/src/command/run.rs b/crates/omnix-ci/src/command/run.rs index 4f437671..19388857 100644 --- a/crates/omnix-ci/src/command/run.rs +++ b/crates/omnix-ci/src/command/run.rs @@ -71,18 +71,6 @@ impl Default for RunCommand { } impl RunCommand { - /// Preprocess this command - pub fn preprocess(&mut self) { - if !self.is_remote() { - self.steps_args.build_step_args.preprocess(); - } - } - - /// Whether this command is to be run remotely - pub fn is_remote(&self) -> bool { - self.on.is_some() - } - /// Get the out-link path pub fn get_out_link(&self) -> Option<&PathBuf> { if self.no_out_link { diff --git a/crates/omnix-ci/src/nix/devour_flake.rs b/crates/omnix-ci/src/nix/devour_flake.rs index d948212d..94c74559 100644 --- a/crates/omnix-ci/src/nix/devour_flake.rs +++ b/crates/omnix-ci/src/nix/devour_flake.rs @@ -1,25 +1,44 @@ //! Rust support for invoking -// TODO: Create a more general version of this module, where a function body is defined in Nix, but FFI invoked (as it were) from Rust. - -use anyhow::{bail, Context, Result}; -use nix_rs::{command::NixCmd, flake::url::FlakeUrl, store::path::StorePath}; -use serde::{Deserialize, Serialize}; -use std::{ - collections::HashMap, - ffi::OsString, - os::unix::ffi::OsStringExt, - path::{Path, PathBuf}, - process::Stdio, +use lazy_static::lazy_static; +use nix_rs::{ + flake::{functions::FlakeFn, url::FlakeUrl}, + store::path::StorePath, }; -use tokio::io::{AsyncBufReadExt, BufReader}; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, path::Path}; + +/// Devour all outputs of a flake producing their store paths +pub struct DevourFlake; + +lazy_static! { + /// devour flake URL + static ref DEVOUR_FLAKE: FlakeUrl = { + let path = env!("DEVOUR_FLAKE"); + Into::::into(Path::new(path)).with_attr("json") + }; +} + +impl FlakeFn for DevourFlake { + type Input = DevourFlakeInput; + type Output = DevourFlakeOutput; + + fn flake() -> &'static FlakeUrl { + &DEVOUR_FLAKE + } -/// Absolute path to the devour-flake flake source -pub const DEVOUR_FLAKE: &str = env!("DEVOUR_FLAKE"); + fn init(out: &mut DevourFlakeOutput) { + // Remove duplicates, which is possible in user's flake + // e.g., when doing `packages.foo = self'.packages.default` + out.out_paths.sort(); + out.out_paths.dedup(); + } +} /// Input arguments to devour-flake +#[derive(Serialize)] pub struct DevourFlakeInput { - /// The flake devour-flake will build + /// The flake whose outputs will be built pub flake: FlakeUrl, /// The systems it will build for. An empty list means all allowed systems. pub systems: Option, @@ -36,91 +55,3 @@ pub struct DevourFlakeOutput { #[serde(rename = "byName")] pub by_name: HashMap, } - -impl DevourFlakeOutput { - fn from_drv(drv_out: &Path) -> anyhow::Result { - // Read drv_out file as JSON, decoding it into DevourFlakeOutput - let mut out: DevourFlakeOutput = serde_json::from_reader(std::fs::File::open(drv_out)?) - .context("Failed to parse devour-flake output")?; - // Remove duplicates, which is possible in user's flake - // e.g., when doing `packages.foo = self'.packages.default` - out.out_paths.sort(); - out.out_paths.dedup(); - Ok(out) - } -} - -/// Run `devour-flake` -pub async fn devour_flake( - nixcmd: &NixCmd, - verbose: bool, - input: DevourFlakeInput, - extra_args: Vec, -) -> Result { - // TODO: Use nix_rs here as well - // In the context of doing https://github.com/srid/nixci/issues/15 - let devour_flake_url = format!("{}#json", env!("DEVOUR_FLAKE")); - let mut cmd = nixcmd.command(); - - let mut args = vec![ - "build", - &devour_flake_url, - "-L", - "--no-link", - "--print-out-paths", - "--override-input", - "flake", - &input.flake, - ]; - // Specify only if the systems is not the default - if let Some(systems) = input.systems.as_ref() { - args.extend(&["--override-input", "systems", &systems.0]); - } - args.extend(extra_args.iter().map(|s| s.as_str())); - cmd.args(args); - - nix_rs::command::trace_cmd(&cmd); - let mut output_fut = cmd.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; - let stderr_handle = output_fut.stderr.take().unwrap(); - tokio::spawn(async move { - let mut reader = BufReader::new(stderr_handle).lines(); - while let Some(line) = reader.next_line().await.expect("read stderr") { - if !verbose { - if line.starts_with("• Added input") { - // Consume the input logging itself - reader.next_line().await.expect("read stderr"); - continue; - } else if line.starts_with("warning: not writing modified lock file of flake") { - continue; - } - } - eprintln!("{}", line); - } - }); - let output = output_fut - .wait_with_output() - .await - .context("Unable to spawn devour-flake process")?; - if output.status.success() { - let drv_out = PathBuf::from(OsString::from_vec(output.stdout.trim_ascii_end().into())); - let v = DevourFlakeOutput::from_drv(&drv_out)?; - Ok(v) - } else { - let exit_code = output.status.code().unwrap_or(1); - bail!("devour-flake failed to run (exited: {})", exit_code); - } -} - -/// Transform `--override-input` arguments to use `flake/` prefix, which -/// devour_flake expects. -pub fn transform_override_inputs(args: &mut [String]) { - let mut iter = args.iter_mut().peekable(); - - while let Some(arg) = iter.next() { - if *arg == "--override-input" { - if let Some(next_arg) = iter.next() { - *next_arg = format!("flake/{}", next_arg); - } - } - } -} diff --git a/crates/omnix-ci/src/step/build.rs b/crates/omnix-ci/src/step/build.rs index d2285247..be616382 100644 --- a/crates/omnix-ci/src/step/build.rs +++ b/crates/omnix-ci/src/step/build.rs @@ -3,7 +3,7 @@ use clap::Parser; use colored::Colorize; use nix_rs::{ command::NixCmd, - flake::url::FlakeUrl, + flake::{functions::FlakeFn, url::FlakeUrl}, store::{command::NixStoreCmd, path::StorePath}, }; use serde::{Deserialize, Serialize}; @@ -11,10 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::{ command::run::RunCommand, config::subflake::SubflakeConfig, - nix::{ - self, - devour_flake::{self, DevourFlakeInput}, - }, + nix::devour_flake::{DevourFlake, DevourFlakeInput, DevourFlakeOutput}, }; /// Represents a build step in the CI pipeline @@ -48,12 +45,16 @@ impl BuildStep { format!("⚒️ Building subflake: {}", subflake.dir).bold() ); let nix_args = subflake_extra_args(subflake, &run_cmd.steps_args.build_step_args); - let devour_input = DevourFlakeInput { - flake: url.sub_flake_url(subflake.dir.clone()), - systems: run_cmd.systems.clone().map(|l| l.0), - }; - let output = - nix::devour_flake::devour_flake(nixcmd, verbose, devour_input, nix_args).await?; + let output = DevourFlake::call( + nixcmd, + verbose, + nix_args, + DevourFlakeInput { + flake: url.sub_flake_url(subflake.dir.clone()), + systems: run_cmd.systems.clone().map(|l| l.0), + }, + ) + .await?; let mut res = BuildStepResult { devour_flake_output: output, @@ -79,7 +80,7 @@ fn subflake_extra_args(subflake: &SubflakeConfig, build_step_args: &BuildStepArg for (k, v) in &subflake.override_inputs { args.extend([ "--override-input".to_string(), - format!("flake/{}", k), + k.to_string(), v.0.to_string(), ]) } @@ -109,12 +110,6 @@ pub struct BuildStepArgs { } impl BuildStepArgs { - /// Preprocess the arguments - pub fn preprocess(&mut self) { - // Adjust to devour_flake's expectations - devour_flake::transform_override_inputs(&mut self.extra_nix_build_args); - } - /// Convert this type back to the user-facing command line arguments pub fn to_cli_args(&self) -> Vec { let mut args = vec![]; @@ -139,7 +134,7 @@ impl BuildStepArgs { pub struct BuildStepResult { /// Output of devour-flake #[serde(flatten)] - pub devour_flake_output: devour_flake::DevourFlakeOutput, + pub devour_flake_output: DevourFlakeOutput, /// All dependencies of the out paths, if available #[serde(skip_serializing_if = "Option::is_none", rename = "allDeps")] diff --git a/crates/omnix-cli/src/command/ci.rs b/crates/omnix-cli/src/command/ci.rs index 07fd2b65..0540984e 100644 --- a/crates/omnix-cli/src/command/ci.rs +++ b/crates/omnix-cli/src/command/ci.rs @@ -27,8 +27,6 @@ impl CICommand { /// /// If the user has not provided one, return the build command by default. fn command(&self) -> Command { - let mut cmd = self.command.clone().unwrap_or_default(); - cmd.preprocess(); - cmd + self.command.clone().unwrap_or_default() } }