Add support for external resources (#22)

* add support for reading environment variables

* add support for properties and add https: to allowed resources to allow reading packages

* add docs + update readme

* version bump to 0.3.5

* include example for evaluator options

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rpkl"
-version = "0.3.4"
+version = "0.3.5"
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/z-jxy/rpkl"

README.md

@@ -31,7 +31,18 @@ struct Database {
     password: String,
 }
 
-let config: Config = rpkl::value_from_config("./config.pkl")?;
+let config: Config = rpkl::from_config("./config.pkl")?;
+```
+
+### Evaluator Options
+
+You can pass options to the evaluator, such as properties, by using [`from_config_with_options`].
+
+```rust
+let options = EvaluatorOptions::default()
+.properties([("username", "root"), ("password", "password123")]);
+
+let config: Config = rpkl::from_config_with_options("./config.pkl", Some(options))?;
 ```
 
 ## Codegen

examples/evaluator_options.rs

@@ -0,0 +1,23 @@
+use std::path::PathBuf;
+
+use rpkl::api::evaluator::EvaluatorOptions;
+use serde::Deserialize;
+
+#[allow(dead_code)]
+#[derive(Debug, Deserialize)]
+struct Config {
+    path: String,
+    name: String,
+}
+
+fn main() {
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("pkl")
+        .join("allowed-resources.pkl");
+
+    let options = EvaluatorOptions::default().properties([("name", "Ferris")]);
+    let config: Config = rpkl::from_config_with_options(path, Some(options)).unwrap();
+
+    println!("{:?}", config);
+}

src/api/evaluator/mod.rs

@@ -1,19 +1,16 @@
 use std::{
+    collections::HashMap,
     io::Write,
     path::PathBuf,
     process::{Child, Command, Stdio},
 };
 
 use crate::{
     error::{Error, Result},
-    utils,
+    utils::{self, macros::_trace},
 };
 
-pub const EVALUATE_RESPONSE: u64 = 0x24;
-
-use outgoing::{
-    pack_msg, ClientModuleReader, CloseEvaluator, CreateEvaluator, EvaluateRequest, OutgoingMessage,
-};
+use outgoing::{pack_msg, CloseEvaluator, CreateEvaluator, EvaluateRequest, OutgoingMessage};
 use responses::PklServerResponseRaw;
 
 use crate::{api::decoder::pkl_eval_module, pkl::PklMod};
@@ -24,6 +21,58 @@ pub mod responses;
 #[cfg(feature = "trace")]
 use tracing::debug;
 
+pub(crate) const EVALUATE_RESPONSE: u64 = 0x24;
+pub(crate) const CREATE_EVALUATOR_REQUEST_ID: u64 = 135;
+pub(crate) const OUTGOING_MESSAGE_REQUEST_ID: u64 = 9805131;
+
+// options that can be provided to the evaluator, such as properties (-p flag from CLI)
+pub struct EvaluatorOptions {
+    /// Properties to pass to the evaluator. Used to read from `props:` in `.pkl` files.
+    pub properties: Option<HashMap<String, String>>,
+}
+
+impl Default for EvaluatorOptions {
+    fn default() -> Self {
+        Self {
+            properties: Some(HashMap::new()),
+        }
+    }
+}
+
+impl EvaluatorOptions {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    // Add a property to the evaluator options map
+    pub fn property(mut self, key: String, value: String) -> Self {
+        if let Some(properties) = self.properties.as_mut() {
+            properties.insert(key, value);
+        } else {
+            let mut map = HashMap::new();
+            map.insert(key, value);
+            self.properties = Some(map);
+        }
+        self
+    }
+
+    // Set properties for the evaluator. This will replace any existing properties
+    pub fn properties<I, K, V>(mut self, properties: I) -> Self
+    where
+        I: IntoIterator<Item = (K, V)>,
+        K: Into<String>,
+        V: Into<String>,
+    {
+        self.properties = Some(
+            properties
+                .into_iter()
+                .map(|(k, v)| (k.into(), v.into()))
+                .collect(),
+        );
+        self
+    }
+}
+
 pub struct Evaluator {
     pub evaluator_id: i64,
     stdin: std::process::ChildStdin,
@@ -36,25 +85,22 @@ impl Evaluator {
     }
 
     pub fn new() -> Result<Self> {
+        return Self::new_from_options(None);
+    }
+
+    pub fn new_from_options(options: Option<EvaluatorOptions>) -> Result<Self> {
         let mut child = start_pkl(false).map_err(|_e| Error::PklProcessStart)?;
         let child_stdin = child.stdin.as_mut().unwrap();
         let mut child_stdout = child.stdout.take().unwrap();
 
-        let request = OutgoingMessage::CreateEvaluator(CreateEvaluator {
-            request_id: 135,
-            allowed_modules: vec![
-                "pkl:".to_string(),
-                "repl:".to_string(),
-                "file:".to_string(),
-                "customfs:".to_string(),
-            ],
-            client_module_readers: vec![ClientModuleReader {
-                scheme: "customfs".to_string(),
-                has_hierarchical_uris: true,
-                is_globbable: true,
-                is_local: true,
-            }],
-        });
+        let mut evaluator_message = CreateEvaluator::default();
+        if let Some(options) = options {
+            if let Some(props) = options.properties {
+                evaluator_message.properties = Some(props);
+            }
+        }
+
+        let request = OutgoingMessage::CreateEvaluator(evaluator_message);
 
         let serialized_request = pack_msg(request);
 
@@ -94,7 +140,7 @@ impl Evaluator {
             .map_err(|_e| Error::Message("failed to canonicalize pkl module path".into()))?;
 
         let msg = OutgoingMessage::EvaluateRequest(EvaluateRequest {
-            request_id: 9805131,
+            request_id: OUTGOING_MESSAGE_REQUEST_ID,
             evaluator_id: evaluator_id,
             module_uri: format!("file://{}", path.to_str().unwrap()),
         });
@@ -108,6 +154,8 @@ impl Evaluator {
             // return Err(anyhow::anyhow!("expected 0x24, got 0x{:X}", eval_res.header));
         }
 
+        _trace!("eval_res header: {:?}", eval_res.header);
+
         let res = eval_res.response.as_map().unwrap();
         let Some((_, result)) = res.iter().find(|(k, _v)| k.as_str() == Some("result")) else {
             // pkl module evaluation failed, return the error message from pkl

src/api/evaluator/outgoing.rs

@@ -1,6 +1,10 @@
+use std::collections::HashMap;
+
 use codes::*;
 use serde::{Deserialize, Serialize};
 
+use super::CREATE_EVALUATOR_REQUEST_ID;
+
 mod codes {
     pub const CREATE_EVALUATOR: u64 = 0x20;
     pub const EVALUATE_REQUEST: u64 = 0x23;
@@ -12,7 +16,41 @@ mod codes {
 pub struct CreateEvaluator {
     pub request_id: u64,
     pub allowed_modules: Vec<String>,
+    pub allowed_resources: Vec<String>,
     pub client_module_readers: Vec<ClientModuleReader>,
+    pub env: Option<HashMap<String, String>>,
+    pub properties: Option<HashMap<String, String>>,
+}
+
+impl Default for CreateEvaluator {
+    fn default() -> Self {
+        let env_vars: HashMap<String, String> = std::env::vars().collect();
+        Self {
+            request_id: CREATE_EVALUATOR_REQUEST_ID,
+            allowed_modules: vec![
+                "pkl:".into(),
+                "repl:".into(),
+                "file:".into(),
+                "https:".into(),
+                "package:".into(),
+            ],
+            allowed_resources: vec![
+                "env:".into(),
+                "prop:".into(),
+                "package:".into(),
+                "https:".into(),
+                "projectpackage:".into(),
+            ],
+            client_module_readers: vec![ClientModuleReader {
+                scheme: "customfs".to_string(),
+                has_hierarchical_uris: true,
+                is_globbable: true,
+                is_local: true,
+            }],
+            env: Some(env_vars),
+            properties: Some(HashMap::new()),
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize)]

src/lib.rs

@@ -1,3 +1,4 @@
+use api::evaluator::EvaluatorOptions;
 use pkl::Deserializer;
 use pkl::PklSerialize;
 
@@ -17,7 +18,7 @@ use tracing::{debug, error, span, trace, Level};
 #[cfg(feature = "trace")]
 use tracing_subscriber::FmtSubscriber;
 
-/// Evaluates a `.pkl` file and deserializes it as `T`.
+/// Evaluates a `.pkl` file and deserializes it as `T`. If you need to pass options to the evaluator, such as properties, use [`from_config_with_options`].
 ///
 /// `path`: The path to the `.pkl` file
 ///
@@ -56,25 +57,68 @@ pub fn from_config<T>(path: impl AsRef<std::path::Path>) -> Result<T>
 where
     T: Sized + for<'de> serde::Deserialize<'de>,
 {
+    return from_config_with_options(path, None);
+}
+
+/// Allows for passing options to the evaluator, such as properties (e.g. `read("prop:username")`). See [`EvaluatorOptions`] for more information.
+///
+/// # Example
+///
+/// ```ignore
+/// ip = "127.0.0.1"
+/// credentials {
+///     username = read("prop:username")
+///     password = read("prop:password")
+/// }
+/// ```
+/// -------------
+/// ```no_run
+///
+/// use serde::Deserialize;
+///
+/// #[derive(Deserialize)]
+/// struct Config {
+///     ip: String,
+///     database: Database,
+/// }
+///
+/// #[derive(Deserialize)]
+/// struct Credentials {
+///     username: String,
+///     password: String,
+/// }
+///
+/// # fn main() -> Result<(), rpkl::Error> {
+/// let options = EvaluatorOptions::default()
+///     .properties([("username", "root"), ("password", "password123")]);
+/// let config: Database = rpkl::from_config("config.pkl")?;
+/// #    Ok(())
+/// # }
+/// ```
+pub fn from_config_with_options<T>(
+    path: impl AsRef<std::path::Path>,
+    options: Option<EvaluatorOptions>,
+) -> Result<T>
+where
+    T: Sized + for<'de> serde::Deserialize<'de>,
+{
+    #[cfg(feature = "trace")]
     {
-        #[cfg(feature = "trace")]
-        {
-            let subscriber = tracing_subscriber::FmtSubscriber::builder()
-                .with_max_level(Level::TRACE)
-                .finish();
-            tracing::subscriber::set_global_default(subscriber)
-                .expect("setting default subscriber failed");
-        }
+        let subscriber = tracing_subscriber::FmtSubscriber::builder()
+            .with_max_level(Level::TRACE)
+            .finish();
+        tracing::subscriber::set_global_default(subscriber)
+            .expect("setting default subscriber failed");
+    }
 
-        let mut evaluator = api::Evaluator::new()?;
-        let pkl_mod = evaluator.evaluate_module(path.as_ref().to_path_buf())?;
+    let mut evaluator = api::Evaluator::new_from_options(options)?;
+    let pkl_mod = evaluator.evaluate_module(path.as_ref().to_path_buf())?;
 
-        let mut pkld = pkl_mod.serialize_pkl_ast()?;
+    let mut pkld = pkl_mod.serialize_pkl_ast()?;
 
-        #[cfg(feature = "trace")]
-        trace!("serialized pkl data {:?}", pkld);
+    #[cfg(feature = "trace")]
+    trace!("serialized pkl data {:?}", pkld);
 
-        T::deserialize(&mut Deserializer::from_pkl_map(&mut pkld))
-            .map_err(|e| Error::DeserializeError(format!("{}", e)))
-    }
+    T::deserialize(&mut Deserializer::from_pkl_map(&mut pkld))
+        .map_err(|e| Error::DeserializeError(format!("{}", e)))
 }

tests/deserializer.rs

@@ -9,6 +9,7 @@ mod tests {
 
     use rmpv::Value;
     use rpkl::api;
+    use rpkl::api::evaluator::EvaluatorOptions;
     use rpkl::pkl::Deserializer;
     use rpkl::pkl::PklSerialize;
     use serde::Deserialize;
@@ -70,6 +71,31 @@ mod tests {
         assert_eq!(config.pet_name, Some("Doggo".into()));
     }
 
+    #[test]
+    fn resources() {
+        #[cfg(feature = "dhat-heap")]
+        let _profiler = dhat::Profiler::new_heap();
+
+        #[allow(dead_code)]
+        #[derive(Debug, Deserialize)]
+        struct Config {
+            path: String,
+            name: String,
+            // package: rpkl::Value,
+        }
+
+        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("tests")
+            .join("pkl")
+            .join("allowed-resources.pkl");
+
+        let options = EvaluatorOptions::default().properties([("name", "zjxy")]);
+
+        let config = rpkl::from_config_with_options::<Config>(path, Some(options)).unwrap();
+
+        assert_eq!(config.name, "zjxy");
+    }
+
     #[test]
     fn deserialize_time() {
         #[cfg(feature = "dhat-heap")]

tests/pkl/allowed-resources.pkl

@@ -0,0 +1,4 @@
+path = read("env:PATH")
+name = read("prop:name")
+
+// package = read("package://pkg.pkl-lang.org/pkl-pantry/[email protected]#/toml.pkl")