Config logging (#71)

* Changed the release version of the histogram to compile for webpack * Updated percent-encodings to v2 - The predefined encodings have been removed in v2 (https://github.com/servo/rust-url/blob/master/UPGRADING.md#upgrading-from-percent-encoding-1x-to-2x) - Created our own encodings to match the old ones and added a new 'non-alphanumeric' encoding that has been added - Updated the docs to include the new encoding * Updated percent-encodings to v2 - The predefined encodings have been removed in v2 (https://github.com/servo/rust-url/blob/master/UPGRADING.md#upgrading-from-percent-encoding-1x-to-2x) - Created our own encodings to match the old ones and added a new 'non-alphanumeric' encoding that has been added - Updated the docs to include the new encoding * Added logging to the config parser 'from yaml' functions * Added debug and trace logs to the config parser * cargo fmt changes * Fixed wasm build issues by deriving debug so we can log
FamilySearch · Nov 8, 2021 · 20a6304 · 20a6304
1 parent af83556
commit 20a6304
Show file tree

Hide file tree

Showing 10 changed files with 288 additions and 69 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -10,6 +10,10 @@ Changes:
 - Changed the default try script output to log `headers_all` rather than `headers`. There were complaints about not seeing duplicate headers causing confusion over what was being sent.
 - Change try script output to go through stdout instead of stderr.
 - Modified the Config WebAssembly (config-wasm) to also return file body paths from the `getInputFiles()` method.
+- Added a new `encode()` option. `encode(value, "non-alphanumeric")` will encode all characters that are not an ASCII letter or digit.
+
+Bug fixes:
+- Upgrade percent-encoding and other dependencies
 
 ### v0.5.9
 Bug fixes:

diff --git a/guide/build-guide.sh b/guide/build-guide.sh
@@ -30,7 +30,7 @@ WASM_OUTPUT_DIR=$(realpath $RESULTS_VIEWER_DIR/lib/hdr-histogram-wasm)
 cd $WASM_LIB_DIR
 wasm-pack build --release -t web -d $WASM_OUTPUT_DIR --scope fs
 cd $WASM_OUTPUT_DIR
-sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\ninput = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js
+sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\n        input = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js
 
 # build the results viewer (which includes putting the output into the book's src)
 cd $RESULTS_VIEWER_DIR

diff --git a/guide/serve-guide.sh b/guide/serve-guide.sh
@@ -30,7 +30,7 @@ WASM_OUTPUT_DIR=$(realpath $RESULTS_VIEWER_DIR/lib/hdr-histogram-wasm)
 cd $WASM_LIB_DIR
 wasm-pack build --release -t web -d $WASM_OUTPUT_DIR --scope fs
 cd $WASM_OUTPUT_DIR
-sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\ninput = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js
+sed -i 's/input = .*import\.meta\.url.*/import(".\/hdr_histogram_wasm_bg.wasm");\n        input = require.resolve(".\/hdr_histogram_wasm_bg.wasm")[0][0];/' hdr_histogram_wasm.js
 
 # build the results viewer (which includes putting the output into the book's src)
 cd $RESULTS_VIEWER_DIR

diff --git a/guide/src/config/common-types/expressions.md b/guide/src/config/common-types/expressions.md
@@ -68,6 +68,7 @@ Encode a string with the given encoding.
 - `"percent"` - Percent encodes every ASCII character less than hexidecimal 20 and greater than 7E in addition to ` `, `"`, `#`, `>`, `<`, `` ` ``, `?`, `{` and `}` (space, doublequote, hash, greater than, less than, backtick, question mark, open curly brace and close curly brace).
 - `"percent-path"` - Percent encodes every ASCII character less than hexidecimal 20 and greater than 7E in addition to ` `, `"`, `#`, `>`, `<`, `` ` ``, `?`, `{`, `}`, `%` and `/` (space, doublequote, hash, greater than, less than, backtick, question mark, open curly brace, close curly brace, percent and forward slash).
 - `"percent-userinfo"` - Percent encodes every ASCII character less than hexidecimal 20 and greater than 7E in addition to ` `, `"`, `#`, `>`, `<`, `` ` ``, `?`, `{`, `}`, `/`, `:`, `;`, `=`, `@`, `\`, `[`, `]`, `^`, and `|` (space, doublequote, hash, greater than, less than, backtick, question mark, open curly brace, close curly brace, forward slash, colon, semi-colon, equal sign, at sign, backslash, open square bracket, close square bracket, caret and pipe).<br/><br/>
+- `"non-alphanumeric"` - Non-Alphanumeric encodes every ASCII character that is not an ASCII letter or digit.
 
 **Example**: with the value `foo=bar` from a provider named `baz`, then the template `https://localhost/abc?${encode(baz, "percent-userinfo"}` would resolve to `https://localhost/abc?foo%3Dbar`.
 

diff --git a/lib/config/Cargo.toml b/lib/config/Cargo.toml
@@ -17,7 +17,7 @@ itertools = "0.10"
 # pinned to 0.2.3 because newer versions made a semver incompatible change by making `Node` no longer
 # public, which we depend on https://github.com/freestrings/jsonpath/issues/36
 jsonpath_lib = "=0.2.3"
-percent-encoding = "1"
+percent-encoding = "2"
 pest = "2"
 pest_derive = "2"
 rand = "0.8"

diff --git a/lib/config/src/expression_functions.rs b/lib/config/src/expression_functions.rs
@@ -7,6 +7,7 @@ use crate::select_parser::ProviderStream;
 use ether::{Either, Either3, EitherExt};
 use futures::{stream, Stream, StreamExt, TryStreamExt};
 use jsonpath_lib as json_path;
+use percent_encoding::AsciiSet;
 use rand::distributions::{Distribution, Uniform};
 use regex::Regex;
 use serde_json as json;
@@ -144,31 +145,79 @@ enum Encoding {
     Percent,
     PercentPath,
     PercentUserinfo,
+    NonAlphanumeric,
 }
 
+// https://github.com/servo/rust-url/blob/master/UPGRADING.md
+// Prepackaged encoding sets, like QUERY_ENCODE_SET and PATH_SEGMENT_ENCODE_SET, are no longer provided.
+// You will need to read the specifications relevant to your domain and construct your own encoding sets
+// by using the percent_encoding::AsciiSet builder methods on either of the base encoding sets,
+// percent_encoding::CONTROLS or percent_encoding::NON_ALPHANUMERIC.
+
+/// This encode set is used in the URL parser for query strings.
+///
+/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+/// space, double quote ("), hash (#), and inequality qualifiers (<), (>) are encoded.
+const QUERY_ENCODE_SET: &AsciiSet = &percent_encoding::CONTROLS
+    .add(b' ')
+    .add(b'"')
+    .add(b'#')
+    .add(b'<')
+    .add(b'>');
+/// This encode set is used for path components.
+///
+/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+/// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
+/// question mark (?), and curly brackets ({), (}) are encoded.
+const DEFAULT_ENCODE_SET: &AsciiSet = &QUERY_ENCODE_SET.add(b'`').add(b'?').add(b'{').add(b'}');
+/// This encode set is used for on '/'-separated path segment
+///
+/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+/// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
+/// question mark (?), and curly brackets ({), (}), percent sign (%), forward slash (/) are
+/// encoded.
+const PATH_SEGMENT_ENCODE_SET: &AsciiSet = &DEFAULT_ENCODE_SET.add(b'%').add(b'/');
+/// This encode set is used for username and password.
+///
+/// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+/// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
+/// question mark (?), and curly brackets ({), (}), forward slash (/), colon (:), semi-colon (;),
+/// equality (=), at (@), backslash (\\), square brackets ([), (]), caret (\^), and pipe (|) are
+/// encoded.
+const USERINFO_ENCODE_SET: &AsciiSet = &DEFAULT_ENCODE_SET
+    .add(b'/')
+    .add(b':')
+    .add(b';')
+    .add(b'=')
+    .add(b'@')
+    .add(b'[')
+    .add(b'\\')
+    .add(b']')
+    .add(b'^')
+    .add(b'|');
+
 impl Encoding {
     fn encode(self, d: &json::Value) -> String {
         let s = json_value_to_string(Cow::Borrowed(d));
         match self {
             Encoding::Base64 => base64::encode(s.as_str()),
             Encoding::PercentSimple => {
-                percent_encoding::utf8_percent_encode(&s, percent_encoding::SIMPLE_ENCODE_SET)
-                    .to_string()
+                percent_encoding::utf8_percent_encode(&s, percent_encoding::CONTROLS).to_string()
             }
             Encoding::PercentQuery => {
-                percent_encoding::utf8_percent_encode(&s, percent_encoding::QUERY_ENCODE_SET)
-                    .to_string()
+                percent_encoding::utf8_percent_encode(&s, QUERY_ENCODE_SET).to_string()
             }
             Encoding::Percent => {
-                percent_encoding::utf8_percent_encode(&s, percent_encoding::DEFAULT_ENCODE_SET)
-                    .to_string()
+                percent_encoding::utf8_percent_encode(&s, DEFAULT_ENCODE_SET).to_string()
             }
             Encoding::PercentPath => {
-                percent_encoding::utf8_percent_encode(&s, percent_encoding::PATH_SEGMENT_ENCODE_SET)
-                    .to_string()
+                percent_encoding::utf8_percent_encode(&s, PATH_SEGMENT_ENCODE_SET).to_string()
             }
             Encoding::PercentUserinfo => {
-                percent_encoding::utf8_percent_encode(&s, percent_encoding::USERINFO_ENCODE_SET)
+                percent_encoding::utf8_percent_encode(&s, USERINFO_ENCODE_SET).to_string()
+            }
+            Encoding::NonAlphanumeric => {
+                percent_encoding::utf8_percent_encode(&s, percent_encoding::NON_ALPHANUMERIC)
                     .to_string()
             }
         }
@@ -182,6 +231,7 @@ impl Encoding {
             "percent" => Ok(Encoding::Percent),
             "percent-path" => Ok(Encoding::PercentPath),
             "percent-userinfo" => Ok(Encoding::PercentUserinfo),
+            "non-alphanumeric" => Ok(Encoding::NonAlphanumeric),
             _ => Err(ExecutingExpressionError::InvalidFunctionArguments("encode", marker).into()),
         }
     }
@@ -1678,6 +1728,11 @@ mod tests {
                 None,
                 j!("asd%20jkl%7C"),
             ),
+            (
+                vec![j!("asd 123~").into(), j!("non-alphanumeric").into()],
+                None,
+                j!("asd%20123%7E"),
+            ),
             (
                 vec!["a".into(), j!("percent-path").into()],
                 Some(j!({"a": "asd/jkl%"})),
@@ -1698,6 +1753,11 @@ mod tests {
                 Some(j!({"a": "asd jkl|"})),
                 j!("asd%20jkl%7C"),
             ),
+            (
+                vec!["a".into(), j!("non-alphanumeric").into()],
+                Some(j!({"a": "asd 123~"})),
+                j!("asd%20123%7E"),
+            ),
             (
                 vec!["a".into(), j!("base64").into()],
                 Some(j!({"a": "foo"})),
@@ -1741,6 +1801,11 @@ mod tests {
                 j!({"a": "asd jkl|"}),
                 vec![j!("asd%20jkl%7C")],
             ),
+            (
+                vec!["a".into(), j!("non-alphanumeric").into()],
+                j!({"a": "asd 123~"}),
+                vec![j!("asd%20123%7E")],
+            ),
             (
                 vec!["a".into(), j!("base64").into()],
                 j!({"a": "foo"}),
@@ -1783,15 +1848,20 @@ mod tests {
                 vec!["d".into(), j!("percent-userinfo").into()],
                 j!("asd%20jkl%7C"),
             ),
-            (vec!["e".into(), j!("base64").into()], j!("Zm9v")),
+            (
+                vec!["e".into(), j!("non-alphanumeric").into()],
+                j!("asd%20123%7E"),
+            ),
+            (vec!["f".into(), j!("base64").into()], j!("Zm9v")),
         ];
 
         let providers = btreemap!(
             "a".to_string() => literals(vec!(j!("asd/jkl%"))),
             "b".to_string() => literals(vec!(j!("asd\njkl#"))),
             "c".to_string() => literals(vec!(j!("asd\njkl{"))),
             "d".to_string() => literals(vec!(j!("asd jkl|"))),
-            "e".to_string() => literals(vec!(j!("foo"))),
+            "e".to_string() => literals(vec!(j!("asd 123~"))),
+            "f".to_string() => literals(vec!(j!("foo"))),
         );
 
         let providers = Arc::new(providers);

diff --git a/lib/config/src/from_yaml.rs b/lib/config/src/from_yaml.rs
@@ -340,7 +340,8 @@ impl<V> Insert for Vec<V> {
     }
 }
 
-#[cfg_attr(debug_assertions, derive(Debug, PartialEq))]
+#[cfg_attr(debug_assertions, derive(PartialEq))]
+#[derive(Debug)]
 pub struct TupleVec<K, V>(pub Vec<(K, V)>);
 
 impl<K, V> Default for TupleVec<K, V> {