Initial test harness

Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "crates/data",
     "crates/cli",
+    "crates/harness",
 ]
 
 [package]

crates/cli/src/main.rs

@@ -15,21 +15,21 @@ fn run() -> proc_exit::ExitResult {
     let matches = Matches::new(args.ignore.iter().map(|s| s.as_str()))
         .with_code(proc_exit::Code::USAGE_ERR)?;
     for bin in args.bin.iter() {
+        let bin = toml_test::verify::Command::new(bin);
         if args.encoder {
-            verify_encoder(bin.as_path(), &matches)?;
+            unimplemented!("Not yet implemented, waiting on a verified decoder");
         } else {
-            verify_decoder(bin.as_path(), &matches)?;
+            verify_decoder(&bin, &matches)?;
         }
     }
 
     Ok(())
 }
 
-fn verify_encoder(_bin: &std::path::Path, _matches: &Matches) -> proc_exit::ExitResult {
-    unimplemented!("Not yet implemented");
-}
-
-fn verify_decoder(bin: &std::path::Path, matches: &Matches) -> proc_exit::ExitResult {
+fn verify_decoder(
+    bin: &dyn toml_test::verify::Decoder,
+    matches: &Matches,
+) -> proc_exit::ExitResult {
     let mut passed = 0;
     let mut failed = 0;
 
@@ -38,26 +38,9 @@ fn verify_decoder(bin: &std::path::Path, matches: &Matches) -> proc_exit::ExitRe
             log::debug!("Skipped {}", case.name.display());
             continue;
         }
-        match run_decoder(bin, case.fixture) {
-            Ok(actual) => {
-                let expected = toml_test::encoded::Encoded::from_slice(&case.expected)
-                    .with_code(proc_exit::Code::USAGE_ERR)?;
-                if actual == expected {
-                    passed += 1;
-                } else {
-                    log::debug!("{}: failed", case.name.display());
-                    log::trace!(
-                        "{}: expected\n{}",
-                        case.name.display(),
-                        expected.to_string_pretty().unwrap()
-                    );
-                    log::trace!(
-                        "{}: actual\n{}",
-                        case.name.display(),
-                        actual.to_string_pretty().unwrap()
-                    );
-                    failed += 1;
-                }
+        match bin.verify_valid_case(case.fixture, case.expected) {
+            Ok(()) => {
+                passed += 1;
             }
             Err(err) => {
                 log::debug!("{}: failed", case.name.display());
@@ -72,28 +55,24 @@ fn verify_decoder(bin: &std::path::Path, matches: &Matches) -> proc_exit::ExitRe
             log::debug!("Skipped {}", case.name.display());
             continue;
         }
-        match run_decoder(bin, case.fixture) {
-            Ok(actual) => {
-                log::debug!("{}: should have failed", case.name.display());
-                log::trace!(
-                    "{}: actual\n{}",
-                    case.name.display(),
-                    actual.to_string_pretty().unwrap()
-                );
-                failed += 1;
-            }
-            Err(err) => {
+        match bin.verify_invalid_case(case.fixture) {
+            Ok(err) => {
                 log::debug!("{}: failed successfully", case.name.display());
                 log::trace!("{}: {}", case.name.display(), err);
                 passed += 1;
             }
+            Err(err) => {
+                log::debug!("{}: should have failed", case.name.display());
+                log::trace!("{}: {}", case.name.display(), err);
+                failed += 1;
+            }
         }
     }
 
     let _ = writeln!(
         std::io::stdout(),
         "toml-test [{}]: using embedded tests: {} passed, {} failed",
-        bin.file_name().unwrap().to_string_lossy(),
+        bin.name(),
         passed,
         failed
     );
@@ -104,32 +83,6 @@ fn verify_decoder(bin: &std::path::Path, matches: &Matches) -> proc_exit::ExitRe
     Ok(())
 }
 
-fn run_decoder(
-    bin: &std::path::Path,
-    toml: &[u8],
-) -> Result<toml_test::encoded::Encoded, eyre::Error> {
-    let mut cmd = std::process::Command::new(bin);
-    cmd.stdin(std::process::Stdio::piped())
-        .stdout(std::process::Stdio::piped())
-        .stderr(std::process::Stdio::piped());
-    let child = cmd.spawn()?;
-    child.stdin.as_ref().unwrap().write_all(toml)?;
-
-    let output = child.wait_with_output()?;
-    if output.status.success() {
-        let output = toml_test::encoded::Encoded::from_slice(&output.stdout)?;
-        Ok(output)
-    } else {
-        let message = String::from_utf8_lossy(&output.stderr);
-        eyre::bail!(
-            "{} failed with {:?}: {}",
-            bin.display(),
-            output.status.code(),
-            message
-        )
-    }
-}
-
 fn init_logging(mut level: clap_verbosity_flag::Verbosity) {
     level.set_default(Some(log::Level::Info));
 

crates/harness/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "toml-test-harness"
+version = "0.1.0"
+description = "Cargo test harness for verifying TOML parsers"
+repository = "https://github.com/epage/toml-test-rs"
+readme = "README.md"
+categories = ["testing", "development-tools", "text-processing", "encoding"]
+keywords = ["development", "toml"]
+license = "MIT OR Apache-2.0"
+edition = "2018"
+
+[dependencies]
+toml-test-data = { version = "1.0", path = "../data" }
+toml-test = { version = "0.1", path = "../../" }
+ignore = "0.4"
+libtest-mimic = "0.3.0"

crates/harness/README.md

@@ -0,0 +1,16 @@
+# toml-test-harness
+
+> **Cargo test harness for verifying TOML parsers**
+
+[![Documentation](https://img.shields.io/badge/docs-master-blue.svg)][Documentation]
+![License](https://img.shields.io/crates/l/toml-test.svg)
+[![Crates Status](https://img.shields.io/crates/v/toml-test.svg)](https://crates.io/crates/toml-test)
+
+Dual-licensed under [MIT](LICENSE-MIT) or [Apache 2.0](LICENSE-APACHE)
+
+## About
+
+[toml-test](https://github.com/BurntSushi/toml-test) is a language-agnostic
+toml parser spec verification.  This crate exposes the test cases as Rust tests.
+
+[Documentation]: https://docs.rs/toml-test

crates/harness/src/lib.rs

@@ -0,0 +1,124 @@
+use std::io::Write;
+
+pub use toml_test::encoded::Encoded;
+pub use toml_test::verify::Decoder;
+pub use toml_test::verify::Encoder;
+pub use toml_test::Error;
+
+pub struct DecoderHarness<D> {
+    decoder: D,
+    matches: Option<Matches>,
+}
+
+impl<D: toml_test::verify::Decoder + Send + Sync + 'static> DecoderHarness<D> {
+    pub fn new(decoder: D) -> Self {
+        Self {
+            decoder,
+            matches: None,
+        }
+    }
+
+    pub fn ignore<'p>(
+        &mut self,
+        patterns: impl IntoIterator<Item = &'p str>,
+    ) -> Result<&mut Self, toml_test::Error> {
+        self.matches = Some(Matches::new(patterns.into_iter())?);
+        Ok(self)
+    }
+
+    pub fn test(self) -> ! {
+        let args = libtest_mimic::Arguments::from_args();
+        let mut tests = Vec::new();
+        tests.extend(toml_test_data::valid().map(|c| {
+            libtest_mimic::Test {
+                name: c.name.display().to_string(),
+                kind: "".into(),
+                is_ignored: self
+                    .matches
+                    .as_ref()
+                    .map(|m| !m.matched(c.name))
+                    .unwrap_or_default(),
+                is_bench: false,
+                data: Case::from(c),
+            }
+        }));
+        tests.extend(toml_test_data::invalid().map(|c| {
+            libtest_mimic::Test {
+                name: c.name.display().to_string(),
+                kind: "".into(),
+                is_ignored: self
+                    .matches
+                    .as_ref()
+                    .map(|m| !m.matched(c.name))
+                    .unwrap_or_default(),
+                is_bench: false,
+                data: Case::from(c),
+            }
+        }));
+
+        let nocapture = args.nocapture;
+        libtest_mimic::run_tests(&args, tests, move |test| match test.data {
+            Case::Valid(case) => {
+                match self.decoder.verify_valid_case(case.fixture, case.expected) {
+                    Ok(()) => libtest_mimic::Outcome::Passed,
+                    Err(err) => libtest_mimic::Outcome::Failed {
+                        msg: Some(err.to_string()),
+                    },
+                }
+            }
+            Case::Invalid(case) => match self.decoder.verify_invalid_case(case.fixture) {
+                Ok(err) => {
+                    if nocapture {
+                        let _ = writeln!(std::io::stdout(), "{}", err);
+                    }
+                    libtest_mimic::Outcome::Passed
+                }
+                Err(err) => libtest_mimic::Outcome::Failed {
+                    msg: Some(err.to_string()),
+                },
+            },
+        })
+        .exit()
+    }
+}
+
+enum Case {
+    Valid(toml_test_data::Valid<'static>),
+    Invalid(toml_test_data::Invalid<'static>),
+}
+
+impl From<toml_test_data::Valid<'static>> for Case {
+    fn from(other: toml_test_data::Valid<'static>) -> Self {
+        Self::Valid(other)
+    }
+}
+
+impl From<toml_test_data::Invalid<'static>> for Case {
+    fn from(other: toml_test_data::Invalid<'static>) -> Self {
+        Self::Invalid(other)
+    }
+}
+
+struct Matches {
+    ignores: ignore::gitignore::Gitignore,
+}
+
+impl Matches {
+    fn new<'p>(patterns: impl Iterator<Item = &'p str>) -> Result<Self, toml_test::Error> {
+        let mut ignores = ignore::gitignore::GitignoreBuilder::new(".");
+        for line in patterns {
+            ignores
+                .add_line(None, line)
+                .map_err(|e| toml_test::Error::new(e))?;
+        }
+        let ignores = ignores.build().map_err(|e| toml_test::Error::new(e))?;
+        Ok(Self { ignores })
+    }
+
+    fn matched(&self, path: &std::path::Path) -> bool {
+        match self.ignores.matched_path_or_any_parents(path, false) {
+            ignore::Match::None | ignore::Match::Whitelist(_) => true,
+            ignore::Match::Ignore(_) => false,
+        }
+    }
+}

src/encoded.rs

@@ -8,11 +8,11 @@ pub enum Encoded {
 
 impl Encoded {
     pub fn from_slice(v: &[u8]) -> Result<Self, crate::Error> {
-        serde_json::from_slice(v).map_err(|e| crate::Error::new(e.to_string()))
+        serde_json::from_slice(v).map_err(|e| crate::Error::new(e))
     }
 
     pub fn to_string_pretty(&self) -> Result<String, crate::Error> {
-        serde_json::to_string_pretty(self).map_err(|e| crate::Error::new(e.to_string()))
+        serde_json::to_string_pretty(self).map_err(|e| crate::Error::new(e))
     }
 }
 

src/lib.rs

@@ -4,19 +4,22 @@ use std::io::Write;
 mod error;
 
 pub mod encoded;
+pub mod verify;
 pub use error::Error;
 
+/// External parser helper for [verify]
 pub fn encoder_in() -> Result<encoded::Encoded, Error> {
     let mut buf = Vec::new();
     std::io::stdin()
         .read_to_end(&mut buf)
-        .map_err(|e| crate::Error::new(e.to_string()))?;
+        .map_err(|e| crate::Error::new(e))?;
     encoded::Encoded::from_slice(&buf)
 }
 
+/// External parser helper for [verify]
 pub fn decoder_out(e: encoded::Encoded) -> Result<(), Error> {
     let s = e.to_string_pretty()?;
     std::io::stdout()
         .write_all(s.as_bytes())
-        .map_err(|e| crate::Error::new(e.to_string()))
+        .map_err(|e| crate::Error::new(e))
 }