Issue 382/ support X-Robots-Tag as a typed http header XRobotsTag

Cargo.toml

@@ -42,6 +42,7 @@ bitflags = "2.4"
 md5 = "0.7.0"
 brotli = "7"
 bytes = "1"
+chrono = "0.4.39"
 clap = { version = "4.5.15", features = ["derive"] }
 crossterm = "0.27"
 csv = "1.3.1"

rama-http-types/src/lib.rs

@@ -131,6 +131,12 @@ pub mod header {
         "x-real-ip",
     ];
 
+    // non-std web-crawler info headers
+    //
+    // More information at
+    // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Robots-Tag>.
+    static_header!["x-robots-tag"];
+
     /// Static Header Value that is can be used as `User-Agent` or `Server` header.
     pub static RAMA_ID_HEADER_VALUE: HeaderValue = HeaderValue::from_static(
         const_format::formatcp!("{}/{}", rama_utils::info::NAME, rama_utils::info::VERSION),

rama-http/Cargo.toml

@@ -30,6 +30,7 @@ async-compression = { workspace = true, features = [
 base64 = { workspace = true }
 bitflags = { workspace = true }
 bytes = { workspace = true }
+chrono = { workspace = true }
 const_format = { workspace = true }
 csv = { workspace = true }
 futures-lite = { workspace = true }

rama-http/src/headers/mod.rs

@@ -102,4 +102,10 @@ pub mod authorization {
 pub use ::rama_http_types::headers::HeaderExt;
 
 pub(crate) mod util;
+
+pub mod x_robots_tag_components;
+
+mod x_robots_tag;
+pub use x_robots_tag::XRobotsTag;
+
 pub use util::quality_value::{Quality, QualityValue};

rama-http/src/headers/util/csv.rs

@@ -11,24 +11,28 @@ use crate::HeaderValue;
 pub(crate) fn from_comma_delimited<'i, I, T, E>(values: &mut I) -> Result<E, Error>
 where
     I: Iterator<Item = &'i HeaderValue>,
-    T: ::std::str::FromStr,
-    E: ::std::iter::FromIterator<T>,
+    T: std::str::FromStr,
+    E: FromIterator<T>,
 {
     values
         .flat_map(|value| {
-            value.to_str().into_iter().flat_map(|string| {
-                string
-                    .split(',')
-                    .filter_map(|x| match x.trim() {
-                        "" => None,
-                        y => Some(y),
-                    })
-                    .map(|x| x.parse().map_err(|_| Error::invalid()))
-            })
+            value
+                .to_str()
+                .into_iter()
+                .flat_map(|string| split_csv_str(string))
         })
         .collect()
 }
 
+pub(crate) fn split_csv_str<T: std::str::FromStr>(
+    string: &str,
+) -> impl Iterator<Item = Result<T, Error>> + use<'_, T> {
+    string.split(',').filter_map(|x| match x.trim() {
+        "" => None,
+        y => Some(y.parse().map_err(|_| Error::invalid())),
+    })
+}
+
 /// Format an array into a comma-delimited string.
 pub(crate) fn fmt_comma_delimited<T: fmt::Display>(
     f: &mut fmt::Formatter,

rama-http/src/headers/util/mod.rs

@@ -1,3 +1,5 @@
 pub(crate) mod csv;
 /// Internal utility functions for headers.
 pub(crate) mod quality_value;
+
+pub(crate) mod value_string;

rama-http/src/headers/util/value_string.rs

@@ -0,0 +1,63 @@
+use http::header::HeaderValue;
+use std::fmt::{Display, Formatter};
+use std::{
+    fmt,
+    str::{self, FromStr},
+};
+
+/// A value that is both a valid `HeaderValue` and `String`.
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct HeaderValueString {
+    /// Care must be taken to only set this value when it is also
+    /// a valid `String`, since `as_str` will convert to a `&str`
+    /// in an unchecked manner.
+    value: HeaderValue,
+}
+
+impl HeaderValueString {
+    pub(crate) fn as_str(&self) -> &str {
+        // HeaderValueString is only created from HeaderValues
+        // that have validated they are also UTF-8 strings.
+        unsafe { str::from_utf8_unchecked(self.value.as_bytes()) }
+    }
+}
+
+impl fmt::Debug for HeaderValueString {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        fmt::Debug::fmt(self.as_str(), f)
+    }
+}
+
+impl Display for HeaderValueString {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        fmt::Display::fmt(self.as_str(), f)
+    }
+}
+
+impl<'a> From<&'a HeaderValueString> for HeaderValue {
+    fn from(src: &'a HeaderValueString) -> HeaderValue {
+        src.value.clone()
+    }
+}
+
+#[derive(Debug)]
+pub struct FromStrError(&'static str);
+
+impl FromStr for HeaderValueString {
+    type Err = FromStrError;
+
+    fn from_str(src: &str) -> Result<Self, Self::Err> {
+        // A valid `str` (the argument)...
+        src.parse()
+            .map(|value| HeaderValueString { value })
+            .map_err(|_| FromStrError("failed to parse header value from string"))
+    }
+}
+
+impl Display for FromStrError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        writeln!(f, "{}", self.0)
+    }
+}
+
+impl std::error::Error for FromStrError {}

rama-http/src/headers/x_robots_tag.rs

@@ -0,0 +1,57 @@
+use crate::headers::x_robots_tag_components::robots_tag_components::Parser;
+use crate::headers::x_robots_tag_components::RobotsTag;
+use crate::headers::Error;
+use headers::Header;
+use http::{HeaderName, HeaderValue};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct XRobotsTag(Vec<RobotsTag>);
+
+impl Header for XRobotsTag {
+    fn name() -> &'static HeaderName {
+        &crate::header::X_ROBOTS_TAG
+    }
+
+    fn decode<'i, I>(values: &mut I) -> Result<Self, Error>
+    where
+        Self: Sized,
+        I: Iterator<Item = &'i HeaderValue>,
+    {
+        let elements = values.try_fold(Vec::new(), |mut acc, value| {
+            acc.extend(Parser::parse_value(value).map_err(|err| {
+                tracing::debug!(?err, "x-robots-tag header element decoding failure");
+                Error::invalid()
+            })?);
+
+            Ok(acc)
+        })?;
+
+        Ok(XRobotsTag(elements))
+    }
+
+    fn encode<E: Extend<HeaderValue>>(&self, values: &mut E) {
+        use std::fmt;
+        struct Format<F>(F);
+        impl<F> fmt::Display for Format<F>
+        where
+            F: Fn(&mut fmt::Formatter<'_>) -> fmt::Result,
+        {
+            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                self.0(f)
+            }
+        }
+        let s = format!(
+            "{}",
+            Format(|f: &mut fmt::Formatter<'_>| {
+                crate::headers::util::csv::fmt_comma_delimited(&mut *f, self.0.iter())
+            })
+        );
+        values.extend(Some(HeaderValue::from_str(&s).unwrap()))
+    }
+}
+
+impl FromIterator<RobotsTag> for XRobotsTag {
+    fn from_iter<T: IntoIterator<Item = RobotsTag>>(iter: T) -> Self {
+        Self(iter.into_iter().collect())
+    }
+}

rama-http/src/headers/x_robots_tag_components/custom_rule.rs

@@ -0,0 +1,28 @@
+use crate::headers::util::value_string::HeaderValueString;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(super) struct CustomRule {
+    key: HeaderValueString,
+    value: Option<HeaderValueString>,
+}
+
+impl CustomRule {
+    pub(super) fn as_tuple(&self) -> (&HeaderValueString, &Option<HeaderValueString>) {
+        (&self.key, &self.value)
+    }
+}
+
+impl From<HeaderValueString> for CustomRule {
+    fn from(key: HeaderValueString) -> Self {
+        Self { key, value: None }
+    }
+}
+
+impl From<(HeaderValueString, HeaderValueString)> for CustomRule {
+    fn from(key_value: (HeaderValueString, HeaderValueString)) -> Self {
+        Self {
+            key: key_value.0,
+            value: Some(key_value.1),
+        }
+    }
+}