From c6bd566c902050b54d5178d414cba1a86aacec5a Mon Sep 17 00:00:00 2001 From: Martin Dickopp Date: Wed, 2 Mar 2022 09:41:50 +0100 Subject: [PATCH] Fix parsing of `Accept-Encoding` request header (#220) * Fix parsing of Accept-Encoding request header * Add unit tests to content_encoding * Represent quality values (qvalues) by a separate type * Parse encodings case-insensitively * Parse qvalues as specified in RFC 7231 section 5.3.1 Refs: #215 * Do not use or-pattern syntax This syntax is not supported in rust 1.51 (the minimum toolchain version). * Add comments to QValue::parse * Remove redundant SupportedEncodingsAll::new function * Add unit tests for all content-encodings (gzip, deflate, br) --- tower-http/src/content_encoding.rs | 482 +++++++++++++++++++++++++++-- tower-http/src/services/fs/mod.rs | 8 +- 2 files changed, 457 insertions(+), 33 deletions(-) diff --git a/tower-http/src/content_encoding.rs b/tower-http/src/content_encoding.rs index af3da136..ac3d0326 100644 --- a/tower-http/src/content_encoding.rs +++ b/tower-http/src/content_encoding.rs @@ -52,16 +52,26 @@ impl Encoding { feature = "fs", ))] fn parse(s: &str, _supported_encoding: impl SupportedEncodings) -> Option { - match s { - #[cfg(any(feature = "fs", feature = "compression-gzip"))] - "gzip" if _supported_encoding.gzip() => Some(Encoding::Gzip), - #[cfg(any(feature = "fs", feature = "compression-deflate"))] - "deflate" if _supported_encoding.deflate() => Some(Encoding::Deflate), - #[cfg(any(feature = "fs", feature = "compression-br"))] - "br" if _supported_encoding.br() => Some(Encoding::Brotli), - "identity" => Some(Encoding::Identity), - _ => None, + #[cfg(any(feature = "fs", feature = "compression-gzip"))] + if s.eq_ignore_ascii_case("gzip") && _supported_encoding.gzip() { + return Some(Encoding::Gzip); + } + + #[cfg(any(feature = "fs", feature = "compression-deflate"))] + if s.eq_ignore_ascii_case("deflate") && _supported_encoding.deflate() { + return Some(Encoding::Deflate); + } + + #[cfg(any(feature = "fs", feature = "compression-br"))] + if s.eq_ignore_ascii_case("br") && _supported_encoding.br() { + return Some(Encoding::Brotli); } + + if s.eq_ignore_ascii_case("identity") { + return Some(Encoding::Identity); + } + + None } #[cfg(any( @@ -84,23 +94,102 @@ impl Encoding { feature = "compression-deflate", feature = "fs", ))] - pub(crate) fn preferred_encoding(accepted_encodings: &[(Encoding, f32)]) -> Option { + pub(crate) fn preferred_encoding(accepted_encodings: &[(Encoding, QValue)]) -> Option { let mut preferred_encoding = None; - let mut max_qval = 0.0; + let mut max_qval = 0; for (encoding, qval) in accepted_encodings { - if (qval - 1.0f32).abs() < 0.01 { + if qval.0 > max_qval { preferred_encoding = Some(*encoding); - break; - } else if *qval > max_qval { - preferred_encoding = Some(*encoding); - max_qval = *qval; + max_qval = qval.0; } } preferred_encoding } } +// Allowed q-values are numbers between 0 and 1 with at most 3 digits in the fractional part. They +// are presented here as an unsigned integer between 0 and 1000. +#[cfg(any( + feature = "compression-gzip", + feature = "compression-br", + feature = "compression-deflate", + feature = "fs", +))] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct QValue(u16); + +#[cfg(any( + feature = "compression-gzip", + feature = "compression-br", + feature = "compression-deflate", + feature = "fs", +))] +impl QValue { + #[inline] + fn one() -> Self { + Self(1000) + } + + // Parse a q-value as specified in RFC 7231 section 5.3.1. + fn parse(s: &str) -> Option { + let mut c = s.chars(); + // Parse "q=" (case-insensitively). + match c.next() { + Some('q') | Some('Q') => (), + _ => return None, + }; + match c.next() { + Some('=') => (), + _ => return None, + }; + + // Parse leading digit. Since valid q-values are between 0.000 and 1.000, only "0" and "1" + // are allowed. + let mut value = match c.next() { + Some('0') => 0, + Some('1') => 1000, + _ => return None, + }; + + // Parse optional decimal point. + match c.next() { + Some('.') => (), + None => return Some(Self(value)), + _ => return None, + }; + + // Parse optional fractional digits. The value of each digit is multiplied by `factor`. + // Since the q-value is represented as an integer between 0 and 1000, `factor` is `100` for + // the first digit, `10` for the next, and `1` for the digit after that. + let mut factor = 100; + loop { + match c.next() { + Some(n @ '0'..='9') => { + // If `factor` is less than `1`, three digits have already been parsed. A + // q-value having more than 3 fractional digits is invalid. + if factor < 1 { + return None; + } + // Add the digit's value multiplied by `factor` to `value`. + value += factor * (n as u16 - '0' as u16); + } + None => { + // No more characters to parse. Check that the value representing the q-value is + // in the valid range. + return if value <= 1000 { + Some(Self(value)) + } else { + None + }; + } + _ => return None, + }; + factor /= 10; + } + } +} + #[cfg(any( feature = "compression-gzip", feature = "compression-br", @@ -111,34 +200,369 @@ impl Encoding { pub(crate) fn encodings( headers: &http::HeaderMap, supported_encoding: impl SupportedEncodings, -) -> Vec<(Encoding, f32)> { +) -> Vec<(Encoding, QValue)> { headers .get_all(http::header::ACCEPT_ENCODING) .iter() .filter_map(|hval| hval.to_str().ok()) - .flat_map(|s| s.split(',').map(str::trim)) + .flat_map(|s| s.split(',')) .filter_map(|v| { - let mut v = v.splitn(2, ";q="); + let mut v = v.splitn(2, ';'); - let encoding = match Encoding::parse(v.next().unwrap(), supported_encoding) { + let encoding = match Encoding::parse(v.next().unwrap().trim(), supported_encoding) { Some(encoding) => encoding, None => return None, // ignore unknown encodings }; let qval = if let Some(qval) = v.next() { - let qval = match qval.parse::() { - Ok(f) => f, - Err(_) => return None, - }; - if qval > 1.0 { - return None; // q-values over 1 are unacceptable + if let Some(qval) = QValue::parse(qval.trim()) { + qval + } else { + return None; } - qval } else { - 1.0f32 + QValue::one() }; Some((encoding, qval)) }) - .collect::>() + .collect::>() +} + +#[cfg(all( + test, + feature = "compression-gzip", + feature = "compression-deflate", + feature = "compression-br" +))] +mod tests { + use super::*; + + #[derive(Copy, Clone, Default)] + struct SupportedEncodingsAll; + + impl SupportedEncodings for SupportedEncodingsAll { + fn gzip(&self) -> bool { + true + } + + fn deflate(&self) -> bool { + true + } + + fn br(&self) -> bool { + true + } + } + + #[test] + fn no_accept_encoding_header() { + let encoding = + Encoding::from_headers(&http::HeaderMap::new(), SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + } + + #[test] + fn accept_encoding_header_single_encoding() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + } + + #[test] + fn accept_encoding_header_two_encodings() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip,br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + } + + #[test] + fn accept_encoding_header_three_encodings() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip,deflate,br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + } + + #[test] + fn accept_encoding_header_two_encodings_with_one_qvalue() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5,br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn accept_encoding_header_three_encodings_with_one_qvalue() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5,deflate,br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Deflate, encoding); + } + + #[test] + fn two_accept_encoding_headers_with_one_qvalue() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5"), + ); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn two_accept_encoding_headers_three_encodings_with_one_qvalue() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5,deflate"), + ); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Deflate, encoding); + } + + #[test] + fn three_accept_encoding_headers_with_one_qvalue() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5"), + ); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("deflate"), + ); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("br"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Deflate, encoding); + } + + #[test] + fn accept_encoding_header_two_encodings_with_two_qvalues() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5,br;q=0.8"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.8,br;q=0.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.995,br;q=0.999"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn accept_encoding_header_three_encodings_with_three_qvalues() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5,deflate;q=0.6,br;q=0.8"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.8,deflate;q=0.6,br;q=0.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.6,deflate;q=0.8,br;q=0.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Deflate, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.995,deflate;q=0.997,br;q=0.999"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn accept_encoding_header_invalid_encdoing() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("invalid,gzip"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + } + + #[test] + fn accept_encoding_header_with_qvalue_zero() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0."), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0,br;q=0.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn accept_encoding_header_with_uppercase_letters() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gZiP"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Gzip, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5,br;Q=0.8"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn accept_encoding_header_with_allowed_spaces() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static(" gzip\t; q=0.5 ,\tbr ;\tq=0.8\t"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Brotli, encoding); + } + + #[test] + fn accept_encoding_header_with_invalid_spaces() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q =0.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q= 0.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + } + + #[test] + fn accept_encoding_header_with_invalid_quvalues() { + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=-0.1"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=00.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=0.5000"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=.5"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=1.01"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + + let mut headers = http::HeaderMap::new(); + headers.append( + http::header::ACCEPT_ENCODING, + http::HeaderValue::from_static("gzip;q=1.001"), + ); + let encoding = Encoding::from_headers(&headers, SupportedEncodingsAll::default()); + assert_eq!(Encoding::Identity, encoding); + } } diff --git a/tower-http/src/services/fs/mod.rs b/tower-http/src/services/fs/mod.rs index 21ce0dd8..f5aca03d 100644 --- a/tower-http/src/services/fs/mod.rs +++ b/tower-http/src/services/fs/mod.rs @@ -24,7 +24,7 @@ mod serve_file; // default capacity 64KiB const DEFAULT_CAPACITY: usize = 65536; -use crate::content_encoding::{Encoding, SupportedEncodings}; +use crate::content_encoding::{Encoding, QValue, SupportedEncodings}; pub use self::{ serve_dir::{ @@ -61,7 +61,7 @@ impl SupportedEncodings for PrecompressedVariants { // to the corresponding file extension for the encoding. fn preferred_encoding( path: &mut PathBuf, - negotiated_encoding: &[(Encoding, f32)], + negotiated_encoding: &[(Encoding, QValue)], ) -> Option { let preferred_encoding = Encoding::preferred_encoding(negotiated_encoding); if let Some(file_extension) = @@ -85,7 +85,7 @@ fn preferred_encoding( // file the uncompressed file is used as a fallback. async fn open_file_with_fallback( mut path: PathBuf, - mut negotiated_encoding: Vec<(Encoding, f32)>, + mut negotiated_encoding: Vec<(Encoding, QValue)>, ) -> io::Result<(File, Option)> { let (file, encoding) = loop { // Get the preferred encoding among the negotiated ones. @@ -112,7 +112,7 @@ async fn open_file_with_fallback( // file the uncompressed file is used as a fallback. async fn file_metadata_with_fallback( mut path: PathBuf, - mut negotiated_encoding: Vec<(Encoding, f32)>, + mut negotiated_encoding: Vec<(Encoding, QValue)>, ) -> io::Result<(Metadata, Option)> { let (file, encoding) = loop { // Get the preferred encoding among the negotiated ones.