From 6c7337a8fc727a79938f5c61c27096381ab1432e Mon Sep 17 00:00:00 2001 From: Christopher Kolstad Date: Tue, 19 Dec 2023 16:36:26 +0100 Subject: [PATCH 1/4] fix: Do not remove features cache if upstream goes away If upstream goes away, edge should remain durable and keep the features it has gotten for the tokens. Since we already keep the authentication status for the tokens we've seen this fix allows you to start edge, manage to talk to upstream once, then sever connection to upstream, but keep serving feature toggles for known tokens --- Cargo.lock | 20 ++++---- server/Cargo.toml | 10 ++-- server/src/frontend_api.rs | 4 +- server/src/http/feature_refresher.rs | 71 ++++++++++++++++++++++------ 4 files changed, 74 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce973c3c..9134c52d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-cors" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b340e9cfa5b08690aae90fb61beb44e9b06f44fe3d0f93781aaa58cfba86245e" +checksum = "0346d8c1f762b41b458ed3145eea914966bb9ad20b9be0d6d463b20d45586370" dependencies = [ "actix-utils", "actix-web", @@ -569,9 +569,9 @@ checksum = "8d18b093eba54c9aaa1e3784d4361eb2ba944cf7d0a932a830132238f483e8d8" [[package]] name = "clap" -version = "4.4.10" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", "clap_derive", @@ -588,9 +588,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", @@ -2452,9 +2452,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9198caff1c94f1a5df6664bddbc379896b51b98a55b0b3fedcb23078fe00c77" +checksum = "615d846f7174a0850dca101bca72f6913e3376a64c5fda2b965d7fc3d1ff60cb" dependencies = [ "const_format", "git2", @@ -3153,9 +3153,9 @@ dependencies = [ [[package]] name = "utoipa-swagger-ui" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154517adf0d0b6e22e8e1f385628f14fcaa3db43531dc74303d3edef89d6dfe5" +checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51" dependencies = [ "actix-web", "mime_guess", diff --git a/server/Cargo.toml b/server/Cargo.toml index b4cd3a66..cae6a16a 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -15,7 +15,7 @@ repository = "https://github.com/Unleash/unleash-edge" version = "16.0.6" [dependencies] -actix-cors = "0.6.4" +actix-cors = "0.6.5" actix-http = { version = "3.4.0", features = ["compress-zstd", "rustls-0_21"] } actix-middleware-etag = "0.3.0" actix-service = "2.0.2" @@ -26,7 +26,7 @@ anyhow = "1.0.75" async-trait = "0.1.74" chrono = { version = "0.4.31", features = ["serde"] } cidr = "0.2.2" -clap = { version = "4.4.8", features = ["derive", "env"] } +clap = { version = "4.4.11", features = ["derive", "env"] } clap-markdown = "0.1.3" dashmap = "5.5.3" futures = "0.3.29" @@ -62,7 +62,7 @@ rustls-pemfile = "1.0.4" serde = { version = "1.0.192", features = ["derive"] } serde_json = "1.0.108" serde_qs = { version = "0.12.0", features = ["actix4", "tracing"] } -shadow-rs = "0.24.1" +shadow-rs = { version = "0.25.0" } tokio = { version = "1.34.0", features = [ "macros", "rt-multi-thread", @@ -75,7 +75,7 @@ ulid = "1.1.0" unleash-types = { version = "0.10", features = ["openapi", "hashes"] } unleash-yggdrasil = { version = "0.8.0" } utoipa = { version = "4.1.0", features = ["actix_extras", "chrono"] } -utoipa-swagger-ui = { version = "4", features = ["actix-web"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [dev-dependencies] actix-http = "3.4.0" actix-http-test = "3.1.0" @@ -89,4 +89,4 @@ testcontainers-modules = { version = "0.2.0", features = ["redis"] } tracing-test = "0.2.4" [build-dependencies] -shadow-rs = "0.24.1" +shadow-rs = "0.25.0" diff --git a/server/src/frontend_api.rs b/server/src/frontend_api.rs index 7065d704..7122706d 100644 --- a/server/src/frontend_api.rs +++ b/server/src/frontend_api.rs @@ -11,7 +11,7 @@ use actix_web::{ }; use dashmap::DashMap; use serde_qs::actix::QsQuery; -use tracing::{debug, instrument}; +use tracing::{debug, instrument, warn}; use unleash_types::client_features::Context; use unleash_types::client_metrics::{ClientApplication, ConnectVia}; use unleash_types::{ @@ -214,7 +214,7 @@ security( ) )] #[get("")] -#[instrument(skip(engine_cache, token_cache))] +#[instrument(skip(edge_token, req, engine_cache, token_cache))] async fn get_enabled_frontend( edge_token: EdgeToken, engine_cache: Data>, diff --git a/server/src/http/feature_refresher.rs b/server/src/http/feature_refresher.rs index 23b57247..8bdfd428 100644 --- a/server/src/http/feature_refresher.rs +++ b/server/src/http/feature_refresher.rs @@ -5,7 +5,7 @@ use actix_web::http::header::EntityTag; use chrono::Utc; use dashmap::DashMap; use reqwest::StatusCode; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use unleash_types::client_features::Segment; use unleash_types::client_metrics::ClientApplication; use unleash_types::{ @@ -376,11 +376,11 @@ impl FeatureRefresher { self.backoff(&refresh.token); } _ => { - warn!("Couldn't refresh features, but will retry next go") + info!("Couldn't refresh features, but will retry next go") } }, FeatureError::AccessDenied => { - warn!("Token used to fetch features was Forbidden, will remove from list of refresh tasks"); + info!("Token used to fetch features was Forbidden, will remove from list of refresh tasks"); self.tokens_to_refresh.remove(&refresh.token.token); if !self.tokens_to_refresh.iter().any(|e| { e.value().token.environment == refresh.token.environment @@ -392,23 +392,15 @@ impl FeatureRefresher { } } FeatureError::NotFound => { - warn!("Had a bad URL when trying to fetch features. Removing ourselves"); + info!("Had a bad URL when trying to fetch features. Removing ourselves from the list of refresh tasks"); self.tokens_to_refresh.remove(&refresh.token.token); - if !self.tokens_to_refresh.iter().any(|e| { - e.value().token.environment == refresh.token.environment - }) { - let cache_key = cache_key(&refresh.token); - // No tokens left that access the environment of our current refresh. Deleting client features and engine cache - self.features_cache.remove(&cache_key); - self.engine_cache.remove(&cache_key); - } } } } EdgeError::ClientCacheError => { - warn!("Couldn't refresh features, but will retry next go") + info!("Couldn't refresh features, but will retry next go") } - _ => warn!("Couldn't refresh features: {e:?}. Will retry next pass"), + _ => info!("Couldn't refresh features: {e:?}. Will retry next pass"), } } } @@ -448,6 +440,7 @@ mod tests { use chrono::{Duration, Utc}; use dashmap::DashMap; use reqwest::Url; + use tracing_test::traced_test; use unleash_types::client_features::{ClientFeature, ClientFeatures}; use unleash_yggdrasil::EngineState; @@ -898,6 +891,56 @@ mod tests { assert!(feature_refresher.engine_cache.is_empty()); } + #[tokio::test] + #[traced_test] + pub async fn getting_404_removes_tokens_from_token_to_refresh_but_not_its_features() { + let mut token = EdgeToken::try_from("*:development.secret123".to_string()).unwrap(); + token.status = Validated; + token.token_type = Some(TokenType::Client); + let token_cache = DashMap::default(); + token_cache.insert(token.token.clone(), token.clone()); + let upstream_features_cache: Arc> = + Arc::new(DashMap::default()); + let upstream_engine_cache: Arc> = Arc::new(DashMap::default()); + let upstream_token_cache: Arc> = Arc::new(token_cache); + let example_features = features_from_disk("../examples/features.json"); + let cache_key = cache_key(&token); + let mut engine_state = EngineState::default(); + engine_state.take_state(example_features.clone()); + upstream_features_cache.insert(cache_key.clone(), example_features.clone()); + upstream_engine_cache.insert(cache_key.clone(), engine_state); + let mut server = client_api_test_server( + upstream_token_cache, + upstream_features_cache, + upstream_engine_cache, + ) + .await; + let unleash_client = UnleashClient::new(server.url("/").as_str(), None).unwrap(); + let features_cache: Arc> = Arc::new(DashMap::default()); + let engine_cache: Arc> = Arc::new(DashMap::default()); + let feature_refresher = FeatureRefresher::new( + Arc::new(unleash_client), + features_cache, + engine_cache, + Duration::milliseconds(1), + None, + ); + feature_refresher + .register_token_for_refresh(token, None) + .await; + assert!(!feature_refresher.tokens_to_refresh.is_empty()); + feature_refresher.refresh_features().await; + assert!(!feature_refresher.tokens_to_refresh.is_empty()); + assert!(!feature_refresher.features_cache.is_empty()); + assert!(!feature_refresher.engine_cache.is_empty()); + server.stop().await; + tokio::time::sleep(std::time::Duration::from_millis(5)).await; // To ensure our refresh is due + feature_refresher.refresh_features().await; + assert!(feature_refresher.tokens_to_refresh.is_empty()); + assert!(!feature_refresher.features_cache.is_empty()); + assert!(!feature_refresher.engine_cache.is_empty()); + } + #[tokio::test] pub async fn when_we_have_a_cache_and_token_gets_removed_caches_are_emptied() { let upstream_features_cache: Arc> = From 3509e5f48caf26da0e66f17bb8d840f28191e643 Mon Sep 17 00:00:00 2001 From: Christopher Kolstad Date: Tue, 19 Dec 2023 16:44:34 +0100 Subject: [PATCH 2/4] cargo fix --- server/src/frontend_api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/frontend_api.rs b/server/src/frontend_api.rs index 7122706d..9462aee6 100644 --- a/server/src/frontend_api.rs +++ b/server/src/frontend_api.rs @@ -11,7 +11,7 @@ use actix_web::{ }; use dashmap::DashMap; use serde_qs::actix::QsQuery; -use tracing::{debug, instrument, warn}; +use tracing::{debug, instrument}; use unleash_types::client_features::Context; use unleash_types::client_metrics::{ClientApplication, ConnectVia}; use unleash_types::{ From 06eb4d40bae306a8f05874c602256135d8c8b9e3 Mon Sep 17 00:00:00 2001 From: Christopher Kolstad Date: Wed, 20 Dec 2023 10:51:38 +0100 Subject: [PATCH 3/4] fix: Change to backoff when receiving NotFound --- server/src/http/feature_refresher.rs | 4 ++-- server/src/main.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/http/feature_refresher.rs b/server/src/http/feature_refresher.rs index 8bdfd428..484d5f95 100644 --- a/server/src/http/feature_refresher.rs +++ b/server/src/http/feature_refresher.rs @@ -392,8 +392,8 @@ impl FeatureRefresher { } } FeatureError::NotFound => { - info!("Had a bad URL when trying to fetch features. Removing ourselves from the list of refresh tasks"); - self.tokens_to_refresh.remove(&refresh.token.token); + info!("Had a bad URL when trying to fetch features. Increasing waiting period for the token before trying again"); + self.backoff(&refresh.token); } } } diff --git a/server/src/main.rs b/server/src/main.rs index 1d4d93c0..82f1afa6 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -163,10 +163,10 @@ async fn main() -> Result<(), anyhow::Error> { tracing::info!("Persister was unexpectedly shut down"); } _ = validator.schedule_validation_of_known_tokens(edge.token_revalidation_interval_seconds) => { - tracing::info!("Token validator validator was unexpectedly shut down"); + tracing::info!("Token validator validation of known tokens was unexpectedly shut down"); } _ = validator.schedule_revalidation_of_startup_tokens(edge.tokens, lazy_feature_refresher) => { - tracing::info!("Token validator validator was unexpectedly shut down"); + tracing::info!("Token validator validation of startup tokens was unexpectedly shut down"); } } } From c5d119e383fb0b37a81441c4d224ab7fdd319367 Mon Sep 17 00:00:00 2001 From: Christopher Kolstad Date: Wed, 20 Dec 2023 10:56:20 +0100 Subject: [PATCH 4/4] Updated test for feature_refresher --- server/src/http/feature_refresher.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/http/feature_refresher.rs b/server/src/http/feature_refresher.rs index 484d5f95..1a008ed2 100644 --- a/server/src/http/feature_refresher.rs +++ b/server/src/http/feature_refresher.rs @@ -440,7 +440,6 @@ mod tests { use chrono::{Duration, Utc}; use dashmap::DashMap; use reqwest::Url; - use tracing_test::traced_test; use unleash_types::client_features::{ClientFeature, ClientFeatures}; use unleash_yggdrasil::EngineState; @@ -892,7 +891,6 @@ mod tests { } #[tokio::test] - #[traced_test] pub async fn getting_404_removes_tokens_from_token_to_refresh_but_not_its_features() { let mut token = EdgeToken::try_from("*:development.secret123".to_string()).unwrap(); token.status = Validated; @@ -936,7 +934,7 @@ mod tests { server.stop().await; tokio::time::sleep(std::time::Duration::from_millis(5)).await; // To ensure our refresh is due feature_refresher.refresh_features().await; - assert!(feature_refresher.tokens_to_refresh.is_empty()); + assert_eq!(feature_refresher.tokens_to_refresh.get("*:development.secret123").unwrap().failure_count, 1); assert!(!feature_refresher.features_cache.is_empty()); assert!(!feature_refresher.engine_cache.is_empty()); }