From 6c7337a8fc727a79938f5c61c27096381ab1432e Mon Sep 17 00:00:00 2001 From: Christopher Kolstad Date: Tue, 19 Dec 2023 16:36:26 +0100 Subject: [PATCH] fix: Do not remove features cache if upstream goes away If upstream goes away, edge should remain durable and keep the features it has gotten for the tokens. Since we already keep the authentication status for the tokens we've seen this fix allows you to start edge, manage to talk to upstream once, then sever connection to upstream, but keep serving feature toggles for known tokens --- Cargo.lock | 20 ++++---- server/Cargo.toml | 10 ++-- server/src/frontend_api.rs | 4 +- server/src/http/feature_refresher.rs | 71 ++++++++++++++++++++++------ 4 files changed, 74 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce973c3c..9134c52d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-cors" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b340e9cfa5b08690aae90fb61beb44e9b06f44fe3d0f93781aaa58cfba86245e" +checksum = "0346d8c1f762b41b458ed3145eea914966bb9ad20b9be0d6d463b20d45586370" dependencies = [ "actix-utils", "actix-web", @@ -569,9 +569,9 @@ checksum = "8d18b093eba54c9aaa1e3784d4361eb2ba944cf7d0a932a830132238f483e8d8" [[package]] name = "clap" -version = "4.4.10" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", "clap_derive", @@ -588,9 +588,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.9" +version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", @@ -2452,9 +2452,9 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9198caff1c94f1a5df6664bddbc379896b51b98a55b0b3fedcb23078fe00c77" +checksum = "615d846f7174a0850dca101bca72f6913e3376a64c5fda2b965d7fc3d1ff60cb" dependencies = [ "const_format", "git2", @@ -3153,9 +3153,9 @@ dependencies = [ [[package]] name = "utoipa-swagger-ui" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154517adf0d0b6e22e8e1f385628f14fcaa3db43531dc74303d3edef89d6dfe5" +checksum = "f839caa8e09dddc3ff1c3112a91ef7da0601075ba5025d9f33ae99c4cb9b6e51" dependencies = [ "actix-web", "mime_guess", diff --git a/server/Cargo.toml b/server/Cargo.toml index b4cd3a66..cae6a16a 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -15,7 +15,7 @@ repository = "https://github.com/Unleash/unleash-edge" version = "16.0.6" [dependencies] -actix-cors = "0.6.4" +actix-cors = "0.6.5" actix-http = { version = "3.4.0", features = ["compress-zstd", "rustls-0_21"] } actix-middleware-etag = "0.3.0" actix-service = "2.0.2" @@ -26,7 +26,7 @@ anyhow = "1.0.75" async-trait = "0.1.74" chrono = { version = "0.4.31", features = ["serde"] } cidr = "0.2.2" -clap = { version = "4.4.8", features = ["derive", "env"] } +clap = { version = "4.4.11", features = ["derive", "env"] } clap-markdown = "0.1.3" dashmap = "5.5.3" futures = "0.3.29" @@ -62,7 +62,7 @@ rustls-pemfile = "1.0.4" serde = { version = "1.0.192", features = ["derive"] } serde_json = "1.0.108" serde_qs = { version = "0.12.0", features = ["actix4", "tracing"] } -shadow-rs = "0.24.1" +shadow-rs = { version = "0.25.0" } tokio = { version = "1.34.0", features = [ "macros", "rt-multi-thread", @@ -75,7 +75,7 @@ ulid = "1.1.0" unleash-types = { version = "0.10", features = ["openapi", "hashes"] } unleash-yggdrasil = { version = "0.8.0" } utoipa = { version = "4.1.0", features = ["actix_extras", "chrono"] } -utoipa-swagger-ui = { version = "4", features = ["actix-web"] } +utoipa-swagger-ui = { version = "5", features = ["actix-web"] } [dev-dependencies] actix-http = "3.4.0" actix-http-test = "3.1.0" @@ -89,4 +89,4 @@ testcontainers-modules = { version = "0.2.0", features = ["redis"] } tracing-test = "0.2.4" [build-dependencies] -shadow-rs = "0.24.1" +shadow-rs = "0.25.0" diff --git a/server/src/frontend_api.rs b/server/src/frontend_api.rs index 7065d704..7122706d 100644 --- a/server/src/frontend_api.rs +++ b/server/src/frontend_api.rs @@ -11,7 +11,7 @@ use actix_web::{ }; use dashmap::DashMap; use serde_qs::actix::QsQuery; -use tracing::{debug, instrument}; +use tracing::{debug, instrument, warn}; use unleash_types::client_features::Context; use unleash_types::client_metrics::{ClientApplication, ConnectVia}; use unleash_types::{ @@ -214,7 +214,7 @@ security( ) )] #[get("")] -#[instrument(skip(engine_cache, token_cache))] +#[instrument(skip(edge_token, req, engine_cache, token_cache))] async fn get_enabled_frontend( edge_token: EdgeToken, engine_cache: Data>, diff --git a/server/src/http/feature_refresher.rs b/server/src/http/feature_refresher.rs index 23b57247..8bdfd428 100644 --- a/server/src/http/feature_refresher.rs +++ b/server/src/http/feature_refresher.rs @@ -5,7 +5,7 @@ use actix_web::http::header::EntityTag; use chrono::Utc; use dashmap::DashMap; use reqwest::StatusCode; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use unleash_types::client_features::Segment; use unleash_types::client_metrics::ClientApplication; use unleash_types::{ @@ -376,11 +376,11 @@ impl FeatureRefresher { self.backoff(&refresh.token); } _ => { - warn!("Couldn't refresh features, but will retry next go") + info!("Couldn't refresh features, but will retry next go") } }, FeatureError::AccessDenied => { - warn!("Token used to fetch features was Forbidden, will remove from list of refresh tasks"); + info!("Token used to fetch features was Forbidden, will remove from list of refresh tasks"); self.tokens_to_refresh.remove(&refresh.token.token); if !self.tokens_to_refresh.iter().any(|e| { e.value().token.environment == refresh.token.environment @@ -392,23 +392,15 @@ impl FeatureRefresher { } } FeatureError::NotFound => { - warn!("Had a bad URL when trying to fetch features. Removing ourselves"); + info!("Had a bad URL when trying to fetch features. Removing ourselves from the list of refresh tasks"); self.tokens_to_refresh.remove(&refresh.token.token); - if !self.tokens_to_refresh.iter().any(|e| { - e.value().token.environment == refresh.token.environment - }) { - let cache_key = cache_key(&refresh.token); - // No tokens left that access the environment of our current refresh. Deleting client features and engine cache - self.features_cache.remove(&cache_key); - self.engine_cache.remove(&cache_key); - } } } } EdgeError::ClientCacheError => { - warn!("Couldn't refresh features, but will retry next go") + info!("Couldn't refresh features, but will retry next go") } - _ => warn!("Couldn't refresh features: {e:?}. Will retry next pass"), + _ => info!("Couldn't refresh features: {e:?}. Will retry next pass"), } } } @@ -448,6 +440,7 @@ mod tests { use chrono::{Duration, Utc}; use dashmap::DashMap; use reqwest::Url; + use tracing_test::traced_test; use unleash_types::client_features::{ClientFeature, ClientFeatures}; use unleash_yggdrasil::EngineState; @@ -898,6 +891,56 @@ mod tests { assert!(feature_refresher.engine_cache.is_empty()); } + #[tokio::test] + #[traced_test] + pub async fn getting_404_removes_tokens_from_token_to_refresh_but_not_its_features() { + let mut token = EdgeToken::try_from("*:development.secret123".to_string()).unwrap(); + token.status = Validated; + token.token_type = Some(TokenType::Client); + let token_cache = DashMap::default(); + token_cache.insert(token.token.clone(), token.clone()); + let upstream_features_cache: Arc> = + Arc::new(DashMap::default()); + let upstream_engine_cache: Arc> = Arc::new(DashMap::default()); + let upstream_token_cache: Arc> = Arc::new(token_cache); + let example_features = features_from_disk("../examples/features.json"); + let cache_key = cache_key(&token); + let mut engine_state = EngineState::default(); + engine_state.take_state(example_features.clone()); + upstream_features_cache.insert(cache_key.clone(), example_features.clone()); + upstream_engine_cache.insert(cache_key.clone(), engine_state); + let mut server = client_api_test_server( + upstream_token_cache, + upstream_features_cache, + upstream_engine_cache, + ) + .await; + let unleash_client = UnleashClient::new(server.url("/").as_str(), None).unwrap(); + let features_cache: Arc> = Arc::new(DashMap::default()); + let engine_cache: Arc> = Arc::new(DashMap::default()); + let feature_refresher = FeatureRefresher::new( + Arc::new(unleash_client), + features_cache, + engine_cache, + Duration::milliseconds(1), + None, + ); + feature_refresher + .register_token_for_refresh(token, None) + .await; + assert!(!feature_refresher.tokens_to_refresh.is_empty()); + feature_refresher.refresh_features().await; + assert!(!feature_refresher.tokens_to_refresh.is_empty()); + assert!(!feature_refresher.features_cache.is_empty()); + assert!(!feature_refresher.engine_cache.is_empty()); + server.stop().await; + tokio::time::sleep(std::time::Duration::from_millis(5)).await; // To ensure our refresh is due + feature_refresher.refresh_features().await; + assert!(feature_refresher.tokens_to_refresh.is_empty()); + assert!(!feature_refresher.features_cache.is_empty()); + assert!(!feature_refresher.engine_cache.is_empty()); + } + #[tokio::test] pub async fn when_we_have_a_cache_and_token_gets_removed_caches_are_emptied() { let upstream_features_cache: Arc> =