diff --git a/config/config.example.toml b/config/config.example.toml index 999266321144..c7a75f9e83c5 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -23,11 +23,10 @@ certificate = "/path/to/certificate.pem" # Proxy server configuration for connecting to payment gateways. # Don't define the fields if a Proxy isn't needed. Empty strings will cause failure. [proxy] -# http_url = "http proxy url" # Proxy all HTTP traffic via this proxy -# https_url = "https proxy url" # Proxy all HTTPS traffic via this proxy -idle_pool_connection_timeout = 90 # Timeout for idle pool connections (defaults to 90s) -bypass_proxy_urls = [] # A list of URLs that should bypass the proxy - +# http_url = "http proxy url" # Proxy all HTTP traffic via this proxy +# https_url = "https proxy url" # Proxy all HTTPS traffic via this proxy +idle_pool_connection_timeout = 90 # Timeout for idle pool connections (defaults to 90s) +bypass_proxy_hosts = "localhost, cluster.local" # A comma-separated list of domains or IP addresses that should not use the proxy. Whitespace between entries would be ignored. # Configuration for the Key Manager Service [key_manager] @@ -468,8 +467,8 @@ bank_redirect.giropay = { connector_list = "adyen,globalpay" } [mandates.update_mandate_supported] -card.credit = { connector_list = "cybersource" } # Update Mandate supported payment method type and connector for card -card.debit = { connector_list = "cybersource" } # Update Mandate supported payment method type and connector for card +card.credit = { connector_list = "cybersource" } # Update Mandate supported payment method type and connector for card +card.debit = { connector_list = "cybersource" } # Update Mandate supported payment method type and connector for card # Required fields info used while listing the payment_method_data [required_fields.pay_later] # payment_method = "pay_later" diff --git a/config/deployments/env_specific.toml b/config/deployments/env_specific.toml index 967b847dae51..a4e7db3725e3 100644 --- a/config/deployments/env_specific.toml +++ b/config/deployments/env_specific.toml @@ -195,9 +195,9 @@ pm_auth_key = "pm_auth_key" # Payment method auth key used for authorization redis_expiry = 900 # Redis expiry time in milliseconds [proxy] -http_url = "http://proxy_http_url" # Outgoing proxy http URL to proxy the HTTP traffic -https_url = "https://proxy_https_url" # Outgoing proxy https URL to proxy the HTTPS traffic -bypass_proxy_urls = [] # A list of URLs that should bypass the proxy +http_url = "http://proxy_http_url" # Proxy all HTTP traffic via this proxy +https_url = "https://proxy_https_url" # Proxy all HTTPS traffic via this proxy +bypass_proxy_hosts = "localhost, cluster.local" # A comma-separated list of domains or IP addresses that should not use the proxy. Whitespace between entries would be ignored. # Redis credentials [redis] @@ -306,8 +306,8 @@ enabled = false global_tenant = { schema = "public", redis_key_prefix = "", clickhouse_database = "default"} [multitenancy.tenants.public] -base_url = "http://localhost:8080" -schema = "public" +base_url = "http://localhost:8080" +schema = "public" redis_key_prefix = "" clickhouse_database = "default" diff --git a/crates/common_enums/src/enums.rs b/crates/common_enums/src/enums.rs index 0833ab37e3f8..7a3ff787497b 100644 --- a/crates/common_enums/src/enums.rs +++ b/crates/common_enums/src/enums.rs @@ -63,6 +63,8 @@ pub enum ApiClientError { #[error("Unexpected state reached/Invariants conflicted")] UnexpectedState, + #[error("Failed to parse URL")] + UrlParsingFailed, #[error("URL encoding of request payload failed")] UrlEncodingFailed, #[error("Failed to send request to connector {0}")] diff --git a/crates/router/src/bin/scheduler.rs b/crates/router/src/bin/scheduler.rs index 62c8c03e1a97..9f617d097545 100644 --- a/crates/router/src/bin/scheduler.rs +++ b/crates/router/src/bin/scheduler.rs @@ -35,15 +35,8 @@ async fn main() -> CustomResult<(), ProcessTrackerError> { let conf = Settings::with_config_path(cmd_line.config_path) .expect("Unable to construct application configuration"); let api_client = Box::new( - services::ProxyClient::new( - conf.proxy.clone(), - services::proxy_bypass_urls( - conf.key_manager.get_inner(), - &conf.locker, - &conf.proxy.bypass_proxy_urls, - ), - ) - .change_context(ProcessTrackerError::ConfigurationError)?, + services::ProxyClient::new(&conf.proxy) + .change_context(ProcessTrackerError::ConfigurationError)?, ); // channel for listening to redis disconnect events let (redis_shutdown_signal_tx, redis_shutdown_signal_rx) = oneshot::channel(); diff --git a/crates/router/src/configs/defaults.rs b/crates/router/src/configs/defaults.rs index 74f6502159da..8989c3386002 100644 --- a/crates/router/src/configs/defaults.rs +++ b/crates/router/src/configs/defaults.rs @@ -58,7 +58,7 @@ impl Default for super::settings::Proxy { http_url: Default::default(), https_url: Default::default(), idle_pool_connection_timeout: Some(90), - bypass_proxy_urls: Vec::new(), + bypass_proxy_hosts: Default::default(), } } } diff --git a/crates/router/src/configs/settings.rs b/crates/router/src/configs/settings.rs index ad5d9e89aaae..3544839756e8 100644 --- a/crates/router/src/configs/settings.rs +++ b/crates/router/src/configs/settings.rs @@ -623,7 +623,7 @@ pub struct Proxy { pub http_url: Option, pub https_url: Option, pub idle_pool_connection_timeout: Option, - pub bypass_proxy_urls: Vec, + pub bypass_proxy_hosts: Option, } #[derive(Debug, Deserialize, Clone)] @@ -804,7 +804,6 @@ impl Settings { .with_list_parse_key("log.telemetry.route_to_trace") .with_list_parse_key("redis.cluster_urls") .with_list_parse_key("events.kafka.brokers") - .with_list_parse_key("proxy.bypass_proxy_urls") .with_list_parse_key("connectors.supported.wallets") .with_list_parse_key("connector_request_reference_id_config.merchant_ids_send_payment_id_as_connector_request_id"), diff --git a/crates/router/src/lib.rs b/crates/router/src/lib.rs index 4fe9318f64bb..f41ee8874bb5 100644 --- a/crates/router/src/lib.rs +++ b/crates/router/src/lib.rs @@ -236,19 +236,9 @@ pub async fn start_server(conf: settings::Settings) -> Applicatio logger::debug!(startup_config=?conf); let server = conf.server.clone(); let (tx, rx) = oneshot::channel(); - let api_client = Box::new( - services::ProxyClient::new( - conf.proxy.clone(), - services::proxy_bypass_urls( - conf.key_manager.get_inner(), - &conf.locker, - &conf.proxy.bypass_proxy_urls, - ), - ) - .map_err(|error| { - errors::ApplicationError::ApiClientError(error.current_context().clone()) - })?, - ); + let api_client = Box::new(services::ProxyClient::new(&conf.proxy).map_err(|error| { + errors::ApplicationError::ApiClientError(error.current_context().clone()) + })?); let state = Box::pin(AppState::new(conf, tx, api_client)).await; let request_body_limit = server.request_body_limit; diff --git a/crates/router/src/services/api.rs b/crates/router/src/services/api.rs index cac856b2c48b..12a48d980bbd 100644 --- a/crates/router/src/services/api.rs +++ b/crates/router/src/services/api.rs @@ -17,7 +17,7 @@ use actix_web::{ http::header::{HeaderName, HeaderValue}, web, FromRequest, HttpRequest, HttpResponse, Responder, ResponseError, }; -pub use client::{proxy_bypass_urls, ApiClient, MockApiClient, ProxyClient}; +pub use client::{ApiClient, MockApiClient, ProxyClient}; pub use common_enums::enums::PaymentAction; pub use common_utils::request::{ContentType, Method, Request, RequestBuilder}; use common_utils::{ @@ -415,29 +415,11 @@ pub async fn send_request( ) -> CustomResult { logger::info!(method=?request.method, headers=?request.headers, payload=?request.body, ?request); - let url = reqwest::Url::parse(&request.url) - .change_context(errors::ApiClientError::UrlEncodingFailed)?; - - #[cfg(feature = "dummy_connector")] - let should_bypass_proxy = url - .as_str() - .starts_with(&state.conf.connectors.dummyconnector.base_url) - || proxy_bypass_urls( - state.conf.key_manager.get_inner(), - &state.conf.locker, - &state.conf.proxy.bypass_proxy_urls, - ) - .contains(&url.to_string()); - #[cfg(not(feature = "dummy_connector"))] - let should_bypass_proxy = proxy_bypass_urls( - &state.conf.key_manager.get_inner(), - &state.conf.locker, - &state.conf.proxy.bypass_proxy_urls, - ) - .contains(&url.to_string()); + let url = + url::Url::parse(&request.url).change_context(errors::ApiClientError::UrlParsingFailed)?; + let client = client::create_client( &state.conf.proxy, - should_bypass_proxy, request.certificate, request.certificate_key, )?; diff --git a/crates/router/src/services/api/client.rs b/crates/router/src/services/api/client.rs index 9a496c18d9a6..096c5ed45d15 100644 --- a/crates/router/src/services/api/client.rs +++ b/crates/router/src/services/api/client.rs @@ -10,18 +10,16 @@ use router_env::tracing_actix_web::RequestId; use super::{request::Maskable, Request}; use crate::{ - configs::settings::{Locker, Proxy}, - consts::{BASE64_ENGINE, LOCKER_HEALTH_CALL_PATH}, + configs::settings::Proxy, + consts::BASE64_ENGINE, core::errors::{ApiClientError, CustomResult}, - routes::{app::settings::KeyManagerConfig, SessionState}, + routes::SessionState, }; -static NON_PROXIED_CLIENT: OnceCell = OnceCell::new(); -static PROXIED_CLIENT: OnceCell = OnceCell::new(); +static DEFAULT_CLIENT: OnceCell = OnceCell::new(); fn get_client_builder( proxy_config: &Proxy, - should_bypass_proxy: bool, ) -> CustomResult { let mut client_builder = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) @@ -31,16 +29,16 @@ fn get_client_builder( .unwrap_or_default(), )); - if should_bypass_proxy { - return Ok(client_builder); - } + let proxy_exclusion_config = + reqwest::NoProxy::from_string(&proxy_config.bypass_proxy_hosts.clone().unwrap_or_default()); // Proxy all HTTPS traffic through the configured HTTPS proxy if let Some(url) = proxy_config.https_url.as_ref() { client_builder = client_builder.proxy( reqwest::Proxy::https(url) .change_context(ApiClientError::InvalidProxyConfiguration) - .attach_printable("HTTPS proxy configuration error")?, + .attach_printable("HTTPS proxy configuration error")? + .no_proxy(proxy_exclusion_config.clone()), ); } @@ -49,44 +47,35 @@ fn get_client_builder( client_builder = client_builder.proxy( reqwest::Proxy::http(url) .change_context(ApiClientError::InvalidProxyConfiguration) - .attach_printable("HTTP proxy configuration error")?, + .attach_printable("HTTP proxy configuration error")? + .no_proxy(proxy_exclusion_config), ); } Ok(client_builder) } -fn get_base_client( - proxy_config: &Proxy, - should_bypass_proxy: bool, -) -> CustomResult { - Ok(if should_bypass_proxy - || (proxy_config.http_url.is_none() && proxy_config.https_url.is_none()) - { - &NON_PROXIED_CLIENT - } else { - &PROXIED_CLIENT - } - .get_or_try_init(|| { - get_client_builder(proxy_config, should_bypass_proxy)? - .build() - .change_context(ApiClientError::ClientConstructionFailed) - .attach_printable("Failed to construct base client") - })? - .clone()) +fn get_base_client(proxy_config: &Proxy) -> CustomResult { + Ok(DEFAULT_CLIENT + .get_or_try_init(|| { + get_client_builder(proxy_config)? + .build() + .change_context(ApiClientError::ClientConstructionFailed) + .attach_printable("Failed to construct base client") + })? + .clone()) } // We may need to use outbound proxy to connect to external world. // Precedence will be the environment variables, followed by the config. pub fn create_client( proxy_config: &Proxy, - should_bypass_proxy: bool, client_certificate: Option>, client_certificate_key: Option>, ) -> CustomResult { match (client_certificate, client_certificate_key) { (Some(encoded_certificate), Some(encoded_certificate_key)) => { - let client_builder = get_client_builder(proxy_config, should_bypass_proxy)?; + let client_builder = get_client_builder(proxy_config)?; let identity = create_identity_from_certificate_and_key( encoded_certificate.clone(), @@ -105,7 +94,7 @@ pub fn create_client( .change_context(ApiClientError::ClientConstructionFailed) .attach_printable("Failed to construct client with certificate and certificate key") } - _ => get_base_client(proxy_config, should_bypass_proxy), + _ => get_base_client(proxy_config), } } @@ -145,35 +134,6 @@ pub fn create_certificate( .change_context(ApiClientError::CertificateDecodeFailed) } -pub fn proxy_bypass_urls( - key_manager: &KeyManagerConfig, - locker: &Locker, - config_whitelist: &[String], -) -> Vec { - let key_manager_host = key_manager.url.to_owned(); - let locker_host = locker.host.to_owned(); - let locker_host_rs = locker.host_rs.to_owned(); - - let proxy_list = [ - format!("{locker_host}/cards/add"), - format!("{locker_host}/cards/fingerprint"), - format!("{locker_host}/cards/retrieve"), - format!("{locker_host}/cards/delete"), - format!("{locker_host_rs}/cards/add"), - format!("{locker_host_rs}/cards/retrieve"), - format!("{locker_host_rs}/cards/delete"), - format!("{locker_host_rs}{}", LOCKER_HEALTH_CALL_PATH), - format!("{locker_host}/card/addCard"), - format!("{locker_host}/card/getCard"), - format!("{locker_host}/card/deleteCard"), - format!("{key_manager_host}/data/encrypt"), - format!("{key_manager_host}/data/decrypt"), - format!("{key_manager_host}/key/create"), - format!("{key_manager_host}/key/rotate"), - ]; - [&proxy_list, config_whitelist].concat().to_vec() -} - pub trait RequestBuilder: Send + Sync { fn json(&mut self, body: serde_json::Value); fn url_encoded_form(&mut self, body: serde_json::Value); @@ -201,6 +161,7 @@ where method: Method, url: String, ) -> CustomResult, ApiClientError>; + fn request_with_certificate( &self, method: Method, @@ -218,7 +179,9 @@ where ) -> CustomResult; fn add_request_id(&mut self, request_id: RequestId); + fn get_request_id(&self) -> Option; + fn add_flow_name(&mut self, flow_name: String); } @@ -226,60 +189,31 @@ dyn_clone::clone_trait_object!(ApiClient); #[derive(Clone)] pub struct ProxyClient { - proxy_client: reqwest::Client, - non_proxy_client: reqwest::Client, - whitelisted_urls: Vec, + proxy_config: Proxy, + client: reqwest::Client, request_id: Option, } impl ProxyClient { - pub fn new( - proxy_config: Proxy, - whitelisted_urls: Vec, - ) -> CustomResult { - let non_proxy_client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .change_context(ApiClientError::ClientConstructionFailed)?; - - let mut proxy_builder = - reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()); - - if let Some(url) = proxy_config.https_url.as_ref() { - proxy_builder = proxy_builder.proxy( - reqwest::Proxy::https(url) - .change_context(ApiClientError::InvalidProxyConfiguration)?, - ); - } - - if let Some(url) = proxy_config.http_url.as_ref() { - proxy_builder = proxy_builder.proxy( - reqwest::Proxy::http(url) - .change_context(ApiClientError::InvalidProxyConfiguration)?, - ); - } - - let proxy_client = proxy_builder + pub fn new(proxy_config: &Proxy) -> CustomResult { + let client = get_client_builder(proxy_config)? .build() .change_context(ApiClientError::InvalidProxyConfiguration)?; Ok(Self { - proxy_client, - non_proxy_client, - whitelisted_urls, + proxy_config: proxy_config.clone(), + client, request_id: None, }) } pub fn get_reqwest_client( &self, - base_url: String, client_certificate: Option>, client_certificate_key: Option>, ) -> CustomResult { match (client_certificate, client_certificate_key) { (Some(certificate), Some(certificate_key)) => { - let client_builder = - reqwest::Client::builder().redirect(reqwest::redirect::Policy::none()); + let client_builder = get_client_builder(&self.proxy_config)?; let identity = create_identity_from_certificate_and_key(certificate, certificate_key)?; Ok(client_builder @@ -290,13 +224,7 @@ impl ProxyClient { "Failed to construct client with certificate and certificate key", )?) } - (_, _) => { - if self.whitelisted_urls.contains(&base_url) { - Ok(self.non_proxy_client.clone()) - } else { - Ok(self.proxy_client.clone()) - } - } + (_, _) => Ok(self.client.clone()), } } } @@ -355,8 +283,6 @@ impl RequestBuilder for RouterRequestBuilder { } } -// TODO: remove this when integrating this trait -#[allow(dead_code)] #[async_trait::async_trait] impl ApiClient for ProxyClient { fn request( @@ -375,7 +301,7 @@ impl ApiClient for ProxyClient { certificate_key: Option>, ) -> CustomResult, ApiClientError> { let client_builder = self - .get_reqwest_client(url.clone(), certificate, certificate_key) + .get_reqwest_client(certificate, certificate_key) .change_context(ApiClientError::ClientConstructionFailed)?; Ok(Box::new(RouterRequestBuilder { inner: Some(client_builder.request(method, url)), diff --git a/crates/router/src/services/openidconnect.rs b/crates/router/src/services/openidconnect.rs index 4f5056f32734..0c0f86431a1a 100644 --- a/crates/router/src/services/openidconnect.rs +++ b/crates/router/src/services/openidconnect.rs @@ -155,7 +155,7 @@ async fn get_oidc_reqwest_client( state: &SessionState, request: oidc::HttpRequest, ) -> Result { - let client = client::create_client(&state.conf.proxy, false, None, None) + let client = client::create_client(&state.conf.proxy, None, None) .map_err(|e| e.current_context().to_owned())?; let mut request_builder = client