From a8f24926ae98cb3466d6993e088941c3a31eaefb Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:49:13 +0530 Subject: [PATCH 1/8] refactor aws --- .env.example | 5 +- .env.test | 44 +++++++++------ crates/orchestrator/src/config.rs | 22 ++++++-- .../src/data_storage/aws_s3/config.rs | 51 +---------------- .../src/data_storage/aws_s3/mod.rs | 56 +++---------------- crates/orchestrator/src/tests/common/mod.rs | 7 ++- .../src/tests/data_storage/mod.rs | 8 ++- 7 files changed, 68 insertions(+), 125 deletions(-) diff --git a/.env.example b/.env.example index 9d229975..13b54a21 100644 --- a/.env.example +++ b/.env.example @@ -24,15 +24,16 @@ MONGODB_CONNECTION_STRING= # AWS AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= -AWS_DEFAULT_REGION= +AWS_REGION= +AWS_ENDPOINT_URL= # SQS +QUEUE_PROVIDER = SQS_JOB_PROCESSING_QUEUE_URL= SQS_JOB_VERIFICATION_QUEUE_URL= # S3 AWS_S3_BUCKET_NAME= -AWS_S3_BUCKET_REGION= # Sharp Services SHARP_CUSTOMER_ID= diff --git a/.env.test b/.env.test index 0e8f36a3..cc6e29fe 100644 --- a/.env.test +++ b/.env.test @@ -1,32 +1,28 @@ -##### AWS config ##### +##### AWS CONFIG ##### AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY" -AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket" -AWS_S3_BUCKET_REGION="us-east-1" +AWS_REGION="localhost" AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566" + +##### STORAGE ##### + +DATA_STORAGE="s3" +AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket" + +##### QUEUE ##### + +QUEUE_PROVIDER = "sqs" SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue" SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue" -AWS_DEFAULT_REGION="localhost" -##### On chain config ##### +##### DATABASE ##### -MADARA_RPC_URL="http://localhost:3000" -ETHEREUM_RPC_URL="http://localhost:3001" -MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead" -PRIVATE_KEY="0xdead" -ETHEREUM_PRIVATE_KEY="0x000000000000000000000000000000000000000000000000000000000000beef" -STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead" +MONGODB_CONNECTION_STRING="mongodb://localhost:27017" -##### Config URLs ##### +##### PROVER ##### -DA_LAYER="ethereum" PROVER_SERVICE="sharp" -SETTLEMENT_LAYER="ethereum" -DATA_STORAGE="s3" -MONGODB_CONNECTION_STRING="mongodb://localhost:27017" - -# Sharp Services SHARP_CUSTOMER_ID="sharp_consumer_id" SHARP_URL="http://127.0.0.1:5000" # [IMP!!!] These are test certificates (they don't work) @@ -34,3 +30,15 @@ SHARP_USER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR4ekNDQXErZ0F3SUJBZ0lV SHARP_USER_KEY="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFJ5bVQzWTJQdkFJL0gKVkwxaTBNU011ZDVQanhuRE9UbW5tODBscWd4aXhTVHpiM1Z0cUdkUVNiQTZJWVpueEIvc25RSVFLdjZPL09QWApKMStrOWt2VjVwSjZiN0o4SUVLeFBZTW95WHNOU3dCdHhwclVpRGRXeUNhSXNtOEFBOHVKVHNDNUZ1ZDdWcDhhClVaT1ZGWDAwbHltazBEY2hzSSt3cFN4NUdxOG1HbU8vV1BjaHRFYnhRSGFFZDdOS01wR3VnQ2xocm1mRGJvTWUKZXVxUHRtV0VNNjVzbmpFa2hXeHpMcG1VUzE2QjRwUEFDU25xN1o3QUEveXRVVUc2T0hKcDVhb1RFMmdTbXl3RgpGZVpzVktObWFNVHlGRHBnd1BlNjl0blVtdzNSVkhNZjNmRDZkTTFMSzZzRDZ3WHFwMGRjQ3ZEdU5TQm8rZHhiCnQxWlFzdjhkQWdNQkFBRUNnZ0VBQU9mcDFiT2xLOVFKeXVlUHhjeDIvTkNVcUMxTEJDL01FdkEyUzVKWGFWbkcKbGhLR0pFb1U0Q0RoVk83dUlLYVZLTFZvMjk4RHFHUnBLM1d0RVE1TE40bytXYTcveTA5c1drMlVzbWxrVWFOZwpSaGtVZEJSK2dLNXVsQ3FKRml2dUJoTEQvRWlnQ1VWUGZKS2JtNG96TnpYcjVSMU5ENlV1aWFtODdtenlFcTBLCmZsVXlhR0RZNGdIdFNBOVBENVBFYlUveFpKeitKaHk5T2l3aVRXV0MrSHoyb2c3UWRDRDE2RlhGcit2VHpQN0MKb2tFb0VDZFNPRWlMalVENjBhS2ZxRmFCVm5MTkVudC9QSytmY1RBM05mNGtSMnFDNk9ZWjVFb09zYm1ka29ZTgpyU3NJZW9XblMxOEhvekZud2w3Z05wTUtjNmRzQzRBTldOVDFsTkhCb1FLQmdRRHlaUDFJSlppZUh6NlExaUVTCm5zd2tnblZCQUQ0SlVLR1ZDMHA3dk4yclNDZXh4c05ZZXFPTEEyZGZCUGpOVjd3blFKcUgxT05XellOMUJVSUUKeThLTCtFZVl6Q3RZa21LL21wSGJIMzNjd2tJODBuMHJROU1BalZMTlJ2YVVEOWp1NFBsRzFqaEFZUVVyTkViZQpKRlVpSk83aDVQa1llZG50SitqSHFpQnRoUUtCZ1FEZGtPbndmL0szYk4xenR0bXZQd0VicjhkVWJjRVh5NDFOCkl5VWwrZW1WSlgzYktKM0duNDZnQ2RsTTdkYmpwS3JVZ3oxL2JsZTgvMkVFckJvSEFRNkMrU2pEaGhvL01CbnIKekZheTBoK3YxbjBnZnNNVzRoOEF4cEFwc25OYnh6K2g1Wm5uSnRTd0srUjB3U0VJVVEzRjAxL2hMWWhLQ2l5OApwbW5HQi9hU3VRS0JnRzdxd1cvVExGd214ZlYyMXBsenFzeUdHZXVObGRXalhOMGIxcEI2b3lDdW11TmhwYUFHCk5uSDFNOGNxT2tPVWd4ZWZHMWRPbGx6eEc5ZGZlWTlDUWhyVW1NYVZucndmK0NuZkxDRU43d1VtcXpLenl1MFMKVXlwc2dOaElRYXNNK1dLTjllTnhRVHBNYXhZVERONjMxM0VSWDNKazJZdFdydDh6cFBSQXFDZ1ZBb0dCQU54egpUa0NMbmJ6aFphbTNlZm9DenlCMEVma3dSdHBkSGxkc3E0NlFqTmRuK1VSd3NpTXBLR2lWeEE3bDZsU1B4NlV3CmU2VHA3Z1JQZUlHRWwxVDJ1VENacGZSODNtcVdlb1FCeVJXZE9nZmplcFkxYWZpL3ZhY3c2Y21ERTRKeXloNVUKYTMveFE5ZVJwSHFDbWxKREMxZ1V5eVlwL3B2a2FjUytNeW5sVEhHSkFvR0FQekdTSzdXOHBUYldSVEFoaTVrSQpwZk5kWk1tcnRodUxNT3F6TGhyRjZublpldk9OdTBoYXVhZktlVElFd2w0clhYZHFKQlJBaWZKMFFsLzZKWFFkCmd1VzFrZWk1Ui8rUFZ5eUhab042c3NXSTNWYklwUUloUmt6UENnTDZhbHEwSzFpT1dlV1lIOHdORGRRdlB1T2UKRkZPOEovSzNxV0NtWjU0ODBBbTNhT0U9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K" SHARP_SERVER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQWxPZ0F3SUJBZ0lVRUR0Rjd0YVNTUnVQQTJ6Uk1aNWNzY2JCRm5jd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TkRBNE1UTXhNekEzTVROYUZ3MHlOVEE0Ck1UTXhNekEzTVROYU1FVXhDekFKQmdOVkJBWVRBa2xPTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFRHcEEwNEZ1QlNFaE5PNVYvMGxTaDkvSEgxeVRZT2dRVFdoOG43eDlRCnZGMHpvZFZueVFIdjE5elU5eVdia2xvOEkvOXFBVm9lRzdXTnpUVFg2Q295ZlNjb1YvazN0Q2UwVnVWMlFJTVQKdW82SzJSU05CVHB1TlNqNTlzUiszVTQ2OFRBQnY0YVpsYjU4TU5CRXM3MVRieVpLRHBGRVRkMkg3T0ZKajg4QQpNRi9MaXJkeDZPOFdZL0tDeisxd1ZXL1JRdytYYjRJSWx4bXJFOC9UZ3FNSEo4dFUxYkZiOWJNcTEvOTN5YWtJClU1V2J2NVhXKzFwZFVyTUFNcTFFaC9vZThMN2pFaFdvZXZrNzgyU0kwUk0xeG5MaEtrUUVBYXd6Zkg2ODZiR2YKUHQ3RkFIQ1pGaWJ4KzZzSkg0R1M3S25iK0x5bk9ud3phMWZPUXZEZmcvRm5BZ01CQUFHalV6QlJNQjBHQTFVZApEZ1FXQkJUYlFUdmlUTW1xNXlNK2ZJRVI4VjdTZk1pK3B6QWZCZ05WSFNNRUdEQVdnQlRiUVR2aVRNbXE1eU0rCmZJRVI4VjdTZk1pK3B6QVBCZ05WSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRREYKTllyRnpBa2RIVkhjUkd5SUNsTi9IVGswaldOcTVSdTB1RUpDQ21Dbm9ZY1pRSTlDQlcwTkl3dGpZUkpTKzR1UwordWh4VWpSYTA5YXdOWDhvYmU0dDZjK25HRnhZMGZqamk0cGZnbU1kMWNJeGdsM3E3Nlp0ZkllRGR6alRLRXN1CjRFUTVadnEwMnJvTEZ0ZjEvL3dRVG0xNkNKdFpGWnhNZ1phYnNxc2JRc3M2dWdMUGtTTmdBWjI1L2VhcWhnQ20KTjFUV2FxL0xJMVBLSkxPK085NFlMa2FsNVpyOTJCOXk4Q0VKVUVuSTA1R1N1MmJUOFM2a0ZBMEpadEszTW9SbwpqRWZWV1lQVHR5TFR4amNvRndCcDlHaXZYSDdSdHBxMDlmSmFhU1pNekxmNGlyNHpBdXprbExBNWZvampPNXlKCllnYlVaQUU2aS81N1NFWjR3VmxTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" SHARP_PROOF_LAYOUT="small" + +##### ON CHAIN CONFIG ##### + +DA_LAYER="ethereum" +SETTLEMENT_LAYER="ethereum" +ETHEREUM_RPC_URL="http://localhost:3001" +MADARA_RPC_URL="http://localhost:3000" +MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead" +PRIVATE_KEY="0xdead" +ETHEREUM_PRIVATE_KEY="0x000000000000000000000000000000000000000000000000000000000000beef" +STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead" + diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index 447b25ec..f4460552 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -4,6 +4,7 @@ use crate::data_storage::aws_s3::config::{AWSS3Config, AWSS3ConfigType}; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use arc_swap::{ArcSwap, Guard}; +use aws_config::SdkConfig; use da_client_interface::{DaClient, DaConfig}; use dotenvy::dotenv; use ethereum_da_client::config::EthereumDaConfig; @@ -56,8 +57,14 @@ pub async fn init_config() -> Config { // init database let database = Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await); + // init AWS + let aws_config = aws_config::load_from_env().await; + // init the queue - let queue = Box::new(SqsQueue {}); + // TODO: we use omniqueue for now which doesn't support loading AWS config + // from `SdkConfig`. We can later move to using `aws_sdk_sqs`. This would require + // us stop using the generic omniqueue abstractions for message ack/nack + let queue = build_queue_client(&aws_config); let da_client = build_da_client().await; @@ -65,7 +72,7 @@ pub async fn init_config() -> Config { let settlement_client = build_settlement_client(&settings_provider).await; let prover_client = build_prover_service(&settings_provider); - let storage_client = build_storage_client().await; + let storage_client = build_storage_client(&aws_config).await; Config::new(Arc::new(provider), da_client, prover_client, settlement_client, database, queue, storage_client) } @@ -177,9 +184,16 @@ pub async fn build_settlement_client( } } -pub async fn build_storage_client() -> Box { +pub async fn build_storage_client(aws_config: &SdkConfig) -> Box { match get_env_var_or_panic("DATA_STORAGE").as_str() { - "s3" => Box::new(AWSS3::new(AWSS3ConfigType::WithoutEndpoint(AWSS3Config::new_from_env())).await), + "s3" => Box::new(AWSS3::new(AWSS3Config::new_from_env(), aws_config)), _ => panic!("Unsupported Storage Client"), } } + +pub async fn build_queue_client(_aws_config: &SdkConfig) -> Box { + match get_env_var_or_panic("QUEUE_PROVIDER").as_str() { + "sqs" => Box::new(SqsQueue {}), + _ => panic!("Unsupported Queue Client"), + } +} diff --git a/crates/orchestrator/src/data_storage/aws_s3/config.rs b/crates/orchestrator/src/data_storage/aws_s3/config.rs index 0970d80d..665caeb4 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/config.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/config.rs @@ -2,64 +2,17 @@ use utils::env_utils::get_env_var_or_panic; use crate::data_storage::DataStorageConfig; -/// Represents the type of the config which one wants to pass to create the client -#[derive(Clone)] -pub enum AWSS3ConfigType { - WithEndpoint(S3LocalStackConfig), - WithoutEndpoint(AWSS3Config), -} - /// Represents AWS S3 config struct with all the necessary variables. #[derive(Clone)] pub struct AWSS3Config { - /// AWS ACCESS KEY ID - pub s3_key_id: String, - /// AWS ACCESS KEY SECRET - pub s3_key_secret: String, /// S3 Bucket Name - pub s3_bucket_name: String, - /// S3 Bucket region - pub s3_bucket_region: String, -} - -/// Represents AWS S3 config struct with all the necessary variables. -#[derive(Clone)] -pub struct S3LocalStackConfig { - /// AWS ACCESS KEY ID - pub s3_key_id: String, - /// AWS ACCESS KEY SECRET - pub s3_key_secret: String, - /// S3 Bucket Name - pub s3_bucket_name: String, - /// S3 Bucket region - pub s3_bucket_region: String, - /// Endpoint url - pub endpoint_url: String, + pub bucket_name: String, } /// Implementation of `DataStorageConfig` for `AWSS3Config` impl DataStorageConfig for AWSS3Config { /// To return the config struct by creating it from the environment variables. fn new_from_env() -> Self { - Self { - s3_key_id: get_env_var_or_panic("AWS_ACCESS_KEY_ID"), - s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"), - s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"), - s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"), - } - } -} - -/// Implementation of `DataStorageConfig` for `S3LocalStackConfig` -impl DataStorageConfig for S3LocalStackConfig { - /// To return the config struct by creating it from the environment variables. - fn new_from_env() -> Self { - Self { - s3_key_id: get_env_var_or_panic("AWS_ACCESS_KEY_ID"), - s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"), - s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"), - s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"), - endpoint_url: get_env_var_or_panic("AWS_ENDPOINT_URL"), - } + Self { bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME") } } } diff --git a/crates/orchestrator/src/data_storage/aws_s3/mod.rs b/crates/orchestrator/src/data_storage/aws_s3/mod.rs index 990a5d0a..13193afc 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/mod.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/mod.rs @@ -1,7 +1,8 @@ -use crate::data_storage::aws_s3::config::AWSS3ConfigType; +use crate::data_storage::aws_s3::config::{AWSS3Config, AWSS3ConfigType}; use crate::data_storage::DataStorage; use async_trait::async_trait; -use aws_sdk_s3::config::{Builder, Credentials, Region}; +use aws_config::SdkConfig; +use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::Client; use bytes::Bytes; @@ -13,7 +14,7 @@ pub mod config; /// AWSS3 represents AWS S3 client object containing the client and the config itself. pub struct AWSS3 { client: Client, - config: AWSS3ConfigType, + bucket: String, } /// Implementation for AWS S3 client. Contains the function for : @@ -22,49 +23,10 @@ pub struct AWSS3 { impl AWSS3 { /// Initializes a new AWS S3 client by passing the config /// and returning it. - pub async fn new(config: AWSS3ConfigType) -> Self { - let (config_builder, config) = match config { - AWSS3ConfigType::WithoutEndpoint(config) => { - let (credentials, region) = get_credentials_and_region_from_config( - config.s3_key_id.clone(), - config.s3_key_secret.clone(), - config.s3_bucket_region.clone(), - ); - ( - Builder::new().region(region).credentials_provider(credentials).force_path_style(true), - AWSS3ConfigType::WithoutEndpoint(config), - ) - } - AWSS3ConfigType::WithEndpoint(config) => { - let (credentials, region) = get_credentials_and_region_from_config( - config.s3_key_id.clone(), - config.s3_key_secret.clone(), - config.s3_bucket_region.clone(), - ); - ( - Builder::new() - .region(region) - .credentials_provider(credentials) - .force_path_style(true) - .endpoint_url(config.endpoint_url.clone()), - AWSS3ConfigType::WithEndpoint(config), - ) - } - }; - - let conf = config_builder.build(); - + pub fn new(s3_config: AWSS3Config, aws_config: &SdkConfig) -> Self { // Building AWS S3 config - let client = Client::from_conf(conf); - - Self { client, config } - } - - pub fn get_bucket_name(&self) -> String { - match self.config.clone() { - AWSS3ConfigType::WithEndpoint(config) => config.s3_bucket_name, - AWSS3ConfigType::WithoutEndpoint(config) => config.s3_bucket_name, - } + let client = Client::new(aws_config); + Self { client, bucket: s3_config.bucket_name } } } @@ -86,7 +48,7 @@ fn get_credentials_and_region_from_config( impl DataStorage for AWSS3 { /// Function to get the data from S3 bucket by Key. async fn get_data(&self, key: &str) -> Result { - let response = self.client.get_object().bucket(self.get_bucket_name()).key(key).send().await?; + let response = self.client.get_object().bucket(&self.bucket).key(key).send().await?; let data_stream = response.body.collect().await.expect("Failed to convert body into AggregatedBytes."); let data_bytes = data_stream.into_bytes(); Ok(data_bytes) @@ -96,7 +58,7 @@ impl DataStorage for AWSS3 { async fn put_data(&self, data: Bytes, key: &str) -> Result<()> { self.client .put_object() - .bucket(self.get_bucket_name()) + .bucket(&self.bucket) .key(key) .body(ByteStream::from(data)) .content_type("application/json") diff --git a/crates/orchestrator/src/tests/common/mod.rs b/crates/orchestrator/src/tests/common/mod.rs index 9d56b7e3..ee29ac34 100644 --- a/crates/orchestrator/src/tests/common/mod.rs +++ b/crates/orchestrator/src/tests/common/mod.rs @@ -7,8 +7,9 @@ use aws_config::Region; use mongodb::Client; use rstest::*; use serde::Deserialize; +use utils::env_utils::get_env_var_or_panic; -use crate::data_storage::aws_s3::config::{AWSS3ConfigType, S3LocalStackConfig}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use crate::database::mongodb::config::MongoDbConfig; @@ -84,5 +85,7 @@ pub struct MessagePayloadType { } pub async fn get_storage_client() -> Box { - Box::new(AWSS3::new(AWSS3ConfigType::WithEndpoint(S3LocalStackConfig::new_from_env())).await) + let aws_config = + aws_config::load_from_env().await.into_builder().endpoint_url(get_env_var_or_panic("AWS_ENDPOINT_URL")).build(); + Box::new(AWSS3::new(AWSS3Config::new_from_env(), &aws_config)) } diff --git a/crates/orchestrator/src/tests/data_storage/mod.rs b/crates/orchestrator/src/tests/data_storage/mod.rs index d127917a..477e5c3f 100644 --- a/crates/orchestrator/src/tests/data_storage/mod.rs +++ b/crates/orchestrator/src/tests/data_storage/mod.rs @@ -1,4 +1,4 @@ -use crate::data_storage::aws_s3::config::{AWSS3ConfigType, S3LocalStackConfig}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use crate::tests::config::TestConfigBuilder; @@ -18,8 +18,10 @@ async fn test_put_and_get_data_s3() -> color_eyre::Result<()> { dotenvy::from_filename("../.env.test")?; - let config = S3LocalStackConfig::new_from_env(); - let s3_client = AWSS3::new(AWSS3ConfigType::WithEndpoint(config)).await; + let config = AWSS3Config::new_from_env(); + let aws_config = + aws_config::load_from_env().await.into_builder().endpoint_url(get_env_var_or_panic("AWS_ENDPOINT_URL")).build(); + let s3_client = AWSS3::new(config, &aws_config); s3_client.build_test_bucket(&get_env_var_or_panic("AWS_S3_BUCKET_NAME")).await.unwrap(); let mock_data = json!( From c827ed6c2ec8006b554327decc85d344920c3cd5 Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:58:50 +0530 Subject: [PATCH 2/8] env cleanup and build_database_client --- .env.example | 59 +++++++++++++++++-------------- .env.test | 8 ++++- CHANGELOG.md | 1 + crates/orchestrator/src/config.rs | 11 ++++-- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index 13b54a21..fd03c031 100644 --- a/.env.example +++ b/.env.example @@ -1,43 +1,50 @@ +##### ORCHESTRATOR ##### + HOST= PORT= -DATABASE_URL= -MADARA_RPC_URL= -DA_LAYER= -SETTLEMENT_LAYER= -# Ethereum -ETHEREUM_PRIVATE_KEY= -ETHEREUM_RPC_URL= -MEMORY_PAGES_CONTRACT_ADDRESS= -STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS= +##### AWS CONFIG ##### - -# Starknet -STARKNET_PUBLIC_KEY= -STARNET_PRIVATE_KEY= -STARKNET_RPC_URL= -STARKNET_CAIRO_CORE_CONTRACT_ADDRESS= - -# MongoDB connection string -MONGODB_CONNECTION_STRING= - -# AWS AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_REGION= AWS_ENDPOINT_URL= -# SQS -QUEUE_PROVIDER = +##### STORAGE ##### + +DATA_STORAGE= +AWS_S3_BUCKET_NAME= + +##### QUEUE ##### + +QUEUE_PROVIDER= SQS_JOB_PROCESSING_QUEUE_URL= SQS_JOB_VERIFICATION_QUEUE_URL= -# S3 -AWS_S3_BUCKET_NAME= +##### DATABASE ##### + +DATABASE= +MONGODB_CONNECTION_STRING= -# Sharp Services +##### PROVER ##### + +PROVER_SERVICE= SHARP_CUSTOMER_ID= +SHARP_URL= +# [IMP!!!] These are test certificates (they don't work) SHARP_USER_CRT= SHARP_USER_KEY= SHARP_SERVER_CRT= -SHARP_PROOF_LAYOUT= \ No newline at end of file +SHARP_PROOF_LAYOUT= + +##### ON CHAIN CONFIG ##### + +DA_LAYER= +SETTLEMENT_LAYER= +ETHEREUM_RPC_URL= +MADARA_RPC_URL= +MEMORY_PAGES_CONTRACT_ADDRESS= +PRIVATE_KEY= +ETHEREUM_PRIVATE_KEY= +STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS= + diff --git a/.env.test b/.env.test index cc6e29fe..854f2a1b 100644 --- a/.env.test +++ b/.env.test @@ -1,3 +1,8 @@ +##### ORCHESTRATOR ##### + +HOST= +PORT= + ##### AWS CONFIG ##### AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID" @@ -12,12 +17,13 @@ AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket" ##### QUEUE ##### -QUEUE_PROVIDER = "sqs" +QUEUE_PROVIDER="sqs" SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue" SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue" ##### DATABASE ##### +DATABASE="mongodb" MONGODB_CONNECTION_STRING="mongodb://localhost:27017" ##### PROVER ##### diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aa715a1..11e32b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Changed +- refactor AWS config usage and clean .env files - GitHub's coverage CI yml file for localstack and db testing. - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder. - `.env` file requires two more variables which are queue urls for processing diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index f4460552..64b40a22 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -55,7 +55,7 @@ pub async fn init_config() -> Config { )); // init database - let database = Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await); + let database = build_database_client().await; // init AWS let aws_config = aws_config::load_from_env().await; @@ -191,9 +191,16 @@ pub async fn build_storage_client(aws_config: &SdkConfig) -> Box Box { +pub fn build_queue_client(_aws_config: &SdkConfig) -> Box { match get_env_var_or_panic("QUEUE_PROVIDER").as_str() { "sqs" => Box::new(SqsQueue {}), _ => panic!("Unsupported Queue Client"), } } + +pub async fn build_database_client() -> Box { + match get_env_var_or_panic("DATABASE").as_str() { + "mongodb" => Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await), + _ => panic!("Unsupported Database Client"), + } +} From 645d035595cb0b9ac9da9a989411776299d916b3 Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:08:05 +0530 Subject: [PATCH 3/8] remove get_credentials_and_region_from_config --- crates/orchestrator/src/config.rs | 2 +- crates/orchestrator/src/data_storage/aws_s3/mod.rs | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index 64b40a22..e3b7b975 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::data_storage::aws_s3::config::{AWSS3Config, AWSS3ConfigType}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use arc_swap::{ArcSwap, Guard}; diff --git a/crates/orchestrator/src/data_storage/aws_s3/mod.rs b/crates/orchestrator/src/data_storage/aws_s3/mod.rs index 13193afc..ddb60f01 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/mod.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/mod.rs @@ -1,4 +1,4 @@ -use crate::data_storage::aws_s3::config::{AWSS3Config, AWSS3ConfigType}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::DataStorage; use async_trait::async_trait; use aws_config::SdkConfig; @@ -30,17 +30,6 @@ impl AWSS3 { } } -/// Return the constructed `Credentials` and `Region` -fn get_credentials_and_region_from_config( - s3_key_id: String, - s3_key_secret: String, - s3_bucket_region: String, -) -> (Credentials, Region) { - let credentials = Credentials::new(s3_key_id, s3_key_secret, None, None, "loaded_from_custom_env"); - let region = Region::new(s3_bucket_region); - (credentials, region) -} - /// Implementation of `DataStorage` for `AWSS3` /// contains the function for getting the data and putting the data /// by taking the key as an argument. From ef231a5634b4c2b8b710a46e112c941db8e76229 Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:58:13 +0530 Subject: [PATCH 4/8] region for test case --- .env.test | 2 +- crates/orchestrator/src/data_storage/aws_s3/mod.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.env.test b/.env.test index 854f2a1b..1aadf808 100644 --- a/.env.test +++ b/.env.test @@ -7,7 +7,7 @@ PORT= AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY" -AWS_REGION="localhost" +AWS_REGION="us-east-1" AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566" ##### STORAGE ##### diff --git a/crates/orchestrator/src/data_storage/aws_s3/mod.rs b/crates/orchestrator/src/data_storage/aws_s3/mod.rs index ddb60f01..db7912fd 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/mod.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/mod.rs @@ -2,7 +2,6 @@ use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::DataStorage; use async_trait::async_trait; use aws_config::SdkConfig; -use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::Client; use bytes::Bytes; From 7a9ba7661ac53d396e9089a59f26ed9e818254b7 Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Fri, 23 Aug 2024 23:09:10 +0530 Subject: [PATCH 5/8] fix localstack not working --- crates/orchestrator/src/data_storage/aws_s3/mod.rs | 6 +++++- crates/orchestrator/src/tests/data_storage/mod.rs | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/orchestrator/src/data_storage/aws_s3/mod.rs b/crates/orchestrator/src/data_storage/aws_s3/mod.rs index db7912fd..0f920c7a 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/mod.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/mod.rs @@ -24,7 +24,11 @@ impl AWSS3 { /// and returning it. pub fn new(s3_config: AWSS3Config, aws_config: &SdkConfig) -> Self { // Building AWS S3 config - let client = Client::new(aws_config); + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(aws_config); + + // this is necessary for it to work with localstack in test cases + s3_config_builder.set_force_path_style(Some(true)); + let client = Client::from_conf(s3_config_builder.build()); Self { client, bucket: s3_config.bucket_name } } } diff --git a/crates/orchestrator/src/tests/data_storage/mod.rs b/crates/orchestrator/src/tests/data_storage/mod.rs index 477e5c3f..a3055acb 100644 --- a/crates/orchestrator/src/tests/data_storage/mod.rs +++ b/crates/orchestrator/src/tests/data_storage/mod.rs @@ -1,7 +1,6 @@ use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; -use crate::tests::config::TestConfigBuilder; use bytes::Bytes; use rstest::rstest; use serde_json::json; @@ -14,8 +13,6 @@ use utils::env_utils::get_env_var_or_panic; #[rstest] #[tokio::test] async fn test_put_and_get_data_s3() -> color_eyre::Result<()> { - TestConfigBuilder::new().build().await; - dotenvy::from_filename("../.env.test")?; let config = AWSS3Config::new_from_env(); From 42ec7ce02d15fd022a9152fa502b6817076ac742 Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Fri, 23 Aug 2024 23:36:04 +0530 Subject: [PATCH 6/8] add port and host to .env.test --- .env.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.test b/.env.test index 24978d4b..b0fbb065 100644 --- a/.env.test +++ b/.env.test @@ -1,7 +1,7 @@ ##### ORCHESTRATOR ##### -HOST= -PORT= +HOST=127.0.0.1 +PORT=3000 ##### AWS CONFIG ##### From 476476f734cea4e562140b324d9c480fb528ee54 Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Sat, 24 Aug 2024 17:18:51 +0530 Subject: [PATCH 7/8] fix eth pk --- .env.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.test b/.env.test index 9b35bda1..87a89d4c 100644 --- a/.env.test +++ b/.env.test @@ -47,5 +47,5 @@ ETHEREUM_RPC_URL="http://localhost:3001" MADARA_RPC_URL="http://localhost:3000" MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead" PRIVATE_KEY="0xdead" -ETHEREUM_PRIVATE_KEY="0x000000000000000000000000000000000000000000000000000000000000beef" +ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead" From 6475ed8356334d980d895c34f68ea8cd35db871a Mon Sep 17 00:00:00 2001 From: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> Date: Sat, 24 Aug 2024 19:23:41 +0530 Subject: [PATCH 8/8] fix CI --- .env.test | 3 ++- crates/da-clients/ethereum/src/config.rs | 4 ++-- crates/settlement-clients/ethereum/src/config.rs | 13 ++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.env.test b/.env.test index 87a89d4c..b87267ed 100644 --- a/.env.test +++ b/.env.test @@ -43,8 +43,9 @@ SHARP_PROOF_LAYOUT="small" DA_LAYER="ethereum" SETTLEMENT_LAYER="ethereum" -ETHEREUM_RPC_URL="http://localhost:3001" +SETTLEMENT_RPC_URL="http://localhost:3001" MADARA_RPC_URL="http://localhost:3000" +L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4" MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead" PRIVATE_KEY="0xdead" ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" diff --git a/crates/da-clients/ethereum/src/config.rs b/crates/da-clients/ethereum/src/config.rs index b50604b2..34613486 100644 --- a/crates/da-clients/ethereum/src/config.rs +++ b/crates/da-clients/ethereum/src/config.rs @@ -19,14 +19,14 @@ pub struct EthereumDaConfig { impl DaConfig for EthereumDaConfig { fn new_from_env() -> Self { Self { - rpc_url: get_env_var_or_panic("ETHEREUM_RPC_URL"), + rpc_url: get_env_var_or_panic("SETTLEMENT_RPC_URL"), memory_pages_contract: get_env_var_or_panic("MEMORY_PAGES_CONTRACT_ADDRESS"), private_key: get_env_var_or_panic("PRIVATE_KEY"), } } async fn build_client(&self) -> EthereumDaClient { let client = - RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse ETHEREUM_RPC_URL")); + RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL")); let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client); EthereumDaClient { provider } diff --git a/crates/settlement-clients/ethereum/src/config.rs b/crates/settlement-clients/ethereum/src/config.rs index 294c2a54..3d038390 100644 --- a/crates/settlement-clients/ethereum/src/config.rs +++ b/crates/settlement-clients/ethereum/src/config.rs @@ -6,8 +6,8 @@ use url::Url; use utils::env_utils::get_env_var_or_panic; pub const ENV_CORE_CONTRACT_ADDRESS: &str = "STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS"; -pub const DEFAULT_SETTLEMENT_CLIENT_RPC: &str = "DEFAULT_SETTLEMENT_CLIENT_RPC"; -pub const DEFAULT_L1_CORE_CONTRACT_ADDRESS: &str = "DEFAULT_L1_CORE_CONTRACT_ADDRESS"; +pub const SETTLEMENT_RPC_URL: &str = "SETTLEMENT_RPC_URL"; +pub const L1_CORE_CONTRACT_ADDRESS: &str = "L1_CORE_CONTRACT_ADDRESS"; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EthereumSettlementConfig { @@ -17,9 +17,8 @@ pub struct EthereumSettlementConfig { impl SettlementConfig for EthereumSettlementConfig { fn new_from_env() -> Self { - let rpc_url = get_env_var_or_panic(DEFAULT_SETTLEMENT_CLIENT_RPC); - let rpc_url = - Url::from_str(&rpc_url).unwrap_or_else(|_| panic!("Failed to parse {}", DEFAULT_SETTLEMENT_CLIENT_RPC)); + let rpc_url = get_env_var_or_panic(SETTLEMENT_RPC_URL); + let rpc_url = Url::from_str(&rpc_url).unwrap_or_else(|_| panic!("Failed to parse {}", SETTLEMENT_RPC_URL)); let core_contract_address = get_env_var_or_panic(ENV_CORE_CONTRACT_ADDRESS); Self { rpc_url, core_contract_address } } @@ -28,8 +27,8 @@ impl SettlementConfig for EthereumSettlementConfig { impl Default for EthereumSettlementConfig { fn default() -> Self { Self { - rpc_url: get_env_var_or_panic(DEFAULT_SETTLEMENT_CLIENT_RPC).parse().unwrap(), - core_contract_address: get_env_var_or_panic(DEFAULT_L1_CORE_CONTRACT_ADDRESS), + rpc_url: get_env_var_or_panic(SETTLEMENT_RPC_URL).parse().unwrap(), + core_contract_address: get_env_var_or_panic(L1_CORE_CONTRACT_ADDRESS), } } }