diff --git a/Cargo.lock b/Cargo.lock index 34457046..06be14e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -595,7 +595,7 @@ dependencies = [ "fastrand 2.1.0", "hex", "http 0.2.12", - "hyper", + "hyper 0.14.29", "ring 0.17.8", "time", "tokio", @@ -895,8 +895,12 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "http-body 1.0.0", +<<<<<<< HEAD "httparse", "hyper", +======= + "hyper 0.14.29", +>>>>>>> 4effe3f (use google-cloud-storage) "hyper-rustls", "once_cell", "pin-project-lite", @@ -985,7 +989,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -994,7 +998,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -1516,11 +1520,11 @@ dependencies = [ "dotenv", "futures-util", "hex", - "jsonwebtoken", + "jsonwebtoken 7.2.0", "lazy_static", "openssl", "percent-encoding", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "tokio", @@ -1573,6 +1577,7 @@ dependencies = [ "aws-sdk-s3", "aws-types", "dotenv", + "google-cloud-storage", "lazy_static", "near-lake-framework", "opentelemetry 0.19.0", @@ -2145,6 +2150,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -2428,7 +2444,7 @@ version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" dependencies = [ - "der", + "der 0.6.1", "elliptic-curve", "rfc6979", "signature 1.6.4", @@ -2479,12 +2495,12 @@ checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" dependencies = [ "base16ct", "crypto-bigint 0.4.9", - "der", + "der 0.6.1", "digest 0.10.7", "ff", "generic-array 0.14.7", "group", - "pkcs8", + "pkcs8 0.9.0", "rand_core 0.6.4", "sec1", "subtle", @@ -2937,8 +2953,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if 1.0.0", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -2964,6 +2982,81 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "google-cloud-auth" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1112c453c2e155b3e683204ffff52bcc6d6495d04b68d9e90cd24161270c5058" +dependencies = [ + "async-trait", + "base64 0.21.7", + "google-cloud-metadata", + "google-cloud-token", + "home", + "jsonwebtoken 9.3.0", + "reqwest 0.12.5", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "tracing", + "urlencoding", +] + +[[package]] +name = "google-cloud-metadata" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f945a208886a13d07636f38fb978da371d0abc3e34bad338124b9f8c135a8f" +dependencies = [ + "reqwest 0.12.5", + "thiserror", + "tokio", +] + +[[package]] +name = "google-cloud-storage" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc0c5b7469142d91bd77959e69375bede324a5def07c7f29aa0d582586cba305" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "base64 0.21.7", + "bytes", + "futures-util", + "google-cloud-auth", + "google-cloud-metadata", + "google-cloud-token", + "hex", + "once_cell", + "percent-encoding", + "pkcs8 0.10.2", + "regex", + "reqwest 0.12.5", + "reqwest-middleware", + "ring 0.17.8", + "serde", + "serde_json", + "sha2", + "thiserror", + "time", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "google-cloud-token" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c12ba8b21d128a2ce8585955246977fbce4415f680ebf9199b6f9d6d725f" +dependencies = [ + "async-trait", +] + [[package]] name = "group" version = "0.12.1" @@ -3198,6 +3291,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -3206,7 +3318,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper", + "hyper 0.14.29", "log", "rustls", "rustls-native-certs", @@ -3220,7 +3332,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -3233,10 +3345,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.29", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.4.0", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.4.0", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -3476,11 +3624,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afabcc15e437a6484fc4f12d0fd63068fe457bf93f1c148d3d9649c60b103f32" dependencies = [ "base64 0.12.3", - "pem", + "pem 0.8.3", "ring 0.16.20", "serde", "serde_json", - "simple_asn1", + "simple_asn1 0.4.1", +] + +[[package]] +name = "jsonwebtoken" +version = "9.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" +dependencies = [ + "base64 0.21.7", + "js-sys", + "pem 3.0.4", + "ring 0.17.8", + "serde", + "serde_json", + "simple_asn1 0.6.2", ] [[package]] @@ -4163,7 +4326,7 @@ dependencies = [ "rayon", "reed-solomon-erasure", "regex", - "reqwest", + "reqwest 0.11.27", "rust-s3", "serde", "serde_json", @@ -4415,7 +4578,7 @@ dependencies = [ "near-crypto 0.20.1", "near-jsonrpc-primitives 0.20.1", "near-primitives 0.20.1", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "thiserror", @@ -4433,7 +4596,7 @@ dependencies = [ "near-crypto 1.40.0", "near-jsonrpc-primitives 1.40.0", "near-primitives 1.40.0", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "thiserror", @@ -5200,8 +5363,8 @@ dependencies = [ "easy-ext", "futures", "hex", - "hyper", - "hyper-tls", + "hyper 0.14.29", + "hyper-tls 0.5.0", "indicatif", "near-async", "near-chain", @@ -5232,7 +5395,7 @@ dependencies = [ "rand 0.8.5", "rayon", "regex", - "reqwest", + "reqwest 0.11.27", "rlimit", "rust-s3", "serde", @@ -5966,6 +6129,25 @@ dependencies = [ "regex", ] +[[package]] +name = "pem" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" +dependencies = [ + "base64 0.22.1", + "serde", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -6057,8 +6239,18 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" dependencies = [ - "der", - "spki", + "der 0.6.1", + "spki 0.6.0", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.9", + "spki 0.7.3", ] [[package]] @@ -6780,8 +6972,8 @@ dependencies = [ "h2", "http 0.2.12", "http-body 0.4.6", - "hyper", - "hyper-tls", + "hyper 0.14.29", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -6790,11 +6982,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -6805,7 +6997,65 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.4.0", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.1.2", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tokio-native-tls", + "tokio-util 0.7.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg 0.52.0", +] + +[[package]] +name = "reqwest-middleware" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39346a33ddfe6be00cbc17a34ce996818b97b230b87229f10114693becca1268" +dependencies = [ + "anyhow", + "async-trait", + "http 1.1.0", + "reqwest 0.12.5", + "serde", + "thiserror", + "tower-service", ] [[package]] @@ -6942,7 +7192,7 @@ dependencies = [ "md5", "minidom", "percent-encoding", - "reqwest", + "reqwest 0.11.27", "serde", "serde-xml-rs", "serde_derive", @@ -7022,7 +7272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -7036,6 +7286,22 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -7191,9 +7457,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" dependencies = [ "base16ct", - "der", + "der 0.6.1", "generic-array 0.14.7", - "pkcs8", + "pkcs8 0.9.0", "subtle", "zeroize", ] @@ -7511,6 +7777,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "simple_asn1" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" +dependencies = [ + "num-bigint 0.4.6", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -7616,7 +7894,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" dependencies = [ "base64ct", - "der", + "der 0.6.1", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.9", ] [[package]] @@ -7792,6 +8080,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "sysinfo" version = "0.24.7" @@ -8234,7 +8528,7 @@ dependencies = [ "h2", "http 0.2.12", "http-body 0.4.6", - "hyper", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -8264,7 +8558,7 @@ dependencies = [ "h2", "http 0.2.12", "http-body 0.4.6", - "hyper", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -8524,9 +8818,7 @@ name = "tx-details-storage" version = "0.2.11" dependencies = [ "anyhow", - "aws-credential-types", - "aws-sdk-s3", - "aws-types", + "google-cloud-storage", ] [[package]] @@ -9539,6 +9831,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/configuration/Cargo.toml b/configuration/Cargo.toml index 20a21cb5..f1024e18 100644 --- a/configuration/Cargo.toml +++ b/configuration/Cargo.toml @@ -13,6 +13,7 @@ aws-credential-types = "1.1.4" aws-sdk-s3 = { version = "1.14.0", features = ["behavior-version-latest"] } aws-types = "1.1.4" dotenv = "0.15.0" +google-cloud-storage = "0.20.0" lazy_static = "1.4.0" regex = "1.10.2" serde = "1.0.145" diff --git a/configuration/src/configs/tx_details_storage.rs b/configuration/src/configs/tx_details_storage.rs index 7d58d9fd..94f033c1 100644 --- a/configuration/src/configs/tx_details_storage.rs +++ b/configuration/src/configs/tx_details_storage.rs @@ -3,8 +3,6 @@ use serde_derive::Deserialize; use crate::configs::{deserialize_optional_data_or_env, required_value_or_panic}; -const DEFAULT_GCP_ENDPOINT_URL: &str = "https://storage.googleapis.com"; - #[derive(Debug, Clone)] pub struct TxDetailsStorageConfig { pub aws_access_key_id: String, @@ -15,31 +13,16 @@ pub struct TxDetailsStorageConfig { } impl TxDetailsStorageConfig { - pub async fn s3_config(&self) -> aws_sdk_s3::Config { - let credentials = aws_credential_types::Credentials::new( - &self.aws_access_key_id, - &self.aws_secret_access_key, - None, - None, - "", - ); - aws_sdk_s3::Config::builder() - .stalled_stream_protection(StalledStreamProtectionConfig::disabled()) - .credentials_provider(credentials) - .endpoint_url( - self.aws_endpoint - .clone() - .unwrap_or_else(|| DEFAULT_GCP_ENDPOINT_URL.to_string()), - ) - .region(aws_types::region::Region::new( - self.aws_default_region.clone(), - )) - .build() + pub async fn gcs_config(&self) -> google_cloud_storage::client::ClientConfig { + google_cloud_storage::client::ClientConfig::default() + .with_auth() + .await + .unwrap() } - pub async fn storage_client(&self) -> aws_sdk_s3::Client { - let s3_config = self.s3_config().await; - aws_sdk_s3::Client::from_conf(s3_config) + pub async fn storage_client(&self) -> google_cloud_storage::client::Client { + let gcs_config = self.gcs_config().await; + google_cloud_storage::client::Client::new(gcs_config) } } diff --git a/tx-details-storage/Cargo.toml b/tx-details-storage/Cargo.toml index 60351eb5..89bf52f3 100644 --- a/tx-details-storage/Cargo.toml +++ b/tx-details-storage/Cargo.toml @@ -10,7 +10,5 @@ license.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -aws-credential-types = "1.1.4" anyhow = "1.0.70" -aws-sdk-s3 = { version = "1.14.0", features = ["behavior-version-latest"] } -aws-types = "1.1.4" +google-cloud-storage = "0.20.0" diff --git a/tx-details-storage/src/lib.rs b/tx-details-storage/src/lib.rs index f7099ed5..ea9cf09a 100644 --- a/tx-details-storage/src/lib.rs +++ b/tx-details-storage/src/lib.rs @@ -1,14 +1,15 @@ -use anyhow::Context; -use aws_sdk_s3::primitives::ByteStream; +use google_cloud_storage::http::objects::download::Range; +use google_cloud_storage::http::objects::get::GetObjectRequest; +use google_cloud_storage::http::objects::upload::{Media, UploadObjectRequest, UploadType}; pub struct TxDetailsStorage { - client: aws_sdk_s3::Client, + client: google_cloud_storage::client::Client, bucket_name: String, } impl TxDetailsStorage { /// Create a new instance of the `TxDetailsStorage` struct. - pub fn new(client: aws_sdk_s3::Client, bucket_name: String) -> Self { + pub fn new(client: google_cloud_storage::client::Client, bucket_name: String) -> Self { Self { client, bucket_name, @@ -17,31 +18,30 @@ impl TxDetailsStorage { pub async fn store(&self, key: &str, data: Vec) -> anyhow::Result<()> { self.client - .put_object() - .bucket(self.bucket_name.clone()) - .key(key) - .body(ByteStream::from(data)) - .send() - .await - .map(|_| ()) - .context("Failed to store the data") + .upload_object( + &UploadObjectRequest { + bucket: self.bucket_name.to_string(), + ..Default::default() + }, + data, + &UploadType::Simple(Media::new(key.to_string())), + ) + .await?; + Ok(()) } pub async fn retrieve(&self, key: &str) -> anyhow::Result> { - let response = self + let data = self .client - .get_object() - .bucket(self.bucket_name.clone()) - .key(key) - .send() - .await - .context("Failed to retrieve the data")?; - - let body = response - .body - .collect() - .await - .context("Failed to read the body")?; - Ok(body.to_vec()) + .download_object( + &GetObjectRequest { + bucket: self.bucket_name.to_string(), + object: key.to_string(), + ..Default::default() + }, + &Range::default(), + ) + .await?; + Ok(data) } }