Disable auto pick-up of S3 config from file or env when explicit opti…

…ons are passed in
splitgraph · Dec 16, 2024 · c0ca6df · c0ca6df
1 parent 361765a
commit c0ca6df
Showing 1 changed file with 20 additions and 13 deletions.
diff --git a/object_store_factory/src/utils.rs b/object_store_factory/src/utils.rs
@@ -1,6 +1,6 @@
 use iceberg::io::{
-    S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_DISABLE_EC2_METADATA, S3_ENDPOINT,
-    S3_REGION, S3_SECRET_ACCESS_KEY,
+    S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_DISABLE_CONFIG_LOAD,
+    S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY,
 };
 use object_store::aws::AmazonS3ConfigKey;
 use std::collections::HashMap;
@@ -18,18 +18,25 @@ pub fn object_store_opts_to_file_io_props(
 
     for (key, val) in opts.iter() {
         let key = match AmazonS3ConfigKey::from_str(key) {
-            Ok(AmazonS3ConfigKey::AccessKeyId) => S3_ACCESS_KEY_ID,
-            Ok(AmazonS3ConfigKey::SecretAccessKey) => S3_SECRET_ACCESS_KEY,
-            Ok(AmazonS3ConfigKey::SkipSignature)
-                if ["true", "t", "1"].contains(&val.to_lowercase().as_str()) =>
-            {
-                // We need two options on the opendal client in this case
-                props.insert(S3_ALLOW_ANONYMOUS.to_string(), val.clone());
-                props.insert(S3_DISABLE_EC2_METADATA.to_string(), val.clone());
-                continue;
+            Ok(s3_key) => {
+                // If any S3 key is detected skip picking up config from config file or env vars
+                props.insert(S3_DISABLE_CONFIG_LOAD.to_string(), val.clone());
+                match s3_key {
+                    AmazonS3ConfigKey::AccessKeyId => S3_ACCESS_KEY_ID,
+                    AmazonS3ConfigKey::SecretAccessKey => S3_SECRET_ACCESS_KEY,
+                    AmazonS3ConfigKey::SkipSignature
+                        if ["true", "t", "1"].contains(&val.to_lowercase().as_str()) =>
+                    {
+                        // We need two options on the opendal client in this case
+                        props.insert(S3_ALLOW_ANONYMOUS.to_string(), val.clone());
+                        props.insert(S3_DISABLE_EC2_METADATA.to_string(), val.clone());
+                        continue;
+                    }
+                    AmazonS3ConfigKey::Region => S3_REGION,
+                    AmazonS3ConfigKey::Endpoint => S3_ENDPOINT,
+                    _ => key, // for now just propagate any non-matched keys
+                }
             }
-            Ok(AmazonS3ConfigKey::Region) => S3_REGION,
-            Ok(AmazonS3ConfigKey::Endpoint) => S3_ENDPOINT,
             _ => key, // for now just propagate any non-matched keys
         };