Add support for public S3 buckets

If the access_key/secret_key aren't set, skip signing the requests and
treat the bucket as public.

docker-compose.yml

@@ -20,7 +20,10 @@ services:
       /usr/bin/mc rm -r --force test-minio/seafowl-test-bucket;  /usr/bin/mc mb
       test-minio/seafowl-test-bucket; /usr/bin/mc cp test-data/table_with_ns_column.parquet
       test-minio/seafowl-test-bucket/table_with_ns_column.parquet; /usr/bin/mc anonymous set public
-      test-minio/seafowl-test-bucket/table_with_ns_column.parquet; exit 0; "
+      test-minio/seafowl-test-bucket/table_with_ns_column.parquet; 
+
+      /usr/bin/mc mb test-minio/seafowl-test-bucket-public; /usr/bin/mc anonymous set public
+      test-minio/seafowl-test-bucket-public; exit 0; "
 
   fake-gcs:
     image: tustvold/fake-gcs-server

src/config/context.rs

@@ -102,8 +102,6 @@ pub fn build_object_store(
             ..
         }) => {
             let mut builder = AmazonS3Builder::new()
-                .with_access_key_id(access_key_id)
-                .with_secret_access_key(secret_access_key)
                 .with_region(region.clone().unwrap_or_default())
                 .with_bucket_name(bucket)
                 .with_allow_http(true);
@@ -112,6 +110,16 @@ pub fn build_object_store(
                 builder = builder.with_endpoint(endpoint);
             }
 
+            if let (Some(access_key_id), Some(secret_access_key)) =
+                (&access_key_id, &secret_access_key)
+            {
+                builder = builder
+                    .with_access_key_id(access_key_id)
+                    .with_secret_access_key(secret_access_key)
+            } else {
+                builder = builder.with_skip_signature(true)
+            }
+
             let store = builder.build()?;
 
             if let Some(props) = cache_properties {

src/config/schema.rs

@@ -126,8 +126,8 @@ pub struct InMemory {}
 #[derive(Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct S3 {
     pub region: Option<String>,
-    pub access_key_id: String,
-    pub secret_access_key: String,
+    pub access_key_id: Option<String>,
+    pub secret_access_key: Option<String>,
     pub endpoint: Option<String>,
     pub bucket: String,
     pub prefix: Option<String>,
@@ -141,18 +141,8 @@ impl S3 {
     ) -> Result<Self, ConfigError> {
         Ok(S3 {
             region: map.get("region").cloned(),
-            access_key_id: map
-                .remove("access_key_id")
-                .ok_or(ConfigError::Message(
-                    "'access_key_id' not found in provided options".to_string(),
-                ))?
-                .clone(),
-            secret_access_key: map
-                .remove("secret_access_key")
-                .ok_or(ConfigError::Message(
-                    "'secret_access_key' not found in provided options".to_string(),
-                ))?
-                .clone(),
+            access_key_id: map.remove("access_key_id"),
+            secret_access_key: map.remove("secret_access_key"),
             endpoint: map.remove("endpoint"),
             bucket,
             prefix: None,
@@ -531,6 +521,17 @@ secret_access_key = "ABC..."
 endpoint = "https://s3.amazonaws.com:9000"
 bucket = "seafowl"
 
+[catalog]
+type = "postgres"
+dsn = "postgresql://user:pass@localhost:5432/somedb"
+"#;
+
+    const TEST_CONFIG_S3_PUBLIC: &str = r#"
+[object_store]
+type = "s3"
+endpoint = "https://s3.amazonaws.com:9000"
+bucket = "seafowl"
+
 [catalog]
 type = "postgres"
 dsn = "postgresql://user:pass@localhost:5432/somedb"
@@ -629,8 +630,8 @@ cache_control = "private, max-age=86400"
             config.object_store,
             ObjectStore::S3(S3 {
                 region: None,
-                access_key_id: "AKI...".to_string(),
-                secret_access_key: "ABC...".to_string(),
+                access_key_id: Some("AKI...".to_string()),
+                secret_access_key: Some("ABC...".to_string()),
                 endpoint: Some("https://s3.amazonaws.com:9000".to_string()),
                 bucket: "seafowl".to_string(),
                 prefix: None,
@@ -639,6 +640,24 @@ cache_control = "private, max-age=86400"
         );
     }
 
+    #[test]
+    fn test_parse_public_config_with_s3() {
+        let config = load_config_from_string(TEST_CONFIG_S3_PUBLIC, false, None).unwrap();
+
+        assert_eq!(
+            config.object_store,
+            ObjectStore::S3(S3 {
+                region: None,
+                access_key_id: None,
+                secret_access_key: None,
+                endpoint: Some("https://s3.amazonaws.com:9000".to_string()),
+                bucket: "seafowl".to_string(),
+                prefix: None,
+                cache_properties: None,
+            })
+        );
+    }
+
     #[test]
     fn test_parse_config_basic() {
         let config = load_config_from_string(TEST_CONFIG_BASIC, false, None).unwrap();

src/object_store/wrapped.rs

@@ -358,8 +358,8 @@ mod tests {
     ) -> Result<()> {
         let config = ObjectStore::S3(S3 {
             region: None,
-            access_key_id: "access_key_id".to_string(),
-            secret_access_key: "secret_access_key".to_string(),
+            access_key_id: Some("access_key_id".to_string()),
+            secret_access_key: Some("secret_access_key".to_string()),
             bucket: bucket.to_string(),
             prefix: prefix.map(|p| p.to_string()),
             endpoint: endpoint.clone(),

tests/statements/ddl.rs

@@ -3,7 +3,11 @@ use crate::statements::*;
 #[rstest]
 #[tokio::test]
 async fn test_create_table(
-    #[values(ObjectStoreType::InMemory, ObjectStoreType::Gcs)]
+    #[values(
+        ObjectStoreType::InMemory,
+        ObjectStoreType::Gcs,
+        ObjectStoreType::S3Public
+    )]
     object_store_type: ObjectStoreType,
 ) {
     let (context, _) = make_context_with_pg(object_store_type).await;

tests/statements/mod.rs

@@ -50,6 +50,8 @@ enum ObjectStoreType {
     InMemory,
     // S3 object store with an optional path to the actual data folder
     S3(Option<&'static str>),
+    // Publicly-accessible S3 bucket
+    S3Public,
 }
 
 /// Make a SeafowlContext that's connected to a real PostgreSQL database
@@ -92,6 +94,16 @@ ttl = 30
             ),
             None,
         ),
+        ObjectStoreType::S3Public => (
+            r#"type = "s3"
+endpoint = "http://127.0.0.1:9000"
+bucket = "seafowl-test-bucket-public"
+[object_store.cache_properties]
+ttl = 30
+"#
+            .to_string(),
+            None,
+        ),
         ObjectStoreType::Gcs => {
             let creds_json = json!({"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": "", "private_key_id": ""});
             // gcs_base_url should match docker-compose.yml:fake-gcs-server