Handle prefixes for AmazonS3 and GoogleCloudStorage object stores (#704)

object_store_factory/src/lib.rs

@@ -131,14 +131,20 @@ pub async fn build_object_store_from_opts(
             aws::add_amazon_s3_specific_options(url, &mut s3_options).await;
             aws::add_amazon_s3_environment_variables(&mut s3_options);
 
-            let (store, _) = parse_url_opts(url, s3_options)?;
+            let (mut store, _) = parse_url_opts(url, s3_options)?;
+            if !url.path().is_empty() {
+                store = Box::new(PrefixStore::new(store, url.path()));
+            }
             Ok(store)
         }
         ObjectStoreScheme::GoogleCloudStorage => {
             let mut gcs_options = google::map_options_into_google_config_keys(options)?;
             google::add_google_cloud_storage_environment_variables(&mut gcs_options);
 
-            let (store, _) = parse_url_opts(url, gcs_options)?;
+            let (mut store, _) = parse_url_opts(url, gcs_options)?;
+            if !url.path().is_empty() {
+                store = Box::new(PrefixStore::new(store, url.path()));
+            }
             Ok(store)
         }
         _ => {

src/object_store/factory.rs

@@ -9,7 +9,7 @@ use deltalake::{
     storage::{FactoryRegistry, ObjectStoreRef, StorageOptions},
     DeltaResult, DeltaTableError, Path,
 };
-use object_store::{prefix::PrefixStore, ObjectStore};
+use object_store::ObjectStore;
 use object_store_factory;
 use url::Url;
 
@@ -103,6 +103,16 @@ impl ObjectStoreFactory {
         options: HashMap<String, String>,
         table_path: String,
     ) -> Result<Arc<dyn LogStore>, object_store::Error> {
+        // This is the least surprising way to extend the path, and make the url point to the table
+        // root: https://github.com/servo/rust-url/issues/333
+        url.path_segments_mut()
+            .map_err(|_| object_store::Error::Generic {
+                store: "object_store_factory",
+                source: "The provided URL is a cannot-be-a-base URL".into(),
+            })?
+            .pop_if_empty()
+            .extend(table_path.split("/"));
+
         let store = {
             let used_options = options.clone();
             let key = StoreCacheKey {
@@ -134,35 +144,7 @@ impl ObjectStoreFactory {
             }
         };
 
-        // Any path provided in the url has not been included in the object store root, so it
-        // needs to become a part of the prefix, alongside with the actual table name (unless
-        // it's a file/memory store)
-        let prefix = if !url.path().is_empty()
-            && url.scheme() != "file"
-            && url.scheme() != "memory"
-        {
-            format!("{}/{table_path}", url.path())
-        } else {
-            table_path.clone()
-        };
-
-        // This is the least surprising way to extend the path, and make the url point to the table
-        // root: https://github.com/servo/rust-url/issues/333
-        url.path_segments_mut()
-            .map_err(|_| object_store::Error::Generic {
-                store: "object_store_factory",
-                source: "The provided URL is a cannot-be-a-base URL".into(),
-            })?
-            .push(&table_path);
-
-        let prefixed_store: PrefixStore<Arc<dyn ObjectStore>> =
-            PrefixStore::new(store, prefix);
-
-        Ok(default_logstore(
-            Arc::from(prefixed_store),
-            &url,
-            &Default::default(),
-        ))
+        Ok(default_logstore(store, &url, &Default::default()))
     }
 
     pub fn get_default_log_store(&self, path: &str) -> Option<Arc<dyn LogStore>> {