feat: Adds support for scanning parquet from GCP

pola-rs · Apr 23, 2024 · 12bc03f · 12bc03f
1 parent 636daa0
commit 12bc03f
Show file tree

Hide file tree

Showing 20 changed files with 655 additions and 19 deletions.
diff --git a/R/extendr-wrappers.R b/R/extendr-wrappers.R
@@ -102,7 +102,7 @@ import_arrow_ipc <- function(path, n_rows, cache, rechunk, row_name, row_index,
 
 new_from_ndjson <- function(path, infer_schema_length, batch_size, n_rows, low_memory, rechunk, row_index_name, row_index_offset, ignore_errors) .Call(wrap__new_from_ndjson, path, infer_schema_length, batch_size, n_rows, low_memory, rechunk, row_index_name, row_index_offset, ignore_errors)
 
-new_from_parquet <- function(path, n_rows, cache, parallel, rechunk, row_name, row_index, use_statistics, low_memory, hive_partitioning) .Call(wrap__new_from_parquet, path, n_rows, cache, parallel, rechunk, row_name, row_index, use_statistics, low_memory, hive_partitioning)
+new_from_parquet <- function(path, n_rows, cache, parallel, rechunk, row_name, row_index, cloud_options, use_statistics, low_memory, hive_partitioning) .Call(wrap__new_from_parquet, path, n_rows, cache, parallel, rechunk, row_name, row_index, cloud_options, use_statistics, low_memory, hive_partitioning)
 
 test_rpolarserr <- function() .Call(wrap__test_rpolarserr)
 

diff --git a/R/io_parquet.R b/R/io_parquet.R
@@ -46,6 +46,7 @@ pl_scan_parquet = function(
     hive_partitioning = TRUE,
     rechunk = FALSE,
     low_memory = FALSE,
+    cloud_options = NULL,
     use_statistics = TRUE,
     cache = TRUE) {
   new_from_parquet(
@@ -58,7 +59,8 @@ pl_scan_parquet = function(
     row_index = row_index_offset,
     low_memory = low_memory,
     use_statistics = use_statistics,
-    hive_partitioning = hive_partitioning
+    hive_partitioning = hive_partitioning,
+    cloud_options = cloud_options
   ) |>
     unwrap("in pl$scan_parquet():")
 }

diff --git a/man/DataFrame_class.Rd b/man/DataFrame_class.Rd
diff --git a/man/DataFrame_to_data_frame.Rd b/man/DataFrame_to_data_frame.Rd
diff --git a/man/DataFrame_to_list.Rd b/man/DataFrame_to_list.Rd
diff --git a/man/IO_scan_parquet.Rd b/man/IO_scan_parquet.Rd
diff --git a/man/LazyFrame_class.Rd b/man/LazyFrame_class.Rd
diff --git a/man/S3_as.data.frame.Rd b/man/S3_as.data.frame.Rd
diff --git a/man/S3_as.vector.Rd b/man/S3_as.vector.Rd
diff --git a/man/S3_as_arrow_table.Rd b/man/S3_as_arrow_table.Rd
diff --git a/man/S3_as_nanoarrow_array_stream.Rd b/man/S3_as_nanoarrow_array_stream.Rd
diff --git a/man/S3_as_record_batch_reader.Rd b/man/S3_as_record_batch_reader.Rd
diff --git a/man/S3_infer_nanoarrow_schema.Rd b/man/S3_infer_nanoarrow_schema.Rd
diff --git a/man/S3_knit_print.Rd b/man/S3_knit_print.Rd
diff --git a/man/Series_class.Rd b/man/Series_class.Rd
diff --git a/man/Series_to_r.Rd b/man/Series_to_r.Rd