CLI arrow 53 (#251)

mwlon · Nov 15, 2024 · fabff73 · fabff73
1 parent f48e7b8
commit fabff73
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 57 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/pco_cli/Cargo.toml b/pco_cli/Cargo.toml
@@ -17,14 +17,14 @@ path = "src/main.rs"
 
 [dependencies]
 anyhow = "1.0.71"
-arrow = { version = "49.0.0", features = ["csv"], default-features = false }
+arrow = { version = "53.0.0", features = ["csv"], default-features = false }
 better_io = { version = "0.1.0", path = "../better_io" }
 blosc-src = { version = "0.3", features = ["zstd"], optional = true }
 bytes = "1.4.0"
 clap = { version = "4.5.3", features = ["derive"] }
 half = { version = "2.1.0", features = ["std"] }
 indicatif = "0.17.8"
-parquet = { version = "49.0.0", features = [
+parquet = { version = "53.0.0", features = [
   "arrow",
   "base64",
   "snap",

diff --git a/pco_cli/src/bench/codecs/parquet.rs b/pco_cli/src/bench/codecs/parquet.rs
@@ -115,18 +115,13 @@ impl CodecInternal for ParquetConfig {
     }
 
     let mut res = Vec::with_capacity(n as usize);
-    unsafe {
-      res.set_len(n as usize);
-    }
-    let mut start = 0;
     for i in 0..parquet_meta.num_row_groups() {
       let row_group_reader = reader.get_row_group(i).unwrap();
       let mut col_reader =
         get_typed_column_reader::<T::Parquet>(row_group_reader.get_column_reader(0).unwrap());
-      let (n_records_read, _, _) = col_reader
-        .read_records(usize::MAX, None, None, &mut res[start..])
+      col_reader
+        .read_records(usize::MAX, None, None, &mut res)
         .unwrap();
-      start += n_records_read
     }
 
     T::parquet_to_nums(res)