Merge branch 'main' into add_fsst

.github/workflows/rust.yml

@@ -167,8 +167,9 @@ jobs:
           rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
           rustup component add rustfmt
       - name: Run tests
-        run: |
-          cargo build --all-features
+        # Check all benches, even though we aren't going to run them.
+        run: |  
+          cargo build --tests --benches --all-features --workspace
           cargo test --all-features
   windows-build:
     runs-on: windows-latest
@@ -190,6 +191,7 @@ jobs:
           Add-Content $env:GITHUB_PATH "C:\protoc\bin"
         shell: powershell
       - name: Run tests
+        # Check all benches, even though we aren't going to run them.
         run: |
-          cargo build
+          cargo build --tests --benches --all-features --workspace
           cargo test

Cargo.toml

@@ -21,7 +21,7 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
-version = "0.14.0"
+version = "0.14.1"
 edition = "2021"
 authors = ["Lance Devs <[email protected]>"]
 license = "Apache-2.0"
@@ -44,20 +44,20 @@ categories = [
 rust-version = "1.78"
 
 [workspace.dependencies]
-lance = { version = "=0.14.0", path = "./rust/lance" }
-lance-arrow = { version = "=0.14.0", path = "./rust/lance-arrow" }
-lance-core = { version = "=0.14.0", path = "./rust/lance-core" }
-lance-datafusion = { version = "=0.14.0", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=0.14.0", path = "./rust/lance-datagen" }
-lance-encoding = { version = "=0.14.0", path = "./rust/lance-encoding" }
-lance-encoding-datafusion = { version = "=0.14.0", path = "./rust/lance-encoding-datafusion" }
-lance-file = { version = "=0.14.0", path = "./rust/lance-file" }
-lance-index = { version = "=0.14.0", path = "./rust/lance-index" }
-lance-io = { version = "=0.14.0", path = "./rust/lance-io" }
-lance-linalg = { version = "=0.14.0", path = "./rust/lance-linalg" }
-lance-table = { version = "=0.14.0", path = "./rust/lance-table" }
-lance-test-macros = { version = "=0.14.0", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=0.14.0", path = "./rust/lance-testing" }
+lance = { version = "=0.14.1", path = "./rust/lance" }
+lance-arrow = { version = "=0.14.1", path = "./rust/lance-arrow" }
+lance-core = { version = "=0.14.1", path = "./rust/lance-core" }
+lance-datafusion = { version = "=0.14.1", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=0.14.1", path = "./rust/lance-datagen" }
+lance-encoding = { version = "=0.14.1", path = "./rust/lance-encoding" }
+lance-encoding-datafusion = { version = "=0.14.1", path = "./rust/lance-encoding-datafusion" }
+lance-file = { version = "=0.14.1", path = "./rust/lance-file" }
+lance-index = { version = "=0.14.1", path = "./rust/lance-index" }
+lance-io = { version = "=0.14.1", path = "./rust/lance-io" }
+lance-linalg = { version = "=0.14.1", path = "./rust/lance-linalg" }
+lance-table = { version = "=0.14.1", path = "./rust/lance-table" }
+lance-test-macros = { version = "=0.14.1", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=0.14.1", path = "./rust/lance-testing" }
 fsst = { version = "=0.1.0", path = "./rust/lance-encoding/compression-algo/fsst" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
@@ -112,6 +112,7 @@ deepsize = "0.2.0"
 either = "1.0"
 futures = "0.3"
 http = "0.2.9"
+hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }
 itertools = "0.12"
 lazy_static = "1"
 log = "0.4"
@@ -138,6 +139,7 @@ serde = { version = "^1" }
 serde_json = { version = "1" }
 shellexpand = "3.0"
 snafu = "0.7.5"
+tantivy = "0.22.0"
 tempfile = "3"
 test-log = { version = "0.2.15" }
 tokio = { version = "1.23", features = [

java/pom.xml

@@ -10,6 +10,27 @@
     <packaging>pom</packaging>
 
     <name>Lance Parent</name>
+    <description>Lance Format Java API</description>
+    <url>http://lancedb.com/</url>
+
+    <developers>
+        <developer>
+            <name>Lance DB Dev Group</name>
+            <email>[email protected]</email>
+        </developer>
+    </developers>
+    <licenses>
+        <license>
+            <name>The Apache Software License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        </license>
+    </licenses>
+
+    <scm>
+      <developerConnection>scm:git:[email protected]:lancedb/lance.git</developerConnection>
+      <tag>HEAD</tag>
+      <url>scm:git:[email protected]:lancedb/lance.git</url>
+    </scm>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -193,6 +214,16 @@
             <id>deploy-to-ossrh</id>
             <build>
                 <plugins>
+                    <plugin>
+                        <groupId>org.sonatype.central</groupId>
+                        <artifactId>central-publishing-maven-plugin</artifactId>
+                        <version>0.4.0</version>
+                        <extensions>true</extensions>
+                        <configuration>
+                            <publishingServerId>ossrh</publishingServerId>
+                            <tokenAuth>true</tokenAuth>
+                        </configuration>
+                    </plugin>
                     <plugin>
                         <groupId>org.sonatype.plugins</groupId>
                         <artifactId>nexus-staging-maven-plugin</artifactId>
@@ -222,4 +253,4 @@
             </build>
         </profile>
     </profiles>
-</project>
+</project>

protos/encodings.proto

@@ -191,6 +191,13 @@ message Fsst {
   bytes symbol_table = 2;
 }
 
+// An array encoding for dictionary-encoded fields
+message Dictionary {
+  ArrayEncoding indices = 1;
+  ArrayEncoding items = 2;
+  uint32 num_dictionary_items = 3;
+}
+
 // Encodings that decode into an Arrow array
 message ArrayEncoding {
     oneof array_encoding {
@@ -201,6 +208,7 @@ message ArrayEncoding {
         SimpleStruct struct = 5;
         Binary binary = 6;
         Fsst fsst = 7;
+        Dictionary dictionary = 7;
     }
 }
 

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pylance"
-version = "0.14.0"
+version = "0.14.1"
 edition = "2021"
 authors = ["Lance Devs <[email protected]>"]
 rust-version = "1.65"

python/python/benchmarks/test_index.py

@@ -6,6 +6,7 @@
 import pyarrow as pa
 import pyarrow.compute as pc
 import pytest
+from lance.indices import IndicesBuilder
 
 N_DIMS = 512
 
@@ -95,3 +96,16 @@ def test_optimize_index(
         lance.write_dataset(small_table, test_large_dataset.uri, mode="append")
 
     benchmark(test_large_dataset.optimize.optimize_indices)
+
+
+@pytest.mark.benchmark(group="optimize_index")
+@pytest.mark.parametrize("num_partitions", [100, 300])
+def test_train_ivf(test_large_dataset, benchmark, num_partitions):
+    builder = IndicesBuilder(test_large_dataset)
+    benchmark.pedantic(
+        builder.train_ivf,
+        args=["vector"],
+        kwargs={"num_partitions": num_partitions},
+        iterations=1,
+        rounds=1,
+    )