Split load partition and search partition into two different steps and parallelize based on different configs #8753
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rust | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - rust/** | |
| - protos/** | |
| - .github/workflows/rust.yml | |
| - Cargo.toml | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # This env var is used by Swatinem/rust-cache@v2 for the cache | |
| # key, so we set it to make sure it is always consistent. | |
| CARGO_TERM_COLOR: always | |
| # Disable full debug symbol generation to speed up CI build and keep memory down | |
| # "1" means line tables only, which is useful for panic tracebacks. | |
| RUSTFLAGS: "-C debuginfo=1" | |
| RUST_BACKTRACE: "1" | |
| # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html | |
| # CI builds are faster with incremental disabled. | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_BUILD_JOBS: "1" | |
| jobs: | |
| format: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check formatting | |
| run: cargo fmt -- --check | |
| clippy: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y protobuf-compiler libssl-dev | |
| - name: Run clippy | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo clippy --version | |
| cargo clippy --locked --features ${ALL_FEATURES} --tests --benches -- -D warnings | |
| cargo-deny: | |
| name: Check Rust dependencies (cargo-deny) | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: EmbarkStudios/cargo-deny-action@v2 | |
| with: | |
| log-level: warn | |
| command: check | |
| linux-build: | |
| runs-on: "ubuntu-24.04" | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| toolchain: | |
| - stable | |
| - nightly | |
| env: | |
| # Need up-to-date compilers for kernels | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # pin the toolchain version to avoid surprises | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: ${{ matrix.toolchain }} | |
| - uses: rui314/setup-mold@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y protobuf-compiler libssl-dev | |
| rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} | |
| - name: Start DynamoDB local for tests | |
| run: | | |
| docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Run tests | |
| if: ${{ matrix.toolchain == 'stable' }} | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo llvm-cov --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES} | |
| - name: Build tests (nightly) | |
| if: ${{ matrix.toolchain != 'stable' }} | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo test --locked --features ${ALL_FEATURES} --workspace --no-run | |
| - name: Run tests (nightly) | |
| if: ${{ matrix.toolchain != 'stable' }} | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo test --features ${ALL_FEATURES} --workspace | |
| - name: Upload coverage to Codecov | |
| if: ${{ matrix.toolchain == 'stable' }} | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| codecov_yml_path: codecov.yml | |
| files: coverage.codecov | |
| flags: unittests | |
| fail_ci_if_error: false | |
| linux-arm: | |
| runs-on: ubuntu-2404-4x-arm64 | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: "stable" | |
| - uses: rui314/setup-mold@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt -y -qq update | |
| sudo apt install -y protobuf-compiler libssl-dev pkg-config | |
| - name: Build tests | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo test --locked --features ${ALL_FEATURES} --no-run | |
| - name: Start DynamoDB local for tests | |
| run: | | |
| docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local | |
| - name: Run tests | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo test --locked --features ${ALL_FEATURES} | |
| build-no-lock: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| env: | |
| # Need up-to-date compilers for kernels | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Remote cargo.lock to force a fresh build | |
| - name: Remove Cargo.lock | |
| run: rm -f Cargo.lock | |
| - uses: rui314/setup-mold@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y protobuf-compiler libssl-dev | |
| - name: Build all | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo build --benches --features ${ALL_FEATURES} --tests | |
| mac-build: | |
| runs-on: "macos-14" | |
| timeout-minutes: 45 | |
| strategy: | |
| matrix: | |
| toolchain: | |
| - stable | |
| - nightly | |
| defaults: | |
| run: | |
| working-directory: ./rust/lance | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Select new xcode | |
| # Default XCode right now is 15.0.1, which contains a bug that causes | |
| # backtraces to not show properly. See: | |
| # https://github.com/rust-lang/rust/issues/113783 | |
| run: sudo xcode-select -s /Applications/Xcode_15.4.app | |
| - name: Install dependencies | |
| run: brew install protobuf | |
| - name: Set up Rust | |
| run: | | |
| rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} | |
| - name: Build tests | |
| run: | | |
| cargo test --locked --features fp16kernels,cli,tensorflow,dynamodb,substrait --no-run | |
| - name: Run tests | |
| run: | | |
| cargo test --features fp16kernels,cli,tensorflow,dynamodb,substrait | |
| - name: Check benchmarks | |
| run: | | |
| cargo check --benches --features fp16kernels,cli,tensorflow,dynamodb,substrait | |
| windows-build: | |
| runs-on: windows-latest | |
| defaults: | |
| run: | |
| working-directory: rust/lance | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install Protoc v21.12 | |
| working-directory: C:\ | |
| run: | | |
| New-Item -Path 'C:\protoc' -ItemType Directory | |
| Set-Location C:\protoc | |
| Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip | |
| 7z x protoc.zip | |
| Add-Content $env:GITHUB_PATH "C:\protoc\bin" | |
| shell: powershell | |
| - name: Build tests | |
| run: cargo test --locked --no-run | |
| - name: Run tests | |
| run: cargo test | |
| - name: Check benchmarks | |
| run: cargo check --benches | |
| msrv: | |
| # Check the minimum supported Rust version | |
| name: MSRV Check - Rust v${{ matrix.msrv }} | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| msrv: ["1.80.1"] # This should match up with rust-version in Cargo.toml | |
| env: | |
| # Need up-to-date compilers for kernels | |
| CC: clang | |
| CXX: clang++ | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y protobuf-compiler libssl-dev | |
| - name: Install ${{ matrix.msrv }} | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ matrix.msrv }} | |
| - name: cargo +${{ matrix.msrv }} check | |
| run: | | |
| ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` | |
| cargo check --workspace --tests --benches --features ${ALL_FEATURES} |