Merge branch 'main' into datafusion-add-map

ibis-project · Dec 17, 2024 · febd745 · febd745
2 parents bf11489 + 945999b
commit febd745
Show file tree

Hide file tree

Showing 88 changed files with 2,178 additions and 2,298 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,4 +1,43 @@
-FROM mcr.microsoft.com/vscode/devcontainers/python:3.11
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.13
+COPY --from=ghcr.io/astral-sh/uv:0.5.9 /uv /uvx /bin/
+ARG USERNAME=vscode
 
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends libgdal-dev
+  apt-get install -y --no-install-recommends libgdal-dev && \
+  rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m pip install pipx --no-cache-dir
+RUN python3 -m pipx ensurepath
+RUN pipx install rust-just
+
+WORKDIR /app
+
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+COPY . /app
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv venv && uv sync --group dev --group tests \
+        --extra duckdb --extra clickhouse  --extra examples --extra geospatial
+
+ENV VENV_DIR=.venv
+RUN chown -R $USERNAME $VENV_DIR && chmod -R 755 $VENV_DIR
+
+ENV IBIS_PROJECT=.
+RUN chown -R $USERNAME $IBIS_PROJECT
+
+# Place executables in the environment at the front of the path
+ENV PATH="/app/.venv/bin:$PATH"
+
+SHELL ["/bin/bash", "-c", "source .venv/bin/activate"]
+
+ENTRYPOINT []
+
+USER $USERNAME
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,6 +1,9 @@
 {
-  "build": { "dockerfile": "Dockerfile" },
-  "postCreateCommand": ".devcontainer/postCreate.sh",
+  "build": { "dockerfile": "Dockerfile", "context": ".." },
+  "containerUser": "vscode",
+  "remoteUser": "vscode",
+  "postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
+  "workspaceFolder": "/app",
   "customizations": {
     "codespaces": {
       "openFiles": ["docs/tutorials/getting_started.qmd"]

diff --git a/.github/workflows/check-generated-files.yml b/.github/workflows/check-generated-files.yml
@@ -39,7 +39,7 @@ jobs:
         uses: actions/setup-python@v5
         id: install_python
         with:
-          python-version: "3.12"
+          python-version: "3.13"
           cache: pip
           cache-dependency-path: requirements-dev.txt
 

diff --git a/.github/workflows/ibis-backends-cloud.yml b/.github/workflows/ibis-backends-cloud.yml
@@ -41,7 +41,7 @@ jobs:
       matrix:
         python-version:
           - "3.10"
-          - "3.12"
+          - "3.13"
         backend:
           - name: snowflake
             title: Snowflake
@@ -58,7 +58,7 @@ jobs:
               title: BigQuery
               extras:
                 - --extra bigquery
-          - python-version: "3.12"
+          - python-version: "3.13"
             backend:
               name: bigquery
               title: BigQuery

diff --git a/.github/workflows/ibis-backends.yml b/.github/workflows/ibis-backends.yml
@@ -53,7 +53,7 @@ jobs:
           - windows-latest
         python-version:
           - "3.10"
-          - "3.12"
+          - "3.13"
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -95,7 +95,7 @@ jobs:
           - windows-latest
         python-version:
           - "3.10"
-          - "3.12"
+          - "3.13"
         backend:
           - name: duckdb
             title: DuckDB
@@ -258,7 +258,42 @@ jobs:
               sys-deps:
                 - cmake
                 - ninja-build
+          # pytorch wheel doesn't exist for windows + python 3.13, so test
+          # against 3.12 until that's shipped
+          - os: windows-latest
+            python-version: "3.12"
+            backend:
+              name: duckdb
+              title: DuckDB
+              serial: true
+              extras:
+                - --extra duckdb
+                - --extra deltalake
+                - --extra geospatial
+                - --extra examples
+                - --extra decompiler
+                - --extra polars
+            additional_deps:
+              - torch
+          # also test duckdb with python 3.13 on windows, *without* pytorch
+          - os: windows-latest
+            python-version: "3.13"
+            backend:
+              name: duckdb
+              title: DuckDB
+              serial: true
+              extras:
+                - --extra duckdb
+                - --extra deltalake
+                - --extra geospatial
+                - --extra examples
+                - --extra decompiler
+                - --extra polars
         exclude:
+          - os: windows-latest
+            python-version: "3.13"
+            backend:
+              name: duckdb
           - os: windows-latest
             backend:
               name: mysql
@@ -317,7 +352,7 @@ jobs:
                 - libgeos-dev
           # TODO(deepyaman): Test whether this works upon releasing https://github.com/cloudera/impyla/commit/bf1f94c3c4106ded6267d2485c1e939775a6a87f
           - os: ubuntu-latest
-            python-version: "3.12"
+            python-version: "3.13"
             backend:
               name: impala
               title: Impala
@@ -383,7 +418,7 @@ jobs:
               services:
                 - oracle
           - os: ubuntu-latest
-            python-version: "3.12"
+            python-version: "3.13"
             backend:
               name: flink
               title: Flink
@@ -526,7 +561,7 @@ jobs:
             tag: local
             deps:
               - delta-spark==3.2.1
-          - python-version: "3.12"
+          - python-version: "3.13"
             pyspark-minor-version: "3.5"
             tag: local
             deps:

diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml
@@ -49,6 +49,7 @@ jobs:
           - "3.10"
           - "3.11"
           - "3.12"
+          - "3.13"
         pyarrow:
           - true
           - false
@@ -118,7 +119,7 @@ jobs:
         uses: actions/setup-python@v5
         id: install_python
         with:
-          python-version: "3.12"
+          python-version: "3.13"
 
       - name: install uv
         uses: astral-sh/[email protected]

diff --git a/.github/workflows/nix-skip-helper.yml b/.github/workflows/nix-skip-helper.yml
@@ -32,16 +32,16 @@ jobs:
       matrix:
         os:
           - ubuntu-latest # x86_64-linux
+          - ubuntu-arm64-24.04 # aarch64-linux
+          - macos-13 # x86_64-darwin
+          - macos-14 # aarch64-darwin
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
+          - "3.13"
         include:
-          - os: ubuntu-arm64-24.04
-            python-version: "3.12"
-          - os: macos-14
-            python-version:
-              - "3.10"
-              - "3.12"
+          - os: ubuntu-latest
+            python-version: "3.10"
+          - os: ubuntu-latest
+            python-version: "3.11"
     steps:
       - run: echo "No build required"
diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
@@ -34,15 +34,17 @@ jobs:
       matrix:
         os:
           - ubuntu-latest # x86_64-linux
+          - ubuntu-arm64-24.04 # aarch64-linux
+          - macos-13 # x86_64-darwin
+          - macos-14 # aarch64-darwin
         python-version:
-          - "3.10"
-          - "3.11"
           - "3.12"
+          - "3.13"
         include:
-          - os: ubuntu-arm64-24.04 # aarch64-linux
-            python-version: "3.12"
-          - os: macos-14 # aarch64-darwin
-            python-version: "3.12"
+          - os: ubuntu-latest
+            python-version: "3.10"
+          - os: ubuntu-latest
+            python-version: "3.11"
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -76,8 +78,6 @@ jobs:
           nix build ".#ibis${version//./}.passthru.tests.pytest" --fallback --keep-going --print-build-logs
 
       - name: nix build devShell
-        # TODO: dev shell doesn't yet build on macos-14 (aarch64-darwin)
-        continue-on-error: ${{ matrix.os == 'macos-14' }}
         run: |
           set -euo pipefail
 

diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -27,7 +27,7 @@ jobs:
       - name: install python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: "3.13"
 
       - name: install uv
         uses: astral-sh/[email protected]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -105,6 +105,7 @@ repos:
         name: shellcheck
         language: system
         entry: shellcheck
+        args: ["--shell", "bash"]
         types_or:
           - sh
           - shell

diff --git a/compose.yaml b/compose.yaml
@@ -99,7 +99,7 @@ services:
       - trino
 
   minio:
-    image: bitnami/minio:2024.11.7
+    image: bitnami/minio:2024.12.13
     environment:
       MINIO_ROOT_USER: accesskey
       MINIO_ROOT_PASSWORD: secretkey
@@ -161,7 +161,7 @@ services:
       test:
         - CMD-SHELL
         - trino --output-format null --execute 'show schemas in hive; show schemas in memory'
-    image: trinodb/trino:466
+    image: trinodb/trino:467
     ports:
       - 8080:8080
     networks:
@@ -346,7 +346,7 @@ services:
       - druid
 
   oracle:
-    image: gvenzl/oracle-free:23.5-slim
+    image: gvenzl/oracle-free:23.6-slim
     environment:
       ORACLE_PASSWORD: ibis
       ORACLE_DATABASE: IBIS_TESTING
@@ -555,7 +555,7 @@ services:
       - impala
 
   risingwave:
-    image: ghcr.io/risingwavelabs/risingwave:v2.0.4
+    image: ghcr.io/risingwavelabs/risingwave:v2.1.0
     command: "standalone --meta-opts=\" \
       --advertise-addr 0.0.0.0:5690 \
       --backend mem \

diff --git a/docs/backend_table_hiearchy.qmd → docs/backend_table_hierarchy.qmd b/docs/backend_table_hiearchy.qmd → docs/backend_table_hierarchy.qmd