[python/ci] Remote-storage CI YAML

.github/workflows/libtiledb-ci.yml

@@ -1,15 +1,18 @@
 name: libTileDB-SOMA CodeCov
 
 on:
-  pull_request:
-    paths-ignore:
-      - "apis/python/**"
-      - "apis/r/**"
-      - ".pre-commit-config.yaml"
-  push:
-    branches:
-      - main
-      - 'release-*'
+# XXX TEMP
+#  pull_request:
+#    paths-ignore:
+#      - "apis/python/**"
+#      - "apis/r/**"
+#      - ".pre-commit-config.yaml"
+#      # XXX TEMP
+#      - ".github/workflows/python-remote-storage.yml"
+#  push:
+#    branches:
+#      - main
+#      - 'release-*'
   workflow_dispatch:
 
 jobs:

.github/workflows/libtiledbsoma-asan-ci.yml

@@ -1,15 +1,18 @@
 name: libtiledbsoma ASAN
 
 on:
-  pull_request:
-    paths-ignore:
-      - "apis/python/**"
-      - "apis/r/**"
-      - ".pre-commit-config.yaml"
-  push:
-    branches:
-      - main
-      - 'release-*'
+# XXX TEMP
+#  pull_request:
+#    paths-ignore:
+#      - "apis/python/**"
+#      - "apis/r/**"
+#      - ".pre-commit-config.yaml"
+#      # XXX TEMP
+#      - ".github/workflows/python-remote-storage.yml"
+#  push:
+#    branches:
+#      - main
+#      - 'release-*'
   workflow_dispatch:
 
 jobs:

.github/workflows/python-ci-minimal.yml

@@ -8,18 +8,19 @@ name: TileDB-SOMA Python CI (Minimal)
 # To test the full matrix on a working branch, invoke ./python-ci-full.yml from
 #   https://github.com/single-cell-data/TileDB-SOMA/actions/workflows/python-ci-full.yml
 on:
-  pull_request:
-    branches:
-      - main
-      - 'release-*'
-    paths:
-      - '**'
-      - '!**.md'
-      - '!apis/r/**'
-      - '!docs/**'
-      - '!.github/**'
-      - '.github/workflows/python-ci-minimal.yml'
-      - '.github/workflows/python-ci-single.yml'
+  # XXX TEMP
+#  pull_request:
+#    branches:
+#      - main
+#      - 'release-*'
+#    paths:
+#      - '**'
+#      - '!**.md'
+#      - '!apis/r/**'
+#      - '!docs/**'
+#      - '!.github/**'
+#      - '.github/workflows/python-ci-minimal.yml'
+#      - '.github/workflows/python-ci-single.yml'
   workflow_dispatch:
 
 jobs:

.github/workflows/python-dependency-variation.yml

@@ -1,15 +1,16 @@
 name: TileDB-SOMA Python CI with varying dependencies
 
 on:
-  push:
-    branches:
-      - main
-      - 'release-*'
-  pull_request:
-    paths-ignore:
-      - '**.md'
-      - 'apis/r/**'
-      - 'docs/**'
+# XXX TEMP
+#  push:
+#    branches:
+#      - main
+#      - 'release-*'
+#  pull_request:
+#    paths-ignore:
+#      - '**.md'
+#      - 'apis/r/**'
+#      - 'docs/**'
   workflow_dispatch:
 
 jobs:

.github/workflows/python-remote-storage.yml

@@ -0,0 +1,104 @@
+name: TileDB-SOMA Python CI (remote storage)
+
+on:
+  workflow_dispatch:
+  #
+  # Not for regular use:
+  # TEMP
+  pull_request:
+  #
+  # TODO: a nightly cron
+
+env:
+  # Don't name this "TILEDB_REST_TOKEN" since that will map into a core
+  # env/config override, as if config key "rest.token" had been set.  One of the
+  # purposes of this CI is to run tests where all config is passed via context
+  # arguments and none via environment variables, in order to flush out
+  # callsites within the code which aren't passing context as they should.
+  TILEDB_REST_UNITTEST_TOKEN: ${{ secrets.TILEDB_REST_UNITTEST_TOKEN}}
+
+jobs:
+  ci:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # This could be run on MacOS too, but, we have enough OS redundancy,
+          # and MacOS-runner availability is a more tightly constrained resource
+          # in GitHub Actions as of 2025-02-06.
+          - name: linux
+            os: ubuntu-24.04
+            # TODO: also on 3.12. But 3.9 is higher-pri, until we drop support
+            # for it. (Note our main CI tests across a broader set of Python
+            # versions.)
+            python_version: 3.9
+            cc: gcc-13
+            cxx: g++-13
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Show matrix OS
+        run: echo "matrix.os:" ${{ matrix.os }}
+
+      - name: Linux CPU info
+        if: ${{ matrix.os == 'ubuntu-24.04' }}
+        run: cat /proc/cpuinfo
+
+      - name: MacOS CPU info
+        if: ${{ matrix.os == 'macOS-latest' }}
+        run: sysctl -a | grep cpu
+
+      - name: Select XCode version
+        if: startsWith(matrix.os, 'macos')
+        uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: '15.4'
+
+      - name: Set up Python ${{ matrix.python_version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python_version }}
+          cache: pip
+          cache-dependency-path: ./apis/python/setup.py
+
+      - name: Show XCode version
+        run: clang --version
+
+      - name: Check out TileDB-SOMA
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # ensure we get all tags to inform package version determination
+
+      - name: Log pip dependencies
+        run: pip list
+
+      - name: Install tiledbsoma
+        # If you'reiterating on the testing framework itself, install from PyPI to avoid the
+        # time-consuming build. If you're trying to evaluate the code modified by a PR,
+        # install from source.
+        #
+        # From PyPI:
+        # run: pip install tiledbsoma==1.15.7
+        #
+        # From source:
+        run: pip -v install -e apis/python[all] -C "--build-option=--no-tiledb-deprecated"
+        #
+        env:
+          CC: ${{ matrix.cc }}
+          CXX: ${{ matrix.cxx }}
+
+      - name: Install dependencies
+        run: pip install --prefer-binary pytest typeguard tiledb.cloud
+
+      - name: Show package versions
+        run: python scripts/show-versions.py
+
+      - name: Show tiledb.cloud version
+        run: python -c 'import tiledb.cloud; print(tiledb.cloud.version.version)'
+
+      - name: Run pytests for Python
+        shell: bash
+        run: python -m pytest apis/python/remote_tests -v --durations=20 --maxfail=50

.github/workflows/r-python-interop-testing.yml

@@ -1,17 +1,21 @@
 name: TileDB-SOMA R-Python interop testing
 
 on:
-  pull_request:
-    # TODO: leave this enabled for pre-merge signal for now. At some point we may want to go back to
-    # only having this signal post-merge.
-    #paths:
-    #  - "apis/python/**"
-    #  - "apis/r/**"
-    #  - "apis/system/**"
-  push:
-    branches:
-      - main
-      - "release-*"
+# XXX TEMP
+#  pull_request:
+#    paths-ignore:
+#      # XXX TEMP
+#      - ".github/workflows/python-remote-storage.yml"
+#    # TODO: leave this enabled for pre-merge signal for now. At some point we may want to go back to
+#    # only having this signal post-merge.
+#    #paths:
+#    #  - "apis/python/**"
+#    #  - "apis/r/**"
+#    #  - "apis/system/**"
+#  push:
+#    branches:
+#      - main
+#      - "release-*"
   workflow_dispatch:
 
 jobs:

apis/python/remote_tests/README.md

@@ -0,0 +1,62 @@
+This is a WIP.
+
+These tests may be later moved into `apis/python/tests/remote`.
+
+## TO DO
+
+* File a task for allowing `.` in names at register
+* Info for provenance `.h5ad`
+* Script info for populating versions
+* Make small stack; provenance
+* Append mode: Monday/Tuesday is fine
+* UDFs:
+  * remote obs schema
+  * collection-mapper test
+* Show, upgrade, resize
+
+# Maybe
+
+* any direct-to-s3 tests?
+
+# Test-data setup
+
+```
+export TILEDB_REST_TOKEN="..." # Get the token for the Saas `unittest` user
+export TILEDB_REST_PAYER_NAMESPACE=unittest
+```
+
+Source data you can find in the sandbox account `unittest` space:
+
+```
+s3://tiledb-unittest/soma-prod-test-data/h5ad/pbmc3k_unprocessed.h5ad
+s3://tiledb-unittest/soma-prod-test-data/h5ad/pbmc3k_processed.h5ad
+```
+
+In a 
+Then use `tiledbsoma.io.from_h5ad` with the following sources and data:
+
+```
+
+ingestor.py tiledb://unittest/s3://tiledb-unittest/soma-prod-test-data/1.15.7/pbmc3k_unprocessed_1.15.7 /var/s/a/pbmc3k_unprocessed.h5ad
+
+ubuntu@segge[prod][][u]$ aws s3 cp ./pbmc3k_unprocessed_1_12.3 s3://tiledb-unittest/soma-prod-test-data/1.12.3/pbmc3k_unprocessed_1_12.3
+
+ubuntu@segge[prod][][u]$ aws s3 sync --delete ./pbmc3k_unprocessed_1.12.3 s3://tiledb-unittest/soma-prod-test-data/1.12.3/pbmc3k_unprocessed_1.12.3
+
+drwxr-xr-x 6 ubuntu ubuntu 4096 Feb  6 18:14 pbmc3k_unprocessed_1.7.3
+drwxr-xr-x 6 ubuntu ubuntu 4096 Feb  6 18:14 pbmc3k_unprocessed_1.12.3
+drwxr-xr-x 6 ubuntu ubuntu 4096 Feb  6 18:12 pbmc3k_unprocessed_1.14.5
+drwxr-xr-x 6 ubuntu ubuntu 4096 Feb  6 18:12 pbmc3k_unprocessed_1.15.0
+drwxr-xr-x 6 ubuntu ubuntu 4096 Feb  6 18:13 pbmc3k_unprocessed_1.15.7
+
+registrations:
+s3://tiledb-unittest/soma-prod-test-data/1.7.3/pbmc3k_unprocessed_1.7.3
+s3://tiledb-unittest/soma-prod-test-data/1.12.3/pbmc3k_unprocessed_1.12.3
+s3://tiledb-unittest/soma-prod-test-data/1.14.5/pbmc3k_unprocessed_1.14.5
+s3://tiledb-unittest/soma-prod-test-data/1.15.0/pbmc3k_unprocessed_1.15.0
+s3://tiledb-unittest/soma-prod-test-data/1.15.7/pbmc3k_unprocessed_1.15.7
+
+
+s3://tiledb-unittest/soma-prod-test-data/h5ad/pbmc3k_unprocessed.h5ad
+s3://tiledb-unittest/soma-prod-test-data/h5ad/pbmc3k_processed.h5ad
+```

apis/python/remote_tests/__init__.py

@@ -0,0 +1,3 @@
+from typeguard import install_import_hook
+
+install_import_hook("tiledbsoma")

apis/python/remote_tests/conftest.py

@@ -0,0 +1,47 @@
+import os
+
+import pytest
+
+import tiledbsoma
+import tiledb.cloud
+
+
+@pytest.fixture
+def conftest_token():
+    env_name = "TILEDB_REST_UNITTEST_TOKEN"
+    token = os.getenv(env_name)
+    if token is None:
+        raise Exception(f'Environment variable "{env_name}" is not set')
+    return token
+
+
+@pytest.fixture
+def conftest_tiledb_cloud_login(conftest_token):
+    print("conftest_tiledb_cloud_login")
+    tiledb.cloud.login(token=conftest_token)
+    return None
+
+
+@pytest.fixture
+def conftest_user_profile(conftest_tiledb_cloud_login):
+    return tiledb.cloud.user_profile()
+
+
+@pytest.fixture
+def conftest_namespace(conftest_user_profile):
+    return conftest_user_profile.username
+
+
+@pytest.fixture
+def conftest_default_s3_path(conftest_user_profile):
+    return conftest_user_profile.default_s3_path
+
+
+@pytest.fixture
+def conftest_context(conftest_token, conftest_namespace):
+    return tiledbsoma.SOMATileDBContext(
+        tiledb_config={
+            "rest.token": conftest_token,
+            "rest.payer_namespace": conftest_namespace,
+        }
+    )