Auto-build binaries for multiple glibc versions and clean up tests

.github/scripts/install-wavpack-linux.sh

@@ -1,10 +1,23 @@
+#!/bin/bash
+
+# Check if TARGET_FOLDER argument is provided
+if [ -n "$1" ]; then
+    TARGET_FOLDER=$1
+    # Create the target folder if it doesn't exist
+    mkdir -p "$TARGET_FOLDER"
+    # Change directory to the target folder
+    cd "$TARGET_FOLDER"
+fi
+
 sudo apt update
 sudo apt install wget
 sudo apt install -y gettext
 
-wget https://www.wavpack.com/wavpack-5.7.0.tar.bz2
-tar -xf wavpack-5.7.0.tar.bz2
-cd wavpack-5.7.0
+WAVPACK_LATEST_VERSION="$(cat .github/wavpack_latest_version.txt)"
+
+wget "https://www.wavpack.com/wavpack-$WAVPACK_LATEST_VERSION.tar.bz2"
+tar -xf wavpack-$WAVPACK_LATEST_VERSION.tar.bz2
+cd wavpack-$WAVPACK_LATEST_VERSION
 ./configure
 sudo make install
 cd ..

.github/wavpack_latest_version.txt

@@ -0,0 +1 @@
+5.7.0

.github/workflows/build-wavpack-binaries.yml

@@ -0,0 +1,57 @@
+name: Python Package using pre-built Linux binaries
+
+on: 
+  workflow_dispatch:
+
+jobs:
+  build-and-test:
+    name: Build wavpack binaries for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: ldd version
+        run: |
+          ldd --version
+          GLIBC_VERSION=$(ldd --version | head -n 1 | awk '{print $NF}')
+          echo "GLIBC Version: $GLIBC_VERSION"
+          echo "GLIBC_VERSION=$GLIBC_VERSION" >> $GITHUB_ENV
+      - name: Build wavpack libraries
+        run: |
+          WAVPACK_LATEST_VERSION="$(cat .github/wavpack_latest_version.txt)"
+          TARGET_FOLDER="./wavpack_src"
+          chmod +x ./.github/scripts/install-wavpack-linux.sh
+          ./.github/scripts/install-wavpack-linux.sh $TARGET_FOLDER
+
+          # copy the built library to the libraries/folder
+          LIB_FOLDER_NAME="$WAVPACK_LATEST_VERSION/linux-x86_64-glibc$GLIBC_VERSION"
+          echo "LIB_FOLDER_NAME=$LIB_FOLDER_NAME" >> $GITHUB_ENV
+          WAVPACK_NUMCODECS_LIB_PATH="src/wavpack_numcodecs/libraries/$LIB_FOLDER_NAME"
+          # find compiled library
+          LIBWAVPACK_PATH="$(find $TARGET_FOLDER/src/.libs -type f | grep libwavpack.so)"
+          mv $LIBWAVPACK_PATH $WAVPACK_NUMCODECS_LIB_PATH/libwavpack.so
+          rm -r $TARGET_FOLDER
+      - name: Install wavpack numcodecs
+        run: |
+          pip install .[test]
+      - name: Test imports
+        run: |
+          pytest -s tests/test_imports.py
+      - name: Test with pytest 
+        run: |
+          pytest -v
+      - name: Make PR with updated binaries
+        uses: peter-evans/create-pull-request@v3
+        with:
+          commit-message: "Update $LIB_FOLDER_NAME wavpack binaries"
+          title: "Update wavpack binaries"
+          body: "This PR updates the wavpack binaries."
+          branch: "update-wavpack-binaries"
+          base: "main"
+          add-paths: "src/wavpack_numcodecs/libraries"

.github/workflows/python-package-binaries.yml

@@ -28,10 +28,7 @@ jobs:
           ldd --version
       - name: Install dependencies
         run: |
-          pip install Cython
-          pip install zarr
-          pip install pytest
-          pip install .
+          pip install .[test]
       - name: Installed wavpack version
         run: |
           python -c "import wavpack_numcodecs; print(wavpack_numcodecs.wavpack_version)"

.github/workflows/python-package-cython.yml

@@ -44,9 +44,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install Cython
-          pip install .
-          pip install zarr
-          pip install pytest
+          pip install .[test]
       - name: Test imports and version
         run: |
           pytest -s tests/test_imports.py

.github/workflows/python-package-multi-threading.yml

@@ -43,10 +43,7 @@ jobs:
           which python
       - name: Install dependencies
         run: |
-          pip install Cython
-          pip install .
-          pip install zarr
-          pip install pytest
+          pip install .[test]
       - name: Test imports and version
         run: |
           pytest -s tests/test_imports.py

.github/workflows/python-package-no-cython.yml

@@ -43,9 +43,7 @@ jobs:
           which python
       - name: Install dependencies
         run: |
-          pip install .
-          pip install zarr
-          pip install pytest
+          pip install .[test_no_cython]
       - name: Test imports and version
         run: |
           pytest -s tests/test_imports.py

pyproject.toml

@@ -20,11 +20,23 @@ dependencies = [
     "packaging"
 ]
 
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "zarr",
+    "Cython"
+]
+
+test_no_cython = [
+    "pytest",
+    "zarr"
+]
+
+
 [project.urls]
 "Homepage" = "https://github.com/AllenNeuralDynamics/wavpack-numcodecs"
 
 
-
 [build-system]
 requires = ["setuptools>=62.0"]
 build-backend = "setuptools.build_meta"

setup.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import platform
 import shutil
-import os
 from pathlib import Path
 from subprocess import check_output
 
@@ -33,14 +32,30 @@ def get_build_extensions():
             runtime_library_dirs = ["/usr/local/lib/", "/usr/bin/"]
         else:
             print("Using shipped libraries")
-            extra_link_args = [f"-L{SRC_FOLDER}/libraries/{LATEST_WAVPACK_VERSION}/linux-x86_64"]
+            wavpack_libraries_folder = Path(f"{SRC_FOLDER}/libraries/{LATEST_WAVPACK_VERSION}")
+            available_glibc_builds = [
+                p.name for p in wavpack_libraries_folder.iterdir() if p.is_dir() and "linux" in p.name
+            ]
+            available_glibc_versions = [
+                p[p.find("glibc") + len("glibc") :] for p in available_glibc_builds
+            ]
+
+            glibc_version = platform.libc_ver()[1]
+            if glibc_version not in available_glibc_versions:
+                raise RuntimeError(
+                    f"Could not find a matching glibc version for the shipped libraries. "
+                    f"Available builds: {available_glibc_versions}"
+                )
+            distr_folder = f"linux-x86_64_glibc{glibc_version}"
+
+            extra_link_args = [f"-L{wavpack_libraries_folder}/{distr_folder}"]
             runtime_library_dirs = [
-                f"$ORIGIN/libraries/{LATEST_WAVPACK_VERSION}/linux-x86_64"
+                f"$ORIGIN/libraries/{LATEST_WAVPACK_VERSION}/{distr_folder}",
             ]
             # hack
             shutil.copy(
-                f"{SRC_FOLDER}/libraries/{LATEST_WAVPACK_VERSION}/linux-x86_64/libwavpack.so",
-                f"{SRC_FOLDER}/libraries/{LATEST_WAVPACK_VERSION}/linux-x86_64/libwavpack.so.1",
+                f"{wavpack_libraries_folder}/{distr_folder}/libwavpack.so",
+                f"{wavpack_libraries_folder}/{distr_folder}/libwavpack.so.1",
             )
     elif platform.system() == "Darwin":
         libraries = ["wavpack"]