diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index e82caddcb..25f3f68bd 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -17,7 +17,7 @@ stages: strategy: matrix: linux: - imageName: 'ubuntu-20.04' + imageName: 'ubuntu-22.04' python.version: '3.x' CXX: g++ BUILD_PYTHON_API: ON @@ -25,22 +25,13 @@ stages: DOWNLOAD_TILEDB_PREBUILT: ON ARTIFACT_NAME: linux linux_tiledb_source_build: - imageName: 'ubuntu-20.04' + imageName: 'ubuntu-22.04' python.version: '3.x' CXX: g++ BUILD_PYTHON_API: OFF BUILD_SPARK_API: OFF DOWNLOAD_TILEDB_PREBUILT: OFF ARTIFACT_NAME: linux_tiledb_source_build - macos: - imageName: 'macOS-11' - python.version: '3.7' - CXX: clang++ - BUILD_PYTHON_API: ON - BUILD_SPARK_API: ON - DOWNLOAD_TILEDB_PREBUILT: ON - #SDKROOT: '/Applications/Xcode_10.3.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk' - ARTIFACT_NAME: macos pool: vmImage: $(imageName) steps: @@ -96,6 +87,7 @@ stages: BUILD_PYTHON_API: ON BUILD_SPARK_API: ON #SDKROOT: '/Applications/Xcode_10.3.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk' + MACOSX_DEPLOYMENT_TARGET: '10.15' pool: vmImage: $(imageName) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 000000000..3cb56558a --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,195 @@ +name: macOS +on: + push: + paths: + - '.github/workflows/macos.yml' + - 'apis/**' + - 'libtiledbvcf/**' + pull_request: + paths: + - '.github/workflows/macos.yml' + - 'apis/**' + - 'libtiledbvcf/**' + workflow_dispatch: +env: + MACOSX_DEPLOYMENT_TARGET: "11" +jobs: + libtiledbvcf: + # Stuck on macos-11 because of the htslib dependency. The htslib + # configuration step fails on macos-12 and macos-13 + runs-on: macos-11 + steps: + - uses: actions/checkout@v3 + - name: Configure libtiledbvcf + run: | + cmake -S libtiledbvcf -B $(pwd)/libtiledbvcf/build \ + -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_INSTALL_PREFIX=$(pwd)/dist \ + -D OVERRIDE_INSTALL_PREFIX=OFF \ + -D DOWNLOAD_TILEDB_PREBUILT=ON + - name: Build libtiledbvcf + run: cmake --build $(pwd)/libtiledbvcf/build -j 2 --config Debug + - name: Upload coredump as artifact if build failed + if: failure() + uses: actions/upload-artifact@v3 + with: + name: coredump + path: libtiledbvcf/build/core + retention-days: 14 + if-no-files-found: error + - name: Install libtiledbvcf + run: cmake --build $(pwd)/libtiledbvcf/build --config Debug --target install-libtiledbvcf + - name: Upload libtiledbvcf as artifact + uses: actions/upload-artifact@v3 + with: + name: libtiledbvcf + path: dist/* + retention-days: 14 + if-no-files-found: error + - name: Confirm linking + run: otool -L dist/lib/libtiledbvcf.dylib + - name: libtiledbvcf version + run: ./dist/bin/tiledbvcf version + - name: Install bcftools + run: brew install bcftools + - name: Unit tests + run: | + make -j 2 -C libtiledbvcf/build/libtiledbvcf tiledb_vcf_unit + ./libtiledbvcf/build/libtiledbvcf/test/tiledb_vcf_unit + - name: CLI tests (require bcftools) + run: | + # USAGE: run-cli-tests.sh + libtiledbvcf/test/run-cli-tests.sh libtiledbvcf/build libtiledbvcf/test/inputs + python: + runs-on: macos-11 + needs: libtiledbvcf + env: + DYLD_LIBRARY_PATH: "${{ github.workspace }}/dist/lib" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetch everything for python setuptools_scm + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: python -m pip install --prefer-binary \ + dask \ + distributed \ + 'fsspec<2023.3.0' \ + pandas \ + pyarrow==10.0.1 \ + pyarrow-hotfix \ + pybind11 \ + pytest \ + setuptools \ + setuptools_scm \ + setuptools_scm_git_archive \ + wheel + - name: Build tiledbvcf-py + env: + LIBTILEDBVCF_PATH: "${{ github.workspace }}/dist" + run: | + echo $DYLD_LIBRARY_PATH + cd apis/python + python -m pip install . + - name: Confirm linking + run: otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so + java: + runs-on: macos-11 + needs: libtiledbvcf + steps: + - uses: actions/checkout@v3 + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Check format + run: cd apis/java && ./gradlew checkFormat + - name: Assemble + run: cd apis/java && ./gradlew assemble + - name: Test + run: cd apis/java && ./gradlew test + spark: + runs-on: macos-11 + needs: libtiledbvcf + steps: + - uses: actions/checkout@v3 + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Assemble + run: cd apis/spark && ./gradlew assemble + - name: Jar + run: cd apis/spark && ./gradlew jar + - name: Test + run: cd apis/spark && ./gradlew test + - name: Check format + run: cd apis/spark && ./gradlew checkFormat + spark3: + runs-on: macos-11 + needs: libtiledbvcf + steps: + - uses: actions/checkout@v3 + - name: Download libtiledbvcf artifact + uses: actions/download-artifact@v3 + with: + name: libtiledbvcf + path: dist + - name: Assemble + run: cd apis/spark3 && ./gradlew assemble + - name: Jar + run: cd apis/spark3 && ./gradlew jar + - name: Test + run: cd apis/spark3 && ./gradlew test + - name: Check format + run: cd apis/spark3 && ./gradlew checkFormat + python-standalone: + runs-on: macos-11 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetch everything for python setuptools_scm + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: python -m pip install --prefer-binary \ + dask \ + distributed \ + 'fsspec<2023.3.0' \ + pandas \ + pyarrow==10.0.1 \ + pyarrow-hotfix \ + pybind11 \ + pytest \ + setuptools \ + setuptools_scm \ + setuptools_scm_git_archive \ + wheel + - name: Build tiledbvcf-py + run: cd apis/python && python -m pip install . + - name: Confirm linking + run: | + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.dylib + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledb.dylib + otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libhts.*.dylib + - name: Version + run: python -c "import tiledbvcf; print(tiledbvcf.version)" + - name: Install bcftools (for tests) + run: brew install bcftools + - name: Install tiledb-py + run: pip install tiledb + - name: Test tiledbvcf-py + run: cd apis/python && pytest diff --git a/apis/java/CMakeLists.txt b/apis/java/CMakeLists.txt index 45af9e6c1..78bb2d8a5 100644 --- a/apis/java/CMakeLists.txt +++ b/apis/java/CMakeLists.txt @@ -37,8 +37,8 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() -# Set C++11 as required standard for all C++ targets. -set(CMAKE_CXX_STANDARD 11) +# Set C++17 as required standard for all C++ targets. +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) # Don't use GNU extensions diff --git a/apis/python/conda-env.yml b/apis/python/conda-env.yml index 3cd7b5f93..fa83ab45a 100644 --- a/apis/python/conda-env.yml +++ b/apis/python/conda-env.yml @@ -14,5 +14,4 @@ dependencies: - pip: - pytest-runner==5.1 - pytest==5.1.2 - - tiledb - zipp<3.16 diff --git a/apis/python/setup.py b/apis/python/setup.py index 65d82456d..edefbfbf8 100644 --- a/apis/python/setup.py +++ b/apis/python/setup.py @@ -224,14 +224,14 @@ class BuildExtCmd(build_ext): def build_extensions(self): if sys.platform != "win32": - opts = ["-std=c++11", "-g"] + opts = ["-std=c++17", "-g"] if TILEDBVCF_DEBUG_BUILD: opts.extend(["-O0"]) else: opts.extend(["-O2"]) else: # windows - # Note: newer versions of msvc cl may not recognize -std:c++11 - opts = ["-std:c++11", "-Zi"] + # Note: newer versions of msvc cl may not recognize -std:c++17 + opts = ["-std:c++17", "-Zi"] if TILEDBVCF_DEBUG_BUILD: opts.extend(["-Od"]) else: diff --git a/apis/spark/CMakeLists.txt b/apis/spark/CMakeLists.txt index 45af9e6c1..a1f4b266e 100644 --- a/apis/spark/CMakeLists.txt +++ b/apis/spark/CMakeLists.txt @@ -37,7 +37,7 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() -# Set C++11 as required standard for all C++ targets. +# Set C++17 as required standard for all C++ targets. set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) # Don't use GNU extensions diff --git a/apis/spark3/CMakeLists.txt b/apis/spark3/CMakeLists.txt index 45af9e6c1..78bb2d8a5 100644 --- a/apis/spark3/CMakeLists.txt +++ b/apis/spark3/CMakeLists.txt @@ -37,8 +37,8 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() -# Set C++11 as required standard for all C++ targets. -set(CMAKE_CXX_STANDARD 11) +# Set C++17 as required standard for all C++ targets. +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) # Don't use GNU extensions diff --git a/ci/azure-linux_mac.yml b/ci/azure-linux_mac.yml index 852461a3d..530185570 100755 --- a/ci/azure-linux_mac.yml +++ b/ci/azure-linux_mac.yml @@ -158,6 +158,8 @@ steps: pushd apis/python micromamba create -f conda-env.yml -y micromamba activate tiledbvcf-py + pip debug --verbose + pip install -vvv tiledb # Install and test the Python API python setup.py install --user