Add CI/CD for building wheels and uploading artifacts

.github/workflows/main.yml

@@ -1,64 +1,217 @@
 name: Python Wheel CI
 
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the master branch
-on: [push]
+# Controls when the action will run. Triggers the workflow on push
+
+###################################
+# WARNING #########################
+# as self-hosted runners are used, and security policy for them has not been yet determined by GitHub
+# pay close attention to not enable workflows on pull_request events
+# TLDR: do NOT add 'pull_request' here for the time being
+on: [push] 
+###################################
+###################################
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
 
+  # This job builds wheels for Windows x86_64 arch
+  build-windows-x86_64:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+    steps:
+      - name: Cache .hunter folder
+        uses: actions/cache@v2
+        with:
+          path: ~/.hunter/
+          key: hunter-msvc
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with: 
+          python-version: ${{ matrix.python-version }}
+      - name: Append build hash if not a tagged commit
+        if: github.ref != 'refs/tags/*'
+        run: echo '::set-env name=BUILD_COMMIT_HASH::${{github.sha}}'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+      - name: Building wheels
+        run: python -m pip wheel . -w ./wheelhouse/
+      - name: Archive wheel artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheelhouse/
+
+  # This job builds wheels for macOS x86_64 arch
+  build-macos-x86_64:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+    steps:
+      - name: Cache .hunter folder
+        uses: actions/cache@v2
+        with:
+          path: ~/.hunter/
+          key: hunter-clang
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with: 
+          python-version: ${{ matrix.python-version }}
+      - name: Append build hash if not a tagged commit
+        if: github.ref != 'refs/tags/*'
+        run: echo '::set-env name=BUILD_COMMIT_HASH::${{github.sha}}'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          brew install libusb
+          python -m pip install delocate
+      - name: Set macos deployment target
+        run: echo '::set-env name=MACOSX_DEPLOYMENT_TARGET::10.9' 
+      - name: Building wheels
+        run: python -m pip wheel . -w ./wheelhouse/
+      - name: Auditing wheels
+        run: ci/repair-whl-macos.sh `pwd`/wheelhouse/* `pwd`/wheelhouse/audited      
+      - name: Archive wheel artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheelhouse/audited/
+
   # This job builds wheels for x86_64 arch
-  build-x86_64:
+  build-linux-x86_64:
     runs-on: ubuntu-latest
-    container: 
+    container:
       image: quay.io/pypa/manylinux2014_x86_64
-      env: 
+      env:
         PLAT: manylinux2014_x86_64
     steps:
-    - name: Cache .hunter folder
-      uses: actions/cache@v2
-      with:
-        path: ~/.hunter/
-        key: hunter-x86_64
-    - uses: actions/checkout@v2
-      with:
-        submodules: 'recursive'
-    - name: Installing libusb1-devel dependency
-      run: yum install -y --disableplugin=fastestmirror libusb1-devel
-    - name: Installing cmake dependency
-      run: |
-        /opt/python/cp38-cp38/bin/python3.8 -m pip install cmake
-        ln -s /opt/python/cp38-cp38/bin/cmake /bin/
-    - name: Building wheels
-      run: for PYBIN in /opt/python/cp3*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/; done
-    - name: Auditing wheels
-      run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done
-    - name: Archive wheel artifacts
-      uses: actions/upload-artifact@v2
-      with:
-        name: audited-wheels
-        path: wheelhouse/audited/
+      - name: Cache .hunter folder
+        uses: actions/cache@v2
+        with:
+          path: ~/.hunter/
+          key: hunter-x86_64
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Installing libusb1-devel dependency
+        run: yum install -y --disableplugin=fastestmirror libusb1-devel
+      - name: Installing cmake dependency
+        run: |
+          /opt/python/cp38-cp38/bin/python3.8 -m pip install cmake
+          ln -s /opt/python/cp38-cp38/bin/cmake /bin/
+      - name: Create folder structure
+        run: mkdir -p wheelhouse/audited/
+      - name: Append build hash if not a tagged commit
+        if: github.ref != 'refs/tags/*'
+        run: echo '::set-env name=BUILD_COMMIT_HASH::${{github.sha}}'
+      - name: Building a source distribution
+        run: |
+          /opt/python/cp38-cp38/bin/python3.8 setup.py sdist --formats=bztar,zip
+          mv dist/* wheelhouse/audited/
+      - name: Building wheels
+        run: for PYBIN in /opt/python/cp3*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/; done
+      - name: Auditing wheels
+        run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done
+      - name: Archive wheel artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheelhouse/audited/
 
   # This job builds wheels for armhf arch
-  build-armhf:
+  build-linux-armhf:
     runs-on: luxonis-armhf
     container: registry.gitlab.com/luxonis/depthai-crosscompile/debian-buster
     steps:
-    - name: Cache .hunter folder
-      uses: actions/cache@v2
-      with:
-        path: ~/.hunter/
-        key: hunter-armhf
-    - uses: actions/checkout@v2
-      with:
-        submodules: 'recursive'
-    - name: Building wheel 
-      run: python3 -m pip wheel . -w ./wheelhouse/
-    - name: Auditing wheel
-      run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/audited/; done
-    - name: Archive wheel artifacts
-      uses: actions/upload-artifact@v2
-      with:
-        name: audited-wheels
-        path: wheelhouse/audited/
+      - name: Cache .hunter folder
+        uses: actions/cache@v2
+        with:
+          path: ~/.hunter/
+          key: hunter-armhf
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Append build hash if not a tagged commit
+        if: github.ref != 'refs/tags/*'
+        run: echo '::set-env name=BUILD_COMMIT_HASH::${{github.sha}}'
+      - name: Building wheel
+        run: python3 -m pip wheel . -w ./wheelhouse/
+      - name: Auditing wheel
+        run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat linux_armv7l -w wheelhouse/audited/; done
+      - name: Archive wheel artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheelhouse/audited/
+
+  # Run tests
+  tests:
+    runs-on: luxonis-arm-tests
+    needs: [build-linux-armhf, build-windows-x86_64, build-macos-x86_64, build-linux-x86_64]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - uses: actions/download-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheels/depthai/
+      - name: List files
+        run: ls -lah
+      - name: install DepthAI
+        run: python3.7 -m pip install depthai --no-index --find-links file://`pwd`/wheels/depthai/
+      - name: Run tests
+        run: python3.7 tests/test.py
+
+  # Deploy to artifactory
+  deploy-artifacts:
+    runs-on: ubuntu-latest
+    needs: tests
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - uses: actions/download-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheelhouse/audited/
+      - name: List files
+        run: ls -lah wheelhouse/audited/
+      - name: Run deploy to artifacts
+        run: bash ./ci/upload-artifactory.sh
+        env:
+          ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
+          ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
+          ARTIFACTORY_PASS: ${{ secrets.ARTIFACTORY_PASS }}
+
+  # Deploy to PyPi. Only when a commit is tagged
+  deploy-pypi:
+    if: github.ref == 'refs/tags/*'
+    runs-on: ubuntu-latest
+    needs: tests
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - uses: actions/download-artifact@v2
+        with:
+          name: audited-wheels
+          path: wheelhouse/audited/
+      - name: List files
+        run: ls -lah
+      - name: Run deploy to PyPi
+        run: bash ./ci/upload-pypi.sh
+        env:
+          PYPI_SERVER: ${{ secrets.PYPI_SERVER }}
+          PYPI_USER: ${{ secrets.PYPI_USER }}
+          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
 

CMakeLists.txt

@@ -8,8 +8,8 @@ set(HUNTER_CONFIGURATION_TYPES "Release" CACHE STRING "Hunter dependencies list
 
 include("cmake/HunterGate.cmake")
 HunterGate(
-    URL "https://github.com/cpp-pm/hunter/archive/v0.23.253.tar.gz"
-    SHA1 "88ea6d37c897a81a080eb9ae0f69d7807bbb3c73"
+    URL "https://github.com/cpp-pm/hunter/archive/v0.23.258.tar.gz"
+    SHA1 "062a19ab13ce8dffa9a882b6ce3e43bdabdf75d3"
     FILEPATH ${CMAKE_CURRENT_LIST_DIR}/depthai-core/cmake/Hunter/config.cmake # Add depthai-core config (hunter limitation)
 )
 
@@ -20,7 +20,7 @@ endif()
 
 # Pybindings project
 set(TARGET_NAME depthai)
-project(${TARGET_NAME})
+project(${TARGET_NAME} VERSION "0") # revision of bindings [depthai-core].[rev]
 
 # Add depthai-cpp dependency
 add_subdirectory(depthai-core EXCLUDE_FROM_ALL)

MANIFEST.in

@@ -2,7 +2,8 @@ include README.md LICENSE
 global-include CMakeLists.txt *.cmake
 recursive-include src *
 recursive-include pybind11/include *.h
-graft depthai-cpp
-prune depthai-cpp/build
-prune example-cpp
+graft depthai-core
+prune depthai-core/build
 prune build
+prune wheelhouse
+prune dist

ci/msvc_toolchain.cmake

@@ -0,0 +1,2 @@
+# Statically compile vcruntime library
+add_compile_options(/MT)

ci/repair-whl-macos.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -ex
+cd $(mktemp -d)
+unzip $1
+DYLD_FALLBACK_LIBRARY_PATH=/usr/local/lib delocate-path -L palace.dylibs .
+wheel=$(basename $1)
+zip -r $wheel *
+mkdir -p $2
+mv $wheel $2
+tempdir=$(pwd)
+cd -
+rm -rf $tempdir

ci/upload-artifactory.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cd wheelhouse/audited/ || exit 1
+export PATH_PREFIX=luxonis-python-snapshot-local/depthai
+
+curl -fL https://getcli.jfrog.io | sh
+./jfrog rt config --url=$ARTIFACTORY_URL --user=$ARTIFACTORY_USER --password=$ARTIFACTORY_PASS
+./jfrog rt u "*" "$PATH_PREFIX/"

ci/upload-pypi.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+python3 -m pip install -U twine
+
+echo "[TEST] PYPI_SERVER: $PYPI_SERVER ; PYPI_USER: $PYPI_USER ; PYPI_PASSWORD: $PYPI_PASSWORD"
+for file in wheelhouse/audited/*.whl; do
+  echo "Uploading $file"
+  python3 -m twine upload --repository-url "$PYPI_SERVER" --username $PYPI_USER --password $PYPI_PASSWORD $file
+done

find_version.py

@@ -0,0 +1,29 @@
+import sys
+import os
+import re
+
+version_depthai_core_path = "depthai-core/CMakeLists.txt"
+version_depthai_python_path = "CMakeLists.txt"
+
+cmake_lists_txt_version_pattern = r'project[\s]*\([^Vv]*version[\s]+((\"(?P<ver1>\S*)\"|(?P<ver2>\S*)\s))'
+
+def get_version_from_cmake_lists(path):
+    with open(path, 'r') as file:
+        content = file.read()
+        match = re.search(cmake_lists_txt_version_pattern, content, flags=re.IGNORECASE)  
+        ver1 = match.group('ver1')
+        ver2 = match.group('ver2')
+        version = ver1
+        if ver1 == None:
+            version = ver2
+        return version
+
+def get_package_version():
+
+    version_core = '0.0.0'
+    version_revision = '0'
+    version_core = get_version_from_cmake_lists(version_depthai_core_path)
+    version_revision = get_version_from_cmake_lists(version_depthai_python_path)
+    package_version = version_core + '.' + version_revision
+
+    return package_version