ci-cd.yml

##
## --------------------------------------------------------------------------------
## SPDX-FileCopyrightText: 2024 Martin Jan Köhler and Harald Pretl
## Johannes Kepler University, Institute for Integrated Circuits.
##
## This file is part of KPEX 
## (see https://github.com/martinjankoehler/klayout-pex).
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
## SPDX-License-Identifier: GPL-3.0-or-later
## --------------------------------------------------------------------------------
##

# This starter workflow is for a CMake project running on multiple platforms. There is a different starter workflow if you just want a single platform.
# See: https://github.com/actions/starter-workflows/blob/main/ci/cmake-single-platform.yml
name: Multi-platform Python/C++ build/test

on:
  push:
    branches: [ "main" ]
    paths-ignore:
       - '.gitignore'
       - '**.md'
       - '*.sh'
  pull_request:
    branches: [ "main" ]
    paths-ignore:
       - '.gitignore'
       - '**.md'
       - '*.sh'
  workflow_call:
  workflow_dispatch:
  release:
    types: [published]

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  build_and_test:
    name: Build/Test ${{ matrix.os }}-${{ matrix.build_type }}-py${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}

    strategy:
      # Set fail-fast to false to ensure that feedback is delivered for all matrix combinations. 
      # Consider changing this to true when your workflow is stable.
      fail-fast: false

      # Set up a matrix to run the following 3 configurations:
      # 1. <Windows, Release, latest MSVC compiler toolchain on the default runner image, default generator>
      # 2. <Linux, Release, latest GCC compiler toolchain on the default runner image, default generator>
      # 3. <Linux, Release, latest Clang compiler toolchain on the default runner image, default generator>
      #
      # To add more build types (Release, Debug, RelWithDebInfo, etc.) customize the build_type list.
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ['3.12']
        build_type: [RelWithDebInfo]
        c_compiler: [clang, cl]
        include:
          - os: windows-latest
            c_compiler: cl
            cpp_compiler: cl
          - os: ubuntu-latest
            c_compiler: clang
            cpp_compiler: clang++
          - os: macos-latest
            c_compiler: clang
            cpp_compiler: clang++
        exclude:
          - os: windows-latest
            c_compiler: gcc
          - os: windows-latest
            c_compiler: clang
          - os: ubuntu-latest
            c_compiler: cl
          - os: ubuntu-latest
            c_compiler: gcc
          - os: macos-latest
            c_compiler: gcc
          - os: macos-latest
            c_compiler: cl

    env:
      ALLURE_RESULTS_PATH: build/allure-results
      COVERAGE_DATA_PATH: build/python-coverage.sqlite

    steps:
    - uses: actions/checkout@v4

    - name: "[🐍] Setup python ${{ matrix.python-version }}"
      id: setup-python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: "[🐍] Install Poetry"
      uses: snok/install-poetry@v1
      with:
        virtualenvs-create: true
        virtualenvs-in-project: true
        virtualenvs-path: .venv
        installer-parallel: true

    - name: "[🐍] Check version"
      if: ${{ github.event.release }}
      run: |
        version=$(poetry version | awk '{print $2}')
        tag=$(echo "${{ github.ref }}" | awk '{split($0,p,"/"); print p[3]}')
        if [ "v$version" != $tag ]; then echo "Release tag ('$tag') and poetry version ('v$version') do not match!"; exit 1; fi;
      shell: bash
    
    - name: "[🐍] Load cached venv"
      id: cached-poetry-dependencies
      uses: actions/cache@v4
      with:
        path: .venv
        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

    - name: "[🐍] Install dependencies"
      run: poetry install --no-interaction --no-root
      shell: bash    # NOTE: required for windows poetry calls

    - name: "[C++] Ccache for gh actions"
      uses: hendrikmuhs/ccache-action@v1.2.14
      with:
        key: ${{ github.job }}-${{ matrix.os }}  # Eg. "some_build-ubuntu-latest"
        
    - name: "Set reusable strings"
      # Turn repeated input strings (such as the build output directory) into step outputs.
      # These step outputs can be used throughout the workflow file.
      id: strings
      shell: bash
      run: |
        echo "build-output-dir=${{ github.workspace }}/build" >> "$GITHUB_OUTPUT"

    - name: "[C++] Cache CMake dependency source code"
      uses: actions/cache@v4
      env:
        cache-name: cache-cmake-dependency-sources
      with:
        # CMake cache is at ${{github.workspace}}/build/_deps but we only will cache folders ending in '-src' to cache source code
        path: ${{github.workspace}}/build/_deps/*-src
        # Cache hash is dependent on CMakeLists files anywhere as these can change what's in the cache, as well as cmake modules files
        key: ${{ env.cache-name }}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
        # it's acceptable to reuse caches for different CMakeLists content if exact match is not available and unlike build caches, we
        # don't need to restrict these by OS or compiler as it's only source code that's being cached
        restore-keys: |
          ${{ env.cache-name }}-

    - name: "[C++] Cache CMake dependency build objects"
      uses: actions/cache@v4
      env:
        cache-name: cache-cmake-dependency-builds
      with:
        # CMake cache is at ${{github.workspace}}/build/_deps but we only care about the folders ending in -build or -subbuild
        path: |
          ${{github.workspace}}/build/_deps/*-build
          ${{github.workspace}}/build/_deps/*-subbuild
        # Cache hash is dependent on CMakeLists files anywhere as these can change what's in the cache, as  well as cmake modules files
        key: ${{ env.cache-name }}-${{ matrix.os }}-${{ matrix.cxx }}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
        # it's acceptable to reuse caches for different CMakeLists content if exact match is not available
        # as long as the OS and Compiler match exactly
        restore-keys: |
          ${{ env.cache-name }}-${{ matrix.os }}-${{ matrix.cxx }}-

    - name: "[C++] Configure CMake"
      # Configure CMake in a 'build' subdirectory.
      # `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
      run: >
        cmake 
        -B ${{ steps.strings.outputs.build-output-dir }}
        -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
        -DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
        -DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
        -DCMAKE_C_COMPILER_LAUNCHER=ccache
        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache        
        -S ${{ github.workspace }}

    - name: "[C++] Build"
      # Build your program with the given configuration.
      # Note that --config is needed because the default Windows generator is a multi-config generator (Visual Studio generator).
      run: >
        cmake 
        --build ${{ steps.strings.outputs.build-output-dir }} 
        --config ${{ matrix.build_type }}

    - name: "CMake: Verify python protobuf generation"
      run: |
        # check if protobuf python modules were created
        ls klayout_pex_protobuf/tech_pb2.py
      shell: bash
      working-directory: ${{github.workspace}}

    - name: "[C++] Test"
      continue-on-error: true
      working-directory: ${{ steps.strings.outputs.build-output-dir }}
      # Execute tests defined by the CMake configuration.
      # Note that --build-config is needed because the default Windows generator is a multi-config generator (Visual Studio generator).
      # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
      run: ctest --build-config ${{ matrix.build_type }}

    - name: "[C++] Generate PB Tech (Windows)"
      if: ${{ matrix.os == 'windows-latest' }}
      run: build\${{matrix.build_type}}\gen_tech_pb.exe klayout_pex_protobuf
      working-directory: ${{github.workspace}}

    - name: "[C++] Generate PB Tech (Unix)"
      if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' }}
      run: build/gen_tech_pb klayout_pex_protobuf
      working-directory: ${{github.workspace}}

    - name: "[🐍] Install project"
      run: poetry install --no-interaction
      shell: bash    # NOTE: required for windows poetry calls

    - name: "[🐍] Build wheel and source archives"
      run: poetry build --output dist
      shell: bash    # NOTE: required for windows poetry calls

    - name: "[🐍] Verify wheel contains generation tech protobuf json messages"
      run: |
        unzip -l dist/*.whl | grep klayout_pex_protobuf | grep .pb.json
      shell: bash

    - name: "[🐍] Verify wheel contains generation protobuf python modules"
      run: |
        unzip -l dist/*.whl | grep klayout_pex_protobuf | grep _pb2.py
      shell: bash

    - name: "[🐍] Store dist artifact"
      uses: actions/upload-artifact@v4
      with:
        name: python-dist-${{ matrix.os }}_py${{ matrix.python-version }}
        path: dist
        retention-days: 7
        if-no-files-found: error

    - name: "[🐍] Run Unit tests"
      run: >
        poetry run coverage run --data-file=${{ env.COVERAGE_DATA_PATH }} -m pytest -m "not slow"
        --alluredir ${{ env.ALLURE_RESULTS_PATH }}
        --color no 
      shell: bash    # NOTE: required for windows poetry calls

    - run: pwd

    - run: ls -aR

    - name: "[🐍] Store coverage results"
      if: success() || failure()
      uses: actions/upload-artifact@v4
      with:
        name: python-unit-tests-coverage-report-${{ matrix.os }}_py${{ matrix.python-version }}
        path: ${{ env.COVERAGE_DATA_PATH }}   # NOTE: pytest-cov coverage binary file, can be combined later!
        retention-days: 1
        if-no-files-found: error

    - name: "[🐍] Store allure test results"
      if: success() || failure()
      uses: actions/upload-artifact@v4
      with:
        name: python-unit-tests-allure-report-${{ matrix.os }}_py${{ matrix.python-version }}
        path: ${{ env.ALLURE_RESULTS_PATH }}
        retention-days: 1
        if-no-files-found: error

  call-publish-pypi:
    name: "Publish distribution 📦s to PyPI/TestPyPI"
    needs: build_and_test
    concurrency:
      group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}-publish-pypi
      cancel-in-progress: true
    uses: ./.github/workflows/publish-pypi.yml

  call-integration-tests:
    name: "Integration Tests"
    needs: build_and_test
    concurrency:
      group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}-integration-tests
      cancel-in-progress: true
    uses: ./.github/workflows/integration-tests.yml

  # NOTE: this is called from integration tests
  # call-publish-reports-github-pages:
  #   name: "Publish reports to GitHub Pages"
  #   needs: build_and_test
  #   concurrency:
  #     group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}-publish-reports-github-pages
  #     cancel-in-progress: true
  #   uses: ./.github/workflows/publish-reports-github-pages.yml