Standardize the results file across jobs #823
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Lumigator pipeline | |
| on: | |
| pull_request: | |
| branches: | |
| - "**" | |
| # synchronized is when you push new commits | |
| types: ["opened", "synchronize"] | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| # Not the right semver regexp, but good enough | |
| - 'v*.*.*' | |
| # required to enable manual triggers on the GH web ui | |
| workflow_dispatch: | |
| permissions: | |
| pull-requests: read | |
| contents: read | |
| packages: write | |
| jobs: | |
| lint: | |
| name: Code linting | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Lint | |
| run: uvx ruff check --config ruff.toml | |
| continue-on-error: false | |
| unit-tests: | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| if: ${{ needs.lint.result == 'success' }} | |
| strategy: | |
| matrix: | |
| test: ["sdk", "backend", "jobs"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Install python | |
| run: uv python install | |
| - name: Run ${{ matrix.test }} unit tests | |
| run: make test-${{ matrix.test }}-unit | |
| integration-tests: | |
| name: Integration tests | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| if: ${{ needs.lint.result == 'success' }} | |
| strategy: | |
| matrix: | |
| test: ["sdk", "backend" ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Install python | |
| run: uv python install | |
| - name: Create HF cache | |
| run: | | |
| mkdir -p ${HOME}/.cache/huggingface | |
| chmod -R 777 ${HOME}/.cache/huggingface | |
| - name: Spinning up containers for the tests | |
| run: make start-lumigator-build | |
| - name: Run ${{ matrix.test }} integration tests | |
| run: make test-${{ matrix.test }}-integration | |
| - name: Collect Ray logs in case of failure | |
| if: failure() | |
| run: | | |
| mkdir /tmp/raylogs | |
| docker cp lumigator-ray-1:/tmp/ray/ /tmp/raylogs/ | |
| - name: Upload Ray logs in case of failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ray-logs | |
| path: /tmp/raylogs/ray/session_*/logs/ | |
| notebook-integration-test: | |
| name: Notebook integration tests | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| if: ${{ needs.lint.result == 'success' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Create a virtual environment | |
| run: uv venv | |
| working-directory: notebooks | |
| - name: Install required packages | |
| run: uv pip install -r requirements.txt | |
| working-directory: notebooks | |
| - name: Install kernel | |
| run: source .venv/bin/activate && python -m ipykernel install --user --name=lumigator | |
| working-directory: notebooks | |
| - name: Create HF cache | |
| run: | | |
| mkdir -p ${HOME}/.cache/huggingface | |
| chmod -R 777 ${HOME}/.cache/huggingface | |
| - name: Setup containers | |
| run: make start-lumigator-build | |
| - name: Run notebook | |
| run: source .venv/bin/activate && jupyter execute --kernel_name=lumigator walkthrough.ipynb | |
| working-directory: notebooks | |
| shell: bash | |
| build-backend-image: | |
| name: Build Backend Docker image | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| if: ${{ needs.lint.result == 'success' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check for modified paths | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| build: | |
| - '.github/**' | |
| - 'lumigator/**' | |
| - name: Truncate commit SHA | |
| if: steps.filter.outputs.build == 'true' || contains(github.ref, 'refs/tags/') | |
| run: echo "GITHUB_SHA_SHORT=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV | |
| - name: Set up QEMU | |
| if: steps.filter.outputs.build == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker buildx | |
| if: steps.filter.outputs.build == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| if: steps.filter.outputs.build == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push backend image | |
| if: steps.filter.outputs.build == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/build-push-action@v6 | |
| with: | |
| platforms: linux/amd64,linux/arm64 | |
| target: "main_image" | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:backend_dev_${{ env.GITHUB_SHA_SHORT }} | |
| build-frontend-image: | |
| name: Build Frontend Docker image | |
| runs-on: ubuntu-latest | |
| needs: lint | |
| if: ${{ needs.lint.result == 'success' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check for modified paths | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| build_fe: | |
| - '.github/**' | |
| - 'lumigator/**' | |
| - name: Truncate commit SHA | |
| if: steps.filter.outputs.build_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| run: echo "GITHUB_SHA_SHORT=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV | |
| - name: Set up QEMU | |
| if: steps.filter.outputs.build_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker buildx | |
| if: steps.filter.outputs.build_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| if: steps.filter.outputs.build_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push frontend Docker image | |
| uses: docker/build-push-action@v6 | |
| if: steps.filter.outputs.build_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| with: | |
| file: "lumigator/frontend/Dockerfile" | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| target: "server" | |
| tags: | | |
| ghcr.io/${{ github.repository }}:frontend_${{ env.GITHUB_SHA_SHORT }} | |
| push-backend-images: | |
| name: Push backend Docker images | |
| needs: [ "build-backend-image", "build-frontend-image", "unit-tests", "integration-tests", "notebook-integration-test" ] | |
| if: ${{ needs.build-backend-image.result == 'success' && needs.build-frontend-image.result == 'success' && needs.unit-tests.result == 'success' && needs.integration-tests.result == 'success' && needs.notebook-integration-test.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check for modified paths | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| push_be: | |
| - '.github/**' | |
| - 'lumigator/**' | |
| - name: Truncate commit SHA | |
| if: steps.filter.outputs.push_be == 'true' || contains(github.ref, 'refs/tags/') | |
| run: echo "GITHUB_SHA_SHORT=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV | |
| - name: Login to GitHub Container Registry | |
| if: steps.filter.outputs.push_be == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to DockerHub | |
| if: steps.filter.outputs.push_be == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Tag and push Docker image (tagged releases) | |
| if: contains(github.ref, 'refs/tags/') | |
| run: skopeo copy --all docker://ghcr.io/${{ github.repository }}:backend_dev_${{ env.GITHUB_SHA_SHORT }} docker://mzdotai/lumigator:${{ github.ref_name }} | |
| - name: Tag and push Docker image (normal build) | |
| if: steps.filter.outputs.push_be == 'true' && !contains(github.ref, 'refs/tags/') | |
| run: skopeo copy --all docker://ghcr.io/${{ github.repository }}:backend_dev_${{ env.GITHUB_SHA_SHORT }} docker://mzdotai/lumigator:backend_dev_${{ env.GITHUB_SHA_SHORT }} | |
| - name: Tag and push Docker image (latest) | |
| if: steps.filter.outputs.push_be == 'true' && github.ref == 'refs/heads/main' | |
| run: skopeo copy --all docker://ghcr.io/${{ github.repository }}:backend_dev_${{ env.GITHUB_SHA_SHORT }} docker://mzdotai/lumigator:latest | |
| push-frontend-images: | |
| name: Push frontend Docker images | |
| needs: [ "build-backend-image", "build-frontend-image", "unit-tests", "integration-tests", "notebook-integration-test" ] | |
| if: ${{ needs.build-backend-image.result == 'success' && needs.build-frontend-image.result == 'success' && needs.unit-tests.result == 'success' && needs.integration-tests.result == 'success' && needs.notebook-integration-test.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check for modified paths | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| push_fe: | |
| - '.github/**' | |
| - 'lumigator/**' | |
| - name: Truncate commit SHA | |
| if: steps.filter.outputs.push_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| run: echo "GITHUB_SHA_SHORT=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_ENV | |
| - name: Login to GitHub Container Registry | |
| if: steps.filter.outputs.push_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to DockerHub | |
| if: steps.filter.outputs.push_fe == 'true' || contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Tag and push Docker image (tagged releases) | |
| if: contains(github.ref, 'refs/tags/') | |
| run: skopeo copy --all docker://ghcr.io/${{ github.repository }}:frontend_${{ env.GITHUB_SHA_SHORT }} docker://mzdotai/lumigator-frontend:${{ github.ref_name }} | |
| - name: Tag and push Docker image (normal build) | |
| if: steps.filter.outputs.push_fe == 'true' && !contains(github.ref, 'refs/tags/') | |
| run: skopeo copy --all docker://ghcr.io/${{ github.repository }}:frontend_${{ env.GITHUB_SHA_SHORT }} docker://mzdotai/lumigator-frontend:frontend_${{ env.GITHUB_SHA_SHORT }} | |
| - name: Tag and push Docker image (latest) | |
| if: steps.filter.outputs.push_fe == 'true' && github.ref == 'refs/heads/main' | |
| run: skopeo copy --all docker://ghcr.io/${{ github.repository }}:frontend_${{ env.GITHUB_SHA_SHORT }} docker://mzdotai/lumigator-frontend:latest | |
| sdk-packaging: | |
| name: Package SDK | |
| needs: [ "notebook-integration-test", "integration-tests", "unit-tests" ] | |
| if: ${{ startsWith(github.ref, 'refs/tags/') && needs.notebook-integration-test.result == 'success' && needs.integration-tests.result == 'success' && needs.unit-tests.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Add tools directory to path | |
| run: echo ${PWD}/tools >> $GITHUB_PATH | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| - name: Install yq package | |
| run: sudo snap install yq | |
| - name: Check tag consistency (SDK) | |
| run: check_version.sh | |
| working-directory: lumigator/sdk | |
| - name: Check tag consistency (schemas) | |
| run: check_version.sh | |
| working-directory: lumigator/schemas | |
| - name: Install python (SDK) | |
| run: uv python install | |
| working-directory: lumigator/sdk | |
| - name: Install python (schemas) | |
| run: uv python install | |
| working-directory: lumigator/schemas | |
| - name: Package SDK | |
| run: uv build | |
| working-directory: lumigator/sdk | |
| - name: Package schemas | |
| run: uv build | |
| working-directory: lumigator/schemas | |
| - name: Upload SDK packages to PyPI | |
| working-directory: lumigator/sdk | |
| run: uv publish --token ${{ secrets.PYPI_TOKEN }} --verbose | |
| - name: Upload schemas packages to PyPI | |
| working-directory: lumigator/schemas | |
| run: uv publish --token ${{ secrets.PYPI_TOKEN }} --verbose |