From f8ca3e466551cb5a3c62d9cb098c4d31a91d2870 Mon Sep 17 00:00:00 2001 From: YannC Date: Mon, 10 Feb 2025 11:56:55 +0100 Subject: [PATCH] fix(): first iteration of new main workflow --- .../action.yml | 0 .../action.yml | 40 +++++---- .../action.yml | 11 ++- .../action.yml | 0 .github/actions/github-release/action.yml | 48 ++++++++++ .github/actions/publish-docker/action.yml | 81 +++++++++++++++++ .github/actions/publish-maven/action.yml | 55 ++++++++++++ .github/workflows/main.yml | 6 +- .../{workflow-pr.yml => pull-request.yml} | 29 +++--- .github/workflows/workflow-main.yml | 52 ++++++++++- .github/workflows/workflow-release.yml | 61 +++++++++++++ .github/workflows/workflow-test.yml | 90 ++++++++++++++++++- 12 files changed, 436 insertions(+), 37 deletions(-) rename .github/actions/{action-backend-test => backend-test}/action.yml (100%) rename .github/actions/{action-build-artifacts => build-artifacts}/action.yml (80%) rename .github/actions/{action-frontend-test => frontend-test}/action.yml (84%) rename .github/actions/{action-generate-translations => generate-translations}/action.yml (100%) create mode 100644 .github/actions/github-release/action.yml create mode 100644 .github/actions/publish-docker/action.yml create mode 100644 .github/actions/publish-maven/action.yml rename .github/workflows/{workflow-pr.yml => pull-request.yml} (80%) create mode 100644 .github/workflows/workflow-release.yml diff --git a/.github/actions/action-backend-test/action.yml b/.github/actions/backend-test/action.yml similarity index 100% rename from .github/actions/action-backend-test/action.yml rename to .github/actions/backend-test/action.yml diff --git a/.github/actions/action-build-artifacts/action.yml b/.github/actions/build-artifacts/action.yml similarity index 80% rename from .github/actions/action-build-artifacts/action.yml rename to .github/actions/build-artifacts/action.yml index a919713f149..c0b2ba72929 100644 --- a/.github/actions/action-build-artifacts/action.yml +++ b/.github/actions/build-artifacts/action.yml @@ -1,24 +1,25 @@ -name: 'Basic Multi-Step Action' -description: 'A basic GitHub Action that runs multiple steps.' +name: 'Build Artifacts' +description: 'Build Jar & Docker image, upload artifacts, and push the docker image to the registry.' + +inputs: + plugin-version: + description: "Kestra version" + default: 'LATEST' + required: true outputs: docker-tag: - description: "The Docker image Tag for Kestra" value: ${{ steps.vars.outputs.tag }} + description: "The Docker image Tag for Kestra" docker-artifact-name: - description: "The GitHub artifact containing the Kestra docker image." value: ${{ steps.vars.outputs.artifact }} + description: "The GitHub artifact containing the Kestra docker image name." plugins: - description: "The Kestra plugins to be used for the build." value: ${{ steps.plugins-list.outputs.plugins }} + description: "The Kestra plugins list used for the build." runs: using: composite - name: Build Artifacts - outputs: - docker-tag: ${{ steps.vars.outputs.tag }} - docker-artifact-name: ${{ steps.vars.outputs.artifact }} - plugins: ${{ steps.plugins-list.outputs.plugins }} steps: - name: Checkout current ref uses: actions/checkout@v4 @@ -44,12 +45,13 @@ runs: if: "!startsWith(github.ref, 'refs/tags/v')" id: plugins-list with: - plugin-version: ${{ env.PLUGIN_VERSION }} + plugin-version: ${{ inputs.plugin-version }} # Set Plugins List - name: Set Plugin List id: plugins if: "!startsWith(github.ref, 'refs/tags/v')" + shell: bash run: | PLUGINS="${{ steps.plugins-list.outputs.plugins }}" TAG=${GITHUB_REF#refs/*/} @@ -59,20 +61,21 @@ runs: echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots $PLUGINS" >> $GITHUB_OUTPUT fi - # Build - name: Build with Gradle + shell: bash run: | ./gradlew executableJar - name: Copy exe to image + shell: bash run: | cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra - # Docker Tag - name: Set up Vars id: vars + shell: bash run: | TAG=${GITHUB_REF#refs/*/} if [[ $TAG = "master" ]] @@ -90,7 +93,6 @@ runs: echo "tag=${TAG}" >> $GITHUB_OUTPUT echo "artifact=docker-kestra-${TAG}" >> $GITHUB_OUTPUT - # Docker setup - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -132,4 +134,12 @@ runs: if: "!startsWith(github.ref, 'refs/tags/v')" with: name: ${{ steps.vars.outputs.artifact }} - path: /tmp/${{ steps.vars.outputs.artifact }}.tar \ No newline at end of file + path: /tmp/${{ steps.vars.outputs.artifact }}.tar + + - name: Set outputs + id: set-outputs + shell: bash + run: | + echo "docker-tag=${{ steps.vars.outputs.tag }}" >> GITHUB_OUTPUT + echo "docker-artifact-name=${{ steps.vars.outputs.artifact }}" >> GITHUB_OUTPUT + echo "plugins=${{ steps.plugins.outputs.plugins }}" >> GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/actions/action-frontend-test/action.yml b/.github/actions/frontend-test/action.yml similarity index 84% rename from .github/actions/action-frontend-test/action.yml rename to .github/actions/frontend-test/action.yml index 7c16714b578..b641c5982dc 100644 --- a/.github/actions/action-frontend-test/action.yml +++ b/.github/actions/frontend-test/action.yml @@ -1,4 +1,4 @@ -name: 'Frontend Test' +name: Frontend Test inputs: github-token: @@ -7,15 +7,14 @@ inputs: codecov-token: description: 'Codecov Token' required: true - - runs: + using: composite steps: - id: checkout uses: actions/checkout@v4 with: - ref: ${{ github.ref }} # Using ref in case translations has committed something + ref: ${{ github.head_ref }} - name: Npm install shell: bash @@ -51,10 +50,10 @@ runs: working-directory: ui run: npm run build-storybook --quiet - - name: Serve Storybook and run tests + - name: Storybook - Run tests shell: bash working-directory: ui run: | npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \ "npx http-server storybook-static --port 6006 --silent" \ - "npx wait-on tcp:127.0.0.1:6006 && npm run test-storybook" + "npx wait-on tcp:127.0.0.1:6006 && npm run test:storybook" \ No newline at end of file diff --git a/.github/actions/action-generate-translations/action.yml b/.github/actions/generate-translations/action.yml similarity index 100% rename from .github/actions/action-generate-translations/action.yml rename to .github/actions/generate-translations/action.yml diff --git a/.github/actions/github-release/action.yml b/.github/actions/github-release/action.yml new file mode 100644 index 00000000000..7da2ae78bfd --- /dev/null +++ b/.github/actions/github-release/action.yml @@ -0,0 +1,48 @@ +name: Github Release +description: Create a Github Release +inputs: + github-token: + description: 'Github Token' + required: true + gh-personal: + description: 'Github Personal Token' + required: true + +runs: + using: composite + name: Github Release + steps: + # Download Exec + - name: Artifacts - Download executable + uses: actions/download-artifact@v4 + if: startsWith(github.ref, 'refs/tags/v') + with: + name: exe + path: build/executable + + # GitHub Release + - name: GitHub - Create release + id: create_github_release + uses: "marvinpinto/action-automatic-releases@latest" + if: startsWith(github.ref, 'refs/tags/v') + continue-on-error: true + with: + repo_token: "${{ inputs.github-token }}" + prerelease: false + files: | + build/executable/* + + # Trigger gha workflow to bump helm chart version + - name: GitHub - Trigger the Helm chart version bump + uses: peter-evans/repository-dispatch@v3 + if: steps.create_github_release.conclusion == 'success' + with: + token: ${{ inputs.gh-personal }} + repository: kestra-io/helm-charts + event-type: update-helm-chart-version + client-payload: |- + { + "new_version": "${{ github.ref_name }}", + "github_repository": "${{ github.repository }}", + "github_actor": "${{ github.actor }}" + } diff --git a/.github/actions/publish-docker/action.yml b/.github/actions/publish-docker/action.yml new file mode 100644 index 00000000000..b7d8e79a673 --- /dev/null +++ b/.github/actions/publish-docker/action.yml @@ -0,0 +1,81 @@ +name: Publish Docker + +inputs: + dockerhub-username: + description: 'DockerHub Username' + required: true + dockerhub-password: + description: 'DockerHub Password' + required: true + tag: + description: 'The Docker image Tag for Kestra' + required: true + plugins: + description: 'The Kestra plugins to be used for the build.' + required: false + packages: + description: 'The packages to be installed in the docker image.' + required: false + default: "" + python-libraries: + description: 'The python libraries to be installed in the docker image.' + required: false + default: "" + +runs: + using: composite + name: Publish Docker + steps: + - name: Checkout - Current ref + uses: actions/checkout@v4 + + # Docker setup + - name: Docker - Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Docker - Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Docker Login + - name: Docker - Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ inputs.dockerhub-username }} + password: ${{ inputs.dockerhub-password }} + + # Vars + - name: Docker - Set image name + shell: bash + id: vars + run: | + TAG=${GITHUB_REF#refs/*/} + if [[ $TAG = "master" || $TAG == v* ]]; then + echo "plugins=${{ inputs.plugins }}" >> $GITHUB_OUTPUT + else + echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ inputs.plugins }}" >> $GITHUB_OUTPUT + fi + + # Build Docker Image + - name: Artifacts - Download executable + uses: actions/download-artifact@v4 + with: + name: exe + path: build/executable + + - name: Docker - Copy exe to image + shell: bash + run: | + cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra + + # Docker Build and push + - name: Docker - Build image + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: kestra/kestra:${{ inputs.tag }} + platforms: linux/amd64,linux/arm64 + build-args: | + KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }} + APT_PACKAGES=${{inputs.packages}} + PYTHON_LIBRARIES=${{inputs.python-libraries}} diff --git a/.github/actions/publish-maven/action.yml b/.github/actions/publish-maven/action.yml new file mode 100644 index 00000000000..e52bedd466d --- /dev/null +++ b/.github/actions/publish-maven/action.yml @@ -0,0 +1,55 @@ +name: Publish Maven +description: Publish Maven artifacts to Maven Central + +inputs: + sonatype-username: + description: 'Sonatype Username' + required: true + sonatype-password: + description: 'Sonatype Password' + required: true + sonatype-gpg-keyid: + description: 'GPG Key ID' + required: true + sonatype-gpg-password: + description: 'GPG Password' + required: true + sonatype-gpg-file: + description: 'GPG File' + required: true + +runs: + using: composite + name: Publish Maven + steps: + - name: Checkout - Current ref + uses: actions/checkout@v4 + + # Setup build + - name: Setup - Build + uses: kestra-io/actions/.github/actions/setup-build@main + id: build + with: + java-enabled: true + node-enabled: true + + # Publish + - name: Publish - Release package to Maven Central + shell: bash + env: + ORG_GRADLE_PROJECT_sonatypeUsername: ${{ inputs.sonatype-username }} + ORG_GRADLE_PROJECT_sonatypePassword: ${{ inputs.sonatype-password }} + SONATYPE_GPG_KEYID: ${{ inputs.sonatype-gpg-keyid }} + SONATYPE_GPG_PASSWORD: ${{ inputs.sonatype-gpg-password }} + SONATYPE_GPG_FILE: ${{ inputs.sonatype-gpg-file }} + run: | + mkdir -p ~/.gradle/ + echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties + echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties + echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties + echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg + ./gradlew publishToSonatype ${{ startsWith(github.ref, 'refs/tags/v') && 'closeAndReleaseSonatypeStagingRepository' || '' }} + + # Gradle dependency + - name: Java - Gradle dependency graph + uses: gradle/actions/dependency-submission@v4 \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ad9283fe63..5d3aa42a636 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,9 +1,9 @@ name: Main on: -# push: -# branches: -# - develop + push: + branches: + - develop # tags: # - v* # pull_request: diff --git a/.github/workflows/workflow-pr.yml b/.github/workflows/pull-request.yml similarity index 80% rename from .github/workflows/workflow-pr.yml rename to .github/workflows/pull-request.yml index b93cacbff31..c4d7d9042ed 100644 --- a/.github/workflows/workflow-pr.yml +++ b/.github/workflows/pull-request.yml @@ -4,17 +4,13 @@ on: pull_request: branches: - develop + concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} cancel-in-progress: true env: - JAVA_VERSION: "21" - DOCKER_APT_PACKAGES: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip - DOCKER_PYTHON_LIBRARIES: kestra PLUGIN_VERSION: ${{ github.event.inputs.plugin-version != null && github.event.inputs.plugin-version || 'LATEST' }} - # to save corepack from itself - COREPACK_INTEGRITY_KEYS: 0 jobs: file-changes: @@ -50,7 +46,7 @@ jobs: - id: generate-translations name: Generate translations if: "needs.file-changes.outputs.translations == 'true'" - uses: ./.github/actions/action-generate-translations + uses: ./.github/actions/generate-translations with: github-token: ${{ secrets.GITHUB_TOKEN }} env: @@ -58,7 +54,10 @@ jobs: - id: frontend-test name: Frontend test - uses: ./.github/actions/action-frontend-test + uses: ./.github/actions/frontend-test + env: + # to save corepack from itself + COREPACK_INTEGRITY_KEYS: 0 with: github-token: ${{ secrets.GITHUB_TOKEN }} codecov-token: ${{ secrets.CODECOV_TOKEN }} @@ -68,12 +67,12 @@ jobs: needs: file-changes runs-on: ubuntu-latest timeout-minutes: 60 - if: needs.file-changes.outputs.backend == 'true' + if: "needs.file-changes.outputs.backend == 'true'" steps: - uses: actions/checkout@v4 - name: Backend test - uses: ./.github/actions/action-backend-test + uses: ./.github/actions/backend-test with: codecov-token: ${{ secrets.CODECOV_TOKEN }} sonar-token: ${{ secrets.SONAR_TOKEN }} @@ -81,14 +80,22 @@ jobs: google-service-account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} build-artifacts: - if: ${{ needs.file-changes.outputs.backend == 'true' || needs.file-changes.outputs.ui == 'true' }} name: Build artifacts + needs: file-changes + if: "needs.file-changes.outputs.backend == 'true' || needs.file-changes.outputs.ui == 'true'" runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + docker-tag: ${{ steps.build-artifacts.outputs.docker-tag }} + docker-artifact-name: ${{ steps.build-artifacts.outputs.docker-artifact-name }} + plugins: ${{ steps.build-artifacts.outputs.plugins }} steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/action-build-artifacts + - uses: ./.github/actions/build-artifacts id: build-artifacts + with: + plugin-version: ${{ env.PLUGIN_VERSION }} e2e: name: Check E2E diff --git a/.github/workflows/workflow-main.yml b/.github/workflows/workflow-main.yml index 9db3a0bf380..84be027b8a8 100644 --- a/.github/workflows/workflow-main.yml +++ b/.github/workflows/workflow-main.yml @@ -1,2 +1,52 @@ +name: Main Workflow + on: -jobs: \ No newline at end of file + push: + branches: + - develop + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + name: Execute tests + uses: ./.github/workflows/workflow-test.yml + + release: + name: Release + needs: [tests] + uses: ./.github/workflows/workflow-release.yml + with: + docker-tag: ${{ needs.tests.outputs.docker-tag }} + plugins: ${{ needs.tests.outputs.plugins }} + + end: + runs-on: ubuntu-latest + needs: + - release + if: always() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + steps: + # Update + - name: Github - Update internal + uses: benc-uk/workflow-dispatch@v1 + if: github.ref == 'refs/heads/develop' && needs.docker.result == 'success' + with: + workflow: oss-build.yml + repo: kestra-io/infra + ref: master + token: ${{ secrets.GH_PERSONAL_TOKEN }} + + # Slack + - name: Slack - Notification + uses: Gamesight/slack-workflow-status@master + if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }} + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }} + name: GitHub Actions + icon_emoji: ":github-actions:" + channel: "C02DQ1A7JLR" # _int_git channel diff --git a/.github/workflows/workflow-release.yml b/.github/workflows/workflow-release.yml new file mode 100644 index 00000000000..5e82577868d --- /dev/null +++ b/.github/workflows/workflow-release.yml @@ -0,0 +1,61 @@ +name: Release + +on: + workflow_call: + inputs: + docker-tag: + description: "The Docker image Tag for Kestra" + type: string + required: true + plugins: + description: "The Kestra plugins to be used for the build." + type: string + required: true +jobs: + Docker: + name: Publish Docker + runs-on: ubuntu-latest + strategy: + matrix: + image: + - tag: ${{inputs.docker-tag}}-no-plugins + packages: "" + python-libraries: "" + + - tag: ${{inputs.docker-tag}} + plugins: ${{inputs.plugins}} + packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip + python-libraries: kestra + steps: + - name: Publish Docker + uses: ./.github/actions/publish-docker + with: + dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-password: ${{ secrets.DOCKERHUB_PASSWORD }} + packages: ${{ matrix.image.packages }} + plugins: ${{ matrix.image.plugins }} + python-libraries: ${{ matrix.image.python-libraries }} + tag: ${{ matrix.image.tag }} + + Maven: + name: Publish Maven + runs-on: ubuntu-latest + steps: + - name: Publish Maven + uses: ./.github/actions/publish-maven + with: + sonatype-username: ${{ secrets.SONATYPE_USER }} + sonatype-password: ${{ secrets.SONATYPE_PASSWORD }} + sonatype-gpg-keyid: ${{ secrets.SONATYPE_GPG_KEYID }} + sonatype-gpg-password: ${{ secrets.SONATYPE_GPG_PASSWORD }} + sonatype-gpg-file: ${{ secrets.SONATYPE_GPG_FILE }} + + Github: + name: Github Release + runs-on: ubuntu-latest + steps: + - name: Github Release + uses: ./.github/actions/github-release + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-personal: ${{ secrets.GH_PERSONAL_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/workflow-test.yml b/.github/workflows/workflow-test.yml index 9db3a0bf380..df843e37e7c 100644 --- a/.github/workflows/workflow-test.yml +++ b/.github/workflows/workflow-test.yml @@ -1,2 +1,90 @@ +name: Tests + on: -jobs: \ No newline at end of file + workflow_call: + +jobs: + frontend: + name: 'Check Frontend' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - id: frontend-test + name: Frontend test + uses: ./.github/actions/frontend-test + env: + # to save corepack from itself + COREPACK_INTEGRITY_KEYS: 0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + codecov-token: ${{ secrets.CODECOV_TOKEN }} + + backend: + name: 'Check Backend' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Backend test + uses: ./.github/actions/backend-test + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + sonar-token: ${{ secrets.SONAR_TOKEN }} + github-token: ${{ secrets.GH_PERSONAL_TOKEN }} + google-service-account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + build-artifacts: + name: Build artifacts + runs-on: ubuntu-latest + outputs: + docker-tag: ${{ steps.build-artifacts.outputs.docker-tag }} + docker-artifact-name: ${{ steps.build-artifacts.outputs.docker-artifact-name }} + plugins: ${{ steps.build-artifacts.outputs.plugins }} + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/build-artifacts + id: build-artifacts + + e2e: + name: Check E2E + needs: [ build-artifacts ] + uses: ./.github/workflows/e2e.yml + strategy: + fail-fast: false + matrix: + backends: [ "postgres" ] + with: + tags: oss + backend: ${{ matrix.backends }} + docker-artifact-name: ${{ needs.build-artifacts.outputs.docker-artifact-name }} + docker-image-tag: kestra/kestra:${{ needs.build-artifacts.outputs.docker-tag }} + secrets: + GITHUB_AUTH_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }} + GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} + + # Output every job status + # To be used in other workflows + report-status: + name: Report Status + runs-on: ubuntu-latest + if: always() + outputs: + frontend_status: ${{ steps.set-frontend-status.outputs.frontend_status }} + backend_status: ${{ steps.set-backend-status.outputs.backend_status }} + e2e_status: ${{ steps.set-e2e-status.outputs.e2e_status }} + steps: + - id: set-frontend-status + name: Set frontend job status + run: echo "::set-output name=frontend_status::${{ jobs.frontend.result }}" + + - id: set-backend-status + name: Set backend job status + run: echo "::set-output name=backend_status::${{ jobs.backend.result }}" + + - id: set-e2e-status + name: Set e2e job status + run: echo "::set-output name=e2e_status::${{ jobs.e2e.result }}" \ No newline at end of file