From 18d1b97b79adb27609489bfed1809120c9b51993 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 24 Nov 2024 11:34:18 +0000 Subject: [PATCH] Refactor package checks to be test and remove duplication (#44318) Package checks are already "tests" not only checks since we introduced compatibility complete tests. This PR updates names of files and also removes duplication of wheel/sdist package build and verification and adding conditional steps. --- .github/workflows/ci.yml | 10 +- ...oviders.yml => test-provider-packages.yml} | 111 +++++------------- dev/breeze/doc/ci/04_selective_checks.md | 2 +- .../src/airflow_breeze/global_constants.py | 2 +- .../airflow_breeze/utils/selective_checks.py | 8 +- dev/breeze/tests/test_selective_checks.py | 8 +- 6 files changed, 46 insertions(+), 95 deletions(-) rename .github/workflows/{check-providers.yml => test-provider-packages.yml} (68%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1f46d74024c2..115d78a49913a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,8 @@ jobs: postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} - providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-string: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-string }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} @@ -323,8 +324,8 @@ jobs: docs-build: ${{ needs.build-info.outputs.docs-build }} providers: - name: "Provider checks" - uses: ./.github/workflows/check-providers.yml + name: "Provider packages tests" + uses: ./.github/workflows/test-provider-packages.yml needs: [build-info, wait-for-ci-images] permissions: contents: read @@ -340,7 +341,8 @@ jobs: default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} - providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} python-versions: ${{ needs.build-info.outputs.python-versions }} providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} diff --git a/.github/workflows/check-providers.yml b/.github/workflows/test-provider-packages.yml similarity index 68% rename from .github/workflows/check-providers.yml rename to .github/workflows/test-provider-packages.yml index a0bf2d316f82a..08715af6b58ba 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/test-provider-packages.yml @@ -44,9 +44,9 @@ on: # yamllint disable-line rule:truthy description: "List of affected providers as string" required: false type: string - providers-compatibility-checks: + providers-compatibility-tests-matrix: description: > - JSON-formatted array of providers compatibility checks in the form of array of dicts + JSON-formatted array of providers compatibility tests in the form of array of dicts (airflow-version, python-versions, remove-providers, run-tests) required: true type: string @@ -63,10 +63,14 @@ on: # yamllint disable-line rule:truthy required: true type: string jobs: - prepare-install-verify-provider-packages-wheel: + prepare-install-verify-provider-packages: timeout-minutes: 80 - name: "Provider packages wheel build and verify" + name: "Providers ${{ matrix.package-format }} tests" runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} + strategy: + fail-fast: false + matrix: + package-format: ["wheel", "sdist"] env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -94,120 +98,65 @@ jobs: run: > breeze release-management prepare-provider-documentation --include-not-ready-providers --non-interactive - - name: "Prepare provider packages: wheel" + if: matrix.package-format == 'wheel' + - name: "Prepare provider packages: ${{ matrix.package-format }}" run: > breeze release-management prepare-provider-packages --include-not-ready-providers - --version-suffix-for-pypi dev0 --package-format wheel - - name: "Prepare airflow package: wheel" - run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 - - name: "Verify wheel packages with twine" + --version-suffix-for-pypi dev0 --package-format ${{ matrix.package-format }} + - name: "Prepare airflow package: ${{ matrix.package-format }}" + run: > + breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 + --package-format ${{ matrix.package-format }} + - name: "Verify ${{ matrix.package-format }} packages with twine" run: | uv tool uninstall twine || true - uv tool install twine && twine check dist/*.whl + uv tool install twine && twine check dist/* - name: "Test providers issue generation automatically" run: > breeze release-management generate-issue-content-providers --only-available-in-dist --disable-progress + if: matrix.package-format == 'wheel' - name: Remove Python 3.9-incompatible provider packages run: | echo "Removing Python 3.9-incompatible provider: cloudant" - rm -vf dist/apache_airflow_providers_cloudant* + rm -vf dist/*cloudant* - name: "Generate source constraints from CI image" shell: bash run: > breeze release-management generate-constraints --airflow-constraints-mode constraints-source-providers --answer yes - - name: "Install and verify all provider packages and airflow via wheel files" + - name: "Install and verify wheel provider packages" run: > breeze release-management verify-provider-packages --use-packages-from-dist - --package-format wheel - --use-airflow-version wheel + --package-format ${{ matrix.package-format }} + --use-airflow-version ${{ matrix.package-format }} --airflow-constraints-reference default --providers-constraints-location /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt env: AIRFLOW_SKIP_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies }}" - - name: "Prepare airflow package: wheel without suffix and skipping the tag check" - run: > - breeze release-management prepare-provider-packages --skip-tag-check --package-format wheel - - prepare-install-provider-packages-sdist: - timeout-minutes: 80 - name: "Provider packages sdist build and install" - runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} - env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - IMAGE_TAG: "${{ inputs.image-tag }}" - INCLUDE_NOT_READY_PROVIDERS: "true" - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - VERBOSE: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: > - Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }} - uses: ./.github/actions/prepare_breeze_and_image - - name: "Cleanup dist files" - run: rm -fv ./dist/* - - name: "Prepare provider packages: sdist" - run: > - breeze release-management prepare-provider-packages --include-not-ready-providers - --version-suffix-for-pypi dev0 --package-format sdist - ${{ inputs.selected-providers-list-as-string }} - - name: "Prepare airflow package: sdist" - run: > - breeze release-management prepare-airflow-package - --version-suffix-for-pypi dev0 --package-format sdist - - name: "Verify sdist packages with twine" - run: | - uv tool uninstall twine || true - uv tool install twine && twine check dist/*.tar.gz - - name: "Generate source constraints from CI image" - shell: bash - run: > - breeze release-management generate-constraints - --airflow-constraints-mode constraints-source-providers --answer yes - - name: "Install all provider packages and airflow via sdist files" - run: > - breeze release-management install-provider-packages - --use-packages-from-dist - --package-format sdist - --use-airflow-version sdist - --airflow-constraints-reference default - --providers-constraints-location - /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt - --run-in-parallel - if: inputs.selected-providers-list-as-string == '' - - name: "Install affected provider packages and airflow via sdist files" + if: matrix.package-format == 'wheel' + - name: "Install all sdist provider packages and airflow" run: > breeze release-management install-provider-packages --use-packages-from-dist - --package-format sdist - --use-airflow-version sdist + --package-format ${{ matrix.package-format }} + --use-airflow-version ${{ matrix.package-format }} --airflow-constraints-reference default --providers-constraints-location /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt --run-in-parallel - if: inputs.selected-providers-list-as-string != '' + if: matrix.package-format == 'sdist' - providers-compatibility-checks: + providers-compatibility-tests-matrix: timeout-minutes: 80 - name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} provider check + name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} providers test runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false matrix: - include: ${{fromJSON(inputs.providers-compatibility-checks)}} + include: ${{fromJSON(inputs.providers-compatibility-tests-matrix)}} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index c39bc5df55fa5..08e5906745cbf 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -225,7 +225,7 @@ Github Actions to pass the list of parameters to a command to execute | postgres-exclude | Which versions of Postgres to exclude for tests as JSON array | [] | | | postgres-versions | Which versions of Postgres to use for tests as JSON array | \['12'\] | | | prod-image-build | Whether PROD image build is needed | true | | -| providers-compatibility-checks | List of dicts: (python_version, airflow_version, removed_providers) for compatibility checks | \[{}\] | | +| providers-compatibility-tests-matrix | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers) | \[{}\] | | | providers-test-types-list-as-string | Which test types should be run for unit tests for providers | Providers Providers\[-google\] | * | | pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | | python-versions | List of python versions to use for that build | \['3.9'\] | | diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 4bc765f255aa5..87e9195be8eb4 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -609,7 +609,7 @@ def get_airflow_extras(): CHICKEN_EGG_PROVIDERS = " ".join(["standard amazon common.sql"]) -BASE_PROVIDERS_COMPATIBILITY_CHECKS: list[dict[str, str | list[str]]] = [ +PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [ { "python-version": "3.9", "airflow-version": "2.8.4", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index c5ecef5526171..5f375e6dc7a33 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -31,7 +31,6 @@ from airflow_breeze.global_constants import ( ALL_PYTHON_MAJOR_MINOR_VERSIONS, APACHE_AIRFLOW_GITHUB_REPOSITORY, - BASE_PROVIDERS_COMPATIBILITY_CHECKS, CHICKEN_EGG_PROVIDERS, COMMITTERS, CURRENT_KUBERNETES_VERSIONS, @@ -45,6 +44,7 @@ DISABLE_TESTABLE_INTEGRATIONS_FROM_CI, HELM_VERSION, KIND_VERSION, + PROVIDERS_COMPATIBILITY_TESTS_MATRIX, RUNS_ON_PUBLIC_RUNNER, RUNS_ON_SELF_HOSTED_ASF_RUNNER, RUNS_ON_SELF_HOSTED_RUNNER, @@ -1376,12 +1376,12 @@ def chicken_egg_providers(self) -> str: return CHICKEN_EGG_PROVIDERS @cached_property - def providers_compatibility_checks(self) -> str: - """Provider compatibility input checks for the current run. Filter out python versions not built""" + def providers_compatibility_tests_matrix(self) -> str: + """Provider compatibility input matrix for the current run. Filter out python versions not built""" return json.dumps( [ check - for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS + for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX if check["python-version"] in self.python_versions ] ) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 2f00013f89f08..f57b156181fb5 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -25,9 +25,9 @@ from rich.console import Console from airflow_breeze.global_constants import ( - BASE_PROVIDERS_COMPATIBILITY_CHECKS, COMMITTERS, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, + PROVIDERS_COMPATIBILITY_TESTS_MATRIX, GithubEvents, ) from airflow_breeze.utils.packages import get_available_packages @@ -2291,10 +2291,10 @@ def test_has_migrations(files: tuple[str, ...], has_migrations: bool): pytest.param( (), { - "providers-compatibility-checks": json.dumps( + "providers-compatibility-tests-matrix": json.dumps( [ check - for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS + for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX if check["python-version"] == DEFAULT_PYTHON_MAJOR_MINOR_VERSION ] ), @@ -2303,7 +2303,7 @@ def test_has_migrations(files: tuple[str, ...], has_migrations: bool): ), pytest.param( ("all versions",), - {"providers-compatibility-checks": json.dumps(BASE_PROVIDERS_COMPATIBILITY_CHECKS)}, + {"providers-compatibility-tests-matrix": json.dumps(PROVIDERS_COMPATIBILITY_TESTS_MATRIX)}, id="full tests", ), ],