From 53afba2ec5a186284054de2d881a334a5f493021 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 15 Mar 2023 21:51:00 +0100 Subject: [PATCH] Add capability of iterating locally on constraint files for CI image (#30002) Whenever you want to upgrade a set of dependencies (for example all google packages) you might deal with a set of dependencies that are conflicting or difficult to upgrade all-at-once. And there might be multiple people working on it in parallel. This brings in the necessity of building the CI image of breeze with a different set of constraints than the ones in "main" branch of Airflow and share it with others working with you. This PR adds the capability of storing a temporary set of constraints in the repository (during PR iteration) and use this constraint file to build the CI image, rather than constraints from the repository. It also provides appropriate README.md file explaining how to iterate and regenerate such constraint files - even multiple times when iterating. --- .dockerignore | 2 +- Dockerfile.ci | 1 + constraints/README.md | 55 +++++ .../commands/ci_image_commands.py | 2 + .../commands/ci_image_commands_config.py | 1 + .../commands/production_image_commands.py | 2 + .../production_image_commands_config.py | 1 + .../airflow_breeze/utils/common_options.py | 8 + .../utils/docker_command_utils.py | 1 + docker-context-files/.README.md | 10 +- images/breeze/output-commands-hash.txt | 8 +- images/breeze/output-commands.svg | 90 ++++---- images/breeze/output_ci-image.svg | 24 +-- images/breeze/output_ci-image_build.svg | 160 +++++++------- images/breeze/output_prod-image.svg | 24 +-- images/breeze/output_prod-image_build.svg | 198 ++++++++++-------- scripts/ci/docker-compose/local.yml | 3 + scripts/in_container/_in_container_utils.sh | 1 + 18 files changed, 353 insertions(+), 238 deletions(-) create mode 100644 constraints/README.md diff --git a/.dockerignore b/.dockerignore index bf1cd0f3b5fb4..4e20efd84ac09 100644 --- a/.dockerignore +++ b/.dockerignore @@ -57,8 +57,8 @@ # This folder is for you if you want to add any packages to the docker context when you build your own # docker image. most of other files and any new folder you add will be excluded by default -# if you need other types of files - please add the extensions here. !docker-context-files +!constraints # Avoid triggering context change on README change (new companies using Airflow) # So please do not uncomment this line ;) diff --git a/Dockerfile.ci b/Dockerfile.ci index c65d45e249c78..22a5e95faa957 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1268,6 +1268,7 @@ COPY setup.py ${AIRFLOW_SOURCES}/setup.py COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ COPY generated/provider_dependencies.json ${AIRFLOW_SOURCES}/generated/ +COPY constraints/* ${AIRFLOW_SOURCES}/constraints/ COPY --from=scripts install_airflow.sh /scripts/docker/ diff --git a/constraints/README.md b/constraints/README.md new file mode 100644 index 0000000000000..1ddfc1a3dfafc --- /dev/null +++ b/constraints/README.md @@ -0,0 +1,55 @@ + + +Sometimes you might want to update dependencies of airflow in bulk, when working on related set of +packages/dependencies. This might lead to excessive time needed to rebuild the image with new versions +of dependencies (due to `pip` backtracking) and necessity to rebuild the image multiple times +and solving conflicting dependencies. + +For development use only, you can store your own version of constraint files used during CI image build +here and refer to them during building of the image using `--airflow-constraints-location constraints/constraints.txt` +This allows you to iterate on dependencies without having to run `--upgrade-to-newer-dependencies` flag continuously. + +Typical workflow in this case is: + +* download and copy the constraint file to the folder (for example via +[The GitHub Raw Link](https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.7.txt) +* modify the constraint file in "constraints" folder +* build the image using this command + +```bash +breeze ci-image build --python 3.7 --airflow-constraints-location constraints/constraints-3.7txt +``` + +You can continue iterating and updating the constraint file (and rebuilding the image) +while iterating, you can also manually add/update the dependencies and after you are done, +you can regenerate the set of constraints based on your currently installed packages, using this command: + +```bash +pip freeze | sort | \ + grep -v "apache_airflow" | \ + grep -v "apache-airflow==" | \ + grep -v "@" | \ + grep -v "/opt/airflow" > /opt/airflow/constraints/constraints-3.7.txt +``` + +If you are working with others on updating the dependencies, you can also commit the constraint +file to the repository and then anyone else working on the same branch/fork, can use it for local +image building (but the file should be removed before the branch you are working on is merged to +the main branch). diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index d37c3e313b1fd..78c8d7d850573 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -35,6 +35,7 @@ option_additional_extras, option_additional_pip_install_flags, option_additional_python_deps, + option_airflow_constraints_location, option_airflow_constraints_mode_ci, option_airflow_constraints_reference_build, option_answer, @@ -188,6 +189,7 @@ def start_building(params: BuildCiParams): @option_dev_apt_deps @option_force_build @option_python_image +@option_airflow_constraints_location @option_airflow_constraints_mode_ci @option_airflow_constraints_reference_build @option_tag_as_latest diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index 1ae0b85e9f798..5f1f9b01468eb 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -55,6 +55,7 @@ "options": [ "--builder", "--install-providers-from-sources", + "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", "--python-image", diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 66256b7e9e30c..ba644702c0e98 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -37,6 +37,7 @@ option_additional_runtime_apt_command, option_additional_runtime_apt_deps, option_additional_runtime_apt_env, + option_airflow_constraints_location, option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, option_builder, @@ -168,6 +169,7 @@ def prod_image(): @option_prepare_buildx_cache @option_push @option_empty_image +@option_airflow_constraints_location @option_airflow_constraints_mode_prod @click.option( "--installation-method", diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index 496166218323d..27b71426592b3 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -56,6 +56,7 @@ "--builder", "--install-providers-from-sources", "--airflow-extras", + "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", "--python-image", diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 1d6459b7349e4..ee517fd163c0a 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -474,6 +474,14 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, envvar="AIRFLOW_CONSTRAINTS_REFERENCE", ) +option_airflow_constraints_location = click.option( + "--airflow-constraints-location", + type=str, + default="", + help="If specified, it is used instead of calculating reference to the constraint file. " + "It could be full remote URL to the location file, or local file placed in `docker-context-files` " + "(in this case it has to start with /opt/airflow/docker-context-files).", +) option_airflow_constraints_reference_build = click.option( "--airflow-constraints-reference", default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index a08fdea87eae9..88d5912ea58db 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -86,6 +86,7 @@ ("NOTICE", "/opt/airflow/NOTICE"), ("RELEASE_NOTES.rst", "/opt/airflow/RELEASE_NOTES.rst"), ("airflow", "/opt/airflow/airflow"), + ("constraints", "/opt/airflow/constraints"), ("provider_packages", "/opt/airflow/provider_packages"), ("dags", "/opt/airflow/dags"), ("dev", "/opt/airflow/dev"), diff --git a/docker-context-files/.README.md b/docker-context-files/.README.md index 2cafbb6dabc71..6623408d79986 100644 --- a/docker-context-files/.README.md +++ b/docker-context-files/.README.md @@ -21,12 +21,12 @@ This folder is part of the Docker context. Most of other folders in Airflow are not part of the context in order to make the context smaller. -The Production [Dockerfile](../Dockerfile) and the CI one [Dockerfile.ci](../Dockerfile.ci) copies -the [docker-context-files](.) folder to the image context - in case of production image it copies it to -the build segment, co content of the folder is available in the `/docker-context-file` folder inside -the build image. You can store constraint files and wheel +The Production [Dockerfile](../Dockerfile) image copies the [docker-context-files](.) folder to the +image context forthe build segment, so content of the folder is available in the `/docker-context-files` +folder inside the build image. You can store constraint files and wheel packages there that you want to install as PYPI packages and refer to those packages using -`--constraint-location` flag for constraints or by using `--install-packages-from-context` flag. +`--airflow-constraints-location /docker-context-files/constraints.txt` flag for +constraints or by using `--install-packages-from-context` flag. By default, the content of this folder is .gitignored so that any binaries and files you put here are only used for local builds and not committed to the repository. diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 01ee85ccebb01..b56c3f38668a5 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -10,10 +10,10 @@ ci:get-workflow-info:01ee34c33ad62fa5dc33e0ac8773223f ci:resource-check:1d4fe47dff9fc64ac1648ec4beb2d85c ci:selective-check:3a085894f24cb909812fbc8253a21e13 ci:9884bf4cef3e70270068de89b9d5df2f -ci-image:build:f8d0fafe7246ca7ac412df8c520908a3 +ci-image:build:3ffe4dd24ae7090415543e27d8504955 ci-image:pull:c16c6e57c748bfe9b365b4ffafb18472 ci-image:verify:aee88f55e8837028d19316356e29b009 -ci-image:0c727b84a764e09c54e461b30ad0410b +ci-image:d53ee5067c46a6d8f3b21d0098c56ca2 cleanup:231de69d5f47ba29c883164e4575e310 compile-www-assets:c8a8c4f002f7246d0541897fc7c70313 exec:42bbd3c1659128b0341ae118c3482da2 @@ -31,10 +31,10 @@ k8s:status:2b8aa0b9597a1790c184c7b776ebd9c3 k8s:tests:dc44e907de908d1c4f11cf5d99915da1 k8s:upload-k8s-image:b012457bb6a98524859ecaf5746b4439 k8s:a04e7db8fb49c16a0633ac32f14463ec -prod-image:build:b2c87adc30320ea70d1b32faf1500959 +prod-image:build:5d0fcf0f0b1f4a8d13378253214162d6 prod-image:pull:e3c89dd908fc44adf6e159c2950ebdd0 prod-image:verify:31bc5efada1d70a0a31990025db1a093 -prod-image:eb1ef0cf6e139d01ceb26f09ca3deaaa +prod-image:2ac8e1e6b4afb259f00390ad0e90a06f release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957 release-management:generate-issue-content-providers:f3c00ba74e3afc054fe29b65156740ac diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 790b9fe55da49..784715a8cce83 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #68a0b3;font-weight: bold } -.breeze-help-r5 { fill: #868887 } +.breeze-help-r4 { fill: #868887 } +.breeze-help-r5 { fill: #68a0b3;font-weight: bold } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -190,50 +190,50 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd |    │ -│statsd | trino)                                                                          │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd |    │ +│statsd | trino)                                                                          │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image.svg b/images/breeze/output_ci-image.svg index bf1df9347a162..3025e6e470c36 100644 --- a/images/breeze/output_ci-image.svg +++ b/images/breeze/output_ci-image.svg @@ -35,8 +35,8 @@ .breeze-ci-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-ci-image-r2 { fill: #c5c8c6 } .breeze-ci-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-ci-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-ci-image-r5 { fill: #868887 } +.breeze-ci-image-r4 { fill: #868887 } +.breeze-ci-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-ci-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage CI images +Tools that developers can use to manually manage CI images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build   Build CI image. Include building multiple images for all python versions.                                 │ -│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ -│verify  Verify CI image.                                                                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build   Build CI image. Include building multiple images for all python versions.                                 │ +│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ +│verify  Verify CI image.                                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 944a6af1da53e..fb6299cb70534 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -246,9 +246,24 @@ + + + + + + + + + + + + + + + - Command: ci-image build + Command: ci-image build @@ -259,73 +274,78 @@ -Usage: breeze ci-image build [OPTIONS] +Usage: breeze ci-image build [OPTIONS] -Build CI image. Include building multiple images for all python versions. +Build CI image. Include building multiple images for all python versions. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building.                             │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for CI image building.                             │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg index 35bf4ae86bb6e..cd179a225efee 100644 --- a/images/breeze/output_prod-image.svg +++ b/images/breeze/output_prod-image.svg @@ -35,8 +35,8 @@ .breeze-prod-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-prod-image-r2 { fill: #c5c8c6 } .breeze-prod-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-prod-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-prod-image-r5 { fill: #868887 } +.breeze-prod-image-r4 { fill: #868887 } +.breeze-prod-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-prod-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage PROD images +Tools that developers can use to manually manage PROD images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ -│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ -│verify Verify Production image.                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ +│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ +│verify Verify Production image.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index 9e8df387bbad8..c3b906463ca15 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -306,9 +306,24 @@ + + + + + + + + + + + + + + + - Command: prod-image build + Command: prod-image build @@ -319,93 +334,98 @@ -Usage: breeze prod-image build [OPTIONS] +Usage: breeze prod-image build [OPTIONS] Build Production image. Include building multiple images for all or selected Python versions sequentially. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building.                           │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for PROD image building.                           │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 0d80695e75d8a..4154187f0bb41 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -69,6 +69,9 @@ services: - type: bind source: ../../../airflow target: /opt/airflow/airflow + - type: bind + source: ../../../constraints + target: /opt/airflow/constraints - type: bind source: ../../../provider_packages target: /opt/airflow/provider_packages diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 32e6b4a7e36ab..b368bb3aca2d9 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -73,6 +73,7 @@ function in_container_fix_ownership() { "${AIRFLOW_SOURCES}/docs" "${AIRFLOW_SOURCES}/dags" "${AIRFLOW_SOURCES}/airflow/" + "${AIRFLOW_SOURCES}/constraints/" "${AIRFLOW_SOURCES}/images/" "${AIRFLOW_SOURCES}/.mypy_cache/" "${AIRFLOW_SOURCES}/dev/"