diff --git a/.dockerignore b/.dockerignore index bf1cd0f3b5fb4..4e20efd84ac09 100644 --- a/.dockerignore +++ b/.dockerignore @@ -57,8 +57,8 @@ # This folder is for you if you want to add any packages to the docker context when you build your own # docker image. most of other files and any new folder you add will be excluded by default -# if you need other types of files - please add the extensions here. !docker-context-files +!constraints # Avoid triggering context change on README change (new companies using Airflow) # So please do not uncomment this line ;) diff --git a/Dockerfile.ci b/Dockerfile.ci index c65d45e249c78..22a5e95faa957 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1268,6 +1268,7 @@ COPY setup.py ${AIRFLOW_SOURCES}/setup.py COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ COPY generated/provider_dependencies.json ${AIRFLOW_SOURCES}/generated/ +COPY constraints/* ${AIRFLOW_SOURCES}/constraints/ COPY --from=scripts install_airflow.sh /scripts/docker/ diff --git a/constraints/README.md b/constraints/README.md new file mode 100644 index 0000000000000..1ddfc1a3dfafc --- /dev/null +++ b/constraints/README.md @@ -0,0 +1,55 @@ + + +Sometimes you might want to update dependencies of airflow in bulk, when working on related set of +packages/dependencies. This might lead to excessive time needed to rebuild the image with new versions +of dependencies (due to `pip` backtracking) and necessity to rebuild the image multiple times +and solving conflicting dependencies. + +For development use only, you can store your own version of constraint files used during CI image build +here and refer to them during building of the image using `--airflow-constraints-location constraints/constraints.txt` +This allows you to iterate on dependencies without having to run `--upgrade-to-newer-dependencies` flag continuously. + +Typical workflow in this case is: + +* download and copy the constraint file to the folder (for example via +[The GitHub Raw Link](https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.7.txt) +* modify the constraint file in "constraints" folder +* build the image using this command + +```bash +breeze ci-image build --python 3.7 --airflow-constraints-location constraints/constraints-3.7txt +``` + +You can continue iterating and updating the constraint file (and rebuilding the image) +while iterating, you can also manually add/update the dependencies and after you are done, +you can regenerate the set of constraints based on your currently installed packages, using this command: + +```bash +pip freeze | sort | \ + grep -v "apache_airflow" | \ + grep -v "apache-airflow==" | \ + grep -v "@" | \ + grep -v "/opt/airflow" > /opt/airflow/constraints/constraints-3.7.txt +``` + +If you are working with others on updating the dependencies, you can also commit the constraint +file to the repository and then anyone else working on the same branch/fork, can use it for local +image building (but the file should be removed before the branch you are working on is merged to +the main branch). diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index d37c3e313b1fd..78c8d7d850573 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -35,6 +35,7 @@ option_additional_extras, option_additional_pip_install_flags, option_additional_python_deps, + option_airflow_constraints_location, option_airflow_constraints_mode_ci, option_airflow_constraints_reference_build, option_answer, @@ -188,6 +189,7 @@ def start_building(params: BuildCiParams): @option_dev_apt_deps @option_force_build @option_python_image +@option_airflow_constraints_location @option_airflow_constraints_mode_ci @option_airflow_constraints_reference_build @option_tag_as_latest diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index 1ae0b85e9f798..5f1f9b01468eb 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -55,6 +55,7 @@ "options": [ "--builder", "--install-providers-from-sources", + "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", "--python-image", diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 66256b7e9e30c..ba644702c0e98 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -37,6 +37,7 @@ option_additional_runtime_apt_command, option_additional_runtime_apt_deps, option_additional_runtime_apt_env, + option_airflow_constraints_location, option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, option_builder, @@ -168,6 +169,7 @@ def prod_image(): @option_prepare_buildx_cache @option_push @option_empty_image +@option_airflow_constraints_location @option_airflow_constraints_mode_prod @click.option( "--installation-method", diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index 496166218323d..27b71426592b3 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -56,6 +56,7 @@ "--builder", "--install-providers-from-sources", "--airflow-extras", + "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", "--python-image", diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 1d6459b7349e4..ee517fd163c0a 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -474,6 +474,14 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, envvar="AIRFLOW_CONSTRAINTS_REFERENCE", ) +option_airflow_constraints_location = click.option( + "--airflow-constraints-location", + type=str, + default="", + help="If specified, it is used instead of calculating reference to the constraint file. " + "It could be full remote URL to the location file, or local file placed in `docker-context-files` " + "(in this case it has to start with /opt/airflow/docker-context-files).", +) option_airflow_constraints_reference_build = click.option( "--airflow-constraints-reference", default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH, diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index a08fdea87eae9..88d5912ea58db 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -86,6 +86,7 @@ ("NOTICE", "/opt/airflow/NOTICE"), ("RELEASE_NOTES.rst", "/opt/airflow/RELEASE_NOTES.rst"), ("airflow", "/opt/airflow/airflow"), + ("constraints", "/opt/airflow/constraints"), ("provider_packages", "/opt/airflow/provider_packages"), ("dags", "/opt/airflow/dags"), ("dev", "/opt/airflow/dev"), diff --git a/docker-context-files/.README.md b/docker-context-files/.README.md index 2cafbb6dabc71..6623408d79986 100644 --- a/docker-context-files/.README.md +++ b/docker-context-files/.README.md @@ -21,12 +21,12 @@ This folder is part of the Docker context. Most of other folders in Airflow are not part of the context in order to make the context smaller. -The Production [Dockerfile](../Dockerfile) and the CI one [Dockerfile.ci](../Dockerfile.ci) copies -the [docker-context-files](.) folder to the image context - in case of production image it copies it to -the build segment, co content of the folder is available in the `/docker-context-file` folder inside -the build image. You can store constraint files and wheel +The Production [Dockerfile](../Dockerfile) image copies the [docker-context-files](.) folder to the +image context forthe build segment, so content of the folder is available in the `/docker-context-files` +folder inside the build image. You can store constraint files and wheel packages there that you want to install as PYPI packages and refer to those packages using -`--constraint-location` flag for constraints or by using `--install-packages-from-context` flag. +`--airflow-constraints-location /docker-context-files/constraints.txt` flag for +constraints or by using `--install-packages-from-context` flag. By default, the content of this folder is .gitignored so that any binaries and files you put here are only used for local builds and not committed to the repository. diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 01ee85ccebb01..b56c3f38668a5 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -10,10 +10,10 @@ ci:get-workflow-info:01ee34c33ad62fa5dc33e0ac8773223f ci:resource-check:1d4fe47dff9fc64ac1648ec4beb2d85c ci:selective-check:3a085894f24cb909812fbc8253a21e13 ci:9884bf4cef3e70270068de89b9d5df2f -ci-image:build:f8d0fafe7246ca7ac412df8c520908a3 +ci-image:build:3ffe4dd24ae7090415543e27d8504955 ci-image:pull:c16c6e57c748bfe9b365b4ffafb18472 ci-image:verify:aee88f55e8837028d19316356e29b009 -ci-image:0c727b84a764e09c54e461b30ad0410b +ci-image:d53ee5067c46a6d8f3b21d0098c56ca2 cleanup:231de69d5f47ba29c883164e4575e310 compile-www-assets:c8a8c4f002f7246d0541897fc7c70313 exec:42bbd3c1659128b0341ae118c3482da2 @@ -31,10 +31,10 @@ k8s:status:2b8aa0b9597a1790c184c7b776ebd9c3 k8s:tests:dc44e907de908d1c4f11cf5d99915da1 k8s:upload-k8s-image:b012457bb6a98524859ecaf5746b4439 k8s:a04e7db8fb49c16a0633ac32f14463ec -prod-image:build:b2c87adc30320ea70d1b32faf1500959 +prod-image:build:5d0fcf0f0b1f4a8d13378253214162d6 prod-image:pull:e3c89dd908fc44adf6e159c2950ebdd0 prod-image:verify:31bc5efada1d70a0a31990025db1a093 -prod-image:eb1ef0cf6e139d01ceb26f09ca3deaaa +prod-image:2ac8e1e6b4afb259f00390ad0e90a06f release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957 release-management:generate-issue-content-providers:f3c00ba74e3afc054fe29b65156740ac diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 790b9fe55da49..784715a8cce83 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #68a0b3;font-weight: bold } -.breeze-help-r5 { fill: #868887 } +.breeze-help-r4 { fill: #868887 } +.breeze-help-r5 { fill: #68a0b3;font-weight: bold } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -190,50 +190,50 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd |    │ -│statsd | trino)                                                                          │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kerberos | mongo | otel | pinot | statsd |    │ +│statsd | trino)                                                                          │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image.svg b/images/breeze/output_ci-image.svg index bf1df9347a162..3025e6e470c36 100644 --- a/images/breeze/output_ci-image.svg +++ b/images/breeze/output_ci-image.svg @@ -35,8 +35,8 @@ .breeze-ci-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-ci-image-r2 { fill: #c5c8c6 } .breeze-ci-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-ci-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-ci-image-r5 { fill: #868887 } +.breeze-ci-image-r4 { fill: #868887 } +.breeze-ci-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-ci-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage CI images +Tools that developers can use to manually manage CI images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build   Build CI image. Include building multiple images for all python versions.                                 │ -│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ -│verify  Verify CI image.                                                                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build   Build CI image. Include building multiple images for all python versions.                                 │ +│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ +│verify  Verify CI image.                                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 944a6af1da53e..fb6299cb70534 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -246,9 +246,24 @@ + + + + + + + + + + + + + + + - Command: ci-image build + Command: ci-image build @@ -259,73 +274,78 @@ -Usage: breeze ci-image build [OPTIONS] +Usage: breeze ci-image build [OPTIONS] -Build CI image. Include building multiple images for all python versions. +Build CI image. Include building multiple images for all python versions. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building.                             │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for CI image building.                             │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg index 35bf4ae86bb6e..cd179a225efee 100644 --- a/images/breeze/output_prod-image.svg +++ b/images/breeze/output_prod-image.svg @@ -35,8 +35,8 @@ .breeze-prod-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-prod-image-r2 { fill: #c5c8c6 } .breeze-prod-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-prod-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-prod-image-r5 { fill: #868887 } +.breeze-prod-image-r4 { fill: #868887 } +.breeze-prod-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-prod-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage PROD images +Tools that developers can use to manually manage PROD images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ -│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ -│verify Verify Production image.                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ +│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ +│verify Verify Production image.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index 9e8df387bbad8..c3b906463ca15 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -306,9 +306,24 @@ + + + + + + + + + + + + + + + - Command: prod-image build + Command: prod-image build @@ -319,93 +334,98 @@ -Usage: breeze prod-image build [OPTIONS] +Usage: breeze prod-image build [OPTIONS] Build Production image. Include building multiple images for all or selected Python versions sequentially. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ -│[default: apache/airflow]                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building.                           │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)│ +│[default: apache/airflow]                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for PROD image building.                           │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 0d80695e75d8a..4154187f0bb41 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -69,6 +69,9 @@ services: - type: bind source: ../../../airflow target: /opt/airflow/airflow + - type: bind + source: ../../../constraints + target: /opt/airflow/constraints - type: bind source: ../../../provider_packages target: /opt/airflow/provider_packages diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 32e6b4a7e36ab..b368bb3aca2d9 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -73,6 +73,7 @@ function in_container_fix_ownership() { "${AIRFLOW_SOURCES}/docs" "${AIRFLOW_SOURCES}/dags" "${AIRFLOW_SOURCES}/airflow/" + "${AIRFLOW_SOURCES}/constraints/" "${AIRFLOW_SOURCES}/images/" "${AIRFLOW_SOURCES}/.mypy_cache/" "${AIRFLOW_SOURCES}/dev/"