Allow to switch easily between Bullseye and Buster debian versions (a…

…pache#21546) We are preparing to switch from Buster to Bullseye and this is the second change that is needed (following apache#21522). This change allows to choose whether we want to use Buster or Bullseye images as a base. We need to be able to choose, because: 1) we want to keep backwards compatibility and continue our users to build Buster-base images 2) we cannot yet fully switch to Bullseye because MsSQL's odbc driver does not yet support Bullseye and we reached out to mysql maintainers to learn about their plans to make the decision on when and how we are going to support Bullseye and MSSQL. Details of this discussion are in: https://github.com/MicrosoftDocs/sql-docs/issues/7255#issuecomment-1037097131 This PR adds the capability of choosing the DEBIAN_VERSION in Breeze when building images but does not yet switch from Buster to Bullseye
tnk-ysk · Feb 28, 2022 · c1abd63 · c1abd63
1 parent f5832a6
commit c1abd63
Show file tree

Hide file tree

Showing 17 changed files with 280 additions and 39 deletions.
diff --git a/.hadolint.yaml b/.hadolint.yaml
@@ -20,3 +20,4 @@ ignored:
   - DL3008
   - DL3005
   - DL3013
+  - SC1091
diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1279,6 +1279,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   -a, --install-airflow-version INSTALL_AIRFLOW_VERSION
           Uses different version of Airflow when building PROD image.
 
@@ -1486,6 +1494,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   -a, --install-airflow-version INSTALL_AIRFLOW_VERSION
           Uses different version of Airflow when building PROD image.
 
@@ -1553,6 +1569,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   -I, --production-image
           Use production image for entering the environment and builds (not for tests).
 
@@ -1627,6 +1651,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   -v, --verbose
           Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
           debugging - when you run breeze with --verbose flags you will be able to see the commands
@@ -1671,6 +1703,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  linux/amd64
 
 
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
+
   ####################################################################################################
 
 
@@ -1872,6 +1912,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   -b, --backend BACKEND
           Backend to use for tests - it determines which database is used.
           One of:
@@ -1948,6 +1996,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   -F, --force-build-images
           Forces building of the local docker images. The images are rebuilt
           automatically for the first time or when changes are detected in
@@ -2354,6 +2410,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  linux/amd64
 
+
+  -d, --debian DEBIAN_VERSION
+          Debian version used for the image. This is always name of the debian distribution version.
+
+          One of:
+
+                 buster bullseye
+
   ****************************************************************************************************
    Choose backend to run for Airflow
 

diff --git a/Dockerfile b/Dockerfile
@@ -95,7 +95,6 @@ ARG DEV_APT_DEPS="\
      lsb-release \
      nodejs \
      openssh-client \
-     python-selinux \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -119,21 +118,20 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \
     DEV_APT_COMMAND=${DEV_APT_COMMAND} \
     ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} \
     ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV}
+COPY scripts/docker/determine_debian_version_specific_variables.sh /scripts/docker/
 
-# Note missing man directories on debian-buster
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
 # Install basic and additional apt dependencies
 RUN apt-get update \
     && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
-    && apt-get install -y --no-install-recommends curl gnupg2 \
-    && mkdir -pv /usr/share/man/man1 \
-    && mkdir -pv /usr/share/man/man7 \
+    && apt-get install -y --no-install-recommends curl gnupg2 lsb-release \
     && export ${ADDITIONAL_DEV_APT_ENV?} \
+    && source /scripts/docker/determine_debian_version_specific_variables.sh \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
            ${DEV_APT_DEPS} \
+           "${DISTRO_SELINUX}" \
            ${ADDITIONAL_DEV_APT_DEPS} \
     && apt-get autoremove -yqq --purge \
     && apt-get clean \
@@ -375,7 +373,6 @@ ARG RUNTIME_APT_DEPS="\
        gosu \
        krb5-user \
        ldap-utils \
-       libffi6 \
        libldap-2.4-2 \
        libsasl2-2 \
        libsasl2-modules \
@@ -407,20 +404,20 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
     GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
     AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}
 
-# Note missing man directories on debian-buster
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+COPY scripts/docker/determine_debian_version_specific_variables.sh /scripts/docker/
+
 # Install basic and additional apt dependencies
 RUN apt-get update \
     && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
-    && apt-get install -y --no-install-recommends curl gnupg2 \
-    && mkdir -pv /usr/share/man/man1 \
-    && mkdir -pv /usr/share/man/man7 \
+    && apt-get install -y --no-install-recommends curl gnupg2 lsb-release \
     && export ${ADDITIONAL_RUNTIME_APT_ENV?} \
+    && source /scripts/docker/determine_debian_version_specific_variables.sh \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
            ${RUNTIME_APT_DEPS} \
+           "${DISTRO_LIBFFI}" \
            ${ADDITIONAL_RUNTIME_APT_DEPS} \
     && apt-get autoremove -yqq --purge \
     && apt-get clean \

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -51,13 +51,16 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
     ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
     ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
 
+COPY scripts/docker/determine_debian_version_specific_variables.sh /scripts/docker/
+
 # Install basic and additional apt dependencies
 RUN apt-get update \
     && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
-    && apt-get install -y --no-install-recommends curl gnupg2 \
+    && apt-get install -y --no-install-recommends curl gnupg2 lsb-release \
     && mkdir -pv /usr/share/man/man1 \
     && mkdir -pv /usr/share/man/man7 \
     && export ${ADDITIONAL_DEV_ENV_VARS?} \
+    && source /scripts/docker/determine_debian_version_specific_variables.sh \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" \
     && apt-get update \
@@ -79,7 +82,7 @@ RUN apt-get update \
            libsasl2-dev \
            libsasl2-modules \
            libssl-dev \
-           libenchant-dev \
+           "${DISTRO_LIBENCHANT}" \
            locales  \
            netcat \
            nodejs \
@@ -112,7 +115,6 @@ RUN bash /scripts/docker/install_mysql.sh prod \
     && chmod 0440 /etc/sudoers.d/airflow
 
 ARG RUNTIME_APT_DEPS="\
-      libgcc-8-dev \
       apt-transport-https \
       bash-completion \
       ca-certificates \
@@ -160,16 +162,14 @@ ENV RUNTIME_APT_DEP=${RUNTIME_APT_DEPS} \
     AIRFLOW_HOME=${AIRFLOW_HOME} \
     AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
 
-# Note missing man directories on debian-buster
-# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
-RUN mkdir -pv /usr/share/man/man1 \
-    && mkdir -pv /usr/share/man/man7 \
-    && export ${ADDITIONAL_DEV_APT_ENV?} \
+RUN export ${ADDITIONAL_DEV_APT_ENV?} \
     && export ${ADDITIONAL_RUNTIME_APT_ENV?} \
+    && source /scripts/docker/determine_debian_version_specific_variables.sh \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \
     && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
+      "${DISTRO_LIBGCC}" \
       ${RUNTIME_APT_DEPS} \
       ${ADDITIONAL_RUNTIME_APT_DEPS} \
     && apt-get autoremove -yqq --purge \

diff --git a/INSTALL b/INSTALL
@@ -5,7 +5,7 @@ This is a generic installation method that requires a number of dependencies to
 Depending on your system you might need different prerequisites, but the following
 systems/prerequisites are known to work:
 
-Linux (Debian Buster and Linux Mint Tricia):
+Linux (Debian Bullseye and Linux Mint Debbie):
 
 sudo apt install build-essential python3-dev libsqlite3-dev openssl \
                  sqlite default-libmysqlclient-dev libmysqlclient-dev postgresql

diff --git a/breeze b/breeze
@@ -515,6 +515,7 @@ EOF
         fi
         cat <<EOF
                                Python version:         ${PYTHON_MAJOR_MINOR_VERSION}
+                               Debian version:         ${DEBIAN_VERSION}
                                Backend:                ${BACKEND} ${backend_version}
 EOF
         if [[ -n ${USE_AIRFLOW_VERSION=} ]]; then
@@ -629,6 +630,7 @@ export HOST_USER_ID=${HOST_USER_ID}
 export HOST_GROUP_ID=${HOST_GROUP_ID}
 export COMPOSE_FILE="${compose_file}"
 export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}"
+export DEBIAN_VERSION="${DEBIAN_VERSION}"
 export BACKEND="${BACKEND}"
 export AIRFLOW_VERSION="${AIRFLOW_VERSION}"
 export INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}"
@@ -679,14 +681,20 @@ EOF
 #######################################################################################################
 function breeze::prepare_command_files() {
     local main_ci_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
-    local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
+    if [[ ${BACKEND} == "mssql" ]]; then
+        local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}-${DEBIAN_VERSION}.yml
+    else
+        local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
+    fi
+
     local backend_port_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}-port.yml
     local local_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local.yml
     local local_all_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local-all-sources.yml
     local files_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/files.yml
     local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
     local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
 
+
     if [[ ${BACKEND} == "mssql" ]]; then
         local docker_filesystem
         docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker 2>/dev/null || echo "unknown")
@@ -816,6 +824,12 @@ function breeze::parse_arguments() {
             echo
             shift 2
             ;;
+        -d | --debian-version)
+            export DEBIAN_VERSION="${2}"
+            echo "Debian version: ${DEBIAN_VERSION}"
+            echo
+            shift 2
+            ;;
         -b | --backend)
             export BACKEND="${2}"
             echo "Backend: ${BACKEND}"
@@ -940,7 +954,7 @@ function breeze::parse_arguments() {
             echo
             shift 2
             ;;
-        -d | --db-reset)
+        --db-reset)
             echo "Resetting the DB!"
             echo
             export DB_RESET="true"
@@ -1544,6 +1558,10 @@ function breeze::prepare_formatted_versions() {
         fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
     readonly FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS
 
+    FORMATTED_DEBIAN_VERSIONS=$(echo "${_breeze_allowed_debian_versions=""}" | tr '\n' ' ' |
+        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
+    readonly FORMATTED_DEBIAN_VERSIONS
+
     FORMATTED_BACKENDS=$(echo "${_breeze_allowed_backends=""}" | tr '\n' ' ' |
         fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
     readonly FORMATTED_BACKENDS
@@ -2260,6 +2278,14 @@ ${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS}
 
 ${FORMATTED_PLATFORMS}
 
+
+-d, --debian DEBIAN_VERSION
+        Debian version used for the image. This is always name of the debian distribution version.
+
+        One of:
+
+${FORMATTED_DEBIAN_VERSIONS}
+
 "
 }
 

diff --git a/breeze-complete b/breeze-complete
@@ -23,6 +23,7 @@
 # Those cannot be made read-only as the breeze-complete must be re-sourceable
 
 _breeze_allowed_python_major_minor_versions="3.7 3.8 3.9"
+_breeze_allowed_debian_versions="buster bullseye"
 _breeze_allowed_backends="sqlite mysql postgres mssql"
 _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq redis statsd trino all"
 _breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers"
@@ -43,6 +44,7 @@ _breeze_allowed_platforms="linux/amd64"
 # shellcheck disable=SC2034
 {
     # Default values for the commands & flags used
+    _breeze_default_debian_version=$(echo "${_breeze_allowed_debian_versions}" | awk '{print $1}')
     _breeze_default_backend=$(echo "${_breeze_allowed_backends}" | awk '{print $1}')
     _breeze_default_generate_providers_mode=$(echo "${_breeze_allowed_generate_constraints_modes}" | awk '{print $1}')
     _breeze_default_kubernetes_mode=$(echo "${_breeze_allowed_kubernetes_modes}" | awk '{print $1}')
@@ -166,7 +168,7 @@ _breeze_default_github_repository="apache/airflow"
 _breeze_default_github_image_id="latest"
 
 _breeze_short_options="
-h p: b: i:
+h p: d: b: i:
 K: V:
 l a: t: d:
 v y n q f
@@ -177,7 +179,7 @@ S: N:
 "
 
 _breeze_long_options="
-help python: backend: integration: platform:
+help python: debian-version: backend: integration: platform:
 kubernetes-mode: kubernetes-version: helm-version: kind-version:
 skip-mounting-local-sources mount-all-local-sources install-airflow-version: install-airflow-reference: db-reset
 verbose assume-yes assume-no assume-quit forward-credentials init-script:
@@ -256,6 +258,9 @@ function breeze_complete::get_known_values_breeze() {
     -p | --python)
         _breeze_known_values=${_breeze_allowed_python_major_minor_versions}
         ;;
+    -d | --debian-version)
+        _breeze_known_values=${_breeze_allowed_debian_versions}
+        ;;
     -b | --backend)
         _breeze_known_values=${_breeze_allowed_backends}
         ;;

diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
@@ -1039,7 +1039,7 @@ def make_sure_remote_apache_exists_and_fetch(git_update: bool, verbose: bool):
         if not git_update:
             return
     except subprocess.CalledProcessError as ex:
-        if ex.returncode == 128:
+        if ex.returncode == 128 or ex.returncode == 2:
             remote_add_command = [
                 "git",
                 "remote",
-Original file line number
+Diff line change
@@ Expand Up / @@ -20,3 +20,4 @@ ignored: @@
       - DL3008
       - DL3005
       - DL3013
+      - SC1091