- Added parallelization for tagging images

- Reverted secrets changes - Fixed #189
spryker · Jun 27, 2023 · 6441e4d · 6441e4d
1 parent 1fb8f5c
commit 6441e4d
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 44 deletions.
diff --git a/bin/sdk/images/common.sh b/bin/sdk/images/common.sh
@@ -19,9 +19,8 @@ function Images::destroy() {
 function Images::_buildImages() {
 
     local -a sshArgument=()
-    local -a secretArgument=()
     local folder=${1}
-    local imagesToBuild="target ${2:-"application-dev cli frontend-dev gateway"}"
+    local imagesToBuild="${2:-"application-dev cli frontend-dev gateway"}"
     local applicationImage="${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}"
     local application_devImage="${SPRYKER_DOCKER_PREFIX}_run_app:${SPRYKER_DOCKER_TAG}"
     local cliImage="${SPRYKER_DOCKER_PREFIX}_run_cli:${SPRYKER_DOCKER_TAG}"
@@ -36,12 +35,21 @@ function Images::_buildImages() {
         sshArgument=('--ssh' 'default')
     fi
 
-    if [ -n "${COMPOSER_AUTH}" ]; then
-        sshArgument=('--secret' 'id=COMPOSER_AUTH')
-    fi
+    # Using temporary file for secrets as `docker secret` is only available for swarm mode.
+    function Images::_prepareSecrets() {
+        env - "${SECRETS_ENVIRONMENT[@]}" env > "${SECRETS_FILE_PATH}"
+    }
+
+    Images::_prepareSecrets
+    Registry::Trap::addExitHook 'removeBuildSecrets' "rm -f ${SECRETS_FILE_PATH}"
 
     Console::verbose "${INFO}Building images${NC}"
 
+    # Primal build including all stages to fully benefit from multistage features
+    Images::_runBuild target "${targetImage}" 1>&2
+
+    Console::verbose "${INFO}Tagging images${NC}"
+    # Tagging all the images in parallel by running the same build targeting different stages
     local IMAGES=(${imagesToBuild})
     for stage in "${IMAGES[@]}"; do
 
@@ -53,38 +61,52 @@ function Images::_buildImages() {
             exit 1
         fi
 
-        docker build \
-            -f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \
-            -t "${tag}" \
-            --target "${stage}" \
-            --progress="${PROGRESS_TYPE}" \
-            "${sshArgument[@]}" \
-            "${secretArgument[@]}" \
-            --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \
-            --build-arg "SPRYKER_SDK_REVISION=${SPRYKER_SDK_REVISION}" \
-            --build-arg "SPRYKER_DOCKER_PREFIX=${SPRYKER_DOCKER_PREFIX}" \
-            --build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \
-            --build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \
-            --build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \
-            --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \
-            --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \
-            --build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \
-            --build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \
-            --build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \
-            --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \
-            --build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \
-            --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \
-            --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \
-            --build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \
-            --build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \
-            --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \
-            --build-arg "USER_UID=${USER_FULL_ID%%:*}" \
-            . 1>&2
+        Images::_runBuild "${stage}" "${tag}" --quiet >/dev/null &
     done
 
+    wait
+
+    Registry::Trap::releaseExitHook 'removeBuildSecrets'
+
     docker rmi -f "${targetImage}" >/dev/null 2>&1 || true
 }
 
+function Images::_runBuild {
+    local stage=${1}
+    local tag=${2}
+    local quiet=$([ -n "${3}" ] && echo '--quiet' || echo '' )
+    local progress=$([ -n "${quiet}" ] && echo 'auto' || echo "${PROGRESS_TYPE}" )
+
+    docker build \
+        -f "${DEPLOYMENT_PATH}/images/${folder}/Dockerfile" \
+        -t "${tag}" \
+        --target "${stage}" \
+        --progress="${progress}" \
+        ${quiet} \
+        "${sshArgument[@]}" \
+        --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \
+        --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \
+        --build-arg "SPRYKER_SDK_REVISION=${SPRYKER_SDK_REVISION}" \
+        --build-arg "SPRYKER_DOCKER_PREFIX=${SPRYKER_DOCKER_PREFIX}" \
+        --build-arg "SPRYKER_PLATFORM_IMAGE=${SPRYKER_PLATFORM_IMAGE}" \
+        --build-arg "SPRYKER_FRONTEND_IMAGE=${SPRYKER_FRONTEND_IMAGE}" \
+        --build-arg "SPRYKER_LOG_DIRECTORY=${SPRYKER_LOG_DIRECTORY}" \
+        --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \
+        --build-arg "APPLICATION_ENV=${APPLICATION_ENV}" \
+        --build-arg "SPRYKER_COMPOSER_MODE=${SPRYKER_COMPOSER_MODE}" \
+        --build-arg "SPRYKER_COMPOSER_AUTOLOAD=${SPRYKER_COMPOSER_AUTOLOAD}" \
+        --build-arg "SPRYKER_ASSETS_MODE=${SPRYKER_ASSETS_MODE}" \
+        --build-arg "SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}" \
+        --build-arg "KNOWN_HOSTS=${KNOWN_HOSTS}" \
+        --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \
+        --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \
+        --build-arg "SPRYKER_NODE_IMAGE_VERSION=${SPRYKER_NODE_IMAGE_VERSION}" \
+        --build-arg "SPRYKER_NODE_IMAGE_DISTRO=${SPRYKER_NODE_IMAGE_DISTRO}" \
+        --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \
+        --build-arg "USER_UID=${USER_FULL_ID%%:*}" \
+        ./
+}
+
 function Images::_tagByApp() {
     local applicationName=$1
     local imageName=$2

diff --git a/generator/Dockerfile b/generator/Dockerfile
@@ -22,7 +22,7 @@ RUN chmod 755 ${srcRoot}/openssl/generate.sh
 USER root
 
 ARG USER_UID
-RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \ || true;
+RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true;
 
 USER spryker
 

diff --git a/generator/deploy-file-generator/Dockerfile b/generator/deploy-file-generator/Dockerfile
@@ -16,7 +16,7 @@ COPY --chown=spryker:spryker deploy-file-generator ${srcRoot}/deploy-file-genera
 USER root
 
 ARG USER_UID
-RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \ || true;
+RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true;
 
 USER spryker
 

diff --git a/generator/src/templates/deploy.bash.twig b/generator/src/templates/deploy.bash.twig
@@ -101,6 +101,7 @@ System::Bootstrap
 readonly PROJECT_DIR="$(pwd)"
 readonly DEPLOYMENT_DIR="$(cd "${BASH_SOURCE%/*}" >/dev/null 2>&1 && pwd)"
 readonly DEPLOYMENT_PATH="${DEPLOYMENT_DIR/$PROJECT_DIR/.}"
+readonly SECRETS_FILE_PATH="${DEPLOYMENT_DIR}/.secrets"
 readonly SPRYKER_SDK_REVISION="{{ _sdk_revision | raw }}"
 readonly PROGRESS_TYPE=${PROGRESS_TYPE:-tty}
 VERBOSE=${VERBOSE:-0}
@@ -162,6 +163,7 @@ readonly SSH_AUTH_SOCK_IN_CLI="$([ -n "${SSH_AUTH_SOCK}" ] && [ -z "${COMPOSER_A
 # Global variables
 readonly USER_FULL_ID=$(Environment::getFullUserId)
 readonly SPRYKER_XDEBUG_HOST_IP=$(Environment::getHostIp)
+readonly SECRETS_ENVIRONMENT=("COMPOSER_AUTH='${COMPOSER_AUTH}'")
 
 command=${1}
 shift || true

diff --git a/images/templates/baked/application.Dockerfile b/images/templates/baked/application.Dockerfile
@@ -13,8 +13,9 @@ USER spryker
 COPY --chown=spryker:spryker composer.json composer.lock ${srcRoot}/
 ARG SPRYKER_COMPOSER_MODE
 RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \
-  --mount=type=ssh,uid=1000  \
-  composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} \
+  --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \
+  set -o allexport && . /run/secrets/secrets-env && set +o allexport \
+  && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} \
   && find ./vendor -type d -name \.git* -prune -exec rm -rf {} +
 
 COPY --chown=spryker:spryker src ${srcRoot}/src
@@ -40,8 +41,7 @@ LABEL "spryker.sdk.revision" "${SPRYKER_SDK_REVISION}"
 
 USER spryker
 
-RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \
-  composer dump-autoload -o
+RUN composer dump-autoload -o
 
 ARG SPRYKER_PIPELINE
 ENV SPRYKER_PIPELINE=${SPRYKER_PIPELINE}
@@ -52,8 +52,7 @@ COPY --chown=spryker:spryker data ${srcRoot}/data
 RUN vendor/bin/install -r ${SPRYKER_PIPELINE} -s build -s build-production
 
 ARG SPRYKER_COMPOSER_AUTOLOAD
-RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \
-  composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD}
+RUN composer dump-autoload ${SPRYKER_COMPOSER_AUTOLOAD}
 
 COPY --chown=spryker:spryker public ${srcRoot}/public
 COPY --chown=spryker:spryker frontend ${srcRoot}/frontend

diff --git a/images/templates/baked/pipeline.Dockerfile b/images/templates/baked/pipeline.Dockerfile
@@ -12,11 +12,11 @@ USER spryker
 
 # Install dev modules for Spryker
 COPY --chown=spryker:spryker composer.json composer.lock ${srcRoot}/
-ARG SPRYKER_COMPOSER_MODE
 ARG SPRYKER_COMPOSER_AUTOLOAD
 RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \
-  --mount=type=ssh,uid=1000 \
-  composer install --no-interaction \
+  --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \
+  set -o allexport && . /run/secrets/secrets-env && set +o allexport \
+  && composer install --no-interaction \
   && find ./vendor -type d -name \.git -prune -exec rm -rf {} +
 
 # Tests contain transfer declaration

diff --git a/images/templates/mount/application.Dockerfile b/images/templates/mount/application.Dockerfile
@@ -17,7 +17,7 @@ ENV SPRYKER_DB_ENGINE=${SPRYKER_DB_ENGINE}
 ENV PATH=${srcRoot}/vendor/bin:$PATH
 
 ARG USER_UID
-RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \ || true;
+RUN usermod -u ${USER_UID} spryker && find / -user 1000 -exec chown -h spryker {} \; || true;
 
 COPY ${DEPLOYMENT_PATH}/context/php/conf.d/91-opcache-dev.ini /usr/local/etc/php/conf.d