From ec3f64d80f49872bc2da9457d688a960f78df2fe Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Fri, 10 Nov 2023 14:43:07 -0600 Subject: [PATCH 01/22] Splitting Dockerfile to use cached layer. * Adding Dockerfile install deps using Alpine Node. * Switching to docker-compose file to maximize caching. --- .../docker-ci/{Dockerfile => Dockerfile.app} | 0 scripts/docker-ci/Dockerfile.deps | 16 ++++++++++++++++ scripts/docker-ci/docker-compose.yml | 17 +++++++++++++++++ 3 files changed, 33 insertions(+) rename scripts/docker-ci/{Dockerfile => Dockerfile.app} (100%) create mode 100644 scripts/docker-ci/Dockerfile.deps create mode 100644 scripts/docker-ci/docker-compose.yml diff --git a/scripts/docker-ci/Dockerfile b/scripts/docker-ci/Dockerfile.app similarity index 100% rename from scripts/docker-ci/Dockerfile rename to scripts/docker-ci/Dockerfile.app diff --git a/scripts/docker-ci/Dockerfile.deps b/scripts/docker-ci/Dockerfile.deps new file mode 100644 index 00000000000..193d9b9d05c --- /dev/null +++ b/scripts/docker-ci/Dockerfile.deps @@ -0,0 +1,16 @@ +FROM node:18.18.2-alpine as base + +WORKDIR /app + +COPY package.json /app/ +COPY yarn.lock /app/ + +RUN apk --no-cache add yarn && yarn + +FROM node:18.18.2-alpine as dependencies + +WORKDIR /app + +COPY --from=base app/package.json /app/ +COPY --from=base app/yarn.lock /app/ +COPY --from=base app/node_modules /app/node_modules/ diff --git a/scripts/docker-ci/docker-compose.yml b/scripts/docker-ci/docker-compose.yml new file mode 100644 index 00000000000..efca8087bc5 --- /dev/null +++ b/scripts/docker-ci/docker-compose.yml @@ -0,0 +1,17 @@ +services: + dependencies: + build: + dockerfile: ./Dockerfile.deps + volumes: + - '.:/app' + - '/app/node_modules' + + app: + image: container-registry-test.elastic.co/eui/ci + build: + dockerfile: ./Dockerfile.app + working_dir: /app + volumes_from: + - "dependencies" + depends_on: + - "dependencies" From 22104b3f9b334a880b96caa2f39e46c8992d7f47 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Fri, 10 Nov 2023 14:50:02 -0600 Subject: [PATCH 02/22] Adding Docker build and push script for CI. --- .buildkite/scripts/pipeline_docker_build.sh | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .buildkite/scripts/pipeline_docker_build.sh diff --git a/.buildkite/scripts/pipeline_docker_build.sh b/.buildkite/scripts/pipeline_docker_build.sh new file mode 100644 index 00000000000..f55b5529396 --- /dev/null +++ b/.buildkite/scripts/pipeline_docker_build.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -euo pipefail +set +x + +function retry { + local retries=$1 + shift + + local count=0 + + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count of $retries exited $exit. Retrying in $wait seconds." + else + >&2 echo "Retry $count of $retries exited $exit. No retries left." + return $exit + fi + + done + return 0 +} + +DOCKER_MACHINE_STAGING=secret/ci/elastic-eui/docker-machine-staging + +echo ":docker: Get Docker Hub credentials & authenticate" +username=$(retry 5 vault read -field=username $DOCKER_MACHINE_STAGING) +password=$(retry 5 vault read -field=password $DOCKER_MACHINE_STAGING) +if [[ -z "${username}" ]] || [[ -z "${password}" ]]; then + echo ":fire: Docker credentials not set." 1>&2 + exit 1 +fi +docker login -u "${username}" -p "${password}" container-registry-test.elastic.co + +echo ":docker: Build and push container" +docker-compose -f ../../scripts/docker-ci/docker-compose.yml build app +docker push container-registry-test.elastic.co/eui/ci:latest + +echo ":docker: Log out of Docker Hub" +docker logout + +# Unset all the thingz +unset DOCKER_MACHINE username password From ab49dd8e98a4d3471b4f28b6d27408d88c038a79 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Fri, 10 Nov 2023 17:07:26 -0600 Subject: [PATCH 03/22] Refactor pipeline to cache container then run tests. --- .../pipelines/pipeline_pull_request_test.yml | 75 +++++++++++++------ 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index 418acd54bb5..d52283179ed 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -1,50 +1,63 @@ # 🏠/.buildkite/pipelines/pipeline_pull_request_test.yml steps: - - command: .buildkite/scripts/pipeline_test.sh - label: ":typescript: Linting" + - label ":docker: Cached container" + key: "build-test-container" + command: ./buildkite/scripts/pipeline_docker_build.sh agents: provider: "gcp" + if: build.branch != "main" # This job is triggered by the combined test and deploy docs for every PR + + - label: ":typescript: Linting" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" # The depends_on creates an implicit "wait" while the container builds + agents: + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'lint' - if: build.branch != "main" # This job is triggered by the combined test and deploy docs for every PR + if: build.branch != "main" - - command: .buildkite/scripts/pipeline_test.sh - label: ":jest: TS unit tests" + - label: ":jest: TS unit tests" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:ts' if: build.branch != "main" - - command: .buildkite/scripts/pipeline_test.sh - label: ":jest: TSX unit tests on React 16" + - label: ":jest: TSX unit tests on React 16" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:tsx:16' if: build.branch != "main" - - command: .buildkite/scripts/pipeline_test.sh - label: ":jest: TSX unit tests on React 17" + - label: ":jest: TSX unit tests on React 17" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:tsx:17' if: build.branch != "main" - - command: .buildkite/scripts/pipeline_test.sh - label: ":jest: TSX unit tests on React 18" + - label: ":jest: TSX unit tests on React 18" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:tsx' if: build.branch != "main" - - command: .buildkite/scripts/pipeline_test.sh - label: ":cypress: Cypress tests on React 16" + - label: ":cypress: Cypress tests on React 16" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:16' if: build.branch != "main" @@ -52,10 +65,11 @@ steps: - "cypress/screenshots/**/*.png" - "cypress/videos/**/*.mp4" - - command: .buildkite/scripts/pipeline_test.sh - label: ":cypress: Cypress tests on React 17" + - label: ":cypress: Cypress tests on React 17" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:17' if: build.branch != "main" @@ -63,13 +77,26 @@ steps: - "cypress/screenshots/**/*.png" - "cypress/videos/**/*.mp4" - - command: .buildkite/scripts/pipeline_test.sh - label: ":cypress: Cypress tests on React 18" + - label: ":cypress: Cypress tests on React 18" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" agents: - provider: "gcp" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:18' if: build.branch != "main" artifact_paths: - "cypress/screenshots/**/*.png" - "cypress/videos/**/*.mp4" + + - label: ":axe: Cypress accessibility (a11y) tests on React 18" + command: .buildkite/scripts/pipeline_test.sh + depends_on: "build-test-container" + agents: + image: "container-registry-test.elastic.co/eui/ci:latest" + env: + TEST_TYPE: 'cypress:a11y' + if: build.branch != "main" + artifact_paths: + - "cypress/screenshots/**/*.png" + - "cypress/videos/**/*.mp4" From c10bf65d653f3fd71735173632d3dabc7c5de7a4 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Fri, 10 Nov 2023 17:08:50 -0600 Subject: [PATCH 04/22] Simplified shell test commands with yarn deps cached. --- .buildkite/scripts/pipeline_test.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/pipeline_test.sh b/.buildkite/scripts/pipeline_test.sh index 7ff953c49ca..7114846e8cc 100644 --- a/.buildkite/scripts/pipeline_test.sh +++ b/.buildkite/scripts/pipeline_test.sh @@ -16,42 +16,47 @@ DOCKER_OPTIONS=( case $TEST_TYPE in lint) echo "[TASK]: Running linters" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn lint") + DOCKER_OPTIONS+=(bash -c "yarn lint") ;; unit:ts) echo "[TASK]: Running .ts and .js unit tests" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react") + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react") ;; unit:tsx:16) echo "[TASK]: Running Jest .tsx tests against React 16" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react") + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react") ;; unit:tsx:17) echo "[TASK]: Running Jest .tsx tests against React 17" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react") + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react") ;; unit:tsx) echo "[TASK]: Running Jest .tsx tests against React 18" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react") + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react") ;; cypress:16) echo "[TASK]: Running Cypress tests against React 16" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16") + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16") ;; cypress:17) echo "[TASK]: Running Cypress tests against React 17" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17") + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17") ;; cypress:18) echo "[TASK]: Running Cypress tests against React 18" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048") + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048") + ;; + + cypress:a11y) + echo "[TASK]: Running Cypress accessibility tests against React 18" + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048") ;; *) From 2ef2e902e387c00fd1d418d8b0291000d375b8e5 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Mon, 13 Nov 2023 10:05:36 -0600 Subject: [PATCH 05/22] Commented out a11y check. Refactored Docker login routine. * Ignoring Cypress a11y routine until test fixes are added. * Changed to docker-login --password-stdin. * Updated Docker image URL in test shell script. --- .../pipelines/pipeline_pull_request_test.yml | 23 ++++++++++--------- .buildkite/scripts/pipeline_docker_build.sh | 11 ++++----- .buildkite/scripts/pipeline_test.sh | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index d52283179ed..5bde319d7c8 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -89,14 +89,15 @@ steps: - "cypress/screenshots/**/*.png" - "cypress/videos/**/*.mp4" - - label: ":axe: Cypress accessibility (a11y) tests on React 18" - command: .buildkite/scripts/pipeline_test.sh - depends_on: "build-test-container" - agents: - image: "container-registry-test.elastic.co/eui/ci:latest" - env: - TEST_TYPE: 'cypress:a11y' - if: build.branch != "main" - artifact_paths: - - "cypress/screenshots/**/*.png" - - "cypress/videos/**/*.mp4" + # TODO: Enable this block after PR #7354 is merged + # - label: ":axe: Cypress accessibility (a11y) tests on React 18" + # command: .buildkite/scripts/pipeline_test.sh + # depends_on: "build-test-container" + # agents: + # image: "container-registry-test.elastic.co/eui/ci:latest" + # env: + # TEST_TYPE: 'cypress:a11y' + # if: build.branch != "main" + # artifact_paths: + # - "cypress/screenshots/**/*.png" + # - "cypress/videos/**/*.mp4" diff --git a/.buildkite/scripts/pipeline_docker_build.sh b/.buildkite/scripts/pipeline_docker_build.sh index f55b5529396..ebfde042543 100644 --- a/.buildkite/scripts/pipeline_docker_build.sh +++ b/.buildkite/scripts/pipeline_docker_build.sh @@ -27,21 +27,20 @@ function retry { DOCKER_MACHINE_STAGING=secret/ci/elastic-eui/docker-machine-staging -echo ":docker: Get Docker Hub credentials & authenticate" +echo "[DOCKER]: Get registry credentials & login" username=$(retry 5 vault read -field=username $DOCKER_MACHINE_STAGING) password=$(retry 5 vault read -field=password $DOCKER_MACHINE_STAGING) if [[ -z "${username}" ]] || [[ -z "${password}" ]]; then echo ":fire: Docker credentials not set." 1>&2 exit 1 fi -docker login -u "${username}" -p "${password}" container-registry-test.elastic.co +echo "${password}" | docker login -u "${username}" --password-stdin container-registry-test.elastic.co -echo ":docker: Build and push container" -docker-compose -f ../../scripts/docker-ci/docker-compose.yml build app +echo "[DOCKER]: Build and push container to registry" +docker-compose -f ./scripts/docker-ci/docker-compose.yml build app docker push container-registry-test.elastic.co/eui/ci:latest -echo ":docker: Log out of Docker Hub" +echo "[DOCKER]: Log out of registry" docker logout -# Unset all the thingz unset DOCKER_MACHINE username password diff --git a/.buildkite/scripts/pipeline_test.sh b/.buildkite/scripts/pipeline_test.sh index 7114846e8cc..a07d2501e3f 100644 --- a/.buildkite/scripts/pipeline_test.sh +++ b/.buildkite/scripts/pipeline_test.sh @@ -10,7 +10,7 @@ DOCKER_OPTIONS=( --user="$(id -u):$(id -g)" --volume="$(pwd):/app" --workdir=/app - docker.elastic.co/eui/ci:5.6 + container-registry-test.elastic.co/eui/ci:latest ) case $TEST_TYPE in From 39f604b2669b44625427e2d06528395e71d62b0e Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Wed, 15 Nov 2023 17:12:26 -0600 Subject: [PATCH 06/22] Restructured env variables and shell script paths. --- .buildkite/scripts/lifecycle/post_command.sh | 2 +- .buildkite/scripts/lifecycle/pre_command.sh | 7 +++ .buildkite/scripts/pipeline_docker_build.sh | 46 ------------------- .../pipelines/pipeline_docker_build.sh | 18 ++++++++ 4 files changed, 26 insertions(+), 47 deletions(-) delete mode 100644 .buildkite/scripts/pipeline_docker_build.sh create mode 100644 .buildkite/scripts/pipelines/pipeline_docker_build.sh diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index 5b5186422c1..76e9f132c0c 100644 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -3,4 +3,4 @@ set -euo pipefail set +x -unset GITHUB_ACCOUNT VAULT_ACCOUNT +unset DOCKER_MACHINE DOCKER_STAGING_PASSWORD DOCKER_STAGING_USERNAME GCE_ACCOUNT GITHUB_ACCOUNT GITHUB_TOKEN GPROJECT VAULT_ACCOUNT diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 9dbaffd6dcd..2e875d7c65e 100644 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -7,10 +7,17 @@ echo '[SOURCE]: Buildkite dependencies' source .buildkite/scripts/common/utils.sh echo '[INSTALL]: Non-exported variables' +DOCKER_MACHINE_STAGING=secret/ci/elastic-eui/docker-machine-staging GITHUB_ACCOUNT=secret/ci/elastic-eui/kibanamachine VAULT_ACCOUNT=secret/ci/elastic-eui/bekitzur-kibana-service-account echo '[INSTALL]: Exported variables' +DOCKER_STAGING_USERNAME=$(retry 5 vault read -field=username $DOCKER_MACHINE_STAGING) +export DOCKER_STAGING_USERNAME + +DOCKER_STAGING_PASSWORD=$(retry 5 vault read -field=password $DOCKER_MACHINE_STAGING) +export DOCKER_STAGING_PASSWORD + GCE_ACCOUNT=$(retry 5 vault read -field=value $VAULT_ACCOUNT) export GCE_ACCOUNT diff --git a/.buildkite/scripts/pipeline_docker_build.sh b/.buildkite/scripts/pipeline_docker_build.sh deleted file mode 100644 index ebfde042543..00000000000 --- a/.buildkite/scripts/pipeline_docker_build.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -set -euo pipefail -set +x - -function retry { - local retries=$1 - shift - - local count=0 - - until "$@"; do - exit=$? - wait=$((2 ** count)) - count=$((count + 1)) - - if [ $count -lt "$retries" ]; then - >&2 echo "Retry $count of $retries exited $exit. Retrying in $wait seconds." - else - >&2 echo "Retry $count of $retries exited $exit. No retries left." - return $exit - fi - - done - return 0 -} - -DOCKER_MACHINE_STAGING=secret/ci/elastic-eui/docker-machine-staging - -echo "[DOCKER]: Get registry credentials & login" -username=$(retry 5 vault read -field=username $DOCKER_MACHINE_STAGING) -password=$(retry 5 vault read -field=password $DOCKER_MACHINE_STAGING) -if [[ -z "${username}" ]] || [[ -z "${password}" ]]; then - echo ":fire: Docker credentials not set." 1>&2 - exit 1 -fi -echo "${password}" | docker login -u "${username}" --password-stdin container-registry-test.elastic.co - -echo "[DOCKER]: Build and push container to registry" -docker-compose -f ./scripts/docker-ci/docker-compose.yml build app -docker push container-registry-test.elastic.co/eui/ci:latest - -echo "[DOCKER]: Log out of registry" -docker logout - -unset DOCKER_MACHINE username password diff --git a/.buildkite/scripts/pipelines/pipeline_docker_build.sh b/.buildkite/scripts/pipelines/pipeline_docker_build.sh new file mode 100644 index 00000000000..0a6c8494174 --- /dev/null +++ b/.buildkite/scripts/pipelines/pipeline_docker_build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -euo pipefail +set +x + +echo "[DOCKER]: Get staging registry credentials & login" +if [[ -z "${DOCKER_STAGING_USERNAME}" ]] || [[ -z "${DOCKER_STAGING_PASSWORD}" ]]; then + echo ":fire: Docker credentials not set." 1>&2 + exit 1 +fi +echo "${DOCKER_STAGING_PASSWORD}" | docker login -u "${DOCKER_STAGING_USERNAME}" --password-stdin container-registry-test.elastic.co + +echo "[DOCKER]: Build and push container to staging registry" +docker-compose -f ./scripts/docker-ci/docker-compose.yml build app +docker push container-registry-test.elastic.co/eui/ci:latest + +echo "[DOCKER]: Log out of staging registry" +docker logout From 988404171e8619ef1615def697e37c5a0120476a Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 09:20:01 -0600 Subject: [PATCH 07/22] Changing the test image name to be more clear about its purpose. --- .buildkite/scripts/pipelines/pipeline_test.sh | 3 ++- scripts/docker-ci/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index b7d3d5b5b5a..2c482d15760 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -10,7 +10,8 @@ DOCKER_OPTIONS=( --user="$(id -u):$(id -g)" --volume="$(pwd):/app" --workdir=/app - container-registry-test.elastic.co/eui/ci:latest + --platform=linux/amd64 + "${DOCKER_STAGING_IMAGE}":latest ) case $TEST_TYPE in diff --git a/scripts/docker-ci/docker-compose.yml b/scripts/docker-ci/docker-compose.yml index efca8087bc5..6da5e821a97 100644 --- a/scripts/docker-ci/docker-compose.yml +++ b/scripts/docker-ci/docker-compose.yml @@ -7,7 +7,7 @@ services: - '/app/node_modules' app: - image: container-registry-test.elastic.co/eui/ci + image: docker-staging/eui/ci build: dockerfile: ./Dockerfile.app working_dir: /app From 0962cafef064b683892d31799a02fc0edfa018f9 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 09:21:09 -0600 Subject: [PATCH 08/22] Refactored registry into secrets, removed a few hardcoded strings. --- .buildkite/scripts/lifecycle/post_command.sh | 2 +- .buildkite/scripts/lifecycle/pre_command.sh | 18 ++++++++++++++++-- .../scripts/pipelines/pipeline_docker_build.sh | 12 ++++++------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index 76e9f132c0c..7e6383ce309 100644 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -3,4 +3,4 @@ set -euo pipefail set +x -unset DOCKER_MACHINE DOCKER_STAGING_PASSWORD DOCKER_STAGING_USERNAME GCE_ACCOUNT GITHUB_ACCOUNT GITHUB_TOKEN GPROJECT VAULT_ACCOUNT +unset DOCKER_MACHINE DOCKER_STAGING_IMAGE DOCKER_STAGING_PASSWORD DOCKER_STAGING_REGISTRY DOCKER_STAGING_USERNAME GCE_ACCOUNT GITHUB_ACCOUNT GITHUB_TOKEN GPROJECT VAULT_ACCOUNT diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 2e875d7c65e..2311cec1fdc 100644 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -6,23 +6,37 @@ set +x echo '[SOURCE]: Buildkite dependencies' source .buildkite/scripts/common/utils.sh -echo '[INSTALL]: Non-exported variables' +echo '[INSTALL]: env variables' +# ======================================== # +# Paths +# ======================================== # DOCKER_MACHINE_STAGING=secret/ci/elastic-eui/docker-machine-staging GITHUB_ACCOUNT=secret/ci/elastic-eui/kibanamachine VAULT_ACCOUNT=secret/ci/elastic-eui/bekitzur-kibana-service-account -echo '[INSTALL]: Exported variables' +# ======================================== # +# Secrets +# ======================================== # DOCKER_STAGING_USERNAME=$(retry 5 vault read -field=username $DOCKER_MACHINE_STAGING) export DOCKER_STAGING_USERNAME DOCKER_STAGING_PASSWORD=$(retry 5 vault read -field=password $DOCKER_MACHINE_STAGING) export DOCKER_STAGING_PASSWORD +DOCKER_STAGING_REGISTRY=$(retry 5 vault read -field=registry $DOCKER_MACHINE_STAGING) +export DOCKER_STAGING_REGISTRY + GCE_ACCOUNT=$(retry 5 vault read -field=value $VAULT_ACCOUNT) export GCE_ACCOUNT GITHUB_TOKEN=$(retry 5 vault read -field=github_token $GITHUB_ACCOUNT) export GITHUB_TOKEN +# ======================================== # +# Known values +# ======================================== # +DOCKER_STAGING_IMAGE=docker-staging/eui/ci +export DOCKER_STAGING_IMAGE + GPROJECT=elastic-bekitzur export GPROJECT diff --git a/.buildkite/scripts/pipelines/pipeline_docker_build.sh b/.buildkite/scripts/pipelines/pipeline_docker_build.sh index 0a6c8494174..970f8e467c9 100644 --- a/.buildkite/scripts/pipelines/pipeline_docker_build.sh +++ b/.buildkite/scripts/pipelines/pipeline_docker_build.sh @@ -3,16 +3,16 @@ set -euo pipefail set +x -echo "[DOCKER]: Get staging registry credentials & login" -if [[ -z "${DOCKER_STAGING_USERNAME}" ]] || [[ -z "${DOCKER_STAGING_PASSWORD}" ]]; then +echo "[DOCKER]: Get registry credentials & login" +if [[ -z "${DOCKER_STAGING_USERNAME}" ]] || [[ -z "${DOCKER_STAGING_PASSWORD}" ]] || [[ -z "${DOCKER_STAGING_REGISTRY}" ]]; then echo ":fire: Docker credentials not set." 1>&2 exit 1 fi -echo "${DOCKER_STAGING_PASSWORD}" | docker login -u "${DOCKER_STAGING_USERNAME}" --password-stdin container-registry-test.elastic.co +echo "${DOCKER_STAGING_PASSWORD}" | docker login -u "${DOCKER_STAGING_USERNAME}" --password-stdin "${DOCKER_STAGING_REGISTRY}" -echo "[DOCKER]: Build and push container to staging registry" +echo "[DOCKER]: Build and push container to registry" docker-compose -f ./scripts/docker-ci/docker-compose.yml build app -docker push container-registry-test.elastic.co/eui/ci:latest +docker push "${DOCKER_STAGING_IMAGE}":latest -echo "[DOCKER]: Log out of staging registry" +echo "[DOCKER]: Log out of registry" docker logout From bcd152181bf11d639acefcfa772ba7f72c25cb31 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 09:54:23 -0600 Subject: [PATCH 09/22] Fixed an incorrect yarn command on a11y test invocation. --- .buildkite/scripts/pipelines/pipeline_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index 2c482d15760..40b80d08e9a 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -62,7 +62,7 @@ case $TEST_TYPE in cypress:a11y) echo "[TASK]: Running Cypress accessibility tests against React 18" - DOCKER_OPTIONS+=(bash -c "/opt/yarn*/bin/yarn && yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048") + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048") ;; *) From d1449b3eea86e4057d25fe6ed227b5eea2b5a476 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 09:56:48 -0600 Subject: [PATCH 10/22] Updating local Docker tests. --- scripts/test-a11y-docker.js | 5 ++--- scripts/test-docker.js | 11 +++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/test-a11y-docker.js b/scripts/test-a11y-docker.js index 65cc57244bf..c366113a710 100644 --- a/scripts/test-a11y-docker.js +++ b/scripts/test-a11y-docker.js @@ -10,9 +10,8 @@ execSync( -e GIT_COMMITTER_NAME=test -e GIT_COMMITTER_EMAIL=test -e HOME=/tmp \ --user=$(id -u):$(id -g) \ docker.elastic.co/eui/ci:5.6 \ - bash -c '/opt/yarn*/bin/yarn \ - && yarn cypress install \ - && yarn run test-cypress-a11y \ + bash -c 'yarn cypress install \ + && yarn test-cypress-a11y \ --node-options=--max_old_space_size=2048 \ --skip-css '", // Skipping CSS because compiling has a tendency to hang on Apple Silicon { diff --git a/scripts/test-docker.js b/scripts/test-docker.js index 2d90213c0b6..443fefac8ea 100644 --- a/scripts/test-docker.js +++ b/scripts/test-docker.js @@ -5,15 +5,14 @@ execSync('docker pull docker.elastic.co/eui/ci:5.6', { }); /* eslint-disable-next-line no-multi-str */ execSync( - 'docker run \ - -i --rm --cap-add=SYS_ADMIN --volume=$(pwd):/app --workdir=/app \ + "docker run \ + -i --rm --cap-add=SYS_ADMIN --volume=$(pwd):/app --workdir=/app --platform=linux/amd64 \ -e GIT_COMMITTER_NAME=test -e GIT_COMMITTER_EMAIL=test -e HOME=/tmp \ --user=$(id -u):$(id -g) \ docker.elastic.co/eui/ci:5.6 \ - bash -c \'/opt/yarn*/bin/yarn \ - && yarn cypress install \ - && NODE_OPTIONS="--max-old-space-size=2048" npm run test-ci \ - && npm run build\'', + bash -c 'yarn cypress install \ + && yarn test-ci --node-options=--max_old_space_size=2048 \ + && yarn build'", { stdio: 'inherit', } From eec4dd43a6e7a4b5728be4f4e3ba19437dac6e5b Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 10:03:52 -0600 Subject: [PATCH 11/22] Forgot to update Docker image path in test config. --- .../pipelines/pipeline_pull_request_test.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index 48f7d26dd9a..8a9297b7036 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -12,7 +12,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" # The depends_on creates an implicit "wait" while the container builds agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'lint' if: build.branch != "main" @@ -21,7 +21,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'unit:ts' if: build.branch != "main" @@ -30,7 +30,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'unit:tsx:16' if: build.branch != "main" @@ -39,7 +39,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'unit:tsx:17' if: build.branch != "main" @@ -48,7 +48,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'unit:tsx' if: build.branch != "main" @@ -57,7 +57,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'cypress:16' if: build.branch != "main" @@ -69,7 +69,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'cypress:17' if: build.branch != "main" @@ -81,7 +81,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'cypress:18' if: build.branch != "main" @@ -93,7 +93,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "container-registry-test.elastic.co/eui/ci:latest" + image: "docker-staging/eui/ci:latest" env: TEST_TYPE: 'cypress:a11y' if: build.branch != "main" From cb30066f25d95f013148cbb2bb17e5d9528b50a9 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 10:13:25 -0600 Subject: [PATCH 12/22] Typo in YML file. --- .buildkite/pipelines/pipeline_pull_request_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index 8a9297b7036..de62786bf49 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -1,7 +1,7 @@ # 🏠/.buildkite/pipelines/pipeline_pull_request_test.yml steps: - - label ":docker: Cached container" + - label: ":docker: Cached container" key: "build-test-container" command: ./buildkite/scripts/pipelines/pipeline_docker_build.sh agents: From 0c9234b37a9022839991c5f511caf5618bf3b746 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 10:20:06 -0600 Subject: [PATCH 13/22] Updated script path typos. --- .../pipelines/pipeline_pull_request_test.yml | 20 +++++++++---------- .buildkite/scripts/lifecycle/pre_command.sh | 2 +- scripts/docker-ci/docker-compose.yml | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index de62786bf49..07f414d3ac7 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -3,7 +3,7 @@ steps: - label: ":docker: Cached container" key: "build-test-container" - command: ./buildkite/scripts/pipelines/pipeline_docker_build.sh + command: .buildkite/scripts/pipelines/pipeline_docker_build.sh agents: provider: "gcp" if: build.branch != "main" # This job is triggered by the combined test and deploy docs for every PR @@ -12,7 +12,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" # The depends_on creates an implicit "wait" while the container builds agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'lint' if: build.branch != "main" @@ -21,7 +21,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:ts' if: build.branch != "main" @@ -30,7 +30,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:tsx:16' if: build.branch != "main" @@ -39,7 +39,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:tsx:17' if: build.branch != "main" @@ -48,7 +48,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'unit:tsx' if: build.branch != "main" @@ -57,7 +57,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:16' if: build.branch != "main" @@ -69,7 +69,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:17' if: build.branch != "main" @@ -81,7 +81,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:18' if: build.branch != "main" @@ -93,7 +93,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: - image: "docker-staging/eui/ci:latest" + image: "container-registry-test.elastic.co/eui/ci:latest" env: TEST_TYPE: 'cypress:a11y' if: build.branch != "main" diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index 2311cec1fdc..d232e776b2e 100644 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -35,7 +35,7 @@ export GITHUB_TOKEN # ======================================== # # Known values # ======================================== # -DOCKER_STAGING_IMAGE=docker-staging/eui/ci +DOCKER_STAGING_IMAGE=container-registry-test.elastic.co/eui/ci export DOCKER_STAGING_IMAGE GPROJECT=elastic-bekitzur diff --git a/scripts/docker-ci/docker-compose.yml b/scripts/docker-ci/docker-compose.yml index 6da5e821a97..efca8087bc5 100644 --- a/scripts/docker-ci/docker-compose.yml +++ b/scripts/docker-ci/docker-compose.yml @@ -7,7 +7,7 @@ services: - '/app/node_modules' app: - image: docker-staging/eui/ci + image: container-registry-test.elastic.co/eui/ci build: dockerfile: ./Dockerfile.app working_dir: /app From 7ba378ab0cd34c8284f550d2a8faaabc434809b1 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 11:39:35 -0600 Subject: [PATCH 14/22] Adding Git and SSH deps to slim Docker image. --- scripts/docker-ci/Dockerfile.app | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker-ci/Dockerfile.app b/scripts/docker-ci/Dockerfile.app index 874ff29854f..2069fb399cb 100644 --- a/scripts/docker-ci/Dockerfile.app +++ b/scripts/docker-ci/Dockerfile.app @@ -17,7 +17,7 @@ ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 # Install latest chrome package and update libs RUN apt-get update \ - && apt-get install -y wget xvfb ca-certificates gnupg \ + && apt-get install -y wget xvfb ca-certificates gnupg git openssh-client \ --no-install-recommends \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ From 94fb7bfa8f76145a33e73bd4f6ed7f702f330b32 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 12:07:12 -0600 Subject: [PATCH 15/22] Adding non-root user ID. --- .buildkite/pipelines/pipeline_pull_request_test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index 07f414d3ac7..da7db1ac3c5 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -13,6 +13,7 @@ steps: depends_on: "build-test-container" # The depends_on creates an implicit "wait" while the container builds agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" # Run as a non-root user env: TEST_TYPE: 'lint' if: build.branch != "main" @@ -22,6 +23,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'unit:ts' if: build.branch != "main" @@ -31,6 +33,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'unit:tsx:16' if: build.branch != "main" @@ -40,6 +43,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'unit:tsx:17' if: build.branch != "main" @@ -49,6 +53,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'unit:tsx' if: build.branch != "main" @@ -58,6 +63,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'cypress:16' if: build.branch != "main" @@ -70,6 +76,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'cypress:17' if: build.branch != "main" @@ -82,6 +89,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'cypress:18' if: build.branch != "main" @@ -94,6 +102,7 @@ steps: depends_on: "build-test-container" agents: image: "container-registry-test.elastic.co/eui/ci:latest" + imageUID: "1000" env: TEST_TYPE: 'cypress:a11y' if: build.branch != "main" From e528d79beef45bfed4a341658bc7ebb8928f3a4d Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 13:25:09 -0600 Subject: [PATCH 16/22] Simplifying yarn invocations in Docker image. --- .buildkite/scripts/lifecycle/post_command.sh | 2 +- .buildkite/scripts/lifecycle/pre_command.sh | 3 -- .../pipelines/pipeline_docker_build.sh | 2 +- .buildkite/scripts/pipelines/pipeline_test.sh | 42 +++++++++---------- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index 7e6383ce309..d7efe569775 100644 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -3,4 +3,4 @@ set -euo pipefail set +x -unset DOCKER_MACHINE DOCKER_STAGING_IMAGE DOCKER_STAGING_PASSWORD DOCKER_STAGING_REGISTRY DOCKER_STAGING_USERNAME GCE_ACCOUNT GITHUB_ACCOUNT GITHUB_TOKEN GPROJECT VAULT_ACCOUNT +unset DOCKER_MACHINE DOCKER_STAGING_PASSWORD DOCKER_STAGING_REGISTRY DOCKER_STAGING_USERNAME GCE_ACCOUNT GITHUB_ACCOUNT GITHUB_TOKEN GPROJECT VAULT_ACCOUNT diff --git a/.buildkite/scripts/lifecycle/pre_command.sh b/.buildkite/scripts/lifecycle/pre_command.sh index d232e776b2e..e1426419460 100644 --- a/.buildkite/scripts/lifecycle/pre_command.sh +++ b/.buildkite/scripts/lifecycle/pre_command.sh @@ -35,8 +35,5 @@ export GITHUB_TOKEN # ======================================== # # Known values # ======================================== # -DOCKER_STAGING_IMAGE=container-registry-test.elastic.co/eui/ci -export DOCKER_STAGING_IMAGE - GPROJECT=elastic-bekitzur export GPROJECT diff --git a/.buildkite/scripts/pipelines/pipeline_docker_build.sh b/.buildkite/scripts/pipelines/pipeline_docker_build.sh index 970f8e467c9..87286ade55c 100644 --- a/.buildkite/scripts/pipelines/pipeline_docker_build.sh +++ b/.buildkite/scripts/pipelines/pipeline_docker_build.sh @@ -12,7 +12,7 @@ echo "${DOCKER_STAGING_PASSWORD}" | docker login -u "${DOCKER_STAGING_USERNAME}" echo "[DOCKER]: Build and push container to registry" docker-compose -f ./scripts/docker-ci/docker-compose.yml build app -docker push "${DOCKER_STAGING_IMAGE}":latest +docker push "${DOCKER_STAGING_REGISTRY}":latest echo "[DOCKER]: Log out of registry" docker logout diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index 40b80d08e9a..2152cabef9a 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -2,17 +2,17 @@ set -euo pipefail -DOCKER_OPTIONS=( - -i --rm - --env GIT_COMMITTER_NAME=test - --env GIT_COMMITTER_EMAIL=test - --env HOME=/tmp - --user="$(id -u):$(id -g)" - --volume="$(pwd):/app" - --workdir=/app - --platform=linux/amd64 - "${DOCKER_STAGING_IMAGE}":latest -) +# DOCKER_OPTIONS=( +# -i --rm +# --env GIT_COMMITTER_NAME=test +# --env GIT_COMMITTER_EMAIL=test +# --env HOME=/tmp +# --user="$(id -u):$(id -g)" +# --volume="$(pwd):/app" +# --workdir=/app +# --platform=linux/amd64 +# "${DOCKER_STAGING_REGISTRY}":latest +# ) case $TEST_TYPE in lint) @@ -22,47 +22,47 @@ case $TEST_TYPE in unit:ts) echo "[TASK]: Running .ts and .js unit tests" - DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react") + yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react ;; unit:tsx:16) echo "[TASK]: Running Jest .tsx tests against React 16" - DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react") + yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react ;; unit:tsx:17) echo "[TASK]: Running Jest .tsx tests against React 17" - DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react") + yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react ;; unit:tsx) echo "[TASK]: Running Jest .tsx tests against React 18" - DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react") + yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react ;; cypress:16) echo "[TASK]: Running Cypress tests against React 16" - DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16") + yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16 ;; cypress:17) echo "[TASK]: Running Cypress tests against React 17" - DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17") + yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17 ;; cypress:18) echo "[TASK]: Running Cypress tests against React 18" - DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048") + yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 ;; cypress:a11y) echo "[TASK]: Running Cypress accessibility tests against React 18" - DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048") + yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 ;; cypress:a11y) echo "[TASK]: Running Cypress accessibility tests against React 18" - DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048") + yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 ;; *) @@ -72,4 +72,4 @@ case $TEST_TYPE in ;; esac -docker run "${DOCKER_OPTIONS[@]}" +# docker run "${DOCKER_OPTIONS[@]}" From 2e06a0bc15fd0f927afca7ca67c74d3a3303feb5 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 13:33:12 -0600 Subject: [PATCH 17/22] Typo in the push location. --- .buildkite/scripts/pipelines/pipeline_docker_build.sh | 2 +- .buildkite/scripts/pipelines/pipeline_test.sh | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.buildkite/scripts/pipelines/pipeline_docker_build.sh b/.buildkite/scripts/pipelines/pipeline_docker_build.sh index 87286ade55c..73ac83ae0ae 100644 --- a/.buildkite/scripts/pipelines/pipeline_docker_build.sh +++ b/.buildkite/scripts/pipelines/pipeline_docker_build.sh @@ -12,7 +12,7 @@ echo "${DOCKER_STAGING_PASSWORD}" | docker login -u "${DOCKER_STAGING_USERNAME}" echo "[DOCKER]: Build and push container to registry" docker-compose -f ./scripts/docker-ci/docker-compose.yml build app -docker push "${DOCKER_STAGING_REGISTRY}":latest +docker push "${DOCKER_STAGING_REGISTRY}"/eui/ci:latest echo "[DOCKER]: Log out of registry" docker logout diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index 2152cabef9a..d9d533968d7 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -59,12 +59,6 @@ case $TEST_TYPE in echo "[TASK]: Running Cypress accessibility tests against React 18" yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 ;; - - cypress:a11y) - echo "[TASK]: Running Cypress accessibility tests against React 18" - yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 - ;; - *) echo "[ERROR]: Unknown task" echo "Exit code: 1" From 72b339295569277500575ad0e55678e1dbb48d13 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 16:28:50 -0600 Subject: [PATCH 18/22] Modifying script to change to app/ directory first to match Docker container. --- .buildkite/scripts/pipelines/pipeline_test.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index d9d533968d7..18b7b7cf121 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -17,48 +17,49 @@ set -euo pipefail case $TEST_TYPE in lint) echo "[TASK]: Running linters" - DOCKER_OPTIONS+=(bash -c "yarn lint") + cd app && yarn lint ;; unit:ts) echo "[TASK]: Running .ts and .js unit tests" - yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react + cd app && yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react ;; unit:tsx:16) echo "[TASK]: Running Jest .tsx tests against React 16" - yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react + cd app && yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react ;; unit:tsx:17) echo "[TASK]: Running Jest .tsx tests against React 17" - yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react + cd app && yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react ;; unit:tsx) echo "[TASK]: Running Jest .tsx tests against React 18" - yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react + cd app && yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react ;; cypress:16) echo "[TASK]: Running Cypress tests against React 16" - yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16 + cd app && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16 ;; cypress:17) echo "[TASK]: Running Cypress tests against React 17" - yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17 + cd app && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17 ;; cypress:18) echo "[TASK]: Running Cypress tests against React 18" - yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 + cd app && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 ;; cypress:a11y) echo "[TASK]: Running Cypress accessibility tests against React 18" - yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 + cd app && yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 ;; + *) echo "[ERROR]: Unknown task" echo "Exit code: 1" From be96f3cfa038e185d86652d61200195a6229c340 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 16:46:02 -0600 Subject: [PATCH 19/22] Adding back GCP to have access to docker CLI commands. --- .../pipelines/pipeline_pull_request_test.yml | 9 ++++ .buildkite/scripts/pipelines/pipeline_test.sh | 42 +++++++++---------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index da7db1ac3c5..f3ebdb8f4c4 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -12,6 +12,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" # The depends_on creates an implicit "wait" while the container builds agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" # Run as a non-root user env: @@ -22,6 +23,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -32,6 +34,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -42,6 +45,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -52,6 +56,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -62,6 +67,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -75,6 +81,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -88,6 +95,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: @@ -101,6 +109,7 @@ steps: command: .buildkite/scripts/pipelines/pipeline_test.sh depends_on: "build-test-container" agents: + provider: "gcp" image: "container-registry-test.elastic.co/eui/ci:latest" imageUID: "1000" env: diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index 18b7b7cf121..87a8f708de3 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -2,62 +2,62 @@ set -euo pipefail -# DOCKER_OPTIONS=( -# -i --rm -# --env GIT_COMMITTER_NAME=test -# --env GIT_COMMITTER_EMAIL=test -# --env HOME=/tmp -# --user="$(id -u):$(id -g)" -# --volume="$(pwd):/app" -# --workdir=/app -# --platform=linux/amd64 -# "${DOCKER_STAGING_REGISTRY}":latest -# ) +DOCKER_OPTIONS=( + -i --rm + --env GIT_COMMITTER_NAME=test + --env GIT_COMMITTER_EMAIL=test + --env HOME=/tmp + --user="$(id -u):$(id -g)" + --volume="$(pwd):/app" + --workdir=/app + --platform=linux/amd64 + "${DOCKER_STAGING_IMAGE}"eui/ci:latest +) case $TEST_TYPE in lint) echo "[TASK]: Running linters" - cd app && yarn lint + DOCKER_OPTIONS+=(bash -c "yarn lint") ;; unit:ts) echo "[TASK]: Running .ts and .js unit tests" - cd app && yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=non-react") ;; unit:tsx:16) echo "[TASK]: Running Jest .tsx tests against React 16" - cd app && yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --react-version=16 --testMatch=react") ;; unit:tsx:17) echo "[TASK]: Running Jest .tsx tests against React 17" - cd app && yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --react-version=17 --testMatch=react") ;; unit:tsx) echo "[TASK]: Running Jest .tsx tests against React 18" - cd app && yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react + DOCKER_OPTIONS+=(bash -c "yarn test-unit --node-options=--max_old_space_size=2048 --testMatch=react") ;; cypress:16) echo "[TASK]: Running Cypress tests against React 16" - cd app && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16 + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=16") ;; cypress:17) echo "[TASK]: Running Cypress tests against React 17" - cd app && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17 + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 --react-version=17") ;; cypress:18) echo "[TASK]: Running Cypress tests against React 18" - cd app && yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048 + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn test-cypress --node-options=--max_old_space_size=2048") ;; cypress:a11y) echo "[TASK]: Running Cypress accessibility tests against React 18" - cd app && yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048 + DOCKER_OPTIONS+=(bash -c "yarn cypress install && yarn run test-cypress-a11y --node-options=--max_old_space_size=2048") ;; *) @@ -67,4 +67,4 @@ case $TEST_TYPE in ;; esac -# docker run "${DOCKER_OPTIONS[@]}" +docker run "${DOCKER_OPTIONS[@]}" From 24582f8e47bbe090e9a6e78e4c29f51966bd49c8 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 16:50:24 -0600 Subject: [PATCH 20/22] Reverted one too many file paths. --- .buildkite/scripts/pipelines/pipeline_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index 87a8f708de3..e99a36b2643 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -11,7 +11,7 @@ DOCKER_OPTIONS=( --volume="$(pwd):/app" --workdir=/app --platform=linux/amd64 - "${DOCKER_STAGING_IMAGE}"eui/ci:latest + "${DOCKER_STAGING_REGISTRY}"eui/ci:latest ) case $TEST_TYPE in From 83ac87b68e12ef4d4ad30e965ccbdd6824123de0 Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 17:10:55 -0600 Subject: [PATCH 21/22] Reverting to simpler GCP agent with optimized image. --- .../pipelines/pipeline_pull_request_test.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.buildkite/pipelines/pipeline_pull_request_test.yml b/.buildkite/pipelines/pipeline_pull_request_test.yml index f3ebdb8f4c4..aed9e8610bd 100644 --- a/.buildkite/pipelines/pipeline_pull_request_test.yml +++ b/.buildkite/pipelines/pipeline_pull_request_test.yml @@ -13,8 +13,6 @@ steps: depends_on: "build-test-container" # The depends_on creates an implicit "wait" while the container builds agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" # Run as a non-root user env: TEST_TYPE: 'lint' if: build.branch != "main" @@ -24,8 +22,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'unit:ts' if: build.branch != "main" @@ -35,8 +31,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'unit:tsx:16' if: build.branch != "main" @@ -46,8 +40,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'unit:tsx:17' if: build.branch != "main" @@ -57,8 +49,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'unit:tsx' if: build.branch != "main" @@ -68,8 +58,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'cypress:16' if: build.branch != "main" @@ -82,8 +70,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'cypress:17' if: build.branch != "main" @@ -96,8 +82,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'cypress:18' if: build.branch != "main" @@ -110,8 +94,6 @@ steps: depends_on: "build-test-container" agents: provider: "gcp" - image: "container-registry-test.elastic.co/eui/ci:latest" - imageUID: "1000" env: TEST_TYPE: 'cypress:a11y' if: build.branch != "main" From 67acc0ec6eadd9301187552f31c058c854ccfdfb Mon Sep 17 00:00:00 2001 From: 1Copenut Date: Thu, 16 Nov 2023 17:22:23 -0600 Subject: [PATCH 22/22] Last Docker image path update. Fingers crossed. --- .buildkite/scripts/pipelines/pipeline_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/pipelines/pipeline_test.sh b/.buildkite/scripts/pipelines/pipeline_test.sh index e99a36b2643..a8fe4124798 100644 --- a/.buildkite/scripts/pipelines/pipeline_test.sh +++ b/.buildkite/scripts/pipelines/pipeline_test.sh @@ -11,7 +11,7 @@ DOCKER_OPTIONS=( --volume="$(pwd):/app" --workdir=/app --platform=linux/amd64 - "${DOCKER_STAGING_REGISTRY}"eui/ci:latest + "${DOCKER_STAGING_REGISTRY}"/eui/ci:latest ) case $TEST_TYPE in