From 0aca2b872d9db06404dac8e0388e38e9259a5c3b Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 20 Jan 2025 13:55:17 +0200 Subject: [PATCH] Use --qualifier in release manager (#16907) This commit uses the new --qualifier parameter in the release manager for publishing dra artifacts. Additionally, simplifies the expected variables to rely on a simple `VERSION_QUALIFIER`. Snapshot builds are skipped when VERSION_QUALIFIER is set. Finally, for helping to test DRA PRs, we also allow passing the `DRA_BRANCH` option/env var to override BUILDKITE_BRANCH. Closes https://github.com/elastic/ingest-dev/issues/4856 --- .buildkite/dra_pipeline.yml | 10 +++- .buildkite/scripts/dra/build_docker.sh | 71 ++++++++---------------- .buildkite/scripts/dra/build_packages.sh | 42 +++++++------- .buildkite/scripts/dra/generatesteps.py | 40 ++++++++----- .buildkite/scripts/dra/publish.sh | 50 ++++++++++++----- 5 files changed, 111 insertions(+), 102 deletions(-) diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml index 4cd3cea931c..10007715ccd 100644 --- a/.buildkite/dra_pipeline.yml +++ b/.buildkite/dra_pipeline.yml @@ -4,8 +4,12 @@ steps: - label: ":pipeline: Generate steps" command: | set -euo pipefail - - echo "--- Building [${WORKFLOW_TYPE}] artifacts" + + echo "--- Building [$${WORKFLOW_TYPE}] artifacts" python3 -m pip install pyyaml echo "--- Building dynamic pipeline steps" - python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload + python3 .buildkite/scripts/dra/generatesteps.py > steps.yml + echo "--- Printing dynamic pipeline steps" + cat steps.yml + echo "--- Uploading dynamic pipeline steps" + cat steps.yml | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 516e991d668..e83e4ae3aa2 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -9,61 +9,38 @@ source ./$(dirname "$0")/common.sh # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values case "$WORKFLOW_TYPE" in snapshot) - info "Building artifacts for the $WORKFLOW_TYPE workflow..." - if [ -z "$VERSION_QUALIFIER_OPT" ]; then - rake artifact:docker || error "artifact:docker build failed." - rake artifact:docker_oss || error "artifact:docker_oss build failed." - rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." - rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi - else - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed." - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed." - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi - # Qualifier is passed from CI as optional field and specify the version postfix - # in case of alpha or beta releases: - # e.g: 8.0.0-alpha1 - STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" - fi - STACK_VERSION=${STACK_VERSION}-SNAPSHOT - info "Build complete, setting STACK_VERSION to $STACK_VERSION." + : # no-op ;; staging) - info "Building artifacts for the $WORKFLOW_TYPE workflow..." - if [ -z "$VERSION_QUALIFIER_OPT" ]; then - RELEASE=1 rake artifact:docker || error "artifact:docker build failed." - RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." - RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." - RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi - else - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed." - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." - if [ "$ARCH" != "aarch64" ]; then - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." - fi - # Qualifier is passed from CI as optional field and specify the version postfix - # in case of alpha or beta releases: - # e.g: 8.0.0-alpha1 - STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" - fi - info "Build complete, setting STACK_VERSION to $STACK_VERSION." + export RELEASE=1 ;; *) error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." ;; esac +rake artifact:docker || error "artifact:docker build failed." +rake artifact:docker_oss || error "artifact:docker_oss build failed." +rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." +rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + +if [[ "$ARCH" != "aarch64" ]]; then + rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." +fi + +if [[ "$WORKFLOW_TYPE" == "staging" ]] && [[ -n "$VERSION_QUALIFIER" ]]; then + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases for staging builds only: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER}" +fi + +if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then + STACK_VERSION="${STACK_VERSION}-SNAPSHOT" +fi + +info "Build complete, setting STACK_VERSION to $STACK_VERSION." + info "Saving tar.gz for docker images" save_docker_tarballs "${ARCH}" "${STACK_VERSION}" diff --git a/.buildkite/scripts/dra/build_packages.sh b/.buildkite/scripts/dra/build_packages.sh index 2bd4ed1750a..327820891ca 100755 --- a/.buildkite/scripts/dra/build_packages.sh +++ b/.buildkite/scripts/dra/build_packages.sh @@ -7,39 +7,35 @@ echo "####################################################################" source ./$(dirname "$0")/common.sh # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values +info "Building artifacts for the $WORKFLOW_TYPE workflow ..." + case "$WORKFLOW_TYPE" in snapshot) - info "Building artifacts for the $WORKFLOW_TYPE workflow..." - if [ -z "$VERSION_QUALIFIER_OPT" ]; then - SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." - else - # Qualifier is passed from CI as optional field and specify the version postfix - # in case of alpha or beta releases: - # e.g: 8.0.0-alpha1 - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." - STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" - fi - STACK_VERSION=${STACK_VERSION}-SNAPSHOT - info "Build complete, setting STACK_VERSION to $STACK_VERSION." + : # no-op ;; staging) - info "Building artifacts for the $WORKFLOW_TYPE workflow..." - if [ -z "$VERSION_QUALIFIER_OPT" ]; then - RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." - else - # Qualifier is passed from CI as optional field and specify the version postfix - # in case of alpha or beta releases: - # e.g: 8.0.0-alpha1 - VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." - STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" - fi - info "Build complete, setting STACK_VERSION to $STACK_VERSION." + export RELEASE=1 ;; *) error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." ;; esac +SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + +if [[ "$WORKFLOW_TYPE" == "staging" ]] && [[ -n "$VERSION_QUALIFIER" ]]; then + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases for staging builds only: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER}" +fi + +if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then + STACK_VERSION="${STACK_VERSION}-SNAPSHOT" +fi + +info "Build complete, setting STACK_VERSION to $STACK_VERSION." + info "Generated Artifacts" for file in build/logstash-*; do shasum $file;done diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py index db4b24ba677..d26e34009e3 100644 --- a/.buildkite/scripts/dra/generatesteps.py +++ b/.buildkite/scripts/dra/generatesteps.py @@ -3,6 +3,8 @@ import yaml +YAML_HEADER = '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n' + def to_bk_key_friendly_string(key): """ Convert and return key to an acceptable format for Buildkite's key: field @@ -106,6 +108,7 @@ def build_steps_to_yaml(branch, workflow_type): if __name__ == "__main__": try: workflow_type = os.environ["WORKFLOW_TYPE"] + version_qualifier = os.environ.get("VERSION_QUALIFIER", "") except ImportError: print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=\n.Exiting.") exit(1) @@ -114,18 +117,25 @@ def build_steps_to_yaml(branch, workflow_type): structure = {"steps": []} - # Group defining parallel steps that build and save artifacts - group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}") - - structure["steps"].append({ - "group": f":Build Artifacts - {workflow_type.upper()}", - "key": group_key, - "steps": build_steps_to_yaml(branch, workflow_type), - }) - - # Final step: pull artifacts built above and publish them via the release-manager - structure["steps"].extend( - yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)), - ) - - print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n' + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False)) + if workflow_type.upper() == "SNAPSHOT" and len(version_qualifier)>0: + structure["steps"].append({ + "label": f"no-op pipeline because prerelease builds (VERSION_QUALIFIER is set to [{version_qualifier}]) don't support the [{workflow_type}] workflow", + "command": ":", + "skip": "VERSION_QUALIFIER (prerelease builds) not supported with SNAPSHOT DRA", + }) + else: + # Group defining parallel steps that build and save artifacts + group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}") + + structure["steps"].append({ + "group": f":Build Artifacts - {workflow_type.upper()}", + "key": group_key, + "steps": build_steps_to_yaml(branch, workflow_type), + }) + + # Final step: pull artifacts built above and publish them via the release-manager + structure["steps"].extend( + yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)), + ) + + print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False)) diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh index 67c6ce895d9..c56fbaedfd5 100755 --- a/.buildkite/scripts/dra/publish.sh +++ b/.buildkite/scripts/dra/publish.sh @@ -7,7 +7,9 @@ echo "####################################################################" source ./$(dirname "$0")/common.sh -PLAIN_STACK_VERSION=$STACK_VERSION +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" # This is the branch selector that needs to be passed to the release-manager # It has to be the name of the branch which originates the artifacts. @@ -15,30 +17,24 @@ RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\. if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then RELEASE_BRANCH=$RELEASE_VER else - RELEASE_BRANCH="${BUILDKITE_BRANCH:="main"}" + RELEASE_BRANCH="${BRANCH:="main"}" fi echo "RELEASE BRANCH: $RELEASE_BRANCH" -if [ -n "$VERSION_QUALIFIER_OPT" ]; then - # Qualifier is passed from CI as optional field and specify the version postfix - # in case of alpha or beta releases: - # e.g: 8.0.0-alpha1 - STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" - PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}" -fi +VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}" case "$WORKFLOW_TYPE" in snapshot) - STACK_VERSION=${STACK_VERSION}-SNAPSHOT + : ;; staging) ;; *) - error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..." + error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." ;; esac -info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}" +info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH} for version: ${STACK_VERSION} with version_qualifier: ${VERSION_QUALIFIER}" if [ "$RELEASE_VER" != "7.17" ]; then # Version 7.17.x doesn't generates ARM artifacts for Darwin @@ -56,7 +52,16 @@ rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz info "Downloaded ARTIFACTS sha report" for file in build/logstash-*; do shasum $file;done -mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv +FINAL_VERSION=$STACK_VERSION +if [[ -n "$VERSION_QUALIFIER" ]]; then + FINAL_VERSION="$FINAL_VERSION-${VERSION_QUALIFIER}" +fi + +if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then + FINAL_VERSION="${STACK_VERSION}-SNAPSHOT" +fi + +mv build/distributions/dependencies-reports/logstash-${FINAL_VERSION}.csv build/distributions/dependencies-${FINAL_VERSION}.csv # set required permissions on artifacts and directory chmod -R a+r build/* @@ -74,6 +79,22 @@ release_manager_login # ensure the latest image has been pulled docker pull docker.elastic.co/infra/release-manager:latest +echo "+++ :clipboard: Listing DRA artifacts for version [$STACK_VERSION], branch [$RELEASE_BRANCH], workflow [$WORKFLOW_TYPE], QUALIFIER [$VERSION_QUALIFIER]" +docker run --rm \ + --name release-manager \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project logstash \ + --branch "${RELEASE_BRANCH}" \ + --commit "$(git rev-parse HEAD)" \ + --workflow "${WORKFLOW_TYPE}" \ + --version "${STACK_VERSION}" \ + --artifact-set main \ + --qualifier "${VERSION_QUALIFIER}" + info "Running the release manager ..." # collect the artifacts for use with the unified build @@ -89,8 +110,9 @@ docker run --rm \ --branch ${RELEASE_BRANCH} \ --commit "$(git rev-parse HEAD)" \ --workflow "${WORKFLOW_TYPE}" \ - --version "${PLAIN_STACK_VERSION}" \ + --version "${STACK_VERSION}" \ --artifact-set main \ + --qualifier "${VERSION_QUALIFIER}" \ ${DRA_DRY_RUN} | tee rm-output.txt # extract the summary URL from a release manager output line like: