diff --git a/buildkite/scripts/promote-docker.sh b/buildkite/scripts/promote-docker.sh index a9b0ec48ab9..5796258e66a 100755 --- a/buildkite/scripts/promote-docker.sh +++ b/buildkite/scripts/promote-docker.sh @@ -1,4 +1,5 @@ #!/bin/bash + set -eo pipefail CLEAR='\033[0m' @@ -8,7 +9,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in -n|--name) NAME="$2"; shift;; -v|--version) VERSION="$2"; shift;; -t|--tag) TAG="$2"; shift;; - -p|--publish) PUBLISH=1; shift;; + -p|--publish) PUBLISH=1 ;; *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done @@ -33,7 +34,7 @@ if [[ -z "$TAG" ]]; then usage "Tag is not set!"; fi; # check for AWS Creds if [ -z "$AWS_ACCESS_KEY_ID" ]; then echo "WARNING: AWS_ACCESS_KEY_ID not set, promote commands won't run" - exit 0 +# exit 0 fi GCR_REPO=gcr.io/o1labs-192920 @@ -45,7 +46,7 @@ docker pull ${GCR_REPO}/${NAME}:${VERSION} if [[ "$PUBLISH" -eq 1 ]]; then TARGET_REPO=docker.io/minaprotocol docker tag ${GCR_REPO}/${NAME}:${VERSION} ${TARGET_REPO}/${NAME}:${TAG} - echo "For security reasons only printing: docker push ${TARGET_REPO}/${NAME}:${TAG}" + docker push "${TARGET_REPO}/${NAME}:${TAG}" else TARGET_REPO=$GCR_REPO echo "retagging ${GCR_REPO}/${NAME}:${VERSION} as ${TARGET_REPO}/${NAME}:${TAG}" diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 79d9a611b73..26feee3e23a 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -63,8 +63,9 @@ if [[ -n "$DEBIANS" ]]; then if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; - if [[ -z "$FROM_CHANNEL" ]]; then usage "'From channel' arg is not set!"; exit 1; fi; - if [[ -z "$TO_CHANNEL" ]]; then usage "'To channel' arg is not set!"; exit 1; fi; + if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; + if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi; + if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi; if [[ -z "$FROM_VERSION" ]]; then usage "Version is not set!"; exit 1; fi; if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$FROM_VERSION; fi; @@ -80,11 +81,17 @@ fi DHALL_DOCKERS="([] : List $DOCKER_DHALL_DEF.Type)" +if [[ $PUBLISH -eq 1 ]]; then + DHALL_PUBLISH="True" + else + DHALL_PUBLISH="False" +fi + if [[ -n "$DOCKERS" ]]; then if [[ -z "$NEW_VERSION" ]]; then usage "New Tag is not set!"; fi; if [[ -z "$FROM_VERSION" ]]; then usage "Version is not set!"; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - + arr_of_dockers=(${DOCKERS//,/ }) DHALL_DOCKERS="" for i in "${arr_of_dockers[@]}"; do @@ -105,4 +112,4 @@ if [[ "${REMOVE_PROFILE_FROM_NAME}" -eq 0 ]]; then else REMOVE_PROFILE_FROM_NAME="True" fi -echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' ' | dhall-to-yaml --quoted +echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall index f47a832bf5e..6e8b4b47c26 100644 --- a/buildkite/src/Command/PromotePackage.dhall +++ b/buildkite/src/Command/PromotePackage.dhall @@ -67,7 +67,8 @@ let PromoteDockerSpec = { new_tag: Text, network: Network.Type, step_key: Text, - `if`: Optional B/If + `if`: Optional B/If, + publish: Bool }, default = { deps = [] : List Command.TaggedKey.Type, @@ -78,7 +79,8 @@ let PromoteDockerSpec = { profile = Profiles.Type.Standard, network = Network.Type.Berkeley, codename = DebianVersions.DebVersion.Bullseye, - `if` = None B/If + `if` = None B/If, + publish = False } } @@ -102,12 +104,13 @@ let promoteDebianStep = \(spec : PromoteDebianSpec.Type) -> let promoteDockerStep = \(spec : PromoteDockerSpec.Type) -> let old_tag = Artifact.dockerTag spec.name spec.version spec.codename spec.profile spec.network - let new_tag = "${spec.new_tag}-${DebianVersions.lowerName spec.codename}" + let new_tag = Artifact.dockerTag spec.name spec.new_tag spec.codename spec.profile spec.network + let publish = if spec.publish then "-p" else "" in Command.build Command.Config::{ commands = [ - Cmd.run "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName spec.name} --version ${old_tag} --tag ${new_tag}" + Cmd.run "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName spec.name} --version ${old_tag} --tag ${new_tag} ${publish}" ], label = "Docker: ${spec.step_key}", key = spec.step_key, diff --git a/buildkite/src/Entrypoints/PromotePackage.dhall b/buildkite/src/Entrypoints/PromotePackage.dhall index e9b87fa2909..75c3e0e95f6 100644 --- a/buildkite/src/Entrypoints/PromotePackage.dhall +++ b/buildkite/src/Entrypoints/PromotePackage.dhall @@ -24,6 +24,7 @@ let promote_artifacts = \(to_channel: DebianChannel.Type ) -> \(tag: Text ) -> \(remove_profile_from_name: Bool) -> + \(publish: Bool) -> let debians_spec = List/map @@ -80,6 +81,7 @@ let promote_artifacts = , codename = codename , new_tag = new_version , network = network + , publish = publish , step_key = "add-tag-to-${Artifact.lowerName docker}-${DebianVersions.lowerName codename}-docker" } ) diff --git a/buildkite/src/README.md b/buildkite/src/README.md index 15f38d59ca1..d9b577e81b5 100644 --- a/buildkite/src/README.md +++ b/buildkite/src/README.md @@ -142,26 +142,66 @@ steps: propagate-environment: true ``` +Above definition one need to paste into steps edit box for given pipeline and then run from branch which contains this README.md (presumably develop). + All list of available parameters: -- DEBIANS - The comma delimitered debian names. For example: `Daemon,Archive` +- DEBIANS - The comma delimitered debian names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/DebianPackage.dhall` files. Only CamelCase format is supported -- DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive` +- DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported -- CODENAMES - The Debian codenames `Bullseye,Buster,Focal` +- CODENAMES - The Debian codenames `Bullseye,Buster,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported - FROM_VERSION - The Source Docker or Debian version - NEW_VERSION - The new Debian version or new Docker tag -- REMOVE_PROFILE_FROM_NAME - Should we remove profile suffix from debian name. For example +- REMOVE_PROFILE_FROM_NAME - Should we remove profile suffix from debian name. For example from package name "mina-devnet-hardfork" it will generate name "mina-devnet" -- PROFILE The Docker and Debian profile (Standard, Lightnet)" +- PROFILE The Docker and Debian profile (Standard, Lightnet)". All available profiles are located in `buildkite/src/Constants/Profiles.dhall` file. Only CamelCase format is supported -- NETWORK The Docker and Debian network (Devnet, Mainnet, Berkeley)" +- NETWORK The Docker and Debian network (Devnet, Mainnet, Berkeley). All available profiles are located in `buildkite/src/Constants/Network.dhall` file. Only CamelCase format is supported -- FROM_CHANNEL Source debian channel" +- FROM_CHANNEL Source debian channel. By default: Unstable. All available channels are located in `buildkite/src/Constants/DebianChannel.dhall` file. Only CamelCase format is supported -- TO_CHANNEL Target debian channel" +- TO_CHANNEL Target debian channel. By default: Unstable. All available profiles are located in `buildkite/src/Constants/DebianChannel.dhall` file. Only CamelCase format is supported - PUBLISH The Publish to docker.io flag. If defined, script will publish docker do docker.io. Otherwise it will still resides in gcr.io + + +#### Examples + +Below examples focus only on environment variables values. We are omitting full pipeline setup. + +##### Promoting Hardfork packages + +We would like to promote all hardfork packages (archive node, daemon, rosetta) from unstable debian channel and gcr to devnet debian channel and dockerhub. We also want easy upgrade from old deamon debian to new one (we would like user experience to be smooth and only required command to update on user side should be `apt-get update mina-daemon`). That is why we want to strip `-hardfork` suffix from debian package. +Pipeline with create 6 jobs for each Docker and Debian component separately. + +``` + - "DOCKERS=Archive,Daemon,Rosetta" + - "DEBIANS=Archive,Daemon,LogProc" + - "REMOVE_PROFILE_FROM_NAME=1" + - "PROFILE=Hardfork" + - "NETWORK=Devnet" + - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e" + - "NEW_VERSION=3.0.0-ddb6fc4" + - "PUBLISH"=1 + - "CODENAMES=Focal,Buster,Bullseye" + - "FROM_CHANNEL=Unstable" + - "TO_CHANNEL=Devnet" + +``` + +#### Promoting dockers form gcr to dockerhub + +We want only to move dockers from gcr to dockerhub without changing version. Current implementation of pipeline is not user friendly so we need to still define `FROM_VERSION` and `TO_VERSION`. They should be equal. + +``` + - "DOCKERS=Archive,Daemon,Rosetta" + - "NETWORK=Devnet" + - "FROM_VERSION=3.0.0-dc6bf78" + - "NEW_VERSION=3.0.0-dc6bf78" + - "CODENAMES=Focal,Buster,Bullseye" + - "PUBLISH=1" +``` \ No newline at end of file