Merge pull request #15569 from MinaProtocol/dkijania/improve_package_…

…promotion_docs

Fix promote pipeline docs issues

buildkite/scripts/promote-docker.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+
 set -eo pipefail
 
 CLEAR='\033[0m'
@@ -8,7 +9,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -n|--name) NAME="$2"; shift;;
   -v|--version) VERSION="$2"; shift;;
   -t|--tag) TAG="$2"; shift;;
-  -p|--publish) PUBLISH=1; shift;;
+  -p|--publish) PUBLISH=1 ;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -33,7 +34,7 @@ if [[ -z "$TAG" ]]; then usage "Tag is not set!"; fi;
 # check for AWS Creds
 if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     echo "WARNING: AWS_ACCESS_KEY_ID not set, promote commands won't run"
-    exit 0
+#    exit 0
 fi
 
 GCR_REPO=gcr.io/o1labs-192920
@@ -45,7 +46,7 @@ docker pull ${GCR_REPO}/${NAME}:${VERSION}
 if [[ "$PUBLISH" -eq 1 ]]; then
   TARGET_REPO=docker.io/minaprotocol
   docker tag ${GCR_REPO}/${NAME}:${VERSION} ${TARGET_REPO}/${NAME}:${TAG}
-  echo "For security reasons only printing: docker push ${TARGET_REPO}/${NAME}:${TAG}"
+  docker push "${TARGET_REPO}/${NAME}:${TAG}"
 else 
   TARGET_REPO=$GCR_REPO
   echo "retagging ${GCR_REPO}/${NAME}:${VERSION} as ${TARGET_REPO}/${NAME}:${TAG}"

buildkite/scripts/run_promote_build_job.sh

@@ -63,8 +63,9 @@ if [[ -n "$DEBIANS" ]]; then
     if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
     if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi;
     if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi;
-    if [[ -z "$FROM_CHANNEL" ]]; then usage "'From channel' arg is not set!"; exit 1;  fi;
-    if [[ -z "$TO_CHANNEL" ]]; then usage "'To channel' arg is not set!"; exit 1; fi;
+    if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi;
+    if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi;
+    if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi;
     if [[ -z "$FROM_VERSION" ]]; then usage "Version is not set!"; exit 1; fi;
     if [[ -z "$NEW_VERSION" ]]; then NEW_VERSION=$FROM_VERSION; fi;
 
@@ -80,11 +81,17 @@ fi
 
 DHALL_DOCKERS="([] : List $DOCKER_DHALL_DEF.Type)"
 
+if [[ $PUBLISH -eq 1 ]]; then
+    DHALL_PUBLISH="True"
+  else 
+    DHALL_PUBLISH="False"
+fi
+
 if [[ -n "$DOCKERS" ]]; then 
     if [[ -z "$NEW_VERSION" ]]; then usage "New Tag is not set!"; fi;
     if [[ -z "$FROM_VERSION" ]]; then usage "Version is not set!"; fi;
     if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
-  
+
   arr_of_dockers=(${DOCKERS//,/ })
   DHALL_DOCKERS=""
   for i in "${arr_of_dockers[@]}"; do
@@ -105,4 +112,4 @@ if [[ "${REMOVE_PROFILE_FROM_NAME}" -eq 0 ]]; then
 else 
   REMOVE_PROFILE_FROM_NAME="True"
 fi 
-echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}'  ' | dhall-to-yaml --quoted 
+echo $PROMOTE_PACKAGE_DHALL_DEF'.promote_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${FROM_VERSION}"'" "'"${NEW_VERSION}"'" "amd64" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${FROM_CHANNEL}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted 

buildkite/src/Command/PromotePackage.dhall

@@ -67,7 +67,8 @@ let PromoteDockerSpec = {
     new_tag: Text,
     network: Network.Type,
     step_key: Text,
-    `if`: Optional B/If
+    `if`: Optional B/If,
+    publish: Bool
   },
   default = {
     deps = [] : List Command.TaggedKey.Type,
@@ -78,7 +79,8 @@ let PromoteDockerSpec = {
     profile = Profiles.Type.Standard,
     network = Network.Type.Berkeley,
     codename = DebianVersions.DebVersion.Bullseye,
-    `if` = None B/If
+    `if` = None B/If,
+    publish = False
   }
 }
 
@@ -102,12 +104,13 @@ let promoteDebianStep = \(spec : PromoteDebianSpec.Type) ->
 
 let promoteDockerStep = \(spec : PromoteDockerSpec.Type) ->
     let old_tag = Artifact.dockerTag spec.name spec.version spec.codename spec.profile spec.network
-    let new_tag = "${spec.new_tag}-${DebianVersions.lowerName spec.codename}"
+    let new_tag = Artifact.dockerTag spec.name spec.new_tag spec.codename spec.profile spec.network
+    let publish = if spec.publish then "-p" else ""
     in
     Command.build
       Command.Config::{
         commands = [ 
-          Cmd.run "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName spec.name} --version ${old_tag} --tag ${new_tag}"
+          Cmd.run "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName spec.name} --version ${old_tag} --tag ${new_tag} ${publish}"
         ],
         label = "Docker: ${spec.step_key}",
         key = spec.step_key,

buildkite/src/Entrypoints/PromotePackage.dhall

@@ -24,6 +24,7 @@ let promote_artifacts =
   \(to_channel: DebianChannel.Type ) ->
   \(tag: Text ) ->
   \(remove_profile_from_name: Bool) ->
+  \(publish: Bool) ->
 
   let debians_spec =
       List/map 
@@ -80,6 +81,7 @@ let promote_artifacts =
               , codename = codename
               , new_tag = new_version
               , network = network
+              , publish = publish
               , step_key = "add-tag-to-${Artifact.lowerName docker}-${DebianVersions.lowerName codename}-docker"
           }
         )

buildkite/src/README.md

@@ -142,26 +142,66 @@ steps:
         propagate-environment: true
 ```
 
+Above definition one need to paste into steps edit box for given pipeline and then run from branch which contains this README.md (presumably develop). 
+
 All list of available parameters: 
 
-- DEBIANS - The comma delimitered debian names. For example: `Daemon,Archive`
+- DEBIANS - The comma delimitered debian names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/DebianPackage.dhall` files. Only CamelCase format is supported
 
-- DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive`
+- DOCKERS - The comma delimitered docker names. For example: `Daemon,Archive`. All available names are located in `buildkite/src/Constans/Artifacts.dhall` files. Only CamelCase format is supported 
 
-- CODENAMES - The Debian codenames `Bullseye,Buster,Focal`
+- CODENAMES - The Debian codenames `Bullseye,Buster,Focal`. All available names are located in `buildkite/src/Constans/DebianVersions.dhall`. Only CamelCase format is supported
 
 - FROM_VERSION - The Source Docker or Debian version
 
 - NEW_VERSION - The new Debian version or new Docker tag
 
-- REMOVE_PROFILE_FROM_NAME - Should we remove profile suffix from debian name. For example 
+- REMOVE_PROFILE_FROM_NAME - Should we remove profile suffix from debian name. For example from package name "mina-devnet-hardfork" it will generate name "mina-devnet"
 
-- PROFILE                     The Docker and Debian profile (Standard, Lightnet)"
+- PROFILE                     The Docker and Debian profile (Standard, Lightnet)". All available profiles are located in `buildkite/src/Constants/Profiles.dhall` file. Only CamelCase format is supported
 
-- NETWORK                     The Docker and Debian network (Devnet, Mainnet, Berkeley)"
+- NETWORK                     The Docker and Debian network (Devnet, Mainnet, Berkeley). All available profiles are located in `buildkite/src/Constants/Network.dhall` file. Only CamelCase format is supported
 
-- FROM_CHANNEL                Source debian channel"
+- FROM_CHANNEL                Source debian channel. By default: Unstable. All available channels  are located in `buildkite/src/Constants/DebianChannel.dhall` file. Only CamelCase format is supported
 
-- TO_CHANNEL                  Target debian channel"
+- TO_CHANNEL                  Target debian channel. By default: Unstable. All available profiles are located in `buildkite/src/Constants/DebianChannel.dhall` file. Only CamelCase format is supported
 
 - PUBLISH                     The Publish to docker.io flag. If defined, script will publish docker do docker.io. Otherwise it will still resides in gcr.io
+
+
+#### Examples 
+
+Below examples focus only on environment variables values. We are omitting full pipeline setup.
+
+##### Promoting Hardfork packages
+
+We would like to promote all hardfork packages (archive node, daemon, rosetta) from unstable debian channel and gcr to devnet debian channel and dockerhub. We also want easy upgrade from old deamon debian to new one (we would like user experience to be smooth and only required command to update on user side should be `apt-get update mina-daemon`). That is why we want to strip `-hardfork` suffix from debian package. 
+Pipeline with create 6 jobs for each Docker and Debian component separately.
+
+```
+  - "DOCKERS=Archive,Daemon,Rosetta"
+  - "DEBIANS=Archive,Daemon,LogProc"
+  - "REMOVE_PROFILE_FROM_NAME=1"
+  - "PROFILE=Hardfork"
+  - "NETWORK=Devnet"
+  - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
+  - "NEW_VERSION=3.0.0-ddb6fc4"
+  - "PUBLISH"=1
+  - "CODENAMES=Focal,Buster,Bullseye"
+  - "FROM_CHANNEL=Unstable"
+  - "TO_CHANNEL=Devnet"
+    
+```
+
+#### Promoting dockers form gcr to dockerhub
+
+We want only to move dockers from gcr to dockerhub without changing version. Current implementation of pipeline is not user friendly so we need to still define `FROM_VERSION` and `TO_VERSION`. They should be equal.
+
+```
+  - "DOCKERS=Archive,Daemon,Rosetta"
+  - "NETWORK=Devnet"
+  - "FROM_VERSION=3.0.0-dc6bf78"
+  - "NEW_VERSION=3.0.0-dc6bf78"
+  - "CODENAMES=Focal,Buster,Bullseye"
+  - "PUBLISH=1"
+```