GEODE-9730: release script improvements (#6989)

* add fan-in for rc pipeline
* disable parallel when publishing to nexus to avoid possibility of upload getting split into two repos
* switch native build from debian to ubuntu for a (less) brittle was to get a recent-enough version of cmake (min 3.12, currently this gets 3.16)
* fix to support pinned geode sha, fix for concourse getting confused and seeing the wrong tag
* Don't imply there's anything to backport for other native branches unless this is a flagship release
* provide instructions to check for and recover from split repo
* remind that issues fixed in e.g. 1.13.1 and 1.14.1 should not be closed if 1.13.1 is released first; wait to close until latest release has fix
* maintain geode-benchmarks versionNumber, even though it doesn't seem to be used for anything
* maintain default benchmark baseline on a new flagship release
* tighten expected file sizes
* template the contents of the GitHub release description
* gpg has been split into two packages
* include a reminder to write the releasenotes while the rc pipeline is running before sending the vote email
* increase upthewaterspout timeout to 2h after observing it can take longer than 1h
* fix benchmark baseline updating and update to the minor for the support branch, unlike develop
* improve prompts for creating release on GitHub

dev-tools/release/commit_rc.sh

@@ -124,6 +124,7 @@ echo "============================================================"
 for DIR in ${GEODE} ${GEODE_EXAMPLES} ${GEODE_NATIVE} ${GEODE_BENCHMARKS} ; do
     set -x
     cd ${DIR}
+    git pull -r
     git push -u origin
     git push origin rel/v${FULL_VERSION}
     set +x
@@ -137,7 +138,8 @@ echo "============================================================"
 cd ${GEODE}/../..
 echo "1. In a separate terminal window, ${0%/*}/deploy_rc_pipeline.sh -v ${VERSION_MM}"
 echo "2. Monitor https://concourse.apachegeode-ci.info/teams/main/pipelines/apache-support-${VERSION_MM//./-}-rc until all green"
-echo "3. Send the following email to announce the RC:"
+echo "3. If you haven't already, add a ${VERSION} section to https://cwiki.apache.org/confluence/display/GEODE/Release+Notes"
+echo "4. Send the following email to announce the RC:"
 echo "To: [email protected]"
 echo "Subject: [VOTE] Apache Geode ${FULL_VERSION}"
 ${0%/*}/print_rc_email.sh -v ${FULL_VERSION} -m ${MAVEN}

dev-tools/release/deploy_rc_pipeline.sh

@@ -108,12 +108,11 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends unzip git
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               SHA=$(cd geode && git rev-parse HEAD)
               java -version
@@ -144,12 +143,11 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends unzip git
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               SHA=$(cd geode && git rev-parse HEAD)
               curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}-src.tgz > src.tgz
@@ -182,12 +180,11 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends git
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               SHA=$(cd geode && git rev-parse HEAD)
               curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > bin.tgz
@@ -219,9 +216,8 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends unzip
               cd geode-examples
@@ -248,12 +244,11 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends unzip git
-              FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##' -e 's#-.*##')
+              FULL_VERSION=$(cd geode-examples && git fetch && git describe --tags | sed -e 's#^rel/v##' -e 's#-.*##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               if [ "${FULL_VERSION}" = "${VERSION}" ] ; then
                 GRADLE_ARGS=""
@@ -280,31 +275,28 @@ jobs:
           image_resource:
             type: docker-image
             source:
-              repository: bellsoft/liberica-openjdk-debian
-              tag: 8
+              repository: adoptopenjdk/openjdk8
+              tag: slim
           inputs:
             - name: geode-native
           platform: linux
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends git
-              FULL_VERSION=$(cd geode-native && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode-native && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               #use geode from binary dist
               curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-${VERSION}.tgz > geode-bin.tgz
               tar xzf geode-bin.tgz
-              # needed to get cmake >= 3.12
-              echo 'APT::Default-Release "stable";' >> /etc/apt/apt.conf.d/99defaultrelease
-              echo 'deb     http://ftp.de.debian.org/debian/    stable main contrib non-free' >> /etc/apt/sources.list.d/stable.list
-              echo 'deb-src http://ftp.de.debian.org/debian/    stable main contrib non-free' >> /etc/apt/sources.list.d/stable.list
-              echo 'deb     http://security.debian.org/         stable/updates  main contrib non-free' >> /etc/apt/sources.list.d/stable.list
               apt-get update || true
               DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y cmake openssl doxygen build-essential libssl-dev zlib1g-dev
+              #cmake wrongly assumes javah wasn't removed until JDK10, but adoptopenjdk removed it in JDK8
+              echo '/opt/java/openjdk/bin/javac -h "$@"' > /opt/java/openjdk/bin/javah
+              chmod +x /opt/java/openjdk/bin/javah
               cd geode-native
               mkdir build
               cd build
@@ -327,33 +319,30 @@ jobs:
           image_resource:
             type: docker-image
             source:
-              repository: bellsoft/liberica-openjdk-debian
-              tag: 8
+              repository: adoptopenjdk/openjdk8
+              tag: slim
           inputs:
             - name: geode-native
             - name: geode
           platform: linux
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends unzip git
-              FULL_VERSION=$(cd geode-native && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode-native && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               # build geode from source
               cd geode
               ./gradlew build -x test -x javadoc -x rat -x pmdMain
               cd ..
-              # needed to get cmake >= 3.12
-              echo 'APT::Default-Release "stable";' >> /etc/apt/apt.conf.d/99defaultrelease
-              echo 'deb     http://ftp.de.debian.org/debian/    stable main contrib non-free' >> /etc/apt/sources.list.d/stable.list
-              echo 'deb-src http://ftp.de.debian.org/debian/    stable main contrib non-free' >> /etc/apt/sources.list.d/stable.list
-              echo 'deb     http://security.debian.org/         stable/updates  main contrib non-free' >> /etc/apt/sources.list.d/stable.list
               apt-get update || true
               DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y cmake openssl doxygen build-essential libssl-dev zlib1g-dev
+              #cmake wrongly assumes javah wasn't removed until JDK10, but adoptopenjdk removed it in JDK8
+              echo '/opt/java/openjdk/bin/javac -h "$@"' > /opt/java/openjdk/bin/javah
+              chmod +x /opt/java/openjdk/bin/javah
               curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-native-${VERSION}-src.tgz > src.tgz
               tar xzf src.tgz
               cd apache-geode-native-${VERSION}-src
@@ -373,7 +362,7 @@ jobs:
         - get: upthewaterspout-tests
         - get: geode-examples
       - task: validate
-        timeout: 1h
+        timeout: 2h
         config:
           image_resource:
             type: docker-image
@@ -388,12 +377,11 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
-              apt install -qq -y --no-install-recommends unzip git gpg wget
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              apt install -qq -y --no-install-recommends unzip git gpg gpg-agent wget
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}')
               cd upthewaterspout-tests
@@ -421,12 +409,11 @@ jobs:
           run:
             path: /bin/sh
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
               apt install -qq -y --no-install-recommends unzip git keychain
-              FULL_VERSION=$(cd geode-benchmarks && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode-benchmarks && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               curl -L -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-benchmarks-${VERSION}-src.tgz > src.tgz
               tar xzf src.tgz
@@ -462,12 +449,11 @@ jobs:
           run:
             path: /bin/bash
             args:
-            - -ec
+            - -ecx
             - |
-              set -ex
               apt update -q
-              apt install -qq -y --no-install-recommends git gpg
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              apt install -qq -y --no-install-recommends git gpg gpg-agent
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               curl -L -s https://dist.apache.org/repos/dist/dev/geode/KEYS > KEYS
               gpg --import KEYS
@@ -512,11 +498,11 @@ jobs:
                 tar xzf $file "${tld}/LICENSE"
                 head -1 "${tld}/LICENSE" | grep -q "Apache License"
               }
-              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION}-src 10000000 30000000
-              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION} 100000000 150000000
-              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-examples-${VERSION}-src 50000 2000000
-              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-native-${VERSION}-src 2000000 4000000
-              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-benchmarks-${VERSION}-src 50000 500000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION}-src          16000000  20000000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-${VERSION}             120000000 135000000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-examples-${VERSION}-src   840000    900000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-native-${VERSION}-src    2400000   3200000
+              verifyArtifactSizeSignatureLicenseNoticeAndCopyright apache-geode-benchmarks-${VERSION}-src  85000    115000
               curl -L -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list
               sort < exp > expected-file-list
               set +x
@@ -555,12 +541,11 @@ jobs:
           run:
             path: /bin/bash
             args:
-            - -ec
+            - -ecx
             - |
-              set -e
               apt update -q
               apt install -qq -y --no-install-recommends git
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//')
               url=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}
               BINARY_EXTENSIONS="jar|war|class|exe|dll|o|so|obj|bin|out|pyc"
@@ -607,13 +592,38 @@ jobs:
           run:
             path: /bin/bash
             args:
-            - -ec
+            - -ecx
             - |
-              set -e
               apt update -q
               apt install -qq -y --no-install-recommends unzip git
-              FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##')
+              FULL_VERSION=$(cd geode && git fetch && git describe --tags | sed -e 's#^rel/v##')
               ./geode-develop/dev-tools/release/license_review.sh -v $FULL_VERSION
+  - name: all-passed
+    serial: true
+    public: true
+    plan:
+      - in_parallel:
+        - get: geode
+          trigger: true
+          passed:
+            - verify-license
+            - upthewaterspout
+            - run-gfsh-from-tgz
+            - verify-no-binaries
+            - build-geode-from-tag
+            - build-geode-from-src-tgz
+            - verify-expected-files-and-keys
+        - get: geode-examples
+          passed:
+            - run-geode-examples-jdk11
+            - run-geode-examples-from-src-tgz-jdk8
+        - get: geode-native
+          passed:
+            - build-geode-native-from-tag
+            - build-geode-native-from-src-tgz
+        - get: geode-benchmarks
+          passed:
+            - benchmarks-test
 EOF
 fly -t concourse.apachegeode-ci.info-main login --team-name main --concourse-url https://concourse.apachegeode-ci.info/
 fly -t concourse.apachegeode-ci.info-main set-pipeline -p apache-support-${VERSION_MM//./-}-rc -c $PIPEYML

dev-tools/release/license_review.sh

@@ -132,6 +132,7 @@ function listJarsInWar() {
 }
 
 function extractLicense() {
+  [ "$SKIP_LICENSES" != "true" ] || return 0
   war=$1
   rm -Rf tmpl
   mkdir tmpl

dev-tools/release/prepare_rc.sh

@@ -165,6 +165,7 @@ git clone --single-branch --branch support/${VERSION_MM} [email protected]:apache/g
 git clone --single-branch --branch support/${VERSION_MM} [email protected]:apache/geode-native.git
 git clone --single-branch --branch develop [email protected]:apache/geode-native.git geode-native-develop
 git clone --single-branch --branch support/${VERSION_MM} [email protected]:apache/geode-benchmarks.git
+git clone --single-branch --branch develop [email protected]:apache/geode-benchmarks.git geode-benchmarks-develop
 git clone --single-branch --branch master [email protected]:Homebrew/homebrew-core.git
 
 svn checkout https://dist.apache.org/repos/dist --depth empty
@@ -347,9 +348,10 @@ echo "============================================================"
 echo "Publishing artifacts to nexus staging manager..."
 echo "PLEASE NOTE, the 2nd prompt will be for your apache (not gpg) password.  Pay attention as the prompts look very similar."
 echo "============================================================"
+publishcmd="./gradlew publish --no-parallel -Pversion=${VERSION} -Paskpass -Psigning.keyId=${SIGNING_KEY} -Psigning.secretKeyRingFile=${HOME}/.gnupg/secring.gpg -PmavenUsername=${APACHE_USERNAME}"
 set -x
 cd ${GEODE}
-./gradlew publish -Pversion=${VERSION} -Paskpass -Psigning.keyId=${SIGNING_KEY} -Psigning.secretKeyRingFile=${HOME}/.gnupg/secring.gpg -PmavenUsername=${APACHE_USERNAME}
+sh -c "$publishcmd"
 set +x
 
 
@@ -360,6 +362,7 @@ echo "============================================================"
 cd ${GEODE}/../..
 echo "1. Go to https://repository.apache.org, login as ${APACHE_USERNAME}, and click on Staging Repositories"
 echo "2. If there is a prior ${VERSION} RC, select it and click Drop."
+echo "2b.If publication got split between two staging repos, drop one of them then run: pushd ${GEODE}; $publishcmd; popd"
 echo '3. Make a note of the 4-digit ID of the current ("implicitly created") staging repo.'
 echo '4. Select the current staging repo and click Close.'
 echo '5. Wait ~10 seconds and then refresh the page to confirm that status has become "Closed"'