build(release): Update release script. (hypermodeinc#5703)

Check for all the commands that are needed for the script from the start and fail early. Simplify the script to clean up unneeded steps since the move to Go Modules (Go Modules is still turned off for Ratel). The script clones the local Dgraph repo instead of a new one so that a release tag can be created and used without first pushing it to GitHub. Use ZIP for Windows binaries since zip files are more common in Windows compared to tarballs. Release builds are built with Go 1.15.5.
brandonheng168 · Feb 17, 2021 · 832ebb9 · 832ebb9
1 parent 589bc54
commit 832ebb9
Show file tree

Hide file tree

Showing 3 changed files with 151 additions and 54 deletions.
diff --git a/contrib/release.sh b/contrib/release.sh
@@ -6,13 +6,61 @@
 # binaries and prepare them such that any human or script can then pick these up
 # and use them as they deem fit.
 
+# Path to this script
+scriptdir="$(cd "$(dirname $0)">/dev/null; pwd)"
+# Path to the root repo directory
+repodir="$(cd "$scriptdir/..">/dev/null; pwd)"
+
 # Output colors
 RED='\033[91;1m'
 RESET='\033[0m'
 
+print_error() {
+    printf "$RED$1$RESET\n"
+}
+
+exit_error() {
+    print_error "$@"
+    exit 1
+}
+check_command_exists() {
+    if ! command -v "$1" > /dev/null; then
+        exit_error "$1: command not found"
+    fi
+}
+
+if [ "$#" -lt 1 ]; then
+    exit_error "Usage: $0 commitish [docker_tag]
+
+Examples:
+Build v1.2.3 release binaries
+  $0 v1.2.3
+Build dev/feature-branch branch and tag as dev-abc123 for the Docker image
+  $0 dev/feature-branch dev-abc123"
+fi
+
+export NVM_DIR="$HOME/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
+[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
+
+# TODO Check if ports 8000, 9080, or 6080 are bound already and error out early.
+
+check_command_exists strip
+check_command_exists make
+check_command_exists gcc
+check_command_exists go
+check_command_exists docker
+check_command_exists docker-compose
+check_command_exists nvm
+check_command_exists npm
+check_command_exists protoc
+check_command_exists shasum
+check_command_exists tar
+check_command_exists zip
+
 # Don't use standard GOPATH. Create a new one.
 unset GOBIN
-GOPATH="/tmp/go"
+export GOPATH="/tmp/go"
 if [ -d $GOPATH ]; then
    chmod -R 755 $GOPATH
 fi
@@ -23,15 +71,14 @@ mkdir $GOPATH
 PATH="$GOPATH/bin:$PATH"
 
 # The Go version used for release builds must match this version.
-GOVERSION="1.14.1"
-
-# Turn off go modules by default. Only enable go modules when needed.
-export GO111MODULE=off
+GOVERSION=${GOVERSION:-"1.15.5"}
 
 TAG=$1
-# The Docker tag should not contain a slash e.g. feature/issue1234
-# The initial slash is taken from the repository name dgraph/dgraph:tag
-DTAG=$(echo "$TAG" | tr '/' '-')
+
+(
+    cd "$repodir"
+    git cat-file -e "$TAG"
+) || exit_error "Ref $TAG does not exist"
 
 # DO NOT change the /tmp/build directory, because Dockerfile also picks up binaries from there.
 TMP="/tmp/build"
@@ -48,97 +95,95 @@ echo "Building Dgraph for tag: $TAG"
 set -e
 set -o xtrace
 
-# Check for existence of strip tool.
-type strip
-type shasum
-
 ratel_release="github.com/dgraph-io/ratel/server.ratelVersion"
 release="github.com/dgraph-io/dgraph/x.dgraphVersion"
+codenameKey="github.com/dgraph-io/dgraph/x.dgraphCodename"
 branch="github.com/dgraph-io/dgraph/x.gitBranch"
 commitSHA1="github.com/dgraph-io/dgraph/x.lastCommitSHA"
 commitTime="github.com/dgraph-io/dgraph/x.lastCommitTime"
+jemallocXgoFlags=
 
-echo "Using Go version"
-go version
-if [[ ! "$(go version)" =~ $GOVERSION ]]; then
-   echo -e "${RED}Go version is NOT expected. Should be $GOVERSION.${RESET}"
-   exit 1
-fi
-
-go get -u github.com/jteeuwen/go-bindata/...
-go get -d google.golang.org/grpc
-go get -u github.com/prometheus/client_golang/prometheus
-go get -u github.com/dgraph-io/dgo
-# go get github.com/stretchr/testify/require
-go get -u github.com/dgraph-io/badger
-go get -u github.com/golang/protobuf/protoc-gen-go
-go get -u github.com/gogo/protobuf/protoc-gen-gofast
-go get -u src.techknowlogick.com/xgo
-
-pushd $GOPATH/src/google.golang.org/grpc
-  git checkout v1.13.0
-popd
+go install src.techknowlogick.com/xgo
+mkdir ~/.xgo-cache || echo "Continuing"
 
 basedir=$GOPATH/src/github.com/dgraph-io
+mkdir -p "$basedir"
+
 # Clone Dgraph repo.
 pushd $basedir
-  git clone https://github.com/dgraph-io/dgraph.git
+  git clone "$repodir"
 popd
 
 pushd $basedir/dgraph
-  git pull
   git checkout $TAG
   # HEAD here points to whatever is checked out.
   lastCommitSHA1=$(git rev-parse --short HEAD)
+  codename="$(awk '/^BUILD_CODENAME/ { print $NF }' ./dgraph/Makefile)"
   gitBranch=$(git rev-parse --abbrev-ref HEAD)
   lastCommitTime=$(git log -1 --format=%ci)
   release_version=$(git describe --always --tags)
 popd
 
+# The Docker tag should not contain a slash e.g. feature/issue1234
+# The initial slash is taken from the repository name dgraph/dgraph:tag
+DOCKER_TAG=${2:-$release_version}
+
 # Regenerate protos. Should not be different from what's checked in.
 pushd $basedir/dgraph/protos
+  # We need to fetch the modules to get the correct proto files. e.g., for
+  # badger and dgo
+  go get -d -v ../dgraph
+
   make regenerate
   if [[ "$(git status --porcelain)" ]]; then
       echo >&2 "Generated protos different in release."
       exit 1
   fi
 popd
 
+# Clone Badger repo.
+pushd $basedir
+  git clone https://github.com/dgraph-io/badger.git
+  # Check out badger version specific to the Dgraph release.
+  cd ./badger
+  ref="$(grep github.com/dgraph-io/badger/v3 $basedir/dgraph/go.mod | grep -v replace |  awk '{ print $2 }')"
+  commitish="$(echo "$ref" | awk -F- '{ print $NF }')"
+  git checkout "$commitish"
+popd
+
 # Clone ratel repo.
 pushd $basedir
   git clone https://github.com/dgraph-io/ratel.git
 popd
 
 pushd $basedir/ratel
-  git pull
-  source ~/.nvm/nvm.sh
   nvm install --lts
-  ./scripts/build.prod.sh
+  (export GO111MODULE=off; ./scripts/build.prod.sh)
   ./scripts/test.sh
 popd
 
 # Build Windows.
 pushd $basedir/dgraph/dgraph
-  xgo -go="go-$GOVERSION" --targets=windows/amd64 -ldflags \
-      "-X $release=$release_version -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" .
+  xgo -go="go-$GOVERSION" --targets=windows/amd64 -buildmode=exe -ldflags \
+      "-X $release=$release_version -X $codenameKey=$codename -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" .
   mkdir $TMP/windows
   mv dgraph-windows-4.0-amd64.exe $TMP/windows/dgraph.exe
 popd
 
 pushd $basedir/badger/badger
-  xgo -go="go-$GOVERSION" --targets=windows/amd64 .
+  xgo -go="go-$GOVERSION" --targets=windows/amd64  -buildmode=exe .
   mv badger-windows-4.0-amd64.exe $TMP/windows/badger.exe
 popd
 
 pushd $basedir/ratel
-  xgo -go="go-$GOVERSION" --targets=windows/amd64 -ldflags "-X $ratel_release=$release_version" .
+  xgo -go="go-$GOVERSION" --targets=windows/amd64 -ldflags "-X $ratel_release=$release_version"  -buildmode=exe .
   mv ratel-windows-4.0-amd64.exe $TMP/windows/dgraph-ratel.exe
 popd
 
 # Build Darwin.
 pushd $basedir/dgraph/dgraph
   xgo -go="go-$GOVERSION" --targets=darwin-10.9/amd64 -ldflags \
-  "-X $release=$release_version -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" .
+  "-X $release=$release_version -X $codenameKey=$codename -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" .
   mkdir $TMP/darwin
   mv dgraph-darwin-10.9-amd64 $TMP/darwin/dgraph
 popd
@@ -156,14 +201,14 @@ popd
 # Build Linux.
 pushd $basedir/dgraph/dgraph
   xgo -go="go-$GOVERSION" --targets=linux/amd64 -ldflags \
-      "-X $release=$release_version -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" .
+      "-X $release=$release_version -X $codenameKey=$codename -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" --tags=jemalloc -deps=https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2  --depsargs='--with-jemalloc-prefix=je_ --with-malloc-conf=background_thread:true,metadata_thp:auto --enable-prof' .
   strip -x dgraph-linux-amd64
   mkdir $TMP/linux
   mv dgraph-linux-amd64 $TMP/linux/dgraph
 popd
 
 pushd $basedir/badger/badger
-  xgo -go="go-$GOVERSION" --targets=linux/amd64 .
+  xgo -go="go-$GOVERSION" --targets=linux/amd64 --tags=jemalloc -deps=https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2  --depsargs='--with-jemalloc-prefix=je_ --with-malloc-conf=background_thread:true,metadata_thp:auto --enable-prof' .
   strip -x badger-linux-amd64
   mv badger-linux-amd64 $TMP/linux/badger
 popd
@@ -177,25 +222,45 @@ popd
 createSum () {
   os=$1
   echo "Creating checksum for $os"
-  pushd $TMP/$os
-    csum=$(shasum -a 256 dgraph | awk '{print $1}')
-    echo $csum /usr/local/bin/dgraph >> ../dgraph-checksum-$os-amd64.sha256
-    csum=$(shasum -a 256 dgraph-ratel | awk '{print $1}')
-    echo $csum /usr/local/bin/dgraph-ratel >> ../dgraph-checksum-$os-amd64.sha256
-  popd
+  if [[ "$os" != "windows" ]]; then
+    pushd $TMP/$os
+      csum=$(shasum -a 256 dgraph | awk '{print $1}')
+      echo $csum /usr/local/bin/dgraph >> ../dgraph-checksum-$os-amd64.sha256
+      csum=$(shasum -a 256 dgraph-ratel | awk '{print $1}')
+      echo $csum /usr/local/bin/dgraph-ratel >> ../dgraph-checksum-$os-amd64.sha256
+    popd
+  else
+    pushd $TMP/$os
+      csum=$(shasum -a 256 dgraph.exe | awk '{print $1}')
+      echo $csum dgraph.exe >> ../dgraph-checksum-$os-amd64.sha256
+      csum=$(shasum -a 256 dgraph-ratel.exe | awk '{print $1}')
+      echo $csum dgraph-ratel.exe >> ../dgraph-checksum-$os-amd64.sha256
+    popd
+  fi
 }
 
 createSum darwin
 createSum linux
+createSum windows
 
 # Create Docker image.
 cp $basedir/dgraph/contrib/Dockerfile $TMP
 pushd $TMP
-  docker build -t dgraph/dgraph:$DTAG .
+  # Get a fresh ubuntu:latest image each time
+  # Don't rely on whatever "latest" version
+  # happens to be on the machine.
+  docker pull ubuntu:latest
+
+  docker build -t dgraph/dgraph:$DOCKER_TAG .
 popd
 rm $TMP/Dockerfile
 
-# Create the tars and delete the binaries.
+# Create Docker standalone image.
+pushd $basedir/dgraph/contrib/standalone
+  make DGRAPH_VERSION=$DOCKER_TAG
+popd
+
+# Create the tar and delete the binaries.
 createTar () {
   os=$1
   echo "Creating tar for $os"
@@ -205,10 +270,39 @@ createTar () {
   rm -Rf $TMP/$os
 }
 
-createTar windows
+# Create the zip and delete the binaries.
+createZip () {
+  os=$1
+  echo "Creating zip for $os"
+  pushd $TMP/$os
+    zip -r ../dgraph-$os-amd64.zip *
+  popd
+  rm -Rf $TMP/$os
+}
+
+createZip windows
 createTar darwin
 createTar linux
 
 echo "Release $TAG is ready."
-docker run -it dgraph/dgraph:$DTAG dgraph
+docker run dgraph/dgraph:$DOCKER_TAG dgraph
 ls -alh $TMP
+
+set +o xtrace
+echo "To release:"
+if git show-ref -q --verify "refs/tags/$TAG"; then
+    echo
+    echo "Push the git tag"
+    echo "  git push origin $TAG"
+fi
+echo
+echo "Push the Docker tag:"
+echo "  docker push dgraph/dgraph:$DOCKER_TAG"
+echo "  docker push dgraph/standalone:$DOCKER_TAG"
+echo
+echo "If this should be the latest release, then tag"
+echo "the image as latest too."
+echo "  docker tag dgraph/dgraph:$DOCKER_TAG dgraph/dgraph:latest"
+echo "  docker tag dgraph/standalone:$DOCKER_TAG dgraph/standalone:latest"
+echo "  docker push dgraph/dgraph:latest"
+echo "  docker push dgraph/standalone:latest"
diff --git a/go.mod b/go.mod
@@ -71,4 +71,5 @@ require (
 	gopkg.in/DataDog/dd-trace-go.v1 v1.13.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.3.1
 	gopkg.in/yaml.v2 v2.2.4
+	src.techknowlogick.com/xgo v1.0.1-0.20200714173810-96de19c19b14
 )
diff --git a/go.sum b/go.sum
@@ -893,3 +893,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k=
+src.techknowlogick.com/xgo v1.0.1-0.20200714173810-96de19c19b14 h1:e8P02bjvCTmi33s1MoinoTwMwrCq7+omCS3/3+GZWXA=
+src.techknowlogick.com/xgo v1.0.1-0.20200714173810-96de19c19b14/go.mod h1:31CE1YKtDOrKTk9PSnjTpe6YbO6W/0LTYZ1VskL09oU=