From 196da729eaa46630b06c4ec07bf7f9b5f27b8917 Mon Sep 17 00:00:00 2001 From: Victor Yang Date: Wed, 24 Oct 2018 14:18:06 -0700 Subject: [PATCH] use scripted Jenkinsfile to solve version bump issue --- Jenkinsfile | 105 ++++++++++++++++++-------------------------- scripts/validate.sh | 60 +++++++++++++++---------- 2 files changed, 79 insertions(+), 86 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 19af5d4..40cb4d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,4 @@ +#!/usr/bin/env groovy /* Copyright 2018 Google LLC @@ -19,12 +20,13 @@ limitations under the License. // define containerTemplate but that has been deprecated in favor of the yaml // format // Reference: https://github.com/jenkinsci/kubernetes-plugin -pipeline { - agent { - kubernetes { - label 'k8s-infra' - defaultContainer 'jnlp' - yaml """ + +// set up pod label and GOOGLE_APPLICATION_CREDENTIALS (for Terraform) +def label = "k8s-infra" +def containerName = "k8s-node" +def GOOGLE_APPLICATION_CREDENTIALS = '/home/jenkins/dev/jenkins-deploy-dev-infra.json' + +podTemplate(label: label, yaml: """ apiVersion: v1 kind: Pod metadata: @@ -32,94 +34,71 @@ metadata: jenkins: build-node spec: containers: - - name: k8s-node - image: gcr.io/pso-helmsman-cicd/jenkins-k8s-node:1.1.0 - imagePullPolicy: Always - command: - - cat + - name: ${containerName} + image: gcr.io/pso-helmsman-cicd/jenkins-k8s-node:${env.CONTAINER_VERSION} + command: ['cat'] tty: true volumeMounts: - # Mount the docker.sock file so we can communicate with the local docker - # daemon - - name: docker-sock-volume - mountPath: /var/run/docker.sock - # Mount the local docker binary - - name: docker-bin-volume - mountPath: /usr/bin/docker # Mount the dev service account key - name: dev-key mountPath: /home/jenkins/dev volumes: - - name: docker-sock-volume - hostPath: - path: /var/run/docker.sock - - name: docker-bin-volume - hostPath: - path: /usr/bin/docker # Create a volume that contains the dev json key that was saved as a secret - name: dev-key secret: secretName: jenkins-deploy-dev-infra """ - } - } - - environment { - GOOGLE_APPLICATION_CREDENTIALS = '/home/jenkins/dev/jenkins-deploy-dev-infra.json' - } - - - stages { - stage('Lint') { - steps { - container('k8s-node') { - sh "make lint" - } - } - } + ) { + node(label) { + try { + // set env variable GOOGLE_APPLICATION_CREDENTIALS for Terraform + env.GOOGLE_APPLICATION_CREDENTIALS=GOOGLE_APPLICATION_CREDENTIALS stage('Setup') { - steps { - container('k8s-node') { - script { - env.ZONE = "${ZONE}" - env.PROJECT_ID = "${PROJECT_ID}" - env.REGION = "${REGION}" - env.KEYFILE = GOOGLE_APPLICATION_CREDENTIALS - } + container(containerName) { + // checkout code from scm i.e. commits related to the PR + checkout scm + // Setup gcloud service account access - sh "gcloud auth activate-service-account --key-file=${env.KEYFILE}" + sh "gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}" sh "gcloud config set compute/zone ${env.ZONE}" sh "gcloud config set core/project ${env.PROJECT_ID}" sh "gcloud config set compute/region ${env.REGION}" - } - } + } + stage('Lint') { + container(containerName) { + sh "make lint" + } } stage('Create') { - steps { - container('k8s-node') { + container(containerName) { sh "make create" } - } } stage('Validate') { - steps { - container('k8s-node') { + container(containerName) { sh "make validate" } - } } - } - post { - always { - container('k8s-node') { + } + catch (err) { + // if any exception occurs, mark the build as failed + // and display a detailed message on the Jenkins console output + currentBuild.result = 'FAILURE' + echo "FAILURE caught echo ${err}" + throw err + } + finally { + stage('Teardown') { + container(containerName) { sh "make teardown" sh "gcloud auth revoke" } - } + } + } } -} +} \ No newline at end of file diff --git a/scripts/validate.sh b/scripts/validate.sh index 8d06696..ff23421 100644 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -25,13 +25,18 @@ set -o nounset set -o pipefail -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +# Define retry constants +readonly MAX_COUNT=60 +readonly RETRY_COUNT=0 +readonly SLEEP=2 + +readonly ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" # shellcheck source=scripts/common.sh source "$ROOT/scripts/common.sh" -APP_NAME=$(kubectl get deployments -n default \ +readonly APP_NAME=$(kubectl get deployments -n default \ -ojsonpath='{.items[0].metadata.labels.app}') -APP_MESSAGE="deployment \"$APP_NAME\" successfully rolled out" +readonly APP_MESSAGE="deployment \"$APP_NAME\" successfully rolled out" cd "$ROOT/terraform" || exit; CLUSTER_NAME=$(terraform output cluster_name) \ ZONE=$(terraform output primary_zone) @@ -60,10 +65,11 @@ fi echo "Step 1 of the validation passed. App is deployed." + # Loop for up to 60 seconds waiting for service's IP address -EXT_IP="" for _ in {1..60} do + # Get service's ip EXT_IP=$(kubectl get svc "$APP_NAME" -n default \ -ojsonpath='{.status.loadBalancer.ingress[0].ip}') [ ! -z "$EXT_IP" ] && break @@ -82,25 +88,33 @@ EXT_PORT=$(kubectl get service "$APP_NAME" -n default \ echo "App is available at: http://$EXT_IP:$EXT_PORT" -STATUS_CODE="" -for _ in {1..60} -do - # Test service availability - STATUS_CODE=$(curl -s -o /dev/null -w '%{http_code}' "$EXT_IP:$EXT_PORT/") - [ ! -z "$STATUS_CODE" ] && break - sleep 2 - echo "Waiting for service availability..." +# Curl for the service with retries +STATUS_CODE=$(curl -s -o /dev/null -w '%{http_code}' "$EXT_IP:$EXT_PORT/") +until [[ $STATUS_CODE -eq 200 ]]; do + if [[ "${RETRY_COUNT}" -gt "${MAX_COUNT}" ]]; then + # failed with retry, lets check whatz wrong and bail + echo "Retry count exceeded. Exiting..." + # Timed out? + if [ -z "$STATUS_CODE" ] + then + echo "ERROR - Timed out waiting for service" + exit 1 + fi + # HTTP status not okay? + if [ "$STATUS_CODE" != "200" ] + then + echo "ERROR - Service is returning error" + exit 1 + fi + fi + NUM_SECONDS="$(( RETRY_COUNT * SLEEP ))" + echo "Waiting for service availability..." + echo "service / did not return an HTTP 200 response code after ${NUM_SECONDS} seconds" + sleep "${SLEEP}" + RETRY_COUNT="$(( RETRY_COUNT + 1 ))" + STATUS_CODE=$(curl -s -o /dev/null -w '%{http_code}' "$EXT_IP:$EXT_PORT/") done -if [ -z "$STATUS_CODE" ] -then - echo "ERROR - Timed out waiting for service" - exit 1 -fi - -if [ "$STATUS_CODE" != "200" ] -then - echo "ERROR - Service is returning error" - exit 1 -fi +# succeeded, let's report it +echo "service / returns an HTTP 200 response code" echo "Step 2 of the validation passed. App handles requests."