GEODE-8348: Add benchmarks EC2 image builder job. (#5367)

* Add EC2 builder job to images. * Benchmarks job uses branch-specific image. * Change benchmarks source repository location to the deployed fork's repo instead of forcing apache. * download our own copy of fly to use when deploying pipelines via deploy_meta.sh.
apache · Jul 15, 2020 · 0a5a629 · 0a5a629
1 parent 1ebd307
commit 0a5a629
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 14 deletions.
diff --git a/.gitignore b/.gitignore
@@ -34,3 +34,4 @@ bin/
 .metadata/
 /ci/pipelines/meta/meta.properties.local
 /dev-tools/docker/docs/build-docs-output.txt
+fly
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
@@ -176,7 +176,7 @@ resources:
   source:
     branch: {{benchmarks.benchmark_branch}}
     depth: 1
-    uri: https://github.com/apache/geode-benchmarks.git
+    uri: https://github.com/((geode-fork))/geode-benchmarks.git
 - name: geode-build-version
   type: semver
   source:
@@ -487,6 +487,7 @@ jobs:
           FLAGS: {{ run_var.flag }}
           TAG_POSTFIX: {{ run_var.title }}
           TEST_OPTIONS: {{ run_var.options }}
+          PURPOSE: ((pipeline-prefix))geode-benchmarks
         run:
           path: geode-ci/ci/scripts/run_benchmarks.sh
         inputs:

diff --git a/ci/pipelines/images/jinja.template.yml b/ci/pipelines/images/jinja.template.yml
@@ -63,6 +63,14 @@ resources:
     repository: openjdk
     tag: 8
 
+- name: geode-benchmarks-image
+  type: git
+  source:
+    branch: ((geode-build-branch))
+    paths:
+    - infrastructure/scripts/aws/image
+    uri: https://github.com/((geode-fork))/geode-benchmarks.git
+
 - name: google-geode-builder
   type: git
   source:
@@ -228,6 +236,39 @@ jobs:
       build: build-concourse-dockerfile/ci/docker
       tag_as_latest: true
 
+- name: build-geode-benchmarks-image
+  public: ((public-pipelines))
+  serial: true
+  plan:
+  - in_parallel:
+    - get: geode-benchmarks-image
+      trigger: true
+    - get: alpine-tools-docker-image
+      passed: [build-alpine-tools-docker-image]
+  - task: build-image
+    timeout: 1h
+    image: alpine-tools-docker-image
+    config:
+      inputs:
+      - name: geode-benchmarks-image
+      outputs:
+      - name: results
+      platform: linux
+      params:
+        AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id))
+        AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key))
+        AWS_DEFAULT_REGION: us-west-2
+        AWS_REGION: us-west-2
+        PURPOSE: ((pipeline-prefix))geode-benchmarks
+      run:
+        path: bash
+        args:
+          - -ec
+          - |-
+            pushd geode-benchmarks-image/infrastructure/scripts/aws/image
+            packer build -var "purpose=${PURPOSE}" packer.json
+            popd
+
 - name: build-google-geode-builder
   public: ((public-pipelines))
   serial: true

diff --git a/ci/pipelines/meta/deploy_meta.sh b/ci/pipelines/meta/deploy_meta.sh
@@ -108,16 +108,37 @@ YML
 
   python3 ../render.py jinja.template.yml --variable-file ../shared/jinja.variables.yml repository.yml pipelineProperties.yml --environment ../shared/ --output ${SCRIPTDIR}/generated-pipeline.yml --debug || exit 1
 
+  unamestr=$(uname)
+  platform='unknown'
+  if [[ "${unamestr}" == 'Darwin' ]]; then
+    platform='darwin'
+  elif [[ "${unamestr}" == 'Linux' ]]; then
+    platform='linux'
+  fi
+
+  FLY_URL="${CONCOURSE_URL}/api/v1/cli?arch=amd64&platform=${platform}"
+  FLY="${SCRIPTDIR}/fly"
+  if [[ ! -e "${FLY}" ]]; then
+    curl -so ${FLY} ${FLY_URL}
+  fi
+  chmod +x ${FLY}
+  set +e
+  if [[ ! $(${FLY} targets | grep "${FLY_TARGET}") ]]; then
+    echo "Creating target for ${FLY_TARGET}"
+    ${FLY} -t ${FLY_TARGET} login -c "${CONCOURSE_URL}"
+  else
+    echo "Target ${FLY_TARGET} already exists."
+  fi
+
   set -e
   if [[ ${UPSTREAM_FORK} != "apache" ]]; then
-    fly -t ${FLY_TARGET} status || \
-    fly -t ${FLY_TARGET} login \
+    ${FLY} -t ${FLY_TARGET} status || \
+    ${FLY} -t ${FLY_TARGET} login \
            --team-name ${CONCOURSE_TEAM} \
            --concourse-url=${CONCOURSE_URL}
   fi
 
-  fly -t ${FLY_TARGET} sync
-  fly -t ${FLY_TARGET} set-pipeline \
+  ${FLY} -t ${FLY_TARGET} set-pipeline \
     -p ${META_PIPELINE} \
     --config ${SCRIPTDIR}/generated-pipeline.yml \
     --var artifact-bucket=${ARTIFACT_BUCKET} \
@@ -145,19 +166,19 @@ popd 2>&1 > /dev/null
 function jobStatus {
   PIPELINE=$1
   JOB=$2
-  fly jobs -t ${FLY_TARGET} -p ${PIPELINE}|awk "/${JOB}/"'{if($2=="yes")print "paused";else if($4!="n/a")print $4; else print $3}'
+  ${FLY} jobs -t ${FLY_TARGET} -p ${PIPELINE}|awk "/${JOB}/"'{if($2=="yes")print "paused";else if($4!="n/a")print $4; else print $3}'
 }
 
 function triggerJob {
   PIPELINE=$1
   JOB=$2
-  (set -x ; fly trigger-job -t ${FLY_TARGET} -j ${PIPELINE}/${JOB})
+  (set -x ; ${FLY} trigger-job -t ${FLY_TARGET} -j ${PIPELINE}/${JOB})
 }
 
 function pauseJob {
   PIPELINE=$1
   JOB=$2
-  (set -x ; fly pause-job -t ${FLY_TARGET} -j ${PIPELINE}/${JOB})
+  (set -x ; ${FLY} pause-job -t ${FLY_TARGET} -j ${PIPELINE}/${JOB})
 }
 
 function pauseJobs {
@@ -180,7 +201,7 @@ function pauseNewJobs {
 function unpauseJob {
   PIPELINE=$1
   JOB=$2
-  (set -x ; fly unpause-job -t ${FLY_TARGET} -j ${PIPELINE}/${JOB})
+  (set -x ; ${FLY} unpause-job -t ${FLY_TARGET} -j ${PIPELINE}/${JOB})
 }
 
 function unpauseJobs {
@@ -193,12 +214,12 @@ function unpauseJobs {
 
 function unpausePipeline {
   PIPELINE=$1
-  (set -x ; fly -t ${FLY_TARGET} unpause-pipeline -p ${PIPELINE})
+  (set -x ; ${FLY} -t ${FLY_TARGET} unpause-pipeline -p ${PIPELINE})
 }
 
 function exposePipeline {
   PIPELINE=$1
-  (set -x ; fly -t ${FLY_TARGET} expose-pipeline -p ${PIPELINE})
+  (set -x ; ${FLY} -t ${FLY_TARGET} expose-pipeline -p ${PIPELINE})
 }
 
 function exposePipelines {

diff --git a/ci/pipelines/meta/jinja.template.yml b/ci/pipelines/meta/jinja.template.yml
@@ -416,7 +416,7 @@ jobs:
   public: ((!public-pipelines))
   serial: true
   plan:
-  - aggregate:
+  - in_parallel:
     - get: meta-mini-dockerfile
       trigger: true
   - put: meta-mini-image

diff --git a/ci/scripts/run_benchmarks.sh b/ci/scripts/run_benchmarks.sh
@@ -34,10 +34,15 @@ source concourse-metadata-resource/concourse_metadata
 CLUSTER_TAG="${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}-${BUILD_ID}${TAG_POSTFIX}"
 RESULTS_DIR=$(pwd)/results/benchmarks-${CLUSTER_TAG}
 
+if [[ ! -z "${PURPOSE}" ]]; then
+  PURPOSE_OPTION="-p ${PURPOSE}"
+fi
+
 CLUSTER_COUNT=4
 BENCHMARKS_BRANCH=${BENCHMARKS_BRANCH:-develop}
 
 GEODE_REPO=${GEODE_REPO:-$(cd geode && git remote get-url origin)}
+BENCHMARKS_REPO=${BENCHMARKS_REPO:-$(cd geode-benchmarks && git remote get-url origin)}
 BASELINE_REPO=${BASELINE_REPO:-${GEODE_REPO}}
 
 pushd geode
@@ -47,7 +52,7 @@ popd
 input="$(pwd)/results/failedTests"
 
 pushd geode-benchmarks/infrastructure/scripts/aws/
-./launch_cluster.sh -t ${CLUSTER_TAG} -c ${CLUSTER_COUNT} --ci
+./launch_cluster.sh -t ${CLUSTER_TAG} -c ${CLUSTER_COUNT} ${PURPOSE_OPTION} --ci
 
 # test retry loop - Check if any tests have failed. If so, overwrite the TEST_OPTIONS with only the
 # failed tests. Test failures only result in an exit code of 1 when on the last iteration of loop.
@@ -80,7 +85,7 @@ do
 
   ./run_on_cluster.sh -t ${CLUSTER_TAG} -- pkill -9 java
   ./run_on_cluster.sh -t ${CLUSTER_TAG} -- rm /home/geode/locator10334view.dat;
-  ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} -r ${GEODE_REPO} ${BASELINE_OPTION} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci',${METADATA_BASELINE},'baseline_branch':'${BASELINE_BRANCH}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS}
+  ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} -r ${GEODE_REPO} -p ${BENCHMARKS_REPO} ${BASELINE_OPTION} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci',${METADATA_BASELINE},'baseline_branch':'${BASELINE_BRANCH}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS}
 
   if [[ $? -eq 0 ]]; then
     break;