From 621d933717300cfcfeda33e716379a68ce9e63b0 Mon Sep 17 00:00:00 2001 From: kentzeng12 <114431467+kentzeng12@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:05:31 -0700 Subject: [PATCH] [buildkite] Migrate buildkite pipeline from AWS to GCP (#4273) * Migrate buildkite pipeline from aws to gcp * Add common block to unit test step * Add kubernetes plugin to Codegen step * Make unit test pass in pipeline * Make unit test pass in pipeline --- .buildkite/pipeline.yml | 211 ++++++++++------------------------------ 1 file changed, 50 insertions(+), 161 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index df282a1d68..ef9f98471c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -49,6 +49,8 @@ common: &common env: SKIP_CODECOV: "true" +# Buildkite pipeline is currently broken, and we are working on fixing it. In the meantime, we are temporarily +# disabling steps until it is fixed. ISSUE: https://github.com/m3db/m3/issues/4274 steps: # - name: "Codegen" # env: @@ -57,6 +59,11 @@ steps: # plugins: # gopath-checkout#v1.0.1: # import: github.com/m3db/m3 +# docker#v3.0.1: +# image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1 +# volumes: +# - .:/go/src/github.com/m3db/m3 +# - /var/run/docker.sock:/var/run/docker.sock # kubernetes: # <<: *kubernetes # podSpec: @@ -67,128 +74,75 @@ steps: # - |- # make clean install-vendor-m3 test-all-gen # <<: *common -# - name: "Unit %n" -# parallelism: 4 -# plugins: -# docker-compose#v2.5.1: -# run: app -# workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-base -# <<: *common + - name: "Unit %n" + parallelism: 4 + plugins: + docker-compose#v2.5.1: + run: app + workdir: /go/src/github.com/m3db/m3 + kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make clean install-vendor-m3 test-base + <<: *common # - name: "Big Unit %n" +# command: make clean install-vendor-m3 test-big-base # parallelism: 2 # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-big-base # <<: *common # - name: "Services, Tools" +# command: make clean install-vendor-m3 services tools # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 services tools # <<: *common # - name: "Lint" +# command: make clean lint # env: # CGO_ENABLED: 0 # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean lint # <<: *common # - name: "Integration (:docker:)" +# command: make clean install-vendor-m3 docker-integration-test # parallelism: 2 # env: # CGO_ENABLED: 0 # GIMME_GO_VERSION: 1.18.x # plugins: -# docker#v3.0.1: -# image: golang:1.18-bullseye -# always-pull: true -# volumes: -# - .:/go/src/github.com/m3db/m3 -# - /var/run/docker.sock:/var/run/docker.sock -# gopath-checkout#v1.0.1: -# import: github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 docker-integration-test +# gopath-checkout#v1.0.1: +# import: github.com/m3db/m3 # <<: *common # - label: "M3 Cluster Integration Tests" +# command: make clean test-ci-cluster-integration # env: # CGO_ENABLED: 0 # GIMME_GO_VERSION: 1.18.x # plugins: # gopath-checkout#v1.0.1: # import: github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean test-ci-cluster-integration # <<: *common # - label: "M3 Cluster Integration Test Harness Tests" +# command: make clean test-ci-test-harness # skip: "NB(nate): temporarily disabling to resolve some tests flakes" # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean test-ci-test-harness # <<: *common # - name: "Prometheus compatibility (:docker:)" +# command: make clean install-vendor-m3 docker-compatibility-test # parallelism: 1 # env: # CGO_ENABLED: 0 @@ -196,138 +150,73 @@ steps: # plugins: # gopath-checkout#v1.0.1: # import: github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 docker-compatibility-test # <<: *common # - name: "Integration (dbnode Recently Read) %n" # parallelism: 2 +# command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read # <<: *common # - name: "Integration (dbnode LRU) %n" # parallelism: 2 +# command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru # <<: *common # - name: "Integration (aggregator TCP client) %n" # parallelism: 1 +# command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp # <<: *common # - name: "Integration (aggregator m3msg client) %n" # parallelism: 1 +# command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg # <<: *common # - label: "Integration (m3em, cluster, msg, metrics) %n" # parallelism: 4 +# command: make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics # <<: *common # - name: "Documentation tests" +# command: make clean install-vendor-m3 docs-test # env: # CGO_ENABLED: 0 # GIMME_GO_VERSION: 1.18.x # plugins: # gopath-checkout#v1.0.1: # import: github.com/m3db/m3 -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 docs-test # <<: *common # - label: "FOSSA license scan" +# command: make clean install-vendor-m3 fossa # plugins: # docker-compose#v2.5.1: # run: app # workdir: /go/src/github.com/m3db/m3 # env: # - FOSSA_API_KEY -# kubernetes: -# <<: *kubernetes -# podSpec: -# <<: *podSpec -# containers: -# - <<: *commandContainer -# command: -# - |- -# make clean install-vendor-m3 fossa # <<: *common - - name: "Check for docker and docs builds :docker: :books:" - commands: - - ".ci/docker/check_do_docker.sh" - - ".buildkite/scripts/check_do_docs.sh" - agents: - queue: "buildkite-gcp" - timeout_in_minutes: 10 - retry: - automatic: - limit: 1 - manual: true +# - name: "Check for docker and docs builds :docker: :books:" +# commands: +# - ".ci/docker/check_do_docker.sh" +# - ".buildkite/scripts/check_do_docs.sh" +# agents: +# queue: init +# timeout_in_minutes: 10 +# retry: +# automatic: +# limit: 1 +# manual: true