[buildkite] Migrate buildkite pipeline from AWS to GCP (#4273)

* Migrate buildkite pipeline from aws to gcp * Add common block to unit test step * Add kubernetes plugin to Codegen step * Make unit test pass in pipeline * Make unit test pass in pipeline
m3db · Jun 25, 2024 · 621d933 · 621d933
1 parent b7297e8
commit 621d933
Showing 1 changed file with 50 additions and 161 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -49,6 +49,8 @@ common: &common
 env:
   SKIP_CODECOV: "true"
 
+# Buildkite pipeline is currently broken, and we are working on fixing it. In the meantime, we are temporarily
+# disabling steps until it is fixed. ISSUE: https://github.com/m3db/m3/issues/4274
 steps:
 #  - name: "Codegen"
 #    env:
@@ -57,6 +59,11 @@ steps:
 #    plugins:
 #      gopath-checkout#v1.0.1:
 #        import: github.com/m3db/m3
+#      docker#v3.0.1:
+#        image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1
+#        volumes:
+#          - .:/go/src/github.com/m3db/m3
+#          - /var/run/docker.sock:/var/run/docker.sock
 #      kubernetes:
 #        <<: *kubernetes
 #        podSpec:
@@ -67,267 +74,149 @@ steps:
 #                - |-
 #                  make clean install-vendor-m3 test-all-gen
 #    <<: *common
-#  - name: "Unit %n"
-#    parallelism: 4
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-base
-#    <<: *common
+  - name: "Unit %n"
+    parallelism: 4
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-base
+    <<: *common
 #  - name: "Big Unit %n"
+#    command: make clean install-vendor-m3 test-big-base
 #    parallelism: 2
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-big-base
 #    <<: *common
 #  - name: "Services, Tools"
+#    command: make clean install-vendor-m3 services tools
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 services tools
 #    <<: *common
 #  - name: "Lint"
+#    command: make clean lint
 #    env:
 #      CGO_ENABLED: 0
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean lint
 #    <<: *common
 #  - name: "Integration (:docker:)"
+#    command: make clean install-vendor-m3 docker-integration-test
 #    parallelism: 2
 #    env:
 #      CGO_ENABLED: 0
 #      GIMME_GO_VERSION: 1.18.x
 #    plugins:
-#      docker#v3.0.1:
-#        image: golang:1.18-bullseye
-#        always-pull: true
-#        volumes:
-#          - .:/go/src/github.com/m3db/m3
-#          - /var/run/docker.sock:/var/run/docker.sock
-#      gopath-checkout#v1.0.1:
-#        import: github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 docker-integration-test
+#        gopath-checkout#v1.0.1:
+#          import: github.com/m3db/m3
 #    <<: *common
 #  - label: "M3 Cluster Integration Tests"
+#    command: make clean test-ci-cluster-integration
 #    env:
 #      CGO_ENABLED: 0
 #      GIMME_GO_VERSION: 1.18.x
 #    plugins:
 #      gopath-checkout#v1.0.1:
 #        import: github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean test-ci-cluster-integration
 #    <<: *common
 #  - label: "M3 Cluster Integration Test Harness Tests"
+#    command: make clean test-ci-test-harness
 #    skip: "NB(nate): temporarily disabling to resolve some tests flakes"
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean test-ci-test-harness
 #    <<: *common
 #  - name: "Prometheus compatibility (:docker:)"
+#    command: make clean install-vendor-m3 docker-compatibility-test
 #    parallelism: 1
 #    env:
 #      CGO_ENABLED: 0
 #      GIMME_GO_VERSION: 1.18.x
 #    plugins:
 #        gopath-checkout#v1.0.1:
 #          import: github.com/m3db/m3
-#        kubernetes:
-#          <<: *kubernetes
-#          podSpec:
-#            <<: *podSpec
-#            containers:
-#              - <<: *commandContainer
-#                command:
-#                  - |-
-#                    make clean install-vendor-m3 docker-compatibility-test
 #    <<: *common
 #  - name: "Integration (dbnode Recently Read) %n"
 #    parallelism: 2
+#    command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
 #    <<: *common
 #  - name: "Integration (dbnode LRU) %n"
 #    parallelism: 2
+#    command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
 #    <<: *common
 #  - name: "Integration (aggregator TCP client) %n"
 #    parallelism: 1
+#    command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp
 #    <<: *common
 #  - name: "Integration (aggregator m3msg client) %n"
 #    parallelism: 1
+#    command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg
 #    <<: *common
 #  - label: "Integration (m3em, cluster, msg, metrics) %n"
 #    parallelism: 4
+#    command: make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
 #    <<: *common
 #  - name: "Documentation tests"
+#    command: make clean install-vendor-m3 docs-test
 #    env:
 #      CGO_ENABLED: 0
 #      GIMME_GO_VERSION: 1.18.x
 #    plugins:
 #        gopath-checkout#v1.0.1:
 #          import: github.com/m3db/m3
-#        kubernetes:
-#          <<: *kubernetes
-#          podSpec:
-#            <<: *podSpec
-#            containers:
-#              - <<: *commandContainer
-#                command:
-#                  - |-
-#                    make clean install-vendor-m3 docs-test
 #    <<: *common
 #  - label: "FOSSA license scan"
+#    command: make clean install-vendor-m3 fossa
 #    plugins:
 #      docker-compose#v2.5.1:
 #        run: app
 #        workdir: /go/src/github.com/m3db/m3
 #        env:
 #          - FOSSA_API_KEY
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 fossa
 #    <<: *common
-  - name: "Check for docker and docs builds :docker: :books:"
-    commands:
-    - ".ci/docker/check_do_docker.sh"
-    - ".buildkite/scripts/check_do_docs.sh"
-    agents:
-      queue: "buildkite-gcp"
-    timeout_in_minutes: 10
-    retry:
-      automatic:
-        limit: 1
-      manual: true
+#  - name: "Check for docker and docs builds :docker: :books:"
+#    commands:
+#    - ".ci/docker/check_do_docker.sh"
+#    - ".buildkite/scripts/check_do_docs.sh"
+#    agents:
+#      queue: init
+#    timeout_in_minutes: 10
+#    retry:
+#      automatic:
+#        limit: 1
+#      manual: true