diff --git a/.buildkite/docs-build-pipeline.yml b/.buildkite/docs-build-pipeline.yml index 8137f93095..55574781db 100644 --- a/.buildkite/docs-build-pipeline.yml +++ b/.buildkite/docs-build-pipeline.yml @@ -6,7 +6,7 @@ steps: - name: "Docs build and push :books:" command: ".buildkite/scripts/docs_push.sh" agents: - queue: workers + queue: "buildkite-gcp" timeout_in_minutes: 20 # Don't let doc failures block other steps (like docker builds) soft_fail: diff --git a/.buildkite/image-release-pipeline.yml b/.buildkite/image-release-pipeline.yml index de12751191..924861c900 100644 --- a/.buildkite/image-release-pipeline.yml +++ b/.buildkite/image-release-pipeline.yml @@ -7,7 +7,7 @@ steps: env: M3_DOCKER_REPO: quay.io/m3db agents: - queue: builders + queue: "buildkite-gcp" timeout_in_minutes: 60 retry: automatic: @@ -23,7 +23,7 @@ steps: env: M3_DOCKER_REPO: quay.io/m3 agents: - queue: builders + queue: "buildkite-gcp" timeout_in_minutes: 60 retry: automatic: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 547e536c68..ef9f98471c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,7 +1,43 @@ +container: + kubernetes: &kubernetes + gitEnvFrom: + - secretRef: + name: oss-github-ssh-credentials + sidecars: + - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1 + volumeMounts: + - mountPath: /var/run/ + name: docker-sock + securityContext: + privileged: true + allowPrivilegeEscalation: true + mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars + podSpec: &podSpec + containers: + - &commandContainer + image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2 + command: + - |- + echo "Command step was not overridden." + exit 1 + volumeMounts: + - mountPath: /var/run/ + name: docker-sock + resources: + requests: + cpu: 7500m + memory: 28G + volumes: + - name: docker-sock + emptyDir: {} + +agents: + queue: "buildkite-gcp" + common: &common timeout_in_minutes: 20 agents: - queue: workers + queue: "buildkite-gcp" retry: # Automatically retry failures one time. automatic: @@ -13,151 +49,174 @@ common: &common env: SKIP_CODECOV: "true" +# Buildkite pipeline is currently broken, and we are working on fixing it. In the meantime, we are temporarily +# disabling steps until it is fixed. ISSUE: https://github.com/m3db/m3/issues/4274 steps: - - name: "Codegen" - command: make clean install-vendor-m3 test-all-gen - env: - CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.18.x - plugins: - gopath-checkout#v1.0.1: - import: github.com/m3db/m3 - <<: *common +# - name: "Codegen" +# env: +# CGO_ENABLED: 0 +# GIMME_GO_VERSION: 1.18.x +# plugins: +# gopath-checkout#v1.0.1: +# import: github.com/m3db/m3 +# docker#v3.0.1: +# image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1 +# volumes: +# - .:/go/src/github.com/m3db/m3 +# - /var/run/docker.sock:/var/run/docker.sock +# kubernetes: +# <<: *kubernetes +# podSpec: +# <<: *podSpec +# containers: +# - <<: *commandContainer +# command: +# - |- +# make clean install-vendor-m3 test-all-gen +# <<: *common - name: "Unit %n" - command: make clean install-vendor-m3 test-base - parallelism: 4 - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Big Unit %n" - command: make clean install-vendor-m3 test-big-base - parallelism: 2 - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Services, Tools" - command: make clean install-vendor-m3 services tools - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Lint" - command: make clean lint - env: - CGO_ENABLED: 0 - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Integration (:docker:)" - command: make clean install-vendor-m3 docker-integration-test - parallelism: 2 - env: - CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.18.x - plugins: - gopath-checkout#v1.0.1: - import: github.com/m3db/m3 - <<: *common - - label: "M3 Cluster Integration Tests" - command: make clean test-ci-cluster-integration - env: - CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.18.x - plugins: - gopath-checkout#v1.0.1: - import: github.com/m3db/m3 - <<: *common - - label: "M3 Cluster Integration Test Harness Tests" - command: make clean test-ci-test-harness - skip: "NB(nate): temporarily disabling to resolve some tests flakes" - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Prometheus compatibility (:docker:)" - command: make clean install-vendor-m3 docker-compatibility-test - parallelism: 1 - env: - CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.18.x - plugins: - gopath-checkout#v1.0.1: - import: github.com/m3db/m3 - <<: *common - - name: "Integration (dbnode Recently Read) %n" - parallelism: 2 - command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Integration (dbnode LRU) %n" - parallelism: 2 - command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Integration (aggregator TCP client) %n" - parallelism: 1 - command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Integration (aggregator m3msg client) %n" - parallelism: 1 - command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - <<: *common - - label: "Integration (m3em, cluster, msg, metrics) %n" parallelism: 4 - command: make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics plugins: docker-compose#v2.5.1: run: app workdir: /go/src/github.com/m3db/m3 - <<: *common - - name: "Documentation tests" - command: make clean install-vendor-m3 docs-test - env: - CGO_ENABLED: 0 - GIMME_GO_VERSION: 1.18.x - plugins: - gopath-checkout#v1.0.1: - import: github.com/m3db/m3 - <<: *common - - label: "FOSSA license scan" - command: make clean install-vendor-m3 fossa - plugins: - docker-compose#v2.5.1: - run: app - workdir: /go/src/github.com/m3db/m3 - env: - - FOSSA_API_KEY - <<: *common - - name: "Check for docker and docs builds :docker: :books:" - commands: - - ".ci/docker/check_do_docker.sh" - - ".buildkite/scripts/check_do_docs.sh" - agents: - queue: init - timeout_in_minutes: 10 - retry: - automatic: - limit: 1 - manual: true + kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make clean install-vendor-m3 test-base + <<: *common +# - name: "Big Unit %n" +# command: make clean install-vendor-m3 test-big-base +# parallelism: 2 +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Services, Tools" +# command: make clean install-vendor-m3 services tools +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Lint" +# command: make clean lint +# env: +# CGO_ENABLED: 0 +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Integration (:docker:)" +# command: make clean install-vendor-m3 docker-integration-test +# parallelism: 2 +# env: +# CGO_ENABLED: 0 +# GIMME_GO_VERSION: 1.18.x +# plugins: +# gopath-checkout#v1.0.1: +# import: github.com/m3db/m3 +# <<: *common +# - label: "M3 Cluster Integration Tests" +# command: make clean test-ci-cluster-integration +# env: +# CGO_ENABLED: 0 +# GIMME_GO_VERSION: 1.18.x +# plugins: +# gopath-checkout#v1.0.1: +# import: github.com/m3db/m3 +# <<: *common +# - label: "M3 Cluster Integration Test Harness Tests" +# command: make clean test-ci-test-harness +# skip: "NB(nate): temporarily disabling to resolve some tests flakes" +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Prometheus compatibility (:docker:)" +# command: make clean install-vendor-m3 docker-compatibility-test +# parallelism: 1 +# env: +# CGO_ENABLED: 0 +# GIMME_GO_VERSION: 1.18.x +# plugins: +# gopath-checkout#v1.0.1: +# import: github.com/m3db/m3 +# <<: *common +# - name: "Integration (dbnode Recently Read) %n" +# parallelism: 2 +# command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Integration (dbnode LRU) %n" +# parallelism: 2 +# command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Integration (aggregator TCP client) %n" +# parallelism: 1 +# command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Integration (aggregator m3msg client) %n" +# parallelism: 1 +# command: make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - label: "Integration (m3em, cluster, msg, metrics) %n" +# parallelism: 4 +# command: make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# <<: *common +# - name: "Documentation tests" +# command: make clean install-vendor-m3 docs-test +# env: +# CGO_ENABLED: 0 +# GIMME_GO_VERSION: 1.18.x +# plugins: +# gopath-checkout#v1.0.1: +# import: github.com/m3db/m3 +# <<: *common +# - label: "FOSSA license scan" +# command: make clean install-vendor-m3 fossa +# plugins: +# docker-compose#v2.5.1: +# run: app +# workdir: /go/src/github.com/m3db/m3 +# env: +# - FOSSA_API_KEY +# <<: *common +# - name: "Check for docker and docs builds :docker: :books:" +# commands: +# - ".ci/docker/check_do_docker.sh" +# - ".buildkite/scripts/check_do_docs.sh" +# agents: +# queue: init +# timeout_in_minutes: 10 +# retry: +# automatic: +# limit: 1 +# manual: true