From 301446f715df19038ac4776f4fc9601884790fd5 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 13:20:53 -0700
Subject: [PATCH 01/92] comment out other tests

---
 .buildkite/pipeline.yml | 410 ++++++++++++++++++++--------------------
 1 file changed, 205 insertions(+), 205 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 12ed82dc28..77c9c1a0a7 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -76,203 +76,203 @@ steps:
 #                - |-
 #                  make clean install-vendor-m3 test-all-gen
 #    <<: *common
-  - name: "Unit %n"
-    parallelism: 4
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-base
-    <<: *common
-  - name: "Big Unit %n"
-    parallelism: 2
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-big-base
-    <<: *common
-  - name: "Services, Tools"
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 services tools
-    <<: *common
-  - name: "Lint"
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean lint
-    <<: *common
-  - label: "Integration (:docker:)"
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 docker-integration-test
-    <<: *common
-  - label: "M3 Cluster Integration Tests"
-    plugins:
-      gopath-checkout#v1.0.1:
-        import: github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean test-ci-cluster-integration
-    <<: *common
-  - label: "M3 Cluster Integration Test Harness Tests"
-    skip: "NB(nate): temporarily disabling to resolve some tests flakes"
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean test-ci-test-harness
-    <<: *common
-#  - name: "Prometheus compatibility (:docker:)"
-#    command: make clean install-vendor-m3 docker-compatibility-test
-#    parallelism: 1
-#    env:
-#      CGO_ENABLED: 0
-#      GIMME_GO_VERSION: 1.22.x
+#  - name: "Unit %n"
+#    parallelism: 4
 #    plugins:
-#        gopath-checkout#v1.0.1:
-#          import: github.com/m3db/m3
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-base
 #    <<: *common
-  - name: "Integration (dbnode Recently Read) %n"
-    parallelism: 2
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
-    <<: *common
-  - name: "Integration (dbnode LRU) %n"
-    parallelism: 2
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
-    <<: *common
-  - name: "Integration (aggregator TCP client) %n"
-    parallelism: 1
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp
-    <<: *common
-  - name: "Integration (aggregator m3msg client) %n"
+#  - name: "Big Unit %n"
+#    parallelism: 2
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-big-base
+#    <<: *common
+#  - name: "Services, Tools"
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 services tools
+#    <<: *common
+#  - name: "Lint"
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean lint
+#    <<: *common
+#  - label: "Integration (:docker:)"
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 docker-integration-test
+#    <<: *common
+#  - label: "M3 Cluster Integration Tests"
+#    plugins:
+#      gopath-checkout#v1.0.1:
+#        import: github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean test-ci-cluster-integration
+#    <<: *common
+#  - label: "M3 Cluster Integration Test Harness Tests"
+#    skip: "NB(nate): temporarily disabling to resolve some tests flakes"
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean test-ci-test-harness
+#    <<: *common
+  - name: "Prometheus compatibility (:docker:)"
+    command: make clean install-vendor-m3 docker-compatibility-test
     parallelism: 1
+    env:
+      CGO_ENABLED: 0
+      GIMME_GO_VERSION: 1.22.x
     plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg
-    <<: *common
-  - label: "Integration (m3em, cluster, msg, metrics) %n"
-    parallelism: 4
-    plugins:
-      docker-compose#v2.5.1:
-        run: app
-        workdir: /go/src/github.com/m3db/m3
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
+        gopath-checkout#v1.0.1:
+          import: github.com/m3db/m3
     <<: *common
+#  - name: "Integration (dbnode Recently Read) %n"
+#    parallelism: 2
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
+#    <<: *common
+#  - name: "Integration (dbnode LRU) %n"
+#    parallelism: 2
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
+#    <<: *common
+#  - name: "Integration (aggregator TCP client) %n"
+#    parallelism: 1
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp
+#    <<: *common
+#  - name: "Integration (aggregator m3msg client) %n"
+#    parallelism: 1
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg
+#    <<: *common
+#  - label: "Integration (m3em, cluster, msg, metrics) %n"
+#    parallelism: 4
+#    plugins:
+#      docker-compose#v2.5.1:
+#        run: app
+#        workdir: /go/src/github.com/m3db/m3
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
+#    <<: *common
 # Commenting out Documentation test for now because of issue with access tokens. Also a low priority test, as it checks
 # for broken links in the documentation. ISSUE: https://github.com/m3db/m3/issues/4288
 #  - name: "Documentation tests"
@@ -284,16 +284,16 @@ steps:
 #        gopath-checkout#v1.0.1:
 #          import: github.com/m3db/m3
 #    <<: *common
-  - name: "Check for docker and docs builds :docker: :books:"
-    plugins:
-      kubernetes:
-        <<: *kubernetes
-        podSpec:
-          <<: *podSpec
-          containers:
-            - <<: *commandContainer
-              command:
-                - |-
-                  .buildkite/scripts/check_do_docs.sh
-                  .ci/docker/check_do_docker.sh
-    <<: *common
+#  - name: "Check for docker and docs builds :docker: :books:"
+#    plugins:
+#      kubernetes:
+#        <<: *kubernetes
+#        podSpec:
+#          <<: *podSpec
+#          containers:
+#            - <<: *commandContainer
+#              command:
+#                - |-
+#                  .buildkite/scripts/check_do_docs.sh
+#                  .ci/docker/check_do_docker.sh
+#    <<: *common

From 3e0fe6878282e624c3ba6e81ba7f74a5a5419e20 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 13:22:23 -0700
Subject: [PATCH 02/92] add make to dockerfile

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 98b67e1b25..b909dc5364 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq make

From ea440fbb854171faea8a81fa1d2589b61c74a5f0 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 13:24:31 -0700
Subject: [PATCH 03/92] remove make, and add kubernetes plugin

---
 .buildkite/pipeline.yml | 10 +++++++++-
 Dockerfile              |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 77c9c1a0a7..ee4341c312 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -184,7 +184,6 @@ steps:
 #                  make clean test-ci-test-harness
 #    <<: *common
   - name: "Prometheus compatibility (:docker:)"
-    command: make clean install-vendor-m3 docker-compatibility-test
     parallelism: 1
     env:
       CGO_ENABLED: 0
@@ -192,6 +191,15 @@ steps:
     plugins:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3
+        kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                  - |-
+                    make clean install-vendor-m3 docker-compatibility-test
     <<: *common
 #  - name: "Integration (dbnode Recently Read) %n"
 #    parallelism: 2
diff --git a/Dockerfile b/Dockerfile
index b909dc5364..98b67e1b25 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq make
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq

From 9f722d7c8cd41d182097901f2be7aad5d1be2f5c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 13:29:07 -0700
Subject: [PATCH 04/92] add docker-compose

---
 .buildkite/pipeline.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index ee4341c312..40fc03836f 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -191,6 +191,9 @@ steps:
     plugins:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3
+        docker-compose#v2.5.1:
+          run: app
+          workdir: /go/src/github.com/m3db/m3
         kubernetes:
           <<: *kubernetes
           podSpec:

From c4c6b126ef248cf1d27feb9a29990d58027f5174 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 13:32:04 -0700
Subject: [PATCH 05/92] add docker-compose to dockerfile

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 98b67e1b25..66e5c81d6e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose

From c67f91d7a72ba82af6fb38a5cf9816b434f58578 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:08:46 -0700
Subject: [PATCH 06/92] update m3query volumes in docker-compose

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 79bd8d4f2a..882778c6d9 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - "./m3query.yml:/etc/m3query/m3query.yml"
+      - "./:/etc/m3query/"
   prometheus:
     expose:
       - "9090"

From d922e6c89d7fd548cb71fe85c35b52cd9b9f4867 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:17:52 -0700
Subject: [PATCH 07/92] comment out trap defer EXIT command in run.sh

---
 scripts/comparator/run.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index ea5c05d781..0bd5f8a279 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -63,10 +63,10 @@ function defer {
 	fi
 }
 
-if [[ "$RUN_ONLY" == "false" ]]
-then
-	trap defer EXIT 
-fi
+#if [[ "$RUN_ONLY" == "false" ]]
+#then
+#	trap defer EXIT
+#fi
 
 $comparator -input=$QUERY_FILE \
 -s=$START \

From c10fd72b98fcbd154112786949925a825ec0b7d5 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:25:06 -0700
Subject: [PATCH 08/92] uncomment previous code

---
 scripts/comparator/run.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 0bd5f8a279..addb8bbbbc 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -63,10 +63,10 @@ function defer {
 	fi
 }
 
-#if [[ "$RUN_ONLY" == "false" ]]
-#then
-#	trap defer EXIT
-#fi
+if [[ "$RUN_ONLY" == "false" ]]
+then
+	trap defer EXIT
+fi
 
 $comparator -input=$QUERY_FILE \
 -s=$START \

From fc865d86fb0f641a296dfde5f8d7089f57a3966c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:27:53 -0700
Subject: [PATCH 09/92] experimenting with commenting out rest of run.sh

---
 scripts/comparator/run.sh | 56 +++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index addb8bbbbc..7d03403d0e 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -48,31 +48,31 @@ then
 	setup_docker $CI
 fi
 
-comparator=$COMPARATOR/compare.out
-go build -o $comparator $COMPARATOR/.
-function defer {
-	rm $comparator
-	if [[ "$CI" == "true" ]]
-	then
-		teardown_docker $CI
-	else
-		if [[ "$RUN_ONLY" == "false" ]]
-		then
-			rm $DASHBOARD
-		fi
-	fi
-}
-
-if [[ "$RUN_ONLY" == "false" ]]
-then
-	trap defer EXIT
-fi
-
-$comparator -input=$QUERY_FILE \
--s=$START \
--e=$END \
--comparator=$COMPARATOR_WRITE \
--regressionDir=$REGRESSION_DIR
-
-# Run PromQL testdata tests
-go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/
+#comparator=$COMPARATOR/compare.out
+#go build -o $comparator $COMPARATOR/.
+#function defer {
+#	rm $comparator
+#	if [[ "$CI" == "true" ]]
+#	then
+#		teardown_docker $CI
+#	else
+#		if [[ "$RUN_ONLY" == "false" ]]
+#		then
+#			rm $DASHBOARD
+#		fi
+#	fi
+#}
+#
+#if [[ "$RUN_ONLY" == "false" ]]
+#then
+#	trap defer EXIT
+#fi
+#
+#$comparator -input=$QUERY_FILE \
+#-s=$START \
+#-e=$END \
+#-comparator=$COMPARATOR_WRITE \
+#-regressionDir=$REGRESSION_DIR
+#
+## Run PromQL testdata tests
+#go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From 66e7dc7e6eaa376e64adb26934143146b06869c1 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:45:46 -0700
Subject: [PATCH 10/92] comment out everything except for line 51 and 52 at the
 end of run.sh

---
 scripts/comparator/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 7d03403d0e..044811449e 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -48,8 +48,8 @@ then
 	setup_docker $CI
 fi
 
-#comparator=$COMPARATOR/compare.out
-#go build -o $comparator $COMPARATOR/.
+comparator=$COMPARATOR/compare.out
+go build -o $comparator $COMPARATOR/.
 #function defer {
 #	rm $comparator
 #	if [[ "$CI" == "true" ]]

From 52281cf37f430224b7560fe346d01dbf9a5b138f Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:50:34 -0700
Subject: [PATCH 11/92] uncomment lines 53-69

---
 scripts/comparator/run.sh | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 044811449e..8ea4f4be14 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -50,24 +50,24 @@ fi
 
 comparator=$COMPARATOR/compare.out
 go build -o $comparator $COMPARATOR/.
-#function defer {
-#	rm $comparator
-#	if [[ "$CI" == "true" ]]
-#	then
-#		teardown_docker $CI
-#	else
-#		if [[ "$RUN_ONLY" == "false" ]]
-#		then
-#			rm $DASHBOARD
-#		fi
-#	fi
-#}
-#
-#if [[ "$RUN_ONLY" == "false" ]]
-#then
-#	trap defer EXIT
-#fi
-#
+function defer {
+	rm $comparator
+	if [[ "$CI" == "true" ]]
+	then
+		teardown_docker $CI
+	else
+		if [[ "$RUN_ONLY" == "false" ]]
+		then
+			rm $DASHBOARD
+		fi
+	fi
+}
+
+if [[ "$RUN_ONLY" == "false" ]]
+then
+	trap defer EXIT
+fi
+
 #$comparator -input=$QUERY_FILE \
 #-s=$START \
 #-e=$END \

From c907f2abcb2162b4bc7ecf92773edf09b546d8f2 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:56:00 -0700
Subject: [PATCH 12/92] uncomment lines 71-75

---
 scripts/comparator/run.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 8ea4f4be14..9d3e5c4220 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -68,11 +68,11 @@ then
 	trap defer EXIT
 fi
 
-#$comparator -input=$QUERY_FILE \
-#-s=$START \
-#-e=$END \
-#-comparator=$COMPARATOR_WRITE \
-#-regressionDir=$REGRESSION_DIR
-#
+$comparator -input=$QUERY_FILE \
+-s=$START \
+-e=$END \
+-comparator=$COMPARATOR_WRITE \
+-regressionDir=$REGRESSION_DIR
+
 ## Run PromQL testdata tests
 #go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From ead72c2b2f6105b2ca563e77589cdae2a19f741c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 14:59:44 -0700
Subject: [PATCH 13/92] comment out lines 71 to 75 and uncomment go test in
 run.sh

---
 scripts/comparator/run.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 9d3e5c4220..077191b644 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -68,11 +68,11 @@ then
 	trap defer EXIT
 fi
 
-$comparator -input=$QUERY_FILE \
--s=$START \
--e=$END \
--comparator=$COMPARATOR_WRITE \
--regressionDir=$REGRESSION_DIR
+#$comparator -input=$QUERY_FILE \
+#-s=$START \
+#-e=$END \
+#-comparator=$COMPARATOR_WRITE \
+#-regressionDir=$REGRESSION_DIR
 
-## Run PromQL testdata tests
-#go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/
+# Run PromQL testdata tests
+go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From 055b09a56a33b5503a08d7d709d6170ee595b09b Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 15:19:57 -0700
Subject: [PATCH 14/92] check if prometheus container is running

---
 scripts/comparator/run.sh | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 077191b644..68ca17d1eb 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -68,11 +68,12 @@ then
 	trap defer EXIT
 fi
 
-#$comparator -input=$QUERY_FILE \
-#-s=$START \
-#-e=$END \
-#-comparator=$COMPARATOR_WRITE \
-#-regressionDir=$REGRESSION_DIR
+docker ps
+$comparator -input=$QUERY_FILE \
+-s=$START \
+-e=$END \
+-comparator=$COMPARATOR_WRITE \
+-regressionDir=$REGRESSION_DIR
 
-# Run PromQL testdata tests
-go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/
+## Run PromQL testdata tests
+#go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From 40dd69d753e90f28630eb1501fb761b089113e5c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 15:38:51 -0700
Subject: [PATCH 15/92] adding logs to check prometheus container not running

---
 scripts/comparator/docker-setup.sh | 2 ++
 scripts/comparator/run.sh          | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index 0fe3c9e0ce..6d4c3e991c 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -10,6 +10,8 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
+  docker-compose -f ${COMPOSE_FILE} logs prometheus
+  docker-compose -f ${COMPOSE_FILE} ps
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query 
   
   CI=$1
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 68ca17d1eb..6cc471dce4 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -68,7 +68,7 @@ then
 	trap defer EXIT
 fi
 
-docker ps
+docker ps -a
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \

From d016524ca7ff063b25af178f6ef87847a45d270e Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 15:56:29 -0700
Subject: [PATCH 16/92] update volume mount for prometheus in docker-compose

---
 scripts/comparator/docker-compose.yml | 2 +-
 scripts/comparator/docker-setup.sh    | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 882778c6d9..84523ed93c 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./:/etc/prometheus/"
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
   grafana:
     build:
       context: ./grafana
diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index 6d4c3e991c..655c596d47 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -10,8 +10,7 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
-  docker-compose -f ${COMPOSE_FILE} logs prometheus
-  docker-compose -f ${COMPOSE_FILE} ps
+  docker-compose -f ${COMPOSE_FILE} logs comparator_prometheus_1
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query 
   
   CI=$1

From dff9a73c81edd0a91f3f43ed28ec5419220e08d6 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 16:02:58 -0700
Subject: [PATCH 17/92] modify right hand side of volume mount for prometheus
 in docker-compose

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 84523ed93c..cd0da65364 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+      - "./prometheus.yml:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From 6ada8c59f4b3e5629052f6464050833fc80d658d Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 16:09:48 -0700
Subject: [PATCH 18/92] try another volume mount for prometheus

---
 scripts/comparator/docker-setup.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index 655c596d47..0fe3c9e0ce 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -10,7 +10,6 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
-  docker-compose -f ${COMPOSE_FILE} logs comparator_prometheus_1
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query 
   
   CI=$1

From b91746f873d987f3a65cc04c5ccd2b1f1a2adba8 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 16:15:33 -0700
Subject: [PATCH 19/92] remove volume mounts to see what happens in
 docker-compose

---
 scripts/comparator/docker-compose.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index cd0da65364..6120fdedc0 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,8 +20,6 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
-    volumes:
-      - "./:/etc/m3query/"
   prometheus:
     expose:
       - "9090"
@@ -30,8 +28,6 @@ services:
     networks:
       - backend
     image: prom/prometheus:v2.31.2
-    volumes:
-      - "./prometheus.yml:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From b9ea790ea1b9e66b601c0b037b47e610e2a0fed4 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 16:19:48 -0700
Subject: [PATCH 20/92] add back logs in docker-setup

---
 scripts/comparator/docker-setup.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index 0fe3c9e0ce..d55973defe 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -10,7 +10,10 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
-  docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query 
+  docker-compose -f ${COMPOSE_FILE} logs prometheus
+  docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query
+  docker-compose -f ${COMPOSE_FILE} logs m3query
+  docker-compose -f ${COMPOSE_FILE} ps
   
   CI=$1
   if [[ "$CI" != "true" ]]

From fc8e96713b44fc7cca077faf99b75be157ea80e1 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Mon, 26 Aug 2024 16:28:44 -0700
Subject: [PATCH 21/92] try adding prometheusURL to run.sh

---
 scripts/comparator/run.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 6cc471dce4..e2f696e4aa 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -73,7 +73,8 @@ $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \
 -comparator=$COMPARATOR_WRITE \
--regressionDir=$REGRESSION_DIR
+-regressionDir=$REGRESSION_DIR \
+-prometheusURL=http://prometheus:9090
 
 ## Run PromQL testdata tests
 #go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From 4790b1e74d69c5a044064b09449d080a4b2bcf48 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:06:59 -0700
Subject: [PATCH 22/92] change localhost to host.docker.internal

---
 scripts/comparator/run.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index e2f696e4aa..b2490234e6 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="localhost:9001"
+export COMPARATOR_WRITE="host.docker.internal:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl
@@ -73,8 +73,7 @@ $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \
 -comparator=$COMPARATOR_WRITE \
--regressionDir=$REGRESSION_DIR \
--prometheusURL=http://prometheus:9090
+-regressionDir=$REGRESSION_DIR
 
 ## Run PromQL testdata tests
 #go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From 31f1737acf68a4355529cf39d15a84d468d683cc Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:12:16 -0700
Subject: [PATCH 23/92] check for firewall

---
 scripts/comparator/run.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index b2490234e6..5423841f3f 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="host.docker.internal:9001"
+export COMPARATOR_WRITE="localhost:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl
@@ -69,6 +69,7 @@ then
 fi
 
 docker ps -a
+sudo ufw status verbose
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \

From af547b8b279e19c1387e1426c8a9476d055958aa Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:15:59 -0700
Subject: [PATCH 24/92] add sudo to dockerfile

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 66e5c81d6e..bf6a08f017 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose sudo

From 12c6f20c59823f7e5d1a399837f9d133b6355c55 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:20:05 -0700
Subject: [PATCH 25/92] add ufw to dockerfile

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index bf6a08f017..a1380c952d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose sudo
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose sudo ufw

From 63956f46add1195cbadb4fbf31d01af2fd6dce41 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:33:08 -0700
Subject: [PATCH 26/92] inspect prometheus docker container for network issues

---
 Dockerfile                | 2 +-
 scripts/comparator/run.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a1380c952d..66e5c81d6e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose sudo ufw
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 5423841f3f..b95fb8309b 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -69,7 +69,7 @@ then
 fi
 
 docker ps -a
-sudo ufw status verbose
+docker inspect e1f1df2c496c
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \

From c7e52095b0f9e44a5761a507c03b7769146de326 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:41:24 -0700
Subject: [PATCH 27/92] modify docker inspect command

---
 scripts/comparator/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index b95fb8309b..c8ec5c0517 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -69,7 +69,7 @@ then
 fi
 
 docker ps -a
-docker inspect e1f1df2c496c
+docker inspect comparator_prometheus_1
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \

From 5d0d90d1977984b429cc2613230023749cf8f3ef Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 10:54:36 -0700
Subject: [PATCH 28/92] add back volume mount for prometheus

---
 scripts/comparator/docker-compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 6120fdedc0..3050d18257 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -28,6 +28,8 @@ services:
     networks:
       - backend
     image: prom/prometheus:v2.31.2
+    volumes:
+      - "./:etc/prometheus"
   grafana:
     build:
       context: ./grafana

From 71d5c5d079be84f52a32b78812f664014cfb0abc Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:03:09 -0700
Subject: [PATCH 29/92] fix typo in volume mount

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 3050d18257..4142318d6e 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -29,7 +29,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./:etc/prometheus"
+      - "./:/etc/prometheus"
   grafana:
     build:
       context: ./grafana

From 099edb82e30a9e64f10c6cd5ce8acfb4be357c38 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:08:35 -0700
Subject: [PATCH 30/92] fix another typo in volume mount for prometheus

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 4142318d6e..bacba32734 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -29,7 +29,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./:/etc/prometheus"
+      - "./:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From dc39fcc829daa3884079ba78b9b20755999cbac0 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:20:34 -0700
Subject: [PATCH 31/92] find the current directory to fix volume mount

---
 scripts/comparator/docker-setup.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index d55973defe..3a9bc66673 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -9,6 +9,7 @@ export REVISION
 function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
+  pwd
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
   docker-compose -f ${COMPOSE_FILE} logs prometheus
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query

From cb173acd8cfe49acf6519de4e17c29058e848466 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:32:59 -0700
Subject: [PATCH 32/92] update volume mount for prometheus with new directory

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index bacba32734..2a0d4d8f22 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -29,7 +29,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./:/etc/prometheus/"
+      - "/go/src/github.com/m3db/m3/scripts/comparator:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From 033e0fd253a8e393c59f6bcf70c63baab27ba331 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:40:12 -0700
Subject: [PATCH 33/92] fix typo

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 2a0d4d8f22..96a49ab8f6 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -29,7 +29,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator:/etc/prometheus/"
+      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From 9b10e0ccc1b74e4758168d4af03b68e1b329b0ae Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:46:07 -0700
Subject: [PATCH 34/92] rewrite volume mount

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 96a49ab8f6..e94eeea39f 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -29,7 +29,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/prometheus/"
+      - "./scripts/comparator/:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From 725706568aab7be1506f0874a0b38917d210b770 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:51:13 -0700
Subject: [PATCH 35/92] list out contents in host machine

---
 scripts/comparator/docker-setup.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index 3a9bc66673..c5f2c69554 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -10,6 +10,7 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   pwd
+  ls /go/src/github.com/m3db/m3/scripts/comparator/
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
   docker-compose -f ${COMPOSE_FILE} logs prometheus
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query

From 74650d9f597d80738490e5350cd5205fd56626a2 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 11:59:44 -0700
Subject: [PATCH 36/92] move ls, and pwd

---
 scripts/comparator/docker-setup.sh | 2 --
 scripts/comparator/run.sh          | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index c5f2c69554..d55973defe 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -9,8 +9,6 @@ export REVISION
 function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
-  pwd
-  ls /go/src/github.com/m3db/m3/scripts/comparator/
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
   docker-compose -f ${COMPOSE_FILE} logs prometheus
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index c8ec5c0517..98cd26a069 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -45,6 +45,8 @@ then
 	$COMPARATOR/setup.sh
 
 	echo "setting up docker"
+	pwd
+	ls /go/src/github.com/m3db/m3/scripts/comparator/
 	setup_docker $CI
 fi
 

From 38cb15d6c12e7adfbba673bf5414acbd2fc0fbcf Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 12:04:30 -0700
Subject: [PATCH 37/92] use absolute path for volume mount

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index e94eeea39f..96a49ab8f6 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -29,7 +29,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./scripts/comparator/:/etc/prometheus/"
+      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From 6814f39147e939b9b23e32be13a637780c223c9a Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 12:06:38 -0700
Subject: [PATCH 38/92] add volume mount for m3query

---
 scripts/comparator/docker-compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 96a49ab8f6..b6addca4cb 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,6 +20,8 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
+    volumes:
+      - "./m3query.yml:/etc/m3query/m3query.yml"
   prometheus:
     expose:
       - "9090"

From 91c406dd037e27758fef519577b2a72d1ede1f1a Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 12:19:33 -0700
Subject: [PATCH 39/92] update m3query volume mount

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index b6addca4cb..fc5ae5422e 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - "./m3query.yml:/etc/m3query/m3query.yml"
+      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/m3query/"
   prometheus:
     expose:
       - "9090"

From 43da6190567855e2ef6928c2b0aa88b4131b752b Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 12:25:15 -0700
Subject: [PATCH 40/92] update volume mount again for prometheus

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index fc5ae5422e..bb7f3151a4 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/prometheus/"
+      - "/go/src/github.com/m3db/m3/scripts/comparator/prometheus.yml:/etc/prometheus/"
   grafana:
     build:
       context: ./grafana

From aba59e25a895199d3297671a7c1ddb04ab3cdd13 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 12:25:55 -0700
Subject: [PATCH 41/92] another volume mount configuration

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index bb7f3151a4..4589f8a49f 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator/prometheus.yml:/etc/prometheus/"
+      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/prometheus/prometheus.yml"
   grafana:
     build:
       context: ./grafana

From 55754303c2e0399c2ebf21a66d3b37f375730ce6 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 12:26:19 -0700
Subject: [PATCH 42/92] another volume mount configuration

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 4589f8a49f..cf9ae80d27 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/prometheus/prometheus.yml"
+      - "/go/src/github.com/m3db/m3/scripts/comparator/prometheus.yml:/etc/prometheus/prometheus.yml"
   grafana:
     build:
       context: ./grafana

From 10520cdd476178f0e70f7dae6f1b1ad8774eeb09 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 15:29:40 -0700
Subject: [PATCH 43/92] try another way to volume mount

---
 scripts/comparator/docker-compose.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index cf9ae80d27..4becf15ac0 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator/:/etc/m3query/"
+      - app:/etc/m3query/
   prometheus:
     expose:
       - "9090"
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "/go/src/github.com/m3db/m3/scripts/comparator/prometheus.yml:/etc/prometheus/prometheus.yml"
+      - app:/etc/prometheus/
   grafana:
     build:
       context: ./grafana
@@ -45,3 +45,5 @@ services:
     image: m3grafana:latest
 networks:
   backend:
+volumes:
+  app:

From 3a3fd3b744775a8433f34d3ace0cd0a2c8978d51 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 15:48:37 -0700
Subject: [PATCH 44/92] add m3query.yml

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 4becf15ac0..0fcebfaee2 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - app:/etc/m3query/
+      - app:/etc/m3query/m3query.yml
   prometheus:
     expose:
       - "9090"

From a70f7d912d2d38c4c0c85e0e4c3b0e2c807e23e1 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 15:49:51 -0700
Subject: [PATCH 45/92] add external:true

---
 scripts/comparator/docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 0fcebfaee2..adb5caa4ce 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - app:/etc/m3query/m3query.yml
+      - app:/etc/m3query/
   prometheus:
     expose:
       - "9090"
@@ -47,3 +47,4 @@ networks:
   backend:
 volumes:
   app:
+    external: true

From 94a1a56de7312d96aa37b71704ef4b8a81a4e5da Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 15:57:30 -0700
Subject: [PATCH 46/92] try using quotes

---
 scripts/comparator/docker-compose.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index adb5caa4ce..4becf15ac0 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -47,4 +47,3 @@ networks:
   backend:
 volumes:
   app:
-    external: true

From 014c46d907f12c44200ca2afec9debd503c404c2 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 16:01:09 -0700
Subject: [PATCH 47/92] remove volume mount for m3query

---
 scripts/comparator/docker-compose.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 4becf15ac0..67992ea4d7 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,8 +20,6 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
-    volumes:
-      - app:/etc/m3query/
   prometheus:
     expose:
       - "9090"

From faf9a345e3f0ca09d22937beb3e58be4b48a90f1 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 16:08:19 -0700
Subject: [PATCH 48/92] add docker inspect for m3query

---
 scripts/comparator/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 98cd26a069..1541292bc9 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -71,7 +71,7 @@ then
 fi
 
 docker ps -a
-docker inspect comparator_prometheus_1
+docker inspect comparator_m3query_1
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \

From a7bb0ab803a90960031b0fc0a01ea37e9d224f1c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 16:15:57 -0700
Subject: [PATCH 49/92] try adding another app

---
 scripts/comparator/docker-compose.yml | 3 +++
 scripts/comparator/run.sh             | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 67992ea4d7..1ed3e6d513 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,6 +20,8 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
+    volumes:
+      - app2:/etc/m3query/
   prometheus:
     expose:
       - "9090"
@@ -45,3 +47,4 @@ networks:
   backend:
 volumes:
   app:
+  app2:
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 1541292bc9..25d6dd9bc8 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -71,7 +71,6 @@ then
 fi
 
 docker ps -a
-docker inspect comparator_m3query_1
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \

From 076044d84e3673461d80465c5f9767b9b6d9d8fb Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 27 Aug 2024 17:09:09 -0700
Subject: [PATCH 50/92] change to localhost

---
 scripts/comparator/compare.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 9ebbc38a06..02b6999fb2 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -52,7 +52,7 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "0.0.0.0:9090", "prom address")
+		pPromAddress  = flag.String("promAdress", "localhost:9090", "prom address")
 		pQueryAddress = flag.String("queryAddress", "0.0.0.0:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")

From 121cb624f23dce0a97c35f0bbb9b3c5a47c31c5b Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 10:33:32 -0700
Subject: [PATCH 51/92] use external to create volume mounts

---
 docker-compose.yml                    | 3 +++
 scripts/comparator/compare.go         | 2 +-
 scripts/comparator/docker-compose.yml | 8 ++++----
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 26a76cdbe5..5c189ab496 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,6 +11,7 @@ services:
       # Support running docker within docker. That is, buildkite jobs themselves run in a container; that container
       # needs to be able to spin up functioning docker containers.
       - /var/run/docker.sock:/var/run/docker.sock
+      - comparator_volume:/go/src/github.com/m3db/m3/scripts/comparator
     extra_hosts:
       # Allow routing from the buildkite container to the host machine, as host.docker.internal. This allows us to do
       # the following:
@@ -38,3 +39,5 @@ services:
       - SPLIT_IDX
       - TOTAL_SPLITS
       - GOPATH=/go
+volumes:
+  comparator_volume:
diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 02b6999fb2..9ebbc38a06 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -52,7 +52,7 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "localhost:9090", "prom address")
+		pPromAddress  = flag.String("promAdress", "0.0.0.0:9090", "prom address")
 		pQueryAddress = flag.String("queryAddress", "0.0.0.0:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")
diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 1ed3e6d513..9b8ab3eea3 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - app2:/etc/m3query/
+      - comparator_volume:/etc/m3query/
   prometheus:
     expose:
       - "9090"
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - app:/etc/prometheus/
+      - comparator_volume:/etc/prometheus/
   grafana:
     build:
       context: ./grafana
@@ -46,5 +46,5 @@ services:
 networks:
   backend:
 volumes:
-  app:
-  app2:
+  comparator_volume:
+    external: true

From ca242dec5d01b91eae10d6ec2a316d54e2e4e6e6 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 10:37:50 -0700
Subject: [PATCH 52/92] add pwd to see why we can't find run.sh

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 62d31c7403..b1d65afe56 100644
--- a/Makefile
+++ b/Makefile
@@ -225,6 +225,7 @@ endif
 .PHONY: docker-integration-test
 docker-integration-test:
 	@echo "--- Running Docker integration test"
+	pwd
 	./scripts/docker-integration-tests/run.sh
 
 .PHONY: docker-compatibility-test

From 6230a00e783b0a327bd3be37b50c56863bc16a7d Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 11:05:57 -0700
Subject: [PATCH 53/92] try another volume mount

---
 Makefile                              | 1 -
 docker-compose.yml                    | 3 ---
 scripts/comparator/docker-compose.yml | 7 ++-----
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index b1d65afe56..62d31c7403 100644
--- a/Makefile
+++ b/Makefile
@@ -225,7 +225,6 @@ endif
 .PHONY: docker-integration-test
 docker-integration-test:
 	@echo "--- Running Docker integration test"
-	pwd
 	./scripts/docker-integration-tests/run.sh
 
 .PHONY: docker-compatibility-test
diff --git a/docker-compose.yml b/docker-compose.yml
index 5c189ab496..26a76cdbe5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,7 +11,6 @@ services:
       # Support running docker within docker. That is, buildkite jobs themselves run in a container; that container
       # needs to be able to spin up functioning docker containers.
       - /var/run/docker.sock:/var/run/docker.sock
-      - comparator_volume:/go/src/github.com/m3db/m3/scripts/comparator
     extra_hosts:
       # Allow routing from the buildkite container to the host machine, as host.docker.internal. This allows us to do
       # the following:
@@ -39,5 +38,3 @@ services:
       - SPLIT_IDX
       - TOTAL_SPLITS
       - GOPATH=/go
-volumes:
-  comparator_volume:
diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 9b8ab3eea3..c30b99715a 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - comparator_volume:/etc/m3query/
+      - .:/go/src/github.com/m3db/m3
   prometheus:
     expose:
       - "9090"
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - comparator_volume:/etc/prometheus/
+      - .:/go/src/github.com/m3db/m3
   grafana:
     build:
       context: ./grafana
@@ -45,6 +45,3 @@ services:
     image: m3grafana:latest
 networks:
   backend:
-volumes:
-  comparator_volume:
-    external: true

From cb5b41b1bf4b3e4a14e519cb9396bf4694613c12 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 11:32:50 -0700
Subject: [PATCH 54/92] use host docker internal

---
 scripts/comparator/compare.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 9ebbc38a06..c01d6939ed 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -52,7 +52,7 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "0.0.0.0:9090", "prom address")
+		pPromAddress  = flag.String("promAdress", "host.docker.internal:9090", "prom address")
 		pQueryAddress = flag.String("queryAddress", "0.0.0.0:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")

From 5c5dbe5b70d766e9f2d2974bf559bcce2f72d0e8 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 11:39:03 -0700
Subject: [PATCH 55/92] add host.docker.internal to queryaddress

---
 scripts/comparator/compare.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index c01d6939ed..3fc8d956ac 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -53,7 +53,7 @@ func main() {
 
 		pQueryFile    = flag.String("input", "", "the query file")
 		pPromAddress  = flag.String("promAdress", "host.docker.internal:9090", "prom address")
-		pQueryAddress = flag.String("queryAddress", "0.0.0.0:7201/m3query", "M3 query address")
+		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")
 		pRegressionDir     = flag.String("regressionDir", "", "optional directory for regression tests")

From 69d9dc7db0d7dc4eec6fe0426c96eeb86940c1b3 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 11:44:40 -0700
Subject: [PATCH 56/92] did we get here?

---
 scripts/comparator/compare.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 3fc8d956ac..da5f9ed0eb 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -129,6 +129,8 @@ func main() {
 		}
 	}
 
+	log.Info("did we get here?")
+
 	if !multiErr.Empty() {
 		log.Fatal("mismatched queries detected in base queries")
 	}

From 475b747639cba2a21c5b18f7e9bd184ab00149d0 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 12:06:48 -0700
Subject: [PATCH 57/92] add logging for response body

---
 scripts/comparator/compare.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index da5f9ed0eb..3cd7d4082b 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -26,6 +26,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -129,8 +130,6 @@ func main() {
 		}
 	}
 
-	log.Info("did we get here?")
-
 	if !multiErr.Empty() {
 		log.Fatal("mismatched queries detected in base queries")
 	}
@@ -272,13 +271,13 @@ func runComparison(
 	queryURL string,
 	log *zap.Logger,
 ) error {
-	promResult, err := parseResult(promURL)
+	promResult, err := parseResult(promURL, log)
 	if err != nil {
 		log.Error("failed to parse Prometheus result", zap.Error(err))
 		return err
 	}
 
-	queryResult, err := parseResult(queryURL)
+	queryResult, err := parseResult(queryURL, log)
 	if err != nil {
 		log.Error("failed to parse M3Query result", zap.Error(err))
 		return err
@@ -293,7 +292,7 @@ func runComparison(
 	return nil
 }
 
-func parseResult(endpoint string) (prometheus.Response, error) {
+func parseResult(endpoint string, log *zap.Logger) (prometheus.Response, error) {
 	var result prometheus.Response
 	response, err := http.Get(endpoint)
 	if err != nil {
@@ -301,6 +300,9 @@ func parseResult(endpoint string) (prometheus.Response, error) {
 	}
 
 	if response.StatusCode != http.StatusOK {
+		bodyBytes, _ := io.ReadAll(response.Body)
+		bodyString := string(bodyBytes)
+		log.Info("response_body", zap.String("response_body", bodyString))
 		return result, fmt.Errorf("response failed with code %s", response.Status)
 	}
 

From d2bdff26bbcea009423ff2cf4c8ed19a290217c4 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 12:37:02 -0700
Subject: [PATCH 58/92] test

---
 scripts/comparator/m3query.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/m3query.yml b/scripts/comparator/m3query.yml
index 5e67225baa..3ad978dc69 100644
--- a/scripts/comparator/m3query.yml
+++ b/scripts/comparator/m3query.yml
@@ -5,7 +5,7 @@ backend: grpc
 rpc:
   remotes:
     - name: "remote"
-      remoteListenAddresses: ["m3comparator:9000"]
+      remoteListenAddresses: ["random:9000"]
 
 metrics:
   scope:

From ca574ac7ef756b87b8adcb411bf66a21104715ae Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 12:41:43 -0700
Subject: [PATCH 59/92] update volume mount for m3query

---
 scripts/comparator/docker-compose.yml | 2 +-
 scripts/comparator/m3query.yml        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index c30b99715a..90f3653710 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - .:/go/src/github.com/m3db/m3
+      - ./scripts/comparator:/etc/m3query
   prometheus:
     expose:
       - "9090"
diff --git a/scripts/comparator/m3query.yml b/scripts/comparator/m3query.yml
index 3ad978dc69..5e67225baa 100644
--- a/scripts/comparator/m3query.yml
+++ b/scripts/comparator/m3query.yml
@@ -5,7 +5,7 @@ backend: grpc
 rpc:
   remotes:
     - name: "remote"
-      remoteListenAddresses: ["random:9000"]
+      remoteListenAddresses: ["m3comparator:9000"]
 
 metrics:
   scope:

From ab8fe4e443e822bbe4e78003f2b56ae8f785e1dd Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 12:47:07 -0700
Subject: [PATCH 60/92] add slash to volume mount

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 90f3653710..fca7940941 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - ./scripts/comparator:/etc/m3query
+      - ./scripts/comparator/:/etc/m3query/
   prometheus:
     expose:
       - "9090"

From 4b0174fe4d69017f2ddc796e2a835ec50326f5a7 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 13:44:46 -0700
Subject: [PATCH 61/92] comment out volume mount to see what happens

---
 scripts/comparator/docker-compose.yml | 4 ++--
 scripts/comparator/run.sh             | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index fca7940941..3d6eaf634a 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,8 +20,8 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
-    volumes:
-      - ./scripts/comparator/:/etc/m3query/
+#    volumes:
+#      - ./scripts/comparator/:/etc/m3query/
   prometheus:
     expose:
       - "9090"
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 25d6dd9bc8..6cc471dce4 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -45,8 +45,6 @@ then
 	$COMPARATOR/setup.sh
 
 	echo "setting up docker"
-	pwd
-	ls /go/src/github.com/m3db/m3/scripts/comparator/
 	setup_docker $CI
 fi
 

From ab35f11278bbc853e70cd70c5bca1926067c7624 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 13:52:33 -0700
Subject: [PATCH 62/92] try deleting random CMD in m3comparator

---
 scripts/comparator/m3comparator.Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/comparator/m3comparator.Dockerfile b/scripts/comparator/m3comparator.Dockerfile
index ee89a6102f..e0b2a73dcd 100755
--- a/scripts/comparator/m3comparator.Dockerfile
+++ b/scripts/comparator/m3comparator.Dockerfile
@@ -8,4 +8,3 @@ ADD ./m3comparator /bin/
 EXPOSE 9000/tcp 9001/tcp
 
 ENTRYPOINT [ "/bin/m3comparator" ]
-CMD

From 5dd2f18af3c7f668fb355f4069efd77e12c2ee49 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 13:57:04 -0700
Subject: [PATCH 63/92] add m3comparator

---
 src/query/config/m3query-dev-remote.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/config/m3query-dev-remote.yml b/src/query/config/m3query-dev-remote.yml
index 560830e9fd..1f9c91596f 100644
--- a/src/query/config/m3query-dev-remote.yml
+++ b/src/query/config/m3query-dev-remote.yml
@@ -5,7 +5,7 @@ backend: grpc
 rpc:
   remotes:
     - name: "remote"
-      remoteListenAddresses: ["localhost:9000"]
+      remoteListenAddresses: ["m3comparator:9000"]
 
 metrics:
   scope:

From 6eb180fd92012d7437f9d6d9042c45ad271ecef0 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 14:10:54 -0700
Subject: [PATCH 64/92] add logs to see promResult and queryResult

---
 scripts/comparator/compare.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 3cd7d4082b..ff5edcfed5 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -283,6 +283,9 @@ func runComparison(
 		return err
 	}
 
+	log.Info("Prometheus result", zap.Any("result", promResult))
+	log.Info("M3Query result", zap.Any("result", queryResult))
+
 	_, err = promResult.Matches(queryResult)
 	if err != nil {
 		log.Error("mismatch", zap.Error(err))

From 3d0751b508623ae393806a9be3ea5f2fd348d377 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 14:13:23 -0700
Subject: [PATCH 65/92] don't use deprecated readall

---
 scripts/comparator/compare.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index ff5edcfed5..782eebd48e 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -27,7 +27,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"time"
@@ -314,7 +313,7 @@ func parseResult(endpoint string, log *zap.Logger) (prometheus.Response, error)
 		body.Close()
 	}()
 
-	data, err := ioutil.ReadAll(body)
+	data, err := io.ReadAll(body)
 	if err != nil {
 		return result, err
 	}

From 37155fd9db1716bc93f6298a9196394b9a838a7b Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 14:18:45 -0700
Subject: [PATCH 66/92] add log for querygroups

---
 scripts/comparator/compare.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 782eebd48e..65f426e923 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -104,6 +104,7 @@ func main() {
 	}
 
 	var multiErr xerrors.MultiError
+	log.Info("queries", zap.Any("queries", queries))
 	for _, queryGroup := range queries {
 		runs := 1
 		if queryGroup.Reruns > 1 {

From bc23d992f2390d15d386c106833da864e93528fd Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 14:29:15 -0700
Subject: [PATCH 67/92] remove fetch querygroup

---
 scripts/comparator/basic_queries/queries.json | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/scripts/comparator/basic_queries/queries.json b/scripts/comparator/basic_queries/queries.json
index 0f1dc99e26..1a87012292 100644
--- a/scripts/comparator/basic_queries/queries.json
+++ b/scripts/comparator/basic_queries/queries.json
@@ -9,18 +9,6 @@
       "1m"
     ]
   },
-  {
-    "queryGroup": "fetch",
-    "queries": [
-      "quail",
-      "quail offset 60s"
-    ],
-    "steps": [
-      "15s",
-      "30s",
-      "1m"
-    ]
-  },
   {
     "queryGroup": "temporal",
     "queries": [

From 644828768f338e46f0937a5470a3bdf000241701 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 14:39:29 -0700
Subject: [PATCH 68/92] add back fetch query group

---
 scripts/comparator/basic_queries/queries.json | 12 ++++++++++++
 scripts/comparator/compare.go                 |  1 -
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/basic_queries/queries.json b/scripts/comparator/basic_queries/queries.json
index 1a87012292..0f1dc99e26 100644
--- a/scripts/comparator/basic_queries/queries.json
+++ b/scripts/comparator/basic_queries/queries.json
@@ -9,6 +9,18 @@
       "1m"
     ]
   },
+  {
+    "queryGroup": "fetch",
+    "queries": [
+      "quail",
+      "quail offset 60s"
+    ],
+    "steps": [
+      "15s",
+      "30s",
+      "1m"
+    ]
+  },
   {
     "queryGroup": "temporal",
     "queries": [
diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 65f426e923..782eebd48e 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -104,7 +104,6 @@ func main() {
 	}
 
 	var multiErr xerrors.MultiError
-	log.Info("queries", zap.Any("queries", queries))
 	for _, queryGroup := range queries {
 		runs := 1
 		if queryGroup.Reruns > 1 {

From d93e50f52d9ab11fba730a93deb5247c2022d63c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 14:52:44 -0700
Subject: [PATCH 69/92] change comparator_write

---
 scripts/comparator/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 6cc471dce4..251d6cbfc0 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="localhost:9001"
+export COMPARATOR_WRITE="m3comparator:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl

From bd1225f0626554bcf068a34b8c1fe863ca65025e Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 16:53:35 -0700
Subject: [PATCH 70/92] add log to find endpoint

---
 scripts/comparator/compare.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 782eebd48e..c3fd7a1f28 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -270,6 +270,8 @@ func runComparison(
 	queryURL string,
 	log *zap.Logger,
 ) error {
+	log.Info("Running comparison", zap.String("promURL", promURL), zap.String("queryURL", queryURL))
+
 	promResult, err := parseResult(promURL, log)
 	if err != nil {
 		log.Error("failed to parse Prometheus result", zap.Error(err))
@@ -302,9 +304,6 @@ func parseResult(endpoint string, log *zap.Logger) (prometheus.Response, error)
 	}
 
 	if response.StatusCode != http.StatusOK {
-		bodyBytes, _ := io.ReadAll(response.Body)
-		bodyString := string(bodyBytes)
-		log.Info("response_body", zap.String("response_body", bodyString))
 		return result, fmt.Errorf("response failed with code %s", response.Status)
 	}
 

From 05db6e24920726d73c406bca3716abed73630365 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 17:07:56 -0700
Subject: [PATCH 71/92] only log the promResult

---
 scripts/comparator/compare.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index c3fd7a1f28..99da835cb6 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -272,13 +272,13 @@ func runComparison(
 ) error {
 	log.Info("Running comparison", zap.String("promURL", promURL), zap.String("queryURL", queryURL))
 
-	promResult, err := parseResult(promURL, log)
+	promResult, err := parseResult(promURL)
 	if err != nil {
 		log.Error("failed to parse Prometheus result", zap.Error(err))
 		return err
 	}
 
-	queryResult, err := parseResult(queryURL, log)
+	queryResult, err := parseResult(queryURL)
 	if err != nil {
 		log.Error("failed to parse M3Query result", zap.Error(err))
 		return err
@@ -296,7 +296,7 @@ func runComparison(
 	return nil
 }
 
-func parseResult(endpoint string, log *zap.Logger) (prometheus.Response, error) {
+func parseResult(endpoint string) (prometheus.Response, error) {
 	var result prometheus.Response
 	response, err := http.Get(endpoint)
 	if err != nil {

From 77ead0d657b7fff94a9909eb9734963e2bf03a56 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 28 Aug 2024 17:14:12 -0700
Subject: [PATCH 72/92] only log the prometheus result actually

---
 scripts/comparator/compare.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 99da835cb6..3eefa1a526 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -285,7 +285,6 @@ func runComparison(
 	}
 
 	log.Info("Prometheus result", zap.Any("result", promResult))
-	log.Info("M3Query result", zap.Any("result", queryResult))
 
 	_, err = promResult.Matches(queryResult)
 	if err != nil {

From c338de83bcf41e600ca1f22729dfdbfc7d4c72d5 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Tue, 10 Sep 2024 21:47:34 -0700
Subject: [PATCH 73/92] log queries

---
 scripts/comparator/compare.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 3eefa1a526..c48487f58d 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -82,6 +82,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	log.Info("log queryFile", zap.String("queryFile", queryFile))
+
 	if len(promAddress) == 0 {
 		paramError("No prom address found", log)
 		os.Exit(1)
@@ -114,6 +116,7 @@ func main() {
 			log.Info("running query group",
 				zap.String("group", queryGroup.QueryGroup),
 				zap.Int("run", i+1))
+			log.Info("log query group", zap.Any("group", queryGroup))
 			if err := runQueryGroup(
 				queryGroup,
 				promAddress,
@@ -250,6 +253,7 @@ func runQueryGroup(
 ) error {
 	var multiErr xerrors.MultiError
 	for _, query := range queryGroup.Queries {
+		log.Info("running query", zap.String("query", query))
 		promURL := fmt.Sprintf("http://%s%s", promAddress, query)
 		queryURL := fmt.Sprintf("http://%s%s", queryAddress, query)
 		if err := runComparison(promURL, queryURL, log); err != nil {

From a9a68197330950e96dec1a8d56afb3273eb0f299 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 10:46:53 -0700
Subject: [PATCH 74/92] switch to using ioutil, and remove some logs

---
 scripts/comparator/compare.go | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index c48487f58d..8034dea512 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -26,7 +26,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"time"
@@ -82,8 +82,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	log.Info("log queryFile", zap.String("queryFile", queryFile))
-
 	if len(promAddress) == 0 {
 		paramError("No prom address found", log)
 		os.Exit(1)
@@ -116,7 +114,6 @@ func main() {
 			log.Info("running query group",
 				zap.String("group", queryGroup.QueryGroup),
 				zap.Int("run", i+1))
-			log.Info("log query group", zap.Any("group", queryGroup))
 			if err := runQueryGroup(
 				queryGroup,
 				promAddress,
@@ -253,7 +250,6 @@ func runQueryGroup(
 ) error {
 	var multiErr xerrors.MultiError
 	for _, query := range queryGroup.Queries {
-		log.Info("running query", zap.String("query", query))
 		promURL := fmt.Sprintf("http://%s%s", promAddress, query)
 		queryURL := fmt.Sprintf("http://%s%s", queryAddress, query)
 		if err := runComparison(promURL, queryURL, log); err != nil {
@@ -289,6 +285,7 @@ func runComparison(
 	}
 
 	log.Info("Prometheus result", zap.Any("result", promResult))
+	log.Info("M3Query result", zap.Any("result", queryResult))
 
 	_, err = promResult.Matches(queryResult)
 	if err != nil {
@@ -315,7 +312,7 @@ func parseResult(endpoint string) (prometheus.Response, error) {
 		body.Close()
 	}()
 
-	data, err := io.ReadAll(body)
+	data, err := ioutil.ReadAll(body)
 	if err != nil {
 		return result, err
 	}

From 0302873ea7934054ef695ec75072bf8098749142 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 10:54:03 -0700
Subject: [PATCH 75/92] test revert back to localhost:9000 for
 m3query-dev-remote

---
 src/query/config/m3query-dev-remote.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/config/m3query-dev-remote.yml b/src/query/config/m3query-dev-remote.yml
index 1f9c91596f..560830e9fd 100644
--- a/src/query/config/m3query-dev-remote.yml
+++ b/src/query/config/m3query-dev-remote.yml
@@ -5,7 +5,7 @@ backend: grpc
 rpc:
   remotes:
     - name: "remote"
-      remoteListenAddresses: ["m3comparator:9000"]
+      remoteListenAddresses: ["localhost:9000"]
 
 metrics:
   scope:

From 5fbef1d752f0cf4c92a8840b7be90195dd7b0fb9 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 11:08:33 -0700
Subject: [PATCH 76/92] revert back to m3comparator:9000, else we get 500 error

---
 src/query/config/m3query-dev-remote.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/config/m3query-dev-remote.yml b/src/query/config/m3query-dev-remote.yml
index 560830e9fd..1f9c91596f 100644
--- a/src/query/config/m3query-dev-remote.yml
+++ b/src/query/config/m3query-dev-remote.yml
@@ -5,7 +5,7 @@ backend: grpc
 rpc:
   remotes:
     - name: "remote"
-      remoteListenAddresses: ["localhost:9000"]
+      remoteListenAddresses: ["m3comparator:9000"]
 
 metrics:
   scope:

From 640eb7ef0305586e1c996027d918d0ffcb69dc0f Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 11:18:26 -0700
Subject: [PATCH 77/92] revert back to localhost for comparator in run.sh to
 test

---
 scripts/comparator/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 251d6cbfc0..6cc471dce4 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="m3comparator:9001"
+export COMPARATOR_WRITE="localhost:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl

From 1f0f2cde58ae6f5cc73afd962432cbd663ccd7f0 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 11:47:19 -0700
Subject: [PATCH 78/92] revert promAddress to 0.0.0.0 to test

---
 scripts/comparator/compare.go         | 2 +-
 scripts/comparator/docker-compose.yml | 2 --
 scripts/comparator/docker-setup.sh    | 1 -
 3 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 8034dea512..e096e7dd6e 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -52,7 +52,7 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "host.docker.internal:9090", "prom address")
+		pPromAddress  = flag.String("promAdress", "0.0.0.0:9090", "prom address")
 		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")
diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 3d6eaf634a..6995dbfae8 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,8 +20,6 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
-#    volumes:
-#      - ./scripts/comparator/:/etc/m3query/
   prometheus:
     expose:
       - "9090"
diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index d55973defe..358c296c3f 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -12,7 +12,6 @@ function setup_docker {
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
   docker-compose -f ${COMPOSE_FILE} logs prometheus
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query
-  docker-compose -f ${COMPOSE_FILE} logs m3query
   docker-compose -f ${COMPOSE_FILE} ps
   
   CI=$1

From 2b8907f35e221e54a7d445e43017616f5d0c2f0a Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 11:59:40 -0700
Subject: [PATCH 79/92] add back volume mount to m3query container

---
 scripts/comparator/compare.go         | 2 +-
 scripts/comparator/docker-compose.yml | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index e096e7dd6e..8034dea512 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -52,7 +52,7 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "0.0.0.0:9090", "prom address")
+		pPromAddress  = flag.String("promAdress", "host.docker.internal:9090", "prom address")
 		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")
diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index 6995dbfae8..de973e707e 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -20,6 +20,8 @@ services:
     networks:
       - backend
     image: "m3query:${REVISION}"
+    volumes:
+      - "./m3query.yml:/etc/m3query/m3query.yml"
   prometheus:
     expose:
       - "9090"

From 562715cb28eb31fe48c911c675580af1a4164249 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 12:05:09 -0700
Subject: [PATCH 80/92] update volume mount for m3query

---
 scripts/comparator/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml
index de973e707e..c30b99715a 100755
--- a/scripts/comparator/docker-compose.yml
+++ b/scripts/comparator/docker-compose.yml
@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - "./m3query.yml:/etc/m3query/m3query.yml"
+      - .:/go/src/github.com/m3db/m3
   prometheus:
     expose:
       - "9090"

From 62fae3f033975c6fd0a679a7a497110efb4c415b Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 12:16:05 -0700
Subject: [PATCH 81/92] change promAddress to queryAddress

---
 scripts/comparator/compare.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 8034dea512..9cb91c446a 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -52,7 +52,7 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "host.docker.internal:9090", "prom address")
+		pPromAddress  = flag.String("promAdress", "host.docker.internal:7201/m3query", "prom address")
 		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")

From 85b2354cbe9b336b0d7aa5be26b5a5502fd47d31 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 12:36:35 -0700
Subject: [PATCH 82/92] add comparator address

---
 scripts/comparator/compare.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 9cb91c446a..0f6047ee2e 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -55,7 +55,7 @@ func main() {
 		pPromAddress  = flag.String("promAdress", "host.docker.internal:7201/m3query", "prom address")
 		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
-		pComparatorAddress = flag.String("comparator", "", "comparator address")
+		pComparatorAddress = flag.String("comparator", "host.docker.internal:9001", "comparator address")
 		pRegressionDir     = flag.String("regressionDir", "", "optional directory for regression tests")
 
 		pStart = flag.Int64("s", now.Add(time.Hour*-3).Unix(), "start time")
@@ -270,8 +270,6 @@ func runComparison(
 	queryURL string,
 	log *zap.Logger,
 ) error {
-	log.Info("Running comparison", zap.String("promURL", promURL), zap.String("queryURL", queryURL))
-
 	promResult, err := parseResult(promURL)
 	if err != nil {
 		log.Error("failed to parse Prometheus result", zap.Error(err))
@@ -284,9 +282,6 @@ func runComparison(
 		return err
 	}
 
-	log.Info("Prometheus result", zap.Any("result", promResult))
-	log.Info("M3Query result", zap.Any("result", queryResult))
-
 	_, err = promResult.Matches(queryResult)
 	if err != nil {
 		log.Error("mismatch", zap.Error(err))

From 1aeba01f7ae4ff0fae3cf858bce06887a91ff109 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 13:21:01 -0700
Subject: [PATCH 83/92] change to m3comparator:9001

---
 scripts/comparator/compare.go | 3 ++-
 scripts/comparator/run.sh     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index 0f6047ee2e..c7ebf3b6d6 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -55,7 +55,7 @@ func main() {
 		pPromAddress  = flag.String("promAdress", "host.docker.internal:7201/m3query", "prom address")
 		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
-		pComparatorAddress = flag.String("comparator", "host.docker.internal:9001", "comparator address")
+		pComparatorAddress = flag.String("comparator", "", "comparator address")
 		pRegressionDir     = flag.String("regressionDir", "", "optional directory for regression tests")
 
 		pStart = flag.Int64("s", now.Add(time.Hour*-3).Unix(), "start time")
@@ -212,6 +212,7 @@ func runRegression(
 	}
 
 	comparatorURL := fmt.Sprintf("http://%s", comparatorAddress)
+
 	resp, err := http.Post(comparatorURL, "application/json", bytes.NewReader(data))
 	if err != nil {
 		log.Error("could not seed regression data", zap.Error(err))
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 6cc471dce4..251d6cbfc0 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="localhost:9001"
+export COMPARATOR_WRITE="m3comparator:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl

From a9b9396e05a1cb000598dca407fdcd16ea32540c Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 13:28:56 -0700
Subject: [PATCH 84/92] change to host.docker.internal instead of m3comparator

---
 scripts/comparator/compare.go | 1 -
 scripts/comparator/run.sh     | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go
index c7ebf3b6d6..8ea99a1e47 100644
--- a/scripts/comparator/compare.go
+++ b/scripts/comparator/compare.go
@@ -212,7 +212,6 @@ func runRegression(
 	}
 
 	comparatorURL := fmt.Sprintf("http://%s", comparatorAddress)
-
 	resp, err := http.Post(comparatorURL, "application/json", bytes.NewReader(data))
 	if err != nil {
 		log.Error("could not seed regression data", zap.Error(err))
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index 251d6cbfc0..b2490234e6 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="m3comparator:9001"
+export COMPARATOR_WRITE="host.docker.internal:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl

From 721609e797cde64f072303f12b773977af0653cb Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 13:32:34 -0700
Subject: [PATCH 85/92] uncomment last command in run.sh

---
 scripts/comparator/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index b2490234e6..b684549099 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -75,5 +75,5 @@ $comparator -input=$QUERY_FILE \
 -comparator=$COMPARATOR_WRITE \
 -regressionDir=$REGRESSION_DIR
 
-## Run PromQL testdata tests
-#go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/
+# Run PromQL testdata tests
+go test -v -timeout 300s -tags=compatibility -count=1 github.com/m3db/m3/src/query/test/compatibility/

From 7e135a298693248ec37d8047fece9eae3d226c73 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 13:48:14 -0700
Subject: [PATCH 86/92] change test to host.docker.internal

---
 src/query/test/compatibility/test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/test/compatibility/test.go b/src/query/test/compatibility/test.go
index d6ae66954c..8a42778c50 100644
--- a/src/query/test/compatibility/test.go
+++ b/src/query/test/compatibility/test.go
@@ -90,7 +90,7 @@ func NewTest(t testutil.T, input string) (*Test, error) {
 	test := &Test{
 		T:            t,
 		cmds:         []testCommand{},
-		m3comparator: newM3ComparatorClient("localhost", 9001),
+		m3comparator: newM3ComparatorClient("host.docker.internal", 9001),
 	}
 	err := test.parse(input)
 	if err != nil {

From afb11c66b521213df70e0229d1e074a1f561871a Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 13:53:31 -0700
Subject: [PATCH 87/92] change m3query to host.docker.internal

---
 src/query/test/compatibility/test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/query/test/compatibility/test.go b/src/query/test/compatibility/test.go
index 8a42778c50..92df93ea85 100644
--- a/src/query/test/compatibility/test.go
+++ b/src/query/test/compatibility/test.go
@@ -371,7 +371,7 @@ func newEvalCmd(expr string, start time.Time, line int) *evalCmd {
 
 		metrics:  map[uint64]labels.Labels{},
 		expected: map[uint64]entry{},
-		m3query:  newM3QueryClient("localhost", 7201),
+		m3query:  newM3QueryClient("host.docker.internal", 7201),
 	}
 }
 

From 0777c6e4e209ecc9a446ce5c485972ca0dfba6b7 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 14:02:55 -0700
Subject: [PATCH 88/92] uncomment rest of pipeline tests

---
 .buildkite/pipeline.yml | 422 ++++++++++++++++++++--------------------
 1 file changed, 211 insertions(+), 211 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 40fc03836f..54c827e1e8 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -76,113 +76,113 @@ steps:
 #                - |-
 #                  make clean install-vendor-m3 test-all-gen
 #    <<: *common
-#  - name: "Unit %n"
-#    parallelism: 4
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-base
-#    <<: *common
-#  - name: "Big Unit %n"
-#    parallelism: 2
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-big-base
-#    <<: *common
-#  - name: "Services, Tools"
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 services tools
-#    <<: *common
-#  - name: "Lint"
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean lint
-#    <<: *common
-#  - label: "Integration (:docker:)"
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 docker-integration-test
-#    <<: *common
-#  - label: "M3 Cluster Integration Tests"
-#    plugins:
-#      gopath-checkout#v1.0.1:
-#        import: github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean test-ci-cluster-integration
-#    <<: *common
-#  - label: "M3 Cluster Integration Test Harness Tests"
-#    skip: "NB(nate): temporarily disabling to resolve some tests flakes"
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean test-ci-test-harness
-#    <<: *common
+  - name: "Unit %n"
+    parallelism: 4
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-base
+    <<: *common
+  - name: "Big Unit %n"
+    parallelism: 2
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-big-base
+    <<: *common
+  - name: "Services, Tools"
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 services tools
+    <<: *common
+  - name: "Lint"
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean lint
+    <<: *common
+  - label: "Integration (:docker:)"
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 docker-integration-test
+    <<: *common
+  - label: "M3 Cluster Integration Tests"
+    plugins:
+      gopath-checkout#v1.0.1:
+        import: github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean test-ci-cluster-integration
+    <<: *common
+  - label: "M3 Cluster Integration Test Harness Tests"
+    skip: "NB(nate): temporarily disabling to resolve some tests flakes"
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean test-ci-test-harness
+    <<: *common
   - name: "Prometheus compatibility (:docker:)"
     parallelism: 1
     env:
@@ -204,107 +204,107 @@ steps:
                   - |-
                     make clean install-vendor-m3 docker-compatibility-test
     <<: *common
-#  - name: "Integration (dbnode Recently Read) %n"
-#    parallelism: 2
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
-#    <<: *common
-#  - name: "Integration (dbnode LRU) %n"
-#    parallelism: 2
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
-#    <<: *common
-#  - name: "Integration (aggregator TCP client) %n"
-#    parallelism: 1
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp
-#    <<: *common
-#  - name: "Integration (aggregator m3msg client) %n"
-#    parallelism: 1
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg
-#    <<: *common
-#  - label: "Integration (m3em, cluster, msg, metrics) %n"
-#    parallelism: 4
-#    plugins:
-#      docker-compose#v2.5.1:
-#        run: app
-#        workdir: /go/src/github.com/m3db/m3
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
-#    <<: *common
-# Commenting out Documentation test for now because of issue with access tokens. Also a low priority test, as it checks
-# for broken links in the documentation. ISSUE: https://github.com/m3db/m3/issues/4288
-#  - name: "Documentation tests"
-#    command: make clean install-vendor-m3 docs-test
-#    env:
-#      CGO_ENABLED: 0
-#      GIMME_GO_VERSION: 1.22.x
-#    plugins:
-#        gopath-checkout#v1.0.1:
-#          import: github.com/m3db/m3
-#    <<: *common
-#  - name: "Check for docker and docs builds :docker: :books:"
-#    plugins:
-#      kubernetes:
-#        <<: *kubernetes
-#        podSpec:
-#          <<: *podSpec
-#          containers:
-#            - <<: *commandContainer
-#              command:
-#                - |-
-#                  .buildkite/scripts/check_do_docs.sh
-#                  .ci/docker/check_do_docker.sh
-#    <<: *common
+  - name: "Integration (dbnode Recently Read) %n"
+    parallelism: 2
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read
+    <<: *common
+  - name: "Integration (dbnode LRU) %n"
+    parallelism: 2
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru
+    <<: *common
+  - name: "Integration (aggregator TCP client) %n"
+    parallelism: 1
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=tcp
+    <<: *common
+  - name: "Integration (aggregator m3msg client) %n"
+    parallelism: 1
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-ci-integration-aggregator aggregator_client=m3msg
+    <<: *common
+  - label: "Integration (m3em, cluster, msg, metrics) %n"
+    parallelism: 4
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
+    <<: *common
+ Commenting out Documentation test for now because of issue with access tokens. Also a low priority test, as it checks
+ for broken links in the documentation. ISSUE: https://github.com/m3db/m3/issues/4288
+  - name: "Documentation tests"
+    command: make clean install-vendor-m3 docs-test
+    env:
+      CGO_ENABLED: 0
+      GIMME_GO_VERSION: 1.22.x
+    plugins:
+        gopath-checkout#v1.0.1:
+          import: github.com/m3db/m3
+    <<: *common
+  - name: "Check for docker and docs builds :docker: :books:"
+    plugins:
+      kubernetes:
+        <<: *kubernetes
+        podSpec:
+          <<: *podSpec
+          containers:
+            - <<: *commandContainer
+              command:
+                - |-
+                  .buildkite/scripts/check_do_docs.sh
+                  .ci/docker/check_do_docker.sh
+    <<: *common

From 943d190ffda6a51d3bfe44f639ecc08ada44686d Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 14:04:51 -0700
Subject: [PATCH 89/92] comment out documentation tests

---
 .buildkite/pipeline.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 54c827e1e8..4c5f9abc26 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -284,17 +284,17 @@ steps:
                 - |-
                   make clean install-vendor-m3 test-ci-integration-m3em test-ci-integration-cluster test-ci-integration-msg test-ci-integration-metrics
     <<: *common
- Commenting out Documentation test for now because of issue with access tokens. Also a low priority test, as it checks
- for broken links in the documentation. ISSUE: https://github.com/m3db/m3/issues/4288
-  - name: "Documentation tests"
-    command: make clean install-vendor-m3 docs-test
-    env:
-      CGO_ENABLED: 0
-      GIMME_GO_VERSION: 1.22.x
-    plugins:
-        gopath-checkout#v1.0.1:
-          import: github.com/m3db/m3
-    <<: *common
+# Commenting out Documentation test for now because of issue with access tokens. Also a low priority test, as it checks
+# for broken links in the documentation. ISSUE: https://github.com/m3db/m3/issues/4288
+#  - name: "Documentation tests"
+#    command: make clean install-vendor-m3 docs-test
+#    env:
+#      CGO_ENABLED: 0
+#      GIMME_GO_VERSION: 1.22.x
+#    plugins:
+#        gopath-checkout#v1.0.1:
+#          import: github.com/m3db/m3
+#    <<: *common
   - name: "Check for docker and docs builds :docker: :books:"
     plugins:
       kubernetes:

From 4d5c8c0da91b40e8a4fe9f0e107415d23279aafb Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 14:10:58 -0700
Subject: [PATCH 90/92] add back CMD in m3comparator.dockerfile

---
 scripts/comparator/m3comparator.Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/comparator/m3comparator.Dockerfile b/scripts/comparator/m3comparator.Dockerfile
index e0b2a73dcd..b8f1f2ceca 100755
--- a/scripts/comparator/m3comparator.Dockerfile
+++ b/scripts/comparator/m3comparator.Dockerfile
@@ -8,3 +8,4 @@ ADD ./m3comparator /bin/
 EXPOSE 9000/tcp 9001/tcp
 
 ENTRYPOINT [ "/bin/m3comparator" ]
+CMD
\ No newline at end of file

From 28a83647cee6e4394b375c4088cbc3ea62f8f18f Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 14:15:07 -0700
Subject: [PATCH 91/92] add space in dockerfile

---
 scripts/comparator/m3comparator.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comparator/m3comparator.Dockerfile b/scripts/comparator/m3comparator.Dockerfile
index b8f1f2ceca..ee89a6102f 100755
--- a/scripts/comparator/m3comparator.Dockerfile
+++ b/scripts/comparator/m3comparator.Dockerfile
@@ -8,4 +8,4 @@ ADD ./m3comparator /bin/
 EXPOSE 9000/tcp 9001/tcp
 
 ENTRYPOINT [ "/bin/m3comparator" ]
-CMD
\ No newline at end of file
+CMD

From 31f18463157a50c38067358d8bdbdb429a5a8db5 Mon Sep 17 00:00:00 2001
From: zengk <zengk@uber.com>
Date: Wed, 11 Sep 2024 14:46:44 -0700
Subject: [PATCH 92/92] remove logs and clean up temp code

---
 scripts/comparator/docker-setup.sh | 4 +---
 scripts/comparator/run.sh          | 1 -
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh
index 358c296c3f..f61c9db922 100755
--- a/scripts/comparator/docker-setup.sh
+++ b/scripts/comparator/docker-setup.sh
@@ -10,10 +10,8 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
-  docker-compose -f ${COMPOSE_FILE} logs prometheus
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query
-  docker-compose -f ${COMPOSE_FILE} ps
-  
+
   CI=$1
   if [[ "$CI" != "true" ]]
   then
diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh
index b684549099..cc45035ee3 100755
--- a/scripts/comparator/run.sh
+++ b/scripts/comparator/run.sh
@@ -68,7 +68,6 @@ then
 	trap defer EXIT
 fi
 
-docker ps -a
 $comparator -input=$QUERY_FILE \
 -s=$START \
 -e=$END \