[buildkite] Fix "Prometheus compatibility (:docker:)" test in buildki…

…te pipeline (#4295)

* comment out other tests

* add make to dockerfile

* remove make, and add kubernetes plugin

* add docker-compose

* add docker-compose to dockerfile

* update m3query volumes in docker-compose

* comment out trap defer EXIT command in run.sh

* uncomment previous code

* experimenting with commenting out rest of run.sh

* comment out everything except for line 51 and 52 at the end of run.sh

* uncomment lines 53-69

* uncomment lines 71-75

* comment out lines 71 to 75 and uncomment go test in run.sh

* check if prometheus container is running

* adding logs to check prometheus container not running

* update volume mount for prometheus in docker-compose

* modify right hand side of volume mount for prometheus in docker-compose

* try another volume mount for prometheus

* remove volume mounts to see what happens in docker-compose

* add back logs in docker-setup

* try adding prometheusURL to run.sh

* change localhost to host.docker.internal

* check for firewall

* add sudo to dockerfile

* add ufw to dockerfile

* inspect prometheus docker container for network issues

* modify docker inspect command

* add back volume mount for prometheus

* fix typo in volume mount

* fix another typo in volume mount for prometheus

* find the current directory to fix volume mount

* update volume mount for prometheus with new directory

* fix typo

* rewrite volume mount

* list out contents in host machine

* move ls, and pwd

* use absolute path for volume mount

* add volume mount for m3query

* update m3query volume mount

* update volume mount again for prometheus

* another volume mount configuration

* another volume mount configuration

* try another way to volume mount

* add m3query.yml

* add external:true

* try using quotes

* remove volume mount for m3query

* add docker inspect for m3query

* try adding another app

* change to localhost

* use external to create volume mounts

* add pwd to see why we can't find run.sh

* try another volume mount

* use host docker internal

* add host.docker.internal to queryaddress

* did we get here?

* add logging for response body

* test

* update volume mount for m3query

* add slash to volume mount

* comment out volume mount to see what happens

* try deleting random CMD in m3comparator

* add m3comparator

* add logs to see promResult and queryResult

* don't use deprecated readall

* add log for querygroups

* remove fetch querygroup

* add back fetch query group

* change comparator_write

* add log to find endpoint

* only log the promResult

* only log the prometheus result actually

* log queries

* switch to using ioutil, and remove some logs

* test revert back to localhost:9000 for m3query-dev-remote

* revert back to m3comparator:9000, else we get 500 error

* revert back to localhost for comparator in run.sh to test

* revert promAddress to 0.0.0.0 to test

* add back volume mount to m3query container

* update volume mount for m3query

* change promAddress to queryAddress

* add comparator address

* change to m3comparator:9001

* change to host.docker.internal instead of m3comparator

* uncomment last command in run.sh

* change test to host.docker.internal

* change m3query to host.docker.internal

* uncomment rest of pipeline tests

* comment out documentation tests

* add back CMD in m3comparator.dockerfile

* add space in dockerfile

* remove logs and clean up temp code

.buildkite/pipeline.yml

@@ -183,16 +183,27 @@ steps:
                 - |-
                   make clean test-ci-test-harness
     <<: *common
-#  - name: "Prometheus compatibility (:docker:)"
-#    command: make clean install-vendor-m3 docker-compatibility-test
-#    parallelism: 1
-#    env:
-#      CGO_ENABLED: 0
-#      GIMME_GO_VERSION: 1.22.x
-#    plugins:
-#        gopath-checkout#v1.0.1:
-#          import: github.com/m3db/m3
-#    <<: *common
+  - name: "Prometheus compatibility (:docker:)"
+    parallelism: 1
+    env:
+      CGO_ENABLED: 0
+      GIMME_GO_VERSION: 1.22.x
+    plugins:
+        gopath-checkout#v1.0.1:
+          import: github.com/m3db/m3
+        docker-compose#v2.5.1:
+          run: app
+          workdir: /go/src/github.com/m3db/m3
+        kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                  - |-
+                    make clean install-vendor-m3 docker-compatibility-test
+    <<: *common
   - name: "Integration (dbnode Recently Read) %n"
     parallelism: 2
     plugins:

Dockerfile

@@ -1,3 +1,3 @@
 FROM golang:1.22-bullseye
 
-RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq
+RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose

scripts/comparator/compare.go

@@ -52,8 +52,8 @@ func main() {
 		now = time.Now()
 
 		pQueryFile    = flag.String("input", "", "the query file")
-		pPromAddress  = flag.String("promAdress", "0.0.0.0:9090", "prom address")
-		pQueryAddress = flag.String("queryAddress", "0.0.0.0:7201/m3query", "M3 query address")
+		pPromAddress  = flag.String("promAdress", "host.docker.internal:7201/m3query", "prom address")
+		pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address")
 
 		pComparatorAddress = flag.String("comparator", "", "comparator address")
 		pRegressionDir     = flag.String("regressionDir", "", "optional directory for regression tests")

scripts/comparator/docker-compose.yml

@@ -21,7 +21,7 @@ services:
       - backend
     image: "m3query:${REVISION}"
     volumes:
-      - "./m3query.yml:/etc/m3query/m3query.yml"
+      - .:/go/src/github.com/m3db/m3
   prometheus:
     expose:
       - "9090"
@@ -31,7 +31,7 @@ services:
       - backend
     image: prom/prometheus:v2.31.2
     volumes:
-      - "./:/etc/prometheus/"
+      - .:/go/src/github.com/m3db/m3
   grafana:
     build:
       context: ./grafana

scripts/comparator/docker-setup.sh

@@ -10,8 +10,8 @@ function setup_docker {
   echo "Run m3query, m3comparator, and prometheus containers"
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator
   docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus
-  docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query 
-  
+  docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query
+
   CI=$1
   if [[ "$CI" != "true" ]]
   then

scripts/comparator/run.sh

@@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out
 export END=${END:-$(date +%s)}
 export START=${START:-$(( $END - 10800 ))}
 # TODO: make this a bit less hacky in the future; e.g. take from config.
-export COMPARATOR_WRITE="localhost:9001"
+export COMPARATOR_WRITE="host.docker.internal:9001"
 
 function generate_dash {
 	TEMPLATE=$GRAFANA_PATH/dashboard.tmpl
@@ -65,7 +65,7 @@ function defer {
 
 if [[ "$RUN_ONLY" == "false" ]]
 then
-	trap defer EXIT 
+	trap defer EXIT
 fi
 
 $comparator -input=$QUERY_FILE \

src/query/config/m3query-dev-remote.yml

@@ -5,7 +5,7 @@ backend: grpc
 rpc:
   remotes:
     - name: "remote"
-      remoteListenAddresses: ["localhost:9000"]
+      remoteListenAddresses: ["m3comparator:9000"]
 
 metrics:
   scope:

src/query/test/compatibility/test.go

@@ -90,7 +90,7 @@ func NewTest(t testutil.T, input string) (*Test, error) {
 	test := &Test{
 		T:            t,
 		cmds:         []testCommand{},
-		m3comparator: newM3ComparatorClient("localhost", 9001),
+		m3comparator: newM3ComparatorClient("host.docker.internal", 9001),
 	}
 	err := test.parse(input)
 	if err != nil {
@@ -371,7 +371,7 @@ func newEvalCmd(expr string, start time.Time, line int) *evalCmd {
 
 		metrics:  map[uint64]labels.Labels{},
 		expected: map[uint64]entry{},
-		m3query:  newM3QueryClient("localhost", 7201),
+		m3query:  newM3QueryClient("host.docker.internal", 7201),
 	}
 }