From d6543ea4d256d59dcf745f6d857edebdc672b2c9 Mon Sep 17 00:00:00 2001 From: kentzeng12 <114431467+kentzeng12@users.noreply.github.com> Date: Thu, 19 Dec 2024 17:08:29 -0800 Subject: [PATCH] [buildkite] Fix "Prometheus compatibility (:docker:)" test in buildkite pipeline (#4295) * comment out other tests * add make to dockerfile * remove make, and add kubernetes plugin * add docker-compose * add docker-compose to dockerfile * update m3query volumes in docker-compose * comment out trap defer EXIT command in run.sh * uncomment previous code * experimenting with commenting out rest of run.sh * comment out everything except for line 51 and 52 at the end of run.sh * uncomment lines 53-69 * uncomment lines 71-75 * comment out lines 71 to 75 and uncomment go test in run.sh * check if prometheus container is running * adding logs to check prometheus container not running * update volume mount for prometheus in docker-compose * modify right hand side of volume mount for prometheus in docker-compose * try another volume mount for prometheus * remove volume mounts to see what happens in docker-compose * add back logs in docker-setup * try adding prometheusURL to run.sh * change localhost to host.docker.internal * check for firewall * add sudo to dockerfile * add ufw to dockerfile * inspect prometheus docker container for network issues * modify docker inspect command * add back volume mount for prometheus * fix typo in volume mount * fix another typo in volume mount for prometheus * find the current directory to fix volume mount * update volume mount for prometheus with new directory * fix typo * rewrite volume mount * list out contents in host machine * move ls, and pwd * use absolute path for volume mount * add volume mount for m3query * update m3query volume mount * update volume mount again for prometheus * another volume mount configuration * another volume mount configuration * try another way to volume mount * add m3query.yml * add external:true * try using quotes * remove volume mount for m3query * add docker inspect for m3query * try adding another app * change to localhost * use external to create volume mounts * add pwd to see why we can't find run.sh * try another volume mount * use host docker internal * add host.docker.internal to queryaddress * did we get here? * add logging for response body * test * update volume mount for m3query * add slash to volume mount * comment out volume mount to see what happens * try deleting random CMD in m3comparator * add m3comparator * add logs to see promResult and queryResult * don't use deprecated readall * add log for querygroups * remove fetch querygroup * add back fetch query group * change comparator_write * add log to find endpoint * only log the promResult * only log the prometheus result actually * log queries * switch to using ioutil, and remove some logs * test revert back to localhost:9000 for m3query-dev-remote * revert back to m3comparator:9000, else we get 500 error * revert back to localhost for comparator in run.sh to test * revert promAddress to 0.0.0.0 to test * add back volume mount to m3query container * update volume mount for m3query * change promAddress to queryAddress * add comparator address * change to m3comparator:9001 * change to host.docker.internal instead of m3comparator * uncomment last command in run.sh * change test to host.docker.internal * change m3query to host.docker.internal * uncomment rest of pipeline tests * comment out documentation tests * add back CMD in m3comparator.dockerfile * add space in dockerfile * remove logs and clean up temp code --- .buildkite/pipeline.yml | 31 +++++++++++++++++-------- Dockerfile | 2 +- scripts/comparator/compare.go | 4 ++-- scripts/comparator/docker-compose.yml | 4 ++-- scripts/comparator/docker-setup.sh | 4 ++-- scripts/comparator/run.sh | 4 ++-- src/query/config/m3query-dev-remote.yml | 2 +- src/query/test/compatibility/test.go | 4 ++-- 8 files changed, 33 insertions(+), 22 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 12ed82dc28..4c5f9abc26 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -183,16 +183,27 @@ steps: - |- make clean test-ci-test-harness <<: *common -# - name: "Prometheus compatibility (:docker:)" -# command: make clean install-vendor-m3 docker-compatibility-test -# parallelism: 1 -# env: -# CGO_ENABLED: 0 -# GIMME_GO_VERSION: 1.22.x -# plugins: -# gopath-checkout#v1.0.1: -# import: github.com/m3db/m3 -# <<: *common + - name: "Prometheus compatibility (:docker:)" + parallelism: 1 + env: + CGO_ENABLED: 0 + GIMME_GO_VERSION: 1.22.x + plugins: + gopath-checkout#v1.0.1: + import: github.com/m3db/m3 + docker-compose#v2.5.1: + run: app + workdir: /go/src/github.com/m3db/m3 + kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make clean install-vendor-m3 docker-compatibility-test + <<: *common - name: "Integration (dbnode Recently Read) %n" parallelism: 2 plugins: diff --git a/Dockerfile b/Dockerfile index 98b67e1b25..66e5c81d6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,3 @@ FROM golang:1.22-bullseye -RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq +RUN apt-get update && apt-get install -y lsof netcat-openbsd docker.io jq docker-compose diff --git a/scripts/comparator/compare.go b/scripts/comparator/compare.go index 9ebbc38a06..8ea99a1e47 100644 --- a/scripts/comparator/compare.go +++ b/scripts/comparator/compare.go @@ -52,8 +52,8 @@ func main() { now = time.Now() pQueryFile = flag.String("input", "", "the query file") - pPromAddress = flag.String("promAdress", "0.0.0.0:9090", "prom address") - pQueryAddress = flag.String("queryAddress", "0.0.0.0:7201/m3query", "M3 query address") + pPromAddress = flag.String("promAdress", "host.docker.internal:7201/m3query", "prom address") + pQueryAddress = flag.String("queryAddress", "host.docker.internal:7201/m3query", "M3 query address") pComparatorAddress = flag.String("comparator", "", "comparator address") pRegressionDir = flag.String("regressionDir", "", "optional directory for regression tests") diff --git a/scripts/comparator/docker-compose.yml b/scripts/comparator/docker-compose.yml index 79bd8d4f2a..c30b99715a 100755 --- a/scripts/comparator/docker-compose.yml +++ b/scripts/comparator/docker-compose.yml @@ -21,7 +21,7 @@ services: - backend image: "m3query:${REVISION}" volumes: - - "./m3query.yml:/etc/m3query/m3query.yml" + - .:/go/src/github.com/m3db/m3 prometheus: expose: - "9090" @@ -31,7 +31,7 @@ services: - backend image: prom/prometheus:v2.31.2 volumes: - - "./:/etc/prometheus/" + - .:/go/src/github.com/m3db/m3 grafana: build: context: ./grafana diff --git a/scripts/comparator/docker-setup.sh b/scripts/comparator/docker-setup.sh index 0fe3c9e0ce..f61c9db922 100755 --- a/scripts/comparator/docker-setup.sh +++ b/scripts/comparator/docker-setup.sh @@ -10,8 +10,8 @@ function setup_docker { echo "Run m3query, m3comparator, and prometheus containers" docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3comparator docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes prometheus - docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query - + docker-compose -f ${COMPOSE_FILE} up -d --build --renew-anon-volumes m3query + CI=$1 if [[ "$CI" != "true" ]] then diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh index ea5c05d781..cc45035ee3 100755 --- a/scripts/comparator/run.sh +++ b/scripts/comparator/run.sh @@ -17,7 +17,7 @@ export DASHBOARD=$GRAFANA_PATH/dash.json.out export END=${END:-$(date +%s)} export START=${START:-$(( $END - 10800 ))} # TODO: make this a bit less hacky in the future; e.g. take from config. -export COMPARATOR_WRITE="localhost:9001" +export COMPARATOR_WRITE="host.docker.internal:9001" function generate_dash { TEMPLATE=$GRAFANA_PATH/dashboard.tmpl @@ -65,7 +65,7 @@ function defer { if [[ "$RUN_ONLY" == "false" ]] then - trap defer EXIT + trap defer EXIT fi $comparator -input=$QUERY_FILE \ diff --git a/src/query/config/m3query-dev-remote.yml b/src/query/config/m3query-dev-remote.yml index 560830e9fd..1f9c91596f 100644 --- a/src/query/config/m3query-dev-remote.yml +++ b/src/query/config/m3query-dev-remote.yml @@ -5,7 +5,7 @@ backend: grpc rpc: remotes: - name: "remote" - remoteListenAddresses: ["localhost:9000"] + remoteListenAddresses: ["m3comparator:9000"] metrics: scope: diff --git a/src/query/test/compatibility/test.go b/src/query/test/compatibility/test.go index d6ae66954c..92df93ea85 100644 --- a/src/query/test/compatibility/test.go +++ b/src/query/test/compatibility/test.go @@ -90,7 +90,7 @@ func NewTest(t testutil.T, input string) (*Test, error) { test := &Test{ T: t, cmds: []testCommand{}, - m3comparator: newM3ComparatorClient("localhost", 9001), + m3comparator: newM3ComparatorClient("host.docker.internal", 9001), } err := test.parse(input) if err != nil { @@ -371,7 +371,7 @@ func newEvalCmd(expr string, start time.Time, line int) *evalCmd { metrics: map[uint64]labels.Labels{}, expected: map[uint64]entry{}, - m3query: newM3QueryClient("localhost", 7201), + m3query: newM3QueryClient("host.docker.internal", 7201), } }