ci(.github): enable self hosted runners for AMD64 E2E tasks (#10945)

* ci(.github): enable self hosted runners for AMD64 e2e tasks * add image pulling steps * add a script to sync docker credentials into k3d/kind clusters * use docker credentials when creating clusters --------- Signed-off-by: Jay Chen <[email protected]> Signed-off-by: Charly Molter <[email protected]> Co-authored-by: Charly Molter <[email protected]>
kumahq · Dec 16, 2024 · 75a063c · 75a063c
1 parent 9c0a3a5
commit 75a063c
Show file tree

Hide file tree

Showing 11 changed files with 133 additions and 35 deletions.
diff --git a/.github/workflows/_e2e.yaml b/.github/workflows/_e2e.yaml
@@ -4,26 +4,35 @@ on:
       matrix:
         required: true
         type: string
-      runnersByArch:
+      runner:
         type: string
+<<<<<<< HEAD
         required: false
         default: '{"amd64": "ubuntu-latest", "arm64": "ubuntu-latest-arm64-kong"}'
+=======
+        required: true
+>>>>>>> 126029d11 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10945))
 permissions:
   contents: read
 env:
-  CI_TOOLS_DIR: ${{ contains(fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch], '-kong') && '/work/kuma/kuma/.ci_tools' || '/home/runner/work/kuma/kuma/.ci_tools' }}
+  CI_TOOLS_DIR: ${{ contains(inputs.runner, '-kong') && '/work/kuma/kuma/.ci_tools' || '/home/runner/work/kuma/kuma/.ci_tools' }}
   E2E_PARAM_K8S_VERSION: ${{ fromJSON(inputs.matrix).k8sVersion }}
   E2E_PARAM_CNI_NETWORK_PLUGIN: ${{ fromJSON(inputs.matrix).cniNetworkPlugin }}
   E2E_PARAM_ARCH: ${{ fromJSON(inputs.matrix).arch }}
   E2E_PARAM_SIDECAR_CONTAINERS: ${{ fromJSON(inputs.matrix).sidecarContainers }}
   E2E_PARAM_TARGET: ${{ fromJSON(inputs.matrix).target }}
   E2E_PARAM_PARALLELISM: ${{ fromJSON(inputs.matrix).parallelism }}
-  E2E_RUN_NAME: ${{ fromJSON(inputs.matrix).target }}_${{ fromJSON(inputs.matrix).k8sVersion }}_${{ fromJSON(inputs.matrix).cniNetworkPlugin }}_${{ fromJSON(inputs.matrix).arch }}_${{ fromJSON(inputs.matrix).parallelism }}
 jobs:
   e2e:
     timeout-minutes: 60
+<<<<<<< HEAD
     # use the runner from the map, if the runner is circleci or '' then use ubuntu-latest
     runs-on: ${{ contains(fromJSON('["circleci", ""]'), fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch]) && 'ubuntu-latest' || fromJSON(inputs.runnersByArch)[fromJSON(inputs.matrix).arch]}}
+=======
+    # can't use env vars here
+    runs-on: ${{ inputs.runner }}
+    if: ${{ inputs.runner != '' }}
+>>>>>>> 126029d11 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10945))
     strategy:
       fail-fast: false
       matrix:
@@ -80,6 +89,8 @@ jobs:
           EOF
           sudo service docker restart
       - name: "Run E2E tests"
+        env:
+          DOCKERHUB_PULL_CREDENTIAL: ${{ secrets.DOCKERHUB_PULL_CREDENTIAL }}
         run: |
           if [[ "${{ env.E2E_PARAM_K8S_VERSION }}" == "kindIpv6" ]]; then
             export IPV6=true
@@ -108,6 +119,21 @@ jobs:
           if [[ "${{ env.E2E_PARAM_TARGET }}" == "multizone" ]]; then
             export KUMA_DEFAULT_RETRIES=60
           fi
+          
+          function on_exit()
+          {
+            docker logout docker.io
+          }
+          
+          # we pull a few images during the E2E run, sometimes we get rate-limited by docker hub
+          # to prevent this, we support specifying a pull credential here
+          if [[ "$DOCKERHUB_PULL_CREDENTIAL" != "" ]]; then
+            DOCKER_USER=$(echo "$DOCKERHUB_PULL_CREDENTIAL" | cut -d ':' -f 1)
+            DOCKER_PWD=$(echo "$DOCKERHUB_PULL_CREDENTIAL" | cut -d ':' -f 2)
+            echo -n "$DOCKER_PWD" | docker login -u "$DOCKER_USER" --password-stdin
+            trap "on_exit" EXIT
+          fi
+          
           if [[ "${{ env.E2E_PARAM_TARGET }}" != "" ]]; then
             target="test/e2e-${{ env.E2E_PARAM_TARGET }}"
           else

diff --git a/.github/workflows/_test.yaml b/.github/workflows/_test.yaml
@@ -4,6 +4,9 @@ on:
       FULL_MATRIX:
         required: true
         type: string
+      RUNNERS_BY_ARCH:
+        required: true
+        type: string
 permissions:
   contents: read
 env:
@@ -105,6 +108,10 @@ jobs:
     uses: ./.github/workflows/_e2e.yaml
     with:
       matrix: ${{ toJSON(matrix) }}
+<<<<<<< HEAD
+=======
+      runner: ${{ fromJSON(inputs.RUNNERS_BY_ARCH)[matrix.arch] }}
+>>>>>>> 126029d11 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10945))
     secrets: inherit
   test_e2e_env:
     needs: ["gen_e2e_matrix"]
@@ -116,4 +123,8 @@ jobs:
     uses: ./.github/workflows/_e2e.yaml
     with:
       matrix: ${{ toJSON(matrix) }}
+<<<<<<< HEAD
+=======
+      runner: ${{ fromJSON(inputs.RUNNERS_BY_ARCH)[matrix.arch] }}
+>>>>>>> 126029d11 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10945))
     secrets: inherit
diff --git a/.github/workflows/build-test-distribute.yaml b/.github/workflows/build-test-distribute.yaml
@@ -85,6 +85,7 @@ jobs:
     uses: ./.github/workflows/_test.yaml
     with:
       FULL_MATRIX: ${{ needs.check.outputs.FULL_MATRIX }}
+      RUNNERS_BY_ARCH: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) && '{"amd64":"ubuntu-latest-kong","arm64":"ubuntu-latest-arm64-kong"}' || '{"amd64":"ubuntu-latest","arm64":""}' }}
     secrets: inherit
   build_publish:
     permissions:

diff --git a/mk/k3d.mk b/mk/k3d.mk
@@ -41,6 +41,7 @@ KUMA_NAMESPACE ?= kuma-system
 PORT_PREFIX := $$(($(patsubst 300-%,300+%-1,$(KIND_CLUSTER_NAME:kuma%=300%))))
 
 K3D_NETWORK_CNI ?= flannel
+K3D_REGISTRY_FILE ?=
 K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \
 	--k3s-arg '--disable=traefik@server:0' \
 	--k3s-arg '--disable=metrics-server@server:0' \
@@ -49,6 +50,7 @@ K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \
     --volume '$(subst @,\@,$(TOP)/$(KUMA_DIR))/test/framework/deployments:/tmp/deployments@server:0' \
 	--network kind \
 	--port "$(PORT_PREFIX)80-$(PORT_PREFIX)99:30080-30099@server:0" \
+	--registry-config "/tmp/.kuma-dev/k3d-registry.yaml" \
 	--timeout 120s
 
 ifeq ($(K3D_NETWORK_CNI),calico)
@@ -82,6 +84,7 @@ k3d/network/create:
 		else docker network create -d=bridge $(KIND_NETWORK_OPTS) kind || true; fi && \
 		rm -f $(BUILD_DIR)/k3d_network.lock
 
+<<<<<<< HEAD
 $(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml:
 	@mkdir -p $(TOP)/$(KUMA_DIR)/test/k3d
 	curl --location --fail --silent --retry 5 \
@@ -91,6 +94,25 @@ $(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml:
 .PHONY: k3d/start
 k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create \
 	$(if $(findstring calico,$(K3D_NETWORK_CNI)),$(TOP)/$(KUMA_DIR)/test/k3d/calico.$(K3D_VERSION).yaml)
+=======
+DOCKERHUB_PULL_CREDENTIAL ?=
+.PHONY: k3d/setup-docker-credentials
+k3d/setup-docker-credentials:
+	@mkdir -p /tmp/.kuma-dev ; \
+	echo '{"configs": {}}' > /tmp/.kuma-dev/k3d-registry.yaml ; \
+	if [[ "$(DOCKERHUB_PULL_CREDENTIAL)" != "" ]]; then \
+  		DOCKER_USER=$$(echo "$(DOCKERHUB_PULL_CREDENTIAL)" | cut -d ':' -f 1); \
+  		DOCKER_PWD=$$(echo "$(DOCKERHUB_PULL_CREDENTIAL)" | cut -d ':' -f 2); \
+  		echo "{\"configs\": {\"registry-1.docker.io\": {\"auth\": {\"username\": \"$${DOCKER_USER}\",\"password\":\"$${DOCKER_PWD}\"}}}}" > /tmp/.kuma-dev/k3d-registry.yaml ; \
+  	fi
+
+.PHONY: k3d/cleanup-docker-credentials
+k3d/cleanup-docker-credentials:
+	@rm -f /tmp/.kuma-dev/k3d-registry.yaml
+
+.PHONY: k3d/start
+k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create k3d/setup-docker-credentials
+>>>>>>> 126029d11 (ci(.github): enable self hosted runners for AMD64 E2E tasks (#10945))
 	@echo "PORT_PREFIX=$(PORT_PREFIX)"
 	@KUBECONFIG=$(KIND_KUBECONFIG) \
 		$(K3D_BIN) cluster create "$(KIND_CLUSTER_NAME)" $(K3D_CLUSTER_CREATE_OPTS)
@@ -136,7 +158,7 @@ k3d/wait:
     done
 
 .PHONY: k3d/stop
-k3d/stop:
+k3d/stop: k3d/cleanup-docker-credentials
 	@KUBECONFIG=$(KIND_KUBECONFIG) $(K3D_BIN) cluster delete "$(KIND_CLUSTER_NAME)"
 
 .PHONY: k3d/stop/all

diff --git a/mk/kind.mk b/mk/kind.mk
@@ -21,8 +21,21 @@ CI_KUBERNETES_VERSION ?= v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb
 KUMA_MODE ?= zone
 KUMA_NAMESPACE ?= kuma-system
 
+DOCKERHUB_PULL_CREDENTIAL ?=
+.PHONY: kind/setup-docker-credentials
+kind/setup-docker-credentials:
+	@mkdir -p /tmp/.kuma-dev ; \
+	echo '{"auths":{}}' > /tmp/.kuma-dev/kind-config.json ; \
+	if [[ "$(DOCKERHUB_PULL_CREDENTIAL)" != "" ]]; then \
+		echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$$(echo -n "$(DOCKERHUB_PULL_CREDENTIAL)" | base64)\"}}}" > /tmp/.kuma-dev/kind-config.json ; \
+	fi
+
+.PHONY: kind/cleanup-docker-credentials
+kind/cleanup-docker-credentials:
+	@rm -f /tmp/.kuma-dev/kind-config.json
+
 .PHONY: kind/start
-kind/start: ${KUBECONFIG_DIR}
+kind/start: ${KUBECONFIG_DIR} kind/setup-docker-credentials
 	$(KIND) get clusters | grep $(KIND_CLUSTER_NAME) >/dev/null 2>&1 && echo "Kind cluster already running." && exit 0 || \
 		($(KIND) create cluster \
 			--name "$(KIND_CLUSTER_NAME)" \
@@ -51,7 +64,7 @@ kind/wait:
     done
 
 .PHONY: kind/stop
-kind/stop:
+kind/stop: kind/cleanup-docker-credentials
 	@$(KIND) delete cluster --name $(KIND_CLUSTER_NAME)
 	@rm -f $(KUBECONFIG_DIR)/$(KIND_KUBECONFIG)
 

diff --git a/test/kind/cluster-ipv6-kuma-1.yaml b/test/kind/cluster-ipv6-kuma-1.yaml
@@ -5,17 +5,20 @@ networking:
   ipFamily: ipv6
   apiServerAddress: 127.0.0.1
 nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: InitConfiguration
-    nodeRegistration:
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  extraPortMappings:
-  - containerPort: 30080
-    hostPort: 30080
-    protocol: TCP
-  - containerPort: 30443
-    hostPort: 30443
-    protocol: TCP
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 30080
+        hostPort: 30080
+        protocol: TCP
+      - containerPort: 30443
+        hostPort: 30443
+        protocol: TCP
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-ipv6-kuma-2.yaml b/test/kind/cluster-ipv6-kuma-2.yaml
@@ -4,3 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   ipFamily: ipv6
   apiServerAddress: 127.0.0.1
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-ipv6-kuma.yaml b/test/kind/cluster-ipv6-kuma.yaml
@@ -4,3 +4,8 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   ipFamily: ipv6
   apiServerAddress: 127.0.0.1
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-kuma-1.yaml b/test/kind/cluster-kuma-1.yaml
@@ -1,18 +1,20 @@
-# an ipv6 cluster
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
-- role: control-plane
-  kubeadmConfigPatches:
-  - |
-    kind: InitConfiguration
-    nodeRegistration:
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  extraPortMappings:
-  - containerPort: 30080
-    hostPort: 30080
-    protocol: TCP
-  - containerPort: 30443
-    hostPort: 30443
-    protocol: TCP
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 30080
+        hostPort: 30080
+        protocol: TCP
+      - containerPort: 30443
+        hostPort: 30443
+        protocol: TCP
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-kuma-2.yaml b/test/kind/cluster-kuma-2.yaml
@@ -1,3 +1,8 @@
 # an ipv6 cluster
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json
diff --git a/test/kind/cluster-kuma.yaml b/test/kind/cluster-kuma.yaml
@@ -1,3 +1,8 @@
 # an ipv6 cluster
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - containerPath: /var/lib/kubelet/config.json
+        hostPath: /tmp/.kuma-dev/kind-config.json