test: Add option to start runtime Cilium from docker

Add PROVISION_EXTERNAL_WORKLOAD environment variable that if not
"false" will cause runtime VM compile and run Cilium in docker with
configuration suitable for external workload to join a Cilium cluster.
This is using the new 'test/provision/externalworkload_install.sh',
which builds and starts a Cilium docker image for a VM.

TLS certs for the VM must be extracted from the k8s cluster before
starting the VM.

Signed-off-by: Jarno Rajahalme <[email protected]>

.gitignore

@@ -68,6 +68,9 @@ test/cilium-istioctl
 # generated test files
 test/k8sT/manifests/cnp-second-namespaces.yaml
 test/cilium.conf.ginkgo
+test/provision/externalworkload-client-ca.crt
+test/provision/externalworkload-client-tls.crt
+test/provision/externalworkload-client-tls.key
 
 # GKE temporary files
 test/gke/cluster-name

test/Vagrantfile

@@ -26,6 +26,7 @@ $CILIUM_OPERATOR_TAG = ENV['CILIUM_OPERATOR_TAG'] || ""
 $HUBBLE_RELAY_IMAGE = ENV['HUBBLE_RELAY_IMAGE'] || ""
 $HUBBLE_RELAY_TAG = ENV['HUBBLE_RELAY_TAG'] || ""
 $PRELOAD_VM = ENV['PRELOAD_VM'] || "false"
+$PROVISION_EXTERNAL_WORKLOAD = ENV['PROVISION_EXTERNAL_WORKLOAD'] || "false"
 $SKIP_K8S_PROVISION = ENV['SKIP_K8S_PROVISION'] || "false"
 $NO_CILIUM_ON_NODE = ENV['NO_CILIUM_ON_NODE'] || ""
 $KUBEPROXY = (ENV['KUBEPROXY'] || "1")
@@ -121,7 +122,8 @@ Vagrant.configure("2") do |config|
             sh.env = {
               "RACE" => "#{$RACE}",
               "LOCKDEBUG" => "#{$LOCKDEBUG}",
-              "BASE_IMAGE" => "#{$BASE_IMAGE}"
+              "BASE_IMAGE" => "#{$BASE_IMAGE}",
+              "PROVISION_EXTERNAL_WORKLOAD" => "#{$PROVISION_EXTERNAL_WORKLOAD}"
             }
         end
     end

test/provision/externalworkload_extract_k8s_certs.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+echo "Extracting VM TLS config from Kubernetes secrets"
+kubectl -n kube-system get secret clustermesh-apiserver-ca-cert -o jsonpath="{.data['ca\.crt']}" | base64 --decode >${DIR}/externalworkload-client-ca.crt
+kubectl -n kube-system get secret clustermesh-apiserver-client-cert -o jsonpath="{.data['tls\.crt']}" | base64 --decode >${DIR}/externalworkload-client-tls.crt
+kubectl -n kube-system get secret clustermesh-apiserver-client-cert -o jsonpath="{.data['tls\.key']}" | base64 --decode >${DIR}/externalworkload-client-tls.key

test/provision/externalworkload_install.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e
+set -x
+
+# IP address at which the clustermesh-apiserver service is reachable at
+# Default to the address of k8s1
+CLUSTER_IP=${CLUSTER_IP:-"192.168.36.11"}
+
+# IP address of the VM itself. This is needed due to avoid Cilium selecting the "10.0.2.15"
+# each vagrant VM has as the first address. Default to the address of the runtime VM
+VM_IP=${VM_IP:-"192.168.36.10"}
+
+PROVISIONSRC="/tmp/provision"
+
+CA_CRT="${PROVISIONSRC}/externalworkload-client-ca.crt"
+TLS_CRT="${PROVISIONSRC}/externalworkload-client-tls.crt"
+TLS_KEY="${PROVISIONSRC}/externalworkload-client-tls.key"
+
+if [[ ! ( -f $CA_CRT && -f $TLS_CRT && -f $TLS_KEY ) ]]; then
+    echo "Client certificates not found. These can be extracted from your Kubernetes"
+    echo "cluster with 'test/provision/externalworkload_extract_k8s_certs.sh'"
+    exit 1
+fi
+
+sudo mkdir -p /var/lib/cilium/etcd
+sudo cp $CA_CRT /var/lib/cilium/etcd/ca.crt
+sudo cp $TLS_CRT /var/lib/cilium/etcd/tls.crt
+sudo cp $TLS_KEY /var/lib/cilium/etcd/tls.key
+sudo tee /var/lib/cilium/etcd/config.yaml <<EOF
+---
+trusted-ca-file: /var/lib/cilium/etcd/ca.crt
+cert-file: /var/lib/cilium/etcd/tls.crt
+key-file: /var/lib/cilium/etcd/tls.key
+endpoints:
+- https://clustermesh-apiserver.cilium.io:32379
+EOF
+
+CILIUM_OPTS=" --debug --ipv4-node $VM_IP"
+CILIUM_OPTS+=" --join-cluster"
+CILIUM_OPTS+=" --kvstore etcd --kvstore-opt etcd.config=/var/lib/cilium/etcd/config.yaml"
+
+# Build docker image
+DOCKER_BUILDKIT=1 make -C /home/vagrant/go/src/github.com/cilium/cilium dev-docker-image
+
+# Etcd TLS config needs hostname IP mapping
+CLUSTER_HOST="clustermesh-apiserver.cilium.io:$CLUSTER_IP"
+
+DOCKER_OPTS=" -d --log-driver syslog --restart always"
+DOCKER_OPTS+=" --privileged --network host --cap-add NET_ADMIN --cap-add SYS_MODULE"
+DOCKER_OPTS+=" --volume /var/lib/cilium/etcd:/var/lib/cilium/etcd"
+DOCKER_OPTS+=" --volume /var/run/cilium:/var/run/cilium"
+DOCKER_OPTS+=" --volume /boot:/boot"
+DOCKER_OPTS+=" --volume /lib/modules:/lib/modules"
+DOCKER_OPTS+=" --volume /sys/fs/bpf:/sys/fs/bpf"
+DOCKER_OPTS+=" --volume /run/xtables.lock:/run/xtables.lock"
+DOCKER_OPTS+=" --add-host $CLUSTER_HOST"
+
+sudo docker run --name cilium $DOCKER_OPTS cilium/cilium-dev:latest cilium-agent $CILIUM_OPTS
+
+# Copy Cilium CLI
+sudo docker cp cilium:/usr/bin/cilium /usr/bin/cilium

test/provision/runtime_install.sh

@@ -12,5 +12,11 @@ sudo bash -c "echo MaxSessions 200 >> /etc/ssh/sshd_config"
 sudo systemctl restart ssh
 
 "${PROVISIONSRC}"/dns.sh
-"${PROVISIONSRC}"/compile.sh
+
+if [[ "${PROVISION_EXTERNAL_WORKLOAD}" == "false" ]]; then
+    "${PROVISIONSRC}"/compile.sh
+else
+    "${PROVISIONSRC}"/externalworkload_install.sh
+fi
+
 "${PROVISIONSRC}"/wait-cilium.sh