From d435763d4f18a29b4838a24ccb0f544cd9c3e842 Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Wed, 10 Oct 2018 22:22:42 -0700 Subject: [PATCH] Build Istio's own kubectl container (#8934) * Build Istio's own kubectl container Hyperkube is really heavyweight at 900mb. Instead build a kubectl container that is about 180mb. I first prototyped a from scratch container here: https://github.com/sdake/scratch-kubectl This contianer is *very* lightweight at about 44mb (12mb compressed). The workflow for creating the continer is a little more difficult though and would benefit from the latest versions of docker which are not yet in the Xenial repositories for all to use. Also there is some concern with regards to maintenance with a from SCRATCH approach. * Address review comments This variant builds the dockerfile, however, the build operation takes roughy 4 seconds on my system if the file has already been built. Bootstrapping this particular container will prove troublesome for most individuals if it were to follow the model of istionightly. A big chunk of time is wasted during the build process transferring docker build context that doesn't need to be transferred. I'm going to take a look at the build a bit and see if there is a way to seperate the built data into more reasonable build contexts. This would significantly improve buildtimes, especially on virtual machines. Sadly kubectl has ballooned to 320MB, however, nearly all of the layers are shared with exception of the final layer that contains the kubectl binary itself. * Add rule for $(GOTOP_BIN) files * Work around problem where testEnvLocalK8S symlinks existing tools --- bin/testEnvLocalK8S.sh | 9 ++++++++- docker/Dockerfile.kubectl | 5 +++++ .../charts/security/templates/cleanup-secrets.yaml | 4 ++-- .../templates/create-custom-resources-job.yaml | 4 ++-- .../templates/install-custom-resources.sh.tpl | 6 +++--- install/kubernetes/helm/istio-remote/values.yaml | 5 ----- .../templates/create-custom-resources-job.yaml | 4 ++-- .../charts/security/templates/cleanup-secrets.yaml | 4 ++-- .../templates/create-custom-resources-job.yaml | 4 ++-- .../istio/templates/install-custom-resources.sh.tpl | 8 ++++---- install/kubernetes/helm/istio/values.yaml | 5 ----- install/updateVersion.sh | 11 ----------- istioctl/cmd/istioctl/gendeployment/cmd.go | 12 +----------- tools/istio-docker.mk | 10 ++++++---- 14 files changed, 37 insertions(+), 54 deletions(-) create mode 100644 docker/Dockerfile.kubectl diff --git a/bin/testEnvLocalK8S.sh b/bin/testEnvLocalK8S.sh index 057167fcc46a..7cc3b71287d8 100755 --- a/bin/testEnvLocalK8S.sh +++ b/bin/testEnvLocalK8S.sh @@ -121,6 +121,12 @@ function getDeps() { fi } +function getLatestDeps() { + curl -Lo "${GO_TOP}/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/${GOOS_LOCAL}/amd64/kubectl" && chmod +x "$GO_TOP/bin/kubectl" + curl -Lo "${GO_TOP}/bin/kube-apiserver" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/${GOOS_LOCAL}/amd64/kube-apiserver" && chmod +x "${GO_TOP}/bin/kube-apiserver" + curl -L "https://github.com/coreos/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz" | tar xz -O "etcd-${ETCD_VER}-linux-amd64/etcd" > "${GO_TOP}/bin/etcd" && chmod +x "${GO_TOP}/bin/etcd" +} + # No root required, run local etcd and kube apiserver for tests. function startLocalApiserver() { ensureK8SCerts @@ -325,10 +331,11 @@ set +xe case "$1" in start) startLocalApiserver ;; stop) stopLocalApiserver ;; + ensure) ensureLocalApiServer ;; startIstio) startIstio ;; stopIstio) stopIstio ;; startMultiCluster) startMultiCluster ;; stopMultiCluster) stopMultiCluster ;; - ensure) ensureLocalApiServer ;; + getDeps) getLatestDeps ;; *) echo "start stop ensure" esac diff --git a/docker/Dockerfile.kubectl b/docker/Dockerfile.kubectl new file mode 100644 index 000000000000..9aca6fa7e975 --- /dev/null +++ b/docker/Dockerfile.kubectl @@ -0,0 +1,5 @@ +FROM istionightly/base_debug +# Image for post install jobs + +# This container should only contain kubectl +ADD kubectl /usr/bin diff --git a/install/kubernetes/helm/istio-remote/charts/security/templates/cleanup-secrets.yaml b/install/kubernetes/helm/istio-remote/charts/security/templates/cleanup-secrets.yaml index ae93b9fc1f5b..8dbe23ca7b90 100644 --- a/install/kubernetes/helm/istio-remote/charts/security/templates/cleanup-secrets.yaml +++ b/install/kubernetes/helm/istio-remote/charts/security/templates/cleanup-secrets.yaml @@ -86,8 +86,8 @@ spec: spec: serviceAccountName: istio-cleanup-secrets-service-account containers: - - name: hyperkube - image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}" + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" command: - /bin/bash - -c diff --git a/install/kubernetes/helm/istio-remote/charts/security/templates/create-custom-resources-job.yaml b/install/kubernetes/helm/istio-remote/charts/security/templates/create-custom-resources-job.yaml index fb547664ea71..1e1a5fac0dfb 100644 --- a/install/kubernetes/helm/istio-remote/charts/security/templates/create-custom-resources-job.yaml +++ b/install/kubernetes/helm/istio-remote/charts/security/templates/create-custom-resources-job.yaml @@ -74,8 +74,8 @@ spec: spec: serviceAccountName: istio-security-post-install-account containers: - - name: hyperkube - image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}" + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" command: [ "/bin/bash", "/tmp/security/run.sh", "/tmp/security/custom-resources.yaml" ] volumeMounts: - mountPath: "/tmp/security" diff --git a/install/kubernetes/helm/istio-remote/templates/install-custom-resources.sh.tpl b/install/kubernetes/helm/istio-remote/templates/install-custom-resources.sh.tpl index a26fc13c296f..2bd9f8d057b9 100644 --- a/install/kubernetes/helm/istio-remote/templates/install-custom-resources.sh.tpl +++ b/install/kubernetes/helm/istio-remote/templates/install-custom-resources.sh.tpl @@ -9,7 +9,7 @@ fi pathToResourceYAML=${1} -/kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null +kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null if [ "$?" -eq 0 ]; then echo "istio-galley validatingwebhookconfiguration found - waiting for istio-galley deployment to be ready" while true; do @@ -19,9 +19,9 @@ if [ "$?" -eq 0 ]; then fi sleep 1 done - /kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley + kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley echo "istio-galley deployment ready for configuration validation" fi sleep 5 -/kubectl apply -f ${pathToResourceYAML} +kubectl apply -f ${pathToResourceYAML} {{ end }} diff --git a/install/kubernetes/helm/istio-remote/values.yaml b/install/kubernetes/helm/istio-remote/values.yaml index 44e07aff79a8..1ec18a983d89 100644 --- a/install/kubernetes/helm/istio-remote/values.yaml +++ b/install/kubernetes/helm/istio-remote/values.yaml @@ -78,11 +78,6 @@ global: # TODO: Switch to Always as default, and override in the local tests. imagePullPolicy: IfNotPresent - # Not recommended for user to configure this. Hyperkube image to use when creating custom resources - hyperkube: - hub: quay.io/coreos - tag: v1.7.6_coreos.0 - # controlPlaneMtls enabled. Will result in delays starting the pods while secrets are # propagated, not recommended for tests. controlPlaneSecurityEnabled: false diff --git a/install/kubernetes/helm/istio/charts/grafana/templates/create-custom-resources-job.yaml b/install/kubernetes/helm/istio/charts/grafana/templates/create-custom-resources-job.yaml index 6d8b93de5689..2c4ad51622d7 100644 --- a/install/kubernetes/helm/istio/charts/grafana/templates/create-custom-resources-job.yaml +++ b/install/kubernetes/helm/istio/charts/grafana/templates/create-custom-resources-job.yaml @@ -64,8 +64,8 @@ spec: spec: serviceAccountName: istio-grafana-post-install-account containers: - - name: hyperkube - image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}" + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" command: [ "/bin/bash", "/tmp/grafana/run.sh", "/tmp/grafana/custom-resources.yaml" ] volumeMounts: - mountPath: "/tmp/grafana" diff --git a/install/kubernetes/helm/istio/charts/security/templates/cleanup-secrets.yaml b/install/kubernetes/helm/istio/charts/security/templates/cleanup-secrets.yaml index ae93b9fc1f5b..8dbe23ca7b90 100644 --- a/install/kubernetes/helm/istio/charts/security/templates/cleanup-secrets.yaml +++ b/install/kubernetes/helm/istio/charts/security/templates/cleanup-secrets.yaml @@ -86,8 +86,8 @@ spec: spec: serviceAccountName: istio-cleanup-secrets-service-account containers: - - name: hyperkube - image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}" + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" command: - /bin/bash - -c diff --git a/install/kubernetes/helm/istio/charts/security/templates/create-custom-resources-job.yaml b/install/kubernetes/helm/istio/charts/security/templates/create-custom-resources-job.yaml index d4e581fe6eca..d68ac92cf634 100644 --- a/install/kubernetes/helm/istio/charts/security/templates/create-custom-resources-job.yaml +++ b/install/kubernetes/helm/istio/charts/security/templates/create-custom-resources-job.yaml @@ -74,8 +74,8 @@ spec: spec: serviceAccountName: istio-security-post-install-account containers: - - name: hyperkube - image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}" + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" command: [ "/bin/bash", "/tmp/security/run.sh", "/tmp/security/custom-resources.yaml" ] volumeMounts: - mountPath: "/tmp/security" diff --git a/install/kubernetes/helm/istio/templates/install-custom-resources.sh.tpl b/install/kubernetes/helm/istio/templates/install-custom-resources.sh.tpl index 6123902c399a..a5525a139145 100644 --- a/install/kubernetes/helm/istio/templates/install-custom-resources.sh.tpl +++ b/install/kubernetes/helm/istio/templates/install-custom-resources.sh.tpl @@ -10,17 +10,17 @@ fi pathToResourceYAML=${1} -/kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null +kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null if [ "$?" -eq 0 ]; then echo "istio-galley validatingwebhookconfiguration found - waiting for istio-galley deployment to be ready" while true; do - /kubectl -n {{ .Release.Namespace }} get deployment istio-galley 2>/dev/null + kubectl -n {{ .Release.Namespace }} get deployment istio-galley 2>/dev/null if [ "$?" -eq 0 ]; then break fi sleep 1 done - /kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley + kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley if [ "$?" -ne 0 ]; then echo "istio-galley deployment rollout status check failed" exit 1 @@ -28,5 +28,5 @@ if [ "$?" -eq 0 ]; then echo "istio-galley deployment ready for configuration validation" fi sleep 5 -/kubectl apply -f ${pathToResourceYAML} +kubectl apply -f ${pathToResourceYAML} {{ end }} diff --git a/install/kubernetes/helm/istio/values.yaml b/install/kubernetes/helm/istio/values.yaml index 910f10e07574..0e0a99f17481 100644 --- a/install/kubernetes/helm/istio/values.yaml +++ b/install/kubernetes/helm/istio/values.yaml @@ -272,11 +272,6 @@ global: # cpu: 100m # memory: 128Mi - # Not recommended for user to configure this. Hyperkube image to use when creating custom resources - hyperkube: - hub: quay.io/coreos - tag: v1.7.6_coreos.0 - # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and # system-node-critical, it is better to configure this in order to make sure your Istio pods # will not be killed because of low priority class. diff --git a/install/updateVersion.sh b/install/updateVersion.sh index 5b3ef0cd4cd2..fc2e9c7579d6 100755 --- a/install/updateVersion.sh +++ b/install/updateVersion.sh @@ -29,8 +29,6 @@ DEST_DIR=$ROOT ISTIO_NAMESPACE="istio-system" FORTIO_HUB="docker.io/fortio" FORTIO_TAG="latest_release" -HYPERKUBE_HUB="quay.io/coreos/hyperkube" -HYPERKUBE_TAG="v1.7.6_coreos.0" while getopts :n:p:x:c:a:h:o:P:d:D: arg; do case ${arg} in @@ -39,7 +37,6 @@ while getopts :n:p:x:c:a:h:o:P:d:D: arg; do x) MIXER_HUB_TAG="${OPTARG}";; # Format: "," c) CITADEL_HUB_TAG="${OPTARG}";; # Format: "," a) ALL_HUB_TAG="${OPTARG}";; # Format: "," - h) HYPERKUBE_HUB_TAG="${OPTARG}";; # Format: "," o) PROXY_HUB_TAG="${OPTARG}";; # Format: "," P) PILOT_DEBIAN_URL="${OPTARG}";; d) DEST_DIR="${OPTARG}";; @@ -79,11 +76,6 @@ if [[ -n ${CITADEL_HUB_TAG} ]]; then CITADEL_TAG="$(echo "${CITADEL_HUB_TAG}"|cut -f2 -d,)" fi -if [[ -n ${HYPERKUBE_HUB_TAG} ]]; then - HYPERKUBE_HUB="$(echo "${HYPERKUBE_HUB_TAG}"|cut -f1 -d,)" - HYPERKUBE_TAG="$(echo "${HYPERKUBE_HUB_TAG}"|cut -f2 -d,)" -fi - function usage() { cat <, for the mixer docker image -c ... , for the citadel docker image -a ... , Specifies same hub and tag for pilot, mixer, proxy, and citadel containers - -h ... , for the hyperkube docker image -o ... , for the proxy docker image -n ... namespace in which to install Istio control plane components -A ... URL to download auth debian packages @@ -139,8 +130,6 @@ export ISTIO_NAMESPACE="${ISTIO_NAMESPACE}" export PILOT_DEBIAN_URL="${PILOT_DEBIAN_URL}" export FORTIO_HUB="${FORTIO_HUB}" export FORTIO_TAG="${FORTIO_TAG}" -export HYPERKUBE_HUB="${HYPERKUBE_HUB}" -export HYPERKUBE_TAG="${HYPERKUBE_TAG}" EOF } diff --git a/istioctl/cmd/istioctl/gendeployment/cmd.go b/istioctl/cmd/istioctl/gendeployment/cmd.go index 35604c9c6f2b..0cff8fd691e9 100644 --- a/istioctl/cmd/istioctl/gendeployment/cmd.go +++ b/istioctl/cmd/istioctl/gendeployment/cmd.go @@ -24,8 +24,7 @@ import ( ) const ( - defaultTag = "master-latest-daily" - defaultHyperkubeTag = "v1.7.6_coreos.0" + defaultTag = "master-latest-daily" ) // Command returns the "gen-deploy" subcommand for istioctl. @@ -75,9 +74,6 @@ func Command(istioNamespaceFlag *string) *cobra.Command { cmd.PersistentFlags().StringVar(&helmChartLocation, "helm-chart-dir", ".", "The directory to find the helm charts used to render Istio deployments. -o yaml uses these to render the helm chart locally.") - cmd.PersistentFlags().StringVar(&install.HyperkubeHub, "hyperkube-hub", install.HyperkubeHub, "The container registry to pull Hyperkube images from") - cmd.PersistentFlags().StringVar(&install.HyperkubeTag, "hyperkube-tag", install.HyperkubeTag, "The tag to use to pull the `Hyperkube` container") - _ = cmd.PersistentFlags().MarkHidden("hub") _ = cmd.PersistentFlags().MarkHidden("mixer-tag") _ = cmd.PersistentFlags().MarkHidden("pilot-tag") @@ -108,9 +104,6 @@ type installation struct { CaTag string ProxyTag string - HyperkubeHub string - HyperkubeTag string - NodePort uint16 Debug bool @@ -138,9 +131,6 @@ func defaultInstall() *installation { PilotTag: defaultTag, CaTag: defaultTag, ProxyTag: defaultTag, - - HyperkubeHub: "quay.io/coreos/hyperkube", - HyperkubeTag: defaultHyperkubeTag, } } diff --git a/tools/istio-docker.mk b/tools/istio-docker.mk index ad2fb384dbe4..21e4c885f3dd 100644 --- a/tools/istio-docker.mk +++ b/tools/istio-docker.mk @@ -22,7 +22,7 @@ docker: build test-bins docker.all DOCKER_TARGETS:=docker.pilot docker.proxy_debug docker.proxytproxy docker.proxyv2 docker.app docker.test_policybackend \ - docker.proxy_init docker.mixer docker.citadel docker.galley docker.sidecar_injector + docker.proxy_init docker.mixer docker.citadel docker.galley docker.sidecar_injector docker.kubectl $(ISTIO_DOCKER) $(ISTIO_DOCKER_TAR): mkdir -p $@ @@ -70,7 +70,9 @@ $(foreach FILE,$(DOCKER_FILES_FROM_SOURCE), \ # cp $(ISTIO_BIN)/kubectl $(ISTIO_DOCKER)/kubectl DOCKER_FILES_FROM_ISTIO_BIN:=kubectl $(foreach FILE,$(DOCKER_FILES_FROM_ISTIO_BIN), \ - $(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_BIN)/$(FILE) | $(ISTIO_DOCKER); bin/testEnvLocalK8S.sh getDeps; cp $(ISTIO_BIN)/$(FILE) $(ISTIO_DOCKER)/$(FILE))) + $(eval $(ISTIO_BIN)/$(FILE): ; bin/testEnvLocalK8S.sh getDeps)) +$(foreach FILE,$(DOCKER_FILES_FROM_ISTIO_BIN), \ + $(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_BIN)/$(FILE) | $(ISTIO_DOCKER); cp $(ISTIO_BIN)/$(FILE) $(ISTIO_DOCKER)/$(FILE))) # pilot docker images @@ -153,8 +155,8 @@ docker.test_policybackend: mixer/docker/Dockerfile.test_policybackend docker.test_policybackend: $(ISTIO_OUT)/mixer-test-policybackend $(DOCKER_RULE) -#docker.kubectl: docker/Dockerfile$$(suffix $$@) $(ISTIO_BIN)/kubectl -# $(DOCKER_RULE) +docker.kubectl: docker/Dockerfile$$(suffix $$@) $(ISTIO_BIN)/kubectl + $(DOCKER_RULE) # addons docker images