Build Istio's own kubectl container (istio#8934)

* Build Istio's own kubectl container

Hyperkube is really heavyweight at 900mb.  Instead build a kubectl
container that is about 180mb.

I first prototyped a from scratch container here:
https://github.com/sdake/scratch-kubectl

This contianer is *very* lightweight at about 44mb (12mb compressed).
The workflow for creating the continer is a little more difficult though
and would benefit from the latest versions of docker which are not
yet in the Xenial repositories for all to use.  Also there is some
concern with regards to maintenance with a from SCRATCH approach.

* Address review comments

This variant builds the dockerfile, however, the build operation
takes roughy 4 seconds on my system if the file has already been
built.  Bootstrapping this particular container will prove troublesome
for most individuals if it were to follow the model of istionightly.

A big chunk of time is wasted during the build process transferring
docker build context that doesn't need to be transferred.  I'm going to
take a look at the build a bit and see if there is a way to seperate
the built data into more reasonable build contexts.  This would
significantly improve buildtimes, especially on virtual machines.

Sadly kubectl has ballooned to 320MB, however, nearly all of the layers
are shared with exception of the final layer that contains the kubectl
binary itself.

* Add rule for $(GOTOP_BIN) files

* Work around problem where testEnvLocalK8S symlinks existing tools

bin/testEnvLocalK8S.sh

@@ -121,6 +121,12 @@ function getDeps() {
    fi
 }
 
+function getLatestDeps() {
+    curl -Lo "${GO_TOP}/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/${GOOS_LOCAL}/amd64/kubectl" && chmod +x "$GO_TOP/bin/kubectl"
+    curl -Lo "${GO_TOP}/bin/kube-apiserver" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VER}/bin/${GOOS_LOCAL}/amd64/kube-apiserver" && chmod +x "${GO_TOP}/bin/kube-apiserver"
+    curl -L "https://github.com/coreos/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz" | tar xz -O "etcd-${ETCD_VER}-linux-amd64/etcd" > "${GO_TOP}/bin/etcd" && chmod +x "${GO_TOP}/bin/etcd"
+}
+
 # No root required, run local etcd and kube apiserver for tests.
 function startLocalApiserver() {
     ensureK8SCerts
@@ -325,10 +331,11 @@ set +xe
 case "$1" in
     start) startLocalApiserver ;;
     stop) stopLocalApiserver ;;
+    ensure) ensureLocalApiServer ;;
     startIstio) startIstio ;;
     stopIstio) stopIstio ;;
     startMultiCluster) startMultiCluster ;;
     stopMultiCluster) stopMultiCluster ;;
-    ensure) ensureLocalApiServer ;;
+    getDeps) getLatestDeps ;;
     *) echo "start stop ensure"
 esac

docker/Dockerfile.kubectl

@@ -0,0 +1,5 @@
+FROM istionightly/base_debug
+# Image for post install jobs
+
+# This container should only contain kubectl
+ADD kubectl /usr/bin

install/kubernetes/helm/istio-remote/charts/security/templates/cleanup-secrets.yaml

@@ -86,8 +86,8 @@ spec:
     spec:
       serviceAccountName: istio-cleanup-secrets-service-account
       containers:
-        - name: hyperkube
-          image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}"
+        - name: kubectl
+          image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}"
           command:
           - /bin/bash
           - -c

install/kubernetes/helm/istio-remote/charts/security/templates/create-custom-resources-job.yaml

@@ -74,8 +74,8 @@ spec:
     spec:
       serviceAccountName: istio-security-post-install-account
       containers:
-        - name: hyperkube
-          image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}"
+        - name: kubectl
+          image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}"
           command: [ "/bin/bash", "/tmp/security/run.sh", "/tmp/security/custom-resources.yaml" ]
           volumeMounts:
             - mountPath: "/tmp/security"

install/kubernetes/helm/istio-remote/templates/install-custom-resources.sh.tpl

@@ -9,7 +9,7 @@ fi
 
 pathToResourceYAML=${1}
 
-/kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null
+kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null
 if [ "$?" -eq 0 ]; then
     echo "istio-galley validatingwebhookconfiguration found - waiting for istio-galley deployment to be ready"
     while true; do
@@ -19,9 +19,9 @@ if [ "$?" -eq 0 ]; then
         fi
         sleep 1
     done
-    /kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley
+    kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley
     echo "istio-galley deployment ready for configuration validation"
 fi
 sleep 5
-/kubectl apply -f ${pathToResourceYAML}
+kubectl apply -f ${pathToResourceYAML}
 {{ end }}

install/kubernetes/helm/istio-remote/values.yaml

@@ -78,11 +78,6 @@ global:
   # TODO: Switch to Always as default, and override in the local tests.
   imagePullPolicy: IfNotPresent
 
-  # Not recommended for user to configure this. Hyperkube image to use when creating custom resources
-  hyperkube:
-    hub: quay.io/coreos
-    tag: v1.7.6_coreos.0
-
   # controlPlaneMtls enabled. Will result in delays starting the pods while secrets are
   # propagated, not recommended for tests.
   controlPlaneSecurityEnabled: false

install/kubernetes/helm/istio/charts/grafana/templates/create-custom-resources-job.yaml

@@ -64,8 +64,8 @@ spec:
     spec:
       serviceAccountName: istio-grafana-post-install-account
       containers:
-        - name: hyperkube
-          image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}"
+        - name: kubectl
+          image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}"
           command: [ "/bin/bash", "/tmp/grafana/run.sh", "/tmp/grafana/custom-resources.yaml" ]
           volumeMounts:
             - mountPath: "/tmp/grafana"

install/kubernetes/helm/istio/charts/security/templates/cleanup-secrets.yaml

@@ -86,8 +86,8 @@ spec:
     spec:
       serviceAccountName: istio-cleanup-secrets-service-account
       containers:
-        - name: hyperkube
-          image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}"
+        - name: kubectl
+          image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}"
           command:
           - /bin/bash
           - -c

install/kubernetes/helm/istio/charts/security/templates/create-custom-resources-job.yaml

@@ -74,8 +74,8 @@ spec:
     spec:
       serviceAccountName: istio-security-post-install-account
       containers:
-        - name: hyperkube
-          image: "{{ .Values.global.hyperkube.hub }}/hyperkube:{{ .Values.global.hyperkube.tag }}"
+        - name: kubectl
+          image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}"
           command: [ "/bin/bash", "/tmp/security/run.sh", "/tmp/security/custom-resources.yaml" ]
           volumeMounts:
             - mountPath: "/tmp/security"

install/kubernetes/helm/istio/templates/install-custom-resources.sh.tpl

@@ -10,23 +10,23 @@ fi
 
 pathToResourceYAML=${1}
 
-/kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null
+kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null
 if [ "$?" -eq 0 ]; then
     echo "istio-galley validatingwebhookconfiguration found - waiting for istio-galley deployment to be ready"
     while true; do
-        /kubectl -n {{ .Release.Namespace }} get deployment istio-galley 2>/dev/null
+        kubectl -n {{ .Release.Namespace }} get deployment istio-galley 2>/dev/null
         if [ "$?" -eq 0 ]; then
             break
         fi
         sleep 1
     done
-    /kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley
+    kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley
     if [ "$?" -ne 0 ]; then
         echo "istio-galley deployment rollout status check failed"
         exit 1
     fi
     echo "istio-galley deployment ready for configuration validation"
 fi
 sleep 5
-/kubectl apply -f ${pathToResourceYAML}
+kubectl apply -f ${pathToResourceYAML}
 {{ end }}

install/kubernetes/helm/istio/values.yaml

@@ -272,11 +272,6 @@ global:
     #   cpu: 100m
     #   memory: 128Mi
 
-  # Not recommended for user to configure this. Hyperkube image to use when creating custom resources
-  hyperkube:
-    hub: quay.io/coreos
-    tag: v1.7.6_coreos.0
-
   # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and
   # system-node-critical, it is better to configure this in order to make sure your Istio pods
   # will not be killed because of low priority class.

install/updateVersion.sh

@@ -29,8 +29,6 @@ DEST_DIR=$ROOT
 ISTIO_NAMESPACE="istio-system"
 FORTIO_HUB="docker.io/fortio"
 FORTIO_TAG="latest_release"
-HYPERKUBE_HUB="quay.io/coreos/hyperkube"
-HYPERKUBE_TAG="v1.7.6_coreos.0"
 
 while getopts :n:p:x:c:a:h:o:P:d:D: arg; do
   case ${arg} in
@@ -39,7 +37,6 @@ while getopts :n:p:x:c:a:h:o:P:d:D: arg; do
     x) MIXER_HUB_TAG="${OPTARG}";;     # Format: "<hub>,<tag>"
     c) CITADEL_HUB_TAG="${OPTARG}";;   # Format: "<hub>,<tag>"
     a) ALL_HUB_TAG="${OPTARG}";;       # Format: "<hub>,<tag>"
-    h) HYPERKUBE_HUB_TAG="${OPTARG}";; # Format: "<hub>,<tag>"
     o) PROXY_HUB_TAG="${OPTARG}";;     # Format: "<hub>,<tag>"
     P) PILOT_DEBIAN_URL="${OPTARG}";;
     d) DEST_DIR="${OPTARG}";;
@@ -79,11 +76,6 @@ if [[ -n ${CITADEL_HUB_TAG} ]]; then
     CITADEL_TAG="$(echo "${CITADEL_HUB_TAG}"|cut -f2 -d,)"
 fi
 
-if [[ -n ${HYPERKUBE_HUB_TAG} ]]; then
-    HYPERKUBE_HUB="$(echo "${HYPERKUBE_HUB_TAG}"|cut -f1 -d,)"
-    HYPERKUBE_TAG="$(echo "${HYPERKUBE_HUB_TAG}"|cut -f2 -d,)"
-fi
-
 function usage() {
   cat <<EOF
 usage: ${BASH_SOURCE[0]} [options ...]"
@@ -93,7 +85,6 @@ usage: ${BASH_SOURCE[0]} [options ...]"
     -x ... <hub>,<tag> for the mixer docker image
     -c ... <hub>,<tag> for the citadel docker image
     -a ... <hub>,<tag> Specifies same hub and tag for pilot, mixer, proxy, and citadel containers
-    -h ... <hub>,<tag> for the hyperkube docker image
     -o ... <hub>,<tag> for the proxy docker image
     -n ... <namespace> namespace in which to install Istio control plane components
     -A ... URL to download auth debian packages
@@ -139,8 +130,6 @@ export ISTIO_NAMESPACE="${ISTIO_NAMESPACE}"
 export PILOT_DEBIAN_URL="${PILOT_DEBIAN_URL}"
 export FORTIO_HUB="${FORTIO_HUB}"
 export FORTIO_TAG="${FORTIO_TAG}"
-export HYPERKUBE_HUB="${HYPERKUBE_HUB}"
-export HYPERKUBE_TAG="${HYPERKUBE_TAG}"
 EOF
 }
 

istioctl/cmd/istioctl/gendeployment/cmd.go

@@ -24,8 +24,7 @@ import (
 )
 
 const (
-	defaultTag          = "master-latest-daily"
-	defaultHyperkubeTag = "v1.7.6_coreos.0"
+	defaultTag = "master-latest-daily"
 )
 
 // Command returns the "gen-deploy" subcommand for istioctl.
@@ -75,9 +74,6 @@ func Command(istioNamespaceFlag *string) *cobra.Command {
 	cmd.PersistentFlags().StringVar(&helmChartLocation, "helm-chart-dir", ".",
 		"The directory to find the helm charts used to render Istio deployments. -o yaml uses these to render the helm chart locally.")
 
-	cmd.PersistentFlags().StringVar(&install.HyperkubeHub, "hyperkube-hub", install.HyperkubeHub, "The container registry to pull Hyperkube images from")
-	cmd.PersistentFlags().StringVar(&install.HyperkubeTag, "hyperkube-tag", install.HyperkubeTag, "The tag to use to pull the `Hyperkube` container")
-
 	_ = cmd.PersistentFlags().MarkHidden("hub")
 	_ = cmd.PersistentFlags().MarkHidden("mixer-tag")
 	_ = cmd.PersistentFlags().MarkHidden("pilot-tag")
@@ -108,9 +104,6 @@ type installation struct {
 	CaTag    string
 	ProxyTag string
 
-	HyperkubeHub string
-	HyperkubeTag string
-
 	NodePort uint16
 	Debug    bool
 
@@ -138,9 +131,6 @@ func defaultInstall() *installation {
 		PilotTag: defaultTag,
 		CaTag:    defaultTag,
 		ProxyTag: defaultTag,
-
-		HyperkubeHub: "quay.io/coreos/hyperkube",
-		HyperkubeTag: defaultHyperkubeTag,
 	}
 }
 

tools/istio-docker.mk

@@ -22,7 +22,7 @@
 docker: build test-bins docker.all
 
 DOCKER_TARGETS:=docker.pilot docker.proxy_debug docker.proxytproxy docker.proxyv2 docker.app docker.test_policybackend \
-	docker.proxy_init docker.mixer docker.citadel docker.galley docker.sidecar_injector
+	docker.proxy_init docker.mixer docker.citadel docker.galley docker.sidecar_injector docker.kubectl
 
 $(ISTIO_DOCKER) $(ISTIO_DOCKER_TAR):
 	mkdir -p $@
@@ -70,7 +70,9 @@ $(foreach FILE,$(DOCKER_FILES_FROM_SOURCE), \
 # 	cp $(ISTIO_BIN)/kubectl $(ISTIO_DOCKER)/kubectl
 DOCKER_FILES_FROM_ISTIO_BIN:=kubectl
 $(foreach FILE,$(DOCKER_FILES_FROM_ISTIO_BIN), \
-        $(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_BIN)/$(FILE) | $(ISTIO_DOCKER); bin/testEnvLocalK8S.sh getDeps; cp $(ISTIO_BIN)/$(FILE) $(ISTIO_DOCKER)/$(FILE)))
+        $(eval $(ISTIO_BIN)/$(FILE): ; bin/testEnvLocalK8S.sh getDeps))
+$(foreach FILE,$(DOCKER_FILES_FROM_ISTIO_BIN), \
+        $(eval $(ISTIO_DOCKER)/$(FILE): $(ISTIO_BIN)/$(FILE) | $(ISTIO_DOCKER); cp $(ISTIO_BIN)/$(FILE) $(ISTIO_DOCKER)/$(FILE)))
 
 # pilot docker images
 
@@ -153,8 +155,8 @@ docker.test_policybackend: mixer/docker/Dockerfile.test_policybackend
 docker.test_policybackend: $(ISTIO_OUT)/mixer-test-policybackend
 	$(DOCKER_RULE)
 
-#docker.kubectl: docker/Dockerfile$$(suffix $$@) $(ISTIO_BIN)/kubectl
-#	$(DOCKER_RULE)
+docker.kubectl: docker/Dockerfile$$(suffix $$@) $(ISTIO_BIN)/kubectl
+	$(DOCKER_RULE)
 
 # addons docker images