From 921e020e33867def19853e8b36247d1d4205ad9e Mon Sep 17 00:00:00 2001 From: Jari Kolehmainen Date: Mon, 28 May 2018 15:14:11 +0300 Subject: [PATCH] Configurable image_repository (#385) --- lib/pharos/config.rb | 1 + lib/pharos/config_schema.rb | 1 + lib/pharos/kubeadm.rb | 1 + lib/pharos/phases/configure_calico.rb | 1 + lib/pharos/phases/configure_etcd.rb | 1 + lib/pharos/phases/configure_host.rb | 3 ++- lib/pharos/phases/configure_kubelet.rb | 2 ++ lib/pharos/phases/configure_metrics.rb | 7 +++++-- lib/pharos/phases/configure_weave.rb | 1 + lib/pharos/resources/calico/daemonset.yml.erb | 4 ++-- lib/pharos/resources/heapster/deployment.yml.erb | 2 +- lib/pharos/resources/weave/daemon-set.yml.erb | 4 ++-- lib/pharos/scripts/configure-cri-o.sh | 2 +- lib/pharos/scripts/configure-etcd.sh | 4 ++-- lib/pharos/scripts/configure-kubelet-proxy.sh | 4 ++-- lib/pharos_cluster.rb | 2 +- spec/pharos/phases/configure_kubelet_spec.rb | 6 +++++- 17 files changed, 31 insertions(+), 15 deletions(-) diff --git a/lib/pharos/config.rb b/lib/pharos/config.rb index 46dfff912..eccddb238 100644 --- a/lib/pharos/config.rb +++ b/lib/pharos/config.rb @@ -42,6 +42,7 @@ def self.load(raw_data) attribute :authentication, Pharos::Configuration::Authentication attribute :audit, Pharos::Configuration::Audit attribute :kubelet, Pharos::Configuration::Kubelet + attribute :image_repository, Pharos::Types::String.default('quay.io/kontena') attribute :addon_paths, Pharos::Types::Array.default([]) attribute :addons, Pharos::Types::Hash.default({}) diff --git a/lib/pharos/config_schema.rb b/lib/pharos/config_schema.rb index 3358739c4..7cbc1c18b 100644 --- a/lib/pharos/config_schema.rb +++ b/lib/pharos/config_schema.rb @@ -112,6 +112,7 @@ def self.messages optional(:kubelet).schema do optional(:read_only_port).filled(:bool?) end + optional(:image_repository).filled(:str?) validate(network_dns_replicas: [:network, :hosts]) do |network, hosts| if network && network[:dns_replicas] diff --git a/lib/pharos/kubeadm.rb b/lib/pharos/kubeadm.rb index 6c40b835d..2a38a382d 100644 --- a/lib/pharos/kubeadm.rb +++ b/lib/pharos/kubeadm.rb @@ -25,6 +25,7 @@ def generate_config 'kind' => 'MasterConfiguration', 'nodeName' => @host.hostname, 'kubernetesVersion' => Pharos::KUBE_VERSION, + 'imageRepository' => @config.image_repository, 'api' => { 'advertiseAddress' => @host.peer_address, 'controlPlaneEndpoint' => 'localhost' diff --git a/lib/pharos/phases/configure_calico.rb b/lib/pharos/phases/configure_calico.rb index 99d6e6017..cb6f46d69 100644 --- a/lib/pharos/phases/configure_calico.rb +++ b/lib/pharos/phases/configure_calico.rb @@ -27,6 +27,7 @@ def call logger.info { "Configuring overlay network ..." } Pharos::Kube.apply_stack( @master.api_address, 'calico', + image_repository: @config.image_repository, ipv4_pool_cidr: @config.network.pod_network_cidr, ipip_mode: @config.network.calico&.ipip_mode || 'Always', ipip_enabled: @config.network.calico&.ipip_mode != 'Never', diff --git a/lib/pharos/phases/configure_etcd.rb b/lib/pharos/phases/configure_etcd.rb index 10db7ffa9..f1b812789 100644 --- a/lib/pharos/phases/configure_etcd.rb +++ b/lib/pharos/phases/configure_etcd.rb @@ -27,6 +27,7 @@ def call 'configure-etcd.sh', PEER_IP: @host.peer_address, INITIAL_CLUSTER: initial_cluster.join(','), + IMAGE_REPO: @config.image_repository, ETCD_VERSION: Pharos::ETCD_VERSION, KUBE_VERSION: Pharos::KUBE_VERSION, ARCH: @host.cpu_arch.name, diff --git a/lib/pharos/phases/configure_host.rb b/lib/pharos/phases/configure_host.rb index b37eaf382..37a70a77f 100644 --- a/lib/pharos/phases/configure_host.rb +++ b/lib/pharos/phases/configure_host.rb @@ -44,7 +44,8 @@ def call CRIO_VERSION: Pharos::CRIO_VERSION, CRICTL_VERSION: Pharos::CRICTL_VERSION, CRIO_STREAM_ADDRESS: @host.peer_address, - CPU_ARCH: @host.cpu_arch.name + CPU_ARCH: @host.cpu_arch.name, + IMAGE_REPO: @config.image_repository ) else raise Pharos::Error, "Unknown container runtime: #{@host.container_runtime}" diff --git a/lib/pharos/phases/configure_kubelet.rb b/lib/pharos/phases/configure_kubelet.rb index 5287f9f8c..59060f170 100644 --- a/lib/pharos/phases/configure_kubelet.rb +++ b/lib/pharos/phases/configure_kubelet.rb @@ -47,6 +47,7 @@ def configure_kubelet_proxy exec_script( 'configure-kubelet-proxy.sh', KUBE_VERSION: Pharos::KUBE_VERSION, + IMAGE_REPO: @config.image_repository, ARCH: @host.cpu_arch.name, VERSION: Pharos::KUBELET_PROXY_VERSION, MASTER_HOSTS: @config.master_hosts.map(&:peer_address).join(','), @@ -100,6 +101,7 @@ def kubelet_extra_args end args += @host.kubelet_args + args << "--pod-infra-container-image=#{@config.image_repository}/pause-#{@host.cpu_arch.name}:3.1" args << "--cloud-provider=#{@config.cloud.provider}" if @config.cloud args << "--cloud-config=#{CLOUD_CONFIG_FILE}" if @config.cloud&.config args diff --git a/lib/pharos/phases/configure_metrics.rb b/lib/pharos/phases/configure_metrics.rb index c20558465..c78bc7a70 100644 --- a/lib/pharos/phases/configure_metrics.rb +++ b/lib/pharos/phases/configure_metrics.rb @@ -5,8 +5,10 @@ module Phases class ConfigureMetrics < Pharos::Phase title "Configure metrics" + HEAPSTER_VERSION = '1.5.1' + register_component( - name: 'heapster', version: '1.5.1', license: 'Apache License 2.0' + name: 'heapster', version: HEAPSTER_VERSION, license: 'Apache License 2.0' ) def call @@ -21,7 +23,8 @@ def configure_heapster logger.info { "Configuring heapster ..." } Pharos::Kube.apply_stack( @master.api_address, 'heapster', - version: '1.5.1', + version: HEAPSTER_VERSION, + image_repository: @config.image_repository, arch: @host.cpu_arch, client_cert: cert.to_pem ) diff --git a/lib/pharos/phases/configure_weave.rb b/lib/pharos/phases/configure_weave.rb index 94aa1b074..aa98bd55f 100644 --- a/lib/pharos/phases/configure_weave.rb +++ b/lib/pharos/phases/configure_weave.rb @@ -41,6 +41,7 @@ def ensure_resources logger.info { "Configuring overlay network ..." } Pharos::Kube.apply_stack( @master.api_address, 'weave', + image_repository: @config.image_repository, trusted_subnets: trusted_subnets, ipalloc_range: @config.network.pod_network_cidr, arch: @host.cpu_arch, diff --git a/lib/pharos/resources/calico/daemonset.yml.erb b/lib/pharos/resources/calico/daemonset.yml.erb index 18dd47cbf..a3fcee822 100644 --- a/lib/pharos/resources/calico/daemonset.yml.erb +++ b/lib/pharos/resources/calico/daemonset.yml.erb @@ -46,7 +46,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v<%= version %> + image: <%= image_repository %>/calico-node:v<%= version %> env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -131,7 +131,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v<%= version %> + image: <%= image_repository %>/calico-cni:v<%= version %> command: ["/install-cni.sh"] env: # Name of the CNI config file to create. diff --git a/lib/pharos/resources/heapster/deployment.yml.erb b/lib/pharos/resources/heapster/deployment.yml.erb index 2581251a6..3576e0d17 100644 --- a/lib/pharos/resources/heapster/deployment.yml.erb +++ b/lib/pharos/resources/heapster/deployment.yml.erb @@ -14,7 +14,7 @@ spec: serviceAccountName: heapster containers: - name: heapster - image: k8s.gcr.io/heapster-<%= arch.name %>:v<%= version %> + image: <%= image_repository %>/heapster-<%= arch.name %>:v<%= version %> imagePullPolicy: IfNotPresent command: - /heapster diff --git a/lib/pharos/resources/weave/daemon-set.yml.erb b/lib/pharos/resources/weave/daemon-set.yml.erb index c343e2651..5ae6d0483 100644 --- a/lib/pharos/resources/weave/daemon-set.yml.erb +++ b/lib/pharos/resources/weave/daemon-set.yml.erb @@ -38,7 +38,7 @@ spec: secretKeyRef: name: weave-passwd key: weave-passwd - image: 'docker.io/weaveworks/weave-kube-<%= arch.name %>:<%= version %>' + image: '<%= image_repository %>/weave-kube-<%= arch.name %>:<%= version %>' livenessProbe: httpGet: host: 127.0.0.1 @@ -73,7 +73,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: 'docker.io/weaveworks/weave-npc-<%= arch.name %>:<%= version %>' + image: '<%= image_repository %>/weave-npc-<%= arch.name %>:<%= version %>' resources: requests: cpu: 10m diff --git a/lib/pharos/scripts/configure-cri-o.sh b/lib/pharos/scripts/configure-cri-o.sh index fea824e50..b32cc0474 100755 --- a/lib/pharos/scripts/configure-cri-o.sh +++ b/lib/pharos/scripts/configure-cri-o.sh @@ -5,7 +5,7 @@ set -ue mkdir -p /etc/systemd/system/crio.service.d cat </etc/systemd/system/crio.service.d/10-cgroup.conf [Service] -Environment='CRIO_STORAGE_OPTIONS=--cgroup-manager=cgroupfs --stream-address=$CRIO_STREAM_ADDRESS --pause-image=k8s.gcr.io/pause-${CPU_ARCH}:3.1' +Environment='CRIO_STORAGE_OPTIONS=--cgroup-manager=cgroupfs --stream-address=$CRIO_STREAM_ADDRESS --pause-image=${IMAGE_REPO}/pause-${CPU_ARCH}:3.1' ExecStartPre=/sbin/sysctl -w net.ipv4.ip_forward=1 EOF diff --git a/lib/pharos/scripts/configure-etcd.sh b/lib/pharos/scripts/configure-etcd.sh index 5ed3c0834..ec4cdf75d 100755 --- a/lib/pharos/scripts/configure-etcd.sh +++ b/lib/pharos/scripts/configure-etcd.sh @@ -47,7 +47,7 @@ spec: - --initial-cluster-token=pharos-etcd-token - --initial-cluster-state=${INITIAL_CLUSTER_STATE} - image: k8s.gcr.io/etcd-${ARCH}:${ETCD_VERSION} + image: ${IMAGE_REPO}/etcd-${ARCH}:${ETCD_VERSION} livenessProbe: exec: command: @@ -85,7 +85,7 @@ if [ ! -e /etc/kubernetes/kubelet.conf ]; then [Service] ExecStartPre=-/sbin/swapoff -a ExecStart= -ExecStart=/usr/bin/kubelet ${KUBELET_ARGS} +ExecStart=/usr/bin/kubelet ${KUBELET_ARGS} --pod-infra-container-image=${IMAGE_REPO}/pause-${ARCH}:3.1 EOF apt-mark unhold kubelet diff --git a/lib/pharos/scripts/configure-kubelet-proxy.sh b/lib/pharos/scripts/configure-kubelet-proxy.sh index ec664a621..f4855152e 100755 --- a/lib/pharos/scripts/configure-kubelet-proxy.sh +++ b/lib/pharos/scripts/configure-kubelet-proxy.sh @@ -17,7 +17,7 @@ metadata: namespace: kube-system spec: containers: - - image: docker.io/kontena/pharos-kubelet-proxy-${ARCH}:${VERSION} + - image: ${IMAGE_REPO}/pharos-kubelet-proxy-${ARCH}:${VERSION} name: proxy env: - name: KUBE_MASTERS @@ -32,7 +32,7 @@ if [ ! -e /etc/kubernetes/kubelet.conf ]; then [Service] ExecStartPre=-/sbin/swapoff -a ExecStart= -ExecStart=/usr/bin/kubelet ${KUBELET_ARGS} +ExecStart=/usr/bin/kubelet ${KUBELET_ARGS} --pod-infra-container-image=${IMAGE_REPO}/pause-${ARCH}:3.1 EOF export DEBIAN_FRONTEND=noninteractive diff --git a/lib/pharos_cluster.rb b/lib/pharos_cluster.rb index 4d263992c..fb4f353b3 100644 --- a/lib/pharos_cluster.rb +++ b/lib/pharos_cluster.rb @@ -14,5 +14,5 @@ module Pharos KUBEADM_VERSION = ENV.fetch('KUBEADM_VERSION') { KUBE_VERSION } ETCD_VERSION = ENV.fetch('ETCD_VERSION') { '3.1.12' } DOCKER_VERSION = '1.13.1' - KUBELET_PROXY_VERSION = '0.3.5' + KUBELET_PROXY_VERSION = '0.3.6' end diff --git a/spec/pharos/phases/configure_kubelet_spec.rb b/spec/pharos/phases/configure_kubelet_spec.rb index 46268f3b2..74a944ba8 100644 --- a/spec/pharos/phases/configure_kubelet_spec.rb +++ b/spec/pharos/phases/configure_kubelet_spec.rb @@ -14,11 +14,15 @@ let(:ssh) { instance_double(Pharos::SSH::Client) } subject { described_class.new(host, config: config, ssh: ssh) } + before(:each) do + allow(host).to receive(:cpu_arch).and_return(double(:cpu_arch, name: 'amd64')) + end + describe '#build_systemd_dropin' do it "returns a systemd unit" do expect(subject.build_systemd_dropin).to eq <<~EOM [Service] - Environment='KUBELET_EXTRA_ARGS=--read-only-port=0 --node-ip=192.168.42.1 --hostname-override=' + Environment='KUBELET_EXTRA_ARGS=--read-only-port=0 --node-ip=192.168.42.1 --hostname-override= --pod-infra-container-image=quay.io/kontena/pause-amd64:3.1' Environment='KUBELET_DNS_ARGS=--cluster-dns=10.96.0.10 --cluster-domain=cluster.local' ExecStartPre=-/sbin/swapoff -a EOM