kubeadm: use config yaml

kvirt/cluster/kubeadm/__init__.py

@@ -8,6 +8,7 @@
 import os
 from random import choice
 from re import match
+from socket import gethostbyname
 from shutil import which
 from string import ascii_lowercase, ascii_letters, digits
 from subprocess import call
@@ -107,6 +108,7 @@ def create(config, plandir, cluster, overrides):
     plan = cluster
     data['kube'] = data['cluster']
     data['kubetype'] = 'generic'
+    domain = data['domain']
     cloud_lb = data['cloud_lb']
     cloud_dns = data['cloud_dns']
     cloud_storage = data['cloud_storage']
@@ -121,7 +123,6 @@ def create(config, plandir, cluster, overrides):
     network = data.get('network', 'default')
     api_ip = data.get('api_ip')
     if provider in cloud_providers:
-        domain = data.get('domain', 'karmalabs.corp')
         api_ip = f"{cluster}-ctlplane.{domain}"
     elif api_ip is None:
         networkinfo = k.info_network(network)
@@ -298,9 +299,11 @@ def create(config, plandir, cluster, overrides):
         else:
             for lbentry in config.list_loadbalancers():
                 if lbentry[0] == f'api-{cluster}':
-                    lb_ip = lbentry[1]
+                    lb_ip = gethostbyname(lbentry[1])
                     update_etc_hosts(cluster, domain, lb_ip)
                     break
+    elif not data['ignore_hosts']:
+        update_etc_hosts(cluster, domain, api_ip)
     os.environ['KUBECONFIG'] = f"{clusterdir}/auth/kubeconfig"
     apps = data.get('apps', [])
     if data.get('metallb', False) and 'metallb' not in apps:

kvirt/cluster/kubeadm/bootstrap.sh

@@ -1,26 +1,12 @@
 #!/usr/bin/env bash
 
+CLUSTER={{ cluster }}
+DOMAIN={{ domain }}
 POD_CIDR={{ cluster_network_ipv4 }}
-SERVICE_CIDR={{ service_network_ipv4 }}
-
-{% if config_type in ['aws', 'gcp', 'ibm'] %}
-API_IP={{ "api.%s.%s" % (cluster, domain) }}
-echo $(hostname -I) api.{{ cluster }}.{{ domain }} >> /etc/hosts
-{% elif sslip|default(False) %}
-API_IP={{ "api.%s.sslip.io" % api_ip.replace('.', '-').replace(':', '-') }}
-{% else %}
-API_IP={{ api_ip }}
-{% endif %}
 
-DOMAIN={{ domain }}
+echo $(hostname -I) api.$CLUSTER.$DOMAIN >> /etc/hosts
 
-# initialize cluster
-CERTKEY={{ cert_key }}
-TOKEN={{ token }}
-K8S_VERSION='{{ "--kubernetes-version %s" % minor_version if minor_version is defined else "" }}'
-REGISTRY='{{ "--image-repository %s" % disconnected_url if disconnected_url != None else "" }}'
-FEATUREGATES='{{ "--feature-gates %s" % ','.join(feature_gates) if feature_gates else "" }}'
-kubeadm init --control-plane-endpoint "${API_IP}:6443" --pod-network-cidr $POD_CIDR --service-cidr $SERVICE_CIDR --certificate-key $CERTKEY --upload-certs --token $TOKEN --token-ttl 0 --apiserver-cert-extra-sans ${API_IP} $K8S_VERSION $REGISTRY $FEATUREGATES
+kubeadm init --config=/root/config.yaml --upload-certs
 
 # config cluster credentials
 cp /etc/kubernetes/admin.conf /root/kubeconfig
@@ -78,7 +64,4 @@ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-c
 
 {% if registry %}
 kubectl create -f /root/registry.yml
-echo """[[registry]]
-location=\"{{ api_ip }}:5000\"
-insecure=true""" > /etc/containers/registries.conf.d/003-{{ cluster }}.conf
 {% endif %}

kvirt/cluster/kubeadm/bootstrap.yml

@@ -29,6 +29,8 @@
  nets: {{ [primary_network] + extra_networks }}
  disks: {{ [disk_size] + extra_disks }}
  files:
+ - path: /root/config.yaml
+   origin: config_bootstrap.yaml
 {% if config_type not in ['aws', 'gcp', 'ibm', 'hcloud'] %}
  - keepalived.conf
  - keepalived.sh

kvirt/cluster/kubeadm/config_bootstrap.yaml

@@ -0,0 +1,46 @@
+{% set api_fqdn = "api.%s.sslip.io" % api_ip.replace('.', '-').replace(':', '-') if sslip|default(False) else "api.%s.%s" % (cluster, domain) %}
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: InitConfiguration
+bootstrapTokens:
+- token: "{{ token }}"
+  ttl: "0"
+localAPIEndpoint:
+  advertiseAddress: 0.0.0.0
+  bindPort: 6443
+certificateKey: "{{ cert_key }}"
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: ClusterConfiguration
+clusterName: "{{ cluster }}"
+{% if minor_version is defined %}
+kubernetesVersion: {{ minor_version }}
+{% endif %}
+controlPlaneEndpoint: {{ api_fqdn }}:6443
+networking:
+  podSubnet: {{ cluster_network_ipv4 }}
+  serviceSubnet: {{ service_network_ipv4 }}
+  dnsDomain: "{{ cluster }}.{{ domain }}"
+apiServer:
+  certSANs:
+  - {{ api_fqdn }}
+{% if disconnected_url != None %}
+imageRepository: {{ disconnected_url }}
+{% endif %}
+{% if not coredns %}
+dns:
+  disabled: true
+{% endif %}
+{% if not kube_proxy %}
+proxy:
+  disabled: true
+{% endif %}
+---
+{% if feature_gates %}
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: systemd
+featureGates:
+{% for feature_gate in feature_gates %}
+  {{ feature_gate }}: true
+{% endfor %}
+kind: KubeletConfiguration
+{% endif %}

kvirt/cluster/kubeadm/config_join.yaml

@@ -0,0 +1,17 @@
+{% set api_ip = api_ip or ('{0}-ctlplane-1'.format(cluster)|kcli_info('ip') if scale|default(False) and 'ctlplane-0' in name else first_ip) %}
+
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: JoinConfiguration
+discovery:
+  tlsBootstrapToken: {{ token }}
+  bootstrapToken:
+    apiServerEndpoint: {{ api_ip }}:6443
+    token: {{ token }}
+    unsafeSkipCAVerification: true
+{% if 'ctlplane' in name %}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: 0.0.0.0
+    bindPort: 6443
+  certificateKey: {{ cert_key }}
+{% endif %}

kvirt/cluster/kubeadm/ctlplanes.yml

@@ -21,6 +21,8 @@
  nets: {{ [primary_network] + extra_networks }}
  disks: {{ [disk_size] + extra_disks }}
  files:
+ - path: /root/config.yaml
+   origin: config_join.yaml
 {% if config_type not in ['aws', 'azure', 'gcp', 'ibm', 'hcloud'] %}
  - keepalived.conf
  - keepalived.sh

kvirt/cluster/kubeadm/join.sh

@@ -1,11 +1,5 @@
 #!/usr/bin/env bash
-
-{% if api_ip == None %}
-{% set api_ip = '{0}-ctlplane-1'.format(cluster)|kcli_info('ip') if scale|default(False) and 'ctlplane-0' in name else first_ip %}
-{% endif %}
-
-TOKEN={{ token }}
-CTLPLANES="{{ '--control-plane --certificate-key %s' % cert_key if 'ctlplane' in name else '' }}"
+{% set api_ip = api_ip or ('{0}-ctlplane-1'.format(cluster)|kcli_info('ip') if scale|default(False) and 'ctlplane-0' in name else first_ip) %}
 
 echo {{ api_ip }} api.{{ cluster }}.{{ domain }} >> /etc/hosts
-kubeadm join {{ api_ip }}:6443 --token $TOKEN --discovery-token-unsafe-skip-ca-verification $CTLPLANES
+kubeadm join --config /root/config.yaml

kvirt/cluster/kubeadm/kcli_default.yml

@@ -88,3 +88,6 @@ disconnected_user: dummy
 disconnected_password: dummy
 disconnected_sync: true
 feature_gates: []
+coredns: true
+kube_proxy: true
+ignore_hosts: false

kvirt/cluster/kubeadm/workers.yml

@@ -17,6 +17,8 @@
  nets: {{ [network] + extra_networks }}
  disks: {{ [disk_size] + extra_disks }}
  files:
+ - path: /root/config.yaml
+   origin: config_join.yaml
  - path: /root/pre.sh
    origin: pre_{{ 'ubuntu' if ubuntu|default(False) else 'el' }}.sh
  - deploy.sh

kvirt/providers/aws/__init__.py

@@ -1583,7 +1583,6 @@ def create_loadbalancer(self, name, ports=[], checkpath='/index.html', vms=[], d
             if dnsclient is not None:
                 return ip
             self.reserve_dns(name, ip=ip, domain=domain, alias=alias)
-        return
 
     def delete_loadbalancer(self, name):
         domain = None