diff --git a/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py b/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py index 6ac52f4e4..bc86e7070 100644 --- a/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py +++ b/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py @@ -1,13 +1,24 @@ import os import yaml import subprocess -import base64 import time import logging from interface import KubernetesClusterPlugin logger = logging.getLogger("PluginClusterStacks") + +# Default configuration values +DEFAULTS = { + 'cs_name': 'scs', +} + +# Keys needed for environment variables +ENV_KEYS = {'cs_name', 'cs_version', 'cs_channel', 'cs_secretname', 'cs_class_name', + 'cs_namespace', 'cs_pod_cidr', 'cs_service_cidr', 'cs_external_id', 'cs_k8s_patch_version', + 'cs_cluster_name', 'cs_k8s_version'} + + # Helper functions def wait_for_pods(self, namespaces, timeout=240, interval=15, kubeconfig=None): """ @@ -27,13 +38,20 @@ def wait_for_pods(self, namespaces, timeout=240, interval=15, kubeconfig=None): for namespace in namespaces: try: # Get pod status in the namespace - wait_pods_command = ( - f"kubectl wait -n {namespace} --for=condition=Ready --timeout={timeout}s pod --all" + wait_pods_command = f"kubectl wait -n {namespace} --for=condition=Ready --timeout={timeout}s pod --all" + result = self._run_subprocess( + wait_pods_command, + f"Error fetching pods in {namespace}", + shell=True, + capture_output=True, + text=True, + kubeconfig=kubeconfig ) - result = self._run_subprocess(wait_pods_command, f"Error fetching pods in {namespace}", shell=True, capture_output=True, text=True, kubeconfig=kubeconfig) if result.returncode != 0: - logger.warning(f"Not all pods in namespace {namespace} are ready. Details: {result.stderr}") + logger.warning( + f"Not all pods in namespace {namespace} are ready. Details: {result.stderr}" + ) all_pods_ready = False else: logger.info(f"All pods in namespace {namespace} are ready.") @@ -49,7 +67,9 @@ def wait_for_pods(self, namespaces, timeout=240, interval=15, kubeconfig=None): logger.info("Waiting for all pods in specified namespaces to become ready...") time.sleep(interval) - raise TimeoutError(f"Timed out after {timeout} seconds waiting for pods in namespaces {namespaces} to become ready.") + raise TimeoutError( + f"Timed out after {timeout} seconds waiting for pods in namespaces {namespaces} to become ready." + ) def load_config(config_path): @@ -57,61 +77,111 @@ def load_config(config_path): Loads the configuration from a YAML file. """ - with open(config_path, 'r') as file: + with open(config_path, "r") as file: config = yaml.safe_load(file) or {} + + base_dir = os.path.dirname(config_path) + if 'kubeconfig' in config: + config['kubeconfig'] = os.path.join(base_dir, config['kubeconfig']) + if 'workloadcluster' in config: + config['workloadcluster'] = os.path.join(base_dir, config['workloadcluster']) + if 'clusterstack' in config: + config['clusterstack'] = os.path.join(base_dir, config['clusterstack']) + return config + +def setup_environment_variables(self): + """ + Constructs and returns a dictionary of required environment variables + based on the configuration. + + :raises ValueError: If the `GIT_ACCESS_TOKEN` environment variable is not set. + + :return: A dictionary of required environment variables with necessary values and + encodings for Kubernetes and Git-related configurations. + """ + # Calculate values that need to be set dynamically + if hasattr(self, 'cluster_version'): + self.config['cs_k8s_version'] = self.cluster_version + self.config['cs_namespace'] = self.cs_namespace + self.config['cs_class_name'] = ( + f"openstack-{self.config['cs_name']}-{str(self.config['cs_k8s_version']).replace('.', '-')}-" + f"{self.config['cs_version']}" + ) + if hasattr(self, 'cluster_name'): + self.config['cs_cluster_name'] = self.cluster_name + + # Construct general environment variables + required_env = {key.upper(): value for key, value in self.config.items() if key in ENV_KEYS} + + return required_env + + class PluginClusterStacksRemoteAPI(KubernetesClusterPlugin): - def __init__(self, config_file=None): + def __init__(self, config_file): self.config = load_config(config_file) if config_file else {} logger.debug(self.config) self.working_directory = os.getcwd() logger.debug(f"Working from {self.working_directory}") - self.kubeconfig_mgmnt = self.config['kubeconfig'] - self.workloadclusters = self.config['workloadcluster'] - self.cs_namespace = self.config['namespace'] - - - def create_cluster(self, cluster_name=None, version=None, kubeconfig_filepath=None): + for key, value in DEFAULTS.items(): + self.config.setdefault(key, value) + self.kubeconfig_mgmnt = self.config["kubeconfig"] + self.workloadclusters = self.config["workloadcluster"] + self.clusterstack = self.config["clusterstack"] + self.cs_namespace = self.config["namespace"] + + def create_cluster(self, cluster_name, version, kubeconfig_filepath): self.cluster_name = cluster_name self.cluster_version = version self.kubeconfig_cs_cluster = kubeconfig_filepath - # Create workload cluster - self._apply_yaml(self.workloadclusters, "Error applying cluster.yaml", kubeconfig=self.kubeconfig_mgmnt) + # Create cluster-stack resource + self._apply_yaml(self.clusterstack, "Error applying clusterstack.yaml", kubeconfig=self.kubeconfig_mgmnt) - #TODO:!!! We also need to introduce a waiting function here - - print("retrieve kubeconfig to path") + # Create workload cluster + self._apply_yaml( + self.workloadclusters, + "Error applying cluster.yaml", + kubeconfig=self.kubeconfig_mgmnt, + ) + + # Get and wait on kubeadmcontrolplane and retrieve workload cluster kubeconfig + kcp_name = self._get_kubeadm_control_plane_name(namespace=self.cs_namespace, kubeconfig=self.kubeconfig_mgmnt) + self._wait_kcp_ready(kcp_name, namespace=self.cs_namespace, kubeconfig=self.kubeconfig_mgmnt) self._retrieve_kubeconfig(namespace=self.cs_namespace, kubeconfig=self.kubeconfig_mgmnt) # Wait for workload system pods to be ready - # wait_for_workload_pods_ready(kubeconfig_path=self.kubeconfig_cs_cluster) - # ~ wait_for_pods(self, ["kube-system"], timeout=600, interval=15, kubeconfig=self.kubeconfig_cs_cluster) - + wait_for_pods(self, ["kube-system"], timeout=600, interval=15, kubeconfig=self.kubeconfig_cs_cluster) - def delete_cluster(self, cluster_name=None): #TODO:!!! need to adjust delete method - self.cluster_name = cluster_name - #Get the name of the workloadcluster from the config file - workload_cluster_config = load_config(self.workloadclusters) - workload_cluster_name = workload_cluster_config['metadata']['name'] + def delete_cluster(self, cluster_name): try: # Check if the cluster exists - check_cluster_command = f"kubectl --kubeconfig={self.kubeconfig_mgmnt} get cluster {workload_cluster_name} -n {self.cs_namespace}" - result = self._run_subprocess(check_cluster_command, "Failed to get cluster resource", shell=True, capture_output=True, text=True) + check_cluster_command = f"kubectl get cluster {cluster_name} -n {self.cs_namespace}" + result = self._run_subprocess( + check_cluster_command, + "Failed to get cluster resource", + shell=True, + capture_output=True, + text=True, + kubeconfig=self.kubeconfig_mgmnt + ) # Proceed with deletion only if the cluster exists if result.returncode == 0: - delete_command = f"kubectl --kubeconfig={self.kubeconfig_mgmnt} delete cluster {workload_cluster_name} --timeout=600s -n {self.cs_namespace}" - self._run_subprocess(delete_command, "Timeout while deleting the cluster", shell=True) + delete_command = f"kubectl delete cluster {cluster_name} --timeout=600s -n {self.cs_namespace}" + self._run_subprocess( + delete_command, "Timeout while deleting the cluster", shell=True, kubeconfig=self.kubeconfig_mgmnt + ) except subprocess.CalledProcessError as error: if "NotFound" in error.stderr: - logger.info(f"Cluster {workload_cluster_name} not found. Skipping deletion.") + logger.info( + f"Cluster {cluster_name} not found. Skipping deletion." + ) else: raise RuntimeError(f"Error checking for cluster existence: {error}") - def _apply_yaml(self, yaml_file, error_msg, kubeconfig=None): """ Applies a Kubernetes YAML configuration file to the cluster, substituting environment variables as needed. @@ -123,15 +193,63 @@ def _apply_yaml(self, yaml_file, error_msg, kubeconfig=None): try: # Determine if the file is a local path or a URL if os.path.isfile(yaml_file): - command = f"kubectl --kubeconfig={self.kubeconfig_mgmnt} apply -f {yaml_file} -n {self.cs_namespace}" + command = f"/tmp/envsubst < {yaml_file} | kubectl apply -f -" else: raise ValueError(f"Unknown file: {yaml_file}") - self._run_subprocess(command, error_msg, shell=True) + self._run_subprocess(command, error_msg, shell=True, kubeconfig=kubeconfig) except subprocess.CalledProcessError as error: raise RuntimeError(f"{error_msg}: {error}") + def _get_kubeadm_control_plane_name(self, namespace="default", kubeconfig=None): + """ + Retrieves the name of the KubeadmControlPlane resource for the Kubernetes cluster + in the specified namespace. + + :param namespace: The namespace to search for the KubeadmControlPlane resource. + :param kubeconfig: Optional path to the kubeconfig file for the target Kubernetes cluster. + + :return: The name of the KubeadmControlPlane resource as a string. + """ + max_retries = 6 + delay_between_retries = 20 + for _ in range(max_retries): + try: + kcp_command = ( + f"kubectl get kubeadmcontrolplane -n {namespace} " + "-o=jsonpath='{.items[0].metadata.name}'" + ) + kcp_name = self._run_subprocess(kcp_command, "Error retrieving kcp_name", shell=True, capture_output=True, text=True, kubeconfig=kubeconfig) + logger.info(kcp_name) + kcp_name_stdout = kcp_name.stdout.strip() + if kcp_name_stdout: + print(f"KubeadmControlPlane name: {kcp_name_stdout}") + return kcp_name_stdout + except subprocess.CalledProcessError as error: + print(f"Error getting kubeadmcontrolplane name: {error}") + # Wait before retrying + time.sleep(delay_between_retries) + else: + raise RuntimeError("Failed to get kubeadmcontrolplane name") + + def _wait_kcp_ready(self, kcp_name, namespace="default", kubeconfig=None): + """ + Waits for the specified KubeadmControlPlane resource to become 'Available'. + + :param kcp_name: The name of the KubeadmControlPlane resource to check for availability. + :param namespace: The namespace where the KubeadmControlPlane resource is. + :param kubeconfig: Optional path to the kubeconfig file for the target Kubernetes cluster. + """ + try: + self._run_subprocess( + f"kubectl wait kubeadmcontrolplane/{kcp_name} --for=condition=Available --timeout=600s -n {namespace}", + "Error waiting for kubeadmcontrolplane availability", + shell=True, + kubeconfig=kubeconfig + ) + except subprocess.CalledProcessError as error: + raise RuntimeError(f"Error waiting for kubeadmcontrolplane to be ready: {error}") def _retrieve_kubeconfig(self, namespace="default", kubeconfig=None): """ @@ -140,26 +258,12 @@ def _retrieve_kubeconfig(self, namespace="default", kubeconfig=None): :param namespace: The namespace of the cluster to retrieve the kubeconfig for. :param kubeconfig: Optional path to the kubeconfig file for the target Kubernetes cluster. """ + kubeconfig_command = ( + f"sudo -E clusterctl get kubeconfig {self.cluster_name} -n {namespace} > {self.kubeconfig_cs_cluster}" + ) + self._run_subprocess(kubeconfig_command, "Error retrieving kubeconfig", shell=True, kubeconfig=kubeconfig) - #Get the name of the workloadcluster from the config file - workload_cluster_config = load_config(self.workloadclusters) - workload_cluster_name = workload_cluster_config['metadata']['name'] - - command_args = [ - "kubectl ", - f"--kubeconfig={self.kubeconfig_mgmnt}", - f"-n {self.cs_namespace}", - f"get secret {workload_cluster_name}-kubeconfig", - "-o go-template='{{.data.value|base64decode}}'", - f"> {self.kubeconfig_cs_cluster}", - ] - kubeconfig_command = "" - for entry in command_args: - kubeconfig_command += entry + " " - self._run_subprocess(kubeconfig_command, "Error retrieving kubeconfig", shell=True) - - - def _run_subprocess(self, command, error_msg, shell=False, capture_output=False, text=False): + def _run_subprocess(self, command, error_msg, shell=False, capture_output=False, text=False, kubeconfig=None): """ Executes a subprocess command with the specified environment variables and parameters. @@ -168,12 +272,24 @@ def _run_subprocess(self, command, error_msg, shell=False, capture_output=False, :param shell: Whether to execute the command through the shell (default: `False`). :param capture_output: Whether to capture the command's standard output and standard error (default: `False`). :param text: Whether to treat the command's output and error as text (default: `False`). + :param kubeconfig: Optional path to the kubeconfig file for the target Kubernetes cluster. + :return: The result of the `subprocess.run` command """ try: - # Run the subprocess - result = subprocess.run(command, shell=shell, capture_output=capture_output, text=text, check=True) + env = setup_environment_variables(self) + env['PATH'] = f'/usr/local/bin:/usr/bin:{self.working_directory}' + # Set env variable DISPLAY which you need to open the oidc window automatically + env['DISPLAY'] = ':0' + env['HOME'] = self.working_directory + if kubeconfig: + env['KUBECONFIG'] = kubeconfig + + # Run the subprocess with the environment + result = subprocess.run(command, shell=shell, capture_output=capture_output, text=text, check=True, env=env) + return result + except subprocess.CalledProcessError as error: logger.error(f"{error_msg}: {error}") raise diff --git a/playbooks/k8s_configs/moin_cluster_clusterstack.yaml b/playbooks/k8s_configs/moin_cluster_clusterstack.yaml new file mode 100644 index 000000000..c25833083 --- /dev/null +++ b/playbooks/k8s_configs/moin_cluster_clusterstack.yaml @@ -0,0 +1,30 @@ +# Cluster-stack and OpenStack cluster-stack release resource templates +apiVersion: clusterstack.x-k8s.io/v1alpha1 +kind: ClusterStack +metadata: + name: clusterstack + namespace: ${CS_NAMESPACE} +spec: + provider: openstack + name: ${CS_NAME} + kubernetesVersion: "${CS_K8S_VERSION}" + channel: ${CS_CHANNEL} + autoSubscribe: false + providerRef: + apiVersion: infrastructure.clusterstack.x-k8s.io/v1alpha1 + kind: OpenStackClusterStackReleaseTemplate + name: cspotemplate + versions: + - ${CS_VERSION} +--- +apiVersion: infrastructure.clusterstack.x-k8s.io/v1alpha1 +kind: OpenStackClusterStackReleaseTemplate +metadata: + name: cspotemplate + namespace: ${CS_NAMESPACE} +spec: + template: + spec: + identityRef: + kind: Secret + name: ${CS_SECRETNAME} diff --git a/playbooks/k8s_configs/moin_cluster_config.yaml b/playbooks/k8s_configs/moin_cluster_config.yaml index 22189ee62..dda27312b 100644 --- a/playbooks/k8s_configs/moin_cluster_config.yaml +++ b/playbooks/k8s_configs/moin_cluster_config.yaml @@ -1,3 +1,16 @@ -kubeconfig: "../../playbooks/k8s_configs/moin_cluster_kubeconfig.yaml" -workloadcluster: "../../playbooks/k8s_configs/moin_cluster_workloadcluster.yaml" -namespace: "kaas-playground1" +kubeconfig: "moin_cluster_kubeconfig.yaml" +workloadcluster: "moin_cluster_workloadcluster.yaml" +clusterstack: "moin_cluster_clusterstack.yaml" +namespace: "kaas-playground8" + +# Cluster-stack related configuration +cs_name: "scs" # Cluster Stack Name +cs_version: "v1" # Cluster Stack Version +cs_channel: "stable" # Release channel +cs_secretname: "openstack" # Cloud name from OpenStack clouds.yaml + +# Cluster Information +cs_pod_cidr: "192.168.0.0/16" # Pod CIDR for networking +cs_service_cidr: "10.96.0.0/12" # Service CIDR for networking +cs_external_id: "ebfe5546-f09f-4f42-ab54-094e457d42ec" # External ID for the Cluster Stack +cs_k8s_patch_version: "9" # Kubernetes patch version to use diff --git a/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml b/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml index 41fdadc48..5b4af619b 100644 --- a/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml +++ b/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml @@ -1,16 +1,23 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: - name: scs-cluster-129 + name: ${CS_CLUSTER_NAME} + namespace: ${CS_NAMESPACE} labels: managed-secret: cloud-config spec: topology: variables: - class: openstack-scs-1-29-v4 + - name: controller_flavor + value: "SCS-2V-4-50" + - name: worker_flavor + value: "SCS-2V-4-50" + - name: external_id + value: ${CS_EXTERNAL_ID} + class: ${CS_CLASS_NAME} controlPlane: replicas: 3 - version: v1.29.10 + version: v${CS_K8S_VERSION}.${CS_K8S_PATCH_VERSION} workers: machineDeployments: - class: default-worker