diff --git a/Tests/kaas/plugin/README.md b/Tests/kaas/plugin/README.md index 521abd5f0..e4f0ae216 100644 --- a/Tests/kaas/plugin/README.md +++ b/Tests/kaas/plugin/README.md @@ -66,17 +66,9 @@ To set the `GIT_ACCESS_TOKEN`, run the following command in your terminal: export GIT_ACCESS_TOKEN= ``` -### Configuring clusterspec.yaml file +### Configuring config file for cluster-stacks plugin -The `clusterspec.yaml` file is used to set parameters for creating a Kubernetes cluster with the `cluster-stacks` plugin. This file allows you to specify details related to the cluster-stack, Git integration, and cluster configurations. - -### Mandatory Parameter - -The only mandatory parameter in `clusterspec.yaml` is `clouds_yaml_path` field, which points to the `clouds.yaml` file for OpenStack. If other parameters are omitted, the default values will be used. - -### Optional Parameters - -You can include additional parameters in `clusterspec.yaml` to customize the cluster setup. These optional parameters are grouped below by their category. +The config file is used to set parameters for creating a Kubernetes cluster with the `cluster-stacks` plugin. This file allows you to specify details related to the cluster-stack, Git integration, and cluster configurations. These optional parameters are grouped below by their category. #### Cluster-Stack Related Parameters @@ -84,7 +76,6 @@ These parameters configure specific settings for the cluster-stack: ```yaml cs_name: # Default: "scs" -cs_k8s_version: # Default: "1.29" cs_version: # Default: "v1" cs_channel: # Default: "stable" cs_cloudname: # Default: "openstack" @@ -105,7 +96,6 @@ git_repo_name: # Default: "cluster-stacks" Set these parameters to customize the configuration for your cluster. ```yaml -cs_cluster_name: # Default: "cs-cluster" cs_pod_cidr: # Default: "192.168.0.0/16" cs_service_cidr: # Default: "10.96.0.0/12" cs_external_id: # Default: A generated UUID diff --git a/Tests/kaas/plugin/cluster.yaml b/Tests/kaas/plugin/cluster.yaml index 17f5a488b..854df210d 100644 --- a/Tests/kaas/plugin/cluster.yaml +++ b/Tests/kaas/plugin/cluster.yaml @@ -1,3 +1,4 @@ +# Cluster resource template apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: diff --git a/Tests/kaas/plugin/clusterstack.yaml b/Tests/kaas/plugin/clusterstack.yaml index ffe33346a..c25833083 100644 --- a/Tests/kaas/plugin/clusterstack.yaml +++ b/Tests/kaas/plugin/clusterstack.yaml @@ -1,3 +1,4 @@ +# Cluster-stack and OpenStack cluster-stack release resource templates apiVersion: clusterstack.x-k8s.io/v1alpha1 kind: ClusterStack metadata: diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index f62e3b3e2..826cc2fbb 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -46,9 +46,10 @@ def create_cluster(self, cluster_name, version, kubeconfig_filepath): """ raise NotImplementedError - def delete_cluster(self, cluster_name): + def delete_cluster(self, cluster_name, kubeconfig_filepath): """ This method is to be called in order to unprovision a cluster :param: cluster_name: + :param: kubeconfig_filepath: """ raise NotImplementedError diff --git a/Tests/kaas/plugin/plugin-cluster-stacks-config.yaml b/Tests/kaas/plugin/plugin-cluster-stacks-config.yaml new file mode 100644 index 000000000..a5056fc8d --- /dev/null +++ b/Tests/kaas/plugin/plugin-cluster-stacks-config.yaml @@ -0,0 +1,19 @@ +# This is an example of the config for a cluster-stacks plugin filled with a default values + +clouds_yaml_path: "~/.config/openstack/clouds.yaml" # Path to OpenStack clouds.yaml file +cs_name: "scs" # Cluster Stack Name +cs_version: "v1" # Cluster Stack Version +cs_channel: "stable" # Release channel +cs_cloudname: "openstack" # Cloud name from OpenStack clouds.yaml +cs_namespace: "default" # Namespace for the Cluster Stack deployment + +# Git Configuration +git_provider: "github" # Git provider (e.g., GitHub, GitLab) +git_org_name: "SovereignCloudStack" # Organization name on Git provider +git_repo_name: "cluster-stacks" # Repository name on Git provider + +# Cluster Information +cs_pod_cidr: "192.168.0.0/16" # Pod CIDR for networking +cs_service_cidr: "10.96.0.0/12" # Service CIDR for networking +cs_external_id: "ebfe5546-f09f-4f42-ab54-094e457d42ec" # External ID for the Cluster Stack +cs_k8s_patch_version: "6" # Kubernetes patch version to use diff --git a/Tests/kaas/plugin/plugin_cluster_stacks.py b/Tests/kaas/plugin/plugin_cluster_stacks.py index 00eb183fd..ff7861e59 100644 --- a/Tests/kaas/plugin/plugin_cluster_stacks.py +++ b/Tests/kaas/plugin/plugin_cluster_stacks.py @@ -1,4 +1,6 @@ import os +import yaml +import shutil import subprocess import base64 import time @@ -45,20 +47,20 @@ def wait_for_capi_pods_ready(timeout=240, interval=15): # Check pod phase and all containers readiness if phase != "Running" or "false" in readiness_states: all_pods_ready = False - print(f"Pod {pod_name} in {namespace} is not ready. Phase: {phase}, Ready: {readiness_states}") + logger.info(f"Pod {pod_name} in {namespace} is not ready. Phase: {phase}, Ready: {readiness_states}") else: - print(f"Error fetching pods in {namespace}: {result.stderr}") + logger.info(f"Error fetching pods in {namespace}: {result.stderr}") all_pods_ready = False except subprocess.CalledProcessError as error: - print(f"Error checking pods in {namespace}: {error}") + logger.error(f"Error checking pods in {namespace}: {error}") all_pods_ready = False if all_pods_ready: - print("All CAPI system pods are ready.") + logger.info("All CAPI system pods are ready.") return True - print("Waiting for all CAPI pods to become ready...") + logger.info("Waiting for all CAPI pods to become ready...") time.sleep(interval) raise TimeoutError(f"Timed out after {timeout} seconds waiting for CAPI and CAPO system pods to become ready.") @@ -93,31 +95,31 @@ def wait_for_cso_pods_ready(timeout=240, interval=15): # Check pod phase and all containers readiness if phase != "Running" or "false" in readiness_states: all_pods_ready = False - print(f"Pod {pod_name} in {cso_namespace} is not ready. Phase: {phase}, Ready: {readiness_states}") + logger.info(f"Pod {pod_name} in {cso_namespace} is not ready. Phase: {phase}, Ready: {readiness_states}") else: - print(f"Error fetching pods in {cso_namespace}: {result.stderr}") + logger.error(f"Error fetching pods in {cso_namespace}: {result.stderr}") all_pods_ready = False except subprocess.CalledProcessError as error: - print(f"Error checking pods in {cso_namespace}: {error}") + logger.error(f"Error checking pods in {cso_namespace}: {error}") all_pods_ready = False if all_pods_ready: - print("All CSO pods in 'cso-system' namespace are ready.") + logger.info("All CSO pods in 'cso-system' namespace are ready.") return True - print("Waiting for CSO pods in 'cso-system' namespace to become ready...") + logger.info("Waiting for CSO pods in 'cso-system' namespace to become ready...") time.sleep(interval) raise TimeoutError(f"Timed out after {timeout} seconds waiting for CSO pods in 'cso-system' namespace to become ready.") -def wait_for_workload_pods_ready(namespace="kube-system", timeout=420, kubeconfig_path=None): +def wait_for_workload_pods_ready(namespace="kube-system", timeout=600, kubeconfig_path=None): """ Waits for all pods in a specific namespace on a workload Kubernetes cluster to become ready. :param namespace: The Kubernetes namespace where pods are located (default is "kube-system"). - :param timeout: The timeout in seconds to wait for pods to become ready (default is 420). + :param timeout: The timeout in seconds to wait for pods to become ready (default is 600). :param kubeconfig_path: Path to the kubeconfig file for the target Kubernetes cluster. :raises RuntimeError: If pods are not ready within the specified timeout. """ @@ -129,87 +131,111 @@ def wait_for_workload_pods_ready(namespace="kube-system", timeout=420, kubeconfi # Run the command subprocess.run(wait_pods_command, shell=True, check=True) - print(f"All pods in namespace '{namespace}' in the workload Kubernetes cluster are ready.") + logger.info("All pods in namespace '{namespace}' in the workload Kubernetes cluster are ready.") except subprocess.CalledProcessError as error: raise RuntimeError(f"Error waiting for pods in namespace '{namespace}' to become ready: {error}") +def load_config(config_path): + """ + Loads the configuration from a YAML file. + """ + if not os.path.exists(config_path): + raise FileNotFoundError(f"Configuration file {config_path} not found.") + + with open(config_path, 'r') as file: + config = yaml.safe_load(file) or {} + return config + + +def setup_environment_variables(self): + # Cluster Stack Parameters + self.clouds_yaml_path = self.config.get('clouds_yaml_path', '~/.config/openstack/clouds.yaml') + self.cs_k8s_version = self.cluster_version + self.cs_name = self.config.get('cs_name', 'scs') + self.cs_version = self.config.get('cs_version', 'v1') + self.cs_channel = self.config.get('cs_channel', 'stable') + self.cs_cloudname = self.config.get('cs_cloudname', 'openstack') + self.cs_secretname = self.cs_cloudname + + # CSP-related variables and additional cluster configuration + self.kubeconfig_cs_cluster_filename = f"kubeconfig-{self.cluster_name}.yaml" + self.cs_class_name = f"openstack-{self.cs_name}-{str(self.cs_k8s_version).replace('.', '-')}-{self.cs_version}" + self.cs_namespace = self.config.get("cs_namespace", "default") + self.cs_pod_cidr = self.config.get('cs_pod_cidr', '192.168.0.0/16') + self.cs_service_cidr = self.config.get('cs_service_cidr', '10.96.0.0/12') + self.cs_external_id = self.config.get('cs_external_id', 'ebfe5546-f09f-4f42-ab54-094e457d42ec') + self.cs_k8s_patch_version = self.config.get('cs_k8s_patch_version', '6') + + if not self.clouds_yaml_path: + raise ValueError("CLOUDS_YAML_PATH environment variable not set.") + + required_env = { + 'CLUSTER_TOPOLOGY': 'true', + 'EXP_CLUSTER_RESOURCE_SET': 'true', + 'EXP_RUNTIME_SDK': 'true', + 'CS_NAME': self.cs_name, + 'CS_K8S_VERSION': self.cs_k8s_version, + 'CS_VERSION': self.cs_version, + 'CS_CHANNEL': self.cs_channel, + 'CS_CLOUDNAME': self.cs_cloudname, + 'CS_SECRETNAME': self.cs_secretname, + 'CS_CLASS_NAME': self.cs_class_name, + 'CS_NAMESPACE': self.cs_namespace, + 'CS_POD_CIDR': self.cs_pod_cidr, + 'CS_SERVICE_CIDR': self.cs_service_cidr, + 'CS_EXTERNAL_ID': self.cs_external_id, + 'CS_K8S_PATCH_VERSION': self.cs_k8s_patch_version, + 'CS_CLUSTER_NAME': self.cluster_name, + } + # Update the environment variables + os.environ.update({key: str(value) for key, value in required_env.items()}) + + +def setup_git_env(self): + # Setup Git environment variables + git_provider = self.config.get('git_provider', 'github') + git_org_name = self.config.get('git_org_name', 'SovereignCloudStack') + git_repo_name = self.config.get('git_repo_name', 'cluster-stacks') + + os.environ.update({ + 'GIT_PROVIDER_B64': base64.b64encode(git_provider.encode()).decode('utf-8'), + 'GIT_ORG_NAME_B64': base64.b64encode(git_org_name.encode()).decode('utf-8'), + 'GIT_REPOSITORY_NAME_B64': base64.b64encode(git_repo_name.encode()).decode('utf-8') + }) + + git_access_token = os.getenv('GIT_ACCESS_TOKEN') + if git_access_token: + os.environ['GIT_ACCESS_TOKEN_B64'] = base64.b64encode(git_access_token.encode()).decode('utf-8') + else: + raise ValueError("GIT_ACCESS_TOKEN environment variable not set.") + + class PluginClusterStacks(KubernetesClusterPlugin): - def __init__(self, config=None): - super().__init__(config) - self.cluster_info = config if config else {} - self._setup_environment_variables() - self._setup_git_env() - - def _setup_environment_variables(self): - # Cluster Stack Parameters - self.clouds_yaml_path = self.cluster_info.get('clouds_yaml_path') - self.cs_k8s_version = self.cluster_info.get('cs_k8s_version', '1.29') - self.cs_name = self.cluster_info.get('cs_name', 'scs') - self.cs_version = self.cluster_info.get('cs_version', 'v1') - self.cs_channel = self.cluster_info.get('cs_channel', 'stable') - self.cs_cloudname = self.cluster_info.get('cs_cloudname', 'openstack') - self.cs_secretname = self.cs_cloudname - - # CSP-related variables and additional cluster configuration - self.cs_cluster_name = self.cluster_info.get('cs_cluster_name', 'cs-cluster') - self.kubeconfig_cs_cluster_filename = f"kubeconfig-{self.cs_cluster_name}" - self.cs_class_name = f"openstack-{self.cs_name}-{str(self.cs_k8s_version).replace('.', '-')}-{self.cs_version}" - self.cs_namespace = os.getenv("CS_NAMESPACE", "default") - self.cs_pod_cidr = self.cluster_info.get('cs_pod_cidr', '192.168.0.0/16') - self.cs_service_cidr = self.cluster_info.get('cs_service_cidr', '10.96.0.0/12') - self.cs_external_id = self.cluster_info.get('cs_external_id', 'ebfe5546-f09f-4f42-ab54-094e457d42ec') - self.cs_k8s_patch_version = self.cluster_info.get('cs_k8s_patch_version', '6') - - if not self.clouds_yaml_path: - raise ValueError("CLOUDS_YAML_PATH environment variable not set.") - - required_env = { - 'CLUSTER_TOPOLOGY': 'true', - 'EXP_CLUSTER_RESOURCE_SET': 'true', - 'EXP_RUNTIME_SDK': 'true', - 'CS_NAME': self.cs_name, - 'CS_K8S_VERSION': self.cs_k8s_version, - 'CS_VERSION': self.cs_version, - 'CS_CHANNEL': self.cs_channel, - 'CS_CLOUDNAME': self.cs_cloudname, - 'CS_SECRETNAME': self.cs_secretname, - 'CS_CLASS_NAME': self.cs_class_name, - 'CS_NAMESPACE': self.cs_namespace, - 'CS_POD_CIDR': self.cs_pod_cidr, - 'CS_SERVICE_CIDR': self.cs_service_cidr, - 'CS_EXTERNAL_ID': self.cs_external_id, - 'CS_K8S_PATCH_VERSION': self.cs_k8s_patch_version, - 'CS_CLUSTER_NAME': self.cs_cluster_name, - } - # Update the environment variables - os.environ.update({key: str(value) for key, value in required_env.items()}) - - def _setup_git_env(self): - # Setup Git environment variables - git_provider = self.cluster_info.get('git_provider', 'github') - git_org_name = self.cluster_info.get('git_org_name', 'SovereignCloudStack') - git_repo_name = self.cluster_info.get('git_repo_name', 'cluster-stacks') - - os.environ.update({ - 'GIT_PROVIDER_B64': base64.b64encode(git_provider.encode()).decode('utf-8'), - 'GIT_ORG_NAME_B64': base64.b64encode(git_org_name.encode()).decode('utf-8'), - 'GIT_REPOSITORY_NAME_B64': base64.b64encode(git_repo_name.encode()).decode('utf-8') - }) - - git_access_token = os.getenv('GIT_ACCESS_TOKEN') - if git_access_token: - os.environ['GIT_ACCESS_TOKEN_B64'] = base64.b64encode(git_access_token.encode()).decode('utf-8') - else: - raise ValueError("GIT_ACCESS_TOKEN environment variable not set.") + def __init__(self, config_file=None): + self.config = load_config(config_file) if config_file else {} + logger.debug(self.config) + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") + + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): + self.cluster_name = cluster_name + self.cluster_version = version + + # Setup variables + setup_environment_variables(self) + setup_git_env(self) - def _create_cluster(self): # Create the Kind cluster - self.cluster = KindCluster(self.cluster_name) + self.cluster = KindCluster(name=cluster_name) self.cluster.create() self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) - os.environ['KUBECONFIG'] = self.kubeconfig + if kubeconfig_filepath: + shutil.move(self.kubeconfig, kubeconfig_filepath) + else: + kubeconfig_filepath = str(self.kubeconfig) + os.environ['KUBECONFIG'] = kubeconfig_filepath # Initialize clusterctl with OpenStack as the infrastructure provider self._run_subprocess(["clusterctl", "init", "--infrastructure", "openstack"], "Error during clusterctl init") @@ -241,34 +267,36 @@ def _create_cluster(self): self._retrieve_kubeconfig() # Wait for workload system pods to be ready + print(self.kubeconfig_cs_cluster_filename) wait_for_workload_pods_ready(kubeconfig_path=self.kubeconfig_cs_cluster_filename) - def _delete_cluster(self): + def delete_cluster(self, cluster_name=None, kubeconfig_filepath=None): + kubeconfig_cs_cluster_filename = f"kubeconfig-{cluster_name}.yaml" try: # Check if the cluster exists - check_cluster_command = f"kubectl get cluster {self.cs_cluster_name} --kubeconfig {self.cluster_info.get('kubeconfig')}" + check_cluster_command = f"kubectl get cluster {cluster_name} --kubeconfig {kubeconfig_filepath}" result = subprocess.run(check_cluster_command, shell=True, check=True, capture_output=True, text=True) # Proceed with deletion only if the cluster exists if result.returncode == 0: - delete_command = f"kubectl delete cluster {self.cs_cluster_name} --timeout=600s --kubeconfig {self.cluster_info.get('kubeconfig')}" + delete_command = f"kubectl delete cluster {cluster_name} --timeout=600s --kubeconfig {kubeconfig_filepath}" self._run_subprocess(delete_command, "Timeout while deleting the cluster", shell=True) except subprocess.CalledProcessError as error: if "NotFound" in error.stderr: - logger.info(f"Cluster {self.cs_cluster_name} not found. Skipping deletion.") + logger.info(f"Cluster {cluster_name} not found. Skipping deletion.") else: raise RuntimeError(f"Error checking for cluster existence: {error}") # Delete kind cluster - self.cluster = KindCluster(self.cluster_name) + self.cluster = KindCluster(cluster_name) self.cluster.delete() # Remove kubeconfigs - if os.path.exists(self.kubeconfig_cs_cluster_filename): - os.remove(self.kubeconfig_cs_cluster_filename) - if os.path.exists(self.cluster_info.get('kubeconfig')): - os.remove(self.cluster_info.get('kubeconfig')) + if os.path.exists(kubeconfig_cs_cluster_filename): + os.remove(kubeconfig_cs_cluster_filename) + if os.path.exists(kubeconfig_filepath): + os.remove(kubeconfig_filepath) def _apply_yaml_with_envsubst(self, yaml_file, error_msg): try: @@ -320,7 +348,7 @@ def _wait_kcp_ready(self, kcp_name): def _retrieve_kubeconfig(self): kubeconfig_command = ( - f"clusterctl get kubeconfig {self.cs_cluster_name} > {self.kubeconfig_cs_cluster_filename}" + f"clusterctl get kubeconfig {self.cluster_name} > {self.kubeconfig_cs_cluster_filename}" ) self._run_subprocess(kubeconfig_command, "Error retrieving kubeconfig", shell=True) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 0b2d78bc9..445996f04 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -45,6 +45,10 @@ def create_cluster(self, cluster_name, version, kubeconfig): else: self.cluster.create(self.config) - def delete_cluster(self, cluster_name): + def delete_cluster(self, cluster_name, kubeconfig_filepath): self.cluster = KindCluster(cluster_name) self.cluster.delete() + + # Remove kubeconfig + if os.path.exists(kubeconfig_filepath): + os.remove(kubeconfig_filepath) diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 08b21db57..5595c8994 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -56,7 +56,7 @@ def delete(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterinfo = clusterspec[clusterspec_cluster] cluster_id = clusterspec_cluster plugin = init_plugin(plugin_kind, plugin_config) - plugin.delete_cluster(cluster_id) + plugin.delete_cluster(cluster_id, os.path.abspath(clusterinfo['kubeconfig'])) if __name__ == '__main__':