diff --git a/Tests/config.toml b/Tests/config.toml index 6833fc5ad..7f8f8e923 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -38,9 +38,12 @@ scopes = [ "scs-compatible-kaas", ] subjects = [ - "cspA-current", - "cspA-current-1", - "cspA-current-2", + #"cspA-current", + #"cspA-current-1", + #"cspA-current-2", + #"moin-cluster-current", + #"moin-cluster-current-1", + "moin-cluster-current-2", ] workers = 4 @@ -81,3 +84,21 @@ kube_plugin = "kind" kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" clusterspec_cluster = "current-k8s-release-2" + +[subjects.moin-cluster-current.kubernetes_setup] +kube_plugin = "cluster-stacks-api" +kube_plugin_config = "../playbooks/k8s_configs/moin_cluster_config.yaml" +clusterspec_cluster = "current-k8s-release" + + +[subjects.moin-cluster-current-1.kubernetes_setup] +kube_plugin = "cluster-stacks-api" +kube_plugin_config = "../playbooks/k8s_configs/moin_cluster_config.yaml" +clusterspec_cluster = "current-k8s-release-1" + + +[subjects.moin-cluster-current-2.kubernetes_setup] +kube_plugin = "cluster-stacks-api" +kube_plugin_config = "../playbooks/k8s_configs/moin_cluster_config.yaml" +clusterspec_cluster = "current-k8s-release-2" + diff --git a/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py b/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py new file mode 100644 index 000000000..6ac52f4e4 --- /dev/null +++ b/Tests/kaas/plugin/plugin_cluster_stacks_remote_api.py @@ -0,0 +1,179 @@ +import os +import yaml +import subprocess +import base64 +import time +import logging +from interface import KubernetesClusterPlugin + +logger = logging.getLogger("PluginClusterStacks") + +# Helper functions +def wait_for_pods(self, namespaces, timeout=240, interval=15, kubeconfig=None): + """ + Waits for all pods in specified namespaces to reach the condition 'Ready'. + + :param namespaces: List of namespaces to check for pod readiness. + :param timeout: Total time to wait in seconds before giving up. + :param interval: Time to wait between checks in seconds. + :param kubeconfig: Optional path to the kubeconfig file for the target Kubernetes cluster. + :return: True if all pods are ready within the given timeout, raises TimeoutError otherwise. + """ + start_time = time.time() + + while time.time() - start_time < timeout: + all_pods_ready = True + + for namespace in namespaces: + try: + # Get pod status in the namespace + wait_pods_command = ( + f"kubectl wait -n {namespace} --for=condition=Ready --timeout={timeout}s pod --all" + ) + result = self._run_subprocess(wait_pods_command, f"Error fetching pods in {namespace}", shell=True, capture_output=True, text=True, kubeconfig=kubeconfig) + + if result.returncode != 0: + logger.warning(f"Not all pods in namespace {namespace} are ready. Details: {result.stderr}") + all_pods_ready = False + else: + logger.info(f"All pods in namespace {namespace} are ready.") + + except subprocess.CalledProcessError as error: + logger.error(f"Error checking pods in {namespace}: {error}") + all_pods_ready = False + + if all_pods_ready: + logger.info("All specified pods are ready in all namespaces.") + return True + + logger.info("Waiting for all pods in specified namespaces to become ready...") + time.sleep(interval) + + raise TimeoutError(f"Timed out after {timeout} seconds waiting for pods in namespaces {namespaces} to become ready.") + + +def load_config(config_path): + """ + Loads the configuration from a YAML file. + """ + + with open(config_path, 'r') as file: + config = yaml.safe_load(file) or {} + return config + +class PluginClusterStacksRemoteAPI(KubernetesClusterPlugin): + def __init__(self, config_file=None): + self.config = load_config(config_file) if config_file else {} + logger.debug(self.config) + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") + self.kubeconfig_mgmnt = self.config['kubeconfig'] + self.workloadclusters = self.config['workloadcluster'] + self.cs_namespace = self.config['namespace'] + + + def create_cluster(self, cluster_name=None, version=None, kubeconfig_filepath=None): + self.cluster_name = cluster_name + self.cluster_version = version + self.kubeconfig_cs_cluster = kubeconfig_filepath + + # Create workload cluster + self._apply_yaml(self.workloadclusters, "Error applying cluster.yaml", kubeconfig=self.kubeconfig_mgmnt) + + #TODO:!!! We also need to introduce a waiting function here + + print("retrieve kubeconfig to path") + self._retrieve_kubeconfig(namespace=self.cs_namespace, kubeconfig=self.kubeconfig_mgmnt) + + # Wait for workload system pods to be ready + # wait_for_workload_pods_ready(kubeconfig_path=self.kubeconfig_cs_cluster) + # ~ wait_for_pods(self, ["kube-system"], timeout=600, interval=15, kubeconfig=self.kubeconfig_cs_cluster) + + + def delete_cluster(self, cluster_name=None): #TODO:!!! need to adjust delete method + self.cluster_name = cluster_name + #Get the name of the workloadcluster from the config file + workload_cluster_config = load_config(self.workloadclusters) + workload_cluster_name = workload_cluster_config['metadata']['name'] + try: + # Check if the cluster exists + check_cluster_command = f"kubectl --kubeconfig={self.kubeconfig_mgmnt} get cluster {workload_cluster_name} -n {self.cs_namespace}" + result = self._run_subprocess(check_cluster_command, "Failed to get cluster resource", shell=True, capture_output=True, text=True) + + # Proceed with deletion only if the cluster exists + if result.returncode == 0: + delete_command = f"kubectl --kubeconfig={self.kubeconfig_mgmnt} delete cluster {workload_cluster_name} --timeout=600s -n {self.cs_namespace}" + self._run_subprocess(delete_command, "Timeout while deleting the cluster", shell=True) + + except subprocess.CalledProcessError as error: + if "NotFound" in error.stderr: + logger.info(f"Cluster {workload_cluster_name} not found. Skipping deletion.") + else: + raise RuntimeError(f"Error checking for cluster existence: {error}") + + + def _apply_yaml(self, yaml_file, error_msg, kubeconfig=None): + """ + Applies a Kubernetes YAML configuration file to the cluster, substituting environment variables as needed. + + :param yaml_file: The name of the YAML file to apply. + :param kubeconfig: Optional path to a kubeconfig file, which specifies which Kubernetes cluster + to apply the YAML configuration to. + """ + try: + # Determine if the file is a local path or a URL + if os.path.isfile(yaml_file): + command = f"kubectl --kubeconfig={self.kubeconfig_mgmnt} apply -f {yaml_file} -n {self.cs_namespace}" + else: + raise ValueError(f"Unknown file: {yaml_file}") + + self._run_subprocess(command, error_msg, shell=True) + + except subprocess.CalledProcessError as error: + raise RuntimeError(f"{error_msg}: {error}") + + + def _retrieve_kubeconfig(self, namespace="default", kubeconfig=None): + """ + Retrieves the kubeconfig for the specified cluster and saves it to a local file. + + :param namespace: The namespace of the cluster to retrieve the kubeconfig for. + :param kubeconfig: Optional path to the kubeconfig file for the target Kubernetes cluster. + """ + + #Get the name of the workloadcluster from the config file + workload_cluster_config = load_config(self.workloadclusters) + workload_cluster_name = workload_cluster_config['metadata']['name'] + + command_args = [ + "kubectl ", + f"--kubeconfig={self.kubeconfig_mgmnt}", + f"-n {self.cs_namespace}", + f"get secret {workload_cluster_name}-kubeconfig", + "-o go-template='{{.data.value|base64decode}}'", + f"> {self.kubeconfig_cs_cluster}", + ] + kubeconfig_command = "" + for entry in command_args: + kubeconfig_command += entry + " " + self._run_subprocess(kubeconfig_command, "Error retrieving kubeconfig", shell=True) + + + def _run_subprocess(self, command, error_msg, shell=False, capture_output=False, text=False): + """ + Executes a subprocess command with the specified environment variables and parameters. + + :param command: The shell command to be executed. This can be a string or a list of arguments to pass to the subprocess. + :param error_msg: A custom error message to be logged and raised if the subprocess fails. + :param shell: Whether to execute the command through the shell (default: `False`). + :param capture_output: Whether to capture the command's standard output and standard error (default: `False`). + :param text: Whether to treat the command's output and error as text (default: `False`). + :return: The result of the `subprocess.run` command + """ + try: + # Run the subprocess + result = subprocess.run(command, shell=shell, capture_output=capture_output, text=text, check=True) + return result + except subprocess.CalledProcessError as error: + logger.error(f"{error_msg}: {error}") + raise diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 5595c8994..3b12155d2 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -8,11 +8,13 @@ from plugin_kind import PluginKind from plugin_static import PluginStatic from plugin_cluster_stacks import PluginClusterStacks +from plugin_cluster_stacks_remote_api import PluginClusterStacksRemoteAPI PLUGIN_LOOKUP = { "kind": PluginKind, "static": PluginStatic, "cluster-stacks": PluginClusterStacks, + "cluster-stacks-api": PluginClusterStacksRemoteAPI, } @@ -56,7 +58,7 @@ def delete(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterinfo = clusterspec[clusterspec_cluster] cluster_id = clusterspec_cluster plugin = init_plugin(plugin_kind, plugin_config) - plugin.delete_cluster(cluster_id, os.path.abspath(clusterinfo['kubeconfig'])) + plugin.delete_cluster(cluster_id) if __name__ == '__main__': diff --git a/playbooks/k8s_configs/moin_cluster_config.yaml b/playbooks/k8s_configs/moin_cluster_config.yaml new file mode 100644 index 000000000..22189ee62 --- /dev/null +++ b/playbooks/k8s_configs/moin_cluster_config.yaml @@ -0,0 +1,3 @@ +kubeconfig: "../../playbooks/k8s_configs/moin_cluster_kubeconfig.yaml" +workloadcluster: "../../playbooks/k8s_configs/moin_cluster_workloadcluster.yaml" +namespace: "kaas-playground1" diff --git a/playbooks/k8s_configs/moin_cluster_kubeconfig.yaml b/playbooks/k8s_configs/moin_cluster_kubeconfig.yaml new file mode 100644 index 000000000..235fa232c --- /dev/null +++ b/playbooks/k8s_configs/moin_cluster_kubeconfig.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +clusters: +- cluster: + server: https://moin.k8s.scs.community + name: moin-cluster +contexts: +- context: + cluster: moin-cluster + user: oidc + name: moin-cluster +current-context: moin-cluster +kind: Config +users: +- name: oidc + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + args: + - oidc-login + - get-token + - --oidc-issuer-url=https://dex.k8s.scs.community + - --oidc-client-id=kubectl + - --oidc-extra-scope=groups,profile + command: kubectl diff --git a/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml b/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml new file mode 100644 index 000000000..41fdadc48 --- /dev/null +++ b/playbooks/k8s_configs/moin_cluster_workloadcluster.yaml @@ -0,0 +1,18 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: scs-cluster-129 + labels: + managed-secret: cloud-config +spec: + topology: + variables: + class: openstack-scs-1-29-v4 + controlPlane: + replicas: 3 + version: v1.29.10 + workers: + machineDeployments: + - class: default-worker + name: md-0 + replicas: 3