diff --git a/config/samples/onpremises/clusters_v1beta1_cassandra.yaml b/config/samples/onpremises/clusters_v1beta1_cassandra.yaml index 8b7968872..d33082acf 100644 --- a/config/samples/onpremises/clusters_v1beta1_cassandra.yaml +++ b/config/samples/onpremises/clusters_v1beta1_cassandra.yaml @@ -1,12 +1,15 @@ apiVersion: clusters.instaclustr.com/v1beta1 kind: Cassandra metadata: - name: cassandra-on-prem-cluster + name: cassandra-on-prem-cluster-test spec: - name: "danylo-on-prem-cassandra" + name: "rostyslp-on-prem-cassandra" version: "4.0.10" privateNetworkCluster: false onPremisesSpec: + cloudInitScriptRef: + name: instaclustr-cloud-init-secret + namespace: default storageClassName: managed-csi-premium osDiskSize: 20Gi dataDiskSize: 200Gi @@ -15,9 +18,6 @@ spec: nodeCPU: 2 nodeMemory: 8192Mi osImageURL: "https://s3.amazonaws.com/debian-bucket/debian-11-generic-amd64-20230601-1398.raw" - cloudInitScriptRef: - namespace: default - name: instaclustr-cloud-init-secret dataCentres: - name: "onPremCassandra" region: "CLIENT_DC" diff --git a/controllers/clusters/cassandra_controller.go b/controllers/clusters/cassandra_controller.go index a1230b7b7..fd3d41f51 100644 --- a/controllers/clusters/cassandra_controller.go +++ b/controllers/clusters/cassandra_controller.go @@ -800,7 +800,58 @@ func (r *CassandraReconciler) newWatchStatusJob(c *v1beta1.Cassandra) scheduler. return err } + // TODO move everything not related to the watch status job to other places if !areStatusesEqual(&iCassandra.Status.ClusterStatus, &c.Status.ClusterStatus) { + if len(c.Status.DataCentres) != 0 && + len(iCassandra.Status.DataCentres) != 0 { + bootstrap := newOnPremisesBootstrap( + r.Client, + c, + r.EventRecorder, + iCassandra.Status.ClusterStatus, + c.Spec.OnPremisesSpec, + newExposePorts(c.GetExposePorts()), + c.GetHeadlessPorts(), + c.Spec.PrivateNetworkCluster, + ) + newNode, oldNode := getReplacedNodes(c.Status.DataCentres[0], iCassandra.Status.DataCentres[0]) + if oldNode != nil { + deleteUsed, err := deleteReplacedNodeResources(context.TODO(), r.Client, oldNode.ID) + if err != nil { + l.Error(err, "Cannot delete replaced node resources", + "old node id", oldNode.ID) + + return err + } + + if deleteUsed { + l.Info("Node replace is in progress. Deleting old vm resources", + "cluster name", c.Name, + "old id", oldNode.ID) + + return nil + } + + l.Info("Node replace is in progress. Creating new resources", + "bootstrap", bootstrap, + "new node", newNode) + + err = handleNodeReplace( + context.TODO(), + bootstrap, + oldNode, + newNode) + if err != nil { + l.Error(err, "Error while handling node replace operation", + "bootstrap struct", bootstrap, + "old node", oldNode, + "new node", newNode) + + return err + } + } + } + l.Info("Updating cluster status", "status from Instaclustr", iCassandra.Status.ClusterStatus, "status from k8s", c.Status.ClusterStatus) diff --git a/controllers/clusters/on_premises.go b/controllers/clusters/on_premises.go index 18fefaf67..49e514ded 100644 --- a/controllers/clusters/on_premises.go +++ b/controllers/clusters/on_premises.go @@ -22,7 +22,6 @@ import ( "strings" k8scorev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -93,102 +92,134 @@ func handleCreateOnPremisesClusterResources(ctx context.Context, b *onPremisesBo return nil } +func handleNodeReplace(ctx context.Context, b *onPremisesBootstrap, newNode, oldNode *v1beta1.Node) error { + objectName := strings.ToLower(b.K8sObject.GetName()) + vm := &virtcorev1.VirtualMachine{} + + for i := 0; i < 20; i++ { + nodeName := fmt.Sprintf("%s-%d-%s", models.NodeVMPrefix, i, objectName) + err := b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: nodeName, + }, vm) + if client.IgnoreNotFound(err) != nil { + return err + } + if err == nil { + continue + } + + osDVName := fmt.Sprintf("%s-%d-%s", models.NodeOSDVPrefix, i, objectName) + osDV, err := reconcileDV(ctx, b, newNode.ID, osDVName, models.OSDVLabel) + if err != nil { + return err + } + + storageDVList := &cdiv1beta1.DataVolumeList{} + err = b.K8sClient.List(ctx, storageDVList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.NodeIDLabel: oldNode.ID, + models.DVRoleLabel: models.StorageDVLabel, + }), + }) + if err != nil { + return err + } + + storageDV := &cdiv1beta1.DataVolume{} + storageDVName := fmt.Sprintf("%s-%d-%s", models.NodeDVPrefix, i, objectName) + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: storageDVName, + }, storageDV) + if err != nil { + return err + } + + patch := client.MergeFrom(storageDV.DeepCopy()) + storageDV.Labels[models.NodeIDLabel] = newNode.ID + err = b.K8sClient.Patch(ctx, storageDV, patch) + if err != nil { + return err + } + + vm, err = reconcileVM(ctx, b, newNode.ID, nodeName, newNode.Rack, osDV.Name, storageDV.Name) + if err != nil { + return err + } + + return nil + } + + return models.ErrGenerateAvailableVMNameFailed +} + func reconcileSSHGatewayResources(ctx context.Context, b *onPremisesBootstrap) error { - gatewayDVSize, err := resource.ParseQuantity(b.OnPremisesSpec.OSDiskSize) + gatewayDVName := fmt.Sprintf("%s-%s", models.GatewayDVPrefix, strings.ToLower(b.K8sObject.GetName())) + gatewayDV, err := reconcileDV(ctx, b, b.ClusterStatus.DataCentres[0].ID, gatewayDVName, models.OSDVLabel) if err != nil { return err } - gatewayDVName := fmt.Sprintf("%s-%s", models.GatewayDVPrefix, strings.ToLower(b.K8sObject.GetName())) - gatewayDV, err := createDV( + gatewayName := fmt.Sprintf("%s-%s", models.GatewayVMPrefix, strings.ToLower(b.K8sObject.GetName())) + gatewayVM, err := reconcileVM( ctx, b, - gatewayDVName, b.ClusterStatus.DataCentres[0].ID, - gatewayDVSize, - true, - ) + gatewayName, + models.GatewayRack, + gatewayDV.Name) if err != nil { return err } - gatewayCPU := resource.Quantity{} - gatewayCPU.Set(b.OnPremisesSpec.SSHGatewayCPU) - - gatewayMemory, err := resource.ParseQuantity(b.OnPremisesSpec.SSHGatewayMemory) + _, err = reconcileService(ctx, b, b.ClusterStatus.DataCentres[0].ID, gatewayVM.Name, models.ExposeServiceRoleLabel) if err != nil { return err } - gatewayName := fmt.Sprintf("%s-%s", models.GatewayVMPrefix, strings.ToLower(b.K8sObject.GetName())) - - gatewayVM := &virtcorev1.VirtualMachine{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: gatewayName, - }, gatewayVM) - if client.IgnoreNotFound(err) != nil { + _, err = reconcileService(ctx, b, b.ClusterStatus.DataCentres[0].ID, gatewayVM.Name, models.ClusterIPServiceRoleLabel) + if err != nil { return err } - if k8serrors.IsNotFound(err) { - gatewayVM, err = newVM( - ctx, - b, - gatewayName, - b.ClusterStatus.DataCentres[0].ID, - models.GatewayRack, - gatewayDV.Name, - gatewayCPU, - gatewayMemory) + + return nil +} + +func reconcileNodesResources(ctx context.Context, b *onPremisesBootstrap) error { + for i, node := range b.ClusterStatus.DataCentres[0].Nodes { + objectName := strings.ToLower(b.K8sObject.GetName()) + newOSDVName := fmt.Sprintf("%s-%d-%s", models.NodeOSDVPrefix, i, objectName) + nodeOSDV, err := reconcileDV(ctx, b, node.ID, newOSDVName, models.OSDVLabel) if err != nil { return err } - err = b.K8sClient.Create(ctx, gatewayVM) + + nodeDataDiskDVName := fmt.Sprintf("%s-%d-%s", models.NodeDVPrefix, i, objectName) + nodeDataDV, err := reconcileDV(ctx, b, node.ID, nodeDataDiskDVName, models.StorageDVLabel) if err != nil { return err } - } - gatewaySvcName := fmt.Sprintf("%s-%s", models.GatewaySvcPrefix, gatewayName) - gatewayExposeService := &k8scorev1.Service{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: gatewaySvcName, - }, gatewayExposeService) + nodeName := fmt.Sprintf("%s-%d-%s", models.NodeVMPrefix, i, objectName) + nodeVM, err := reconcileVM(ctx, b, node.ID, nodeName, node.Rack, nodeOSDV.Name, nodeDataDV.Name) + if err != nil { + return err + } - if client.IgnoreNotFound(err) != nil { - return err - } - if k8serrors.IsNotFound(err) { - gatewayExposeService = newExposeService( - b, - gatewaySvcName, - gatewayName, - b.ClusterStatus.DataCentres[0].ID, - ) - err = b.K8sClient.Create(ctx, gatewayExposeService) + if !b.PrivateNetworkCluster { + _, err = reconcileService(ctx, b, node.ID, nodeVM.Name, models.ExposeServiceRoleLabel) + if err != nil { + return err + } + } + + _, err = reconcileService(ctx, b, node.ID, nodeVM.Name, models.HeadlessServiceRoleLabel) if err != nil { return err } - } - clusterIPServiceName := fmt.Sprintf("cluster-ip-%s", gatewayName) - nodeExposeService := &k8scorev1.Service{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: clusterIPServiceName, - }, nodeExposeService) - if client.IgnoreNotFound(err) != nil { - return err - } - if k8serrors.IsNotFound(err) { - nodeExposeService = newClusterIPService( - b, - clusterIPServiceName, - gatewayName, - b.ClusterStatus.DataCentres[0].ID, - ) - err = b.K8sClient.Create(ctx, nodeExposeService) + _, err = reconcileService(ctx, b, node.ID, nodeVM.Name, models.ClusterIPServiceRoleLabel) if err != nil { return err } @@ -197,198 +228,220 @@ func reconcileSSHGatewayResources(ctx context.Context, b *onPremisesBootstrap) e return nil } -func reconcileNodesResources(ctx context.Context, b *onPremisesBootstrap) error { - for i, node := range b.ClusterStatus.DataCentres[0].Nodes { - nodeOSDiskSize, err := resource.ParseQuantity(b.OnPremisesSpec.OSDiskSize) +func reconcileVM( + ctx context.Context, + b *onPremisesBootstrap, + nodeID, + nodeName, + rack, + osDVName string, + storageDVName ...string) (*virtcorev1.VirtualMachine, error) { + nodeCPU := resource.Quantity{} + nodeMemory := resource.Quantity{} + var err error + + switch rack { + case models.GatewayRack: + nodeCPU.Set(b.OnPremisesSpec.SSHGatewayCPU) + nodeMemory, err = resource.ParseQuantity(b.OnPremisesSpec.SSHGatewayMemory) if err != nil { - return err + return nil, err } - - nodeOSDiskDVName := fmt.Sprintf("%s-%d-%s", models.NodeOSDVPrefix, i, strings.ToLower(b.K8sObject.GetName())) - nodeOSDV, err := createDV( - ctx, - b, - nodeOSDiskDVName, - node.ID, - nodeOSDiskSize, - true, - ) + default: + nodeCPU.Set(b.OnPremisesSpec.NodeCPU) + nodeMemory, err = resource.ParseQuantity(b.OnPremisesSpec.NodeMemory) if err != nil { - return err + return nil, err } + } - nodeDataDiskDVSize, err := resource.ParseQuantity(b.OnPremisesSpec.DataDiskSize) - if err != nil { - return err - } + vmList := &virtcorev1.VirtualMachineList{} + err = b.K8sClient.List(ctx, vmList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.NodeIDLabel: nodeID, + }), + }) - nodeDataDiskDVName := fmt.Sprintf("%s-%d-%s", models.NodeDVPrefix, i, strings.ToLower(b.K8sObject.GetName())) - nodeDataDV, err := createDV( + vm := &virtcorev1.VirtualMachine{} + if len(vmList.Items) == 0 { + vm, err = newVM( ctx, b, - nodeDataDiskDVName, - node.ID, - nodeDataDiskDVSize, - false, + nodeName, + nodeID, + rack, + osDVName, + nodeCPU, + nodeMemory, + storageDVName..., ) if err != nil { - return err + return nil, err } - nodeCPU := resource.Quantity{} - nodeCPU.Set(b.OnPremisesSpec.NodeCPU) - - nodeMemory, err := resource.ParseQuantity(b.OnPremisesSpec.NodeMemory) + err = b.K8sClient.Create(ctx, vm) if err != nil { - return err + + return nil, err } - nodeName := fmt.Sprintf("%s-%d-%s", models.NodeVMPrefix, i, strings.ToLower(b.K8sObject.GetName())) + return vm, nil + } - nodeVM := &virtcorev1.VirtualMachine{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: nodeName, - }, nodeVM) - if client.IgnoreNotFound(err) != nil { - return err - } - if k8serrors.IsNotFound(err) { - nodeVM, err = newVM( - ctx, - b, - nodeName, - node.ID, - node.Rack, - nodeOSDV.Name, - nodeCPU, - nodeMemory, - nodeDataDV.Name, - ) + return &vmList.Items[0], nil +} + +func reconcileDV(ctx context.Context, b *onPremisesBootstrap, nodeID, diskName, diskRole string) (*cdiv1beta1.DataVolume, error) { + dvList := &cdiv1beta1.DataVolumeList{} + err := b.K8sClient.List(ctx, dvList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.NodeIDLabel: nodeID, + models.DVRoleLabel: diskRole, + }), + }) + if err != nil { + return nil, err + } + + dv := &cdiv1beta1.DataVolume{} + if len(dvList.Items) == 0 { + dvSize := resource.Quantity{} + switch diskRole { + case models.OSDVLabel: + dvSize, err = resource.ParseQuantity(b.OnPremisesSpec.OSDiskSize) if err != nil { - return err + return nil, err } - err = b.K8sClient.Create(ctx, nodeVM) + case models.StorageDVLabel: + dvSize, err = resource.ParseQuantity(b.OnPremisesSpec.DataDiskSize) if err != nil { - return err + return nil, err } } - if !b.PrivateNetworkCluster { - nodeExposeName := fmt.Sprintf("%s-%s", models.NodeSvcPrefix, nodeName) - nodeExposeService := &k8scorev1.Service{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: nodeExposeName, - }, nodeExposeService) - if client.IgnoreNotFound(err) != nil { - return err - } - if k8serrors.IsNotFound(err) { - nodeExposeService = newExposeService( - b, - nodeExposeName, - nodeName, - node.ID, - ) - err = b.K8sClient.Create(ctx, nodeExposeService) - if err != nil { - return err - } - } + dv, err = createDV( + ctx, + b, + diskName, + nodeID, + diskRole, + dvSize, + ) + if err != nil { + return nil, err } - headlessServiceName := fmt.Sprintf("%s-%s", models.KubevirtSubdomain, strings.ToLower(b.K8sObject.GetName())) - headlessSVC := &k8scorev1.Service{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: headlessServiceName, - }, headlessSVC) + return dv, nil + } - if client.IgnoreNotFound(err) != nil { - return err - } - if k8serrors.IsNotFound(err) { - headlessSVC = newHeadlessService( + return &dvList.Items[0], nil +} + +func reconcileService(ctx context.Context, b *onPremisesBootstrap, nodeID, nodeName, role string) (*k8scorev1.Service, error) { + svcList := &k8scorev1.ServiceList{} + svcLabels := map[string]string{ + models.ServiceRoleLabel: role, + models.ClusterIDLabel: b.ClusterStatus.ID, + } + + var svcName string + if role == models.HeadlessServiceRoleLabel { + svcName = fmt.Sprintf("%s-%s", role, strings.ToLower(b.K8sObject.GetName())) + } else { + svcName = fmt.Sprintf("%s-%s", role, nodeName) + svcLabels[models.NodeIDLabel] = nodeID + } + + err := b.K8sClient.List(ctx, svcList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(svcLabels), + }) + if err != nil { + return nil, err + } + + svc := &k8scorev1.Service{} + if len(svcList.Items) == 0 { + switch role { + case models.ExposeServiceRoleLabel: + svc = newExposeService( b, - headlessServiceName, + svcName, + nodeName, + nodeID, ) - err = b.K8sClient.Create(ctx, headlessSVC) - if err != nil { - return err - } - } - - clusterIPServiceName := fmt.Sprintf("cluster-ip-%s", nodeName) - nodeExposeService := &k8scorev1.Service{} - err = b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: clusterIPServiceName, - }, nodeExposeService) - if client.IgnoreNotFound(err) != nil { - return err - } - if k8serrors.IsNotFound(err) { - nodeExposeService = newClusterIPService( + case models.HeadlessServiceRoleLabel: + svc = newHeadlessService( b, - clusterIPServiceName, + svcName, + ) + case models.ClusterIPServiceRoleLabel: + svc = newClusterIPService( + b, + svcName, nodeName, - node.ID, + nodeID, ) - err = b.K8sClient.Create(ctx, nodeExposeService) - if err != nil { - return err - } + } + err = b.K8sClient.Create(ctx, svc) + if err != nil { + return nil, err } + return svc, nil } - return nil + + err = b.K8sClient.Get(ctx, types.NamespacedName{ + Namespace: b.K8sObject.GetNamespace(), + Name: svcName, + }, svc) + if err != nil { + return nil, err + } + + return svc, nil } func createDV( ctx context.Context, b *onPremisesBootstrap, name, - nodeID string, + nodeID, + diskRole string, size resource.Quantity, - isOSDisk bool, ) (*cdiv1beta1.DataVolume, error) { - dv := &cdiv1beta1.DataVolume{} - err := b.K8sClient.Get(ctx, types.NamespacedName{ - Namespace: b.K8sObject.GetNamespace(), - Name: name, - }, dv) - if client.IgnoreNotFound(err) != nil { + dv := newDataDiskDV( + b, + name, + nodeID, + diskRole, + size, + ) + err := b.K8sClient.Create(ctx, dv) + if err != nil { return nil, err } - if k8serrors.IsNotFound(err) { - dv = newDataDiskDV( - b, - name, - nodeID, - size, - isOSDisk, - ) - err = b.K8sClient.Create(ctx, dv) - if err != nil { - return nil, err - } - } + return dv, nil } func newDataDiskDV( b *onPremisesBootstrap, name, - nodeID string, + nodeID, + diskRole string, storageSize resource.Quantity, - isOSDisk bool, ) *cdiv1beta1.DataVolume { dvSource := &cdiv1beta1.DataVolumeSource{} + dvLabels := map[string]string{ + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + models.DVRoleLabel: diskRole, + } - if isOSDisk { + switch diskRole { + case models.OSDVLabel: dvSource.HTTP = &cdiv1beta1.DataVolumeSourceHTTP{URL: b.OnPremisesSpec.OSImageURL} - } else { + case models.StorageDVLabel: dvSource.Blank = &cdiv1beta1.DataVolumeBlankImage{} } @@ -398,12 +451,9 @@ func newDataDiskDV( APIVersion: models.CDIKubevirtV1beta1APIVersion, }, ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: b.K8sObject.GetNamespace(), - Labels: map[string]string{ - models.ClusterIDLabel: b.ClusterStatus.ID, - models.NodeIDLabel: nodeID, - }, + Name: name, + Namespace: b.K8sObject.GetNamespace(), + Labels: dvLabels, Finalizers: []string{models.DeletionFinalizer}, }, Spec: cdiv1beta1.DataVolumeSpec{ @@ -517,10 +567,8 @@ func newVM( { Name: models.Boot, VolumeSource: virtcorev1.VolumeSource{ - PersistentVolumeClaim: &virtcorev1.PersistentVolumeClaimVolumeSource{ - PersistentVolumeClaimVolumeSource: k8scorev1.PersistentVolumeClaimVolumeSource{ - ClaimName: OSDiskDVName, - }, + DataVolume: &virtcorev1.DataVolumeSource{ + Name: OSDiskDVName, }, }, }, @@ -566,10 +614,8 @@ func newVM( vm.Spec.Template.Spec.Volumes = append(vm.Spec.Template.Spec.Volumes, virtcorev1.Volume{ Name: diskName, VolumeSource: virtcorev1.VolumeSource{ - PersistentVolumeClaim: &virtcorev1.PersistentVolumeClaimVolumeSource{ - PersistentVolumeClaimVolumeSource: k8scorev1.PersistentVolumeClaimVolumeSource{ - ClaimName: dvName, - }, + DataVolume: &virtcorev1.DataVolumeSource{ + Name: dvName, }, }, }) @@ -593,8 +639,9 @@ func newExposeService( Name: svcName, Namespace: b.K8sObject.GetNamespace(), Labels: map[string]string{ - models.ClusterIDLabel: b.ClusterStatus.ID, - models.NodeIDLabel: nodeID, + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + models.ServiceRoleLabel: models.ExposeServiceRoleLabel, }, Finalizers: []string{models.DeletionFinalizer}, }, @@ -622,7 +669,8 @@ func newHeadlessService( Name: svcName, Namespace: b.K8sObject.GetNamespace(), Labels: map[string]string{ - models.ClusterIDLabel: b.ClusterStatus.ID, + models.ClusterIDLabel: b.ClusterStatus.ID, + models.ServiceRoleLabel: models.HeadlessServiceRoleLabel, }, Finalizers: []string{models.DeletionFinalizer}, }, @@ -837,7 +885,7 @@ func newWatchOnPremisesIPsJob(kind string, b *onPremisesBootstrap) scheduler.Job b.EventRecorder.Eventf( b.K8sObject, models.Warning, models.CreationFailed, - "The private IP addresses of the node are not matching. Reason: %v", + "The private IP addresses of the node are not matching. Contact Instaclustr support to resolve the issue. Reason: %v", err, ) return err @@ -882,7 +930,7 @@ func newWatchOnPremisesIPsJob(kind string, b *onPremisesBootstrap) scheduler.Job b.EventRecorder.Eventf( b.K8sObject, models.Warning, models.CreationFailed, - "The public IP addresses of the node are not matching. Reason: %v", + "The public IP addresses of the node are not matching. Contact Instaclustr support to resolve the issue. Reason: %v", err, ) return err @@ -910,8 +958,9 @@ func newClusterIPService( Name: svcName, Namespace: b.K8sObject.GetNamespace(), Labels: map[string]string{ - models.ClusterIDLabel: b.ClusterStatus.ID, - models.NodeIDLabel: nodeID, + models.ClusterIDLabel: b.ClusterStatus.ID, + models.NodeIDLabel: nodeID, + models.ServiceRoleLabel: models.ClusterIPServiceRoleLabel, }, Finalizers: []string{models.DeletionFinalizer}, }, @@ -925,3 +974,104 @@ func newClusterIPService( }, } } + +func getReplacedNodes(oldNodesDC, newNodesDC *v1beta1.DataCentreStatus) (newNode, oldNode *v1beta1.Node) { + for _, aNode := range oldNodesDC.Nodes { + for _, bNode := range newNodesDC.Nodes { + if aNode.ID == bNode.ID { + oldNode = nil + newNode = nil + break + } + oldNode = aNode + newNode = bNode + } + + if oldNode != nil { + return + } + } + + return nil, nil +} + +func deleteReplacedNodeResources(ctx context.Context, k8sClient client.Client, oldID string) (deleteUsed bool, err error) { + vmList := &virtcorev1.VirtualMachineList{} + err = k8sClient.List(ctx, vmList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.NodeIDLabel: oldID, + }), + }) + if err != nil { + return false, err + } + + for _, vm := range vmList.Items { + deleteUsed = true + err = k8sClient.Delete(ctx, &vm) + if err != nil { + return false, err + } + + patch := client.MergeFrom(vm.DeepCopy()) + controllerutil.RemoveFinalizer(&vm, models.DeletionFinalizer) + err = k8sClient.Patch(ctx, &vm, patch) + if err != nil { + return false, err + } + } + + dvList := &cdiv1beta1.DataVolumeList{} + err = k8sClient.List(context.TODO(), dvList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.NodeIDLabel: oldID, + models.DVRoleLabel: models.OSDVLabel, + }), + }) + if err != nil { + return false, err + } + + for _, dv := range dvList.Items { + deleteUsed = true + err = k8sClient.Delete(ctx, &dv) + if err != nil { + return false, err + } + + patch := client.MergeFrom(dv.DeepCopy()) + controllerutil.RemoveFinalizer(&dv, models.DeletionFinalizer) + err = k8sClient.Patch(ctx, &dv, patch) + if err != nil { + return false, err + } + } + + svcList := &k8scorev1.ServiceList{} + err = k8sClient.List(context.TODO(), dvList, &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + models.NodeIDLabel: oldID, + models.ServiceRoleLabel: models.ClusterIPServiceRoleLabel, + }), + }) + if err != nil { + return false, err + } + + for _, svc := range svcList.Items { + deleteUsed = true + err = k8sClient.Delete(ctx, &svc) + if err != nil { + return false, err + } + + patch := client.MergeFrom(svc.DeepCopy()) + controllerutil.RemoveFinalizer(&svc, models.DeletionFinalizer) + err = k8sClient.Patch(ctx, &svc, patch) + if err != nil { + return false, err + } + } + + return deleteUsed, nil +} diff --git a/pkg/models/errors.go b/pkg/models/errors.go index a86d5ca0f..09e49b284 100644 --- a/pkg/models/errors.go +++ b/pkg/models/errors.go @@ -70,4 +70,5 @@ var ( ErrDebeziumImmutable = errors.New("debezium array is immutable") ErrCreateClusterWithMultiDC = errors.New("Multiple data center is still not supported. Please create a cluster with one data centre and add a second one when the cluster is in the running state") ErrOnPremicesWithMultiDC = errors.New("on-premises cluster can be provisioned with only one data centre") + ErrGenerateAvailableVMNameFailed = errors.New("failed to generate available VM name") ) diff --git a/pkg/models/on_premises.go b/pkg/models/on_premises.go index 7766ae2ca..e85bed65d 100644 --- a/pkg/models/on_premises.go +++ b/pkg/models/on_premises.go @@ -10,22 +10,30 @@ const ( KubevirtV1APIVersion = "kubevirt.io/v1" CDIKubevirtV1beta1APIVersion = "cdi.kubevirt.io/v1beta1" - KubevirtSubdomain = "kubevirt" - KubevirtDomainLabel = "kubevirt.io/domain" - NodeIDLabel = "nodeID" - NodeRackLabel = "nodeRack" - NodeLabel = "node" - NodeOSDVPrefix = "node-os-data-volume-pvc" - NodeDVPrefix = "node-data-volume-pvc" - NodeVMPrefix = "node-vm" - NodeSvcPrefix = "node-service" - WorkerNode = "worker-node" - GatewayDVPrefix = "gateway-data-volume-pvc" - GatewayVMPrefix = "gateway-vm" - GatewaySvcPrefix = "gateway-service" - GatewayRack = "ssh-gateway-rack" - IgnitionScriptSecretPrefix = "ignition-script-secret" - DataDisk = "data-disk" + KubevirtSubdomain = "kubevirt" + KubevirtDomainLabel = "kubevirt.io/domain" + NodeReplaceActiveAnnotation = "instaclustr.com/node-replace-active" + NodeIDLabel = "nodeID" + NodeRackLabel = "nodeRack" + NodeLabel = "node" + ServiceRoleLabel = "service-role" + ExposeServiceRoleLabel = "expose" + HeadlessServiceRoleLabel = "headless" + ClusterIPServiceRoleLabel = "cluster-ip" + DVRoleLabel = "role" + OSDVLabel = "osdv" + StorageDVLabel = "storagedv" + NodeOSDVPrefix = "os-volume" + NodeDVPrefix = "data-volume" + NodeVMPrefix = "node-vm" + NodeSvcPrefix = "node-service" + WorkerNode = "worker-node" + GatewayDVPrefix = "gateway-data-volume-pvc" + GatewayVMPrefix = "gateway-vm" + GatewaySvcPrefix = "gateway-service" + GatewayRack = "ssh-gateway-rack" + IgnitionScriptSecretPrefix = "ignition-script-secret" + DataDisk = "data-disk" Boot = "boot" Storage = "storage"