From 39c077d7600ac0393daea4a602e1bd8b8e6800fd Mon Sep 17 00:00:00 2001 From: oraz Date: Mon, 30 Oct 2023 12:13:23 +0200 Subject: [PATCH 1/2] Delete pods to acclerate pods rescheduling SNR and FAR can use DeletePods function to fully remediate a node after a reboot has finished --- pkg/resources/resources.go | 52 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 pkg/resources/resources.go diff --git a/pkg/resources/resources.go b/pkg/resources/resources.go new file mode 100644 index 0000000..4d34f31 --- /dev/null +++ b/pkg/resources/resources.go @@ -0,0 +1,52 @@ +package resources + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// DeletePods deletes all the pods from the unhealthy node +func DeletePods(ctx context.Context, r client.Client, nodeName string) error { + log := ctrl.Log.WithName("commons-resource") + zero := int64(0) + backgroundDeletePolicy := metav1.DeletePropagationBackground + + deleteOptions := &client.DeleteAllOfOptions{ + ListOptions: client.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), + Namespace: "", + Limit: 0, + }, + DeleteOptions: client.DeleteOptions{ + GracePeriodSeconds: &zero, + PropagationPolicy: &backgroundDeletePolicy, + }, + } + + namespaces := corev1.NamespaceList{} + if err := r.List(ctx, &namespaces); err != nil { + log.Error(err, "failed to list namespaces") + return err + } + + log.Info("starting to delete pods", "node name", nodeName) + + pod := &corev1.Pod{} + for _, ns := range namespaces.Items { + deleteOptions.Namespace = ns.Name + err := r.DeleteAllOf(ctx, pod, deleteOptions) + if err != nil { + log.Error(err, "failed to delete pods of unhealthy node", "namespace", ns.Name) + return err + } + } + + log.Info("done deleting pods", "node name", nodeName) + + return nil +} From 15a76c808cf37f246e44588ca54d05ae930e79f6 Mon Sep 17 00:00:00 2001 From: oraz Date: Mon, 30 Oct 2023 14:12:51 +0200 Subject: [PATCH 2/2] Delete pods from node We don't know the node state- whether it is unhealthy or not --- pkg/resources/resources.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resources/resources.go b/pkg/resources/resources.go index 4d34f31..33b9d70 100644 --- a/pkg/resources/resources.go +++ b/pkg/resources/resources.go @@ -10,7 +10,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// DeletePods deletes all the pods from the unhealthy node +// DeletePods deletes all the pods from the node func DeletePods(ctx context.Context, r client.Client, nodeName string) error { log := ctrl.Log.WithName("commons-resource") zero := int64(0) @@ -41,7 +41,7 @@ func DeletePods(ctx context.Context, r client.Client, nodeName string) error { deleteOptions.Namespace = ns.Name err := r.DeleteAllOf(ctx, pod, deleteOptions) if err != nil { - log.Error(err, "failed to delete pods of unhealthy node", "namespace", ns.Name) + log.Error(err, "failed to delete pods of node", "namespace", ns.Name) return err } }