CORE-3242: skip node delete on id mismatch (#73)

* skip node delete on id mismatch Co-authored-by: Marcin Kaciuba <[email protected]> --------- Co-authored-by: Mangirdas <[email protected]> Co-authored-by: Marcin Kaciuba <[email protected]>
castai · Jul 13, 2023 · 862c92d · 862c92d
1 parent e3cea91
commit 862c92d
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 1 deletion.
diff --git a/actions/delete_node_handler.go b/actions/delete_node_handler.go
@@ -62,7 +62,23 @@ func (h *deleteNodeHandler) Handle(ctx context.Context, action *castai.ClusterAc
 
 	b := backoff.WithContext(backoff.WithMaxRetries(backoff.NewConstantBackOff(h.cfg.deleteRetryWait), h.cfg.deleteRetries), ctx)
 	err := backoff.Retry(func() error {
-		err := h.clientset.CoreV1().Nodes().Delete(ctx, req.NodeName, metav1.DeleteOptions{})
+		current, err := h.clientset.CoreV1().Nodes().Get(ctx, req.NodeName, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				log.Info("node not found, skipping delete")
+				return nil
+			}
+			return fmt.Errorf("error getting node: %w", err)
+		}
+
+		if val, ok := current.Labels[castai.LabelNodeID]; ok {
+			if val != "" && val != req.NodeID {
+				log.Infof("node id mismatch, expected %q got %q. Skipping delete.", req.NodeID, val)
+				return nil
+			}
+		}
+
+		err = h.clientset.CoreV1().Nodes().Delete(ctx, current.Name, metav1.DeleteOptions{})
 		if apierrors.IsNotFound(err) {
 			log.Info("node not found, skipping delete")
 			return nil

diff --git a/actions/delete_node_handler_test.go b/actions/delete_node_handler_test.go
@@ -82,6 +82,41 @@ func TestDeleteNodeHandler(t *testing.T) {
 		r.NoError(err)
 	})
 
+	t.Run("skip delete when node id do not match", func(t *testing.T) {
+		r := require.New(t)
+		nodeName := "node1"
+		node := &v1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: nodeName,
+				Labels: map[string]string{
+					castai.LabelNodeID: "node-id",
+				},
+			},
+		}
+		clientset := fake.NewSimpleClientset(node)
+
+		action := &castai.ClusterAction{
+			ID: uuid.New().String(),
+			ActionDeleteNode: &castai.ActionDeleteNode{
+				NodeName: "node1",
+				NodeID:   "another-node-id",
+			},
+		}
+
+		h := deleteNodeHandler{
+			log:       log,
+			clientset: clientset,
+			cfg:       deleteNodeConfig{},
+		}
+
+		err := h.Handle(context.Background(), action)
+		r.NoError(err)
+
+		existing, err := clientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
+		r.NoError(err)
+		existing.Labels[castai.LabelNodeID] = "node-id"
+	})
+
 	t.Run("delete node with pods", func(t *testing.T) {
 		r := require.New(t)
 		nodeName := "node1"

diff --git a/castai/types.go b/castai/types.go
@@ -8,6 +8,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 )
 
+const (
+	LabelNodeID = "provisioner.cast.ai/node-id"
+)
+
 type GetClusterActionsResponse struct {
 	Items []*ClusterAction `json:"items"`
 }