feat: add vm-live-migrate-detector

.gitignore

@@ -0,0 +1,23 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+
+.vscode

cmd/vm-live-migrate-detector/README.md

@@ -0,0 +1,54 @@
+# vm-live-migrate-detector
+
+A simple command line tool that helps detect if the VMs on a specific node can be live-migrated to other nodes considering the following:
+
+- Node selectors: If a VM is running on a specific node and has a node selector configured, the VM is considered non-live-migratable
+- PCIe devices: If a VM has PCIe devices attached, it is considered non-live-migratable
+- Node affinities: If a VM has no place to go after the node affinities are deduced, it is considered non-live-migratable
+
+Besides detecting the live-migratability of the VMs on the node, the tool can also shut down the VMs that are not live-migratable if the `--shutdown` flag is given.
+
+## Usage
+
+```
+$ vm-live-migrate-detector --help
+A simple VM detector and executor for Harvester upgrades
+
+The detector accepts a node name and inferences the possible places the VMs on top of it could be live migrated to.
+If there is no place to go, it can optionally shut down the VMs.
+
+Usage:
+  vm-live-migrate-detector NODENAME [flags]
+
+Flags:
+      --debug                set logging level to debug
+  -h, --help                 help for vm-live-migrate-detector
+      --kubeconfig string    Path to the kubeconfig file (default "/Users/starbops/.kube/foxtrot.yaml")
+      --kubecontext string   Context name
+      --shutdown             Do not shutdown non-migratable VMs
+      --trace                set logging level to trace
+  -v, --version              version for vm-live-migrate-detector
+```
+
+Given a node name, it can iterate all the VMs that are running on top of the node, and return with a list of non-live-migratable VMs.
+
+```
+$ export KUBECONFIG=~/.kube/foxtrot.yaml
+
+$ vm-live-migrate-detector harvester-z5hd8
+INFO[0000] Starting VM Live Migrate Detector
+INFO[0000] Checking vms on node harvester-z5hd8...
+INFO[0000] default/test-vm
+INFO[0000] Non-migratable VM(s): [default/test-vm]
+```
+
+It can help you shut down those non-live-migratable VMs if you have the `--shutdown` flag specified.
+
+```
+$ vm-live-migrate-detector harvester-z5hd8 --shutdown
+INFO[0000] Starting VM Live Migrate Detector
+INFO[0000] Checking vms on node harvester-z5hd8...
+INFO[0000] default/test-vm
+INFO[0000] Non-migratable VM(s): [default/test-vm]
+INFO[0000] vm default/test-vm was administratively stopped
+```

cmd/vm-live-migrate-detector/detector.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/rancher/wrangler/pkg/kv"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/component-helpers/scheduling/corev1"
+	kubevirtv1 "kubevirt.io/api/core/v1"
+	"kubevirt.io/client-go/kubecli"
+)
+
+type vmLiveMigrateDetector struct {
+	kubeConfig  string
+	kubeContext string
+
+	nodeName string
+	shutdown bool
+
+	virtClient kubecli.KubevirtClient
+}
+
+func newVMLiveMigrateDetector(options detectorOptions) *vmLiveMigrateDetector {
+	return &vmLiveMigrateDetector{
+		kubeConfig:  options.kubeConfigPath,
+		kubeContext: options.kubeContext,
+		nodeName:    options.nodeName,
+		shutdown:    options.shutdown,
+	}
+}
+
+func (d *vmLiveMigrateDetector) init() (err error) {
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{
+			ExplicitPath: d.kubeConfig,
+		},
+		&clientcmd.ConfigOverrides{
+			ClusterInfo:    clientcmdapi.Cluster{},
+			CurrentContext: d.kubeContext,
+		},
+	)
+
+	d.virtClient, err = kubecli.GetKubevirtClientFromClientConfig(clientConfig)
+	if err != nil {
+		logrus.Fatalf("cannot obtain KubeVirt client: %v\n", err)
+	}
+
+	return
+}
+
+func (d *vmLiveMigrateDetector) getFilteredNodes(nodes []v1.Node) ([]v1.Node, error) {
+	var found bool
+	var filteredNodes []v1.Node
+	for _, node := range nodes {
+		if d.nodeName != node.Name {
+			filteredNodes = append(filteredNodes, node)
+			continue
+		}
+		found = true
+	}
+	if !found {
+		return filteredNodes, fmt.Errorf("node %s not found", d.nodeName)
+	}
+	return filteredNodes, nil
+}
+
+func (d *vmLiveMigrateDetector) getNonMigratableVMs(ctx context.Context, nodes []v1.Node) ([]string, error) {
+	var vmNames []string
+
+	labelSelector := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"kubevirt.io/nodeName": d.nodeName,
+		},
+	}
+	listOptions := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+	}
+	vmis, err := d.virtClient.VirtualMachineInstance("").List(ctx, &listOptions)
+	if err != nil {
+		return vmNames, err
+	}
+
+	for _, vmi := range vmis.Items {
+		logrus.Debugf("%s/%s", vmi.Namespace, vmi.Name)
+
+		// Check nodeSelector
+		if vmi.Spec.NodeSelector != nil {
+			vmNames = append(vmNames, vmi.Namespace+"/"+vmi.Name)
+			continue
+		}
+
+		// Check pcidevices
+		if len(vmi.Spec.Domain.Devices.HostDevices) > 0 {
+			vmNames = append(vmNames, vmi.Namespace+"/"+vmi.Name)
+			continue
+		}
+
+		// Check nodeAffinity
+		var matched bool
+		nodeSelectorTerms := vmi.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
+		for i := range nodes {
+			matched, err = corev1.MatchNodeSelectorTerms(&nodes[i], nodeSelectorTerms)
+			if err != nil {
+				return vmNames, fmt.Errorf(err.Error())
+			}
+			if !matched {
+				continue
+			}
+			if nodes[i].Spec.Unschedulable {
+				matched = false
+				continue
+			}
+		}
+		if !matched {
+			vmNames = append(vmNames, vmi.Namespace+"/"+vmi.Name)
+		}
+	}
+	return vmNames, nil
+}
+
+func (d *vmLiveMigrateDetector) run(ctx context.Context) error {
+	if d.nodeName == "" {
+		return fmt.Errorf("please specify a node name")
+	}
+
+	nodeList, err := d.virtClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	filteredNodes, err := d.getFilteredNodes(nodeList.Items)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("Checking vms on node %s...", d.nodeName)
+
+	nonLiveMigratableVMNames, err := d.getNonMigratableVMs(ctx, filteredNodes)
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("Non-migratable VM(s): %v", nonLiveMigratableVMNames)
+
+	if d.shutdown {
+		for _, namespacedName := range nonLiveMigratableVMNames {
+			namespace, name := kv.RSplit(namespacedName, "/")
+			if err := d.virtClient.VirtualMachine(namespace).Stop(ctx, name, &kubevirtv1.StopOptions{}); err != nil {
+				return err
+			}
+			logrus.Infof("vm %s was administratively stopped", namespacedName)
+		}
+	}
+
+	return nil
+}

cmd/vm-live-migrate-detector/main.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+type detectorOptions struct {
+	kubeConfigPath string
+	kubeContext    string
+	shutdown       bool
+	nodeName       string
+}
+
+var (
+	AppVersion = "dev"
+	GitCommit  = "commit"
+
+	logDebug bool
+	logTrace bool
+
+	kubeConfigPath string
+	kubeContext    string
+	shutdown       bool
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "vm-live-migrate-detector NODENAME",
+	Short: "VM Live Migrate Detector",
+	Long: `A simple VM detector and executor for Harvester upgrades
+
+The detector accepts a node name and inferences the possible places the VMs on top of it could be live migrated to.
+If there is no place to go, it can optionally shut down the VMs.
+	`,
+	Version: AppVersion,
+	Args:    cobra.ExactArgs(1),
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		logrus.SetOutput(os.Stdout)
+		if logDebug {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+		if logTrace {
+			logrus.SetLevel(logrus.TraceLevel)
+		}
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := context.Context(context.Background())
+		options := detectorOptions{
+			kubeConfigPath: kubeConfigPath,
+			kubeContext:    kubeContext,
+			shutdown:       shutdown,
+			nodeName:       args[0],
+		}
+		if err := run(ctx, options); err != nil {
+			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+			os.Exit(1)
+		}
+	},
+}
+
+func init() {
+	debug := envGetBool("DEBUG", false)
+	trace := envGetBool("TRACE", false)
+
+	rootCmd.PersistentFlags().BoolVar(&logDebug, "debug", debug, "set logging level to debug")
+	rootCmd.PersistentFlags().BoolVar(&logTrace, "trace", trace, "set logging level to trace")
+
+	rootCmd.Flags().StringVar(&kubeConfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to the kubeconfig file")
+	rootCmd.Flags().StringVar(&kubeContext, "kubecontext", os.Getenv("KUBECONTEXT"), "Context name")
+	rootCmd.Flags().BoolVar(&shutdown, "shutdown", false, "Do not shutdown non-migratable VMs")
+}
+
+func run(ctx context.Context, options detectorOptions) error {
+	logrus.Info("Starting VM Live Migrate Detector")
+	detector := newVMLiveMigrateDetector(options)
+	detector.init()
+	return detector.run(ctx)
+}
+
+func main() {
+	cobra.CheckErr(rootCmd.Execute())
+}

cmd/vm-live-migrate-detector/util.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"os"
+	"strconv"
+)
+
+func envGetBool(key string, defaultValue bool) bool {
+	if parsed, err := strconv.ParseBool(os.Getenv(key)); err == nil {
+		return parsed
+	}
+	return defaultValue
+}