e2e: Add PersistentIPs tests

Signed-off-by: Enrique Llorente <[email protected]>

Makefile

@@ -21,7 +21,7 @@ CONTAINER_TOOL ?= docker
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
-
+	
 .PHONY: all
 all: build
 
@@ -192,6 +192,11 @@ cluster-down:
 cluster-sync:
 	./hack/cluster.sh sync
 
+e2e:
+	export KUBECONFIG=$$(pwd)/.output/kubeconfig && \
+	cd test/e2e && \
+	go test -v --ginkgo.v
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary (ideally with version)
 # $2 - package url which can be installed

test/e2e/e2e_suite_test.go

@@ -21,8 +21,14 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
+	testenv "github.com/maiqueb/kubevirt-ipam-claims/test/env"
 )
 
+var _ = BeforeSuite(func() {
+	testenv.Start()
+})
+
 // Run e2e tests using the Ginkgo runner.
 func TestE2E(t *testing.T) {
 	RegisterFailHandler(Fail)

test/e2e/persistentips_test.go

@@ -0,0 +1,247 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2024 Red Hat, Inc.
+ *
+ */
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	kubevirtv1 "kubevirt.io/api/core/v1"
+
+	testenv "github.com/maiqueb/kubevirt-ipam-claims/test/env"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("Persistent IPs", func() {
+	When("network attachment definition created with allowPersistentIPs=true", func() {
+		var (
+			td                   testenv.TestData
+			networkInterfaceName = "multus"
+			vm                   *kubevirtv1.VirtualMachine
+			vmi                  *kubevirtv1.VirtualMachineInstance
+			nad                  *nadv1.NetworkAttachmentDefinition
+		)
+		BeforeEach(func() {
+			td = testenv.GenerateTestData()
+			td.SetUp()
+			DeferCleanup(func() {
+				td.TearDown()
+			})
+
+			nadName := testenv.RandomName("l2", 16)
+			nad = &nadv1.NetworkAttachmentDefinition{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      nadName,
+					Namespace: td.Namespace,
+				},
+				Spec: nadv1.NetworkAttachmentDefinitionSpec{
+					Config: fmt.Sprintf(`
+{
+        "cniVersion": "0.3.0",
+        "name": "%[2]s",
+        "type": "ovn-k8s-cni-overlay",
+        "topology": "layer2",
+        "subnets": "10.100.200.0/24",
+        "netAttachDefName": "%[1]s/%[2]s",
+        "allowPersistentIPs": true
+}
+`, td.Namespace, nadName),
+				},
+			}
+			vmi = &kubevirtv1.VirtualMachineInstance{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: td.Namespace,
+					Name:      testenv.RandomName("alpine", 16),
+				},
+				Spec: kubevirtv1.VirtualMachineInstanceSpec{
+					Domain: kubevirtv1.DomainSpec{
+						Resources: kubevirtv1.ResourceRequirements{
+							Requests: corev1.ResourceList{
+								corev1.ResourceMemory: resource.MustParse("128Mi"),
+							},
+						},
+						Devices: kubevirtv1.Devices{
+							Disks: []kubevirtv1.Disk{
+								{
+									DiskDevice: kubevirtv1.DiskDevice{
+										Disk: &kubevirtv1.DiskTarget{
+											Bus: kubevirtv1.DiskBusVirtio,
+										},
+									},
+									Name: "containerdisk",
+								},
+							},
+							Interfaces: []kubevirtv1.Interface{
+								{
+									Name: networkInterfaceName,
+									InterfaceBindingMethod: kubevirtv1.InterfaceBindingMethod{
+										Bridge: &kubevirtv1.InterfaceBridge{},
+									},
+								},
+							},
+						},
+					},
+					Networks: []kubevirtv1.Network{
+						{
+							Name: networkInterfaceName,
+							NetworkSource: kubevirtv1.NetworkSource{
+								Multus: &kubevirtv1.MultusNetwork{
+									NetworkName: nad.Name,
+								},
+							},
+						},
+					},
+					TerminationGracePeriodSeconds: pointer.Int64(5),
+					Volumes: []kubevirtv1.Volume{
+						{
+							Name: "containerdisk",
+							VolumeSource: kubevirtv1.VolumeSource{
+								ContainerDisk: &kubevirtv1.ContainerDiskSource{
+									Image: "quay.io/kubevirtci/alpine-container-disk-demo:devel_alt",
+								},
+							},
+						},
+					},
+				},
+			}
+			vm = testenv.NewVirtualMachine(vmi, testenv.WithRunning())
+
+			By("Create NetworkAttachmentDefinition")
+			Expect(testenv.Client.Create(context.Background(), nad)).To(Succeed())
+		})
+		Context("and a virtual machine using it is also created", func() {
+			BeforeEach(func() {
+				By("Creating VM using the nad")
+				Expect(testenv.Client.Create(context.Background(), vm)).To(Succeed())
+
+				By(fmt.Sprintf("Waiting for readiness at virtual machine %s", vm.Name))
+				Eventually(testenv.ThisVMReadiness(vm)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(BeTrue())
+
+				By("Wait for IPAMClaim to get created")
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					ShouldNot(BeEmpty())
+
+				Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed())
+			})
+
+			It("should keep ips after live migration", func() {
+				Expect(vmi.Status.Interfaces).ToNot(BeEmpty())
+				Expect(vmi.Status.Interfaces[0].IPs).ToNot(BeEmpty())
+
+				vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs
+
+				testenv.LiveMigrateVirtualMachine(td.Namespace, vm.Name)
+				testenv.CheckLiveMigrationSucceeded(td.Namespace, vm.Name)
+
+				By("Wait for VMI to be ready after live migration")
+				vmi = testenv.WaitVirtualMachineInstanceReadiness(vmi)
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration)))
+
+			})
+
+			It("should garbage collect IPAMClaims after virtual machine deletion", func() {
+				Expect(testenv.Client.Delete(context.Background(), vm)).To(Succeed())
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeEmpty())
+			})
+
+			It("should keep ips after restart", func() {
+				vmiIPsBeforeRestart := vmi.Status.Interfaces[0].IPs
+				vmiUUIDBeforeRestart := vmi.UID
+
+				By("Re-starting the VM")
+				output, err := exec.Command("virtctl", "restart", "-n", td.Namespace, vmi.Name).CombinedOutput()
+				Expect(err).ToNot(HaveOccurred(), output)
+
+				By("Wait for a new VMI to be re-started")
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(90 * time.Second).
+					Should(testenv.BeRestarted(vmiUUIDBeforeRestart))
+
+				By("Wait for VMI to be ready after restart")
+				vmi = testenv.WaitVirtualMachineInstanceReadiness(vmi)
+
+				Expect(vmi).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeRestart)))
+			})
+		})
+		Context("and a virtual machine instance using it is also created", func() {
+			BeforeEach(func() {
+				By("Creating VMI using the nad")
+				Expect(testenv.Client.Create(context.Background(), vmi)).To(Succeed())
+
+				By(fmt.Sprintf("Waiting for readiness at virtual machine instance %s", vmi.Name))
+				Eventually(testenv.ThisVMI(vmi)).
+					WithPolling(time.Second).
+					WithTimeout(5 * time.Minute).
+					Should(testenv.ContainConditionVMIReady())
+
+				By("Wait for IPAMClaim to get created")
+				Eventually(testenv.IPAMClaimsFromNamespace(vm.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					ShouldNot(BeEmpty())
+
+				Expect(testenv.Client.Get(context.Background(), client.ObjectKeyFromObject(vmi), vmi)).To(Succeed())
+			})
+
+			It("should keep ips after live migration", func() {
+				Expect(vmi.Status.Interfaces).ToNot(BeEmpty())
+				Expect(vmi.Status.Interfaces[0].IPs).ToNot(BeEmpty())
+
+				vmiIPsBeforeMigration := vmi.Status.Interfaces[0].IPs
+
+				testenv.LiveMigrateVirtualMachine(td.Namespace, vmi.Name)
+				testenv.CheckLiveMigrationSucceeded(td.Namespace, vmi.Name)
+
+				Expect(testenv.ThisVMI(vmi)()).Should(testenv.MatchIPsAtInterfaceByName(networkInterfaceName, ConsistOf(vmiIPsBeforeMigration)))
+
+			})
+
+			It("should garbage collect IPAMClaims after virtual machine deletion", func() {
+				Expect(testenv.Client.Delete(context.Background(), vmi)).To(Succeed())
+				Eventually(testenv.IPAMClaimsFromNamespace(vmi.Namespace)).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeEmpty())
+			})
+		})
+
+	})
+})