bin: Port storage and network tests to Incus

Signed-off-by: Stéphane Graber <[email protected]>

bin/test-incus-network

@@ -0,0 +1,270 @@
+#!/bin/sh
+set -eu
+
+# Default to loop storage pool if /dev/nvme0n1p3 doesn't exist.
+storageDevice=""
+if [ -f "/dev/nvme0n1p3" ]; then
+  storageDevice="/dev/nvme0n1p3"
+fi
+
+parentNIC="enp5s0"
+
+waitVMAgent() (
+  set +x
+  # shellcheck disable=SC3043
+  local vmName="$1"
+  for i in $(seq 90) # Wait up to 90s.
+  do
+    if incus info "${vmName}" | grep -qF 127.0.0.1; then
+      return 0 # Success.
+    fi
+
+    sleep 1
+  done
+
+  echo "VM ${vmName} agent not running after ${i}s"
+  return 1 # Failed.
+)
+
+cleanup() {
+    echo ""
+    if [ "${FAIL}" = "1" ]; then
+        echo "Test failed"
+        exit 1
+    fi
+
+    echo "Test passed"
+    exit 0
+}
+
+FAIL=1
+trap cleanup EXIT HUP INT TERM
+
+# Install test dependencies
+apt-get remove --purge cloud-init --yes
+apt-get install --yes curl jq zfsutils-linux
+
+# Install Incus
+curl -sL https://pkgs.zabbly.com/get/incus-daily | sh
+
+# Enable SR-IOV on nic and bring up
+echo 7 > "/sys/class/net/${parentNIC}/device/sriov_numvfs"
+ip link set "${parentNIC}" up
+sleep 10
+ethtool "${parentNIC}"
+ip a
+
+# Configure Incus
+incus storage create default zfs source="${storageDevice}"
+incus profile device add default root disk path=/ pool=default
+
+(
+cat << EOF
+version: 2
+ethernets:
+  eth0:
+    optional: true
+    accept-ra: true
+    dhcp4: true
+    dhcp6: true
+  enp5s0:
+    optional: true
+    accept-ra: true
+    dhcp4: true
+    dhcp6: true
+EOF
+) | incus profile set default user.network-config -
+
+# Launch instances with physical NICs
+echo "==> VM on default VLAN with physical"
+incus init images:ubuntu/22.04/cloud v1-physical --vm -c limits.cpu=3
+incus config device add v1-physical eth0 nic nictype=physical parent="${parentNIC}v1" name=eth0
+incus start v1-physical
+
+echo "==> Container on default VLAN with physical"
+incus init images:ubuntu/22.04/cloud c1-physical
+incus config device add c1-physical eth0 nic nictype=physical parent="${parentNIC}v2" name=eth0
+incus start c1-physical
+
+# Launch instances with macvlan NICs
+echo "==> VM on default VLAN with macvlan"
+incus init images:ubuntu/22.04/cloud v1-macvlan --vm -c limits.cpu=3
+incus config device add v1-macvlan eth0 nic nictype=macvlan parent="${parentNIC}" name=eth0
+incus start v1-macvlan
+
+echo "==> Container on default VLAN with macvlan"
+incus init images:ubuntu/22.04/cloud c1-macvlan
+incus config device add c1-macvlan eth0 nic nictype=macvlan parent="${parentNIC}" name=eth0
+incus start c1-macvlan
+
+# Launch instances with sriov NICs
+echo "==> VM on default VLAN with sriov"
+incus init images:ubuntu/22.04/cloud v1-sriov --vm -c limits.cpu=3
+incus config device add v1-sriov eth0 nic nictype=sriov parent="${parentNIC}" name=eth0
+incus start v1-sriov
+
+echo "==> Container on default VLAN with sriov"
+incus init images:ubuntu/22.04/cloud c1-sriov
+incus config device add c1-sriov eth0 nic nictype=sriov parent="${parentNIC}" name=eth0
+incus start c1-sriov
+
+# Wait for VMs to start.
+echo "==> Waiting for VMs to start"
+sleep 30
+waitVMAgent v1-physical
+waitVMAgent v1-macvlan
+waitVMAgent v1-sriov
+
+# Check that all instances have an IPv4 and IPv6 address
+networkTests() {
+    # shellcheck disable=SC3043
+    local FAIL=0
+
+    echo "=> Performing network tests"
+    for url in $(incus query "/1.0/instances" | jq -r .[]); do
+        name=$(echo "${url}" | cut -d/ -f4)
+        echo ""
+
+        # Get the addresses
+        address=$(incus query "${url}/state" | jq -r ".network.eth0.addresses | .[] | select(.scope | contains(\"global\")) | .address" 2>/dev/null || true)
+        if [ -z "${address}" ]; then
+            address=$(incus query "${url}/state" | jq -r ".network.enp5s0.addresses | .[] | select(.scope | contains(\"global\")) | .address" 2>/dev/null || true)
+        fi
+
+        if [ -z "${address}" ]; then
+            echo "FAIL: No network interface address: ${name}"
+            FAIL=1
+            continue
+        fi
+
+        # IPv4 address
+        if echo "${address}" | grep -qF "."; then
+            echo "PASS: IPv4 address: ${name}"
+        else
+            echo "FAIL: IPv4 address: ${name}"
+            FAIL=1
+        fi
+
+        # IPv6 address
+        if echo "${address}" | grep -qF ":"; then
+            echo "PASS: IPv6 address: ${name}"
+        else
+            echo "FAIL: IPv6 address: ${name}"
+            FAIL=1
+        fi
+
+        # DNS resolution
+        if incus exec "${name}" -- getent hosts linuxcontainers.org >/dev/null 2>&1 || incus exec "${name}" -- ping -c1 -W1 linuxcontainers.org >/dev/null 2>&1; then
+            echo "PASS: DNS resolution: ${name}"
+        else
+            echo "FAIL: DNS resolution: ${name}"
+            FAIL=1
+        fi
+    done
+
+    if [ "${FAIL}" = "1" ]; then
+        return 1
+    fi
+
+    return 0
+}
+
+incus list
+networkTests
+
+# Reload Incus and check connectivity still works.
+# Ensures VM NICs are not dependent on any fds/fdsets from Incus.
+systemctl restart incus
+sleep 10
+incus admin waitready --timeout=300
+networkTests
+
+# Check VM macvlan multi-queue support.
+incus exec v1-macvlan -- apt-get install --no-install-recommends --yes ethtool
+incus exec v1-macvlan -- ethtool -l enp5s0 | grep -c '^Combined:\s\+3$'  | grep -Fx 2
+
+# Hot unplug the NICs and check they are removed
+incus config device remove v1-physical eth0
+! incus exec v1-physical -- ip a show enp5s0 || false
+incus config device remove c1-physical eth0
+! incus exec c1-physical -- ip a show eth0 || false
+incus config device remove v1-macvlan eth0
+! incus exec v1-macvlan -- ip a show enp5s0 || false
+incus config device remove c1-macvlan eth0
+! incus exec c1-macvlan -- ip a show eth0 || false
+incus config device remove v1-sriov eth0
+! incus exec v1-sriov -- ip a show enp5s0 || false
+incus config device remove c1-sriov eth0
+! incus exec c1-sriov -- ip a show eth0 || false
+
+incus list
+
+# Hot plug the NICs back
+incus config device add v1-physical eth0 nic nictype=physical parent="${parentNIC}v1" name=eth0
+incus config device add c1-physical eth0 nic nictype=physical parent="${parentNIC}v2" name=eth0
+incus config device add v1-macvlan eth0 nic nictype=macvlan parent="${parentNIC}" name=eth0
+incus config device add c1-macvlan eth0 nic nictype=macvlan parent="${parentNIC}" name=eth0
+incus config device add v1-sriov eth0 nic nictype=sriov parent="${parentNIC}" name=eth0
+incus config device add c1-sriov eth0 nic nictype=sriov parent="${parentNIC}" name=eth0
+
+# Wait for DHCP
+sleep 10
+
+# Check the IPs are re-added
+incus list
+networkTests
+
+# Remove instances, leaving one VM and container for bridge hotplug test
+incus delete -f v1-macvlan
+incus delete -f c1-macvlan
+incus delete -f v1-sriov
+incus delete -f c1-sriov
+
+# Check bridged NIC type works (also test hotlugging different NIC type).
+echo "=> Performing bridged NIC tests"
+incus network create incusbr0
+incus config device remove v1-physical eth0
+incus config device remove c1-physical eth0
+incus config device add v1-physical eth0 nic network=incusbr0 name=eth0
+incus config device add c1-physical eth0 nic network=incusbr0 name=eth0
+
+# Wait for DHCP
+sleep 10
+
+# Check the IPs are re-added
+incus list
+networkTests
+
+# Check VM "agent.nic_config" works by reconfiguring eth0 to use parent and mtu settings.
+echo "=> Performing VM bridged NIC agent.nic_config tests"
+incus stop v1-physical -f
+incus config set v1-physical agent.nic_config=true
+incus config device set v1-physical eth0 nictype=bridged parent=incusbr0 network= mtu=1400 name=eth0
+incus start v1-physical
+
+# Wait for incus-agent to rename the interface.
+waitVMAgent v1-physical
+
+# Interface "eth0" should exist in the VM with the correct MTU.
+incus exec v1-physical -- test -d /sys/class/net/eth0
+incus exec v1-physical -- grep -Fx 1400 /sys/class/net/eth0/mtu
+
+# Check VM bridged multi-queue support.
+incus exec v1-physical -- apt-get install --no-install-recommends --yes ethtool
+incus exec v1-physical -- ethtool -l eth0 | grep -c '^Combined:\s\+3$'  | grep -Fx 2
+
+# Default VM interface enp5s0 should not exist in the VM.
+! incus exec v1-physical -- test -d /sys/class/net/enp5s0 || false
+
+# Cleanup
+echo "=> Cleanup"
+for url in $(incus query "/1.0/instances" | jq -r .[]); do
+    name=$(echo "${url}" | cut -d/ -f4)
+    incus delete -f "${name}"
+done
+
+incus profile device remove default root
+incus storage delete default
+incus network delete incusbr0
+
+FAIL=0