diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 0e929aa2..fc840acd 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -4,7 +4,7 @@ unset_interactive_vars() { unset LOOKUP_IFACE LIMIT_SUBNET SKIP_SERVICE EXPECT_PEERS REUSE_EXISTING REUSE_EXISTING_COUNT \ SETUP_ZFS ZFS_FILTER ZFS_WIPE \ - SETUP_CEPH CEPH_WARNING CEPH_FILTER CEPH_WIPE SETUP_CEPHFS \ + SETUP_CEPH CEPH_WARNING CEPH_FILTER CEPH_WIPE SETUP_CEPHFS CEPH_CLUSTER_NETWORK IGNORE_CEPH_NETWORKING \ SETUP_OVN OVN_WARNING OVN_FILTER IPV4_SUBNET IPV4_START IPV4_END DNS_ADDRESSES IPV6_SUBNET } @@ -13,28 +13,30 @@ unset_interactive_vars() { # The lines that are output are based on the values passed to the listed environment variables. # Any unset variables will be omitted. microcloud_interactive() { - LOOKUP_IFACE=${LOOKUP_IFACE:-} # filter string for the lookup interface table. - LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. - SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. - EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. + LOOKUP_IFACE=${LOOKUP_IFACE:-} # filter string for the lookup interface table. + LIMIT_SUBNET=${LIMIT_SUBNET:-} # (yes/no) input for limiting lookup of systems to the above subnet. + SKIP_SERVICE=${SKIP_SERVICE:-} # (yes/no) input to skip any missing services. Should be unset if all services are installed. + EXPECT_PEERS=${EXPECT_PEERS:-} # wait for this number of systems to be available to join the cluster. REUSE_EXISTING=${REUSE_EXISTING:-} # (yes/no) incorporate an existing clustered service. REUSE_EXISTING_COUNT=${REUSE_EXISTING_COUNT:-0} # (number) number of existing clusters to incorporate. - SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. - ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. - ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. - SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. - SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. - CEPH_WARNING=${CEPH_WARNING:-} # (yes/no) input for warning about eligible disk detection. - CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. - CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. - SETUP_OVN=${SETUP_OVN:-} # (yes/no) input for initiating OVN network setup. - OVN_WARNING=${OVN_WARNING:-} # (yes/no) input for warning about eligible interface detection. - OVN_FILTER=${OVN_FILTER:-} # filter string for OVN interfaces. - IPV4_SUBNET=${IPV4_SUBNET:-} # OVN ipv4 gateway subnet. - IPV4_START=${IPV4_START:-} # OVN ipv4 range start. - IPV4_END=${IPV4_END:-} # OVN ipv4 range end. - DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. - IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. + SETUP_ZFS=${SETUP_ZFS:-} # (yes/no) input for initiating ZFS storage pool setup. + ZFS_FILTER=${ZFS_FILTER:-} # filter string for ZFS disks. + ZFS_WIPE=${ZFS_WIPE:-} # (yes/no) to wipe all disks. + SETUP_CEPH=${SETUP_CEPH:-} # (yes/no) input for initiating CEPH storage pool setup. + SETUP_CEPHFS=${SETUP_CEPHFS:-} # (yes/no) input for initialising CephFS storage pool setup. + CEPH_WARNING=${CEPH_WARNING:-} # (yes/no) input for warning about eligible disk detection. + CEPH_FILTER=${CEPH_FILTER:-} # filter string for CEPH disks. + CEPH_WIPE=${CEPH_WIPE:-} # (yes/no) to wipe all disks. + CEPH_CLUSTER_NETWORK=${CEPH_CLUSTER_NETWORK:-} # (default: MicroCloud internal subnet or Ceph public network if specified previously) input for setting up a cluster network. + IGNORE_CEPH_NETWORKING=${IGNORE_CEPH_NETWORKING:-} # (yes/no) input for ignoring Ceph network setup. Set it to `yes` during `microcloud add` . + SETUP_OVN=${SETUP_OVN:-} # (yes/no) input for initiating OVN network setup. + OVN_WARNING=${OVN_WARNING:-} # (yes/no) input for warning about eligible interface detection. + OVN_FILTER=${OVN_FILTER:-} # filter string for OVN interfaces. + IPV4_SUBNET=${IPV4_SUBNET:-} # OVN ipv4 gateway subnet. + IPV4_START=${IPV4_START:-} # OVN ipv4 range start. + IPV4_END=${IPV4_END:-} # OVN ipv4 range end. + DNS_ADDRESSES=${DNS_ADDRESSES:-} # OVN custom DNS addresses. + IPV6_SUBNET=${IPV6_SUBNET:-} # OVN ipv6 range. setup=" ${LOOKUP_IFACE} # filter the lookup interface @@ -86,6 +88,14 @@ $(true) # workaround for set -e " fi +if [ -z "${IGNORE_CEPH_NETWORKING}" ]; then + if [ -n "${CEPH_CLUSTER_NETWORK}" ]; then + setup="${setup} +${CEPH_CLUSTER_NETWORK} +$(true) # workaround for set -e +" + fi +fi if [ -n "${SETUP_OVN}" ]; then setup="${setup} @@ -163,9 +173,15 @@ validate_system_microceph() { shift 1 fi + cluster_ceph_subnet="" + if echo "${1}" | grep -Pq '^([0-9]{1,3}\.){3}[0-9]{1,3}/([0-9]|[1-2][0-9]|3[0-2])$'; then + cluster_ceph_subnet="${1}" + shift 1 + fi + disks="${*}" - echo "==> ${name} Validating MicroCeph. Using disks: {${disks}}" + echo "==> ${name} Validating MicroCeph. Using disks: {${disks}}, Using CephFS: {${cephfs}}, Cluster Ceph Subnet: {${cluster_ceph_subnet}}" lxc remote switch local lxc exec "${name}" -- sh -ceu " @@ -191,6 +207,12 @@ validate_system_microceph() { microceph.ceph osd pool get lxd_cephfs_data size " > /dev/null fi + + if [ -n "${cluster_ceph_subnet}" ]; then + lxc exec "${name}" -- sh -ceu " + microceph.ceph config show osd.1 cluster_network | grep -q ${cluster_ceph_subnet} + " > /dev/null + fi } # validate_system_microovn: Ensures the node with the given name has correctly set up MicroOVN with the given resources. @@ -1104,3 +1126,30 @@ lxd_wait_vm() { echo " ${name} VM failed to start" return 1 } + +# ip_prefix_by_netmask: Returns the prefix length of the given netmask. +ip_prefix_by_netmask () { + # shellcheck disable=SC2048,SC2086 + c=0 x=0$( printf '%o' ${1//./ } ) + # shellcheck disable=SC2048,SC2086 + while [ $x -gt 0 ]; do + (( c += x % 2, x >>= 1 )) + done + + echo /$c ; +} + +# ip_config_to_netaddr: Returns the IPv4 network address of the given interface. +# e.g: ip_config_to_netaddr lxdbr0 (with inet: 10.233.6.X/24)-> 10.233.6.0/24 +ip_config_to_netaddr () { + local line ip mask net_addr + line=$(ifconfig -a "$1" | grep netmask | tr -s " ") + ip=$(echo "$line" | cut -f 3 -d " ") + mask=$(echo "$line" | cut -f 5 -d " ") + + IFS=. read -r io1 io2 io3 io4 <<< "$ip" + IFS=. read -r mo1 mo2 mo3 mo4 <<< "$mask" + net_addr="$((io1 & mo1)).$((io2 & mo2)).$((io3 & mo3)).$((io4 & mo4))" + + echo "${net_addr}$(ip_prefix_by_netmask "${mask}")" +} diff --git a/test/suites/add.sh b/test/suites/add.sh index 2df58c5c..350b2cf3 100644 --- a/test/suites/add.sh +++ b/test/suites/add.sh @@ -101,7 +101,15 @@ test_add_auto() { } test_add_interactive() { - reset_systems 4 2 1 + reset_systems 4 2 2 + + ceph_cluster_subnet_prefix="10.0.1" + ceph_cluster_subnet_iface="enp7s0" + + for n in $(seq 2 5); do + cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + done # Disable extra nodes so we don't add them yet. # shellcheck disable=SC2043 @@ -120,6 +128,7 @@ test_add_interactive() { export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" export CEPH_WIPE="yes" + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" export SETUP_OVN="yes" export OVN_FILTER="enp6s0" export IPV4_SUBNET="10.1.123.1/24" @@ -145,6 +154,7 @@ test_add_interactive() { export ZFS_WIPE="yes" export SETUP_CEPH="yes" export CEPH_WIPE="yes" + export IGNORE_CEPH_NETWORKING="yes" export SETUP_OVN="yes" export OVN_FILTER="enp6s0" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" @@ -152,7 +162,7 @@ test_add_interactive() { for m in micro01 micro02 micro03 micro04 ; do validate_system_lxd "${m}" 4 disk1 1 1 enp6s0 10.1.123.1/24 10.1.123.100-10.1.123.254 fd42:1:1234:1234::1/64 10.1.123.1,fd42:1:1234:1234::1 - validate_system_microceph "${m}" 1 disk2 + validate_system_microceph "${m}" 1 "${ceph_cluster_subnet_prefix}.0/24" disk2 validate_system_microovn "${m}" done @@ -179,6 +189,7 @@ test_add_interactive() { export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" + export IGNORE_CEPH_NETWORKING="yes" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud add > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q diff --git a/test/suites/basic.sh b/test/suites/basic.sh index 155af598..5d636f0c 100644 --- a/test/suites/basic.sh +++ b/test/suites/basic.sh @@ -3,6 +3,8 @@ test_interactive() { reset_systems 3 3 1 + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + echo "Creating a MicroCloud with all services but no devices" export LOOKUP_IFACE="enp5s0" export LIMIT_SUBNET="yes" @@ -10,6 +12,7 @@ test_interactive() { export SETUP_ZFS="no" export SETUP_CEPH="no" export SETUP_OVN="no" + export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -28,11 +31,14 @@ test_interactive() { lxc exec "${m}" -- snap restart microcloud done + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + echo "Creating a MicroCloud with ZFS storage" export SKIP_SERVICE="yes" export SETUP_ZFS="yes" export ZFS_FILTER="lxd_disk1" export ZFS_WIPE="yes" + export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" unset SETUP_CEPH SETUP_OVN microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" @@ -44,6 +50,8 @@ test_interactive() { # Reset the systems and install microceph. reset_systems 3 3 1 + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + for m in micro01 micro02 micro03 ; do lxc exec "${m}" -- snap disable microovn || true lxc exec "${m}" -- snap restart microcloud @@ -54,6 +62,7 @@ test_interactive() { export SETUP_CEPHFS="yes" export CEPH_FILTER="lxd_disk2" export CEPH_WIPE="yes" + export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -65,6 +74,8 @@ test_interactive() { # Reset the systems and install microovn. reset_systems 3 3 1 + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + for m in micro01 micro02 micro03 ; do lxc exec "${m}" -- snap disable microceph || true lxc exec "${m}" -- snap restart microcloud @@ -80,6 +91,7 @@ test_interactive() { export IPV4_END="10.1.123.254" export IPV6_SUBNET="fd42:1:1234:1234::1/64" export DNS_ADDRESSES="10.1.123.1,8.8.8.8" + export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -91,12 +103,15 @@ test_interactive() { # Reset the systems and install microovn and microceph. reset_systems 3 3 1 + microcloud_internal_net_addr="$(ip_config_to_netaddr lxdbr0)" + echo "Creating a MicroCloud with ZFS and Ceph storage, and OVN network" unset SKIP_SERVICE export SETUP_CEPH="yes" export SETUP_CEPHFS="yes" export CEPH_FILTER="lxd_disk2" export CEPH_WIPE="yes" + export CEPH_CLUSTER_NETWORK="${microcloud_internal_net_addr}" microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q @@ -105,6 +120,74 @@ test_interactive() { validate_system_microceph "${m}" 1 disk2 validate_system_microovn "${m}" done + + # Reset the systems and install microovn and microceph with a partially disaggregated ceph network setup. + reset_systems 3 3 3 + + ceph_dedicated_subnet_prefix="10.0.1" + ceph_dedicated_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + dedicated_ip="${ceph_dedicated_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- ip addr add "${dedicated_ip}" dev "${ceph_dedicated_subnet_iface}" + done + + echo "Creating a MicroCloud with ZFS, Ceph storage with a partially disaggregated Ceph networking setup, and OVN network" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="yes" + export SETUP_CEPHFS="yes" + export CEPH_FILTER="lxd_disk2" + export CEPH_WIPE="yes" + export CEPH_CLUSTER_NETWORK="${ceph_dedicated_subnet_prefix}.0/24" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + validate_system_microceph "${m}" 1 "${ceph_dedicated_subnet_prefix}.0/24" disk2 + validate_system_microovn "${m}" + done + + # Reset the systems and install microovn and microceph with a partially disaggregated ceph network setup. + reset_systems 3 3 2 + + ceph_cluster_subnet_prefix="10.0.1" + ceph_cluster_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + cluster_ip="${ceph_cluster_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- ip addr add "${cluster_ip}" dev "${ceph_cluster_subnet_iface}" + done + + echo "Creating a MicroCloud with ZFS, Ceph storage with a fully disaggregated Ceph networking setup, and OVN network" + export SETUP_ZFS="yes" + export ZFS_FILTER="lxd_disk1" + export ZFS_WIPE="yes" + export SETUP_CEPH="yes" + export SETUP_CEPHFS="yes" + export CEPH_FILTER="lxd_disk2" + export CEPH_WIPE="yes" + export CEPH_CLUSTER_NETWORK="${ceph_cluster_subnet_prefix}.0/24" + export SETUP_OVN="yes" + export OVN_FILTER="enp6s0" + export IPV4_SUBNET="10.1.123.1/24" + export IPV4_START="10.1.123.100" + export IPV4_END="10.1.123.254" + microcloud_interactive | lxc exec micro01 -- sh -c "microcloud init > out" + + lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q + for m in micro01 micro02 micro03 ; do + validate_system_lxd "${m}" 3 disk1 3 1 "${OVN_FILTER}" "${IPV4_SUBNET}" "${IPV4_START}"-"${IPV4_END}" "${IPV6_SUBNET}" + validate_system_microceph "${m}" 1 "${CEPH_CLUSTER_NETWORK}" disk2 + validate_system_microovn "${m}" + done } test_instances_config() { @@ -400,6 +483,106 @@ EOF fi lxc exec micro01 -- lxc delete -f c1 c2 + + # Create a MicroCloud with ceph, partially disaggregated ceph networking and ovn setup. + reset_systems 3 3 3 + addr=$(lxc ls micro01 -f json -c4 | jq -r '.[0].state.network.enp5s0.addresses[] | select(.family == "inet") | .address') + + ceph_dedicated_subnet_prefix="10.0.1" + ceph_dedicated_subnet_iface="enp7s0" + + for n in $(seq 2 4); do + dedicated_ip="${ceph_dedicated_subnet_prefix}.${n}/24" + lxc exec "micro0$((n-1))" -- ip addr add "${dedicated_ip}" dev "${ceph_dedicated_subnet_iface}" + done + + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed < out" lxc exec micro01 -- tail -1 out | grep "MicroCloud is ready" -q for m in micro01 micro02 micro03 micro04 ; do diff --git a/test/suites/preseed.sh b/test/suites/preseed.sh index 64895fd8..ee5d7487 100644 --- a/test/suites/preseed.sh +++ b/test/suites/preseed.sh @@ -2,7 +2,7 @@ test_preseed() { reset_systems 4 3 2 - lookup_addr=$(lxc ls micro01 -f csv -c4 | grep enp5s0 | cut -d' ' -f1) + lookup_addr=$(lxc ls micro01 -f json -c4 | jq -r '.[0].state.network.enp5s0.addresses[] | select(.family == "inet") | .address') # Create a MicroCloud with storage directly given by-path on one node, and by filter on other nodes. lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud init --preseed << EOF @@ -61,7 +61,7 @@ EOF validate_system_microovn micro02 # Grow the MicroCloud with a new node, with filter-based storage selection. - lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud add --preseed << EOF + lxc exec micro01 --env TEST_CONSOLE=0 -- microcloud add --preseed <