Add TestVMWithStorageNetwork

harvester_e2e_tests/fixtures/base.py

@@ -0,0 +1,18 @@
+from datetime import datetime, timedelta
+from time import sleep
+
+
+def wait_until(timeout, snooze=3):
+    def wait_until_decorator(api_func):
+        def wrapped(*args, **kwargs):
+            endtime = datetime.now() + timedelta(seconds=timeout)
+            while endtime > datetime.now():
+                qualified, (code, data) = api_func(*args, **kwargs)
+                if qualified:
+                    break
+                sleep(snooze)
+            return qualified, (code, data)
+
+        return wrapped
+
+    return wait_until_decorator

harvester_e2e_tests/fixtures/images.py

@@ -1,6 +1,7 @@
 from urllib.parse import urlparse, urljoin
 
 import pytest
+from .base import wait_until
 
 pytest_plugins = ["harvester_e2e_tests.fixtures.api_client"]
 
@@ -68,3 +69,26 @@ def url(self):
         if self.is_file:
             return self.url_result.geturl().split("file://", 1)[-1]
         return self.url_result.geturl()
+
+
+@pytest.fixture(scope="session")
+def image_checker(api_client, wait_timeout, sleep_timeout):
+    class ImageChecker:
+        def __init__(self):
+            self.images = api_client.images
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_downloaded(self, image_name):
+            code, data = self.images.get(image_name)
+            if data.get('status', {}).get('progress') == 100:
+                return True, (code, data)
+            return False, (code, data)
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_deleted(self, image_name):
+            code, data = self.images.get(image_name)
+            if code == 404:
+                return True, (code, data)
+            return False, (code, data)
+
+    return ImageChecker()

harvester_e2e_tests/fixtures/networks.py

@@ -1,4 +1,7 @@
+import json
+
 import pytest
+from .base import wait_until
 
 
 @pytest.fixture(scope="session")
@@ -13,3 +16,21 @@ def vlan_nic(request):
     vlan_nic = request.config.getoption('--vlan-nic')
     assert vlan_nic, f"VLAN NIC {vlan_nic} not configured correctly."
     return vlan_nic
+
+
+@pytest.fixture(scope="session")
+def network_checker(api_client, wait_timeout, sleep_timeout):
+    class NetworkChecker:
+        def __init__(self):
+            self.networks = api_client.networks
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_routed(self, vnet_name):
+            code, data = self.networks.get(vnet_name)
+            annotations = data['metadata'].get('annotations', {})
+            route = json.loads(annotations.get('network.harvesterhci.io/route', '{}'))
+            if code == 200 and route.get('connectivity') == 'true':
+                return True, (code, data)
+            return False, (code, data)
+
+    return NetworkChecker()

harvester_e2e_tests/fixtures/settings.py

@@ -0,0 +1,93 @@
+import json
+from ipaddress import ip_address, ip_network
+
+import pytest
+from .base import wait_until
+
+
+@pytest.fixture(scope="session")
+def setting_checker(api_client, wait_timeout, sleep_timeout):
+    class SettingChecker:
+        def __init__(self):
+            self.settings = api_client.settings
+            self.network_annotation = 'k8s.v1.cni.cncf.io/network-status'
+
+        def _storage_net_configured(self):
+            code, data = self.settings.get('storage-network')
+            if (cs := data.get('status', {}).get('conditions')):
+                if 'True' == cs[-1].get('status') and 'Completed' == cs[-1].get('reason'):
+                    return True, (code, data)
+            return False, (code, data)
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_storage_net_enabled_on_harvester(self):
+            snet_configured, (code, data) = self._storage_net_configured()
+            if snet_configured and data.get('value'):
+                return True, (code, data)
+            return False, (code, data)
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_storage_net_disabled_on_harvester(self):
+            snet_configured, (code, data) = self._storage_net_configured()
+            if snet_configured and not data.get('value'):
+                return True, (code, data)
+            return False, (code, data)
+
+        def _lh_instance_mgrs_running(self):
+            code, data = api_client.get_pods(namespace='longhorn-system')
+            if not (code == 200):
+                return False, (code, data)
+
+            lh_instance_mgrs = [pod for pod in data['data'] if 'instance-manager' in pod['id']]
+            if not lh_instance_mgrs:
+                return False, ("No instance-manager pods", data)
+
+            for imgr in lh_instance_mgrs:
+                if 'Running' != imgr['status']['phase']:
+                    return False, (f"Pod {imgr['id']} is NOT Running", imgr)
+
+                if not (self.network_annotation in imgr['metadata']['annotations']):
+                    return False, (f"No annotation '{self.network_annotation}' on pod", imgr)
+
+                networks = json.loads(imgr['metadata']['annotations'][self.network_annotation])
+                if not networks:
+                    return False, (f"Pod annotation '{self.network_annotation}' is empty", imgr)
+
+            return True, (None, lh_instance_mgrs)
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_storage_net_enabled_on_longhorn(self, snet_cidr):
+            imgrs_running, (code, data) = self._lh_instance_mgrs_running()
+            if not imgrs_running:
+                return False, (code, data)
+
+            for imgr in data:
+                networks = json.loads(imgr['metadata']['annotations'][self.network_annotation])
+                try:
+                    snet_network = next(n for n in networks if 'lhnet1' == n.get('interface'))
+                except StopIteration:
+                    return False, ("No dedicated interface interface 'lhnet1'", imgr)
+
+                snet_ips = snet_network.get('ips', ['::1'])
+                if not all(ip_address(sip) in ip_network(snet_cidr) for sip in snet_ips):
+                    return False, (f"Dedicated IPs {snet_ips} does NOT fits {snet_cidr}", imgr)
+
+            return True, (None, None)
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_storage_net_disabled_on_longhorn(self):
+            imgrs_running, (code, data) = self._lh_instance_mgrs_running()
+            if not imgrs_running:
+                return False, (code, data)
+
+            for imgr in data:
+                networks = json.loads(imgr['metadata']['annotations'][self.network_annotation])
+                try:
+                    next(n for n in networks if 'lhnet1' == n.get('interface'))
+                    return False, ("No dedicated interface 'lhnet1'", imgr)
+                except StopIteration:
+                    continue
+
+            return True, (None, None)
+
+    return SettingChecker()

harvester_e2e_tests/fixtures/volumes.py

@@ -0,0 +1,29 @@
+
+import pytest
+from .base import wait_until
+
+pytest_plugins = ["harvester_e2e_tests.fixtures.api_client"]
+
+
+@pytest.fixture(scope="session")
+def volume_checker(api_client, wait_timeout, sleep_timeout):
+    class VolumeChecker:
+        def __init__(self):
+            self.volumes = api_client.volumes
+            self.lhvolumes = api_client.lhvolumes
+
+        @wait_until(wait_timeout, sleep_timeout)
+        def wait_volumes_detached(self, vol_names):
+            for vol_name in vol_names:
+                code, data = self.volumes.get(name=vol_name)
+                if not (code == 200):
+                    return False, (code, data)
+
+                pvc_name = data["spec"]["volumeName"]
+                code, data = self.lhvolumes.get(pvc_name)
+                if not (200 == code and "detached" == data['status']['state']):
+                    return False, (code, data)
+
+            return True, (code, data)
+
+    return VolumeChecker()

harvester_e2e_tests/integrations/test_0_storage_network.py

@@ -9,7 +9,8 @@
 
 pytest_plugins = [
     "harvester_e2e_tests.fixtures.api_client",
-    "harvester_e2e_tests.fixtures.networks"
+    "harvester_e2e_tests.fixtures.networks",
+    "harvester_e2e_tests.fixtures.settings"
 ]
 
 
@@ -78,7 +79,9 @@ def cluster_network(request, api_client, unique_name):
 @pytest.mark.settings
 @pytest.mark.networks
 @pytest.mark.skip_version_before('v1.0.3')
-def test_storage_network(api_client, cluster_network, vlan_id, unique_name, wait_timeout):
+def test_storage_network(
+    api_client, cluster_network, vlan_id, unique_name, wait_timeout, setting_checker
+):
     '''
     To cover test:
     - https://harvester.github.io/tests/manual/_incoming/1055_dedicated_storage_network/
@@ -129,12 +132,7 @@ def test_storage_network(api_client, cluster_network, vlan_id, unique_name, wait
     cidr = route['cidr']
 
     # Create storage-network
-    code, data = api_client.settings.get('storage-network')
-    assert 200 == code, (code, data)
-    origin_spec = api_client.settings.Spec.from_dict(data)
-    spec = api_client.settings.StorageNetworkSpec.enable_with(
-        vlan_id, cluster_network, cidr
-    )
+    spec = api_client.settings.StorageNetworkSpec.enable_with(vlan_id, cluster_network, cidr)
     code, data = api_client.settings.update('storage-network', spec)
     assert 200 == code, (code, data)
 
@@ -184,6 +182,11 @@ def test_storage_network(api_client, cluster_network, vlan_id, unique_name, wait
             f"Not completed: {retries}"
         )
 
-    # Teardown
-    code, data = api_client.settings.update('storage-network', origin_spec)
+    # teardown
+    disable_spec = api_client.settings.StorageNetworkSpec.disable()
+    code, data = api_client.settings.update('storage-network', disable_spec)
     assert 200 == code, (code, data)
+    snet_disabled, (code, data) = setting_checker.wait_storage_net_disabled_on_harvester()
+    assert snet_disabled, (code, data)
+    snet_disabled, (code, data) = setting_checker.wait_storage_net_disabled_on_longhorn()
+    assert snet_disabled, (code, data)

harvester_e2e_tests/integrations/test_1_images.py

@@ -3,18 +3,17 @@
 import re
 import zlib
 from datetime import datetime, timedelta
-from ipaddress import ip_address, ip_network
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from time import sleep
 
 import pytest
 
-
 pytest_plugins = [
     "harvester_e2e_tests.fixtures.api_client",
     "harvester_e2e_tests.fixtures.images",
-    "harvester_e2e_tests.fixtures.networks"
+    "harvester_e2e_tests.fixtures.networks",
+    "harvester_e2e_tests.fixtures.settings"
 ]
 
 
@@ -163,65 +162,29 @@ def vlan_cidr(api_client, cluster_network, vlan_id, wait_timeout, sleep_timeout)
 
 
 @pytest.fixture(scope="class")
-def storage_network(api_client, cluster_network, vlan_id, vlan_cidr, wait_timeout, sleep_timeout):
-    code, data = api_client.settings.get('storage-network')
-    assert 200 == code, (code, data)
-
-    # Enable from Harvester side
-    spec_orig = api_client.settings.Spec.from_dict(data)
-    spec = api_client.settings.StorageNetworkSpec.enable_with(vlan_id, cluster_network, vlan_cidr)
-    code, data = api_client.settings.update('storage-network', spec)
+def storage_network(api_client, cluster_network, vlan_id, vlan_cidr, setting_checker):
+    ''' Ref. https://docs.harvesterhci.io/v1.3/advanced/storagenetwork/#configuration-example
+    '''
+    enable_spec = api_client.settings.StorageNetworkSpec.enable_with(
+        vlan_id, cluster_network, vlan_cidr
+    )
+    code, data = api_client.settings.update('storage-network', enable_spec)
     assert 200 == code, (code, data)
-
-    endtime = datetime.now() + timedelta(seconds=wait_timeout)
-    while endtime > datetime.now():
-        code, data = api_client.settings.get('storage-network')
-        conds = data.get('status', {}).get('conditions', [])
-        if conds and 'True' == conds[-1].get('status') and 'Completed' == conds[-1].get('reason'):
-            break
-        sleep(sleep_timeout)
-    else:
-        raise AssertionError(
-            f"Fail to enable storage-network with error: {code}, {data}"
-        )
-
-    # Check on Longhorn side
-    done, ip_range = [], ip_network(vlan_cidr)
-    endtime = datetime.now() + timedelta(seconds=wait_timeout)
-    while endtime > datetime.now():
-        code, data = api_client.get_pods(namespace='longhorn-system')
-        lh_instance_mgrs = [d for d in data['data']
-                            if 'instance-manager' in d['id'] and d['id'] not in done]
-        retries = []
-        for im in lh_instance_mgrs:
-            if 'Running' != im['status']['phase']:
-                retries.append(im)
-                continue
-            nets = json.loads(im['metadata']['annotations']['k8s.v1.cni.cncf.io/network-status'])
-            try:
-                dedicated = next(n for n in nets if 'lhnet1' == n.get('interface'))
-            except StopIteration:
-                retries.append(im)
-                continue
-
-            if not all(ip_address(ip) in ip_range for ip in dedicated.get('ips', ['::1'])):
-                retries.append(im)
-                continue
-
-        if not retries:
-            break
-        sleep(sleep_timeout)
-    else:
-        raise AssertionError(
-            f"{len(retries)} Longhorn's instance manager not be updated after {wait_timeout}s\n"
-            f"Not completed: {retries}"
-        )
+    snet_enabled, (code, data) = setting_checker.wait_storage_net_enabled_on_harvester()
+    assert snet_enabled, (code, data)
+    snet_enabled, (code, data) = setting_checker.wait_storage_net_enabled_on_longhorn(vlan_cidr)
+    assert snet_enabled, (code, data)
 
     yield
 
     # Teardown
-    code, data = api_client.settings.update('storage-network', spec_orig)
+    disable_spec = api_client.settings.StorageNetworkSpec.disable()
+    code, data = api_client.settings.update('storage-network', disable_spec)
     assert 200 == code, (code, data)
+    snet_disabled, (code, data) = setting_checker.wait_storage_net_disabled_on_harvester()
+    assert snet_disabled, (code, data)
+    snet_disabled, (code, data) = setting_checker.wait_storage_net_disabled_on_longhorn()
+    assert snet_disabled, (code, data)
 
 
 @pytest.mark.p0