Add storage manager support

Signed-off-by: Xu Tian <[email protected]>

virttest/virt_storage/backend/base.py

@@ -0,0 +1,135 @@
+import re
+import uuid
+
+from virttest.virt_storage import utils
+from virttest.virt_storage import virt_source
+from virttest.virt_storage import virt_target
+
+
+class BaseStoragePool(object):
+    TYPE = "none"
+
+    def __init__(self, name, params):
+        self.name = name
+        self.uuid = uuid.uuid1()
+        self.source = None
+        self.target = None
+        self._capacity = None
+        self._available = None
+        self._params = params
+        self._helper = None
+        self._volumes = set()
+
+    @property
+    def capacity(self):
+        if self._capacity is None:
+            self._capacity = self.helper.capacity
+        return self._capacity
+
+    @property
+    def available(self):
+        if self._available is None:
+            self._available = self.helper.available
+        return self._available
+
+    @property
+    def helper(self):
+        if self._helper is None:
+            self._helper = utils.get_pool_helper(self)
+        return self._helper
+
+    @classmethod
+    def pool_define_by_params(cls, name, params):
+        inst = cls(name, params)
+        inst.target = virt_target.PoolTarget.target_define_by_params(params)
+        if params.get("source"):
+            source_params = params.object_params(params.get("source"))
+            inst.source = virt_source.PoolSource.source_define_by_params(
+                params.get("source"), source_params)
+        inst.set_special_opts_by_params(params)
+        return inst
+
+    def set_special_opts_by_params(self, params=None):
+        params = params or self._params
+        pattern = re.compile(r"(\w+)\s*=(\w+)\s*")
+        options = params.get("config_opts", "").split(",")
+        for option in options:
+            match = pattern.search(option)
+            if match:
+                key, val = match.groups()
+                setattr(self, key, val)
+
+    def start(self):
+        raise NotImplementedError
+
+    def stop(self):
+        raise NotImplementedError
+
+    def destroy(self):
+        """Destroy storage pools"""
+        self.stop()
+        self._volumes.clear()
+
+    def find_sources(self):
+        raise NotImplementedError
+
+    def create_volume(self, volume):
+        raise NotImplementedError
+
+    def refresh(self):
+        raise NotImplementedError
+
+    def remove_volume(self, volume):
+        raise NotImplementedError
+
+    def find_volume_by_name(self, name):
+        """find volume by name"""
+        return self.__find_volume_by_attr("name", name)
+
+    def find_volume_by_path(self, path):
+        """find volume by path"""
+        return self.__find_volume_by_attr("path", path)
+
+    def find_volume_by_key(self, key):
+        """find volume by key"""
+        return self.__find_volume_by_attr("key", key)
+
+    def find_volume_by_url(self, url):
+        """find volume by url"""
+        return self.__find_volume_by_attr("url", url)
+
+    def __find_volume_by_attr(self, attr, val):
+        """
+        Find the volume attribute is match given value
+
+        :param attr:  attribute name
+        :param val: attribute value
+        :return:  StorageVolume object or None
+        :raise:
+        """
+
+        matched_volumes = filter(
+            lambda x: str(
+                getattr(
+                    x,
+                    attr)) == str(val),
+            self.get_volumes())
+        return matched_volumes[0] if matched_volumes else None
+
+    def get_volumes(self):
+        return self._volumes
+
+    def add_volume(self, volume):
+        self._volumes.add(volume)
+
+    def acquire_volume(self, volume):
+        if volume.is_allocated:
+            return
+        self.create_volume(volume)
+        self.refresh()
+
+    def info(self):
+        return utils.get_instance_info(self)
+
+    def __str__(self):
+        return "%s:%s" % (self.__class__.__name__, self.name)

virttest/virt_storage/backend/direct_iscsi.py

@@ -0,0 +1,66 @@
+from avocado.core import exceptions
+
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.backend import base
+from virttest.virt_storage.utils import storage_util
+
+
+class IscsiDriectPool(base.BaseStoragePool):
+    TYPE = "iscsi-direct"
+
+    def find_sources(self):
+        """find lun in iscsi target"""
+        # Used host path as key of the volume
+        return self.helper.list_disks()
+
+    def start(self):
+        self.helper.login()
+        self.refresh()
+
+    def stop(self):
+        self.helper.logout()
+
+    def refresh(self):
+        for path in self.find_sources():
+            if self.find_volume_by_path(path):
+                continue
+            else:
+                self.create_volume_from_path(path)
+
+    def create_volume_from_path(self, path):
+        capacity = self.helper.get_size(path)
+        url = self.helper.path_to_url(path)
+        volume = storage_volume.StorageVolume(self)
+        volume.path = path
+        volume.url = url
+        volume.capacity = capacity
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        """map exists lun to volume object"""
+        vol = self.__find_appropriate_lun(volume)
+        volume.path = vol.path
+        volume.url = vol.url
+        self._volumes.remove(vol)
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self._volumes.remove(volume)
+
+    def __find_appropriate_lun(self, vol):
+        """find appropriate lun for logical volume"""
+        volumes = filter(
+            lambda x: x.capacity - vol.capacity >= 0 and x.name is None, self._volumes)
+        try:
+            return sorted(volumes, key=lambda x: x.capacity)[0]
+        except Exception:
+            raise exceptions.TestError(
+                "No appropriate lun found for volume %s: %s" %
+                (vol.name, vol.info()))
+
+    @property
+    def available(self):
+        return sum(filter(lambda x: x.url and x.name is None, self._volumes))

virttest/virt_storage/backend/directory.py

@@ -0,0 +1,51 @@
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.backend import base
+from virttest.virt_storage.utils import storage_util
+
+
+class DirectoryPool(base.BaseStoragePool):
+    TYPE = "directory"
+
+    def find_sources(self):
+        return self.helper.list_files()
+
+    def start(self):
+        self.helper.create()
+        self.refresh()
+
+    def stop(self):
+        pass
+
+    def delete(self):
+        self.helper.remove()
+
+    def refresh(self):
+        files = filter(
+            lambda x: not self.find_volume_by_path,
+            self.find_sources())
+        return map(self.create_volume_from_local, files)
+
+    def create_volume_from_local(self, path):
+        """
+        Create logical volume from local file
+        file size maybe mismatch, but need to resize in here
+        it will be recreate by qemu-img in next step.
+
+        """
+        volume = storage_volume.StorageVolume(self)
+        volume.path = path
+        volume.url = self.helper.get_url(path)
+        volume.capacity = self.helper.get_size(path)
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        if volume.path is None:
+            volume.path = self.helper.get_path_by_name(volume.name)
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self.helper.remove_file(volume.path)
+        self._volumes.remove(volume)

virttest/virt_storage/backend/gluster.py

@@ -0,0 +1,32 @@
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.backend import nfs
+from virttest.virt_storage.utils import storage_util
+
+
+class GlusterPool(nfs.NfsPool):
+    TYPE = "gluster"
+
+    def find_sources(self):
+        urls = list()
+        for path in self.helper.list_files():
+            urls.append(self.helper.get_url(path))
+        return urls
+
+    def create_volume_from_remote(self, url):
+        volume = storage_volume.StorageVolume(self)
+        volume.url = volume.path = url
+        volume.capacity = self.helper.get_size(url)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self.helper.remove_image(volume.url)
+        self._volumes.remove(volume)
+
+    def create_volume(self, volume):
+        if volume.url is None:
+            url = self.helper.get_url_by_name(volume.name)
+            volume.url = volume.path = url
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume

virttest/virt_storage/backend/nfs.py

@@ -0,0 +1,45 @@
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.backend import directory
+from virttest.virt_storage.utils import storage_util
+
+
+class NfsPool(directory.DirectoryPool):
+    TYPE = "nfs"
+
+    def find_sources(self):
+        files = super(NfsPool, self).find_sources()
+        return map(self.helper.get_url, files)
+
+    def start(self):
+        self.helper.mount()
+        self.refresh()
+
+    def stop(self):
+        return self.helper.umount()
+
+    def delete(self):
+        self.helper.remove()
+
+    def refresh(self):
+        urls = filter(
+            lambda x: not self.find_volume_by_url,
+            self.find_sources())
+        return map(self.create_volume_from_remote, urls)
+
+    def create_volume_from_remote(self, url):
+        volume = storage_volume.StorageVolume(self)
+        volume.url = url
+        volume.path = self.helper.get_path(url)
+        volume.capacity = self.helper.get_size(volume.path)
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        if volume.url is None:
+            url = self.helper.get_url_by_name(volume.name)
+            volume.url = url
+            path = self.helper.get_path(volume.url)
+            volume.path = path
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume

virttest/virt_storage/backend/rbd.py

@@ -0,0 +1,42 @@
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.backend import base
+from virttest.virt_storage.utils import storage_util
+
+
+class RbdPool(base.BaseStoragePool):
+    TYPE = 'rbd'
+
+    def find_sources(self):
+        return map(self.helper.get_url_by_name, self.helper.list_images())
+
+    def start(self):
+        self.helper.connect()
+        self.refresh()
+
+    def stop(self):
+        return self.helper.shutdown()
+
+    def refresh(self):
+        urls = filter(lambda x: not self.find_volume_by_url(x), self.find_sources())
+        return list(map(self.create_volume_from_remote, urls))
+
+    def create_volume_from_remote(self, url):
+        volume = storage_volume.StorageVolume(self)
+        volume.path = volume.url = url
+        volume.capacity = self.helper.get_size(url)
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        if volume.url is None:
+            url = self.helper.get_url_by_name(volume.name)
+            volume.url = volume.path = url
+        if volume.is_allocated:
+            self.helper.remove_image(volume.url)
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self.helper.remove_image(volume.url)
+        self._volumes.remove(volume)