Add storage manager support

Signed-off-by: Xu Tian <[email protected]>

virttest/virt_storage/backend/base.py

@@ -0,0 +1,149 @@
+import re
+import uuid
+
+from virttest.virt_storage import utils
+from virttest.virt_storage import virt_source
+from virttest.virt_storage import virt_target
+
+
+class BaseStoragePool(object):
+    TYPE = "none"
+
+    def __init__(self, name, params):
+        self.name = name
+        self.uuid = uuid.uuid1()
+        self._volumes = set()
+        self.sources = []
+        self.target = None
+        self._capacity = None
+        self._available = None
+        self._params = params
+        self._helper = None
+
+    @property
+    def capacity(self):
+        if self._capacity is None:
+            self._capacity = self.helper.capacity
+        return self._capacity
+
+    @property
+    def available(self):
+        if self._available is None:
+            self._available = self.helper.available
+        return self._available
+
+    @property
+    def helper(self):
+        if self._helper is None:
+            self._helper = utils.get_pool_helper(self)
+        return self._helper
+
+    def sources_define_by_params(self, params=None):
+        sources = list()
+        sources_params = params or self._params
+        for item in sources_params.objects("sources"):
+            _params = sources_params.object_params(item)
+            source = virt_source.VirtStoragePoolSource.source_define_by_params(
+                _params)
+            source.name = item
+            sources.append(source)
+        return sources
+
+    @classmethod
+    def pool_define_by_params(cls, name, params):
+        instance = cls(name, params)
+        instance.target = virt_target.VirtStoragePoolTarget.target_define_by_params(
+            params)
+        instance = utils.get_instance_by_params(cls, params, instance)
+        instance.set_special_opts_by_params(params)
+        return instance
+
+    def set_special_opts_by_params(self, params=None):
+        params = params or self._params
+        pattern = re.compile(r"(\w+)\s*=(\w+)\s*")
+        options = params.get("config_opts", "").split(",")
+        for option in options:
+            match = pattern.search(option)
+            if match:
+                key, val = match.groups()
+                setattr(self, key, val)
+
+    def start(self):
+        raise NotImplementedError
+
+    def stop(self):
+        raise NotImplementedError
+
+    def destroy(self):
+        """Destroy storage pools"""
+        self.stop()
+        self._volumes.clear()
+
+    def find_sources(self):
+        raise NotImplementedError
+
+    def create_volume(self, volume):
+        raise NotImplementedError
+
+    def refresh(self):
+        raise NotImplementedError
+
+    def remove_volume(self, volume):
+        raise NotImplementedError
+
+    def generate_volume_key_by_name(self, name):
+        raise NotImplementedError
+
+    def find_volume_by_name(self, name):
+        """find volume by name"""
+        return self.__find_volume_by_attr("name", name)
+
+    def find_volume_by_path(self, path):
+        """find volume by path"""
+        return self.__find_volume_by_attr("path", path)
+
+    def find_volume_by_key(self, key):
+        """find volume by key"""
+        return self.__find_volume_by_attr("key", key)
+
+    def find_volume_by_url(self, url):
+        """find volume by url"""
+        return self.__find_volume_by_attr("url", url)
+
+    def __find_volume_by_attr(self, attr, val):
+        """
+        Find the volume attribute is match given value
+
+        :param attr:  attribute name
+        :param val: attribute value
+        :return:  StorageVolume object or None
+        :raise:
+        """
+
+        matched_volumes = filter(
+            lambda x: str(
+                getattr(
+                    x,
+                    attr)) == str(val),
+            self.get_volumes())
+        return matched_volumes[0] if matched_volumes else None
+
+    def get_volumes(self):
+        return self._volumes
+
+    def add_volume(self, volume):
+        self._volumes.add(volume)
+
+    def acquire_volume(self, volume):
+        if volume.is_allocated:
+            return
+        assert self.state == "running", "Pool '%s' is not running!" % (
+            self.name)
+        self.create_volume(volume)
+        self.refresh()
+
+    def info(self):
+        return utils.get_instance_info(self)
+
+    def __str__(self):
+        return "%s:%s" % (self.__class__.__name__, self.name)

virttest/virt_storage/backend/direct_iscsi.py

@@ -0,0 +1,69 @@
+from avocado.core import exceptions
+
+from virttest.virt_storage.backend import base
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.utils import storage_util
+
+
+class IscsiDriectPool(base.BaseStoragePool):
+    TYPE = "iscsi-direct"
+
+    def find_sources(self):
+        """find lun in iscsi target"""
+        # Used host path as key of the volume
+        return self.helper.list_luns()
+
+    def start(self):
+        self.helper.login()
+        self.refresh()
+
+    def stop(self):
+        self.helper.logout()
+
+
+    def refresh(self):
+        lun = filter(
+            lambda x: not self.find_volume_by_key(
+                self.generate_volume_key_by_name(x)),
+            self.find_sources())
+        return map(self.create_volume_from_lun, lun)
+
+    def create_volume_from_lun(self, lun):
+        volume = storage_volume.StorageVolume(self)
+        volume.path = self.helper.get_host_path(lun)
+        volume.url = volume.key = self.generate_volume_key_by_name(lun)
+        volume.capacity = self.helper.get_size(lun)
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        """map exists lun to volume object"""
+        vol = self.__find_appropriate_lun(volume)
+        volume.path = vol.path
+        volume.url = volume.key = vol.url
+        self._volumes.remove(vol)
+        volume.is_allocated = True
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self._volumes.remove(volume)
+
+    def __find_appropriate_lun(self, vol):
+        """find appropriate lun for logical volume"""
+        volumes = filter(
+            lambda x: x.capacity - vol.capacity >= 0 and x.name is None, self._volumes)
+        try:
+            return sorted(volumes, key=lambda x: x.capacity)[0]
+        except Exception:
+            raise exceptions.TestError(
+                "No appropriate lun found for volume %s: %s" %
+                (vol.name, vol.info()))
+
+    @property
+    def available(self):
+        return sum(filter(lambda x: x.url and x.name is None, self._volumes))
+
+    def generate_volume_key_by_name(self, name):
+        return self.helper.get_url(name)

virttest/virt_storage/backend/directory.py

@@ -0,0 +1,54 @@
+import os
+
+from virttest.virt_storage.backend import base
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.utils import storage_util
+
+
+class DirectoryPool(base.BaseStoragePool):
+    TYPE = "directory"
+
+    def find_sources(self):
+        return self.helper.list_files()
+
+    def start(self):
+        self.helper.create()
+        self.refresh()
+
+    def stop(self):
+        pass
+
+    def delete(self):
+        self.helper.remove()
+
+    def refresh(self):
+        files = filter(
+            lambda x: not self.find_volume_by_key,
+            self.find_sources())
+        return map(self.create_volume_from_local, files)
+
+    def create_volume_from_local(self, path):
+        """
+        Create logical volume from local file
+        file size maybe mismatch, but need to resize in here
+        it will be recreate by qemu-img in next step.
+
+        """
+        volume = storage_volume.StorageVolume(self)
+        volume.path = volume.key = path
+        volume.url = self.helper.get_url(path)
+        volume.capacity = self.helper.get_size(path)
+        volume.is_allocated = True
+        return volume
+
+    def generate_volume_key_by_name(self, name):
+        return os.path.join(self.target.path, name)
+
+    def create_volume(self, volume):
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self.helper.remove_file(volume.path)
+        self._volumes.remove(volume)

virttest/virt_storage/backend/gluster.py

@@ -0,0 +1,34 @@
+from virttest.virt_storage.backend import nfs
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.utils import storage_util
+
+
+class GlusterPool(nfs.NfsPool):
+    TYPE = "gluster"
+
+    def find_sources(self):
+        urls = list()
+        for path in self.helper.list_files():
+            urls.append(self.helper.get_url(path))
+        return urls
+
+    def __create_volume_from_url(self, url):
+        volume = storage_volume.StorageVolume(self)
+        volume.url = volume.key = volume.path = url
+        volume.capacity = self.helper.get_size(url)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self.helper.remove_image(volume.key)
+        self._volumes.remove(volume)
+
+    def generate_volume_key_by_name(self, name):
+        return self.helper.get_url_by_name(name)
+
+    def create_volume(self, volume):
+        storage_util.create_volume(volume)
+        volume.url = volume.key
+        volume.path = self.helper.get_path(volume.url)
+        volume.is_allocated = True
+        return volume

virttest/virt_storage/backend/nfs.py

@@ -0,0 +1,39 @@
+from virttest.virt_storage.backend import directory
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.utils import storage_util
+
+
+class NfsPool(directory.DirectoryPool):
+    TYPE = "nfs"
+
+    def start(self):
+        self.helper.mount()
+        self.refresh()
+
+    def stop(self):
+        return self.helper.umount()
+
+    def delete(self):
+        self.helper.remove()
+
+    def refresh(self):
+        urls = filter(
+            lambda x: not self.find_volume_by_key,
+            self.find_sources())
+        return map(self.__create_volume_from_url, urls)
+
+    def __create_volume_from_url(self, url):
+        volume = storage_volume.StorageVolume(self)
+        volume.url = volume.key = url
+        volume.path = self.helper.get_path(url)
+        volume.capacity = self.helper.get_size(volume.path)
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def generate_volume_key_by_name(self, name):
+        return self.helper.get_url_by_name(name)

virttest/virt_storage/backend/rbd.py

@@ -0,0 +1,43 @@
+from virttest.virt_storage.backend import base
+from virttest.virt_storage import storage_volume
+from virttest.virt_storage.utils import storage_util
+
+
+class RbdPool(base.BaseStoragePool):
+    TYPE = 'rbd'
+
+    def find_sources(self):
+        return self.helper.list_images()
+
+    def start(self):
+        self.helper.connect()
+        self.refresh()
+
+    def stop(self):
+        return self.helper.shutdown()
+
+    def refresh(self):
+        keys = filter(lambda x: not self.find_volume_by_key(x),
+                      map(self.generate_volume_key_by_name, self.find_sources()))
+        return map(self.__create_volume_from_key, keys)
+
+    def __create_volume_from_key(self, key):
+        volume = storage_volume.StorageVolume(self)
+        volume.key = volume.url = volume.path = key
+        volume.capacity = self.helper.get_size(key)
+        volume.is_allocated = True
+        return volume
+
+    def create_volume(self, volume):
+        if volume.is_allocated:
+            self.helper.remove_image(volume.key)
+        storage_util.create_volume(volume)
+        volume.is_allocated = True
+        return volume
+
+    def remove_volume(self, volume):
+        self.helper.remove_image(volume.key)
+        self._volumes.remove(volume)
+
+    def generate_volume_key_by_name(self, name):
+        return self.helper.get_path(name)