Merge branch '3.2-devel' into 3.2-release

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "po"]
+	path = po
+	url = https://github.com/storaged-project/blivet-weblate.git

MANIFEST.in

@@ -1,3 +1,3 @@
-include COPYING ChangeLog Makefile python-blivet.spec release_notes.rst
+include README.md COPYING ChangeLog Makefile python-blivet.spec release_notes.rst
 recursive-include po *.mo *.po *.pot Makefile
 recursive-include examples *.py

Makefile

@@ -1,6 +1,8 @@
 PYTHON?=python3
 PKG_INSTALL?=dnf
 
+L10N_REPOSITORY=https://github.com/storaged-project/blivet-weblate
+
 PKGNAME=blivet
 SPECFILE=python-blivet.spec
 VERSION=$(shell $(PYTHON) setup.py --version)
@@ -12,28 +14,46 @@ VERSION_TAG=$(PKGNAME)-$(VERSION)
 
 COVERAGE=$(PYTHON) -m coverage
 
-ZANATA_PULL_ARGS = --transdir ./po/
-ZANATA_PUSH_ARGS = --srcdir ./po/ --push-type source --force
-
 MOCKCHROOT ?= fedora-rawhide-$(shell uname -m)
 
 all:
 	$(MAKE) -C po
 
 po-pull:
-	@which zanata >/dev/null 2>&1 || ( echo "You need to install Zanata client to download translation files"; exit 1 )
-	zanata pull $(ZANATA_PULL_ARGS)
-
-po-empty:
-	for lingua in $$(gawk 'match($$0, /locale>(.*)<\/locale/, ary) {print ary[1]}' ./zanata.xml) ; do \
-		[ -f ./po/$$lingua.po ] || \
-		msginit -i ./po/$(PKGNAME).pot -o ./po/$$lingua.po --no-translator || \
-		exit 1 ; \
-	done
+	git submodule update --init po
+	git submodule update --remote --merge po
 
-# Try to fetch the real .po files, but if that fails use the empty ones
-po-fallback:
-	$(MAKE) po-pull || $(MAKE) po-empty
+potfile:
+	make -C po $(PKGNAME).pot
+	# This algorithm will make these steps:
+	# - clone localization repository
+	# - copy pot file to this repository
+	# - check if pot file is changed (ignore the POT-Creation-Date otherwise it's always changed)
+	# - if not changed:
+	#   - remove cloned repository
+	# - if changed:
+	#   - add pot file
+	#   - commit pot file
+	#   - tell user to verify this file and push to the remote from the temp dir
+	TEMP_DIR=$$(mktemp --tmpdir -d $(PKGNAME)-localization-XXXXXXXXXX) || exit 1 ; \
+	git clone --depth 1 -b master -- $(L10N_REPOSITORY) $$TEMP_DIR || exit 2 ; \
+	cp po/$(PKGNAME).pot $$TEMP_DIR/ || exit 3 ; \
+	pushd $$TEMP_DIR ; \
+	git difftool --trust-exit-code -y -x "diff -u -I '^\"POT-Creation-Date: .*$$'" HEAD ./$(PKGNAME).pot &>/dev/null ; \
+	if [ $$? -eq 0  ] ; then \
+		popd ; \
+		echo "Pot file is up to date" ; \
+		rm -rf $$TEMP_DIR ; \
+		git submodule foreach git checkout -- blivet.pot ; \
+	else \
+		git add ./$(PKGNAME).pot && \
+		git commit -m "Update $(PKGNAME).pot" && \
+		popd && \
+		git submodule foreach git checkout -- blivet.pot ; \
+		echo "Pot file updated for the localization repository $(L10N_REPOSITORY)" && \
+		echo "Please confirm and push:" && \
+		echo "$$TEMP_DIR" ; \
+	fi ;
 
 install-requires:
 	@echo "*** Installing the dependencies required for testing and analysis ***"
@@ -65,11 +85,15 @@ pep8:
 	else \
 		echo "You need to install pycodestyle/pep8 to run this check."; exit 1; \
 	fi ; \
-	$$pep8 --ignore=E501,E402,E731,W504 blivet/ tests/ examples/
+	$$pep8 --ignore=E501,E402,E731,W504,E741 blivet/ tests/ examples/
 
-canary: po-fallback
+canary:
 	@echo "*** Running translation-canary tests ***"
-	PYTHONPATH=translation-canary:$(PYTHONPATH) $(PYTHON) -m translation_canary.translatable po/blivet.pot
+	@if [ ! -e po/$(PKGNAME).pot ]; then \
+		echo "Translation files not present. Skipping" ; \
+	else \
+		PYTHONPATH=translation-canary:$(PYTHONPATH) $(PYTHON) -m translation_canary.translatable po/$(PKGNAME).pot; \
+	fi ; \
 
 check:
 	@status=0; \
@@ -113,7 +137,6 @@ archive: po-pull
 	cp ChangeLog $(PKGNAME)-$(VERSION)/
 	( cd $(PKGNAME)-$(VERSION) && $(PYTHON) setup.py -q sdist --dist-dir .. )
 	rm -rf $(PKGNAME)-$(VERSION)
-	git checkout -- po/$(PKGNAME).pot
 	@echo "The archive is in $(PKGNAME)-$(VERSION).tar.gz"
 	@make tests-archive
 
@@ -147,10 +170,8 @@ bumpver: po-pull
 		opts="$${opts} -d" ; \
 	fi ; \
 	( scripts/makebumpver $${opts} ) || exit 1 ; \
-	make -C po $(PKGNAME).pot ; \
-	zanata push $(ZANATA_PUSH_ARGS)
 
-scratch-bumpver: po-empty
+scratch-bumpver:
 	@opts="-n $(PKGNAME) -v $(RPMVERSION) -r $(RPMRELEASE) --newrelease $(RC_RELEASE)" ; \
 	if [ ! -z "$(IGNORE)" ]; then \
 		opts="$${opts} -i $(IGNORE)" ; \
@@ -165,9 +186,8 @@ scratch-bumpver: po-empty
 		opts="$${opts} -d" ; \
 	fi ; \
 	( scripts/makebumpver $${opts} ) || exit 1 ; \
-	make -C po $(PKGNAME).pot
 
-scratch: po-empty
+scratch:
 	@rm -f ChangeLog
 	@make ChangeLog
 	@rm -rf $(PKGNAME)-$(VERSION).tar.gz

README.md

@@ -0,0 +1,70 @@
+Blivet is a python module for system storage configuration.
+
+### CI status
+
+<img alt="CI status" src="https://fedorapeople.org/groups/storage_apis/statuses/blivet-master.svg" width="100%" height="300ex" />
+
+### Licence
+
+See [COPYING](https://github.com/storaged-project/blivet/blob/master/COPYING)
+
+### Installation
+
+#### From Fedora repositories
+
+Blivet is available in Fedora repositories. You can install it using
+
+    # dnf install python3-blivet
+
+#### Daily builds (for Fedora)
+
+Daily builds of Blivet are available in `@storage/blivet-daily` Copr repository.
+You can enable it using
+
+    # dnf copr enable @storage/blivet-daily
+
+Daily builds of _libblockdev_ and _libbytesize_ are also in this repo.
+
+#### OBS repository (for Ubuntu and Debian)
+
+Packages for Debian (testing and unstable) and Ubuntu (19.04 and newer) are available through the Open Build Service.
+Instructions for adding the repository are available [here](https://software.opensuse.org/download.html?project=home:vtrefny&package=python3-blivet).
+
+#### PyPI
+
+Blivet is also available through the [Python Package Index](https://pypi.org/project/blivet/).
+You can install it using
+
+    $ pip3 install blivet
+
+Blivet depends on some C libraries that are not available on PyPI so you need to install these manually.
+
+The main dependencies include [libblockdev](https://github.com/storaged-project/libblockdev), [libbytesize](https://github.com/storaged-project/libbytesize), parted and their Python bindings.
+These libraries should be available on most distributions in the standard repositories.
+
+To install these dependencies use following commands:
+
+ * On Fedora and RHEL/CentOS based distributions:
+
+       # dnf install python3-blockdev libblockdev-plugins-all python3-bytesize libbytesize python3-pyparted parted libselinux-python3
+ * On Debian and Ubuntu based distributions:
+
+       # apt-get install python3-blockdev python3-bytesize python3-parted python3-selinux gir1.2-blockdev-2.0 libblockdev-lvm2 libblockdev-btrfs2 libblockdev-swap2 libblockdev-loop2 libblockdev-crypto2 libblockdev-mpath2 libblockdev-dm2 libblockdev-mdraid2 libblockdev-nvdimm2
+
+### Development
+
+See [CONTRIBUTING](https://github.com/storaged-project/blivet/blob/master/CONTRIBUTING)
+
+Developer documentation is available on our [website](http://storaged.org/blivet/docs/) or on [Read the Docs](https://blivet.readthedocs.io/en/latest/).
+
+Additional information about the release process, roadmap and other development-related materials are also available in the [GitHub Wiki](https://github.com/storaged-project/blivet/wiki).
+
+### Localization
+
+[![Translation](https://translate.fedoraproject.org/widgets/blivet/-/blivet-master/287x66-grey.png)](https://translate.fedoraproject.org/engage/blivet/?utm_source=widget)
+
+### Bug reporting
+
+Bugs should be reported to [bugzilla.redhat.com](https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora&component=python-blivet).
+
+You can also report bug using the [GitHub issues](https://github.com/storaged-project/blivet/issues).

TODO

@@ -1,16 +1,11 @@
-- Port to python3
 - More docs
 - More unit tests
 - More xample programs
 - optimize for large systems (>1000 block devices)
   - action sorting
   - clearpart?
   - audit device setup/teardown policies to reduce unnecessary churn
-- API stability
 - aynschronous actions
-- event handling
-- snapshot support
-- support for managing container member set
 - API for managing fstab and crypttab
 - improve interaction with the environment
   - disallow partitioning changes to disks with in-use partitions?
@@ -25,6 +20,3 @@
 - add StorageConfigurationError or similar so autopart failures due to the
   sanity check can be handled by offering the option of keeping the layout to
   attempt to fix the problem instead of always resetting
-- resize luks
-- check for read-only before trying to use disks
-

blivet/deviceaction.py

@@ -20,6 +20,8 @@
 # Red Hat Author(s): Dave Lehman <[email protected]>
 #
 
+import copy
+
 from six import add_metaclass
 
 from . import util
@@ -670,6 +672,8 @@ def execute(self, callbacks=None):
             msg = _("Created %(type)s on %(device)s") % {"type": self.device.format.type, "device": self.device.path}
             callbacks.create_format_post(CreateFormatPostData(msg))
 
+        self.device.original_format = copy.deepcopy(self.device.format)
+
     def cancel(self):
         if not self._applied:
             return

blivet/devices/cache.py

@@ -53,12 +53,6 @@ def stats(self):
         :rtype: :class:`CacheStats`
         """
 
-    @abc.abstractproperty
-    def mode(self):
-        """Mode of the cache (writeback/writethrough...)
-        :rtype: str
-        """
-
     @abc.abstractproperty
     def backing_device_name(self):
         """Name of the backing (big/slow) device of the cache (if any)"""

blivet/devices/lvm.py

@@ -1103,7 +1103,10 @@ def cache(self):
             # check if we have a cache pool internal LV
             for lv in self._internal_lvs:
                 if lv.int_lv_type == LVMInternalLVtype.cache_pool:
-                    self._cache = LVMCache(self, size=lv.size, exists=True)
+                    if self.seg_type == "cache":
+                        self._cache = LVMCache(self, size=lv.size, exists=True)
+                    elif self.seg_type == "writecache":
+                        self._cache = LVMWriteCache(self, size=lv.size, exists=True)
 
         return self._cache
 
@@ -2265,6 +2268,8 @@ def attach_cache(self, cache_pool_lv):
 
 class LVMCache(Cache):
 
+    type = "cache"
+
     """Class providing the cache-related functionality of a cached LV"""
 
     def __init__(self, cached_lv, size=None, md_size=None, exists=False, pvs=None, mode=None):
@@ -2408,6 +2413,52 @@ def detach(self):
         return ret
 
 
+class LVMWriteCache(Cache):
+
+    type = "writecache"
+
+    def __init__(self, cached_lv, size, exists):
+        self._cached_lv = cached_lv
+        self._exists = exists
+        self._size = size
+
+        if not self._exists:
+            raise ValueError("Only preexisting LVM writecache devices are currently supported.")
+
+    @property
+    def size(self):
+        return self._size
+
+    @property
+    def md_size(self):
+        # there are no metadata for writecache
+        return Size(0)
+
+    @property
+    def vg_space_used(self):
+        return self.size
+
+    @property
+    def exists(self):
+        return self._exists
+
+    @property
+    def stats(self):
+        return None
+
+    @property
+    def backing_device_name(self):
+        return self._cached_lv.name
+
+    @property
+    def cache_device_name(self):
+        vg_name = self._cached_lv.vg.name
+        return "%s-%s" % (vg_name, blockdev.lvm.cache_pool_name(vg_name, self._cached_lv.lvname))
+
+    def detach(self):
+        raise NotImplementedError
+
+
 class LVMCacheStats(CacheStats):
 
     def __init__(self, stats_data):

blivet/devices/partition.py

@@ -35,7 +35,7 @@
 from ..storage_log import log_method_call
 from .. import udev
 from ..formats import DeviceFormat, get_format
-from ..size import Size, MiB
+from ..size import Size, MiB, ROUND_DOWN
 
 import logging
 log = logging.getLogger("blivet")
@@ -934,8 +934,14 @@ def max_size(self):
 
     @property
     def resizable(self):
-        return super(PartitionDevice, self).resizable and \
-            self.disk.type != 'dasd' and self.disklabel_supported
+        if not self.exists:
+            return False
+        elif self.disk.type == 'dasd' or not self.disklabel_supported:
+            return False
+        elif self.is_extended:
+            return True
+        else:
+            return super(PartitionDevice, self).resizable
 
     def check_size(self):
         """ Check to make sure the size of the device is allowed by the
@@ -961,7 +967,8 @@ def populate_ksdata(self, data):
         data.resize = (self.exists and self.target_size and
                        self.target_size != self.current_size)
         if not self.exists:
-            data.size = self.req_base_size.convert_to(MiB)
+            # round this to nearest MiB before doing anything else
+            data.size = self.req_base_size.round_to_nearest(MiB, rounding=ROUND_DOWN).convert_to(spec=MiB)
             data.grow = self.req_grow
             if self.req_grow:
                 data.max_size_mb = self.req_max_size.convert_to(MiB)
@@ -974,4 +981,6 @@ def populate_ksdata(self, data):
             data.on_part = self.name                     # by-id
 
             if data.resize:
-                data.size = self.size.convert_to(MiB)
+                # on s390x in particular, fractional sizes are reported, which
+                # cause issues when writing to ks.cfg
+                data.size = self.size.round_to_nearest(MiB, rounding=ROUND_DOWN).convert_to(spec=MiB)