From 23f047710cc44cebb01372d0a7425876530ad3d4 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 8 Mar 2023 09:59:07 -0500 Subject: [PATCH 01/32] Add field for metadata version in static header --- src/engine/strat_engine/backstore/blockdev.rs | 7 ++++- src/engine/strat_engine/backstore/devices.rs | 3 +- src/engine/strat_engine/liminal/setup.rs | 12 ++++++++ src/engine/strat_engine/metadata/bda.rs | 21 ++++++++++++-- .../strat_engine/metadata/static_header.rs | 19 ++++++------ src/engine/types/mod.rs | 29 +++++++++++++++++++ 6 files changed, 76 insertions(+), 15 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev.rs index 36550a08d3..0de0e17c75 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev.rs @@ -37,7 +37,7 @@ use crate::{ }, types::{ Compare, DevUuid, DevicePath, Diff, EncryptionInfo, KeyDescription, Name, PoolUuid, - StateDiff, StratBlockDevDiff, + StateDiff, StratBlockDevDiff, StratSigblockVersion, }, }, stratis::{StratisError, StratisResult}, @@ -570,6 +570,11 @@ impl StratBlockDev { Ok(()) } } + + /// Get metadata version from static header + pub fn metadata_version(&self) -> StratSigblockVersion { + self.bda.sigblock_version() + } } impl<'a> Into for &'a StratBlockDev { diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index 5ed501edfd..f21f93f06f 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -37,7 +37,7 @@ use crate::{ STRATIS_FS_TYPE, }, }, - types::{DevUuid, DevicePath, EncryptionInfo, Name, PoolUuid}, + types::{DevUuid, DevicePath, EncryptionInfo, Name, PoolUuid, StratSigblockVersion}, }, stratis::{StratisError, StratisResult}, }; @@ -579,6 +579,7 @@ pub fn initialize_devices( }; let bda = BDA::new( + StratSigblockVersion::V1, StratisIdentifiers::new(pool_uuid, dev_uuid), mda_data_size, data_size, diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index 90f6e056ec..67b857c326 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -299,11 +299,13 @@ pub fn get_blockdevs( ) -> BDARecordResult> { let mut uuids = HashSet::new(); let mut duplicate_uuids = Vec::new(); + let mut metadata_version = HashSet::new(); for dev in &devs { let dev_uuid = dev.uuid(); if !uuids.insert(dev_uuid) { duplicate_uuids.push(dev_uuid); } + metadata_version.insert(dev.metadata_version()); } if !duplicate_uuids.is_empty() { @@ -314,6 +316,16 @@ pub fn get_blockdevs( return Err((StratisError::Msg(err_msg), bds_to_bdas(devs))); } + if metadata_version.len() > 1 { + return Err(( + StratisError::Msg(format!( + "Found mismatching metadata versions across block devices: {:?}", + metadata_version, + )), + bds_to_bdas(devs), + )); + } + let recorded_uuids: HashSet<_> = dev_map.keys().cloned().collect(); if uuids != recorded_uuids { let err_msg = format!( diff --git a/src/engine/strat_engine/metadata/bda.rs b/src/engine/strat_engine/metadata/bda.rs index 731e91bb9f..850b90da99 100644 --- a/src/engine/strat_engine/metadata/bda.rs +++ b/src/engine/strat_engine/metadata/bda.rs @@ -16,7 +16,7 @@ use crate::{ }, writing::SyncAll, }, - types::{DevUuid, PoolUuid}, + types::{DevUuid, PoolUuid, StratSigblockVersion}, }, stratis::StratisResult, }; @@ -30,6 +30,7 @@ pub struct BDA { impl Default for BDA { fn default() -> BDA { BDA::new( + StratSigblockVersion::V1, StratisIdentifiers::new(PoolUuid::nil(), DevUuid::nil()), MDADataSize::default(), BlockdevSize::default(), @@ -40,13 +41,19 @@ impl Default for BDA { impl BDA { pub fn new( + sigblock_version: StratSigblockVersion, identifiers: StratisIdentifiers, mda_data_size: MDADataSize, blkdev_size: BlockdevSize, initialization_time: DateTime, ) -> BDA { - let header = - StaticHeader::new(identifiers, mda_data_size, blkdev_size, initialization_time); + let header = StaticHeader::new( + sigblock_version, + identifiers, + mda_data_size, + blkdev_size, + initialization_time, + ); let regions = mda::MDARegions::new(header.mda_size); @@ -148,6 +155,11 @@ impl BDA { pub fn initialization_time(&self) -> DateTime { self.header.initialization_time } + + /// Get the sigblock version for this device. + pub fn sigblock_version(&self) -> StratSigblockVersion { + self.header.sigblock_version + } } #[cfg(test)] @@ -172,6 +184,7 @@ mod tests { let mut buf = Cursor::new(vec![0; buf_size]); let bda = BDA::new( + StratSigblockVersion::V1, sh.identifiers, sh.mda_size.region_size().data_size(), sh.blkdev_size, @@ -203,6 +216,7 @@ mod tests { ) ]); let mut bda = BDA::new( + StratSigblockVersion::V1, sh.identifiers, sh.mda_size.region_size().data_size(), sh.blkdev_size, @@ -253,6 +267,7 @@ mod tests { let buf_size = convert_test!(*sh.mda_size.bda_size().sectors().bytes(), u128, usize); let mut buf = Cursor::new(vec![0; buf_size]); let mut bda = BDA::new( + StratSigblockVersion::V1, sh.identifiers, sh.mda_size.region_size().data_size(), sh.blkdev_size, diff --git a/src/engine/strat_engine/metadata/static_header.rs b/src/engine/strat_engine/metadata/static_header.rs index 6ba80aa99e..aeb8f45cd8 100644 --- a/src/engine/strat_engine/metadata/static_header.rs +++ b/src/engine/strat_engine/metadata/static_header.rs @@ -25,7 +25,7 @@ use crate::{ }, writing::SyncAll, }, - types::{DevUuid, PoolUuid}, + types::{DevUuid, PoolUuid, StratSigblockVersion}, }, stratis::{StratisError, StratisResult}, }; @@ -34,8 +34,6 @@ const RESERVED_SECTORS: Sectors = Sectors(3 * IEC::Mi / (SECTOR_SIZE as u64)); / const STRAT_MAGIC: &[u8] = b"!Stra0tis\x86\xff\x02^\x41rh"; -const STRAT_SIGBLOCK_VERSION: u8 = 1; - const CASTAGNOLI: Crc = Crc::::new(&CRC_32_ISCSI); /// Data structure to hold results of reading and parsing a signature buffer. @@ -133,6 +131,7 @@ where #[derive(Debug, Eq, PartialEq)] pub struct StaticHeader { pub blkdev_size: BlockdevSize, + pub sigblock_version: StratSigblockVersion, pub identifiers: StratisIdentifiers, pub mda_size: MDASize, pub reserved_size: ReservedSize, @@ -142,6 +141,7 @@ pub struct StaticHeader { impl StaticHeader { pub fn new( + sigblock_version: StratSigblockVersion, identifiers: StratisIdentifiers, mda_data_size: MDADataSize, blkdev_size: BlockdevSize, @@ -149,6 +149,7 @@ impl StaticHeader { ) -> StaticHeader { StaticHeader { blkdev_size, + sigblock_version, identifiers, mda_size: mda_data_size.region_size().mda_size(), reserved_size: ReservedSize::new(RESERVED_SECTORS), @@ -496,7 +497,7 @@ impl StaticHeader { let mut buf = [0u8; bytes!(static_header_size::SIGBLOCK_SECTORS)]; buf[4..20].clone_from_slice(STRAT_MAGIC); LittleEndian::write_u64(&mut buf[20..28], *self.blkdev_size.sectors()); - buf[28] = STRAT_SIGBLOCK_VERSION; + buf[28] = u8::from(self.sigblock_version); buf[32..64].clone_from_slice(uuid_to_string!(self.identifiers.pool_uuid).as_bytes()); buf[64..96].clone_from_slice(uuid_to_string!(self.identifiers.device_uuid).as_bytes()); LittleEndian::write_u64(&mut buf[96..104], *self.mda_size.sectors()); @@ -530,12 +531,8 @@ impl StaticHeader { let blkdev_size = BlockdevSize::new(Sectors(LittleEndian::read_u64(&buf[20..28]))); - let version = buf[28]; - if version != STRAT_SIGBLOCK_VERSION { - return Err(StratisError::Msg(format!( - "Unknown sigblock version: {version}" - ))); - } + let version_buf = buf[28]; + let version = StratSigblockVersion::try_from(version_buf)?; let pool_uuid = PoolUuid::parse_str(from_utf8(&buf[32..64])?)?; let dev_uuid = DevUuid::parse_str(from_utf8(&buf[64..96])?)?; @@ -547,6 +544,7 @@ impl StaticHeader { Ok(Some(StaticHeader { identifiers: StratisIdentifiers::new(pool_uuid, dev_uuid), blkdev_size, + sigblock_version: version, mda_size, reserved_size: ReservedSize::new(Sectors(LittleEndian::read_u64(&buf[104..112]))), flags: 0, @@ -623,6 +621,7 @@ pub mod tests { MDADataSize::new(MDADataSize::default().bytes() + Bytes::from(mda_size_factor * 4)); let blkdev_size = (Bytes::from(IEC::Mi) + Sectors(blkdev_size).bytes()).sectors(); StaticHeader::new( + StratSigblockVersion::V1, StratisIdentifiers::new(pool_uuid, dev_uuid), mda_data_size, BlockdevSize::new(blkdev_size), diff --git a/src/engine/types/mod.rs b/src/engine/types/mod.rs index 4fd9643511..597153f0b4 100644 --- a/src/engine/types/mod.rs +++ b/src/engine/types/mod.rs @@ -500,3 +500,32 @@ impl UuidOrConflict { } } } + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum StratSigblockVersion { + V1 = 1, + V2 = 2, +} + +impl TryFrom for StratSigblockVersion { + type Error = StratisError; + + fn try_from(value: u8) -> Result { + match value { + 1u8 => Ok(StratSigblockVersion::V1), + 2u8 => Ok(StratSigblockVersion::V2), + _ => Err(StratisError::Msg(format!( + "Unknown sigblock version: {value}" + ))), + } + } +} + +impl From for u8 { + fn from(version: StratSigblockVersion) -> Self { + match version { + StratSigblockVersion::V1 => 1u8, + StratSigblockVersion::V2 => 2u8, + } + } +} From fc21a8f5984e0294b4d718a1ecbe800f720d5f37 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 21 Jun 2023 13:17:16 -0400 Subject: [PATCH 02/32] Lift crypt module up one level --- .../strat_engine/backstore/backstore.rs | 14 ++++----- src/engine/strat_engine/backstore/blockdev.rs | 2 +- .../strat_engine/backstore/blockdevmgr.rs | 2 +- src/engine/strat_engine/backstore/devices.rs | 8 ++--- src/engine/strat_engine/backstore/mod.rs | 12 ++------ .../{backstore => }/crypt/consts.rs | 0 .../{backstore => }/crypt/handle.rs | 29 +++++++++---------- .../{backstore => }/crypt/macros.rs | 0 .../strat_engine/{backstore => }/crypt/mod.rs | 2 +- .../{backstore => }/crypt/shared.rs | 27 ++++++++--------- src/engine/strat_engine/engine.rs | 2 +- src/engine/strat_engine/liminal/identify.rs | 3 +- src/engine/strat_engine/liminal/liminal.rs | 3 +- src/engine/strat_engine/liminal/setup.rs | 3 +- src/engine/strat_engine/mod.rs | 3 +- src/engine/strat_engine/tests/loopbacked.rs | 2 +- src/engine/strat_engine/tests/real.rs | 2 +- 17 files changed, 50 insertions(+), 64 deletions(-) rename src/engine/strat_engine/{backstore => }/crypt/consts.rs (100%) rename src/engine/strat_engine/{backstore => }/crypt/handle.rs (96%) rename src/engine/strat_engine/{backstore => }/crypt/macros.rs (100%) rename src/engine/strat_engine/{backstore => }/crypt/mod.rs (99%) rename src/engine/strat_engine/{backstore => }/crypt/shared.rs (98%) diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index 39931818ca..2b03f0febb 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -17,15 +17,11 @@ use crate::{ shared::gather_encryption_info, strat_engine::{ backstore::{ - blockdev::StratBlockDev, - blockdevmgr::BlockDevMgr, - cache_tier::CacheTier, - crypt::{ - back_up_luks_header, interpret_clevis_config, restore_luks_header, CryptHandle, - }, - data_tier::DataTier, - devices::UnownedDevices, - shared::BlockSizeSummary, + blockdev::StratBlockDev, blockdevmgr::BlockDevMgr, cache_tier::CacheTier, + data_tier::DataTier, devices::UnownedDevices, shared::BlockSizeSummary, + }, + crypt::{ + back_up_luks_header, interpret_clevis_config, restore_luks_header, CryptHandle, }, dm::{get_dm, list_of_backstore_devices, remove_optional_devices}, metadata::{MDADataSize, BDA}, diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev.rs index 0de0e17c75..1e1697ed46 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev.rs @@ -23,10 +23,10 @@ use crate::{ engine::{BlockDev, DumpState}, strat_engine::{ backstore::{ - crypt::CryptHandle, devices::BlockSizes, range_alloc::{PerDevSegments, RangeAllocator}, }, + crypt::CryptHandle, device::blkdev_size, metadata::{ disown_device, static_header, BDAExtendedSize, BlockdevSize, MDADataSize, diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index 4dff7b4e26..fca9ee1057 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -19,10 +19,10 @@ use crate::{ strat_engine::{ backstore::{ blockdev::StratBlockDev, - crypt::CryptHandle, devices::{initialize_devices, wipe_blockdevs, UnownedDevices}, shared::{BlkDevSegment, Segment}, }, + crypt::CryptHandle, metadata::{MDADataSize, BDA}, serde_structs::{BaseBlockDevSave, Recordable}, shared::bds_to_bdas, diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index f21f93f06f..10b972d040 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -23,10 +23,8 @@ use libblkid_rs::{BlkidCache, BlkidProbe}; use crate::{ engine::{ strat_engine::{ - backstore::{ - blockdev::{StratBlockDev, UnderlyingDevice}, - crypt::CryptHandle, - }, + backstore::blockdev::{StratBlockDev, UnderlyingDevice}, + crypt::CryptHandle, device::{blkdev_logical_sector_size, blkdev_physical_sector_size, blkdev_size}, metadata::{ device_identifiers, disown_device, BlockdevSize, MDADataSize, StratisIdentifiers, @@ -822,7 +820,7 @@ mod tests { use crate::engine::{ strat_engine::{ - backstore::crypt::CryptHandle, + crypt::CryptHandle, metadata::device_identifiers, tests::{crypt, loopbacked, real}, }, diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index 7eb1fc43cd..54198b4e0f 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -7,21 +7,15 @@ mod backstore; mod blockdev; mod blockdevmgr; mod cache_tier; -mod crypt; mod data_tier; mod devices; mod range_alloc; mod shared; +#[cfg(test)] +pub use self::devices::initialize_devices; pub use self::{ backstore::Backstore, blockdev::{StratBlockDev, UnderlyingDevice}, - crypt::{ - crypt_metadata_size, register_clevis_token, set_up_crypt_logging, CryptHandle, - CLEVIS_TANG_TRUST_URL, - }, - devices::{find_stratis_devs_by_uuid, ProcessedPathInfos, UnownedDevices}, + devices::{find_stratis_devs_by_uuid, get_devno_from_path, ProcessedPathInfos, UnownedDevices}, }; - -#[cfg(test)] -pub use self::devices::initialize_devices; diff --git a/src/engine/strat_engine/backstore/crypt/consts.rs b/src/engine/strat_engine/crypt/consts.rs similarity index 100% rename from src/engine/strat_engine/backstore/crypt/consts.rs rename to src/engine/strat_engine/crypt/consts.rs diff --git a/src/engine/strat_engine/backstore/crypt/handle.rs b/src/engine/strat_engine/crypt/handle.rs similarity index 96% rename from src/engine/strat_engine/backstore/crypt/handle.rs rename to src/engine/strat_engine/crypt/handle.rs index dc7330bb15..ec25c1af64 100644 --- a/src/engine/strat_engine/backstore/crypt/handle.rs +++ b/src/engine/strat_engine/crypt/handle.rs @@ -25,24 +25,21 @@ use crate::{ engine::{ engine::MAX_STRATIS_PASS_SIZE, strat_engine::{ - backstore::{ - crypt::{ - consts::{ - CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, - DEFAULT_CRYPT_METADATA_SIZE, LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, - STRATIS_TOKEN_ID, - }, - shared::{ - acquire_crypt_device, activate, add_keyring_keyslot, check_luks2_token, - clevis_info_from_metadata, ensure_inactive, ensure_wiped, - get_keyslot_number, interpret_clevis_config, key_desc_from_metadata, - load_crypt_metadata, replace_pool_name, setup_crypt_device, - setup_crypt_handle, wipe_fallback, StratisLuks2Token, - }, + backstore::get_devno_from_path, + cmd::{clevis_decrypt, clevis_luks_bind, clevis_luks_regen, clevis_luks_unbind}, + crypt::{ + consts::{ + CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, STRATIS_TOKEN_ID, + }, + shared::{ + acquire_crypt_device, activate, add_keyring_keyslot, check_luks2_token, + clevis_info_from_metadata, ensure_inactive, ensure_wiped, get_keyslot_number, + interpret_clevis_config, key_desc_from_metadata, load_crypt_metadata, + replace_pool_name, setup_crypt_device, setup_crypt_handle, wipe_fallback, + StratisLuks2Token, }, - devices::get_devno_from_path, }, - cmd::{clevis_decrypt, clevis_luks_bind, clevis_luks_regen, clevis_luks_unbind}, dm::DEVICEMAPPER_PATH, metadata::StratisIdentifiers, names::format_crypt_name, diff --git a/src/engine/strat_engine/backstore/crypt/macros.rs b/src/engine/strat_engine/crypt/macros.rs similarity index 100% rename from src/engine/strat_engine/backstore/crypt/macros.rs rename to src/engine/strat_engine/crypt/macros.rs diff --git a/src/engine/strat_engine/backstore/crypt/mod.rs b/src/engine/strat_engine/crypt/mod.rs similarity index 99% rename from src/engine/strat_engine/backstore/crypt/mod.rs rename to src/engine/strat_engine/crypt/mod.rs index 2ac2b314b5..3bf75b68f8 100644 --- a/src/engine/strat_engine/backstore/crypt/mod.rs +++ b/src/engine/strat_engine/crypt/mod.rs @@ -38,7 +38,7 @@ mod tests { use crate::engine::{ strat_engine::{ - backstore::crypt::{ + crypt::{ consts::{ CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, diff --git a/src/engine/strat_engine/backstore/crypt/shared.rs b/src/engine/strat_engine/crypt/shared.rs similarity index 98% rename from src/engine/strat_engine/backstore/crypt/shared.rs rename to src/engine/strat_engine/crypt/shared.rs index 6e95c84bbc..d22037461e 100644 --- a/src/engine/strat_engine/backstore/crypt/shared.rs +++ b/src/engine/strat_engine/crypt/shared.rs @@ -37,23 +37,20 @@ use libcryptsetup_rs::{ use crate::{ engine::{ strat_engine::{ - backstore::{ - crypt::{ - consts::{ - CLEVIS_LUKS_TOKEN_ID, CLEVIS_RECURSION_LIMIT, CLEVIS_TANG_TRUST_URL, - CLEVIS_TOKEN_NAME, DEFAULT_CRYPT_KEYSLOTS_SIZE, - DEFAULT_CRYPT_METADATA_SIZE, LUKS2_SECTOR_SIZE, LUKS2_TOKEN_ID, - LUKS2_TOKEN_TYPE, STRATIS_TOKEN_DEVNAME_KEY, STRATIS_TOKEN_DEV_UUID_KEY, - STRATIS_TOKEN_ID, STRATIS_TOKEN_POOLNAME_KEY, STRATIS_TOKEN_POOL_UUID_KEY, - STRATIS_TOKEN_TYPE, TOKEN_KEYSLOTS_KEY, TOKEN_TYPE_KEY, - }, - handle::{CryptHandle, CryptMetadata}, + backstore::get_devno_from_path, + cmd::clevis_decrypt, + crypt::{ + consts::{ + CLEVIS_LUKS_TOKEN_ID, CLEVIS_RECURSION_LIMIT, CLEVIS_TANG_TRUST_URL, + CLEVIS_TOKEN_NAME, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_SECTOR_SIZE, LUKS2_TOKEN_ID, LUKS2_TOKEN_TYPE, STRATIS_TOKEN_DEVNAME_KEY, + STRATIS_TOKEN_DEV_UUID_KEY, STRATIS_TOKEN_ID, STRATIS_TOKEN_POOLNAME_KEY, + STRATIS_TOKEN_POOL_UUID_KEY, STRATIS_TOKEN_TYPE, TOKEN_KEYSLOTS_KEY, + TOKEN_TYPE_KEY, }, - devices::get_devno_from_path, + handle::{CryptHandle, CryptMetadata}, }, - cmd::clevis_decrypt, - dm::get_dm, - dm::DEVICEMAPPER_PATH, + dm::{get_dm, DEVICEMAPPER_PATH}, keys, metadata::StratisIdentifiers, }, diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index cb5082bf2e..a605d19b31 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -752,8 +752,8 @@ mod test { use crate::engine::{ engine::Pool, strat_engine::{ - backstore::crypt_metadata_size, cmd, + crypt::crypt_metadata_size, ns::unshare_mount_namespace, tests::{crypt, loopbacked, real, FailDevice}, }, diff --git a/src/engine/strat_engine/liminal/identify.rs b/src/engine/strat_engine/liminal/identify.rs index b03e96122c..b07291cda1 100644 --- a/src/engine/strat_engine/liminal/identify.rs +++ b/src/engine/strat_engine/liminal/identify.rs @@ -52,7 +52,8 @@ use devicemapper::Device; use crate::engine::{ strat_engine::{ - backstore::{CryptHandle, StratBlockDev}, + backstore::StratBlockDev, + crypt::CryptHandle, metadata::{static_header, StratisIdentifiers, BDA}, udev::{ block_enumerator, decide_ownership, UdevOwnership, CRYPTO_FS_TYPE, FS_TYPE_KEY, diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index 36743874ac..650ad6a0c6 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -17,7 +17,8 @@ use crate::{ engine::{ engine::{DumpState, Pool, StateDiff}, strat_engine::{ - backstore::{find_stratis_devs_by_uuid, CryptHandle, StratBlockDev}, + backstore::{find_stratis_devs_by_uuid, StratBlockDev}, + crypt::CryptHandle, dm::{has_leftover_devices, stop_partially_constructed_pool}, liminal::{ device_info::{ diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index 67b857c326..7f79ad46e8 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -18,7 +18,8 @@ use devicemapper::Sectors; use crate::{ engine::{ strat_engine::{ - backstore::{CryptHandle, StratBlockDev, UnderlyingDevice}, + backstore::{StratBlockDev, UnderlyingDevice}, + crypt::CryptHandle, device::blkdev_size, liminal::device_info::{LStratisDevInfo, LStratisInfo}, metadata::BDA, diff --git a/src/engine/strat_engine/mod.rs b/src/engine/strat_engine/mod.rs index 969ac51127..e4063d7c0d 100644 --- a/src/engine/strat_engine/mod.rs +++ b/src/engine/strat_engine/mod.rs @@ -4,6 +4,7 @@ mod backstore; mod cmd; +mod crypt; mod device; mod devlinks; mod dm; @@ -22,7 +23,7 @@ mod udev; mod writing; pub use self::{ - backstore::{ + crypt::{ crypt_metadata_size, register_clevis_token, set_up_crypt_logging, CLEVIS_TANG_TRUST_URL, }, dm::{get_dm, get_dm_init}, diff --git a/src/engine/strat_engine/tests/loopbacked.rs b/src/engine/strat_engine/tests/loopbacked.rs index 2fbcc96c50..9c6fb0f90f 100644 --- a/src/engine/strat_engine/tests/loopbacked.rs +++ b/src/engine/strat_engine/tests/loopbacked.rs @@ -17,7 +17,7 @@ use devicemapper::{Bytes, Sectors, IEC}; use crate::{ engine::strat_engine::{ - backstore::register_clevis_token, + crypt::register_clevis_token, tests::{logger::init_logger, util::clean_up}, }, stratis::StratisResult, diff --git a/src/engine/strat_engine/tests/real.rs b/src/engine/strat_engine/tests/real.rs index 045a00b03a..e0d88a203c 100644 --- a/src/engine/strat_engine/tests/real.rs +++ b/src/engine/strat_engine/tests/real.rs @@ -20,8 +20,8 @@ use devicemapper::{ }; use crate::engine::strat_engine::{ - backstore::register_clevis_token, cmd::udev_settle, + crypt::register_clevis_token, device::blkdev_size, dm::get_dm, tests::{logger::init_logger, util::clean_up}, From c62b149b1a8075cade56d1111ae09433c0b4ade8 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 30 Jun 2023 12:10:50 -0400 Subject: [PATCH 03/32] Separate new and legacy versions of the crypt handle --- .../strat_engine/backstore/backstore.rs | 3 +- src/engine/strat_engine/backstore/blockdev.rs | 2 +- .../strat_engine/backstore/blockdevmgr.rs | 2 +- src/engine/strat_engine/backstore/devices.rs | 4 +- src/engine/strat_engine/crypt/handle.rs | 722 ------- src/engine/strat_engine/crypt/handle/mod.rs | 6 + src/engine/strat_engine/crypt/handle/v1.rs | 1685 +++++++++++++++++ src/engine/strat_engine/crypt/handle/v2.rs | 1240 ++++++++++++ src/engine/strat_engine/crypt/mod.rs | 574 +----- src/engine/strat_engine/crypt/shared.rs | 438 +---- src/engine/strat_engine/liminal/identify.rs | 2 +- src/engine/strat_engine/liminal/liminal.rs | 2 +- src/engine/strat_engine/liminal/setup.rs | 2 +- src/engine/strat_engine/names.rs | 20 + src/engine/types/mod.rs | 2 + 15 files changed, 2984 insertions(+), 1720 deletions(-) delete mode 100644 src/engine/strat_engine/crypt/handle.rs create mode 100644 src/engine/strat_engine/crypt/handle/mod.rs create mode 100644 src/engine/strat_engine/crypt/handle/v1.rs create mode 100644 src/engine/strat_engine/crypt/handle/v2.rs diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index 2b03f0febb..e9cbabe934 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -21,7 +21,8 @@ use crate::{ data_tier::DataTier, devices::UnownedDevices, shared::BlockSizeSummary, }, crypt::{ - back_up_luks_header, interpret_clevis_config, restore_luks_header, CryptHandle, + back_up_luks_header, handle::v1::CryptHandle, interpret_clevis_config, + restore_luks_header, }, dm::{get_dm, list_of_backstore_devices, remove_optional_devices}, metadata::{MDADataSize, BDA}, diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev.rs index 1e1697ed46..6f69444474 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev.rs @@ -26,7 +26,7 @@ use crate::{ devices::BlockSizes, range_alloc::{PerDevSegments, RangeAllocator}, }, - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, device::blkdev_size, metadata::{ disown_device, static_header, BDAExtendedSize, BlockdevSize, MDADataSize, diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index fca9ee1057..f145df442c 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -22,7 +22,7 @@ use crate::{ devices::{initialize_devices, wipe_blockdevs, UnownedDevices}, shared::{BlkDevSegment, Segment}, }, - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, metadata::{MDADataSize, BDA}, serde_structs::{BaseBlockDevSave, Recordable}, shared::bds_to_bdas, diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index 10b972d040..3f7e1c9ef9 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -24,7 +24,7 @@ use crate::{ engine::{ strat_engine::{ backstore::blockdev::{StratBlockDev, UnderlyingDevice}, - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, device::{blkdev_logical_sector_size, blkdev_physical_sector_size, blkdev_size}, metadata::{ device_identifiers, disown_device, BlockdevSize, MDADataSize, StratisIdentifiers, @@ -820,7 +820,7 @@ mod tests { use crate::engine::{ strat_engine::{ - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, metadata::device_identifiers, tests::{crypt, loopbacked, real}, }, diff --git a/src/engine/strat_engine/crypt/handle.rs b/src/engine/strat_engine/crypt/handle.rs deleted file mode 100644 index ec25c1af64..0000000000 --- a/src/engine/strat_engine/crypt/handle.rs +++ /dev/null @@ -1,722 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -use std::{ - fmt::Debug, - path::{Path, PathBuf}, -}; - -use either::Either; -use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use serde_json::{to_value, Value}; - -use devicemapper::{Device, DmName, DmNameBuf, Sectors}; -use libcryptsetup_rs::{ - c_uint, - consts::{ - flags::{CryptActivate, CryptVolumeKey}, - vals::{EncryptionFormat, KeyslotsSize, MetadataSize}, - }, - CryptDevice, CryptInit, CryptParamsLuks2, CryptParamsLuks2Ref, SafeMemHandle, TokenInput, -}; - -use crate::{ - engine::{ - engine::MAX_STRATIS_PASS_SIZE, - strat_engine::{ - backstore::get_devno_from_path, - cmd::{clevis_decrypt, clevis_luks_bind, clevis_luks_regen, clevis_luks_unbind}, - crypt::{ - consts::{ - CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, - LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, STRATIS_TOKEN_ID, - }, - shared::{ - acquire_crypt_device, activate, add_keyring_keyslot, check_luks2_token, - clevis_info_from_metadata, ensure_inactive, ensure_wiped, get_keyslot_number, - interpret_clevis_config, key_desc_from_metadata, load_crypt_metadata, - replace_pool_name, setup_crypt_device, setup_crypt_handle, wipe_fallback, - StratisLuks2Token, - }, - }, - dm::DEVICEMAPPER_PATH, - metadata::StratisIdentifiers, - names::format_crypt_name, - }, - types::{ - DevUuid, DevicePath, EncryptionInfo, KeyDescription, Name, PoolUuid, SizedKeyMemory, - UnlockMethod, - }, - ClevisInfo, - }, - stratis::{StratisError, StratisResult}, -}; - -#[derive(Debug, Clone)] -pub struct CryptMetadata { - pub physical_path: DevicePath, - pub identifiers: StratisIdentifiers, - pub encryption_info: EncryptionInfo, - pub activation_name: DmNameBuf, - pub activated_path: PathBuf, - pub pool_name: Option, - pub device: Device, -} - -/// Handle for performing all operations on an encrypted device. -/// -/// `Clone` is derived for this data structure because `CryptHandle` acquires -/// a new crypt device context for each operation. -#[derive(Debug, Clone)] -pub struct CryptHandle { - metadata: CryptMetadata, -} - -impl CryptHandle { - pub(super) fn new( - physical_path: DevicePath, - pool_uuid: PoolUuid, - dev_uuid: DevUuid, - encryption_info: EncryptionInfo, - pool_name: Option, - devno: Device, - ) -> CryptHandle { - let activation_name = format_crypt_name(&dev_uuid); - let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] - .into_iter() - .collect::(); - let activated_path = path.canonicalize().unwrap_or(path); - CryptHandle { - metadata: CryptMetadata { - physical_path, - identifiers: StratisIdentifiers { - pool_uuid, - device_uuid: dev_uuid, - }, - encryption_info, - activation_name, - pool_name, - device: devno, - activated_path, - }, - } - } - - /// Check whether the given physical device can be unlocked with the current - /// environment (e.g. the proper key is in the kernel keyring, the device - /// is formatted as a LUKS2 device, etc.) - pub fn can_unlock( - physical_path: &Path, - try_unlock_keyring: bool, - try_unlock_clevis: bool, - ) -> bool { - fn can_unlock_with_failures( - physical_path: &Path, - try_unlock_keyring: bool, - try_unlock_clevis: bool, - ) -> StratisResult { - let mut device = acquire_crypt_device(physical_path)?; - - if try_unlock_keyring { - let key_description = key_desc_from_metadata(&mut device); - - if key_description.is_some() { - check_luks2_token(&mut device)?; - } - } - if try_unlock_clevis { - log_on_failure!( - device.token_handle().activate_by_token::<()>( - None, - Some(CLEVIS_LUKS_TOKEN_ID), - None, - CryptActivate::empty(), - ), - "libcryptsetup reported that the decrypted Clevis passphrase \ - is unable to open the encrypted device" - ); - } - Ok(true) - } - - can_unlock_with_failures(physical_path, try_unlock_keyring, try_unlock_clevis) - .map_err(|e| { - warn!( - "stratisd was unable to simulate opening the given device \ - in the current environment: {}", - e, - ); - }) - .unwrap_or(false) - } - - /// Initialize a device with the provided key description and Clevis info. - pub fn initialize( - physical_path: &Path, - pool_uuid: PoolUuid, - dev_uuid: DevUuid, - pool_name: Name, - encryption_info: &EncryptionInfo, - sector_size: Option, - ) -> StratisResult { - let activation_name = format_crypt_name(&dev_uuid); - - let luks2_params = sector_size.map(|sector_size| CryptParamsLuks2 { - pbkdf: None, - integrity: None, - integrity_params: None, - data_alignment: 0, - data_device: None, - sector_size, - label: None, - subsystem: None, - }); - - let mut device = log_on_failure!( - CryptInit::init(physical_path), - "Failed to acquire context for device {} while initializing; \ - nothing to clean up", - physical_path.display() - ); - device.settings_handle().set_metadata_size( - MetadataSize::try_from(convert_int!(*DEFAULT_CRYPT_METADATA_SIZE, u128, u64)?)?, - KeyslotsSize::try_from(convert_int!(*DEFAULT_CRYPT_KEYSLOTS_SIZE, u128, u64)?)?, - )?; - Self::initialize_with_err(&mut device, physical_path, pool_uuid, dev_uuid, &pool_name, encryption_info, luks2_params.as_ref()) - .and_then(|path| clevis_info_from_metadata(&mut device).map(|ci| (path, ci))) - .and_then(|(_, clevis_info)| { - let encryption_info = - if let Some(info) = EncryptionInfo::from_options((encryption_info.key_description().cloned(), clevis_info)) { - info - } else { - return Err(StratisError::Msg(format!( - "No valid encryption method that can be used to unlock device {} found after initialization", - physical_path.display() - ))); - }; - - let device_path = DevicePath::new(physical_path)?; - let devno = get_devno_from_path(physical_path)?; - Ok(CryptHandle::new( - device_path, - pool_uuid, - dev_uuid, - encryption_info, - Some(pool_name), - devno, - )) - }) - .map_err(|e| { - if let Err(err) = - Self::rollback(&mut device, physical_path, &activation_name) - { - warn!( - "Failed to roll back crypt device initialization; you may need to manually wipe this device: {}", - err - ); - } - e - }) - } - - /// Initialize with a passphrase in the kernel keyring only. - fn initialize_with_keyring( - device: &mut CryptDevice, - key_description: &KeyDescription, - ) -> StratisResult<()> { - add_keyring_keyslot(device, key_description, None)?; - - Ok(()) - } - - /// Initialize with Clevis only. - fn initialize_with_clevis( - device: &mut CryptDevice, - physical_path: &Path, - (pin, json, yes): (&str, &Value, bool), - ) -> StratisResult<()> { - let (_, key_data) = thread_rng() - .sample_iter(Alphanumeric) - .take(MAX_STRATIS_PASS_SIZE) - .fold( - (0, SafeMemHandle::alloc(MAX_STRATIS_PASS_SIZE)?), - |(idx, mut mem), ch| { - mem.as_mut()[idx] = ch; - (idx + 1, mem) - }, - ); - - let key = SizedKeyMemory::new(key_data, MAX_STRATIS_PASS_SIZE); - let keyslot = log_on_failure!( - device - .keyslot_handle() - .add_by_key(None, None, key.as_ref(), CryptVolumeKey::empty(),), - "Failed to initialize keyslot with provided key in keyring" - ); - - clevis_luks_bind( - physical_path, - Either::Right(key), - CLEVIS_LUKS_TOKEN_ID, - pin, - json, - yes, - )?; - - // Need to reload device here to refresh the state of the device - // after being modified by Clevis. - if let Err(e) = device - .context_handle() - .load::<()>(Some(EncryptionFormat::Luks2), None) - { - return Err(wipe_fallback(physical_path, StratisError::from(e))); - } - - device.keyslot_handle().destroy(keyslot)?; - - Ok(()) - } - - /// Initialize with both a passphrase in the kernel keyring and Clevis. - fn initialize_with_both( - device: &mut CryptDevice, - physical_path: &Path, - key_description: &KeyDescription, - (pin, json, yes): (&str, &Value, bool), - ) -> StratisResult<()> { - Self::initialize_with_keyring(device, key_description)?; - - clevis_luks_bind( - physical_path, - Either::Left(LUKS2_TOKEN_ID), - CLEVIS_LUKS_TOKEN_ID, - pin, - json, - yes, - )?; - - // Need to reload device here to refresh the state of the device - // after being modified by Clevis. - if let Err(e) = device - .context_handle() - .load::<()>(Some(EncryptionFormat::Luks2), None) - { - return Err(wipe_fallback(physical_path, StratisError::from(e))); - } - - Ok(()) - } - - fn initialize_with_err( - device: &mut CryptDevice, - physical_path: &Path, - pool_uuid: PoolUuid, - dev_uuid: DevUuid, - pool_name: &Name, - encryption_info: &EncryptionInfo, - luks2_params: Option<&CryptParamsLuks2>, - ) -> StratisResult<()> { - let mut luks2_params_ref: Option> = - luks2_params.map(|lp| lp.try_into()).transpose()?; - - log_on_failure!( - device.context_handle().format::>( - EncryptionFormat::Luks2, - ("aes", "xts-plain64"), - None, - libcryptsetup_rs::Either::Right(STRATIS_MEK_SIZE), - luks2_params_ref.as_mut() - ), - "Failed to format device {} with LUKS2 header", - physical_path.display() - ); - - match encryption_info { - EncryptionInfo::Both(kd, (pin, config)) => { - let mut parsed_config = config.clone(); - let y = interpret_clevis_config(pin, &mut parsed_config)?; - Self::initialize_with_both(device, physical_path, kd, (pin, &parsed_config, y))? - } - EncryptionInfo::KeyDesc(kd) => Self::initialize_with_keyring(device, kd)?, - EncryptionInfo::ClevisInfo((pin, config)) => { - let mut parsed_config = config.clone(); - let y = interpret_clevis_config(pin, &mut parsed_config)?; - Self::initialize_with_clevis(device, physical_path, (pin, &parsed_config, y))? - } - }; - - let activation_name = format_crypt_name(&dev_uuid); - // Initialize stratis token - log_on_failure!( - device.token_handle().json_set(TokenInput::ReplaceToken( - STRATIS_TOKEN_ID, - &to_value(StratisLuks2Token { - devname: activation_name.clone(), - identifiers: StratisIdentifiers { - pool_uuid, - device_uuid: dev_uuid - }, - pool_name: Some(pool_name.clone()), - })?, - )), - "Failed to create the Stratis token" - ); - - activate( - device, - encryption_info.key_description(), - if matches!( - encryption_info, - EncryptionInfo::Both(_, _) | EncryptionInfo::KeyDesc(_) - ) { - UnlockMethod::Keyring - } else { - UnlockMethod::Clevis - }, - &activation_name, - ) - } - - pub fn rollback( - device: &mut CryptDevice, - physical_path: &Path, - name: &DmName, - ) -> StratisResult<()> { - ensure_wiped(device, physical_path, name) - } - - /// Acquire the crypt device handle for the physical path in this `CryptHandle`. - pub(super) fn acquire_crypt_device(&self) -> StratisResult { - acquire_crypt_device(self.luks2_device_path()) - } - - /// Query the device metadata to reconstruct a handle for performing operations - /// on an existing encrypted device. - /// - /// This method will check that the metadata on the given device is - /// for the LUKS2 format and that the LUKS2 metadata is formatted - /// properly as a Stratis encrypted device. If it is properly - /// formatted it will return the device identifiers (pool and device UUIDs). - /// - /// NOTE: This will not validate that the proper key is in the kernel - /// keyring. For that, use `CryptHandle::can_unlock()`. - /// - /// The checks include: - /// * is a LUKS2 device - /// * has a valid Stratis LUKS2 token - /// * has a token of the proper type for LUKS2 keyring unlocking - pub fn setup( - physical_path: &Path, - unlock_method: Option, - ) -> StratisResult> { - match setup_crypt_device(physical_path)? { - Some(ref mut device) => setup_crypt_handle(device, physical_path, unlock_method), - None => Ok(None), - } - } - - /// Load the required information for Stratis from the LUKS2 metadata. - pub fn load_metadata(physical_path: &Path) -> StratisResult> { - match setup_crypt_device(physical_path)? { - Some(ref mut device) => load_crypt_metadata(device, physical_path), - None => Ok(None), - } - } - - /// Get the encryption info for this encrypted device. - pub fn encryption_info(&self) -> &EncryptionInfo { - &self.metadata.encryption_info - } - - /// Return the path to the device node of the underlying storage device - /// for the encrypted device. - pub fn luks2_device_path(&self) -> &Path { - &self.metadata.physical_path - } - - /// Return the name of the activated devicemapper device. - pub fn activation_name(&self) -> &DmName { - &self.metadata.activation_name - } - - /// Return the path of the activated devicemapper device. - pub fn activated_device_path(&self) -> &Path { - &self.metadata.activated_path - } - - /// Return the pool name recorded in the LUKS2 metadata. - pub fn pool_name(&self) -> Option<&Name> { - self.metadata.pool_name.as_ref() - } - - /// Device number for the LUKS2 encrypted device. - pub fn device(&self) -> &Device { - &self.metadata.device - } - - /// Get the Stratis device identifiers for a given encrypted device. - pub fn device_identifiers(&self) -> &StratisIdentifiers { - &self.metadata.identifiers - } - - /// Get the keyslot associated with the given token ID. - pub fn keyslots(&self, token_id: c_uint) -> StratisResult>> { - get_keyslot_number(&mut self.acquire_crypt_device()?, token_id) - } - - /// Get info for the clevis binding. - pub fn clevis_info(&self) -> StratisResult> { - clevis_info_from_metadata(&mut self.acquire_crypt_device()?) - } - - /// Bind the given device using clevis. - pub fn clevis_bind(&mut self, pin: &str, json: &Value) -> StratisResult<()> { - let mut json_owned = json.clone(); - let yes = interpret_clevis_config(pin, &mut json_owned)?; - - clevis_luks_bind( - self.luks2_device_path(), - Either::Left(LUKS2_TOKEN_ID), - CLEVIS_LUKS_TOKEN_ID, - pin, - &json_owned, - yes, - )?; - self.metadata.encryption_info = - self.metadata - .encryption_info - .clone() - .set_clevis_info(self.clevis_info()?.ok_or_else(|| { - StratisError::Msg( - "Clevis reported successfully binding to device but no metadata was found" - .to_string(), - ) - })?); - Ok(()) - } - - /// Unbind the given device using clevis. - pub fn clevis_unbind(&mut self) -> StratisResult<()> { - if self.metadata.encryption_info.key_description().is_none() { - return Err(StratisError::Msg( - "No kernel keyring binding found; removing the Clevis binding \ - would remove the ability to open this device; aborting" - .to_string(), - )); - } - - let keyslots = self.keyslots(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { - StratisError::Msg(format!( - "Token slot {CLEVIS_LUKS_TOKEN_ID} appears to be empty; could not determine keyslots" - )) - })?; - for keyslot in keyslots { - log_on_failure!( - clevis_luks_unbind(self.luks2_device_path(), keyslot), - "Failed to unbind device {} from Clevis", - self.luks2_device_path().display() - ); - } - self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_clevis_info(); - Ok(()) - } - - /// Change the key description and passphrase that a device is bound to - /// - /// This method needs to re-read the cached Clevis information because - /// the config may change specifically in the case where a new thumbprint - /// is provided if Tang keys are rotated. - pub fn rebind_clevis(&mut self) -> StratisResult<()> { - if self.metadata.encryption_info.clevis_info().is_none() { - return Err(StratisError::Msg( - "No Clevis binding found; cannot regenerate the Clevis binding if the device does not already have a Clevis binding".to_string(), - )); - } - - let mut device = self.acquire_crypt_device()?; - let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)? - .and_then(|vec| vec.into_iter().next()) - .ok_or_else(|| { - StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) - })?; - - clevis_luks_regen(self.luks2_device_path(), keyslot)?; - // Need to reload LUKS2 metadata after Clevis metadata modification. - if let Err(e) = device - .context_handle() - .load::<()>(Some(EncryptionFormat::Luks2), None) - { - return Err(StratisError::Chained( - "Failed to reload crypt device state after modification to Clevis data".to_string(), - Box::new(StratisError::from(e)), - )); - } - - let (pin, config) = clevis_info_from_metadata(&mut device)?.ok_or_else(|| { - StratisError::Msg(format!( - "Did not find Clevis metadata on device {}", - self.luks2_device_path().display() - )) - })?; - self.metadata.encryption_info = self - .metadata - .encryption_info - .clone() - .set_clevis_info((pin, config)); - Ok(()) - } - - /// Add a keyring binding to the underlying LUKS2 volume. - pub fn bind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult<()> { - let mut device = self.acquire_crypt_device()?; - let key = Self::clevis_decrypt(&mut device)?.ok_or_else(|| { - StratisError::Msg( - "The Clevis token appears to have been wiped outside of \ - Stratis; cannot add a keyring key binding without an existing \ - passphrase to unlock the device" - .to_string(), - ) - })?; - - add_keyring_keyslot(&mut device, key_desc, Some(Either::Left(key)))?; - - self.metadata.encryption_info = self - .metadata - .encryption_info - .clone() - .set_key_desc(key_desc.clone()); - Ok(()) - } - - /// Add a keyring binding to the underlying LUKS2 volume. - pub fn unbind_keyring(&mut self) -> StratisResult<()> { - if self.metadata.encryption_info.clevis_info().is_none() { - return Err(StratisError::Msg( - "No Clevis binding was found; removing the keyring binding would \ - remove the ability to open this device; aborting" - .to_string(), - )); - } - - let mut device = self.acquire_crypt_device()?; - let keyslots = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? - .ok_or_else(|| StratisError::Msg("No LUKS2 keyring token was found".to_string()))?; - for keyslot in keyslots { - log_on_failure!( - device.keyslot_handle().destroy(keyslot), - "Failed partway through the kernel keyring unbinding operation \ - which cannot be rolled back; manual intervention may be required" - ) - } - device - .token_handle() - .json_set(TokenInput::RemoveToken(LUKS2_TOKEN_ID))?; - - self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_key_desc(); - - Ok(()) - } - - /// Change the key description and passphrase that a device is bound to - pub fn rebind_keyring(&mut self, new_key_desc: &KeyDescription) -> StratisResult<()> { - let mut device = self.acquire_crypt_device()?; - - let old_key_description = self.metadata.encryption_info - .key_description() - .ok_or_else(|| { - StratisError::Msg("Cannot change passphrase because this device is not bound to a passphrase in the kernel keyring".to_string()) - })?; - add_keyring_keyslot( - &mut device, - new_key_desc, - Some(Either::Right(old_key_description)), - )?; - self.metadata.encryption_info = self - .metadata - .encryption_info - .clone() - .set_key_desc(new_key_desc.clone()); - Ok(()) - } - - /// Rename the pool in the LUKS2 token. - pub fn rename_pool_in_metadata(&mut self, pool_name: Name) -> StratisResult<()> { - let mut device = self.acquire_crypt_device()?; - replace_pool_name(&mut device, pool_name) - } - - /// Decrypt a Clevis passphrase and return it securely. - fn clevis_decrypt(device: &mut CryptDevice) -> StratisResult> { - let mut token = match device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).ok() { - Some(t) => t, - None => return Ok(None), - }; - let jwe = token - .as_object_mut() - .and_then(|map| map.remove("jwe")) - .ok_or_else(|| { - StratisError::Msg(format!( - "Token slot {CLEVIS_LUKS_TOKEN_ID} is occupied but does not appear to be a Clevis \ - token; aborting" - )) - })?; - clevis_decrypt(&jwe).map(Some) - } - - /// Deactivate the device referenced by the current device handle. - pub fn deactivate(&self) -> StratisResult<()> { - ensure_inactive(&mut self.acquire_crypt_device()?, self.activation_name()) - } - - /// Wipe all LUKS2 metadata on the device safely using libcryptsetup. - pub fn wipe(&self) -> StratisResult<()> { - ensure_wiped( - &mut self.acquire_crypt_device()?, - self.luks2_device_path(), - self.activation_name(), - ) - } - - /// Get the size of the logical device built on the underlying encrypted physical - /// device. `devicemapper` will return the size in terms of number of sectors. - pub fn logical_device_size(&self) -> StratisResult { - let name = self.activation_name().to_owned(); - let active_device = log_on_failure!( - self.acquire_crypt_device()? - .runtime_handle(&name.to_string()) - .get_active_device(), - "Failed to get device size for encrypted logical device" - ); - Ok(Sectors(active_device.size)) - } - - /// Changed the encrypted device size - /// `None` will fill up the entire underlying physical device. - /// `Some(_)` will resize the device to the given number of sectors. - pub fn resize(&self, size: Option) -> StratisResult<()> { - let processed_size = match size { - Some(s) => { - if s == Sectors(0) { - return Err(StratisError::Msg( - "Cannot specify a crypt device size of zero".to_string(), - )); - } else { - *s - } - } - None => 0, - }; - let mut crypt = self.acquire_crypt_device()?; - crypt.token_handle().activate_by_token::<()>( - None, - None, - None, - CryptActivate::KEYRING_KEY, - )?; - crypt - .context_handle() - .resize(&self.activation_name().to_string(), processed_size) - .map_err(StratisError::Crypt) - } -} diff --git a/src/engine/strat_engine/crypt/handle/mod.rs b/src/engine/strat_engine/crypt/handle/mod.rs new file mode 100644 index 0000000000..9eccf93f66 --- /dev/null +++ b/src/engine/strat_engine/crypt/handle/mod.rs @@ -0,0 +1,6 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +pub mod v1; +pub mod v2; diff --git a/src/engine/strat_engine/crypt/handle/v1.rs b/src/engine/strat_engine/crypt/handle/v1.rs new file mode 100644 index 0000000000..336c162c65 --- /dev/null +++ b/src/engine/strat_engine/crypt/handle/v1.rs @@ -0,0 +1,1685 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{ + fmt::{self, Debug}, + path::{Path, PathBuf}, +}; + +use either::Either; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use serde::{ + de::{Error, MapAccess, Visitor}, + ser::SerializeMap, + Deserialize, Deserializer, Serialize, Serializer, +}; +use serde_json::{from_value, to_value, Value}; + +use devicemapper::{Device, DmName, DmNameBuf, Sectors}; +use libcryptsetup_rs::{ + c_uint, + consts::{ + flags::{CryptActivate, CryptVolumeKey}, + vals::{EncryptionFormat, KeyslotsSize, MetadataSize}, + }, + CryptDevice, CryptInit, CryptParamsLuks2, CryptParamsLuks2Ref, SafeMemHandle, TokenInput, +}; + +use crate::{ + engine::{ + engine::MAX_STRATIS_PASS_SIZE, + strat_engine::{ + backstore::get_devno_from_path, + cmd::{clevis_decrypt, clevis_luks_bind, clevis_luks_regen, clevis_luks_unbind}, + crypt::{ + consts::{ + CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, STRATIS_TOKEN_DEVNAME_KEY, + STRATIS_TOKEN_DEV_UUID_KEY, STRATIS_TOKEN_ID, STRATIS_TOKEN_POOLNAME_KEY, + STRATIS_TOKEN_POOL_UUID_KEY, STRATIS_TOKEN_TYPE, TOKEN_KEYSLOTS_KEY, + TOKEN_TYPE_KEY, + }, + shared::{ + acquire_crypt_device, activate, add_keyring_keyslot, check_luks2_token, + clevis_info_from_metadata, device_from_physical_path, ensure_inactive, + ensure_wiped, get_keyslot_number, interpret_clevis_config, + key_desc_from_metadata, luks2_token_type_is_valid, read_key, wipe_fallback, + }, + }, + dm::DEVICEMAPPER_PATH, + metadata::StratisIdentifiers, + names::format_crypt_name, + }, + types::{ + DevUuid, DevicePath, EncryptionInfo, KeyDescription, Name, PoolUuid, SizedKeyMemory, + UnlockMethod, + }, + ClevisInfo, + }, + stratis::{StratisError, StratisResult}, +}; + +pub struct StratisLuks2Token { + pub devname: DmNameBuf, + pub identifiers: StratisIdentifiers, + pub pool_name: Option, +} + +impl Serialize for StratisLuks2Token { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map_serializer = serializer.serialize_map(None)?; + map_serializer.serialize_entry(TOKEN_TYPE_KEY, STRATIS_TOKEN_TYPE)?; + map_serializer.serialize_entry::<_, [u32; 0]>(TOKEN_KEYSLOTS_KEY, &[])?; + map_serializer.serialize_entry(STRATIS_TOKEN_DEVNAME_KEY, &self.devname.to_string())?; + map_serializer.serialize_entry( + STRATIS_TOKEN_POOL_UUID_KEY, + &self.identifiers.pool_uuid.to_string(), + )?; + map_serializer.serialize_entry( + STRATIS_TOKEN_DEV_UUID_KEY, + &self.identifiers.device_uuid.to_string(), + )?; + if let Some(ref pn) = self.pool_name { + map_serializer.serialize_entry(STRATIS_TOKEN_POOLNAME_KEY, pn)?; + } + map_serializer.end() + } +} + +impl<'de> Deserialize<'de> for StratisLuks2Token { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct StratisTokenVisitor; + + impl<'de> Visitor<'de> for StratisTokenVisitor { + type Value = StratisLuks2Token; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "a Stratis LUKS2 token") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut token_type = None; + let mut token_keyslots = None; + let mut d_name = None; + let mut p_uuid = None; + let mut d_uuid = None; + let mut p_name = None; + + while let Some((k, v)) = map.next_entry::()? { + match k.as_str() { + TOKEN_TYPE_KEY => { + token_type = Some(v); + } + TOKEN_KEYSLOTS_KEY => { + token_keyslots = Some(v); + } + STRATIS_TOKEN_DEVNAME_KEY => { + d_name = Some(v); + } + STRATIS_TOKEN_POOL_UUID_KEY => { + p_uuid = Some(v); + } + STRATIS_TOKEN_DEV_UUID_KEY => { + d_uuid = Some(v); + } + STRATIS_TOKEN_POOLNAME_KEY => { + p_name = Some(v); + } + st => { + return Err(A::Error::custom(format!("Found unrecognized key {st}"))); + } + } + } + + token_type + .ok_or_else(|| A::Error::custom(format!("Missing field {TOKEN_TYPE_KEY}"))) + .and_then(|ty| match ty { + Value::String(s) => { + if s == STRATIS_TOKEN_TYPE { + Ok(()) + } else { + Err(A::Error::custom(format!( + "Incorrect value for {TOKEN_TYPE_KEY}: {s}" + ))) + } + } + _ => Err(A::Error::custom(format!( + "Unrecognized value type for {TOKEN_TYPE_KEY}" + ))), + }) + .and_then(|_| { + let value = token_keyslots.ok_or_else(|| { + A::Error::custom(format!("Missing field {TOKEN_KEYSLOTS_KEY}")) + })?; + match value { + Value::Array(a) => { + if a.is_empty() { + Ok(()) + } else { + Err(A::Error::custom(format!( + "Found non-empty array for {TOKEN_KEYSLOTS_KEY}" + ))) + } + } + _ => Err(A::Error::custom(format!( + "Unrecognized value type for {TOKEN_TYPE_KEY}" + ))), + } + }) + .and_then(|_| { + let value = d_name.ok_or_else(|| { + A::Error::custom(format!("Missing field {STRATIS_TOKEN_DEVNAME_KEY}")) + })?; + match value { + Value::String(s) => DmNameBuf::new(s).map_err(A::Error::custom), + _ => Err(A::Error::custom(format!( + "Unrecognized value type for {STRATIS_TOKEN_DEVNAME_KEY}" + ))), + } + }) + .and_then(|dev_name| { + let value = p_uuid.ok_or_else(|| { + A::Error::custom(format!("Missing field {STRATIS_TOKEN_POOL_UUID_KEY}")) + })?; + match value { + Value::String(s) => PoolUuid::parse_str(&s) + .map(|uuid| (dev_name, uuid)) + .map_err(A::Error::custom), + _ => Err(A::Error::custom(format!( + "Unrecognized value type for {STRATIS_TOKEN_POOL_UUID_KEY}" + ))), + } + }) + .and_then(|(dev_name, pool_uuid)| { + let value = d_uuid.ok_or_else(|| { + A::Error::custom(format!("Missing field {STRATIS_TOKEN_DEV_UUID_KEY}")) + })?; + match value { + Value::String(s) => DevUuid::parse_str(&s) + .map(|uuid| (dev_name, pool_uuid, uuid)) + .map_err(A::Error::custom), + _ => Err(A::Error::custom(format!( + "Unrecognized value type for {STRATIS_TOKEN_DEV_UUID_KEY}" + ))), + } + }) + .and_then(|(devname, pool_uuid, device_uuid)| { + let pool_name = match p_name { + Some(Value::String(s)) => Some(Name::new(s)), + Some(_) => { + return Err(A::Error::custom(format!( + "Unrecognized value type for {STRATIS_TOKEN_POOLNAME_KEY}" + ))) + } + None => None, + }; + Ok(StratisLuks2Token { + devname, + identifiers: StratisIdentifiers { + pool_uuid, + device_uuid, + }, + pool_name, + }) + }) + } + } + + deserializer.deserialize_map(StratisTokenVisitor) + } +} + +/// Query the Stratis metadata for the device identifiers. +fn identifiers_from_metadata(device: &mut CryptDevice) -> StratisResult { + Ok( + from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)? + .identifiers, + ) +} + +/// Query the Stratis metadata for the device activation name. +fn activation_name_from_metadata(device: &mut CryptDevice) -> StratisResult { + Ok(from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)?.devname) +} + +/// Query the Stratis metadata for the pool name. +pub fn pool_name_from_metadata(device: &mut CryptDevice) -> StratisResult> { + Ok( + from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)? + .pool_name, + ) +} + +/// Replace the old pool name in the Stratis LUKS2 token. +pub fn replace_pool_name(device: &mut CryptDevice, new_name: Name) -> StratisResult<()> { + let mut token = + from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)?; + token.pool_name = Some(new_name); + device.token_handle().json_set(TokenInput::ReplaceToken( + STRATIS_TOKEN_ID, + &to_value(token)?, + ))?; + Ok(()) +} + +/// Load crypt device metadata. +pub fn load_crypt_metadata( + device: &mut CryptDevice, + physical_path: &Path, +) -> StratisResult> { + let physical = DevicePath::new(physical_path)?; + + let identifiers = identifiers_from_metadata(device)?; + let activation_name = activation_name_from_metadata(device)?; + let pool_name = pool_name_from_metadata(device)?; + let key_description = key_desc_from_metadata(device); + let devno = get_devno_from_path(physical_path)?; + let key_description = match key_description + .as_ref() + .map(|kd| KeyDescription::from_system_key_desc(kd)) + { + Some(Some(Ok(description))) => Some(description), + Some(Some(Err(e))) => { + return Err(StratisError::Msg(format!( + "key description {} found on devnode {} is not a valid Stratis key description: {}", + key_description.expect("key_desc_from_metadata determined to be Some(_) above"), + physical_path.display(), + e, + ))); + } + Some(None) => { + warn!("Key description stored on device {} does not appear to be a Stratis key description; ignoring", physical_path.display()); + None + } + None => None, + }; + let clevis_info = clevis_info_from_metadata(device)?; + + let encryption_info = + if let Some(info) = EncryptionInfo::from_options((key_description, clevis_info)) { + info + } else { + return Err(StratisError::Msg(format!( + "No valid encryption method that can be used to unlock device {} found", + physical_path.display() + ))); + }; + + let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] + .into_iter() + .collect::(); + let activated_path = path.canonicalize().unwrap_or(path); + Ok(Some(CryptMetadata { + physical_path: physical, + identifiers, + encryption_info, + activation_name, + pool_name, + device: devno, + activated_path, + })) +} + +/// Validate that the Stratis token is present and valid +fn stratis_token_is_valid(json: Value) -> bool { + debug!("Stratis LUKS2 token: {}", json); + + let result = from_value::(json); + if let Err(ref e) = result { + debug!( + "LUKS2 token in the Stratis token slot does not appear \ + to be a Stratis token: {}.", + e, + ); + } + result.is_ok() +} + +/// Check whether the physical device path corresponds to an encrypted +/// Stratis device. +/// +/// This method works on activated and deactivated encrypted devices. +/// +/// This device will only return true if the device was initialized +/// with encryption by Stratis. This requires that: +/// * the device is a LUKS2 encrypted device. +/// * the device has a valid Stratis LUKS2 token. +fn is_encrypted_stratis_device(device: &mut CryptDevice) -> bool { + fn device_operations(device: &mut CryptDevice) -> StratisResult<()> { + let stratis_token = device.token_handle().json_get(STRATIS_TOKEN_ID).ok(); + let luks_token = device.token_handle().json_get(LUKS2_TOKEN_ID).ok(); + let clevis_token = device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).ok(); + if stratis_token.is_none() || (luks_token.is_none() && clevis_token.is_none()) { + return Err(StratisError::Msg( + "Device appears to be missing some of the required Stratis LUKS2 tokens" + .to_string(), + )); + } + if let Some(ref lt) = luks_token { + if !luks2_token_type_is_valid(lt) { + return Err(StratisError::Msg("LUKS2 token is invalid".to_string())); + } + } + if let Some(st) = stratis_token { + if !stratis_token_is_valid(st) { + return Err(StratisError::Msg("Stratis token is invalid".to_string())); + } + } + Ok(()) + } + + device_operations(device) + .map(|_| true) + .map_err(|e| { + debug!( + "Operations querying device to determine if it is a Stratis device \ + failed with an error: {}; reporting as not a Stratis device.", + e + ); + }) + .unwrap_or(false) +} + +/// Set up a libcryptsetup device handle on a device that may or may not be a LUKS2 +/// device. +pub fn setup_crypt_device(physical_path: &Path) -> StratisResult> { + let device_result = device_from_physical_path(physical_path); + match device_result { + Ok(None) => Ok(None), + Ok(Some(mut dev)) => { + if !is_encrypted_stratis_device(&mut dev) { + Ok(None) + } else { + Ok(Some(dev)) + } + } + Err(e) => Err(e), + } +} + +/// Set up a handle to a crypt device using either Clevis or the keyring to activate +/// the device. +pub fn setup_crypt_handle( + device: &mut CryptDevice, + physical_path: &Path, + unlock_method: Option, +) -> StratisResult> { + let metadata = match load_crypt_metadata(device, physical_path)? { + Some(m) => m, + None => return Ok(None), + }; + + if !vec![DEVICEMAPPER_PATH, &metadata.activation_name.to_string()] + .into_iter() + .collect::() + .exists() + { + if let Some(unlock) = unlock_method { + activate( + device, + metadata.encryption_info.key_description(), + unlock, + &metadata.activation_name, + )? + }; + } + + Ok(Some(CryptHandle::new( + metadata.physical_path, + metadata.identifiers.pool_uuid, + metadata.identifiers.device_uuid, + metadata.encryption_info, + metadata.pool_name, + metadata.device, + ))) +} + +#[derive(Debug, Clone)] +pub struct CryptMetadata { + pub physical_path: DevicePath, + pub identifiers: StratisIdentifiers, + pub encryption_info: EncryptionInfo, + pub activation_name: DmNameBuf, + pub activated_path: PathBuf, + pub pool_name: Option, + pub device: Device, +} + +/// Handle for performing all operations on an encrypted device. +/// +/// `Clone` is derived for this data structure because `CryptHandle` acquires +/// a new crypt device context for each operation. +#[derive(Debug, Clone)] +pub struct CryptHandle { + metadata: CryptMetadata, +} + +impl CryptHandle { + pub(super) fn new( + physical_path: DevicePath, + pool_uuid: PoolUuid, + dev_uuid: DevUuid, + encryption_info: EncryptionInfo, + pool_name: Option, + devno: Device, + ) -> CryptHandle { + let activation_name = format_crypt_name(&dev_uuid); + let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] + .into_iter() + .collect::(); + let activated_path = path.canonicalize().unwrap_or(path); + CryptHandle { + metadata: CryptMetadata { + physical_path, + identifiers: StratisIdentifiers { + pool_uuid, + device_uuid: dev_uuid, + }, + encryption_info, + activation_name, + pool_name, + device: devno, + activated_path, + }, + } + } + + /// Check whether the given physical device can be unlocked with the current + /// environment (e.g. the proper key is in the kernel keyring, the device + /// is formatted as a LUKS2 device, etc.) + pub fn can_unlock( + physical_path: &Path, + try_unlock_keyring: bool, + try_unlock_clevis: bool, + ) -> bool { + fn can_unlock_with_failures( + physical_path: &Path, + try_unlock_keyring: bool, + try_unlock_clevis: bool, + ) -> StratisResult { + let mut device = acquire_crypt_device(physical_path)?; + + if try_unlock_keyring { + let key_description = key_desc_from_metadata(&mut device); + + if key_description.is_some() { + check_luks2_token(&mut device)?; + } + } + if try_unlock_clevis { + log_on_failure!( + device.token_handle().activate_by_token::<()>( + None, + Some(CLEVIS_LUKS_TOKEN_ID), + None, + CryptActivate::empty(), + ), + "libcryptsetup reported that the decrypted Clevis passphrase \ + is unable to open the encrypted device" + ); + } + Ok(true) + } + + can_unlock_with_failures(physical_path, try_unlock_keyring, try_unlock_clevis) + .map_err(|e| { + warn!( + "stratisd was unable to simulate opening the given device \ + in the current environment: {}", + e, + ); + }) + .unwrap_or(false) + } + + /// Initialize a device with the provided key description and Clevis info. + pub fn initialize( + physical_path: &Path, + pool_uuid: PoolUuid, + dev_uuid: DevUuid, + pool_name: Name, + encryption_info: &EncryptionInfo, + sector_size: Option, + ) -> StratisResult { + let activation_name = format_crypt_name(&dev_uuid); + + let luks2_params = sector_size.map(|sector_size| CryptParamsLuks2 { + pbkdf: None, + integrity: None, + integrity_params: None, + data_alignment: 0, + data_device: None, + sector_size, + label: None, + subsystem: None, + }); + + let mut device = log_on_failure!( + CryptInit::init(physical_path), + "Failed to acquire context for device {} while initializing; \ + nothing to clean up", + physical_path.display() + ); + device.settings_handle().set_metadata_size( + MetadataSize::try_from(convert_int!(*DEFAULT_CRYPT_METADATA_SIZE, u128, u64)?)?, + KeyslotsSize::try_from(convert_int!(*DEFAULT_CRYPT_KEYSLOTS_SIZE, u128, u64)?)?, + )?; + Self::initialize_with_err(&mut device, physical_path, pool_uuid, dev_uuid, &pool_name, encryption_info, luks2_params.as_ref()) + .and_then(|path| clevis_info_from_metadata(&mut device).map(|ci| (path, ci))) + .and_then(|(_, clevis_info)| { + let encryption_info = + if let Some(info) = EncryptionInfo::from_options((encryption_info.key_description().cloned(), clevis_info)) { + info + } else { + return Err(StratisError::Msg(format!( + "No valid encryption method that can be used to unlock device {} found after initialization", + physical_path.display() + ))); + }; + + let device_path = DevicePath::new(physical_path)?; + let devno = get_devno_from_path(physical_path)?; + Ok(CryptHandle::new( + device_path, + pool_uuid, + dev_uuid, + encryption_info, + Some(pool_name), + devno, + )) + }) + .map_err(|e| { + if let Err(err) = + Self::rollback(&mut device, physical_path, &activation_name) + { + warn!( + "Failed to roll back crypt device initialization; you may need to manually wipe this device: {}", + err + ); + } + e + }) + } + + /// Initialize with a passphrase in the kernel keyring only. + fn initialize_with_keyring( + device: &mut CryptDevice, + key_description: &KeyDescription, + ) -> StratisResult<()> { + add_keyring_keyslot(device, key_description, None)?; + + Ok(()) + } + + /// Initialize with Clevis only. + fn initialize_with_clevis( + device: &mut CryptDevice, + physical_path: &Path, + (pin, json, yes): (&str, &Value, bool), + ) -> StratisResult<()> { + let (_, key_data) = thread_rng() + .sample_iter(Alphanumeric) + .take(MAX_STRATIS_PASS_SIZE) + .fold( + (0, SafeMemHandle::alloc(MAX_STRATIS_PASS_SIZE)?), + |(idx, mut mem), ch| { + mem.as_mut()[idx] = ch; + (idx + 1, mem) + }, + ); + + let key = SizedKeyMemory::new(key_data, MAX_STRATIS_PASS_SIZE); + let keyslot = log_on_failure!( + device + .keyslot_handle() + .add_by_key(None, None, key.as_ref(), CryptVolumeKey::empty(),), + "Failed to initialize keyslot with provided key in keyring" + ); + + clevis_luks_bind( + physical_path, + Either::Right(key), + CLEVIS_LUKS_TOKEN_ID, + pin, + json, + yes, + )?; + + // Need to reload device here to refresh the state of the device + // after being modified by Clevis. + if let Err(e) = device + .context_handle() + .load::<()>(Some(EncryptionFormat::Luks2), None) + { + return Err(wipe_fallback(physical_path, StratisError::from(e))); + } + + device.keyslot_handle().destroy(keyslot)?; + + Ok(()) + } + + /// Initialize with both a passphrase in the kernel keyring and Clevis. + fn initialize_with_both( + device: &mut CryptDevice, + physical_path: &Path, + key_description: &KeyDescription, + (pin, json, yes): (&str, &Value, bool), + ) -> StratisResult<()> { + Self::initialize_with_keyring(device, key_description)?; + + clevis_luks_bind( + physical_path, + Either::Left(LUKS2_TOKEN_ID), + CLEVIS_LUKS_TOKEN_ID, + pin, + json, + yes, + )?; + + // Need to reload device here to refresh the state of the device + // after being modified by Clevis. + if let Err(e) = device + .context_handle() + .load::<()>(Some(EncryptionFormat::Luks2), None) + { + return Err(wipe_fallback(physical_path, StratisError::from(e))); + } + + Ok(()) + } + + fn initialize_with_err( + device: &mut CryptDevice, + physical_path: &Path, + pool_uuid: PoolUuid, + dev_uuid: DevUuid, + pool_name: &Name, + encryption_info: &EncryptionInfo, + luks2_params: Option<&CryptParamsLuks2>, + ) -> StratisResult<()> { + let mut luks2_params_ref: Option> = + luks2_params.map(|lp| lp.try_into()).transpose()?; + + log_on_failure!( + device.context_handle().format::>( + EncryptionFormat::Luks2, + ("aes", "xts-plain64"), + None, + libcryptsetup_rs::Either::Right(STRATIS_MEK_SIZE), + luks2_params_ref.as_mut() + ), + "Failed to format device {} with LUKS2 header", + physical_path.display() + ); + + match encryption_info { + EncryptionInfo::Both(kd, (pin, config)) => { + let mut parsed_config = config.clone(); + let y = interpret_clevis_config(pin, &mut parsed_config)?; + Self::initialize_with_both(device, physical_path, kd, (pin, &parsed_config, y))? + } + EncryptionInfo::KeyDesc(kd) => Self::initialize_with_keyring(device, kd)?, + EncryptionInfo::ClevisInfo((pin, config)) => { + let mut parsed_config = config.clone(); + let y = interpret_clevis_config(pin, &mut parsed_config)?; + Self::initialize_with_clevis(device, physical_path, (pin, &parsed_config, y))? + } + }; + + let activation_name = format_crypt_name(&dev_uuid); + // Initialize stratis token + log_on_failure!( + device.token_handle().json_set(TokenInput::ReplaceToken( + STRATIS_TOKEN_ID, + &to_value(StratisLuks2Token { + devname: activation_name.clone(), + identifiers: StratisIdentifiers { + pool_uuid, + device_uuid: dev_uuid + }, + pool_name: Some(pool_name.clone()), + })?, + )), + "Failed to create the Stratis token" + ); + + activate( + device, + encryption_info.key_description(), + UnlockMethod::Any, + &activation_name, + ) + } + + pub fn rollback( + device: &mut CryptDevice, + physical_path: &Path, + name: &DmName, + ) -> StratisResult<()> { + ensure_wiped(device, physical_path, name) + } + + /// Acquire the crypt device handle for the physical path in this `CryptHandle`. + pub(super) fn acquire_crypt_device(&self) -> StratisResult { + acquire_crypt_device(self.luks2_device_path()) + } + + /// Query the device metadata to reconstruct a handle for performing operations + /// on an existing encrypted device. + /// + /// This method will check that the metadata on the given device is + /// for the LUKS2 format and that the LUKS2 metadata is formatted + /// properly as a Stratis encrypted device. If it is properly + /// formatted it will return the device identifiers (pool and device UUIDs). + /// + /// NOTE: This will not validate that the proper key is in the kernel + /// keyring. For that, use `CryptHandle::can_unlock()`. + /// + /// The checks include: + /// * is a LUKS2 device + /// * has a valid Stratis LUKS2 token + /// * has a token of the proper type for LUKS2 keyring unlocking + pub fn setup( + physical_path: &Path, + unlock_method: Option, + ) -> StratisResult> { + match setup_crypt_device(physical_path)? { + Some(ref mut device) => setup_crypt_handle(device, physical_path, unlock_method), + None => Ok(None), + } + } + + /// Load the required information for Stratis from the LUKS2 metadata. + pub fn load_metadata(physical_path: &Path) -> StratisResult> { + match setup_crypt_device(physical_path)? { + Some(ref mut device) => load_crypt_metadata(device, physical_path), + None => Ok(None), + } + } + + /// Get the encryption info for this encrypted device. + pub fn encryption_info(&self) -> &EncryptionInfo { + &self.metadata.encryption_info + } + + /// Return the path to the device node of the underlying storage device + /// for the encrypted device. + pub fn luks2_device_path(&self) -> &Path { + &self.metadata.physical_path + } + + /// Return the name of the activated devicemapper device. + pub fn activation_name(&self) -> &DmName { + &self.metadata.activation_name + } + + /// Return the path of the activated devicemapper device. + pub fn activated_device_path(&self) -> &Path { + &self.metadata.activated_path + } + + /// Return the pool name recorded in the LUKS2 metadata. + pub fn pool_name(&self) -> Option<&Name> { + self.metadata.pool_name.as_ref() + } + + /// Device number for the LUKS2 encrypted device. + pub fn device(&self) -> &Device { + &self.metadata.device + } + + /// Get the Stratis device identifiers for a given encrypted device. + pub fn device_identifiers(&self) -> &StratisIdentifiers { + &self.metadata.identifiers + } + + /// Get the keyslot associated with the given token ID. + pub fn keyslots(&self, token_id: c_uint) -> StratisResult>> { + get_keyslot_number(&mut self.acquire_crypt_device()?, token_id) + } + + /// Get info for the clevis binding. + pub fn clevis_info(&self) -> StratisResult> { + clevis_info_from_metadata(&mut self.acquire_crypt_device()?) + } + + /// Bind the given device using clevis. + pub fn clevis_bind(&mut self, pin: &str, json: &Value) -> StratisResult<()> { + let mut json_owned = json.clone(); + let yes = interpret_clevis_config(pin, &mut json_owned)?; + + clevis_luks_bind( + self.luks2_device_path(), + Either::Left(LUKS2_TOKEN_ID), + CLEVIS_LUKS_TOKEN_ID, + pin, + &json_owned, + yes, + )?; + self.metadata.encryption_info = + self.metadata + .encryption_info + .clone() + .set_clevis_info(self.clevis_info()?.ok_or_else(|| { + StratisError::Msg( + "Clevis reported successfully binding to device but no metadata was found" + .to_string(), + ) + })?); + Ok(()) + } + + /// Unbind the given device using clevis. + pub fn clevis_unbind(&mut self) -> StratisResult<()> { + if self.metadata.encryption_info.key_description().is_none() { + return Err(StratisError::Msg( + "No kernel keyring binding found; removing the Clevis binding \ + would remove the ability to open this device; aborting" + .to_string(), + )); + } + + let keyslots = self.keyslots(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { + StratisError::Msg(format!( + "Token slot {CLEVIS_LUKS_TOKEN_ID} appears to be empty; could not determine keyslots" + )) + })?; + for keyslot in keyslots { + log_on_failure!( + clevis_luks_unbind(self.luks2_device_path(), keyslot), + "Failed to unbind device {} from Clevis", + self.luks2_device_path().display() + ); + } + self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_clevis_info(); + Ok(()) + } + + /// Change the key description and passphrase that a device is bound to + /// + /// This method needs to re-read the cached Clevis information because + /// the config may change specifically in the case where a new thumbprint + /// is provided if Tang keys are rotated. + pub fn rebind_clevis(&mut self) -> StratisResult<()> { + if self.metadata.encryption_info.clevis_info().is_none() { + return Err(StratisError::Msg( + "No Clevis binding found; cannot regenerate the Clevis binding if the device does not already have a Clevis binding".to_string(), + )); + } + + let mut device = self.acquire_crypt_device()?; + let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)? + .and_then(|vec| vec.into_iter().next()) + .ok_or_else(|| { + StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) + })?; + + clevis_luks_regen(self.luks2_device_path(), keyslot)?; + // Need to reload LUKS2 metadata after Clevis metadata modification. + if let Err(e) = device + .context_handle() + .load::<()>(Some(EncryptionFormat::Luks2), None) + { + return Err(StratisError::Chained( + "Failed to reload crypt device state after modification to Clevis data".to_string(), + Box::new(StratisError::from(e)), + )); + } + + let (pin, config) = clevis_info_from_metadata(&mut device)?.ok_or_else(|| { + StratisError::Msg(format!( + "Did not find Clevis metadata on device {}", + self.luks2_device_path().display() + )) + })?; + self.metadata.encryption_info = self + .metadata + .encryption_info + .clone() + .set_clevis_info((pin, config)); + Ok(()) + } + + /// Add a keyring binding to the underlying LUKS2 volume. + pub fn bind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult<()> { + let mut device = self.acquire_crypt_device()?; + let key = Self::clevis_decrypt(&mut device)?.ok_or_else(|| { + StratisError::Msg( + "The Clevis token appears to have been wiped outside of \ + Stratis; cannot add a keyring key binding without an existing \ + passphrase to unlock the device" + .to_string(), + ) + })?; + + add_keyring_keyslot(&mut device, key_desc, Some(Either::Left(key)))?; + + self.metadata.encryption_info = self + .metadata + .encryption_info + .clone() + .set_key_desc(key_desc.clone()); + Ok(()) + } + + /// Add a keyring binding to the underlying LUKS2 volume. + pub fn unbind_keyring(&mut self) -> StratisResult<()> { + if self.metadata.encryption_info.clevis_info().is_none() { + return Err(StratisError::Msg( + "No Clevis binding was found; removing the keyring binding would \ + remove the ability to open this device; aborting" + .to_string(), + )); + } + + let mut device = self.acquire_crypt_device()?; + let keyslots = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? + .ok_or_else(|| StratisError::Msg("No LUKS2 keyring token was found".to_string()))?; + for keyslot in keyslots { + log_on_failure!( + device.keyslot_handle().destroy(keyslot), + "Failed partway through the kernel keyring unbinding operation \ + which cannot be rolled back; manual intervention may be required" + ) + } + device + .token_handle() + .json_set(TokenInput::RemoveToken(LUKS2_TOKEN_ID))?; + + self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_key_desc(); + + Ok(()) + } + + /// Change the key description and passphrase that a device is bound to + pub fn rebind_keyring(&mut self, new_key_desc: &KeyDescription) -> StratisResult<()> { + let mut device = self.acquire_crypt_device()?; + + let old_key_description = self.metadata.encryption_info + .key_description() + .ok_or_else(|| { + StratisError::Msg("Cannot change passphrase because this device is not bound to a passphrase in the kernel keyring".to_string()) + })?; + add_keyring_keyslot( + &mut device, + new_key_desc, + Some(Either::Right(old_key_description)), + )?; + self.metadata.encryption_info = self + .metadata + .encryption_info + .clone() + .set_key_desc(new_key_desc.clone()); + Ok(()) + } + + /// Rename the pool in the LUKS2 token. + pub fn rename_pool_in_metadata(&mut self, pool_name: Name) -> StratisResult<()> { + let mut device = self.acquire_crypt_device()?; + replace_pool_name(&mut device, pool_name) + } + + /// Decrypt a Clevis passphrase and return it securely. + fn clevis_decrypt(device: &mut CryptDevice) -> StratisResult> { + let mut token = match device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).ok() { + Some(t) => t, + None => return Ok(None), + }; + let jwe = token + .as_object_mut() + .and_then(|map| map.remove("jwe")) + .ok_or_else(|| { + StratisError::Msg(format!( + "Token slot {CLEVIS_LUKS_TOKEN_ID} is occupied but does not appear to be a Clevis \ + token; aborting" + )) + })?; + clevis_decrypt(&jwe).map(Some) + } + + /// Deactivate the device referenced by the current device handle. + pub fn deactivate(&self) -> StratisResult<()> { + ensure_inactive(&mut self.acquire_crypt_device()?, self.activation_name()) + } + + /// Wipe all LUKS2 metadata on the device safely using libcryptsetup. + pub fn wipe(&self) -> StratisResult<()> { + ensure_wiped( + &mut self.acquire_crypt_device()?, + self.luks2_device_path(), + self.activation_name(), + ) + } + + /// Get the size of the logical device built on the underlying encrypted physical + /// device. `devicemapper` will return the size in terms of number of sectors. + pub fn logical_device_size(&self) -> StratisResult { + let name = self.activation_name().to_owned(); + let active_device = log_on_failure!( + self.acquire_crypt_device()? + .runtime_handle(&name.to_string()) + .get_active_device(), + "Failed to get device size for encrypted logical device" + ); + Ok(Sectors(active_device.size)) + } + + /// Changed the encrypted device size + /// `None` will fill up the entire underlying physical device. + /// `Some(_)` will resize the device to the given number of sectors. + pub fn resize(&self, size: Option) -> StratisResult<()> { + let processed_size = match size { + Some(s) => { + if s == Sectors(0) { + return Err(StratisError::Msg( + "Cannot specify a crypt device size of zero".to_string(), + )); + } else { + *s + } + } + None => 0, + }; + let mut crypt = self.acquire_crypt_device()?; + let passphrase = if let Some(kd) = self.encryption_info().key_description() { + read_key(kd)?.ok_or_else(|| { + StratisError::Msg("Failed to find key with key description".to_string()) + })? + } else if self.encryption_info().clevis_info().is_some() { + Self::clevis_decrypt(&mut crypt)?.expect("Already checked token exists") + } else { + unreachable!("Must be encrypted") + }; + crypt.activate_handle().activate_by_passphrase( + None, + None, + passphrase.as_ref(), + CryptActivate::KEYRING_KEY, + )?; + crypt + .context_handle() + .resize(&self.activation_name().to_string(), processed_size) + .map_err(StratisError::Crypt) + } +} + +#[cfg(test)] +mod tests { + use std::{ + env, + ffi::CString, + fs::{File, OpenOptions}, + io::{self, Read, Write}, + mem::MaybeUninit, + path::Path, + ptr, slice, + }; + + use devicemapper::{Bytes, Sectors, IEC}; + use libcryptsetup_rs::{ + consts::vals::{CryptStatusInfo, EncryptionFormat}, + CryptInit, Either, + }; + + use crate::engine::{ + strat_engine::{ + crypt::{ + consts::{ + CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, + }, + shared::acquire_crypt_device, + }, + ns::{unshare_mount_namespace, MemoryFilesystem}, + tests::{crypt, loopbacked, real}, + }, + types::{DevUuid, EncryptionInfo, KeyDescription, Name, PoolUuid, UnlockMethod}, + }; + + use super::*; + + /// If this method is called without a key with the specified key description + /// in the kernel ring, it should always fail and allow us to test the rollback + /// of failed initializations. + fn test_failed_init(paths: &[&Path]) { + assert_eq!(paths.len(), 1); + + let path = paths.first().expect("There must be exactly one path"); + let key_description = + KeyDescription::try_from("I am not a key".to_string()).expect("no semi-colons"); + + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + let dev_uuid = DevUuid::new_v4(); + + let result = CryptHandle::initialize( + path, + pool_uuid, + dev_uuid, + pool_name, + &EncryptionInfo::KeyDesc(key_description), + None, + ); + + // Initialization cannot occur with a non-existent key + assert!(result.is_err()); + + assert!(CryptHandle::load_metadata(path).unwrap().is_none()); + + // TODO: Check actual superblock with libblkid + } + + #[test] + fn loop_test_failed_init() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_failed_init, + ); + } + + #[test] + fn real_test_failed_init() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_failed_init, + ); + } + + /// Test the method `can_unlock` works on an initialized device in both + /// active and inactive states. + fn test_can_unlock(paths: &[&Path]) { + fn crypt_test(paths: &[&Path], key_desc: &KeyDescription) { + let mut handles = vec![]; + + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + for path in paths { + let dev_uuid = DevUuid::new_v4(); + + let handle = CryptHandle::initialize( + path, + pool_uuid, + dev_uuid, + pool_name.clone(), + &EncryptionInfo::KeyDesc(key_desc.clone()), + None, + ) + .unwrap(); + handles.push(handle); + } + + for path in paths { + if !CryptHandle::can_unlock(path, true, false) { + panic!("All devices should be able to be unlocked"); + } + } + + for handle in handles.iter_mut() { + handle.deactivate().unwrap(); + } + + for path in paths { + if !CryptHandle::can_unlock(path, true, false) { + panic!("All devices should be able to be unlocked"); + } + } + + for handle in handles.iter_mut() { + handle.wipe().unwrap(); + } + + for path in paths { + if CryptHandle::can_unlock(path, true, false) { + panic!("All devices should no longer be able to be unlocked"); + } + } + } + + crypt::insert_and_cleanup_key(paths, crypt_test) + } + + #[test] + fn loop_test_can_unlock() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_can_unlock, + ); + } + + #[test] + fn real_test_can_unlock() { + real::test_with_spec( + &real::DeviceLimits::Range(1, 3, None, None), + test_can_unlock, + ); + } + + /// Test initializing and activating an encrypted device using + /// the utilities provided here. + /// + /// The overall format of the test involves generating a random byte buffer + /// of size 1 MiB, encrypting it on disk, and then ensuring that the plaintext + /// cannot be found on the encrypted disk by doing a scan of the disk using + /// a sliding window. + /// + /// The sliding window size of 1 MiB was chosen to lower the number of + /// searches that need to be done compared to a smaller sliding window + /// and also to decrease the probability of the random sequence being found + /// on the disk due to leftover data from other tests. + // TODO: Rewrite libc calls using nix crate. + fn test_crypt_device_ops(paths: &[&Path]) { + fn crypt_test(paths: &[&Path], key_desc: &KeyDescription) { + let path = paths + .first() + .expect("This test only accepts a single device"); + + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + let dev_uuid = DevUuid::new_v4(); + + let handle = CryptHandle::initialize( + path, + pool_uuid, + dev_uuid, + pool_name, + &EncryptionInfo::KeyDesc(key_desc.clone()), + None, + ) + .unwrap(); + let logical_path = handle.activated_device_path(); + + const WINDOW_SIZE: usize = 1024 * 1024; + let mut devicenode = OpenOptions::new().write(true).open(logical_path).unwrap(); + let mut random_buffer = vec![0; WINDOW_SIZE].into_boxed_slice(); + File::open("/dev/urandom") + .unwrap() + .read_exact(&mut random_buffer) + .unwrap(); + devicenode.write_all(&random_buffer).unwrap(); + std::mem::drop(devicenode); + + let dev_path_cstring = + CString::new(path.to_str().expect("Failed to convert path to string")).unwrap(); + let fd = unsafe { libc::open(dev_path_cstring.as_ptr(), libc::O_RDONLY) }; + if fd < 0 { + panic!("{}", io::Error::last_os_error()); + } + + let mut stat: MaybeUninit = MaybeUninit::zeroed(); + let fstat_result = unsafe { libc::fstat(fd, stat.as_mut_ptr()) }; + if fstat_result < 0 { + panic!("{}", io::Error::last_os_error()); + } + let device_size = + convert_int!(unsafe { stat.assume_init() }.st_size, libc::off_t, usize).unwrap(); + let mapped_ptr = unsafe { + libc::mmap( + ptr::null_mut(), + device_size, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapped_ptr.is_null() { + panic!("mmap failed"); + } + + { + let disk_buffer = + unsafe { slice::from_raw_parts(mapped_ptr as *const u8, device_size) }; + for window in disk_buffer.windows(WINDOW_SIZE) { + if window == &*random_buffer as &[u8] { + unsafe { + libc::munmap(mapped_ptr, device_size); + libc::close(fd); + }; + panic!("Disk was not encrypted!"); + } + } + } + + unsafe { + libc::munmap(mapped_ptr, device_size); + libc::close(fd); + }; + + let device_name = handle.activation_name(); + loop { + match libcryptsetup_rs::status( + Some(&mut handle.acquire_crypt_device().unwrap()), + &device_name.to_string(), + ) { + Ok(CryptStatusInfo::Busy) => (), + Ok(CryptStatusInfo::Active) => break, + Ok(s) => { + panic!("Crypt device is in invalid state {s:?}") + } + Err(e) => { + panic!("Checking device status returned error: {e}") + } + } + } + + handle.deactivate().unwrap(); + + let handle = CryptHandle::setup(path, Some(UnlockMethod::Keyring)) + .unwrap() + .unwrap_or_else(|| { + panic!( + "Device {} no longer appears to be a LUKS2 device", + path.display(), + ) + }); + handle.wipe().unwrap(); + } + + assert_eq!(paths.len(), 1); + + crypt::insert_and_cleanup_key(paths, crypt_test); + } + + #[test] + fn real_test_crypt_device_ops() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(2 * IEC::Mi))), + test_crypt_device_ops, + ); + } + + #[test] + fn loop_test_crypt_metadata_defaults() { + fn test_defaults(paths: &[&Path]) { + let mut context = CryptInit::init(paths[0]).unwrap(); + context + .context_handle() + .format::<()>( + EncryptionFormat::Luks2, + ("aes", "xts-plain64"), + None, + Either::Right(STRATIS_MEK_SIZE), + None, + ) + .unwrap(); + let (metadata, keyslot) = context.settings_handle().get_metadata_size().unwrap(); + assert_eq!(DEFAULT_CRYPT_METADATA_SIZE, Bytes::from(*metadata)); + assert_eq!(DEFAULT_CRYPT_KEYSLOTS_SIZE, Bytes::from(*keyslot)); + } + + loopbacked::test_with_spec(&loopbacked::DeviceLimits::Exactly(1, None), test_defaults); + } + + #[test] + // Test passing an unusual, larger sector size for cryptsetup. 4096 should + // be no smaller than the physical sector size of the loop device, and + // should be allowed by cryptsetup. + fn loop_test_set_sector_size() { + fn the_test(paths: &[&Path]) { + fn test_set_sector_size(paths: &[&Path], key_description: &KeyDescription) { + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + let dev_uuid = DevUuid::new_v4(); + + CryptHandle::initialize( + paths[0], + pool_uuid, + dev_uuid, + pool_name, + &EncryptionInfo::KeyDesc(key_description.clone()), + Some(4096u32), + ) + .unwrap(); + } + + crypt::insert_and_cleanup_key(paths, test_set_sector_size); + } + + loopbacked::test_with_spec(&loopbacked::DeviceLimits::Exactly(1, None), the_test); + } + + fn test_both_initialize(paths: &[&Path]) { + fn both_initialize(paths: &[&Path], key_desc: &KeyDescription) { + unshare_mount_namespace().unwrap(); + let _memfs = MemoryFilesystem::new().unwrap(); + let path = paths.first().copied().expect("Expected exactly one path"); + let pool_name = Name::new("pool_name".to_string()); + let handle = CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + pool_name, + &EncryptionInfo::Both( + key_desc.clone(), + ( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + ), + ), + None, + ).unwrap(); + + let mut device = acquire_crypt_device(handle.luks2_device_path()).unwrap(); + device.token_handle().json_get(LUKS2_TOKEN_ID).unwrap(); + device + .token_handle() + .json_get(CLEVIS_LUKS_TOKEN_ID) + .unwrap(); + handle.deactivate().unwrap(); + } + + fn unlock_clevis(paths: &[&Path]) { + let path = paths.first().copied().expect("Expected exactly one path"); + CryptHandle::setup(path, Some(UnlockMethod::Clevis)) + .unwrap() + .unwrap(); + } + + crypt::insert_and_remove_key(paths, both_initialize, unlock_clevis); + } + + #[test] + fn clevis_real_test_both_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_both_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_real_should_fail_test_both_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_both_initialize, + ); + } + + #[test] + fn clevis_loop_test_both_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_both_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_loop_should_fail_test_both_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_both_initialize, + ); + } + + fn test_clevis_initialize(paths: &[&Path]) { + unshare_mount_namespace().unwrap(); + let _memfs = MemoryFilesystem::new().unwrap(); + let path = paths[0]; + let pool_name = Name::new("pool_name".to_string()); + + let handle = CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + pool_name, + &EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + )), + None, + ) + .unwrap(); + + let mut device = acquire_crypt_device(handle.luks2_device_path()).unwrap(); + assert!(device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).is_ok()); + assert!(device.token_handle().json_get(LUKS2_TOKEN_ID).is_err()); + } + + #[test] + fn clevis_real_test_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_clevis_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_real_should_fail_test_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_clevis_initialize, + ); + } + + #[test] + fn clevis_loop_test_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_loop_should_fail_test_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_initialize, + ); + } + + fn test_clevis_tang_configs(paths: &[&Path]) { + let path = paths[0]; + let pool_name = Name::new("pool_name".to_string()); + + assert!(CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + pool_name.clone(), + &EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required")}), + )), + None, + ) + .is_err()); + CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + pool_name, + &EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({ + "stratis:tang:trust_url": true, + "url": env::var("TANG_URL").expect("TANG_URL env var required"), + }), + )), + None, + ) + .unwrap(); + } + + #[test] + fn clevis_real_test_clevis_tang_configs() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_clevis_tang_configs, + ); + } + + #[test] + fn clevis_loop_test_clevis_tang_configs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_tang_configs, + ); + } + + fn test_clevis_sss_configs(paths: &[&Path]) { + let path = paths[0]; + let pool_name = Name::new("pool_name".to_string()); + + assert!(CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + pool_name.clone(), + &EncryptionInfo::ClevisInfo(( + "sss".to_string(), + json!({"t": 1, "pins": {"tang": {"url": env::var("TANG_URL").expect("TANG_URL env var required")}, "tpm2": {}}}), + )), + None, + ) + .is_err()); + CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + pool_name, + &EncryptionInfo::ClevisInfo(( + "sss".to_string(), + json!({ + "t": 1, + "stratis:tang:trust_url": true, + "pins": { + "tang": {"url": env::var("TANG_URL").expect("TANG_URL env var required")}, + "tpm2": {} + } + }), + )), + None, + ) + .unwrap(); + } + + #[test] + fn clevis_real_test_clevis_sss_configs() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_clevis_sss_configs, + ); + } + + #[test] + fn clevis_loop_test_clevis_sss_configs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_sss_configs, + ); + } +} diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs new file mode 100644 index 0000000000..d7743189af --- /dev/null +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -0,0 +1,1240 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#![allow(dead_code)] + +use std::{ + fmt::Debug, + path::{Path, PathBuf}, +}; + +use either::Either; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use serde_json::Value; + +use devicemapper::{Device, DmName, DmNameBuf, Sectors}; +use libcryptsetup_rs::{ + c_uint, + consts::{ + flags::{CryptActivate, CryptVolumeKey}, + vals::{EncryptionFormat, KeyslotsSize, MetadataSize}, + }, + CryptDevice, CryptInit, CryptParamsLuks2, CryptParamsLuks2Ref, SafeMemHandle, TokenInput, +}; + +#[cfg(test)] +use crate::engine::strat_engine::crypt::shared::ensure_inactive; +use crate::{ + engine::{ + engine::MAX_STRATIS_PASS_SIZE, + strat_engine::{ + backstore::get_devno_from_path, + cmd::{clevis_decrypt, clevis_luks_bind, clevis_luks_regen, clevis_luks_unbind}, + crypt::{ + consts::{ + CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, + }, + shared::{ + acquire_crypt_device, activate, add_keyring_keyslot, clevis_info_from_metadata, + device_from_physical_path, ensure_wiped, get_keyslot_number, + interpret_clevis_config, key_desc_from_metadata, luks2_token_type_is_valid, + wipe_fallback, + }, + }, + dm::DEVICEMAPPER_PATH, + names::format_crypt_backstore_name, + }, + types::{ + DevicePath, EncryptionInfo, KeyDescription, PoolUuid, SizedKeyMemory, UnlockMethod, + }, + ClevisInfo, + }, + stratis::{StratisError, StratisResult}, +}; + +/// Load crypt device metadata. +pub fn load_crypt_metadata( + device: &mut CryptDevice, + physical_path: &Path, + pool_uuid: PoolUuid, +) -> StratisResult> { + let physical = DevicePath::new(physical_path)?; + + let activation_name = format_crypt_backstore_name(&pool_uuid); + let key_description = key_desc_from_metadata(device); + let key_description = match key_description + .as_ref() + .map(|kd| KeyDescription::from_system_key_desc(kd)) + { + Some(Some(Ok(description))) => Some(description), + Some(Some(Err(e))) => { + return Err(StratisError::Msg(format!( + "key description {} found on devnode {} is not a valid Stratis key description: {}", + key_description.expect("key_desc_from_metadata determined to be Some(_) above"), + physical_path.display(), + e, + ))); + } + Some(None) => { + warn!("Key description stored on device {} does not appear to be a Stratis key description; ignoring", physical_path.display()); + None + } + None => None, + }; + let clevis_info = clevis_info_from_metadata(device)?; + + let encryption_info = + if let Some(info) = EncryptionInfo::from_options((key_description, clevis_info)) { + info + } else { + return Err(StratisError::Msg(format!( + "No valid encryption method that can be used to unlock device {} found", + physical_path.display() + ))); + }; + + let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] + .into_iter() + .collect::(); + let activated_path = path.canonicalize().unwrap_or(path); + let devno = get_devno_from_path(&activated_path)?; + Ok(Some(CryptMetadata { + physical_path: physical, + pool_uuid, + encryption_info, + activation_name, + activated_path, + device: devno, + })) +} + +#[derive(Debug, Clone)] +pub struct CryptMetadata { + pub physical_path: DevicePath, + pub pool_uuid: PoolUuid, + pub encryption_info: EncryptionInfo, + pub activation_name: DmNameBuf, + pub activated_path: PathBuf, + pub device: Device, +} + +/// Check whether the physical device path corresponds to an encrypted +/// Stratis device. +/// +/// This method works on activated and deactivated encrypted devices. +/// +/// This device will only return true if the device was initialized +/// with encryption by Stratis. This requires that the device is a LUKS2 encrypted device. +fn is_encrypted_stratis_device(device: &mut CryptDevice) -> bool { + fn device_operations(device: &mut CryptDevice) -> StratisResult<()> { + let luks_token = device.token_handle().json_get(LUKS2_TOKEN_ID).ok(); + let clevis_token = device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).ok(); + if luks_token.is_none() && clevis_token.is_none() { + return Err(StratisError::Msg( + "Device appears to be missing some of the required Stratis LUKS2 tokens" + .to_string(), + )); + } + if let Some(ref lt) = luks_token { + if !luks2_token_type_is_valid(lt) { + return Err(StratisError::Msg("LUKS2 token is invalid".to_string())); + } + } + Ok(()) + } + + device_operations(device) + .map(|_| true) + .map_err(|e| { + debug!( + "Operations querying device to determine if it is a Stratis device \ + failed with an error: {}; reporting as not a Stratis device.", + e + ); + }) + .unwrap_or(false) +} + +/// Set up a libcryptsetup device handle on a device that may or may not be a LUKS2 +/// device. +pub fn setup_crypt_device(physical_path: &Path) -> StratisResult> { + let device_result = device_from_physical_path(physical_path); + match device_result { + Ok(None) => Ok(None), + Ok(Some(mut dev)) => { + if !is_encrypted_stratis_device(&mut dev) { + Ok(None) + } else { + Ok(Some(dev)) + } + } + Err(e) => Err(e), + } +} + +/// Set up a handle to a crypt device using either Clevis or the keyring to activate +/// the device. +pub fn setup_crypt_handle( + device: &mut CryptDevice, + physical_path: &Path, + pool_uuid: PoolUuid, + unlock_method: UnlockMethod, +) -> StratisResult> { + let metadata = match load_crypt_metadata(device, physical_path, pool_uuid)? { + Some(m) => m, + None => return Ok(None), + }; + + if !vec![DEVICEMAPPER_PATH, &metadata.activation_name.to_string()] + .into_iter() + .collect::() + .exists() + { + activate( + device, + metadata.encryption_info.key_description(), + unlock_method, + &metadata.activation_name, + )? + } + + Ok(Some(CryptHandle::new( + metadata.physical_path, + metadata.pool_uuid, + metadata.encryption_info, + metadata.device, + ))) +} + +/// Handle for performing all operations on an encrypted device. +/// +/// `Clone` is derived for this data structure because `CryptHandle` acquires +/// a new crypt device context for each operation. +#[derive(Debug, Clone)] +pub struct CryptHandle { + metadata: CryptMetadata, +} + +impl CryptHandle { + pub(super) fn new( + physical_path: DevicePath, + pool_uuid: PoolUuid, + encryption_info: EncryptionInfo, + devno: Device, + ) -> CryptHandle { + let activation_name = format_crypt_backstore_name(&pool_uuid); + let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] + .into_iter() + .collect::(); + let activated_path = path.canonicalize().unwrap_or(path); + CryptHandle { + metadata: CryptMetadata { + physical_path, + pool_uuid, + encryption_info, + activation_name, + device: devno, + activated_path, + }, + } + } + + /// Initialize a device with the provided key description and Clevis info. + pub fn initialize( + physical_path: &Path, + pool_uuid: PoolUuid, + encryption_info: &EncryptionInfo, + sector_size: Option, + ) -> StratisResult { + let activation_name = format_crypt_backstore_name(&pool_uuid); + + let luks2_params = sector_size.map(|sector_size| CryptParamsLuks2 { + pbkdf: None, + integrity: None, + integrity_params: None, + data_alignment: 0, + data_device: None, + sector_size, + label: None, + subsystem: None, + }); + + let mut device = log_on_failure!( + CryptInit::init(physical_path), + "Failed to acquire context for device {} while initializing; \ + nothing to clean up", + physical_path.display() + ); + device.settings_handle().set_metadata_size( + MetadataSize::try_from(convert_int!(*DEFAULT_CRYPT_METADATA_SIZE, u128, u64)?)?, + KeyslotsSize::try_from(convert_int!(*DEFAULT_CRYPT_KEYSLOTS_SIZE, u128, u64)?)?, + )?; + Self::initialize_with_err(&mut device, physical_path, pool_uuid, encryption_info, luks2_params.as_ref()) + .and_then(|path| clevis_info_from_metadata(&mut device).map(|ci| (path, ci))) + .and_then(|(_, clevis_info)| { + let encryption_info = + if let Some(info) = EncryptionInfo::from_options((encryption_info.key_description().cloned(), clevis_info)) { + info + } else { + return Err(StratisError::Msg(format!( + "No valid encryption method that can be used to unlock device {} found after initialization", + physical_path.display() + ))); + }; + + let device_path = DevicePath::new(physical_path)?; + let devno = get_devno_from_path(physical_path)?; + Ok(CryptHandle::new( + device_path, + pool_uuid, + encryption_info, + devno, + )) + }) + .map_err(|e| { + if let Err(err) = + Self::rollback(&mut device, physical_path, &activation_name) + { + warn!( + "Failed to roll back crypt device initialization; you may need to manually wipe this device: {}", + err + ); + } + e + }) + } + + /// Initialize with a passphrase in the kernel keyring only. + fn initialize_with_keyring( + device: &mut CryptDevice, + key_description: &KeyDescription, + ) -> StratisResult<()> { + add_keyring_keyslot(device, key_description, None)?; + + Ok(()) + } + + /// Initialize with Clevis only. + fn initialize_with_clevis( + device: &mut CryptDevice, + physical_path: &Path, + (pin, json, yes): (&str, &Value, bool), + ) -> StratisResult<()> { + let (_, key_data) = thread_rng() + .sample_iter(Alphanumeric) + .take(MAX_STRATIS_PASS_SIZE) + .fold( + (0, SafeMemHandle::alloc(MAX_STRATIS_PASS_SIZE)?), + |(idx, mut mem), ch| { + mem.as_mut()[idx] = ch; + (idx + 1, mem) + }, + ); + + let key = SizedKeyMemory::new(key_data, MAX_STRATIS_PASS_SIZE); + let keyslot = log_on_failure!( + device + .keyslot_handle() + .add_by_key(None, None, key.as_ref(), CryptVolumeKey::empty(),), + "Failed to initialize keyslot with provided key in keyring" + ); + + clevis_luks_bind( + physical_path, + Either::Right(key), + CLEVIS_LUKS_TOKEN_ID, + pin, + json, + yes, + )?; + + // Need to reload device here to refresh the state of the device + // after being modified by Clevis. + if let Err(e) = device + .context_handle() + .load::<()>(Some(EncryptionFormat::Luks2), None) + { + return Err(wipe_fallback(physical_path, StratisError::from(e))); + } + + device.keyslot_handle().destroy(keyslot)?; + + Ok(()) + } + + /// Initialize with both a passphrase in the kernel keyring and Clevis. + fn initialize_with_both( + device: &mut CryptDevice, + physical_path: &Path, + key_description: &KeyDescription, + (pin, json, yes): (&str, &Value, bool), + ) -> StratisResult<()> { + Self::initialize_with_keyring(device, key_description)?; + + clevis_luks_bind( + physical_path, + Either::Left(LUKS2_TOKEN_ID), + CLEVIS_LUKS_TOKEN_ID, + pin, + json, + yes, + )?; + + // Need to reload device here to refresh the state of the device + // after being modified by Clevis. + if let Err(e) = device + .context_handle() + .load::<()>(Some(EncryptionFormat::Luks2), None) + { + return Err(wipe_fallback(physical_path, StratisError::from(e))); + } + + Ok(()) + } + + fn initialize_with_err( + device: &mut CryptDevice, + physical_path: &Path, + pool_uuid: PoolUuid, + encryption_info: &EncryptionInfo, + luks2_params: Option<&CryptParamsLuks2>, + ) -> StratisResult<()> { + let mut luks2_params_ref: Option> = + luks2_params.map(|lp| lp.try_into()).transpose()?; + + log_on_failure!( + device.context_handle().format::>( + EncryptionFormat::Luks2, + ("aes", "xts-plain64"), + None, + libcryptsetup_rs::Either::Right(STRATIS_MEK_SIZE), + luks2_params_ref.as_mut() + ), + "Failed to format device {} with LUKS2 header", + physical_path.display() + ); + + match encryption_info { + EncryptionInfo::Both(kd, (pin, config)) => { + let mut parsed_config = config.clone(); + let y = interpret_clevis_config(pin, &mut parsed_config)?; + Self::initialize_with_both(device, physical_path, kd, (pin, &parsed_config, y))? + } + EncryptionInfo::KeyDesc(kd) => Self::initialize_with_keyring(device, kd)?, + EncryptionInfo::ClevisInfo((pin, config)) => { + let mut parsed_config = config.clone(); + let y = interpret_clevis_config(pin, &mut parsed_config)?; + Self::initialize_with_clevis(device, physical_path, (pin, &parsed_config, y))? + } + }; + + let activation_name = format_crypt_backstore_name(&pool_uuid); + activate( + device, + encryption_info.key_description(), + UnlockMethod::Any, + &activation_name, + ) + } + + pub fn rollback( + device: &mut CryptDevice, + physical_path: &Path, + name: &DmName, + ) -> StratisResult<()> { + ensure_wiped(device, physical_path, name) + } + + /// Acquire the crypt device handle for the physical path in this `CryptHandle`. + pub(super) fn acquire_crypt_device(&self) -> StratisResult { + acquire_crypt_device(self.luks2_device_path()) + } + + /// Query the device metadata to reconstruct a handle for performing operations + /// on an existing encrypted device. + /// + /// This method will check that the metadata on the given device is + /// for the LUKS2 format and that the LUKS2 metadata is formatted + /// properly as a Stratis encrypted device. If it is properly + /// formatted it will return the device identifiers (pool and device UUIDs). + /// + /// The checks include: + /// * is a LUKS2 device + /// * has a valid Stratis LUKS2 token + /// * has a token of the proper type for LUKS2 keyring unlocking + pub fn setup( + physical_path: &Path, + pool_uuid: PoolUuid, + unlock_method: UnlockMethod, + ) -> StratisResult> { + match setup_crypt_device(physical_path)? { + Some(ref mut device) => { + setup_crypt_handle(device, physical_path, pool_uuid, unlock_method) + } + None => Ok(None), + } + } + + /// Load the required information for Stratis from the LUKS2 metadata. + #[cfg(test)] + pub fn load_metadata( + physical_path: &Path, + pool_uuid: PoolUuid, + ) -> StratisResult> { + match setup_crypt_device(physical_path)? { + Some(ref mut device) => load_crypt_metadata(device, physical_path, pool_uuid), + None => Ok(None), + } + } + + /// Get the encryption info for this encrypted device. + pub fn encryption_info(&self) -> &EncryptionInfo { + &self.metadata.encryption_info + } + + /// Return the path to the device node of the underlying storage device + /// for the encrypted device. + pub fn luks2_device_path(&self) -> &Path { + &self.metadata.physical_path + } + + /// Return the name of the activated devicemapper device. + pub fn activation_name(&self) -> &DmName { + &self.metadata.activation_name + } + + /// Return the path of the activated devicemapper device. + #[cfg(test)] + pub fn activated_device_path(&self) -> &Path { + &self.metadata.activated_path + } + + /// Device number for the LUKS2 encrypted device. + pub fn device(&self) -> Device { + self.metadata.device + } + + /// Get the keyslot associated with the given token ID. + pub fn keyslots(&self, token_id: c_uint) -> StratisResult>> { + get_keyslot_number(&mut self.acquire_crypt_device()?, token_id) + } + + /// Get info for the clevis binding. + pub fn clevis_info(&self) -> StratisResult> { + clevis_info_from_metadata(&mut self.acquire_crypt_device()?) + } + + /// Bind the given device using clevis. + pub fn clevis_bind(&mut self, pin: &str, json: &Value) -> StratisResult<()> { + let mut json_owned = json.clone(); + let yes = interpret_clevis_config(pin, &mut json_owned)?; + + clevis_luks_bind( + self.luks2_device_path(), + Either::Left(LUKS2_TOKEN_ID), + CLEVIS_LUKS_TOKEN_ID, + pin, + &json_owned, + yes, + )?; + self.metadata.encryption_info = + self.metadata + .encryption_info + .clone() + .set_clevis_info(self.clevis_info()?.ok_or_else(|| { + StratisError::Msg( + "Clevis reported successfully binding to device but no metadata was found" + .to_string(), + ) + })?); + Ok(()) + } + + /// Unbind the given device using clevis. + pub fn clevis_unbind(&mut self) -> StratisResult<()> { + if self.metadata.encryption_info.key_description().is_none() { + return Err(StratisError::Msg( + "No kernel keyring binding found; removing the Clevis binding \ + would remove the ability to open this device; aborting" + .to_string(), + )); + } + + let keyslots = self.keyslots(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { + StratisError::Msg(format!( + "Token slot {CLEVIS_LUKS_TOKEN_ID} appears to be empty; could not determine keyslots" + )) + })?; + for keyslot in keyslots { + log_on_failure!( + clevis_luks_unbind(self.luks2_device_path(), keyslot), + "Failed to unbind device {} from Clevis", + self.luks2_device_path().display() + ); + } + self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_clevis_info(); + Ok(()) + } + + /// Change the key description and passphrase that a device is bound to + /// + /// This method needs to re-read the cached Clevis information because + /// the config may change specifically in the case where a new thumbprint + /// is provided if Tang keys are rotated. + pub fn rebind_clevis(&mut self) -> StratisResult<()> { + if self.metadata.encryption_info.clevis_info().is_none() { + return Err(StratisError::Msg( + "No Clevis binding found; cannot regenerate the Clevis binding if the device does not already have a Clevis binding".to_string(), + )); + } + + let mut device = self.acquire_crypt_device()?; + let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)? + .and_then(|vec| vec.into_iter().next()) + .ok_or_else(|| { + StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) + })?; + + clevis_luks_regen(self.luks2_device_path(), keyslot)?; + // Need to reload LUKS2 metadata after Clevis metadata modification. + if let Err(e) = device + .context_handle() + .load::<()>(Some(EncryptionFormat::Luks2), None) + { + return Err(StratisError::Chained( + "Failed to reload crypt device state after modification to Clevis data".to_string(), + Box::new(StratisError::from(e)), + )); + } + + let (pin, config) = clevis_info_from_metadata(&mut device)?.ok_or_else(|| { + StratisError::Msg(format!( + "Did not find Clevis metadata on device {}", + self.luks2_device_path().display() + )) + })?; + self.metadata.encryption_info = self + .metadata + .encryption_info + .clone() + .set_clevis_info((pin, config)); + Ok(()) + } + + /// Add a keyring binding to the underlying LUKS2 volume. + pub fn bind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult<()> { + let mut device = self.acquire_crypt_device()?; + let key = Self::clevis_decrypt(&mut device)?.ok_or_else(|| { + StratisError::Msg( + "The Clevis token appears to have been wiped outside of \ + Stratis; cannot add a keyring key binding without an existing \ + passphrase to unlock the device" + .to_string(), + ) + })?; + + add_keyring_keyslot(&mut device, key_desc, Some(Either::Left(key)))?; + + self.metadata.encryption_info = self + .metadata + .encryption_info + .clone() + .set_key_desc(key_desc.clone()); + Ok(()) + } + + /// Add a keyring binding to the underlying LUKS2 volume. + pub fn unbind_keyring(&mut self) -> StratisResult<()> { + if self.metadata.encryption_info.clevis_info().is_none() { + return Err(StratisError::Msg( + "No Clevis binding was found; removing the keyring binding would \ + remove the ability to open this device; aborting" + .to_string(), + )); + } + + let mut device = self.acquire_crypt_device()?; + let keyslots = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? + .ok_or_else(|| StratisError::Msg("No LUKS2 keyring token was found".to_string()))?; + for keyslot in keyslots { + log_on_failure!( + device.keyslot_handle().destroy(keyslot), + "Failed partway through the kernel keyring unbinding operation \ + which cannot be rolled back; manual intervention may be required" + ) + } + device + .token_handle() + .json_set(TokenInput::RemoveToken(LUKS2_TOKEN_ID))?; + + self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_key_desc(); + + Ok(()) + } + + /// Change the key description and passphrase that a device is bound to + pub fn rebind_keyring(&mut self, new_key_desc: &KeyDescription) -> StratisResult<()> { + let mut device = self.acquire_crypt_device()?; + + let old_key_description = self.metadata.encryption_info + .key_description() + .ok_or_else(|| { + StratisError::Msg("Cannot change passphrase because this device is not bound to a passphrase in the kernel keyring".to_string()) + })?; + add_keyring_keyslot( + &mut device, + new_key_desc, + Some(Either::Right(old_key_description)), + )?; + self.metadata.encryption_info = self + .metadata + .encryption_info + .clone() + .set_key_desc(new_key_desc.clone()); + Ok(()) + } + + /// Decrypt a Clevis passphrase and return it securely. + fn clevis_decrypt(device: &mut CryptDevice) -> StratisResult> { + let mut token = match device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).ok() { + Some(t) => t, + None => return Ok(None), + }; + let jwe = token + .as_object_mut() + .and_then(|map| map.remove("jwe")) + .ok_or_else(|| { + StratisError::Msg(format!( + "Token slot {CLEVIS_LUKS_TOKEN_ID} is occupied but does not appear to be a Clevis \ + token; aborting" + )) + })?; + clevis_decrypt(&jwe).map(Some) + } + + /// Deactivate the device referenced by the current device handle. + #[cfg(test)] + pub fn deactivate(&self) -> StratisResult<()> { + ensure_inactive(&mut self.acquire_crypt_device()?, self.activation_name()) + } + + /// Wipe all LUKS2 metadata on the device safely using libcryptsetup. + pub fn wipe(&self) -> StratisResult<()> { + ensure_wiped( + &mut self.acquire_crypt_device()?, + self.luks2_device_path(), + self.activation_name(), + ) + } + + /// Changed the encrypted device size + /// `None` will fill up the entire underlying physical device. + /// `Some(_)` will resize the device to the given number of sectors. + pub fn resize(&self, size: Option) -> StratisResult<()> { + let processed_size = match size { + Some(s) => { + if s == Sectors(0) { + return Err(StratisError::Msg( + "Cannot specify a crypt device size of zero".to_string(), + )); + } else { + *s + } + } + None => 0, + }; + let mut crypt = self.acquire_crypt_device()?; + crypt.token_handle().activate_by_token::<()>( + None, + None, + None, + CryptActivate::KEYRING_KEY, + )?; + crypt + .context_handle() + .resize(&self.activation_name().to_string(), processed_size) + .map_err(StratisError::Crypt) + } +} + +#[cfg(test)] +mod tests { + use std::{ + env, + ffi::CString, + fs::{File, OpenOptions}, + io::{self, Read, Write}, + mem::MaybeUninit, + path::Path, + ptr, slice, + }; + + use devicemapper::{Bytes, Sectors, IEC}; + use libcryptsetup_rs::{ + consts::vals::{CryptStatusInfo, EncryptionFormat}, + CryptInit, Either, + }; + + use crate::engine::{ + strat_engine::{ + crypt::{ + consts::{ + CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, + }, + shared::acquire_crypt_device, + }, + ns::{unshare_mount_namespace, MemoryFilesystem}, + tests::{crypt, loopbacked, real}, + }, + types::{EncryptionInfo, KeyDescription, PoolUuid, UnlockMethod}, + }; + + use super::*; + + /// If this method is called without a key with the specified key description + /// in the kernel ring, it should always fail and allow us to test the rollback + /// of failed initializations. + fn test_failed_init(paths: &[&Path]) { + assert_eq!(paths.len(), 1); + + let path = paths.first().expect("There must be exactly one path"); + let key_description = + KeyDescription::try_from("I am not a key".to_string()).expect("no semi-colons"); + + let pool_uuid = PoolUuid::new_v4(); + + let result = CryptHandle::initialize( + path, + pool_uuid, + &EncryptionInfo::KeyDesc(key_description), + None, + ); + + // Initialization cannot occur with a non-existent key + assert!(result.is_err()); + + assert!(CryptHandle::load_metadata(path, pool_uuid) + .unwrap() + .is_none()); + + // TODO: Check actual superblock with libblkid + } + + #[test] + fn loop_test_failed_init() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_failed_init, + ); + } + + #[test] + fn real_test_failed_init() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_failed_init, + ); + } + + /// Test initializing and activating an encrypted device using + /// the utilities provided here. + /// + /// The overall format of the test involves generating a random byte buffer + /// of size 1 MiB, encrypting it on disk, and then ensuring that the plaintext + /// cannot be found on the encrypted disk by doing a scan of the disk using + /// a sliding window. + /// + /// The sliding window size of 1 MiB was chosen to lower the number of + /// searches that need to be done compared to a smaller sliding window + /// and also to decrease the probability of the random sequence being found + /// on the disk due to leftover data from other tests. + // TODO: Rewrite libc calls using nix crate. + fn test_crypt_device_ops(paths: &[&Path]) { + fn crypt_test(paths: &[&Path], key_desc: &KeyDescription) { + let path = paths + .first() + .expect("This test only accepts a single device"); + + let pool_uuid = PoolUuid::new_v4(); + + let handle = CryptHandle::initialize( + path, + pool_uuid, + &EncryptionInfo::KeyDesc(key_desc.clone()), + None, + ) + .unwrap(); + let logical_path = handle.activated_device_path(); + + const WINDOW_SIZE: usize = 1024 * 1024; + let mut devicenode = OpenOptions::new().write(true).open(logical_path).unwrap(); + let mut random_buffer = vec![0; WINDOW_SIZE].into_boxed_slice(); + File::open("/dev/urandom") + .unwrap() + .read_exact(&mut random_buffer) + .unwrap(); + devicenode.write_all(&random_buffer).unwrap(); + std::mem::drop(devicenode); + + let dev_path_cstring = + CString::new(path.to_str().expect("Failed to convert path to string")).unwrap(); + let fd = unsafe { libc::open(dev_path_cstring.as_ptr(), libc::O_RDONLY) }; + if fd < 0 { + panic!("{}", io::Error::last_os_error()); + } + + let mut stat: MaybeUninit = MaybeUninit::zeroed(); + let fstat_result = unsafe { libc::fstat(fd, stat.as_mut_ptr()) }; + if fstat_result < 0 { + panic!("{}", io::Error::last_os_error()); + } + let device_size = + convert_int!(unsafe { stat.assume_init() }.st_size, libc::off_t, usize).unwrap(); + let mapped_ptr = unsafe { + libc::mmap( + ptr::null_mut(), + device_size, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapped_ptr.is_null() { + panic!("mmap failed"); + } + + { + let disk_buffer = + unsafe { slice::from_raw_parts(mapped_ptr as *const u8, device_size) }; + for window in disk_buffer.windows(WINDOW_SIZE) { + if window == &*random_buffer as &[u8] { + unsafe { + libc::munmap(mapped_ptr, device_size); + libc::close(fd); + }; + panic!("Disk was not encrypted!"); + } + } + } + + unsafe { + libc::munmap(mapped_ptr, device_size); + libc::close(fd); + }; + + let device_name = handle.activation_name(); + loop { + match libcryptsetup_rs::status( + Some(&mut handle.acquire_crypt_device().unwrap()), + &device_name.to_string(), + ) { + Ok(CryptStatusInfo::Busy) => (), + Ok(CryptStatusInfo::Active) => break, + Ok(s) => { + panic!("Crypt device is in invalid state {s:?}") + } + Err(e) => { + panic!("Checking device status returned error: {e}") + } + } + } + + handle.deactivate().unwrap(); + + let handle = CryptHandle::setup(path, pool_uuid, UnlockMethod::Keyring) + .unwrap() + .unwrap_or_else(|| { + panic!( + "Device {} no longer appears to be a LUKS2 device", + path.display(), + ) + }); + handle.wipe().unwrap(); + } + + assert_eq!(paths.len(), 1); + + crypt::insert_and_cleanup_key(paths, crypt_test); + } + + #[test] + fn real_test_crypt_device_ops() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(2 * IEC::Mi))), + test_crypt_device_ops, + ); + } + + #[test] + fn loop_test_crypt_metadata_defaults() { + fn test_defaults(paths: &[&Path]) { + let mut context = CryptInit::init(paths[0]).unwrap(); + context + .context_handle() + .format::<()>( + EncryptionFormat::Luks2, + ("aes", "xts-plain64"), + None, + Either::Right(STRATIS_MEK_SIZE), + None, + ) + .unwrap(); + let (metadata, keyslot) = context.settings_handle().get_metadata_size().unwrap(); + assert_eq!(DEFAULT_CRYPT_METADATA_SIZE, Bytes::from(*metadata)); + assert_eq!(DEFAULT_CRYPT_KEYSLOTS_SIZE, Bytes::from(*keyslot)); + } + + loopbacked::test_with_spec(&loopbacked::DeviceLimits::Exactly(1, None), test_defaults); + } + + #[test] + // Test passing an unusual, larger sector size for cryptsetup. 4096 should + // be no smaller than the physical sector size of the loop device, and + // should be allowed by cryptsetup. + fn loop_test_set_sector_size() { + fn the_test(paths: &[&Path]) { + fn test_set_sector_size(paths: &[&Path], key_description: &KeyDescription) { + let pool_uuid = PoolUuid::new_v4(); + + CryptHandle::initialize( + paths[0], + pool_uuid, + &EncryptionInfo::KeyDesc(key_description.clone()), + Some(4096u32), + ) + .unwrap(); + } + + crypt::insert_and_cleanup_key(paths, test_set_sector_size); + } + + loopbacked::test_with_spec(&loopbacked::DeviceLimits::Exactly(1, None), the_test); + } + + fn test_both_initialize(paths: &[&Path]) { + fn both_initialize(paths: &[&Path], key_desc: &KeyDescription, pool_uuid: PoolUuid) { + unshare_mount_namespace().unwrap(); + let _memfs = MemoryFilesystem::new().unwrap(); + let path = paths.first().copied().expect("Expected exactly one path"); + let handle = CryptHandle::initialize( + path, + pool_uuid, + &EncryptionInfo::Both( + key_desc.clone(), + ( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + ), + ), + None, + ).unwrap(); + + let mut device = acquire_crypt_device(handle.luks2_device_path()).unwrap(); + device.token_handle().json_get(LUKS2_TOKEN_ID).unwrap(); + device + .token_handle() + .json_get(CLEVIS_LUKS_TOKEN_ID) + .unwrap(); + handle.deactivate().unwrap(); + } + + fn unlock_clevis(paths: &[&Path], pool_uuid: PoolUuid) { + let path = paths.first().copied().expect("Expected exactly one path"); + CryptHandle::setup(path, pool_uuid, UnlockMethod::Clevis) + .unwrap() + .unwrap(); + } + + let pool_uuid = PoolUuid::new_v4(); + crypt::insert_and_remove_key( + paths, + |paths, key_desc| both_initialize(paths, key_desc, pool_uuid), + |paths| unlock_clevis(paths, pool_uuid), + ); + } + + #[test] + fn clevis_real_test_both_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_both_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_real_should_fail_test_both_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_both_initialize, + ); + } + + #[test] + fn clevis_loop_test_both_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_both_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_loop_should_fail_test_both_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_both_initialize, + ); + } + + fn test_clevis_initialize(paths: &[&Path]) { + unshare_mount_namespace().unwrap(); + let _memfs = MemoryFilesystem::new().unwrap(); + let path = paths[0]; + + let handle = CryptHandle::initialize( + path, + PoolUuid::new_v4(), + &EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + )), + None, + ) + .unwrap(); + + let mut device = acquire_crypt_device(handle.luks2_device_path()).unwrap(); + assert!(device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).is_ok()); + assert!(device.token_handle().json_get(LUKS2_TOKEN_ID).is_err()); + } + + #[test] + fn clevis_real_test_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_clevis_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_real_should_fail_test_initialize() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), + test_clevis_initialize, + ); + } + + #[test] + fn clevis_loop_test_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_loop_should_fail_test_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_initialize, + ); + } + + fn test_clevis_tang_configs(paths: &[&Path]) { + let path = paths[0]; + + assert!(CryptHandle::initialize( + path, + PoolUuid::new_v4(), + &EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required")}), + )), + None, + ) + .is_err()); + CryptHandle::initialize( + path, + PoolUuid::new_v4(), + &EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({ + "stratis:tang:trust_url": true, + "url": env::var("TANG_URL").expect("TANG_URL env var required"), + }), + )), + None, + ) + .unwrap(); + } + + #[test] + fn clevis_real_test_clevis_tang_configs() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_clevis_tang_configs, + ); + } + + #[test] + fn clevis_loop_test_clevis_tang_configs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_tang_configs, + ); + } + + fn test_clevis_sss_configs(paths: &[&Path]) { + let path = paths[0]; + + assert!(CryptHandle::initialize( + path, + PoolUuid::new_v4(), + &EncryptionInfo::ClevisInfo(( + "sss".to_string(), + json!({"t": 1, "pins": {"tang": {"url": env::var("TANG_URL").expect("TANG_URL env var required")}, "tpm2": {}}}), + )), + None, + ) + .is_err()); + CryptHandle::initialize( + path, + PoolUuid::new_v4(), + &EncryptionInfo::ClevisInfo(( + "sss".to_string(), + json!({ + "t": 1, + "stratis:tang:trust_url": true, + "pins": { + "tang": {"url": env::var("TANG_URL").expect("TANG_URL env var required")}, + "tpm2": {} + } + }), + )), + None, + ) + .unwrap(); + } + + #[test] + fn clevis_real_test_clevis_sss_configs() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_clevis_sss_configs, + ); + } + + #[test] + fn clevis_loop_test_clevis_sss_configs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_clevis_sss_configs, + ); + } +} diff --git a/src/engine/strat_engine/crypt/mod.rs b/src/engine/strat_engine/crypt/mod.rs index 3bf75b68f8..1915b19156 100644 --- a/src/engine/strat_engine/crypt/mod.rs +++ b/src/engine/strat_engine/crypt/mod.rs @@ -6,585 +6,13 @@ mod macros; mod consts; -mod handle; +pub mod handle; mod shared; pub use self::{ consts::CLEVIS_TANG_TRUST_URL, - handle::CryptHandle, shared::{ back_up_luks_header, crypt_metadata_size, interpret_clevis_config, register_clevis_token, restore_luks_header, set_up_crypt_logging, }, }; - -#[cfg(test)] -mod tests { - use std::{ - env, - ffi::CString, - fs::{File, OpenOptions}, - io::{self, Read, Write}, - mem::MaybeUninit, - path::Path, - ptr, slice, - }; - - use devicemapper::{Bytes, Sectors, IEC}; - use libcryptsetup_rs::{ - consts::vals::{CryptStatusInfo, EncryptionFormat}, - CryptInit, Either, - }; - - use crate::engine::{ - strat_engine::{ - crypt::{ - consts::{ - CLEVIS_LUKS_TOKEN_ID, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, - LUKS2_TOKEN_ID, STRATIS_MEK_SIZE, - }, - shared::acquire_crypt_device, - }, - ns::{unshare_mount_namespace, MemoryFilesystem}, - tests::{crypt, loopbacked, real}, - }, - types::{DevUuid, EncryptionInfo, KeyDescription, Name, PoolUuid, UnlockMethod}, - }; - - use super::*; - - /// If this method is called without a key with the specified key description - /// in the kernel ring, it should always fail and allow us to test the rollback - /// of failed initializations. - fn test_failed_init(paths: &[&Path]) { - assert_eq!(paths.len(), 1); - - let path = paths.first().expect("There must be exactly one path"); - let key_description = - KeyDescription::try_from("I am not a key".to_string()).expect("no semi-colons"); - - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - let dev_uuid = DevUuid::new_v4(); - - let result = CryptHandle::initialize( - path, - pool_uuid, - dev_uuid, - pool_name, - &EncryptionInfo::KeyDesc(key_description), - None, - ); - - // Initialization cannot occur with a non-existent key - assert!(result.is_err()); - - assert!(CryptHandle::load_metadata(path).unwrap().is_none()); - - // TODO: Check actual superblock with libblkid - } - - #[test] - fn loop_test_failed_init() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_failed_init, - ); - } - - #[test] - fn real_test_failed_init() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), - test_failed_init, - ); - } - - /// Test the method `can_unlock` works on an initialized device in both - /// active and inactive states. - fn test_can_unlock(paths: &[&Path]) { - fn crypt_test(paths: &[&Path], key_desc: &KeyDescription) { - let mut handles = vec![]; - - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - for path in paths { - let dev_uuid = DevUuid::new_v4(); - - let handle = CryptHandle::initialize( - path, - pool_uuid, - dev_uuid, - pool_name.clone(), - &EncryptionInfo::KeyDesc(key_desc.clone()), - None, - ) - .unwrap(); - handles.push(handle); - } - - for path in paths { - if !CryptHandle::can_unlock(path, true, false) { - panic!("All devices should be able to be unlocked"); - } - } - - for handle in handles.iter_mut() { - handle.deactivate().unwrap(); - } - - for path in paths { - if !CryptHandle::can_unlock(path, true, false) { - panic!("All devices should be able to be unlocked"); - } - } - - for handle in handles.iter_mut() { - handle.wipe().unwrap(); - } - - for path in paths { - if CryptHandle::can_unlock(path, true, false) { - panic!("All devices should no longer be able to be unlocked"); - } - } - } - - crypt::insert_and_cleanup_key(paths, crypt_test) - } - - #[test] - fn loop_test_can_unlock() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, None), - test_can_unlock, - ); - } - - #[test] - fn real_test_can_unlock() { - real::test_with_spec( - &real::DeviceLimits::Range(1, 3, None, None), - test_can_unlock, - ); - } - - /// Test initializing and activating an encrypted device using - /// the utilities provided here. - /// - /// The overall format of the test involves generating a random byte buffer - /// of size 1 MiB, encrypting it on disk, and then ensuring that the plaintext - /// cannot be found on the encrypted disk by doing a scan of the disk using - /// a sliding window. - /// - /// The sliding window size of 1 MiB was chosen to lower the number of - /// searches that need to be done compared to a smaller sliding window - /// and also to decrease the probability of the random sequence being found - /// on the disk due to leftover data from other tests. - // TODO: Rewrite libc calls using nix crate. - fn test_crypt_device_ops(paths: &[&Path]) { - fn crypt_test(paths: &[&Path], key_desc: &KeyDescription) { - let path = paths - .first() - .expect("This test only accepts a single device"); - - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - let dev_uuid = DevUuid::new_v4(); - - let handle = CryptHandle::initialize( - path, - pool_uuid, - dev_uuid, - pool_name, - &EncryptionInfo::KeyDesc(key_desc.clone()), - None, - ) - .unwrap(); - let logical_path = handle.activated_device_path(); - - const WINDOW_SIZE: usize = 1024 * 1024; - let mut devicenode = OpenOptions::new().write(true).open(logical_path).unwrap(); - let mut random_buffer = vec![0; WINDOW_SIZE].into_boxed_slice(); - File::open("/dev/urandom") - .unwrap() - .read_exact(&mut random_buffer) - .unwrap(); - devicenode.write_all(&random_buffer).unwrap(); - std::mem::drop(devicenode); - - let dev_path_cstring = - CString::new(path.to_str().expect("Failed to convert path to string")).unwrap(); - let fd = unsafe { libc::open(dev_path_cstring.as_ptr(), libc::O_RDONLY) }; - if fd < 0 { - panic!("{:?}", io::Error::last_os_error()); - } - - let mut stat: MaybeUninit = MaybeUninit::zeroed(); - let fstat_result = unsafe { libc::fstat(fd, stat.as_mut_ptr()) }; - if fstat_result < 0 { - panic!("{:?}", io::Error::last_os_error()); - } - let device_size = - convert_int!(unsafe { stat.assume_init() }.st_size, libc::off_t, usize).unwrap(); - let mapped_ptr = unsafe { - libc::mmap( - ptr::null_mut(), - device_size, - libc::PROT_READ, - libc::MAP_SHARED, - fd, - 0, - ) - }; - if mapped_ptr.is_null() { - panic!("mmap failed"); - } - - { - let disk_buffer = - unsafe { slice::from_raw_parts(mapped_ptr as *const u8, device_size) }; - for window in disk_buffer.windows(WINDOW_SIZE) { - if window == &*random_buffer as &[u8] { - unsafe { - libc::munmap(mapped_ptr, device_size); - libc::close(fd); - }; - panic!("Disk was not encrypted!"); - } - } - } - - unsafe { - libc::munmap(mapped_ptr, device_size); - libc::close(fd); - }; - - let device_name = handle.activation_name(); - loop { - match libcryptsetup_rs::status( - Some(&mut handle.acquire_crypt_device().unwrap()), - &device_name.to_string(), - ) { - Ok(CryptStatusInfo::Busy) => (), - Ok(CryptStatusInfo::Active) => break, - Ok(s) => { - panic!("Crypt device is in invalid state {s:?}") - } - Err(e) => { - panic!("Checking device status returned error: {e}") - } - } - } - - handle.deactivate().unwrap(); - - let handle = CryptHandle::setup(path, Some(UnlockMethod::Keyring)) - .unwrap() - .unwrap_or_else(|| { - panic!( - "Device {} no longer appears to be a LUKS2 device", - path.display(), - ) - }); - handle.wipe().unwrap(); - } - - assert_eq!(paths.len(), 1); - - crypt::insert_and_cleanup_key(paths, crypt_test); - } - - #[test] - fn real_test_crypt_device_ops() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, Some(Sectors(2 * IEC::Mi))), - test_crypt_device_ops, - ); - } - - #[test] - fn loop_test_crypt_metadata_defaults() { - fn test_defaults(paths: &[&Path]) { - let mut context = CryptInit::init(paths[0]).unwrap(); - context - .context_handle() - .format::<()>( - EncryptionFormat::Luks2, - ("aes", "xts-plain64"), - None, - Either::Right(STRATIS_MEK_SIZE), - None, - ) - .unwrap(); - let (metadata, keyslot) = context.settings_handle().get_metadata_size().unwrap(); - assert_eq!(DEFAULT_CRYPT_METADATA_SIZE, Bytes::from(*metadata)); - assert_eq!(DEFAULT_CRYPT_KEYSLOTS_SIZE, Bytes::from(*keyslot)); - } - - loopbacked::test_with_spec(&loopbacked::DeviceLimits::Exactly(1, None), test_defaults); - } - - #[test] - // Test passing an unusual, larger sector size for cryptsetup. 4096 should - // be no smaller than the physical sector size of the loop device, and - // should be allowed by cryptsetup. - fn loop_test_set_sector_size() { - fn the_test(paths: &[&Path]) { - fn test_set_sector_size(paths: &[&Path], key_description: &KeyDescription) { - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - let dev_uuid = DevUuid::new_v4(); - - CryptHandle::initialize( - paths[0], - pool_uuid, - dev_uuid, - pool_name, - &EncryptionInfo::KeyDesc(key_description.clone()), - Some(4096u32), - ) - .unwrap(); - } - - crypt::insert_and_cleanup_key(paths, test_set_sector_size); - } - - loopbacked::test_with_spec(&loopbacked::DeviceLimits::Exactly(1, None), the_test); - } - - fn test_both_initialize(paths: &[&Path]) { - fn both_initialize(paths: &[&Path], key_desc: &KeyDescription) { - unshare_mount_namespace().unwrap(); - let _memfs = MemoryFilesystem::new().unwrap(); - let path = paths.first().copied().expect("Expected exactly one path"); - let pool_name = Name::new("pool_name".to_string()); - let handle = CryptHandle::initialize( - path, - PoolUuid::new_v4(), - DevUuid::new_v4(), - pool_name, - &EncryptionInfo::Both( - key_desc.clone(), - ( - "tang".to_string(), - json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), - ), - ), - None, - ).unwrap(); - - let mut device = acquire_crypt_device(handle.luks2_device_path()).unwrap(); - device.token_handle().json_get(LUKS2_TOKEN_ID).unwrap(); - device - .token_handle() - .json_get(CLEVIS_LUKS_TOKEN_ID) - .unwrap(); - handle.deactivate().unwrap(); - } - - fn unlock_clevis(paths: &[&Path]) { - let path = paths.first().copied().expect("Expected exactly one path"); - CryptHandle::setup(path, Some(UnlockMethod::Clevis)) - .unwrap() - .unwrap(); - } - - crypt::insert_and_remove_key(paths, both_initialize, unlock_clevis); - } - - #[test] - fn clevis_real_test_both_initialize() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), - test_both_initialize, - ); - } - - #[test] - #[should_panic] - fn clevis_real_should_fail_test_both_initialize() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), - test_both_initialize, - ); - } - - #[test] - fn clevis_loop_test_both_initialize() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_both_initialize, - ); - } - - #[test] - #[should_panic] - fn clevis_loop_should_fail_test_both_initialize() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_both_initialize, - ); - } - - fn test_clevis_initialize(paths: &[&Path]) { - unshare_mount_namespace().unwrap(); - - let _memfs = MemoryFilesystem::new().unwrap(); - let path = paths[0]; - let pool_name = Name::new("pool_name".to_string()); - - let handle = CryptHandle::initialize( - path, - PoolUuid::new_v4(), - DevUuid::new_v4(), - pool_name, - &EncryptionInfo::ClevisInfo(( - "tang".to_string(), - json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), - )), - None, - ) - .unwrap(); - - let mut device = acquire_crypt_device(handle.luks2_device_path()).unwrap(); - assert!(device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).is_ok()); - assert!(device.token_handle().json_get(LUKS2_TOKEN_ID).is_err()); - } - - #[test] - fn clevis_real_test_initialize() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), - test_clevis_initialize, - ); - } - - #[test] - #[should_panic] - fn clevis_real_should_fail_test_initialize() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, Some(Sectors(1024 * 1024 * 1024 / 512))), - test_clevis_initialize, - ); - } - - #[test] - fn clevis_loop_test_initialize() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_clevis_initialize, - ); - } - - #[test] - #[should_panic] - fn clevis_loop_should_fail_test_initialize() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_clevis_initialize, - ); - } - - fn test_clevis_tang_configs(paths: &[&Path]) { - let path = paths[0]; - let pool_name = Name::new("pool_name".to_string()); - - assert!(CryptHandle::initialize( - path, - PoolUuid::new_v4(), - DevUuid::new_v4(), - pool_name.clone(), - &EncryptionInfo::ClevisInfo(( - "tang".to_string(), - json!({"url": env::var("TANG_URL").expect("TANG_URL env var required")}), - )), - None, - ) - .is_err()); - CryptHandle::initialize( - path, - PoolUuid::new_v4(), - DevUuid::new_v4(), - pool_name, - &EncryptionInfo::ClevisInfo(( - "tang".to_string(), - json!({ - "stratis:tang:trust_url": true, - "url": env::var("TANG_URL").expect("TANG_URL env var required"), - }), - )), - None, - ) - .unwrap(); - } - - #[test] - fn clevis_real_test_clevis_tang_configs() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, None), - test_clevis_tang_configs, - ); - } - - #[test] - fn clevis_loop_test_clevis_tang_configs() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_clevis_tang_configs, - ); - } - - fn test_clevis_sss_configs(paths: &[&Path]) { - let path = paths[0]; - let pool_name = Name::new("pool_name".to_string()); - - assert!(CryptHandle::initialize( - path, - PoolUuid::new_v4(), - DevUuid::new_v4(), - pool_name.clone(), - &EncryptionInfo::ClevisInfo(( - "sss".to_string(), - json!({"t": 1, "pins": {"tang": {"url": env::var("TANG_URL").expect("TANG_URL env var required")}, "tpm2": {}}}), - )), - None, - ) - .is_err()); - CryptHandle::initialize( - path, - PoolUuid::new_v4(), - DevUuid::new_v4(), - pool_name, - &EncryptionInfo::ClevisInfo(( - "sss".to_string(), - json!({ - "t": 1, - "stratis:tang:trust_url": true, - "pins": { - "tang": {"url": env::var("TANG_URL").expect("TANG_URL env var required")}, - "tpm2": {} - } - }), - )), - None, - ) - .unwrap(); - } - - #[test] - fn clevis_real_test_clevis_sss_configs() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, None), - test_clevis_sss_configs, - ); - } - - #[test] - fn clevis_loop_test_clevis_sss_configs() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_clevis_sss_configs, - ); - } -} diff --git a/src/engine/strat_engine/crypt/shared.rs b/src/engine/strat_engine/crypt/shared.rs index d22037461e..5fa3488c7e 100644 --- a/src/engine/strat_engine/crypt/shared.rs +++ b/src/engine/strat_engine/crypt/shared.rs @@ -3,7 +3,6 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. use std::{ - fmt::{self, Formatter}, fs::OpenOptions, io::Write, mem::forget, @@ -13,16 +12,11 @@ use std::{ use data_encoding::BASE64URL_NOPAD; use either::Either; -use serde::{ - de::{Error, MapAccess, Visitor}, - ser::SerializeMap, - Deserialize, Deserializer, Serialize, Serializer, -}; -use serde_json::{from_value, to_value, Map, Value}; +use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; use tempfile::TempDir; -use devicemapper::{Bytes, DevId, DmName, DmNameBuf, DmOptions}; +use devicemapper::{Bytes, DevId, DmName, DmOptions}; use libcryptsetup_rs::{ c_uint, consts::{ @@ -31,33 +25,23 @@ use libcryptsetup_rs::{ CryptDebugLevel, CryptLogLevel, CryptStatusInfo, CryptWipePattern, EncryptionFormat, }, }, - register, set_debug_level, set_log_callback, CryptDevice, CryptInit, TokenInput, + register, set_debug_level, set_log_callback, CryptDevice, CryptInit, }; use crate::{ engine::{ strat_engine::{ - backstore::get_devno_from_path, cmd::clevis_decrypt, - crypt::{ - consts::{ - CLEVIS_LUKS_TOKEN_ID, CLEVIS_RECURSION_LIMIT, CLEVIS_TANG_TRUST_URL, - CLEVIS_TOKEN_NAME, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, - LUKS2_SECTOR_SIZE, LUKS2_TOKEN_ID, LUKS2_TOKEN_TYPE, STRATIS_TOKEN_DEVNAME_KEY, - STRATIS_TOKEN_DEV_UUID_KEY, STRATIS_TOKEN_ID, STRATIS_TOKEN_POOLNAME_KEY, - STRATIS_TOKEN_POOL_UUID_KEY, STRATIS_TOKEN_TYPE, TOKEN_KEYSLOTS_KEY, - TOKEN_TYPE_KEY, - }, - handle::{CryptHandle, CryptMetadata}, + crypt::consts::{ + CLEVIS_LUKS_TOKEN_ID, CLEVIS_RECURSION_LIMIT, CLEVIS_TANG_TRUST_URL, + CLEVIS_TOKEN_NAME, DEFAULT_CRYPT_KEYSLOTS_SIZE, DEFAULT_CRYPT_METADATA_SIZE, + LUKS2_SECTOR_SIZE, LUKS2_TOKEN_ID, LUKS2_TOKEN_TYPE, TOKEN_KEYSLOTS_KEY, + TOKEN_TYPE_KEY, }, - dm::{get_dm, DEVICEMAPPER_PATH}, + dm::get_dm, keys, - metadata::StratisIdentifiers, - }, - types::{ - DevUuid, DevicePath, EncryptionInfo, KeyDescription, Name, PoolUuid, SizedKeyMemory, - UnlockMethod, }, + types::{KeyDescription, SizedKeyMemory, UnlockMethod}, }, stratis::{StratisError, StratisResult}, }; @@ -80,185 +64,6 @@ pub fn set_up_crypt_logging() { set_log_callback::<()>(Some(c_logging_callback), None); } -pub struct StratisLuks2Token { - pub devname: DmNameBuf, - pub identifiers: StratisIdentifiers, - pub pool_name: Option, -} - -impl Serialize for StratisLuks2Token { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map_serializer = serializer.serialize_map(None)?; - map_serializer.serialize_entry(TOKEN_TYPE_KEY, STRATIS_TOKEN_TYPE)?; - map_serializer.serialize_entry::<_, [u32; 0]>(TOKEN_KEYSLOTS_KEY, &[])?; - map_serializer.serialize_entry(STRATIS_TOKEN_DEVNAME_KEY, &self.devname.to_string())?; - map_serializer.serialize_entry( - STRATIS_TOKEN_POOL_UUID_KEY, - &self.identifiers.pool_uuid.to_string(), - )?; - map_serializer.serialize_entry( - STRATIS_TOKEN_DEV_UUID_KEY, - &self.identifiers.device_uuid.to_string(), - )?; - if let Some(ref pn) = self.pool_name { - map_serializer.serialize_entry(STRATIS_TOKEN_POOLNAME_KEY, pn)?; - } - map_serializer.end() - } -} - -impl<'de> Deserialize<'de> for StratisLuks2Token { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct StratisTokenVisitor; - - impl<'de> Visitor<'de> for StratisTokenVisitor { - type Value = StratisLuks2Token; - - fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "a Stratis LUKS2 token") - } - - fn visit_map(self, mut map: A) -> Result - where - A: MapAccess<'de>, - { - let mut token_type = None; - let mut token_keyslots = None; - let mut d_name = None; - let mut p_uuid = None; - let mut d_uuid = None; - let mut p_name = None; - - while let Some((k, v)) = map.next_entry::()? { - match k.as_str() { - TOKEN_TYPE_KEY => { - token_type = Some(v); - } - TOKEN_KEYSLOTS_KEY => { - token_keyslots = Some(v); - } - STRATIS_TOKEN_DEVNAME_KEY => { - d_name = Some(v); - } - STRATIS_TOKEN_POOL_UUID_KEY => { - p_uuid = Some(v); - } - STRATIS_TOKEN_DEV_UUID_KEY => { - d_uuid = Some(v); - } - STRATIS_TOKEN_POOLNAME_KEY => { - p_name = Some(v); - } - st => { - return Err(A::Error::custom(format!("Found unrecognized key {st}"))); - } - } - } - - token_type - .ok_or_else(|| A::Error::custom(format!("Missing field {TOKEN_TYPE_KEY}"))) - .and_then(|ty| match ty { - Value::String(s) => { - if s == STRATIS_TOKEN_TYPE { - Ok(()) - } else { - Err(A::Error::custom(format!( - "Incorrect value for {TOKEN_TYPE_KEY}: {s}" - ))) - } - } - _ => Err(A::Error::custom(format!( - "Unrecognized value type for {TOKEN_TYPE_KEY}" - ))), - }) - .and_then(|_| { - let value = token_keyslots.ok_or_else(|| { - A::Error::custom(format!("Missing field {TOKEN_KEYSLOTS_KEY}")) - })?; - match value { - Value::Array(a) => { - if a.is_empty() { - Ok(()) - } else { - Err(A::Error::custom(format!( - "Found non-empty array for {TOKEN_KEYSLOTS_KEY}" - ))) - } - } - _ => Err(A::Error::custom(format!( - "Unrecognized value type for {TOKEN_TYPE_KEY}" - ))), - } - }) - .and_then(|_| { - let value = d_name.ok_or_else(|| { - A::Error::custom(format!("Missing field {STRATIS_TOKEN_DEVNAME_KEY}")) - })?; - match value { - Value::String(s) => DmNameBuf::new(s).map_err(A::Error::custom), - _ => Err(A::Error::custom(format!( - "Unrecognized value type for {STRATIS_TOKEN_DEVNAME_KEY}" - ))), - } - }) - .and_then(|dev_name| { - let value = p_uuid.ok_or_else(|| { - A::Error::custom(format!("Missing field {STRATIS_TOKEN_POOL_UUID_KEY}")) - })?; - match value { - Value::String(s) => PoolUuid::parse_str(&s) - .map(|uuid| (dev_name, uuid)) - .map_err(A::Error::custom), - _ => Err(A::Error::custom(format!( - "Unrecognized value type for {STRATIS_TOKEN_POOL_UUID_KEY}" - ))), - } - }) - .and_then(|(dev_name, pool_uuid)| { - let value = d_uuid.ok_or_else(|| { - A::Error::custom(format!("Missing field {STRATIS_TOKEN_DEV_UUID_KEY}")) - })?; - match value { - Value::String(s) => DevUuid::parse_str(&s) - .map(|uuid| (dev_name, pool_uuid, uuid)) - .map_err(A::Error::custom), - _ => Err(A::Error::custom(format!( - "Unrecognized value type for {STRATIS_TOKEN_DEV_UUID_KEY}" - ))), - } - }) - .and_then(|(devname, pool_uuid, device_uuid)| { - let pool_name = match p_name { - Some(Value::String(s)) => Some(Name::new(s)), - Some(_) => { - return Err(A::Error::custom(format!( - "Unrecognized value type for {STRATIS_TOKEN_POOLNAME_KEY}" - ))) - } - None => None, - }; - Ok(StratisLuks2Token { - devname, - identifiers: StratisIdentifiers { - pool_uuid, - device_uuid, - }, - pool_name, - }) - }) - } - } - - deserializer.deserialize_map(StratisTokenVisitor) - } -} - /// Acquire a crypt device handle or return an error. This serves as a wrapper /// around device_from_physical_path removing the Option type. pub fn acquire_crypt_device(physical_path: &Path) -> StratisResult { @@ -350,121 +155,9 @@ pub fn add_keyring_keyslot( Ok(()) } -/// Set up a libcryptsetup device handle on a device that may or may not be a LUKS2 -/// device. -pub fn setup_crypt_device(physical_path: &Path) -> StratisResult> { - let device_result = device_from_physical_path(physical_path); - match device_result { - Ok(None) => Ok(None), - Ok(Some(mut dev)) => { - if !is_encrypted_stratis_device(&mut dev) { - Ok(None) - } else { - Ok(Some(dev)) - } - } - Err(e) => Err(e), - } -} - -/// Load crypt device metadata. -pub fn load_crypt_metadata( - device: &mut CryptDevice, - physical_path: &Path, -) -> StratisResult> { - let physical = DevicePath::new(physical_path)?; - - let identifiers = identifiers_from_metadata(device)?; - let activation_name = activation_name_from_metadata(device)?; - let pool_name = pool_name_from_metadata(device)?; - let key_description = key_desc_from_metadata(device); - let devno = get_devno_from_path(physical_path)?; - let key_description = match key_description - .as_ref() - .map(|kd| KeyDescription::from_system_key_desc(kd)) - { - Some(Some(Ok(description))) => Some(description), - Some(Some(Err(e))) => { - return Err(StratisError::Msg(format!( - "key description {} found on devnode {} is not a valid Stratis key description: {}", - key_description.expect("key_desc_from_metadata determined to be Some(_) above"), - physical_path.display(), - e, - ))); - } - Some(None) => { - warn!("Key description stored on device {} does not appear to be a Stratis key description; ignoring", physical_path.display()); - None - } - None => None, - }; - let clevis_info = clevis_info_from_metadata(device)?; - - let encryption_info = - if let Some(info) = EncryptionInfo::from_options((key_description, clevis_info)) { - info - } else { - return Err(StratisError::Msg(format!( - "No valid encryption method that can be used to unlock device {} found", - physical_path.display() - ))); - }; - - let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] - .into_iter() - .collect::(); - let activated_path = path.canonicalize().unwrap_or(path); - Ok(Some(CryptMetadata { - physical_path: physical, - identifiers, - encryption_info, - activation_name, - pool_name, - device: devno, - activated_path, - })) -} - -/// Set up a handle to a crypt device using either Clevis or the keyring to activate -/// the device. -pub fn setup_crypt_handle( - device: &mut CryptDevice, - physical_path: &Path, - unlock_method: Option, -) -> StratisResult> { - let metadata = match load_crypt_metadata(device, physical_path)? { - Some(m) => m, - None => return Ok(None), - }; - - if !vec![DEVICEMAPPER_PATH, &metadata.activation_name.to_string()] - .into_iter() - .collect::() - .exists() - { - if let Some(unlock) = unlock_method { - activate( - device, - metadata.encryption_info.key_description(), - unlock, - &metadata.activation_name, - )?; - } - } - - Ok(Some(CryptHandle::new( - metadata.physical_path, - metadata.identifiers.pool_uuid, - metadata.identifiers.device_uuid, - metadata.encryption_info, - metadata.pool_name, - metadata.device, - ))) -} - /// Create a device handle and load the LUKS2 header into memory from /// a physical path. -fn device_from_physical_path(physical_path: &Path) -> StratisResult> { +pub fn device_from_physical_path(physical_path: &Path) -> StratisResult> { let mut device = log_on_failure!( CryptInit::init(physical_path), "Failed to acquire a context for device {}", @@ -725,51 +418,6 @@ fn pin_dispatch(decoded_jwe: &Value, recursion_limit: u64) -> StratisResult<(Str } } -/// Check whether the physical device path corresponds to an encrypted -/// Stratis device. -/// -/// This method works on activated and deactivated encrypted devices. -/// -/// This device will only return true if the device was initialized -/// with encryption by Stratis. This requires that: -/// * the device is a LUKS2 encrypted device. -/// * the device has a valid Stratis LUKS2 token. -fn is_encrypted_stratis_device(device: &mut CryptDevice) -> bool { - fn device_operations(device: &mut CryptDevice) -> StratisResult<()> { - let stratis_token = device.token_handle().json_get(STRATIS_TOKEN_ID).ok(); - let luks_token = device.token_handle().json_get(LUKS2_TOKEN_ID).ok(); - let clevis_token = device.token_handle().json_get(CLEVIS_LUKS_TOKEN_ID).ok(); - if stratis_token.is_none() || (luks_token.is_none() && clevis_token.is_none()) { - return Err(StratisError::Msg( - "Device appears to be missing some of the required Stratis LUKS2 tokens" - .to_string(), - )); - } - if let Some(ref lt) = luks_token { - if !luks2_token_type_is_valid(lt) { - return Err(StratisError::Msg("LUKS2 token is invalid".to_string())); - } - } - if let Some(st) = stratis_token { - if !stratis_token_is_valid(st) { - return Err(StratisError::Msg("Stratis token is invalid".to_string())); - } - } - Ok(()) - } - - device_operations(device) - .map(|_| true) - .map_err(|e| { - debug!( - "Operations querying device to determine if it is a Stratis device \ - failed with an error: {}; reporting as not a Stratis device.", - e - ); - }) - .unwrap_or(false) -} - fn device_is_active(device: Option<&mut CryptDevice>, device_name: &DmName) -> StratisResult<()> { match libcryptsetup_rs::status(device, &device_name.to_string()) { Ok(CryptStatusInfo::Active) => Ok(()), @@ -815,7 +463,7 @@ pub fn activate( unlock_method: UnlockMethod, name: &DmName, ) -> StratisResult<()> { - if let (Some(kd), UnlockMethod::Keyring) = (key_desc, unlock_method) { + if let (Some(kd), UnlockMethod::Keyring | UnlockMethod::Any) = (key_desc, unlock_method) { let key_description_missing = keys::search_key_persistent(kd) .map_err(|_| { StratisError::Msg(format!( @@ -829,6 +477,8 @@ pub fn activate( "Key description {} was not found in the keyring", kd.as_application_str() ); + } + if key_description_missing && unlock_method == UnlockMethod::Keyring { return Err(StratisError::Msg(format!( "The key description \"{}\" is not currently set.", kd.as_application_str(), @@ -838,11 +488,13 @@ pub fn activate( log_on_failure!( device.token_handle().activate_by_token::<()>( Some(&name.to_string()), - Some(if unlock_method == UnlockMethod::Keyring { - LUKS2_TOKEN_ID + if unlock_method == UnlockMethod::Keyring { + Some(LUKS2_TOKEN_ID) + } else if unlock_method == UnlockMethod::Clevis { + Some(CLEVIS_LUKS_TOKEN_ID) } else { - CLEVIS_LUKS_TOKEN_ID - }), + None + }, None, CryptActivate::empty(), ), @@ -1048,28 +700,13 @@ pub fn check_luks2_token(device: &mut CryptDevice) -> StratisResult<()> { /// Validate that the LUKS2 token is present and valid /// /// May not be necessary. See the comment above the invocation. -fn luks2_token_type_is_valid(json: &Value) -> bool { +pub fn luks2_token_type_is_valid(json: &Value) -> bool { json.get(TOKEN_TYPE_KEY) .and_then(|type_val| type_val.as_str()) .map(|type_str| type_str == LUKS2_TOKEN_TYPE) .unwrap_or(false) } -/// Validate that the Stratis token is present and valid -fn stratis_token_is_valid(json: Value) -> bool { - debug!("Stratis LUKS2 token: {}", json); - - let result = from_value::(json); - if let Err(ref e) = result { - debug!( - "LUKS2 token in the Stratis token slot does not appear \ - to be a Stratis token: {}.", - e, - ); - } - result.is_ok() -} - /// Read key from keyring with the given key description. /// /// Returns a safe owned memory segment that will clear itself when dropped. @@ -1090,45 +727,12 @@ pub fn read_key(key_description: &KeyDescription) -> StratisResult StratisResult { - Ok(from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)?.devname) -} - /// Query the Stratis metadata for the key description used to unlock the /// physical device. pub fn key_desc_from_metadata(device: &mut CryptDevice) -> Option { device.token_handle().luks2_keyring_get(LUKS2_TOKEN_ID).ok() } -/// Query the Stratis metadata for the pool name. -pub fn pool_name_from_metadata(device: &mut CryptDevice) -> StratisResult> { - Ok( - from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)? - .pool_name, - ) -} - -/// Replace the old pool name in the Stratis LUKS2 token. -pub fn replace_pool_name(device: &mut CryptDevice, new_name: Name) -> StratisResult<()> { - let mut token = - from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)?; - token.pool_name = Some(new_name); - device.token_handle().json_set(TokenInput::ReplaceToken( - STRATIS_TOKEN_ID, - &to_value(token)?, - ))?; - Ok(()) -} - -/// Query the Stratis metadata for the device identifiers. -fn identifiers_from_metadata(device: &mut CryptDevice) -> StratisResult { - Ok( - from_value::(device.token_handle().json_get(STRATIS_TOKEN_ID)?)? - .identifiers, - ) -} - // Bytes occupied by crypt metadata pub fn crypt_metadata_size() -> Bytes { 2u64 * DEFAULT_CRYPT_METADATA_SIZE + ceiling_sector_size_alignment(DEFAULT_CRYPT_KEYSLOTS_SIZE) diff --git a/src/engine/strat_engine/liminal/identify.rs b/src/engine/strat_engine/liminal/identify.rs index b07291cda1..3bc5f91a66 100644 --- a/src/engine/strat_engine/liminal/identify.rs +++ b/src/engine/strat_engine/liminal/identify.rs @@ -53,7 +53,7 @@ use devicemapper::Device; use crate::engine::{ strat_engine::{ backstore::StratBlockDev, - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, metadata::{static_header, StratisIdentifiers, BDA}, udev::{ block_enumerator, decide_ownership, UdevOwnership, CRYPTO_FS_TYPE, FS_TYPE_KEY, diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index 650ad6a0c6..9b84629ffc 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -18,7 +18,7 @@ use crate::{ engine::{DumpState, Pool, StateDiff}, strat_engine::{ backstore::{find_stratis_devs_by_uuid, StratBlockDev}, - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, dm::{has_leftover_devices, stop_partially_constructed_pool}, liminal::{ device_info::{ diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index 7f79ad46e8..33691bd3af 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -19,7 +19,7 @@ use crate::{ engine::{ strat_engine::{ backstore::{StratBlockDev, UnderlyingDevice}, - crypt::CryptHandle, + crypt::handle::v1::CryptHandle, device::blkdev_size, liminal::device_info::{LStratisDevInfo, LStratisInfo}, metadata::BDA, diff --git a/src/engine/strat_engine/names.rs b/src/engine/strat_engine/names.rs index 18ea42de67..63e79447c5 100644 --- a/src/engine/strat_engine/names.rs +++ b/src/engine/strat_engine/names.rs @@ -70,6 +70,26 @@ pub fn format_crypt_name(dev_uuid: &DevUuid) -> DmNameBuf { DmNameBuf::new(value).expect("FORMAT_VERSION display length < 73") } +/// Get a devicemapper name from the device UUID. +/// +/// Prerequisite: len(format!("{}", FORMAT_VERSION) +/// + len("stratis") 7 +/// + len("private") 7 +/// + len("crypt") 5 +/// + num_dashes 4 +/// + len(dev uuid) 32 +/// < 128 +/// +/// which is equivalent to len(format!("{}", FORMAT_VERSION) < 73 +pub fn format_crypt_backstore_name(pool_uuid: &PoolUuid) -> DmNameBuf { + let value = format!( + "stratis-{}-private-{}-crypt", + FORMAT_VERSION, + uuid_to_string!(pool_uuid) + ); + DmNameBuf::new(value).expect("FORMAT_VERSION display length < 73") +} + #[derive(Clone, Copy)] pub enum FlexRole { MetadataVolume, diff --git a/src/engine/types/mod.rs b/src/engine/types/mod.rs index 597153f0b4..3ea5428053 100644 --- a/src/engine/types/mod.rs +++ b/src/engine/types/mod.rs @@ -134,6 +134,7 @@ impl Display for StratisUuid { pub enum UnlockMethod { Clevis, Keyring, + Any, } impl<'a> TryFrom<&'a str> for UnlockMethod { @@ -143,6 +144,7 @@ impl<'a> TryFrom<&'a str> for UnlockMethod { match s { "keyring" => Ok(UnlockMethod::Keyring), "clevis" => Ok(UnlockMethod::Clevis), + "any" => Ok(UnlockMethod::Any), _ => Err(StratisError::Msg(format!( "{s} is an invalid unlock method" ))), From 451fc4a2d4733cef4a40eb57c06e4edfe56fae62 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 30 Jun 2023 17:37:36 -0400 Subject: [PATCH 04/32] Separate new and legacy versions of Stratis block devices --- src/engine/engine.rs | 8 +- src/engine/sim_engine/blockdev.rs | 10 +- .../strat_engine/backstore/backstore.rs | 8 +- .../strat_engine/backstore/blockdev/mod.rs | 136 ++++++ .../backstore/{blockdev.rs => blockdev/v1.rs} | 296 +++++-------- .../strat_engine/backstore/blockdev/v2.rs | 419 ++++++++++++++++++ .../strat_engine/backstore/blockdevmgr.rs | 2 +- .../strat_engine/backstore/cache_tier.rs | 2 +- .../strat_engine/backstore/data_tier.rs | 2 +- src/engine/strat_engine/backstore/devices.rs | 5 +- src/engine/strat_engine/backstore/mod.rs | 3 +- .../strat_engine/backstore/range_alloc.rs | 1 + src/engine/strat_engine/backstore/shared.rs | 2 +- .../strat_engine/liminal/device_info.rs | 2 +- src/engine/strat_engine/liminal/identify.rs | 6 +- src/engine/strat_engine/liminal/liminal.rs | 5 +- src/engine/strat_engine/liminal/setup.rs | 5 +- src/engine/strat_engine/pool.rs | 5 +- src/engine/strat_engine/shared.rs | 23 +- 19 files changed, 733 insertions(+), 207 deletions(-) create mode 100644 src/engine/strat_engine/backstore/blockdev/mod.rs rename src/engine/strat_engine/backstore/{blockdev.rs => blockdev/v1.rs} (82%) create mode 100644 src/engine/strat_engine/backstore/blockdev/v2.rs diff --git a/src/engine/engine.rs b/src/engine/engine.rs index e988239f42..5002ef741a 100644 --- a/src/engine/engine.rs +++ b/src/engine/engine.rs @@ -25,7 +25,7 @@ use crate::{ MappingCreateAction, MappingDeleteAction, Name, PoolDiff, PoolEncryptionInfo, PoolIdentifier, PoolUuid, RegenAction, RenameAction, ReportType, SetCreateAction, SetDeleteAction, SetUnlockAction, StartAction, StopAction, StoppedPoolsInfo, - StratFilesystemDiff, UdevEngineEvent, UnlockMethod, + StratFilesystemDiff, StratSigblockVersion, UdevEngineEvent, UnlockMethod, }, }, stratis::StratisResult, @@ -118,14 +118,14 @@ pub trait BlockDev: Debug { /// The total size of the device, including space not usable for data. fn size(&self) -> Sectors; - /// Get the status of whether a block device is encrypted or not. - fn is_encrypted(&self) -> bool; - /// Get the newly registered size, if any, of the block device. /// /// If internally the new size is None, the block device size is equal to that /// registered in the BDA. fn new_size(&self) -> Option; + + /// Get metadata version from static header + fn metadata_version(&self) -> StratSigblockVersion; } pub trait Pool: Debug + Send + Sync { diff --git a/src/engine/sim_engine/blockdev.rs b/src/engine/sim_engine/blockdev.rs index bcdbf30352..4b3d474e9c 100644 --- a/src/engine/sim_engine/blockdev.rs +++ b/src/engine/sim_engine/blockdev.rs @@ -12,7 +12,7 @@ use devicemapper::{Bytes, Sectors, IEC}; use crate::engine::{ engine::BlockDev, shared::now_to_timestamp, - types::{DevUuid, EncryptionInfo, KeyDescription}, + types::{DevUuid, EncryptionInfo, KeyDescription, StratSigblockVersion}, }; #[derive(Debug)] @@ -57,13 +57,13 @@ impl BlockDev for SimDev { Bytes::from(IEC::Gi).sectors() } - fn is_encrypted(&self) -> bool { - self.encryption_info.is_some() - } - fn new_size(&self) -> Option { None } + + fn metadata_version(&self) -> StratSigblockVersion { + StratSigblockVersion::V2 + } } impl SimDev { diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index e9cbabe934..4652e4eb03 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -17,8 +17,12 @@ use crate::{ shared::gather_encryption_info, strat_engine::{ backstore::{ - blockdev::StratBlockDev, blockdevmgr::BlockDevMgr, cache_tier::CacheTier, - data_tier::DataTier, devices::UnownedDevices, shared::BlockSizeSummary, + blockdev::{v1::StratBlockDev, InternalBlockDev}, + blockdevmgr::BlockDevMgr, + cache_tier::CacheTier, + data_tier::DataTier, + devices::UnownedDevices, + shared::BlockSizeSummary, }, crypt::{ back_up_luks_header, handle::v1::CryptHandle, interpret_clevis_config, diff --git a/src/engine/strat_engine/backstore/blockdev/mod.rs b/src/engine/strat_engine/backstore/blockdev/mod.rs new file mode 100644 index 0000000000..b90843fa2c --- /dev/null +++ b/src/engine/strat_engine/backstore/blockdev/mod.rs @@ -0,0 +1,136 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{fmt, path::Path}; + +use chrono::{DateTime, Utc}; + +use devicemapper::{Device, Sectors}; + +use crate::{ + engine::{ + strat_engine::{ + backstore::{devices::BlockSizes, range_alloc::PerDevSegments}, + metadata::{BDAExtendedSize, BlockdevSize, MDADataSize, BDA}, + }, + types::{DevUuid, StratSigblockVersion}, + }, + stratis::StratisResult, +}; + +pub mod v1; +pub mod v2; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct StratSectorSizes { + pub base: BlockSizes, + pub crypt: Option, +} + +impl fmt::Display for StratSectorSizes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "base: {}, crypt: {}", + self.base, + &self + .crypt + .map(|sz| sz.to_string()) + .unwrap_or("None".to_string()) + ) + } +} + +pub trait InternalBlockDev { + /// The device's UUID. + fn uuid(&self) -> DevUuid; + + /// Returns the blockdev's Device. For unencrypted devices, this is the physical, + /// unencrypted device. For encrypted devices, this is the logical, unlocked + /// device on top of LUKS2. + /// + /// Practically, this is the device number that should be used when constructing + /// the cap device. + fn device(&self) -> &Device; + + /// Returns the physical path of the block device structure. + fn physical_path(&self) -> &Path; + + /// Block size information + fn blksizes(&self) -> StratSectorSizes; + + /// Return sigblock metadata version for this block device. + fn metadata_version(&self) -> StratSigblockVersion; + + /// The total size of the Stratis block device. + fn total_size(&self) -> BlockdevSize; + + /// The number of Sectors on this device not allocated for any purpose. + /// self.total_allocated_size() - self.metadata_size() >= self.available() + fn available(&self) -> Sectors; + + // ALL SIZE METHODS (except size(), which is in BlockDev impl.) + /// The number of Sectors on this device used by Stratis for metadata + fn metadata_size(&self) -> BDAExtendedSize; + + /// The maximum size of variable length metadata that can be accommodated. + /// self.max_metadata_size() < self.metadata_size() + #[allow(dead_code)] + fn max_metadata_size(&self) -> MDADataSize; + + /// Whether or not the blockdev is in use by upper layers. It is if the + /// sum of the blocks used exceeds the Stratis metadata size. + fn in_use(&self) -> bool; + + /// Find some sector ranges that could be allocated. If more + /// sectors are needed than are available, return partial results. + fn alloc(&mut self, size: Sectors) -> PerDevSegments; + + /// Calculate the new size of the block device specified by physical_path. + /// + /// Returns: + /// * `None` if the size hasn't changed or is equal to the current size recorded + /// in the metadata. + /// * Otherwise, `Some(_)` + fn calc_new_size(&self) -> StratisResult>; + + /// Grow the block device if the underlying physical device has grown in size. + /// Return an error and leave the size as is if the device has shrunk. + /// Do nothing if the device is the same size as recorded in the metadata. + /// + /// This method does not need to block IO to the extended crypt device prior + /// to rollback because of per-pool locking. Growing the device will acquire + /// an exclusive lock on the pool and therefore the thin pool cannot be + /// extended to use the larger or unencrypted block device size until the + /// transaction has been completed successfully. + fn grow(&mut self) -> StratisResult; + + /// Load the pool-level metadata for the given block device. + fn load_state(&self) -> StratisResult, &DateTime)>>; + + /// Save the current metadata state to block device. + fn save_state(&mut self, time: &DateTime, metadata: &[u8]) -> StratisResult<()>; + + /// If a pool is encrypted, tear down the cryptsetup devicemapper devices on the + /// physical device. + fn teardown(&mut self) -> StratisResult<()>; + + /// Remove information that identifies this device as belonging to Stratis + /// + /// If self.is_encrypted() is true, destroy all keyslots and wipe the LUKS2 header. + /// This will render all Stratis and LUKS2 metadata unreadable and unrecoverable + /// from the given device. + /// + /// If self.is_encrypted() is false, wipe the Stratis metadata on the device. + /// This will make the Stratis data and metadata invisible to all standard blkid + /// and stratisd operations. + /// + /// Precondition: if self.is_encrypted() == true, the data on + /// self.devnode.physical_path() has been encrypted with + /// aes-xts-plain64 encryption. + fn disown(&mut self) -> StratisResult<()>; + + /// Consume block device returning BDA. + fn into_bda(self) -> BDA; +} diff --git a/src/engine/strat_engine/backstore/blockdev.rs b/src/engine/strat_engine/backstore/blockdev/v1.rs similarity index 82% rename from src/engine/strat_engine/backstore/blockdev.rs rename to src/engine/strat_engine/backstore/blockdev/v1.rs index 6f69444474..c7a6be205b 100644 --- a/src/engine/strat_engine/backstore/blockdev.rs +++ b/src/engine/strat_engine/backstore/blockdev/v1.rs @@ -6,7 +6,6 @@ use std::{ cmp::Ordering, - fmt, fs::{File, OpenOptions}, io::Seek, path::Path, @@ -23,6 +22,7 @@ use crate::{ engine::{BlockDev, DumpState}, strat_engine::{ backstore::{ + blockdev::{InternalBlockDev, StratSectorSizes}, devices::BlockSizes, range_alloc::{PerDevSegments, RangeAllocator}, }, @@ -79,30 +79,10 @@ impl UnderlyingDevice { } } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub struct StratSectorSizes { - pub base: BlockSizes, - pub crypt: Option, -} - -impl fmt::Display for StratSectorSizes { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "base: {}, crypt: {}", - self.base, - &self - .crypt - .map(|sz| sz.to_string()) - .unwrap_or("None".to_string()) - ) - } -} - #[derive(Debug)] pub struct StratBlockDev { dev: Device, - pub(in super::super) bda: BDA, + bda: BDA, used: RangeAllocator, user_info: Option, hardware_info: Option, @@ -195,26 +175,11 @@ impl StratBlockDev { }) } - /// Returns the blockdev's Device. For unencrypted devices, this is the physical, - /// unencrypted device. For encrypted devices, this is the logical, unlocked - /// device on top of LUKS2. - /// - /// Practically, this is the device number that should be used when constructing - /// the cap device. - pub fn device(&self) -> &Device { - &self.dev - } - /// Returns the LUKS2 device's Device if encrypted pub fn luks_device(&self) -> Option<&Device> { self.underlying_device.crypt_handle().map(|ch| ch.device()) } - /// Returns the physical path of the block device structure. - pub fn physical_path(&self) -> &Path { - self.devnode() - } - /// Returns the path to the unencrypted metadata stored on the block device structure. /// On encrypted devices, this will point to a devicemapper device set up by libcryptsetup. /// On unencrypted devices, this will be the same as the physical device. @@ -222,90 +187,11 @@ impl StratBlockDev { self.underlying_device.metadata_path() } - /// Remove information that identifies this device as belonging to Stratis - /// - /// If self.is_encrypted() is true, destroy all keyslots and wipe the LUKS2 header. - /// This will render all Stratis and LUKS2 metadata unreadable and unrecoverable - /// from the given device. - /// - /// If self.is_encrypted() is false, wipe the Stratis metadata on the device. - /// This will make the Stratis data and metadata invisible to all standard blkid - /// and stratisd operations. - /// - /// Precondition: if self.is_encrypted() == true, the data on - /// self.devnode.physical_path() has been encrypted with - /// aes-xts-plain64 encryption. - pub fn disown(&mut self) -> StratisResult<()> { - if let Some(ref mut handle) = self.underlying_device.crypt_handle_mut() { - handle.wipe()?; - } else { - disown_device( - &mut OpenOptions::new() - .write(true) - .open(self.underlying_device.physical_path())?, - )?; - } - Ok(()) - } - - pub fn save_state(&mut self, time: &DateTime, metadata: &[u8]) -> StratisResult<()> { - let mut f = OpenOptions::new() - .read(true) - .write(true) - .open(self.underlying_device.metadata_path())?; - self.bda.save_state(time, metadata, &mut f)?; - - f.rewind()?; - let header = static_header(&mut f)?.ok_or_else(|| { - StratisError::Msg("Stratis device has no signature buffer".to_string()) - })?; - let bda = BDA::load(header, &mut f)? - .ok_or_else(|| StratisError::Msg("Stratis device has no BDA".to_string()))?; - self.bda = bda; - Ok(()) - } - - pub fn load_state(&self) -> StratisResult, &DateTime)>> { - let mut f = OpenOptions::new() - .read(true) - .open(self.underlying_device.metadata_path())?; - match (self.bda.load_state(&mut f)?, self.bda.last_update_time()) { - (Some(state), Some(time)) => Ok(Some((state, time))), - (None, None) => Ok(None), - _ => Err(StratisError::Msg( - "Stratis metadata written but unknown update time or vice-versa".into(), - )), - } - } - /// The pool's UUID. pub fn pool_uuid(&self) -> PoolUuid { self.bda.pool_uuid() } - /// The device's UUID. - pub fn uuid(&self) -> DevUuid { - self.bda.dev_uuid() - } - - /// Find some sector ranges that could be allocated. If more - /// sectors are needed than are available, return partial results. - pub fn alloc(&mut self, size: Sectors) -> PerDevSegments { - self.used.alloc(size) - } - - // ALL SIZE METHODS (except size(), which is in BlockDev impl.) - /// The number of Sectors on this device used by Stratis for metadata - pub fn metadata_size(&self) -> BDAExtendedSize { - self.bda.extended_size() - } - - /// The number of Sectors on this device not allocated for any purpose. - /// self.total_allocated_size() - self.metadata_size() >= self.available() - pub fn available(&self) -> Sectors { - self.used.available() - } - /// The total size of the Stratis block device. pub fn total_size(&self) -> BlockdevSize { self.bda.dev_size() @@ -317,12 +203,6 @@ impl StratBlockDev { self.bda.max_data_size() } - /// Whether or not the blockdev is in use by upper layers. It is if the - /// sum of the blocks used exceeds the Stratis metadata size. - pub fn in_use(&self) -> bool { - self.used.used() > self.metadata_size().sectors() - } - /// Set the user info on this blockdev. /// The user_info may be None, which unsets user info. /// Returns true if the user info was changed, otherwise false. @@ -357,11 +237,6 @@ impl StratBlockDev { .map(|ch| ch.pool_name()) } - /// Block size information - pub fn blksizes(&self) -> StratSectorSizes { - self.blksizes - } - /// Bind encrypted device using the given clevis configuration. pub fn bind_clevis(&mut self, pin: &str, clevis_info: &Value) -> StratisResult<()> { let crypt_handle = self.underlying_device.crypt_handle_mut().ok_or_else(|| { @@ -413,26 +288,6 @@ impl StratBlockDev { crypt_handle.rebind_clevis() } - /// Calculate the new size of the block device specified by physical_path. - /// - /// Returns: - /// * `None` if the size hasn't changed or is equal to the current size recorded - /// in the metadata. - /// * Otherwise, `Some(_)` - pub fn calc_new_size(&self) -> StratisResult> { - let s = Self::scan_blkdev_size( - self.physical_path(), - self.underlying_device.crypt_handle().is_some(), - )?; - if Some(s) == self.new_size - || (self.new_size.is_none() && s == self.bda.dev_size().sectors()) - { - Ok(None) - } else { - Ok(Some(s)) - } - } - /// Scan the block device specified by physical_path for its size. pub fn scan_blkdev_size(physical_path: &Path, is_encrypted: bool) -> StratisResult { Ok(blkdev_size(&File::open(physical_path)?)?.sectors() @@ -463,16 +318,80 @@ impl StratBlockDev { } } - /// Grow the block device if the underlying physical device has grown in size. - /// Return an error and leave the size as is if the device has shrunk. - /// Do nothing if the device is the same size as recorded in the metadata. - /// - /// This method does not need to block IO to the extended crypt device prior - /// to rollback because of per-pool locking. Growing the device will acquire - /// an exclusive lock on the pool and therefore the thin pool cannot be - /// extended to use the larger or unencrypted block device size until the - /// transaction has been completed successfully. - pub fn grow(&mut self) -> StratisResult { + /// Rename pool in metadata if it is encrypted. + pub fn rename_pool(&mut self, pool_name: Name) -> StratisResult<()> { + match self.underlying_device.crypt_handle_mut() { + Some(handle) => handle.rename_pool_in_metadata(pool_name), + None => Ok(()), + } + } + + #[cfg(test)] + pub fn invariant(&self) { + assert!(self.total_size() == self.used.size()); + } +} + +impl InternalBlockDev for StratBlockDev { + fn uuid(&self) -> DevUuid { + self.bda.dev_uuid() + } + + fn device(&self) -> &Device { + &self.dev + } + + fn physical_path(&self) -> &Path { + self.devnode() + } + + fn blksizes(&self) -> StratSectorSizes { + self.blksizes + } + + fn metadata_version(&self) -> StratSigblockVersion { + self.bda.sigblock_version() + } + + fn total_size(&self) -> BlockdevSize { + self.bda.dev_size() + } + + fn available(&self) -> Sectors { + self.used.available() + } + + fn metadata_size(&self) -> BDAExtendedSize { + self.bda.extended_size() + } + + fn max_metadata_size(&self) -> MDADataSize { + self.bda.max_data_size() + } + + fn in_use(&self) -> bool { + self.used.used() > self.metadata_size().sectors() + } + + fn alloc(&mut self, size: Sectors) -> PerDevSegments { + self.used.alloc(size) + } + + fn calc_new_size(&self) -> StratisResult> { + let s = Self::scan_blkdev_size( + self.physical_path(), + self.underlying_device.crypt_handle().is_some(), + )?; + if Some(s) == self.new_size + || (self.new_size.is_none() && s == self.bda.dev_size().sectors()) + { + Ok(None) + } else { + Ok(Some(s)) + } + } + + fn grow(&mut self) -> StratisResult { /// Precondition: size > h.blkdev_size fn needs_rollback(bd: &mut StratBlockDev, size: BlockdevSize) -> StratisResult<()> { let mut f = OpenOptions::new() @@ -544,22 +463,37 @@ impl StratBlockDev { } } - /// Rename pool in metadata if it is encrypted. - pub fn rename_pool(&mut self, pool_name: Name) -> StratisResult<()> { - match self.underlying_device.crypt_handle_mut() { - Some(handle) => handle.rename_pool_in_metadata(pool_name), - None => Ok(()), + fn load_state(&self) -> StratisResult, &DateTime)>> { + let mut f = OpenOptions::new() + .read(true) + .open(self.underlying_device.metadata_path())?; + match (self.bda.load_state(&mut f)?, self.bda.last_update_time()) { + (Some(state), Some(time)) => Ok(Some((state, time))), + (None, None) => Ok(None), + _ => Err(StratisError::Msg( + "Stratis metadata written but unknown update time or vice-versa".into(), + )), } } - #[cfg(test)] - pub fn invariant(&self) { - assert!(self.total_size() == self.used.size()); + fn save_state(&mut self, time: &DateTime, metadata: &[u8]) -> StratisResult<()> { + let mut f = OpenOptions::new() + .read(true) + .write(true) + .open(self.underlying_device.metadata_path())?; + self.bda.save_state(time, metadata, &mut f)?; + + f.rewind()?; + let header = static_header(&mut f)?.ok_or_else(|| { + StratisError::Msg("Stratis device has no signature buffer".to_string()) + })?; + let bda = BDA::load(header, &mut f)? + .ok_or_else(|| StratisError::Msg("Stratis device has no BDA".to_string()))?; + self.bda = bda; + Ok(()) } - /// If a pool is encrypted, tear down the cryptsetup devicemapper devices on the - /// physical device. - pub fn teardown(&mut self) -> StratisResult<()> { + fn teardown(&mut self) -> StratisResult<()> { if let Some(ch) = self.underlying_device.crypt_handle() { debug!( "Deactivating unlocked encrypted device with UUID {}", @@ -571,9 +505,21 @@ impl StratBlockDev { } } - /// Get metadata version from static header - pub fn metadata_version(&self) -> StratSigblockVersion { - self.bda.sigblock_version() + fn disown(&mut self) -> StratisResult<()> { + if let Some(ref mut handle) = self.underlying_device.crypt_handle_mut() { + handle.wipe()?; + } else { + disown_device( + &mut OpenOptions::new() + .write(true) + .open(self.underlying_device.physical_path())?, + )?; + } + Ok(()) + } + + fn into_bda(self) -> BDA { + self.bda } } @@ -634,13 +580,13 @@ impl BlockDev for StratBlockDev { self.total_size().sectors() } - fn is_encrypted(&self) -> bool { - self.encryption_info().is_some() - } - fn new_size(&self) -> Option { self.new_size } + + fn metadata_version(&self) -> StratSigblockVersion { + self.bda.sigblock_version() + } } impl Recordable for StratBlockDev { diff --git a/src/engine/strat_engine/backstore/blockdev/v2.rs b/src/engine/strat_engine/backstore/blockdev/v2.rs new file mode 100644 index 0000000000..5b5a3451e3 --- /dev/null +++ b/src/engine/strat_engine/backstore/blockdev/v2.rs @@ -0,0 +1,419 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Code to handle a single block device. + +use std::{ + cmp::Ordering, + fs::{File, OpenOptions}, + io::Seek, + path::Path, +}; + +use chrono::{DateTime, Utc}; +use serde_json::Value; + +use devicemapper::{Device, Sectors}; + +use crate::{ + engine::{ + engine::{BlockDev, DumpState}, + strat_engine::{ + backstore::{ + blockdev::{InternalBlockDev, StratSectorSizes}, + devices::BlockSizes, + range_alloc::{PerDevSegments, RangeAllocator}, + }, + device::blkdev_size, + metadata::{ + disown_device, static_header, BDAExtendedSize, BlockdevSize, MDADataSize, + MetadataLocation, StaticHeader, BDA, + }, + serde_structs::{BaseBlockDevSave, Recordable}, + types::BDAResult, + }, + types::{ + Compare, DevUuid, DevicePath, Diff, PoolUuid, StateDiff, StratBlockDevDiff, + StratSigblockVersion, + }, + }, + stratis::{StratisError, StratisResult}, +}; + +#[derive(Debug)] +pub struct StratBlockDev { + dev: Device, + bda: BDA, + used: RangeAllocator, + user_info: Option, + hardware_info: Option, + devnode: DevicePath, + new_size: Option, + blksizes: StratSectorSizes, +} + +impl StratBlockDev { + /// Make a new BlockDev from the parameters. + /// Allocate space for the Stratis metadata on the device. + /// - dev: the device, identified by number + /// - devnode: for encrypted devices, the logical and physical + /// paths; for unencrypted devices, the physical path + /// - bda: the device's BDA + /// - other_segments: segments allocated outside Stratis metadata region + /// - user_info: user settable identifying information + /// - hardware_info: identifying information in the hardware + /// - key_description: optional argument enabling encryption using + /// the specified key in the kernel keyring + /// Returns an error if it is impossible to allocate all segments on the + /// device. + /// NOTE: It is possible that the actual device size is greater than + /// the recorded device size. In that case, the additional space available + /// on the device is simply invisible to the blockdev. Consequently, it + /// is invisible to the engine, and is not part of the total size value + /// reported on the D-Bus. + /// + /// Precondition: segments in other_segments do not overlap with Stratis + /// metadata region. + pub fn new( + dev: Device, + bda: BDA, + other_segments: &[(Sectors, Sectors)], + user_info: Option, + hardware_info: Option, + devnode: DevicePath, + ) -> BDAResult { + let mut segments = vec![(Sectors(0), bda.extended_size().sectors())]; + segments.extend(other_segments); + + let allocator = match RangeAllocator::new(bda.dev_size(), &segments) { + Ok(a) => a, + Err(e) => return Err((e, bda)), + }; + + let base_blksizes = match OpenOptions::new() + .read(true) + .open(&*devnode) + .map_err(StratisError::from) + .and_then(|f| BlockSizes::read(&f)) + { + Ok(blksizes) => blksizes, + Err(e) => return Err((e, bda)), + }; + + let blksizes = StratSectorSizes { + base: base_blksizes, + crypt: None, + }; + + Ok(StratBlockDev { + dev, + bda, + used: allocator, + user_info, + hardware_info, + devnode, + new_size: None, + blksizes, + }) + } + + /// Returns the blockdev's Device. For unencrypted devices, this is the physical, + /// unencrypted device. For encrypted devices, this is the logical, unlocked + /// device on top of LUKS2. + /// + /// Practically, this is the device number that should be used when constructing + /// the cap device. + pub fn device(&self) -> &Device { + &self.dev + } + + pub fn save_state(&mut self, time: &DateTime, metadata: &[u8]) -> StratisResult<()> { + let mut f = OpenOptions::new().write(true).open(self.devnode())?; + self.bda.save_state(time, metadata, &mut f) + } + + /// The pool's UUID. + pub fn pool_uuid(&self) -> PoolUuid { + self.bda.pool_uuid() + } + + /// The device's UUID. + pub fn uuid(&self) -> DevUuid { + self.bda.dev_uuid() + } + + /// Set the user info on this blockdev. + /// The user_info may be None, which unsets user info. + /// Returns true if the user info was changed, otherwise false. + pub fn set_user_info(&mut self, user_info: Option<&str>) -> bool { + set_blockdev_user_info!(self; user_info) + } + + /// Get the physical path for a block device. + pub fn devnode(&self) -> &Path { + &self.devnode + } + + /// Scan the block device specified by physical_path for its size. + pub fn scan_blkdev_size(physical_path: &Path) -> StratisResult { + Ok(blkdev_size(&File::open(physical_path)?)?.sectors()) + } + + /// Set the newly detected size of a block device. + pub fn set_new_size(&mut self, new_size: Sectors) { + match self.bda.dev_size().cmp(&BlockdevSize::new(new_size)) { + Ordering::Greater => { + warn!( + "The given device with path: {}, UUID; {} appears to have shrunk; you may experience data loss", + self.devnode().display(), + self.bda.dev_uuid(), + ); + self.new_size = Some(new_size); + } + Ordering::Less => { + self.new_size = Some(new_size); + } + Ordering::Equal => { + self.new_size = None; + } + } + } + + #[cfg(test)] + pub fn invariant(&self) { + assert!(self.total_size() == self.used.size()); + } +} + +impl InternalBlockDev for StratBlockDev { + fn uuid(&self) -> DevUuid { + self.bda.dev_uuid() + } + + fn device(&self) -> &Device { + &self.dev + } + + fn physical_path(&self) -> &Path { + &self.devnode + } + + fn blksizes(&self) -> StratSectorSizes { + self.blksizes + } + + fn metadata_version(&self) -> StratSigblockVersion { + self.bda.sigblock_version() + } + + fn total_size(&self) -> BlockdevSize { + self.bda.dev_size() + } + + fn available(&self) -> Sectors { + self.used.available() + } + + fn metadata_size(&self) -> BDAExtendedSize { + self.bda.extended_size() + } + + fn max_metadata_size(&self) -> MDADataSize { + self.bda.max_data_size() + } + + fn in_use(&self) -> bool { + self.used.used() > self.metadata_size().sectors() + } + + fn alloc(&mut self, size: Sectors) -> PerDevSegments { + self.used.alloc(size) + } + + fn calc_new_size(&self) -> StratisResult> { + let s = Self::scan_blkdev_size(self.devnode())?; + if Some(s) == self.new_size + || (self.new_size.is_none() && s == self.bda.dev_size().sectors()) + { + Ok(None) + } else { + Ok(Some(s)) + } + } + + fn grow(&mut self) -> StratisResult { + let size = BlockdevSize::new(Self::scan_blkdev_size(self.devnode())?); + let metadata_size = self.bda.dev_size(); + match size.cmp(&metadata_size) { + Ordering::Less => Err(StratisError::Msg( + "The underlying device appears to have shrunk; you may experience data loss" + .to_string(), + )), + Ordering::Equal => Ok(false), + Ordering::Greater => { + let mut f = OpenOptions::new() + .write(true) + .read(true) + .open(self.devnode())?; + let mut h = static_header(&mut f)?.ok_or_else(|| { + StratisError::Msg(format!( + "No static header found on device {}", + self.devnode().display() + )) + })?; + + h.blkdev_size = size; + let h = StaticHeader::write_header(&mut f, h, MetadataLocation::Both)?; + + self.bda.header = h; + self.used.increase_size(size.sectors()); + + Ok(true) + } + } + } + + fn load_state(&self) -> StratisResult, &DateTime)>> { + let mut f = OpenOptions::new().read(true).open(&*self.devnode)?; + match (self.bda.load_state(&mut f)?, self.bda.last_update_time()) { + (Some(state), Some(time)) => Ok(Some((state, time))), + (None, None) => Ok(None), + _ => Err(StratisError::Msg( + "Stratis metadata written but unknown update time or vice-versa".into(), + )), + } + } + + fn save_state(&mut self, time: &DateTime, metadata: &[u8]) -> StratisResult<()> { + let mut f = OpenOptions::new() + .read(true) + .write(true) + .open(&*self.devnode)?; + self.bda.save_state(time, metadata, &mut f)?; + + f.rewind()?; + let header = static_header(&mut f)?.ok_or_else(|| { + StratisError::Msg("Stratis device has no signature buffer".to_string()) + })?; + let bda = BDA::load(header, &mut f)? + .ok_or_else(|| StratisError::Msg("Stratis device has no BDA".to_string()))?; + self.bda = bda; + Ok(()) + } + + fn teardown(&mut self) -> StratisResult<()> { + Ok(()) + } + + fn disown(&mut self) -> StratisResult<()> { + disown_device(&mut OpenOptions::new().write(true).open(self.devnode())?)?; + Ok(()) + } + + fn into_bda(self) -> BDA { + self.bda + } +} + +impl<'a> Into for &'a StratBlockDev { + fn into(self) -> Value { + let mut json = json!({ + "path": self.devnode(), + "uuid": self.bda.dev_uuid().to_string(), + }); + let map = json.as_object_mut().expect("just created above"); + map.insert("size".to_string(), Value::from(self.size().to_string())); + if let Some(new_size) = self.new_size { + map.insert("new_size".to_string(), Value::from(new_size.to_string())); + } + map.insert( + "blksizes".to_string(), + Value::from(self.blksizes.to_string()), + ); + map.insert("in_use".to_string(), Value::from(self.in_use())); + json + } +} + +impl BlockDev for StratBlockDev { + fn devnode(&self) -> &Path { + self.devnode() + } + + fn metadata_path(&self) -> &Path { + self.devnode() + } + + fn user_info(&self) -> Option<&str> { + self.user_info.as_deref() + } + + fn hardware_info(&self) -> Option<&str> { + self.hardware_info.as_deref() + } + + fn initialization_time(&self) -> DateTime { + self.bda.initialization_time() + } + + fn size(&self) -> Sectors { + self.total_size().sectors() + } + + fn new_size(&self) -> Option { + self.new_size + } + + fn metadata_version(&self) -> StratSigblockVersion { + self.bda.sigblock_version() + } +} + +impl Recordable for StratBlockDev { + fn record(&self) -> BaseBlockDevSave { + BaseBlockDevSave { + uuid: self.uuid(), + user_info: self.user_info.clone(), + hardware_info: self.hardware_info.clone(), + } + } +} + +pub struct StratBlockDevState { + new_size: Option, +} + +impl StateDiff for StratBlockDevState { + type Diff = StratBlockDevDiff; + + fn diff(&self, new_state: &Self) -> Self::Diff { + StratBlockDevDiff { + size: self.new_size.compare(&new_state.new_size), + } + } + + fn unchanged(&self) -> Self::Diff { + StratBlockDevDiff { + size: Diff::Unchanged(self.new_size), + } + } +} + +impl<'a> DumpState<'a> for StratBlockDev { + type State = StratBlockDevState; + type DumpInput = Sectors; + + fn cached(&self) -> Self::State { + StratBlockDevState { + new_size: self.new_size, + } + } + + fn dump(&mut self, input: Self::DumpInput) -> Self::State { + self.set_new_size(input); + StratBlockDevState { + new_size: self.new_size, + } + } +} diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index f145df442c..ee4232b58f 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -18,7 +18,7 @@ use crate::{ shared::gather_encryption_info, strat_engine::{ backstore::{ - blockdev::StratBlockDev, + blockdev::{v1::StratBlockDev, InternalBlockDev}, devices::{initialize_devices, wipe_blockdevs, UnownedDevices}, shared::{BlkDevSegment, Segment}, }, diff --git a/src/engine/strat_engine/backstore/cache_tier.rs b/src/engine/strat_engine/backstore/cache_tier.rs index 5439be1288..9742ccd5dc 100644 --- a/src/engine/strat_engine/backstore/cache_tier.rs +++ b/src/engine/strat_engine/backstore/cache_tier.rs @@ -13,7 +13,7 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::StratBlockDev, + blockdev::{v1::StratBlockDev, InternalBlockDev}, blockdevmgr::BlockDevMgr, devices::UnownedDevices, shared::{metadata_to_segment, AllocatedAbove, BlkDevSegment, BlockDevPartition}, diff --git a/src/engine/strat_engine/backstore/data_tier.rs b/src/engine/strat_engine/backstore/data_tier.rs index 532cd7f621..27aa811c81 100644 --- a/src/engine/strat_engine/backstore/data_tier.rs +++ b/src/engine/strat_engine/backstore/data_tier.rs @@ -13,7 +13,7 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::StratBlockDev, + blockdev::{v1::StratBlockDev, InternalBlockDev}, blockdevmgr::BlockDevMgr, devices::UnownedDevices, shared::{metadata_to_segment, AllocatedAbove, BlkDevSegment, BlockDevPartition}, diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index 3f7e1c9ef9..828e1fbbec 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -23,7 +23,10 @@ use libblkid_rs::{BlkidCache, BlkidProbe}; use crate::{ engine::{ strat_engine::{ - backstore::blockdev::{StratBlockDev, UnderlyingDevice}, + backstore::blockdev::{ + v1::{StratBlockDev, UnderlyingDevice}, + InternalBlockDev, + }, crypt::handle::v1::CryptHandle, device::{blkdev_logical_sector_size, blkdev_physical_sector_size, blkdev_size}, metadata::{ diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index 54198b4e0f..f9bd29298a 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -4,7 +4,7 @@ #[allow(clippy::module_inception)] mod backstore; -mod blockdev; +pub mod blockdev; mod blockdevmgr; mod cache_tier; mod data_tier; @@ -16,6 +16,5 @@ mod shared; pub use self::devices::initialize_devices; pub use self::{ backstore::Backstore, - blockdev::{StratBlockDev, UnderlyingDevice}, devices::{find_stratis_devs_by_uuid, get_devno_from_path, ProcessedPathInfos, UnownedDevices}, }; diff --git a/src/engine/strat_engine/backstore/range_alloc.rs b/src/engine/strat_engine/backstore/range_alloc.rs index e00d045b6c..906b2e814f 100644 --- a/src/engine/strat_engine/backstore/range_alloc.rs +++ b/src/engine/strat_engine/backstore/range_alloc.rs @@ -44,6 +44,7 @@ impl PerDevSegments { } /// The number of distinct ranges + #[cfg(test)] pub fn len(&self) -> usize { self.used.len() } diff --git a/src/engine/strat_engine/backstore/shared.rs b/src/engine/strat_engine/backstore/shared.rs index 60b2ad3125..dcd54d4dcc 100644 --- a/src/engine/strat_engine/backstore/shared.rs +++ b/src/engine/strat_engine/backstore/shared.rs @@ -12,7 +12,7 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::{StratBlockDev, StratSectorSizes}, + blockdev::{v1::StratBlockDev, InternalBlockDev, StratSectorSizes}, devices::BlockSizes, }, serde_structs::{BaseDevSave, Recordable}, diff --git a/src/engine/strat_engine/liminal/device_info.rs b/src/engine/strat_engine/liminal/device_info.rs index 9940a04ce4..f64e2abb99 100644 --- a/src/engine/strat_engine/liminal/device_info.rs +++ b/src/engine/strat_engine/liminal/device_info.rs @@ -18,7 +18,7 @@ use crate::{ engine::{ shared::{gather_encryption_info, gather_pool_name}, strat_engine::{ - backstore::StratBlockDev, + backstore::blockdev::{v1::StratBlockDev, InternalBlockDev}, liminal::{ identify::{DeviceInfo, LuksInfo, StratisDevInfo, StratisInfo}, setup::get_name, diff --git a/src/engine/strat_engine/liminal/identify.rs b/src/engine/strat_engine/liminal/identify.rs index 3bc5f91a66..d48a715d92 100644 --- a/src/engine/strat_engine/liminal/identify.rs +++ b/src/engine/strat_engine/liminal/identify.rs @@ -52,7 +52,7 @@ use devicemapper::Device; use crate::engine::{ strat_engine::{ - backstore::StratBlockDev, + backstore::blockdev::{v1::StratBlockDev, InternalBlockDev}, crypt::handle::v1::CryptHandle, metadata::{static_header, StratisIdentifiers, BDA}, udev::{ @@ -197,7 +197,7 @@ impl From for Vec { device_number: *bd.device(), devnode: bd.metadata_path().to_owned(), }, - bda: bd.bda, + bda: bd.into_bda(), })); } } @@ -207,7 +207,7 @@ impl From for Vec { device_number: *bd.device(), devnode: bd.physical_path().to_owned(), }, - bda: bd.bda, + bda: bd.into_bda(), })), (_, _, _) => unreachable!("If bd.is_encrypted(), all are Some(_)"), } diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index 9b84629ffc..1f4079650c 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -17,7 +17,10 @@ use crate::{ engine::{ engine::{DumpState, Pool, StateDiff}, strat_engine::{ - backstore::{find_stratis_devs_by_uuid, StratBlockDev}, + backstore::{ + blockdev::{v1::StratBlockDev, InternalBlockDev}, + find_stratis_devs_by_uuid, + }, crypt::handle::v1::CryptHandle, dm::{has_leftover_devices, stop_partially_constructed_pool}, liminal::{ diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index 33691bd3af..143f90a815 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -18,7 +18,10 @@ use devicemapper::Sectors; use crate::{ engine::{ strat_engine::{ - backstore::{StratBlockDev, UnderlyingDevice}, + backstore::blockdev::{ + v1::{StratBlockDev, UnderlyingDevice}, + InternalBlockDev, + }, crypt::handle::v1::CryptHandle, device::blkdev_size, liminal::device_info::{LStratisDevInfo, LStratisInfo}, diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index 8840d0ea5b..275a32f20f 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -18,7 +18,10 @@ use crate::{ validate_name, validate_paths, }, strat_engine::{ - backstore::{Backstore, ProcessedPathInfos, StratBlockDev, UnownedDevices}, + backstore::{ + blockdev::{v1::StratBlockDev, InternalBlockDev}, + Backstore, ProcessedPathInfos, UnownedDevices, + }, liminal::DeviceSet, metadata::{MDADataSize, BDA}, serde_structs::{FlexDevsSave, PoolSave, Recordable}, diff --git a/src/engine/strat_engine/shared.rs b/src/engine/strat_engine/shared.rs index 403f5e5500..8a11e7e240 100644 --- a/src/engine/strat_engine/shared.rs +++ b/src/engine/strat_engine/shared.rs @@ -5,24 +5,33 @@ use std::collections::HashMap; use crate::engine::{ - strat_engine::{backstore::StratBlockDev, metadata::BDA}, + strat_engine::{backstore::blockdev::InternalBlockDev, metadata::BDA}, types::DevUuid, }; /// Convert a collection of blockdevs to BDAs. -pub fn bds_to_bdas(bds: Vec) -> HashMap { +pub fn bds_to_bdas(bds: Vec) -> HashMap +where + B: InternalBlockDev, +{ bds.into_iter() - .map(|bd| (bd.bda.dev_uuid(), bd.bda)) + .map(|bd| { + let bda = bd.into_bda(); + (bda.dev_uuid(), bda) + }) .collect() } /// Convert datadevs and cachedevs to BDAs on error with the option of adding /// one additional BDA . -pub fn tiers_to_bdas( - datadevs: Vec, - cachedevs: Vec, +pub fn tiers_to_bdas( + datadevs: Vec, + cachedevs: Vec, bda: Option, -) -> HashMap { +) -> HashMap +where + B: InternalBlockDev, +{ bds_to_bdas(datadevs) .into_iter() .chain(bds_to_bdas(cachedevs)) From 42e476baa1f19733e24a444452e95e8f8ff4da19 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 30 Jun 2023 17:49:15 -0400 Subject: [PATCH 05/32] Abstract BlockDevMgr across both block devices --- .../strat_engine/backstore/backstore.rs | 4 +- .../strat_engine/backstore/blockdev/mod.rs | 1 - .../strat_engine/backstore/blockdevmgr.rs | 750 +++++++++----- .../strat_engine/backstore/cache_tier.rs | 8 +- .../strat_engine/backstore/data_tier.rs | 8 +- src/engine/strat_engine/backstore/devices.rs | 963 ++++++++++++------ src/engine/strat_engine/backstore/mod.rs | 2 +- src/engine/strat_engine/liminal/identify.rs | 6 +- 8 files changed, 1155 insertions(+), 587 deletions(-) diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index 4652e4eb03..977ecf9922 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -234,7 +234,7 @@ impl Backstore { mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, ) -> StratisResult { - let data_tier = DataTier::new(BlockDevMgr::initialize( + let data_tier = DataTier::new(BlockDevMgr::::initialize( pool_name, pool_uuid, devices, @@ -275,7 +275,7 @@ impl Backstore { // If it is desired to change a cache dev to a data dev, it // should be removed and then re-added in order to ensure // that the MDA region is set to the correct size. - let bdm = BlockDevMgr::initialize( + let bdm = BlockDevMgr::::initialize( pool_name, pool_uuid, devices, diff --git a/src/engine/strat_engine/backstore/blockdev/mod.rs b/src/engine/strat_engine/backstore/blockdev/mod.rs index b90843fa2c..6d9c736acc 100644 --- a/src/engine/strat_engine/backstore/blockdev/mod.rs +++ b/src/engine/strat_engine/backstore/blockdev/mod.rs @@ -76,7 +76,6 @@ pub trait InternalBlockDev { /// The maximum size of variable length metadata that can be accommodated. /// self.max_metadata_size() < self.metadata_size() - #[allow(dead_code)] fn max_metadata_size(&self) -> MDADataSize; /// Whether or not the blockdev is in use by upper layers. It is if the diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index ee4232b58f..0cb513b17c 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -4,6 +4,8 @@ // Code to handle a collection of block devices. +#![allow(dead_code)] + use std::collections::HashMap; #[cfg(test)] use std::collections::HashSet; @@ -18,8 +20,10 @@ use crate::{ shared::gather_encryption_info, strat_engine::{ backstore::{ - blockdev::{v1::StratBlockDev, InternalBlockDev}, - devices::{initialize_devices, wipe_blockdevs, UnownedDevices}, + blockdev::{v1, v2, InternalBlockDev}, + devices::{ + initialize_devices, initialize_devices_legacy, wipe_blockdevs, UnownedDevices, + }, shared::{BlkDevSegment, Segment}, }, crypt::handle::v1::CryptHandle, @@ -67,26 +71,15 @@ impl TimeStamp { } #[derive(Debug)] -pub struct BlockDevMgr { +pub struct BlockDevMgr { /// All the block devices that belong to this block dev manager. - block_devs: Vec, + block_devs: Vec, /// The most recent time that variable length metadata was saved to the /// devices managed by this block dev manager. last_update_time: TimeStamp, } -impl BlockDevMgr { - /// Make a struct that represents an existing BlockDevMgr. - pub fn new( - block_devs: Vec, - last_update_time: Option>, - ) -> BlockDevMgr { - BlockDevMgr { - block_devs, - last_update_time: last_update_time.into(), - } - } - +impl BlockDevMgr { /// Initialize a new StratBlockDevMgr with specified pool and devices. pub fn initialize( pool_name: Name, @@ -95,9 +88,9 @@ impl BlockDevMgr { mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, sector_size: Option, - ) -> StratisResult { + ) -> StratisResult> { Ok(BlockDevMgr::new( - initialize_devices( + initialize_devices_legacy( devices, pool_name, pool_uuid, @@ -109,24 +102,6 @@ impl BlockDevMgr { )) } - /// Convert the BlockDevMgr into a collection of BDAs. - pub fn into_bdas(self) -> HashMap { - bds_to_bdas(self.block_devs) - } - - /// Drain the BlockDevMgr block devices into a collection of block devices. - pub fn drain_bds(&mut self) -> Vec { - self.block_devs.drain(..).collect::>() - } - - /// Get a hashmap that maps UUIDs to Devices. - pub fn uuid_to_devno(&self) -> HashMap { - self.block_devs - .iter() - .map(|bd| (bd.uuid(), *bd.device())) - .collect() - } - /// Add paths to self. /// Return the uuids of all blockdevs corresponding to paths that were /// added. @@ -166,7 +141,7 @@ impl BlockDevMgr { // variable length metadata requires more than the minimum allocated, // then the necessary amount must be provided or the data can not be // saved. - let bds = initialize_devices( + let bds = initialize_devices_legacy( devices, pool_name, pool_uuid, @@ -179,10 +154,157 @@ impl BlockDevMgr { Ok(bdev_uuids) } + /// Get the encryption information for a whole pool. + pub fn encryption_info(&self) -> Option { + gather_encryption_info( + self.block_devs.len(), + self.block_devs.iter().map(|bd| bd.encryption_info()), + ) + .expect("Cannot create a pool out of both encrypted and unencrypted devices") + } + + pub fn is_encrypted(&self) -> bool { + self.encryption_info().is_some() + } + + #[cfg(test)] + fn invariant(&self) { + let pool_uuids = self + .block_devs + .iter() + .map(|bd| bd.pool_uuid()) + .collect::>(); + assert!(pool_uuids.len() == 1); + + let encryption_infos = self + .block_devs + .iter() + .filter_map(|bd| bd.encryption_info()) + .collect::>(); + if encryption_infos.is_empty() { + assert_eq!(self.encryption_info(), None); + } else { + assert_eq!(encryption_infos.len(), self.block_devs.len()); + + let info_set = encryption_infos.iter().collect::>(); + assert!(info_set.len() == 1); + } + + for bd in self.block_devs.iter() { + bd.invariant(); + } + } +} + +impl BlockDevMgr { + /// Initialize a new StratBlockDevMgr with specified pool and devices. + pub fn initialize( + pool_uuid: PoolUuid, + devices: UnownedDevices, + mda_data_size: MDADataSize, + ) -> StratisResult> { + Ok(BlockDevMgr::new( + initialize_devices(devices, pool_uuid, mda_data_size)?, + None, + )) + } + + /// Add paths to self. + /// Return the uuids of all blockdevs corresponding to paths that were + /// added. + pub fn add( + &mut self, + pool_uuid: PoolUuid, + devices: UnownedDevices, + ) -> StratisResult> { + let this_pool_uuid = self.block_devs.first().map(|bd| bd.pool_uuid()); + if this_pool_uuid.is_some() && this_pool_uuid != Some(pool_uuid) { + return Err(StratisError::Msg( + format!("block devices being managed have pool UUID {} but new devices are to be added with pool UUID {}", + this_pool_uuid.expect("guarded by if-expression"), + pool_uuid) + )); + } + + // FIXME: This is a bug. If new devices are added to a pool, and the + // variable length metadata requires more than the minimum allocated, + // then the necessary amount must be provided or the data can not be + // saved. + let bds = initialize_devices(devices, pool_uuid, MDADataSize::default())?; + let bdev_uuids = bds.iter().map(|bd| bd.uuid()).collect(); + self.block_devs.extend(bds); + Ok(bdev_uuids) + } + + #[cfg(test)] + fn invariant(&self) { + let pool_uuids = self + .block_devs + .iter() + .map(|bd| bd.pool_uuid()) + .collect::>(); + assert!(pool_uuids.len() == 1); + + for bd in self.block_devs.iter() { + bd.invariant(); + } + } +} + +impl BlockDevMgr +where + B: InternalBlockDev, +{ + /// Make a struct that represents an existing BlockDevMgr. + pub fn new(block_devs: Vec, last_update_time: Option>) -> BlockDevMgr { + BlockDevMgr { + block_devs, + last_update_time: last_update_time.into(), + } + } + + /// Convert the BlockDevMgr into a collection of BDAs. + pub fn into_bdas(self) -> HashMap { + bds_to_bdas(self.block_devs) + } + + /// Get a hashmap that maps UUIDs to Devices. + pub fn uuid_to_devno(&self) -> HashMap { + self.block_devs + .iter() + .map(|bd| (bd.uuid(), *bd.device())) + .collect() + } + pub fn destroy_all(&mut self) -> StratisResult<()> { wipe_blockdevs(&mut self.block_devs) } + /// Drain the BlockDevMgr block devices into a collection of block devices. + pub fn drain_bds(&mut self) -> Vec { + self.block_devs.drain(..).collect::>() + } + + /// Get references to managed blockdevs. + pub fn blockdevs(&self) -> Vec<(DevUuid, &B)> { + self.block_devs.iter().map(|bd| (bd.uuid(), bd)).collect() + } + + pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut B)> { + self.block_devs + .iter_mut() + .map(|bd| (bd.uuid(), bd)) + .collect() + } + + pub fn get_blockdev_by_uuid(&self, uuid: DevUuid) -> Option<&B> { + self.block_devs.iter().find(|bd| bd.uuid() == uuid) + } + + pub fn get_mut_blockdev_by_uuid(&mut self, uuid: DevUuid) -> Option<&mut B> { + self.block_devs.iter_mut().find(|bd| bd.uuid() == uuid) + } + /// Remove the specified block devs and erase their metadata. /// /// Precondition: It is the responsibility of the caller to ensure that @@ -324,26 +446,6 @@ impl BlockDevMgr { }) } - /// Get references to managed blockdevs. - pub fn blockdevs(&self) -> Vec<(DevUuid, &StratBlockDev)> { - self.block_devs.iter().map(|bd| (bd.uuid(), bd)).collect() - } - - pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut StratBlockDev)> { - self.block_devs - .iter_mut() - .map(|bd| (bd.uuid(), bd as &mut StratBlockDev)) - .collect() - } - - pub fn get_blockdev_by_uuid(&self, uuid: DevUuid) -> Option<&StratBlockDev> { - self.block_devs.iter().find(|bd| bd.uuid() == uuid) - } - - pub fn get_mut_blockdev_by_uuid(&mut self, uuid: DevUuid) -> Option<&mut StratBlockDev> { - self.block_devs.iter_mut().find(|bd| bd.uuid() == uuid) - } - // SIZE methods /// The number of sectors not allocated for any purpose. @@ -370,47 +472,6 @@ impl BlockDevMgr { .sum() } - /// Get the encryption information for a whole pool. - pub fn encryption_info(&self) -> Option { - gather_encryption_info( - self.block_devs.len(), - self.block_devs.iter().map(|bd| bd.encryption_info()), - ) - .expect("Cannot create a pool out of both encrypted and unencrypted devices") - } - - pub fn is_encrypted(&self) -> bool { - self.encryption_info().is_some() - } - - #[cfg(test)] - fn invariant(&self) { - let pool_uuids = self - .block_devs - .iter() - .map(|bd| bd.pool_uuid()) - .collect::>(); - assert!(pool_uuids.len() == 1); - - let encryption_infos = self - .block_devs - .iter() - .filter_map(|bd| bd.encryption_info()) - .collect::>(); - if encryption_infos.is_empty() { - assert_eq!(self.encryption_info(), None); - } else { - assert_eq!(encryption_infos.len(), self.block_devs.len()); - - let info_set = encryption_infos.iter().collect::>(); - assert!(info_set.len() == 1); - } - - for bd in self.block_devs.iter() { - bd.invariant(); - } - } - pub fn grow(&mut self, dev: DevUuid) -> StratisResult { let bd = self .block_devs @@ -437,7 +498,10 @@ impl BlockDevMgr { } } -impl Recordable> for BlockDevMgr { +impl Recordable> for BlockDevMgr +where + B: Recordable, +{ fn record(&self) -> Vec { self.block_devs.iter().map(|bd| bd.record()).collect() } @@ -449,7 +513,10 @@ mod tests { use crate::engine::{ strat_engine::{ - backstore::devices::{ProcessedPathInfos, UnownedDevices}, + backstore::{ + blockdev, + devices::{ProcessedPathInfos, UnownedDevices}, + }, cmd, tests::{crypt, loopbacked, real}, }, @@ -464,219 +531,342 @@ mod tests { .and_then(|(sds, uds)| sds.error_on_not_empty().map(|_| uds)) } - /// Verify that initially, - /// size() - metadata_size() = avail_space(). - /// After 2 Sectors have been allocated, that amount must also be included - /// in balance. - fn test_blockdevmgr_used(paths: &[&Path]) { - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - let devices = get_devices(paths).unwrap(); - let mut mgr = BlockDevMgr::initialize( - pool_name, - pool_uuid, - devices, - MDADataSize::default(), - None, - None, - ) - .unwrap(); - assert_eq!(mgr.avail_space() + mgr.metadata_size(), mgr.size()); - - let allocated = Sectors(2); - mgr.alloc(&[allocated]).unwrap(); - assert_eq!( - mgr.avail_space() + allocated + mgr.metadata_size(), - mgr.size() - ); - } - - #[test] - fn loop_test_blockdevmgr_used() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, None), - test_blockdevmgr_used, - ); - } - - #[test] - fn real_test_blockdevmgr_used() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_blockdevmgr_used, - ); - } + mod v1 { + use super::*; - /// Test that the `BlockDevMgr` will add devices if the same key - /// is used to encrypted the existing devices and the added devices. - fn test_blockdevmgr_same_key(paths: &[&Path]) { - fn test_with_key(paths: &[&Path], key_desc: &KeyDescription) { + /// Verify that initially, + /// size() - metadata_size() = avail_space(). + /// After 2 Sectors have been allocated, that amount must also be included + /// in balance. + fn test_blockdevmgr_used(paths: &[&Path]) { let pool_uuid = PoolUuid::new_v4(); - - let devices1 = get_devices(&paths[..2]).unwrap(); - let devices2 = get_devices(&paths[2..3]).unwrap(); - let pool_name = Name::new("pool_name".to_string()); - let mut bdm = BlockDevMgr::initialize( - pool_name.clone(), + let devices = get_devices(paths).unwrap(); + let mut mgr = BlockDevMgr::::initialize( + pool_name, pool_uuid, - devices1, + devices, MDADataSize::default(), - Some(&EncryptionInfo::KeyDesc(key_desc.clone())), + None, None, ) .unwrap(); + assert_eq!(mgr.avail_space() + mgr.metadata_size(), mgr.size()); + + let allocated = Sectors(2); + mgr.alloc(&[allocated]).unwrap(); + assert_eq!( + mgr.avail_space() + allocated + mgr.metadata_size(), + mgr.size() + ); + } - if bdm.add(pool_name, pool_uuid, devices2, None).is_err() { - panic!("Adding a blockdev with the same key to an encrypted pool should succeed") + #[test] + fn loop_test_blockdevmgr_used() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_blockdevmgr_used, + ); + } + + #[test] + fn real_test_blockdevmgr_used() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_blockdevmgr_used, + ); + } + + /// Test that the `BlockDevMgr` will add devices if the same key + /// is used to encrypted the existing devices and the added devices. + fn test_blockdevmgr_same_key(paths: &[&Path]) { + fn test_with_key(paths: &[&Path], key_desc: &KeyDescription) { + let pool_uuid = PoolUuid::new_v4(); + + let devices1 = get_devices(&paths[..2]).unwrap(); + let devices2 = get_devices(&paths[2..3]).unwrap(); + + let pool_name = Name::new("pool_name".to_string()); + let mut bdm = BlockDevMgr::::initialize( + pool_name.clone(), + pool_uuid, + devices1, + MDADataSize::default(), + Some(&EncryptionInfo::KeyDesc(key_desc.clone())), + None, + ) + .unwrap(); + + if bdm.add(pool_name, pool_uuid, devices2, None).is_err() { + panic!( + "Adding a blockdev with the same key to an encrypted pool should succeed" + ) + } } + + crypt::insert_and_cleanup_key(paths, test_with_key); } - crypt::insert_and_cleanup_key(paths, test_with_key); - } + #[test] + fn loop_test_blockdevmgr_same_key() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(3, None), + test_blockdevmgr_same_key, + ); + } - #[test] - fn loop_test_blockdevmgr_same_key() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(3, None), - test_blockdevmgr_same_key, - ); - } + #[test] + fn real_test_blockdevmgr_same_key() { + real::test_with_spec( + &real::DeviceLimits::Exactly(3, None, None), + test_blockdevmgr_same_key, + ); + } - #[test] - fn real_test_blockdevmgr_same_key() { - real::test_with_spec( - &real::DeviceLimits::Exactly(3, None, None), - test_blockdevmgr_same_key, - ); - } + /// Test that the `BlockDevMgr` will not add devices if a different key + /// is present in the keyring than was used to encrypted the existing + /// devices. + fn test_blockdevmgr_changed_key(paths: &[&Path]) { + fn test_with_key(paths: &[&Path], key_desc: &KeyDescription) { + let pool_uuid = PoolUuid::new_v4(); + + let devices1 = get_devices(&paths[..2]).unwrap(); + let devices2 = get_devices(&paths[2..3]).unwrap(); + + let pool_name = Name::new("pool_name".to_string()); + let mut bdm = BlockDevMgr::::initialize( + pool_name.clone(), + pool_uuid, + devices1, + MDADataSize::default(), + Some(&EncryptionInfo::KeyDesc(key_desc.clone())), + None, + ) + .unwrap(); + + crypt::change_key(key_desc); + + if bdm.add(pool_name, pool_uuid, devices2, None).is_ok() { + panic!("Adding a blockdev with a new key to an encrypted pool should fail") + } + } - /// Test that the `BlockDevMgr` will not add devices if a different key - /// is present in the keyring than was used to encrypted the existing - /// devices. - fn test_blockdevmgr_changed_key(paths: &[&Path]) { - fn test_with_key(paths: &[&Path], key_desc: &KeyDescription) { - let pool_uuid = PoolUuid::new_v4(); + crypt::insert_and_cleanup_key(paths, test_with_key); + } - let devices1 = get_devices(&paths[..2]).unwrap(); - let devices2 = get_devices(&paths[2..3]).unwrap(); + #[test] + fn loop_test_blockdevmgr_changed_key() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(3, None), + test_blockdevmgr_changed_key, + ); + } - let pool_name = Name::new("pool_name".to_string()); - let mut bdm = BlockDevMgr::initialize( - pool_name.clone(), - pool_uuid, - devices1, + #[test] + fn real_test_blockdevmgr_changed_key() { + real::test_with_spec( + &real::DeviceLimits::Exactly(3, None, None), + test_blockdevmgr_changed_key, + ); + } + + /// Verify that it is impossible to steal blockdevs from another Stratis + /// pool. + /// 1. Initialize devices with pool uuid. + /// 2. Initializing again with different uuid must fail. + /// 3. Adding the devices must succeed, because they already belong. + fn test_initialization_add_stratis(paths: &[&Path]) { + assert!(paths.len() > 1); + let (paths1, paths2) = paths.split_at(paths.len() / 2); + + let uuid = PoolUuid::new_v4(); + let uuid2 = PoolUuid::new_v4(); + let pool_name1 = Name::new("pool_name1".to_string()); + let pool_name2 = Name::new("pool_name2".to_string()); + + let bd_mgr = BlockDevMgr::::initialize( + pool_name1, + uuid, + get_devices(paths1).unwrap(), MDADataSize::default(), - Some(&EncryptionInfo::KeyDesc(key_desc.clone())), + None, + None, + ) + .unwrap(); + cmd::udev_settle().unwrap(); + + assert_matches!(get_devices(paths1), Err(_)); + + assert!(ProcessedPathInfos::try_from(paths1) + .unwrap() + .unpack() + .0 + .partition(uuid2) + .0 + .is_empty()); + + assert!(!ProcessedPathInfos::try_from(paths1) + .unwrap() + .unpack() + .0 + .partition(uuid) + .0 + .is_empty()); + + BlockDevMgr::::initialize( + pool_name2, + uuid, + get_devices(paths2).unwrap(), + MDADataSize::default(), + None, None, ) .unwrap(); - crypt::change_key(key_desc); + cmd::udev_settle().unwrap(); - if bdm.add(pool_name, pool_uuid, devices2, None).is_ok() { - panic!("Adding a blockdev with a new key to an encrypted pool should fail") - } + assert!(!ProcessedPathInfos::try_from(paths2) + .unwrap() + .unpack() + .0 + .partition(uuid) + .0 + .is_empty()); + + bd_mgr.invariant() } - crypt::insert_and_cleanup_key(paths, test_with_key); - } + #[test] + fn loop_test_initialization_stratis() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_initialization_add_stratis, + ); + } - #[test] - fn loop_test_blockdevmgr_changed_key() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(3, None), - test_blockdevmgr_changed_key, - ); + #[test] + fn real_test_initialization_stratis() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_initialization_add_stratis, + ); + } } - #[test] - fn real_test_blockdevmgr_changed_key() { - real::test_with_spec( - &real::DeviceLimits::Exactly(3, None, None), - test_blockdevmgr_changed_key, - ); - } + mod v2 { + use super::*; - /// Verify that it is impossible to steal blockdevs from another Stratis - /// pool. - /// 1. Initialize devices with pool uuid. - /// 2. Initializing again with different uuid must fail. - /// 3. Adding the devices must succeed, because they already belong. - fn test_initialization_add_stratis(paths: &[&Path]) { - assert!(paths.len() > 1); - let (paths1, paths2) = paths.split_at(paths.len() / 2); - - let uuid = PoolUuid::new_v4(); - let uuid2 = PoolUuid::new_v4(); - let pool_name1 = Name::new("pool_name1".to_string()); - let pool_name2 = Name::new("pool_name2".to_string()); - - let bd_mgr = BlockDevMgr::initialize( - pool_name1, - uuid, - get_devices(paths1).unwrap(), - MDADataSize::default(), - None, - None, - ) - .unwrap(); - cmd::udev_settle().unwrap(); - - assert_matches!(get_devices(paths1), Err(_)); - - assert!(ProcessedPathInfos::try_from(paths1) - .unwrap() - .unpack() - .0 - .partition(uuid2) - .0 - .is_empty()); - - assert!(!ProcessedPathInfos::try_from(paths1) - .unwrap() - .unpack() - .0 - .partition(uuid) - .0 - .is_empty()); - - BlockDevMgr::initialize( - pool_name2, - uuid, - get_devices(paths2).unwrap(), - MDADataSize::default(), - None, - None, - ) - .unwrap(); + /// Verify that initially, + /// size() - metadata_size() = avail_space(). + /// After 2 Sectors have been allocated, that amount must also be included + /// in balance. + fn test_blockdevmgr_used(paths: &[&Path]) { + let pool_uuid = PoolUuid::new_v4(); + let devices = get_devices(paths).unwrap(); + let mut mgr = BlockDevMgr::::initialize( + pool_uuid, + devices, + MDADataSize::default(), + ) + .unwrap(); + assert_eq!(mgr.avail_space() + mgr.metadata_size(), mgr.size()); + + let allocated = Sectors(2); + mgr.alloc(&[allocated]).unwrap(); + assert_eq!( + mgr.avail_space() + allocated + mgr.metadata_size(), + mgr.size() + ); + } - cmd::udev_settle().unwrap(); + #[test] + fn loop_test_blockdevmgr_used() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_blockdevmgr_used, + ); + } - assert!(!ProcessedPathInfos::try_from(paths2) - .unwrap() - .unpack() - .0 - .partition(uuid) - .0 - .is_empty()); + #[test] + fn real_test_blockdevmgr_used() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_blockdevmgr_used, + ); + } - bd_mgr.invariant() - } + /// Verify that it is impossible to steal blockdevs from another Stratis + /// pool. + /// 1. Initialize devices with pool uuid. + /// 2. Initializing again with different uuid must fail. + /// 3. Adding the devices must succeed, because they already belong. + fn test_initialization_add_stratis(paths: &[&Path]) { + assert!(paths.len() > 1); + let (paths1, paths2) = paths.split_at(paths.len() / 2); + + let uuid = PoolUuid::new_v4(); + let uuid2 = PoolUuid::new_v4(); + + let bd_mgr = BlockDevMgr::::initialize( + uuid, + get_devices(paths1).unwrap(), + MDADataSize::default(), + ) + .unwrap(); + cmd::udev_settle().unwrap(); + + assert_matches!(get_devices(paths1), Err(_)); + + assert!(ProcessedPathInfos::try_from(paths1) + .unwrap() + .unpack() + .0 + .partition(uuid2) + .0 + .is_empty()); + + assert!(!ProcessedPathInfos::try_from(paths1) + .unwrap() + .unpack() + .0 + .partition(uuid) + .0 + .is_empty()); + + BlockDevMgr::::initialize( + uuid, + get_devices(paths2).unwrap(), + MDADataSize::default(), + ) + .unwrap(); - #[test] - fn loop_test_initialization_stratis() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), - test_initialization_add_stratis, - ); - } + cmd::udev_settle().unwrap(); - #[test] - fn real_test_initialization_stratis() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(2, None, None), - test_initialization_add_stratis, - ); + assert!(!ProcessedPathInfos::try_from(paths2) + .unwrap() + .unpack() + .0 + .partition(uuid) + .0 + .is_empty()); + + bd_mgr.invariant() + } + + #[test] + fn loop_test_initialization_stratis() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_initialization_add_stratis, + ); + } + + #[test] + fn real_test_initialization_stratis() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_initialization_add_stratis, + ); + } } } diff --git a/src/engine/strat_engine/backstore/cache_tier.rs b/src/engine/strat_engine/backstore/cache_tier.rs index 9742ccd5dc..2dcd2410a6 100644 --- a/src/engine/strat_engine/backstore/cache_tier.rs +++ b/src/engine/strat_engine/backstore/cache_tier.rs @@ -38,7 +38,7 @@ const MAX_CACHE_SIZE: Sectors = Sectors(32 * IEC::Ti / SECTOR_SIZE as u64); #[derive(Debug)] pub struct CacheTier { /// Manages the individual block devices - pub(super) block_mgr: BlockDevMgr, + pub(super) block_mgr: BlockDevMgr, /// The list of segments granted by block_mgr and used by the cache /// device. pub(super) cache_segments: AllocatedAbove, @@ -51,7 +51,7 @@ impl CacheTier { /// Setup a previously existing cache layer from the block_mgr and /// previously allocated segments. pub fn setup( - block_mgr: BlockDevMgr, + block_mgr: BlockDevMgr, cache_tier_save: &CacheTierSave, ) -> BDARecordResult { if block_mgr.avail_space() != Sectors(0) { @@ -147,7 +147,7 @@ impl CacheTier { /// sub-device too big. /// /// WARNING: metadata changing event - pub fn new(mut block_mgr: BlockDevMgr) -> StratisResult { + pub fn new(mut block_mgr: BlockDevMgr) -> StratisResult { let avail_space = block_mgr.avail_space(); // FIXME: Come up with a better way to choose metadata device size @@ -298,7 +298,7 @@ mod tests { let devices1 = get_devices(paths1).unwrap(); let devices2 = get_devices(paths2).unwrap(); - let mgr = BlockDevMgr::initialize( + let mgr = BlockDevMgr::::initialize( pool_name.clone(), pool_uuid, devices1, diff --git a/src/engine/strat_engine/backstore/data_tier.rs b/src/engine/strat_engine/backstore/data_tier.rs index 27aa811c81..9259fc0592 100644 --- a/src/engine/strat_engine/backstore/data_tier.rs +++ b/src/engine/strat_engine/backstore/data_tier.rs @@ -30,7 +30,7 @@ use crate::{ #[derive(Debug)] pub struct DataTier { /// Manages the individual block devices - pub(super) block_mgr: BlockDevMgr, + pub(super) block_mgr: BlockDevMgr, /// The list of segments granted by block_mgr and used by dm_device pub(super) segments: AllocatedAbove, } @@ -39,7 +39,7 @@ impl DataTier { /// Setup a previously existing data layer from the block_mgr and /// previously allocated segments. pub fn setup( - block_mgr: BlockDevMgr, + block_mgr: BlockDevMgr, data_tier_save: &DataTierSave, ) -> BDARecordResult { let uuid_to_devno = block_mgr.uuid_to_devno(); @@ -66,7 +66,7 @@ impl DataTier { /// Initially 0 segments are allocated. /// /// WARNING: metadata changing event - pub fn new(block_mgr: BlockDevMgr) -> DataTier { + pub fn new(block_mgr: BlockDevMgr) -> DataTier { DataTier { block_mgr, segments: AllocatedAbove { inner: vec![] }, @@ -241,7 +241,7 @@ mod tests { let devices1 = get_devices(paths1).unwrap(); let devices2 = get_devices(paths2).unwrap(); - let mgr = BlockDevMgr::initialize( + let mgr = BlockDevMgr::::initialize( pool_name.clone(), pool_uuid, devices1, diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index 828e1fbbec..b7ed570626 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -24,8 +24,8 @@ use crate::{ engine::{ strat_engine::{ backstore::blockdev::{ - v1::{StratBlockDev, UnderlyingDevice}, - InternalBlockDev, + v1::{self, UnderlyingDevice}, + v2, InternalBlockDev, }, crypt::handle::v1::CryptHandle, device::{blkdev_logical_sector_size, blkdev_physical_sector_size, blkdev_size}, @@ -501,14 +501,14 @@ impl UnownedDevices { /// /// Precondition: Each device's DeviceInfo struct contains all necessary /// information about the device. -pub fn initialize_devices( +pub fn initialize_devices_legacy( devices: UnownedDevices, pool_name: Name, pool_uuid: PoolUuid, mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, sector_size: Option, -) -> StratisResult> { +) -> StratisResult> { /// Initialize an encrypted device on the given physical device /// using the pool and device UUIDs of the new Stratis block device /// and the key description for the key to use for encrypting the @@ -558,7 +558,7 @@ pub fn initialize_devices( dev_uuid: DevUuid, sizes: (MDADataSize, BlockdevSize), id_wwn: &Option>, - ) -> StratisResult { + ) -> StratisResult { let (mda_data_size, data_size) = sizes; let mut f = OpenOptions::new() .write(true) @@ -589,7 +589,7 @@ pub fn initialize_devices( bda.initialize(&mut f)?; - StratBlockDev::new(devno, bda, &[], None, hw_id, underlying_device).map_err(|(e, _)| e) + v1::StratBlockDev::new(devno, bda, &[], None, hw_id, underlying_device).map_err(|(e, _)| e) } /// Clean up an encrypted device after initialization failure. @@ -655,7 +655,7 @@ pub fn initialize_devices( mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, sector_size: Option, - ) -> StratisResult { + ) -> StratisResult { let dev_uuid = DevUuid::new_v4(); let (handle, devno, blockdev_size) = if let Some(ei) = encryption_info { initialize_encrypted( @@ -731,8 +731,8 @@ pub fn initialize_devices( mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, sector_size: Option, - ) -> StratisResult> { - let mut initialized_blockdevs: Vec = Vec::new(); + ) -> StratisResult> { + let mut initialized_blockdevs: Vec = Vec::new(); for dev_info in devices.inner { match initialize_one( &dev_info, @@ -787,10 +787,173 @@ pub fn initialize_devices( res } +/// Initialize devices in devices. +/// Clean up previously initialized devices if initialization of any single +/// device fails during initialization. Log at the warning level if cleanup +/// fails. +/// +/// Precondition: All devices have been identified as ready to be initialized +/// in a previous step. +/// +/// Precondition: Each device's DeviceInfo struct contains all necessary +/// information about the device. +pub fn initialize_devices( + devices: UnownedDevices, + pool_uuid: PoolUuid, + mda_data_size: MDADataSize, +) -> StratisResult> { + fn initialize_stratis_metadata( + devnode: DevicePath, + devno: Device, + pool_uuid: PoolUuid, + dev_uuid: DevUuid, + sizes: (MDADataSize, BlockdevSize), + id_wwn: &Option>, + ) -> StratisResult { + let (mda_data_size, data_size) = sizes; + let mut f = OpenOptions::new().write(true).open(&*devnode)?; + + // NOTE: Encrypted devices will discard the hardware ID as encrypted devices + // are always represented as logical, software-based devicemapper devices + // which will never have a hardware ID. + let hw_id = match id_wwn { + Some(Ok(ref hw_id)) => Some(hw_id.to_owned()), + Some(Err(_)) => { + warn!("Value for ID_WWN for device {} obtained from the udev database could not be decoded; inserting device into pool with UUID {} anyway", + devnode.display(), + pool_uuid); + None + } + None => None, + }; + + let bda = BDA::new( + StratSigblockVersion::V2, + StratisIdentifiers::new(pool_uuid, dev_uuid), + mda_data_size, + data_size, + Utc::now(), + ); + + bda.initialize(&mut f)?; + + v2::StratBlockDev::new(devno, bda, &[], None, hw_id, devnode).map_err(|(e, _)| e) + } + + /// Clean up an unencrypted device after initialization failure. + fn clean_up(path: &Path, causal_error: StratisError) -> StratisError { + if let Err(e) = OpenOptions::new() + .write(true) + .open(path) + .map_err(StratisError::from) + .and_then(|mut f| disown_device(&mut f)) + { + let msg = format!( + "Failed to clean up unencrypted device {}; cleanup was attempted because initialization of the device failed", + path.display(), + ); + warn!("{}; clean up failure cause: {}", msg, e,); + StratisError::Chained( + msg, + Box::new(StratisError::NoActionRollbackError { + causal_error: Box::new(causal_error), + rollback_error: Box::new(e), + }), + ) + } else { + causal_error + } + } + + // Initialize a single device using information in dev_info. + // If initialization fails at any stage clean up the device. + // Return an error if initialization failed. Log a warning if cleanup + // fails. + // + // This method will clean up after LUKS2 and unencrypted Stratis devices + // in phases. In the case of encryption, if a device has been initialized + // as an encrypted volume, it will either rely on StratBlockDev::disown() + // if the in-memory StratBlockDev object has been created or + // will call out directly to destroy_encrypted_stratis_device() if it + // fails before that. + fn initialize_one( + dev_info: &DeviceInfo, + pool_uuid: PoolUuid, + mda_data_size: MDADataSize, + ) -> StratisResult { + let dev_uuid = DevUuid::new_v4(); + let (devno, blockdev_size) = (dev_info.devno, dev_info.size.sectors()); + + let physical_path = &dev_info.devnode; + let blockdev = initialize_stratis_metadata( + DevicePath::new(physical_path)?, + devno, + pool_uuid, + dev_uuid, + (mda_data_size, BlockdevSize::new(blockdev_size)), + &dev_info.id_wwn, + ); + if let Err(err) = blockdev { + Err(clean_up(physical_path, err)) + } else { + blockdev + } + } + + /// Initialize all provided devices with Stratis metadata. + fn initialize_all( + devices: UnownedDevices, + pool_uuid: PoolUuid, + mda_data_size: MDADataSize, + ) -> StratisResult> { + let mut initialized_blockdevs: Vec = Vec::new(); + for dev_info in devices.inner { + match initialize_one(&dev_info, pool_uuid, mda_data_size) { + Ok(blockdev) => initialized_blockdevs.push(blockdev), + Err(err) => { + if let Err(err) = wipe_blockdevs(&mut initialized_blockdevs) { + warn!("Failed to clean up some devices after initialization of device {} for pool with UUID {} failed: {}", + dev_info.devnode.display(), + pool_uuid, + err); + } + return Err(err); + } + } + } + Ok(initialized_blockdevs) + } + + let device_paths = devices + .inner + .iter() + .map(|d| d.devnode.clone()) + .collect::>(); + { + let mut guard = (*BLOCKDEVS_IN_PROGRESS).lock().expect("Should not panic"); + if device_paths.iter().any(|dev| guard.contains(dev)) { + return Err(StratisError::Msg(format!("An initialization operation is already in progress with at least one of the following devices: {device_paths:?}"))); + } + guard.extend(device_paths.iter().cloned()); + } + + let res = initialize_all(devices, pool_uuid, mda_data_size); + + { + let mut guard = (*BLOCKDEVS_IN_PROGRESS).lock().expect("Should not panic"); + guard.retain(|path| !device_paths.contains(path)); + } + + res +} + /// Wipe some blockdevs of their identifying headers. /// Return an error if any of the blockdevs could not be wiped. /// If an error occurs while wiping a blockdev, attempt to wipe all remaining. -pub fn wipe_blockdevs(blockdevs: &mut [StratBlockDev]) -> StratisResult<()> { +pub fn wipe_blockdevs(blockdevs: &mut [B]) -> StratisResult<()> +where + B: InternalBlockDev, +{ let unerased_devnodes: Vec<_> = blockdevs .iter_mut() .filter_map(|bd| match bd.disown() { @@ -823,7 +986,6 @@ mod tests { use crate::engine::{ strat_engine::{ - crypt::handle::v1::CryptHandle, metadata::device_identifiers, tests::{crypt, loopbacked, real}, }, @@ -832,65 +994,441 @@ mod tests { use super::*; - /// Test that initializing devices claims all and that destroying - /// them releases all. Verify that already initialized devices are - /// rejected or filtered as appropriate. - fn test_ownership(paths: &[&Path], key_description: Option<&KeyDescription>) { - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - let dev_infos: Vec<_> = ProcessedPathInfos::try_from(paths) + // Verify that a non-existent path results in a reasonably elegant + // error, i.e., not an assertion failure. + fn test_nonexistent_path(paths: &[&Path]) { + assert!(!paths.is_empty()); + + let test_paths = [paths, &[Path::new("/srk/cheese")]].concat(); + + assert_matches!(ProcessedPathInfos::try_from(test_paths.as_slice()), Err(_)); + } + + #[test] + fn loop_test_nonexistent_path() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_nonexistent_path, + ); + } + + #[test] + fn real_test_nonexistent_path() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_nonexistent_path, + ); + } + + // Verify that resolve devices simply eliminates duplicate devnodes, + // without returning an error. + fn test_duplicate_devnodes(paths: &[&Path]) { + assert!(!paths.is_empty()); + + let duplicate_paths = paths + .iter() + .chain(paths.iter()) + .copied() + .collect::>(); + + let result = ProcessedPathInfos::try_from(duplicate_paths.as_slice()) .unwrap() .unclaimed_devices; - if dev_infos.len() != paths.len() { - panic!("Some duplicate devices were found"); + assert_eq!(result.len(), paths.len()); + } + + #[test] + fn loop_test_duplicate_devnodes() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 2, None), + test_duplicate_devnodes, + ); + } + + #[test] + fn real_test_duplicate_devnodes() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_duplicate_devnodes, + ); + } + + mod v1 { + use super::*; + + /// Test that initializing devices claims all and that destroying + /// them releases all. Verify that already initialized devices are + /// rejected or filtered as appropriate. + fn test_ownership(paths: &[&Path], key_description: Option<&KeyDescription>) { + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + let dev_infos: Vec<_> = ProcessedPathInfos::try_from(paths) + .unwrap() + .unclaimed_devices; + + if dev_infos.len() != paths.len() { + panic!("Some duplicate devices were found"); + } + + let mut blockdevs = initialize_devices_legacy( + UnownedDevices { inner: dev_infos }, + pool_name, + pool_uuid, + MDADataSize::default(), + key_description + .map(|kd| EncryptionInfo::KeyDesc(kd.clone())) + .as_ref(), + None, + ) + .unwrap(); + + if blockdevs.len() != paths.len() { + panic!("Fewer blockdevices were created than were requested"); + } + + let stratis_devnodes: Vec = blockdevs + .iter() + .map(|bd| bd.metadata_path().to_owned()) + .collect(); + + let stratis_identifiers: Vec> = stratis_devnodes + .iter() + .map(|dev| { + OpenOptions::new() + .read(true) + .open(dev) + .map_err(|err| err.into()) + .and_then(|mut f| device_identifiers(&mut f)) + }) + .collect::>>>() + .unwrap(); + + if stratis_identifiers.iter().any(Option::is_none) { + panic!("Some device which should have had Stratis identifiers on it did not"); + } + + if stratis_identifiers + .iter() + .any(|x| x.expect("returned in line above if any are None").pool_uuid != pool_uuid) + { + panic!("Some device had the wrong pool UUID"); + } + + if key_description.is_none() { + if !ProcessedPathInfos::try_from( + stratis_devnodes + .iter() + .map(|p| p.as_path()) + .collect::>() + .as_slice(), + ) + .unwrap() + .unpack() + .1 + .inner + .is_empty() + { + panic!( + "Failed to eliminate devices already initialized for this pool from list of devices to initialize" + ); + } + + if ProcessedPathInfos::try_from( + stratis_devnodes + .iter() + .map(|p| p.as_path()) + .collect::>() + .as_slice(), + ) + .unwrap() + .unpack() + .0 + .partition(pool_uuid) + .1 + .error_on_not_empty() + .is_err() + { + panic!( + "Failed to return an error when some device processed was not in the set of already initialized devices" + ); + } + } else { + // The devices will be rejected with an errorif they were the + // minimum size when initialized. + if let Ok(infos) = ProcessedPathInfos::try_from( + stratis_devnodes + .iter() + .map(|p| p.as_path()) + .collect::>() + .as_slice(), + ) { + if !infos.unpack().0.partition(pool_uuid).1.inner.is_empty() { + panic!( + "Failed to eliminate devices already initialized for this pool from list of devices to initialize" + ); + } + } + + if ProcessedPathInfos::try_from(paths).is_ok() { + panic!("Failed to return an error when encountering devices that are LUKS2"); + } + } + + if let Ok(infos) = ProcessedPathInfos::try_from( + stratis_devnodes + .iter() + .map(|p| p.as_path()) + .collect::>() + .as_slice(), + ) { + if !infos.unpack().0.partition(PoolUuid::new_v4()).0.is_empty() { + panic!( + "Failed to leave devices in StratisDevices when processing devices for a pool UUID which is not the same as that for which the devices were initialized" + ); + } + }; + + wipe_blockdevs(&mut blockdevs).unwrap(); + + for path in paths { + if key_description.is_some() { + if CryptHandle::load_metadata(path).unwrap().is_some() { + panic!("LUKS2 metadata on Stratis devices was not successfully wiped"); + } + } else if (device_identifiers( + &mut OpenOptions::new().read(true).open(path).unwrap(), + ) + .unwrap()) + .is_some() + { + panic!("Metadata on Stratis devices was not successfully wiped"); + } + } } - let mut blockdevs = initialize_devices( - UnownedDevices { inner: dev_infos }, - pool_name, - pool_uuid, - MDADataSize::default(), - key_description - .map(|kd| EncryptionInfo::KeyDesc(kd.clone())) - .as_ref(), - None, - ) - .unwrap(); + /// Test ownership with encryption + fn test_ownership_crypt(paths: &[&Path]) { + fn call_crypt_test(paths: &[&Path], key_description: &KeyDescription) { + test_ownership(paths, Some(key_description)) + } - if blockdevs.len() != paths.len() { - panic!("Fewer blockdevices were created than were requested"); + crypt::insert_and_cleanup_key(paths, call_crypt_test) } - let stratis_devnodes: Vec = blockdevs - .iter() - .map(|bd| bd.metadata_path().to_owned()) - .collect(); + /// Test ownership with no encryption + fn test_ownership_no_crypt(paths: &[&Path]) { + test_ownership(paths, None) + } - let stratis_identifiers: Vec> = stratis_devnodes - .iter() - .map(|dev| { - OpenOptions::new() - .read(true) - .open(dev) - .map_err(|err| err.into()) - .and_then(|mut f| device_identifiers(&mut f)) - }) - .collect::>>>() - .unwrap(); + #[test] + fn loop_test_ownership() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_ownership_no_crypt, + ); + } - if stratis_identifiers.iter().any(Option::is_none) { - panic!("Some device which should have had Stratis identifiers on it did not"); + #[test] + fn real_test_ownership() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_ownership_no_crypt, + ); } - if stratis_identifiers - .iter() - .any(|x| x.expect("returned in line above if any are None").pool_uuid != pool_uuid) - { - panic!("Some device had the wrong pool UUID"); + #[test] + fn loop_test_crypt_ownership() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_ownership_crypt, + ); + } + + #[test] + fn real_test_crypt_ownership() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_ownership_crypt, + ); + } + + // Verify that if the last device in a list of devices to initialize + // can not be initialized, all the devices previously initialized are + // properly cleaned up. + fn test_failure_cleanup(paths: &[&Path], key_desc: Option<&KeyDescription>) { + if paths.len() <= 1 { + panic!("Test requires more than one device"); + } + + let mut dev_infos = ProcessedPathInfos::try_from(paths) + .unwrap() + .unclaimed_devices; + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + + if dev_infos.len() != paths.len() { + panic!("Some duplicate devices were found"); + } + + // Synthesize a DeviceInfo that will cause initialization to fail. + { + let old_info = dev_infos.pop().expect("Must contain at least two devices"); + + let new_info = DeviceInfo { + devnode: PathBuf::from("/srk/cheese"), + devno: old_info.devno, + id_wwn: None, + size: old_info.size, + blksizes: old_info.blksizes, + }; + + dev_infos.push(new_info); + } + + if initialize_devices_legacy( + UnownedDevices { inner: dev_infos }, + pool_name, + pool_uuid, + MDADataSize::default(), + key_desc + .map(|kd| EncryptionInfo::KeyDesc(kd.clone())) + .as_ref(), + None, + ) + .is_ok() + { + panic!("Initialization should not have succeeded"); + } + + // Check all paths for absence of device identifiers or LUKS2 metadata + // depending on whether or not it is encrypted. Initialization of the + // last path was never attempted, so it should be as bare of Stratis + // identifiers as all the other paths that were initialized. + for path in paths { + if key_desc.is_some() { + if CryptHandle::load_metadata(path).unwrap().is_some() { + panic!("Device {} should have no LUKS2 metadata", path.display()); + } + } else { + let mut f = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + match device_identifiers(&mut f) { + Ok(None) => (), + _ => { + panic!( + "Device {} should have returned nothing for device identifiers", + path.display() + ) + } + } + } + } + } + + // Run test_failure_cleanup for encrypted devices + fn test_failure_cleanup_crypt(paths: &[&Path]) { + fn failure_cleanup_crypt(paths: &[&Path], key_desc: &KeyDescription) { + test_failure_cleanup(paths, Some(key_desc)) + } + + crypt::insert_and_cleanup_key(paths, failure_cleanup_crypt) + } + + // Run test_failure_cleanup for unencrypted devices + fn test_failure_cleanup_no_crypt(paths: &[&Path]) { + test_failure_cleanup(paths, None) } - if key_description.is_none() { + #[test] + fn loop_test_crypt_failure_cleanup() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_failure_cleanup_crypt, + ); + } + + #[test] + fn real_test_crypt_failure_cleanup() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_failure_cleanup_crypt, + ); + } + + #[test] + fn loop_test_failure_cleanup() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_failure_cleanup_no_crypt, + ); + } + + #[test] + fn real_test_failure_cleanup() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_failure_cleanup_no_crypt, + ); + } + } + + mod v2 { + use super::*; + + /// Test that initializing devices claims all and that destroying + /// them releases all. Verify that already initialized devices are + /// rejected or filtered as appropriate. + fn test_ownership(paths: &[&Path]) { + let pool_uuid = PoolUuid::new_v4(); + let dev_infos: Vec<_> = ProcessedPathInfos::try_from(paths) + .unwrap() + .unclaimed_devices; + + if dev_infos.len() != paths.len() { + panic!("Some duplicate devices were found") + } + + let mut blockdevs = initialize_devices( + UnownedDevices { inner: dev_infos }, + pool_uuid, + MDADataSize::default(), + ) + .unwrap(); + + if blockdevs.len() != paths.len() { + panic!("Fewer blockdevices were created than were requested") + } + + let stratis_devnodes: Vec = + blockdevs.iter().map(|bd| bd.devnode().to_owned()).collect(); + + let stratis_identifiers: Vec> = stratis_devnodes + .iter() + .map(|dev| { + OpenOptions::new() + .read(true) + .open(dev) + .map_err(|err| err.into()) + .and_then(|mut f| device_identifiers(&mut f)) + }) + .collect::>>>() + .unwrap(); + + if stratis_identifiers.iter().any(Option::is_none) { + panic!("Some device which should have had Stratis identifiers on it did not") + } + + if stratis_identifiers + .iter() + .any(|x| x.expect("returned in line above if any are None").pool_uuid != pool_uuid) + { + panic!("Some device had the wrong pool UUID") + } + if !ProcessedPathInfos::try_from( stratis_devnodes .iter() @@ -906,7 +1444,7 @@ mod tests { { panic!( "Failed to eliminate devices already initialized for this pool from list of devices to initialize" - ); + ) } if ProcessedPathInfos::try_from( @@ -926,11 +1464,9 @@ mod tests { { panic!( "Failed to return an error when some device processed was not in the set of already initialized devices" - ); + ) } - } else { - // The devices will be rejected with an errorif they were the - // minimum size when initialized. + if let Ok(infos) = ProcessedPathInfos::try_from( stratis_devnodes .iter() @@ -938,178 +1474,85 @@ mod tests { .collect::>() .as_slice(), ) { - if !infos.unpack().0.partition(pool_uuid).1.inner.is_empty() { + if !infos.unpack().0.partition(PoolUuid::new_v4()).0.is_empty() { panic!( - "Failed to eliminate devices already initialized for this pool from list of devices to initialize" - ); + "Failed to leave devices in StratisDevices when processing devices for a pool UUID which is not the same as that for which the devices were initialized" + ) } - } - - if ProcessedPathInfos::try_from(paths).is_ok() { - panic!("Failed to return an error when encountering devices that are LUKS2"); - } - } - - if let Ok(infos) = ProcessedPathInfos::try_from( - stratis_devnodes - .iter() - .map(|p| p.as_path()) - .collect::>() - .as_slice(), - ) { - if !infos.unpack().0.partition(PoolUuid::new_v4()).0.is_empty() { - panic!( - "Failed to leave devices in StratisDevices when processing devices for a pool UUID which is not the same as that for which the devices were initialized" - ); - } - }; + }; - wipe_blockdevs(&mut blockdevs).unwrap(); + wipe_blockdevs(&mut blockdevs).unwrap(); - for path in paths { - if key_description.is_some() { - if CryptHandle::load_metadata(path).unwrap().is_some() { - panic!("LUKS2 metadata on Stratis devices was not successfully wiped"); + for path in paths { + if (device_identifiers(&mut OpenOptions::new().read(true).open(path).unwrap()) + .unwrap()) + .is_some() + { + panic!("Metadata on Stratis devices was not successfully wiped") } - } else if (device_identifiers(&mut OpenOptions::new().read(true).open(path).unwrap()) - .unwrap()) - .is_some() - { - panic!("Metadata on Stratis devices was not successfully wiped"); } } - } - /// Test ownership with encryption - fn test_ownership_crypt(paths: &[&Path]) { - fn call_crypt_test(paths: &[&Path], key_description: &KeyDescription) { - test_ownership(paths, Some(key_description)) + #[test] + fn loop_test_ownership() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, None), + test_ownership, + ); } - crypt::insert_and_cleanup_key(paths, call_crypt_test) - } - - /// Test ownership with no encryption - fn test_ownership_no_crypt(paths: &[&Path]) { - test_ownership(paths, None) - } - - #[test] - fn loop_test_ownership() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, None), - test_ownership_no_crypt, - ); - } - - #[test] - fn real_test_ownership() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_ownership_no_crypt, - ); - } - - #[test] - fn loop_test_crypt_ownership() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, None), - test_ownership_crypt, - ); - } - - #[test] - fn real_test_crypt_ownership() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_ownership_crypt, - ); - } - - // Verify that a non-existent path results in a reasonably elegant - // error, i.e., not an assertion failure. - fn test_nonexistent_path(paths: &[&Path]) { - assert!(!paths.is_empty()); - - let test_paths = [paths, &[Path::new("/srk/cheese")]].concat(); - - assert_matches!(ProcessedPathInfos::try_from(test_paths.as_slice()), Err(_)); - } - - #[test] - fn loop_test_nonexistent_path() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, None), - test_nonexistent_path, - ); - } + #[test] + fn real_test_ownership() { + real::test_with_spec(&real::DeviceLimits::AtLeast(1, None, None), test_ownership); + } - #[test] - fn real_test_nonexistent_path() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_nonexistent_path, - ); - } + // Verify that if the last device in a list of devices to initialize + // can not be initialized, all the devices previously initialized are + // properly cleaned up. + fn test_failure_cleanup(paths: &[&Path]) { + if paths.len() <= 1 { + panic!("Test requires more than one device") + } - // Verify that if the last device in a list of devices to initialize - // can not be initialized, all the devices previously initialized are - // properly cleaned up. - fn test_failure_cleanup(paths: &[&Path], key_desc: Option<&KeyDescription>) { - if paths.len() <= 1 { - panic!("Test requires more than one device"); - } + let mut dev_infos = ProcessedPathInfos::try_from(paths) + .unwrap() + .unclaimed_devices; + let pool_uuid = PoolUuid::new_v4(); - let mut dev_infos = ProcessedPathInfos::try_from(paths) - .unwrap() - .unclaimed_devices; - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); + if dev_infos.len() != paths.len() { + panic!("Some duplicate devices were found") + } - if dev_infos.len() != paths.len() { - panic!("Some duplicate devices were found"); - } + // Synthesize a DeviceInfo that will cause initialization to fail. + { + let old_info = dev_infos.pop().expect("Must contain at least two devices"); - // Synthesize a DeviceInfo that will cause initialization to fail. - { - let old_info = dev_infos.pop().expect("Must contain at least two devices"); - - let new_info = DeviceInfo { - devnode: PathBuf::from("/srk/cheese"), - devno: old_info.devno, - id_wwn: None, - size: old_info.size, - blksizes: old_info.blksizes, - }; + let new_info = DeviceInfo { + devnode: PathBuf::from("/srk/cheese"), + devno: old_info.devno, + id_wwn: None, + size: old_info.size, + blksizes: old_info.blksizes, + }; - dev_infos.push(new_info); - } + dev_infos.push(new_info); + } - if initialize_devices( - UnownedDevices { inner: dev_infos }, - pool_name, - pool_uuid, - MDADataSize::default(), - key_desc - .map(|kd| EncryptionInfo::KeyDesc(kd.clone())) - .as_ref(), - None, - ) - .is_ok() - { - panic!("Initialization should not have succeeded"); - } + if initialize_devices( + UnownedDevices { inner: dev_infos }, + pool_uuid, + MDADataSize::default(), + ) + .is_ok() + { + panic!("Initialization should not have succeeded") + } - // Check all paths for absence of device identifiers or LUKS2 metadata - // depending on whether or not it is encrypted. Initialization of the - // last path was never attempted, so it should be as bare of Stratis - // identifiers as all the other paths that were initialized. - for path in paths { - if key_desc.is_some() { - if CryptHandle::load_metadata(path).unwrap().is_some() { - panic!("Device {} should have no LUKS2 metadata", path.display()); - } - } else { + // Check all paths for absence of device identifiers or LUKS2 metadata + // depending on whether or not it is encrypted. Initialization of the + // last path was never attempted, so it should be as bare of Stratis + // identifiers as all the other paths that were initialized. + for path in paths { let mut f = OpenOptions::new() .read(true) .write(true) @@ -1126,85 +1569,21 @@ mod tests { } } } - } - // Run test_failure_cleanup for encrypted devices - fn test_failure_cleanup_crypt(paths: &[&Path]) { - fn failure_cleanup_crypt(paths: &[&Path], key_desc: &KeyDescription) { - test_failure_cleanup(paths, Some(key_desc)) + #[test] + fn loop_test_failure_cleanup() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_failure_cleanup, + ); } - crypt::insert_and_cleanup_key(paths, failure_cleanup_crypt) - } - - // Run test_failure_cleanup for unencrypted devices - fn test_failure_cleanup_no_crypt(paths: &[&Path]) { - test_failure_cleanup(paths, None) - } - - #[test] - fn loop_test_crypt_failure_cleanup() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), - test_failure_cleanup_crypt, - ); - } - - #[test] - fn real_test_crypt_failure_cleanup() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(2, None, None), - test_failure_cleanup_crypt, - ); - } - - #[test] - fn loop_test_failure_cleanup() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), - test_failure_cleanup_no_crypt, - ); - } - - #[test] - fn real_test_failure_cleanup() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(2, None, None), - test_failure_cleanup_no_crypt, - ); - } - - // Verify that resolve devices simply eliminates duplicate devnodes, - // without returning an error. - fn test_duplicate_devnodes(paths: &[&Path]) { - assert!(!paths.is_empty()); - - let duplicate_paths = paths - .iter() - .chain(paths.iter()) - .copied() - .collect::>(); - - let result = ProcessedPathInfos::try_from(duplicate_paths.as_slice()) - .unwrap() - .unclaimed_devices; - - assert_eq!(result.len(), paths.len()); - } - - #[test] - fn loop_test_duplicate_devnodes() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 2, None), - test_duplicate_devnodes, - ); - } - - #[test] - fn real_test_duplicate_devnodes() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_duplicate_devnodes, - ); + #[test] + fn real_test_failure_cleanup() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_failure_cleanup, + ); + } } } diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index f9bd29298a..b766caad2c 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -13,7 +13,7 @@ mod range_alloc; mod shared; #[cfg(test)] -pub use self::devices::initialize_devices; +pub use self::devices::initialize_devices_legacy; pub use self::{ backstore::Backstore, devices::{find_stratis_devs_by_uuid, get_devno_from_path, ProcessedPathInfos, UnownedDevices}, diff --git a/src/engine/strat_engine/liminal/identify.rs b/src/engine/strat_engine/liminal/identify.rs index d48a715d92..0dea8acda7 100644 --- a/src/engine/strat_engine/liminal/identify.rs +++ b/src/engine/strat_engine/liminal/identify.rs @@ -492,7 +492,7 @@ mod tests { use crate::{ engine::{ strat_engine::{ - backstore::{initialize_devices, ProcessedPathInfos, UnownedDevices}, + backstore::{initialize_devices_legacy, ProcessedPathInfos, UnownedDevices}, cmd::create_fs, metadata::MDADataSize, tests::{crypt, loopbacked, real}, @@ -528,7 +528,7 @@ mod tests { let pool_uuid = PoolUuid::new_v4(); let pool_name = Name::new("pool_name".to_string()); - let devices = initialize_devices( + let devices = initialize_devices_legacy( get_devices(paths).unwrap(), pool_name, pool_uuid, @@ -629,7 +629,7 @@ mod tests { let pool_uuid = PoolUuid::new_v4(); let pool_name = Name::new("pool_name".to_string()); - initialize_devices( + initialize_devices_legacy( get_devices(paths).unwrap(), pool_name, pool_uuid, From c164cc6fb6c29ca5e3278175c893559fba9f12de Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 30 Jun 2023 17:58:32 -0400 Subject: [PATCH 06/32] Abstract CacheTier and DataTier across both blockdev types --- .../strat_engine/backstore/backstore.rs | 8 +- .../strat_engine/backstore/cache_tier.rs | 426 ++++++++++++------ .../strat_engine/backstore/data_tier.rs | 399 +++++++++++----- src/engine/strat_engine/backstore/shared.rs | 15 +- .../strat_engine/liminal/device_info.rs | 2 +- 5 files changed, 590 insertions(+), 260 deletions(-) diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore.rs index 977ecf9922..f8bec491c9 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore.rs @@ -52,7 +52,7 @@ const CACHE_BLOCK_SIZE: Sectors = Sectors(2048); // 1024 KiB /// take extra steps to make it clean. fn make_cache( pool_uuid: PoolUuid, - cache_tier: &CacheTier, + cache_tier: &CacheTier, origin: LinearDev, new: bool, ) -> StratisResult { @@ -102,9 +102,9 @@ pub struct Backstore { cache: Option, /// Coordinate handling of blockdevs that back the cache. Optional, since /// this structure can operate without a cache. - cache_tier: Option, + cache_tier: Option>, /// Coordinates handling of the blockdevs that form the base. - data_tier: DataTier, + data_tier: DataTier, /// A linear DM device. linear: Option, /// Index for managing allocation of cap device @@ -234,7 +234,7 @@ impl Backstore { mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, ) -> StratisResult { - let data_tier = DataTier::new(BlockDevMgr::::initialize( + let data_tier = DataTier::::new(BlockDevMgr::::initialize( pool_name, pool_uuid, devices, diff --git a/src/engine/strat_engine/backstore/cache_tier.rs b/src/engine/strat_engine/backstore/cache_tier.rs index 2dcd2410a6..adf2d38614 100644 --- a/src/engine/strat_engine/backstore/cache_tier.rs +++ b/src/engine/strat_engine/backstore/cache_tier.rs @@ -4,6 +4,8 @@ // Code to handle the backing store of a pool. +#![allow(dead_code)] + #[cfg(test)] use std::collections::HashSet; @@ -13,12 +15,14 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::{v1::StratBlockDev, InternalBlockDev}, + blockdev::{v1, v2, InternalBlockDev}, blockdevmgr::BlockDevMgr, devices::UnownedDevices, shared::{metadata_to_segment, AllocatedAbove, BlkDevSegment, BlockDevPartition}, }, - serde_structs::{BaseDevSave, BlockDevSave, CacheTierSave, Recordable}, + serde_structs::{ + BaseBlockDevSave, BaseDevSave, BlockDevSave, CacheTierSave, Recordable, + }, types::BDARecordResult, }, types::{BlockDevTier, DevUuid, Name, PoolUuid}, @@ -36,9 +40,9 @@ const MAX_CACHE_SIZE: Sectors = Sectors(32 * IEC::Ti / SECTOR_SIZE as u64); /// Handles the cache devices. #[derive(Debug)] -pub struct CacheTier { +pub struct CacheTier { /// Manages the individual block devices - pub(super) block_mgr: BlockDevMgr, + pub(super) block_mgr: BlockDevMgr, /// The list of segments granted by block_mgr and used by the cache /// device. pub(super) cache_segments: AllocatedAbove, @@ -47,51 +51,87 @@ pub struct CacheTier { pub(super) meta_segments: AllocatedAbove, } -impl CacheTier { - /// Setup a previously existing cache layer from the block_mgr and - /// previously allocated segments. - pub fn setup( - block_mgr: BlockDevMgr, - cache_tier_save: &CacheTierSave, - ) -> BDARecordResult { - if block_mgr.avail_space() != Sectors(0) { - let err_msg = format!( - "{} unallocated to device; probable metadata corruption", - block_mgr.avail_space() - ); - return Err((StratisError::Msg(err_msg), block_mgr.into_bdas())); - } +impl CacheTier { + /// Add the given paths to self. Return UUIDs of the new blockdevs + /// corresponding to the specified paths and a pair of Boolean values. + /// The first is true if the cache sub-device's segments were changed, + /// the second is true if the meta sub-device's segments were changed. + /// Adds all additional space to cache sub-device. + /// WARNING: metadata changing event + /// + /// Return an error if the addition of the cachedevs would result in a + /// cache with a cache sub-device size greater than 32 TiB. + /// + // FIXME: That all segments on the newly added device are added to the + // cache sub-device and none to the meta sub-device could lead to failure. + // Presumably, the size required for the meta sub-device varies directly + // with the size of cache sub-device. + pub fn add( + &mut self, + pool_name: Name, + pool_uuid: PoolUuid, + devices: UnownedDevices, + sector_size: Option, + ) -> StratisResult<(Vec, (bool, bool))> { + let uuids = self + .block_mgr + .add(pool_name, pool_uuid, devices, sector_size)?; - let uuid_to_devno = block_mgr.uuid_to_devno(); - let mapper = |ld: &BaseDevSave| -> StratisResult { - metadata_to_segment(&uuid_to_devno, ld) - }; + let avail_space = self.block_mgr.avail_space(); - let meta_segments = match cache_tier_save.blockdev.allocs[1] - .iter() - .map(&mapper) - .collect::>>() - { - Ok(ms) => AllocatedAbove { inner: ms }, - Err(e) => return Err((e, block_mgr.into_bdas())), - }; + // FIXME: This check will become unnecessary when cache metadata device + // can be increased dynamically. + if avail_space + self.cache_segments.size() > MAX_CACHE_SIZE { + self.block_mgr.remove_blockdevs(&uuids)?; + return Err(StratisError::Msg(format!( + "The size of the cache sub-device may not exceed {MAX_CACHE_SIZE}" + ))); + } - let cache_segments = match cache_tier_save.blockdev.allocs[0] + let segments = self + .block_mgr + .alloc(&[avail_space]) + .expect("asked for exactly the space available, must get") .iter() - .map(&mapper) - .collect::>>() - { - Ok(cs) => AllocatedAbove { inner: cs }, - Err(e) => return Err((e, block_mgr.into_bdas())), - }; + .flat_map(|s| s.iter()) + .cloned() + .collect::>(); + self.cache_segments.coalesce_blkdevsegs(&segments); - Ok(CacheTier { - block_mgr, - cache_segments, - meta_segments, - }) + Ok((uuids, (true, false))) + } + + /// Get all the blockdevs belonging to this tier. + pub fn blockdevs(&self) -> Vec<(DevUuid, &v1::StratBlockDev)> { + self.block_mgr.blockdevs() + } + + pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut v1::StratBlockDev)> { + self.block_mgr.blockdevs_mut() + } + + /// Lookup an immutable blockdev by its Stratis UUID. + pub fn get_blockdev_by_uuid( + &self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &v1::StratBlockDev)> { + self.block_mgr + .get_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Cache, bd)) + } + + /// Lookup a mutable blockdev by its Stratis UUID. + pub fn get_mut_blockdev_by_uuid( + &mut self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &mut v1::StratBlockDev)> { + self.block_mgr + .get_mut_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Cache, bd)) } +} +impl CacheTier { /// Add the given paths to self. Return UUIDs of the new blockdevs /// corresponding to the specified paths and a pair of Boolean values. /// The first is true if the cache sub-device's segments were changed, @@ -108,14 +148,10 @@ impl CacheTier { // with the size of cache sub-device. pub fn add( &mut self, - pool_name: Name, pool_uuid: PoolUuid, devices: UnownedDevices, - sector_size: Option, ) -> StratisResult<(Vec, (bool, bool))> { - let uuids = self - .block_mgr - .add(pool_name, pool_uuid, devices, sector_size)?; + let uuids = self.block_mgr.add(pool_uuid, devices)?; let avail_space = self.block_mgr.avail_space(); @@ -141,13 +177,91 @@ impl CacheTier { Ok((uuids, (true, false))) } + /// Get all the blockdevs belonging to this tier. + pub fn blockdevs(&self) -> Vec<(DevUuid, &v2::StratBlockDev)> { + self.block_mgr.blockdevs() + } + + pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut v2::StratBlockDev)> { + self.block_mgr.blockdevs_mut() + } + + /// Lookup an immutable blockdev by its Stratis UUID. + pub fn get_blockdev_by_uuid( + &self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &v2::StratBlockDev)> { + self.block_mgr + .get_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Cache, bd)) + } + + /// Lookup a mutable blockdev by its Stratis UUID. + pub fn get_mut_blockdev_by_uuid( + &mut self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &mut v2::StratBlockDev)> { + self.block_mgr + .get_mut_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Cache, bd)) + } +} + +impl CacheTier +where + B: InternalBlockDev, +{ + /// Setup a previously existing cache layer from the block_mgr and + /// previously allocated segments. + pub fn setup( + block_mgr: BlockDevMgr, + cache_tier_save: &CacheTierSave, + ) -> BDARecordResult> { + if block_mgr.avail_space() != Sectors(0) { + let err_msg = format!( + "{} unallocated to device; probable metadata corruption", + block_mgr.avail_space() + ); + return Err((StratisError::Msg(err_msg), block_mgr.into_bdas())); + } + + let uuid_to_devno = block_mgr.uuid_to_devno(); + let mapper = |ld: &BaseDevSave| -> StratisResult { + metadata_to_segment(&uuid_to_devno, ld) + }; + + let meta_segments = match cache_tier_save.blockdev.allocs[1] + .iter() + .map(&mapper) + .collect::>>() + { + Ok(ms) => AllocatedAbove { inner: ms }, + Err(e) => return Err((e, block_mgr.into_bdas())), + }; + + let cache_segments = match cache_tier_save.blockdev.allocs[0] + .iter() + .map(&mapper) + .collect::>>() + { + Ok(cs) => AllocatedAbove { inner: cs }, + Err(e) => return Err((e, block_mgr.into_bdas())), + }; + + Ok(CacheTier { + block_mgr, + cache_segments, + meta_segments, + }) + } + /// Setup a new CacheTier struct from the block_mgr. /// /// Returns an error if the block devices passed would make the cache /// sub-device too big. /// /// WARNING: metadata changing event - pub fn new(mut block_mgr: BlockDevMgr) -> StratisResult { + pub fn new(mut block_mgr: BlockDevMgr) -> StratisResult> { let avail_space = block_mgr.avail_space(); // FIXME: Come up with a better way to choose metadata device size @@ -189,35 +303,9 @@ impl CacheTier { self.block_mgr.destroy_all() } - /// Get all the blockdevs belonging to this tier. - pub fn blockdevs(&self) -> Vec<(DevUuid, &StratBlockDev)> { - self.block_mgr.blockdevs() - } - - pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut StratBlockDev)> { - self.block_mgr.blockdevs_mut() - } - - /// Lookup an immutable blockdev by its Stratis UUID. - pub fn get_blockdev_by_uuid(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> { - self.block_mgr - .get_blockdev_by_uuid(uuid) - .map(|bd| (BlockDevTier::Cache, bd)) - } - - /// Lookup a mutable blockdev by its Stratis UUID. - pub fn get_mut_blockdev_by_uuid( - &mut self, - uuid: DevUuid, - ) -> Option<(BlockDevTier, &mut StratBlockDev)> { - self.block_mgr - .get_mut_blockdev_by_uuid(uuid) - .map(|bd| (BlockDevTier::Cache, bd)) - } - /// Return the partition of the block devs that are in use and those that /// are not. - pub fn partition_cache_by_use(&self) -> BlockDevPartition<'_> { + pub fn partition_cache_by_use(&self) -> BlockDevPartition<'_, B> { let blockdevs = self.block_mgr.blockdevs(); let (used, unused) = blockdevs.iter().partition(|(_, bd)| bd.in_use()); BlockDevPartition { used, unused } @@ -251,7 +339,10 @@ impl CacheTier { } } -impl Recordable for CacheTier { +impl Recordable for CacheTier +where + B: Recordable, +{ fn record(&self) -> CacheTierSave { CacheTierSave { blockdev: BlockDevSave { @@ -268,7 +359,10 @@ mod tests { use std::path::Path; use crate::engine::strat_engine::{ - backstore::devices::{ProcessedPathInfos, UnownedDevices}, + backstore::{ + blockdev, + devices::{ProcessedPathInfos, UnownedDevices}, + }, metadata::MDADataSize, tests::{loopbacked, real}, }; @@ -284,71 +378,147 @@ mod tests { }) } - /// Do basic testing of the cache. Make a new cache and test some - /// expected properties, then add some additional blockdevs and test - /// some more properties. - fn cache_test_add(paths: &[&Path]) { - assert!(paths.len() > 1); + mod v1 { + use super::*; + + /// Do basic testing of the cache. Make a new cache and test some + /// expected properties, then add some additional blockdevs and test + /// some more properties. + fn cache_test_add(paths: &[&Path]) { + assert!(paths.len() > 1); - let (paths1, paths2) = paths.split_at(paths.len() / 2); + let (paths1, paths2) = paths.split_at(paths.len() / 2); - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); - let devices1 = get_devices(paths1).unwrap(); - let devices2 = get_devices(paths2).unwrap(); + let devices1 = get_devices(paths1).unwrap(); + let devices2 = get_devices(paths2).unwrap(); - let mgr = BlockDevMgr::::initialize( - pool_name.clone(), - pool_uuid, - devices1, - MDADataSize::default(), - None, - None, - ) - .unwrap(); + let mgr = BlockDevMgr::::initialize( + pool_name.clone(), + pool_uuid, + devices1, + MDADataSize::default(), + None, + None, + ) + .unwrap(); - let mut cache_tier = CacheTier::new(mgr).unwrap(); - cache_tier.invariant(); + let mut cache_tier = CacheTier::new(mgr).unwrap(); + cache_tier.invariant(); - // A cache tier w/ some devices and everything promptly allocated to - // the tier. - let cache_metadata_size = cache_tier.meta_segments.size(); + // A cache tier w/ some devices and everything promptly allocated to + // the tier. + let cache_metadata_size = cache_tier.meta_segments.size(); - let mut metadata_size = cache_tier.block_mgr.metadata_size(); - let mut size = cache_tier.block_mgr.size(); - let mut allocated = cache_tier.cache_segments.size(); + let mut metadata_size = cache_tier.block_mgr.metadata_size(); + let mut size = cache_tier.block_mgr.size(); + let mut allocated = cache_tier.cache_segments.size(); - assert_eq!(cache_tier.block_mgr.avail_space(), Sectors(0)); - assert_eq!(size - metadata_size, allocated + cache_metadata_size); + assert_eq!(cache_tier.block_mgr.avail_space(), Sectors(0)); + assert_eq!(size - metadata_size, allocated + cache_metadata_size); - let (_, (cache, meta)) = cache_tier - .add(pool_name, pool_uuid, devices2, None) - .unwrap(); - cache_tier.invariant(); - // TODO: Ultimately, it should be the case that meta can be true. - assert!(cache); - assert!(!meta); + let (_, (cache, meta)) = cache_tier + .add(pool_name, pool_uuid, devices2, None) + .unwrap(); + cache_tier.invariant(); + // TODO: Ultimately, it should be the case that meta can be true. + assert!(cache); + assert!(!meta); - assert_eq!(cache_tier.block_mgr.avail_space(), Sectors(0)); - assert!(cache_tier.block_mgr.size() > size); - assert!(cache_tier.block_mgr.metadata_size() > metadata_size); + assert_eq!(cache_tier.block_mgr.avail_space(), Sectors(0)); + assert!(cache_tier.block_mgr.size() > size); + assert!(cache_tier.block_mgr.metadata_size() > metadata_size); - metadata_size = cache_tier.block_mgr.metadata_size(); - size = cache_tier.block_mgr.size(); - allocated = cache_tier.cache_segments.size(); - assert_eq!(size - metadata_size, allocated + cache_metadata_size); + metadata_size = cache_tier.block_mgr.metadata_size(); + size = cache_tier.block_mgr.size(); + allocated = cache_tier.cache_segments.size(); + assert_eq!(size - metadata_size, allocated + cache_metadata_size); - cache_tier.destroy().unwrap(); - } + cache_tier.destroy().unwrap(); + } + + #[test] + fn loop_cache_test_add() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + cache_test_add, + ); + } - #[test] - fn loop_cache_test_add() { - loopbacked::test_with_spec(&loopbacked::DeviceLimits::Range(2, 3, None), cache_test_add); + #[test] + fn real_cache_test_add() { + real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), cache_test_add); + } } - #[test] - fn real_cache_test_add() { - real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), cache_test_add); + mod v2 { + use super::*; + + /// Do basic testing of the cache. Make a new cache and test some + /// expected properties, then add some additional blockdevs and test + /// some more properties. + fn cache_test_add(paths: &[&Path]) { + assert!(paths.len() > 1); + + let (paths1, paths2) = paths.split_at(paths.len() / 2); + + let pool_uuid = PoolUuid::new_v4(); + + let devices1 = get_devices(paths1).unwrap(); + let devices2 = get_devices(paths2).unwrap(); + + let mgr = BlockDevMgr::::initialize( + pool_uuid, + devices1, + MDADataSize::default(), + ) + .unwrap(); + + let mut cache_tier = CacheTier::new(mgr).unwrap(); + cache_tier.invariant(); + + // A cache tier w/ some devices and everything promptly allocated to + // the tier. + let cache_metadata_size = cache_tier.meta_segments.size(); + + let mut metadata_size = cache_tier.block_mgr.metadata_size(); + let mut size = cache_tier.block_mgr.size(); + let mut allocated = cache_tier.cache_segments.size(); + + assert_eq!(cache_tier.block_mgr.avail_space(), Sectors(0)); + assert_eq!(size - metadata_size, allocated + cache_metadata_size); + + let (_, (cache, meta)) = cache_tier.add(pool_uuid, devices2).unwrap(); + cache_tier.invariant(); + // TODO: Ultimately, it should be the case that meta can be true. + assert!(cache); + assert!(!meta); + + assert_eq!(cache_tier.block_mgr.avail_space(), Sectors(0)); + assert!(cache_tier.block_mgr.size() > size); + assert!(cache_tier.block_mgr.metadata_size() > metadata_size); + + metadata_size = cache_tier.block_mgr.metadata_size(); + size = cache_tier.block_mgr.size(); + allocated = cache_tier.cache_segments.size(); + assert_eq!(size - metadata_size, allocated + cache_metadata_size); + + cache_tier.destroy().unwrap(); + } + + #[test] + fn loop_cache_test_add() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + cache_test_add, + ); + } + + #[test] + fn real_cache_test_add() { + real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), cache_test_add); + } } } diff --git a/src/engine/strat_engine/backstore/data_tier.rs b/src/engine/strat_engine/backstore/data_tier.rs index 9259fc0592..592b725fcf 100644 --- a/src/engine/strat_engine/backstore/data_tier.rs +++ b/src/engine/strat_engine/backstore/data_tier.rs @@ -4,6 +4,8 @@ // Code to handle the backing store of a pool. +#![allow(dead_code)] + #[cfg(test)] use std::collections::HashSet; @@ -13,12 +15,14 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::{v1::StratBlockDev, InternalBlockDev}, + blockdev::{v1, v2, InternalBlockDev}, blockdevmgr::BlockDevMgr, devices::UnownedDevices, shared::{metadata_to_segment, AllocatedAbove, BlkDevSegment, BlockDevPartition}, }, - serde_structs::{BaseDevSave, BlockDevSave, DataTierSave, Recordable}, + serde_structs::{ + BaseBlockDevSave, BaseDevSave, BlockDevSave, DataTierSave, Recordable, + }, types::BDARecordResult, }, types::{BlockDevTier, DevUuid, Name, PoolUuid}, @@ -28,45 +32,20 @@ use crate::{ /// Handles the lowest level, base layer of this tier. #[derive(Debug)] -pub struct DataTier { +pub struct DataTier { /// Manages the individual block devices - pub(super) block_mgr: BlockDevMgr, + pub(super) block_mgr: BlockDevMgr, /// The list of segments granted by block_mgr and used by dm_device pub(super) segments: AllocatedAbove, } -impl DataTier { - /// Setup a previously existing data layer from the block_mgr and - /// previously allocated segments. - pub fn setup( - block_mgr: BlockDevMgr, - data_tier_save: &DataTierSave, - ) -> BDARecordResult { - let uuid_to_devno = block_mgr.uuid_to_devno(); - let mapper = |ld: &BaseDevSave| -> StratisResult { - metadata_to_segment(&uuid_to_devno, ld) - }; - let segments = match data_tier_save.blockdev.allocs[0] - .iter() - .map(&mapper) - .collect::>>() - { - Ok(s) => AllocatedAbove { inner: s }, - Err(e) => return Err((e, block_mgr.into_bdas())), - }; - - Ok(DataTier { - block_mgr, - segments, - }) - } - +impl DataTier { /// Setup a new DataTier struct from the block_mgr. /// /// Initially 0 segments are allocated. /// /// WARNING: metadata changing event - pub fn new(block_mgr: BlockDevMgr) -> DataTier { + pub fn new(block_mgr: BlockDevMgr) -> DataTier { DataTier { block_mgr, segments: AllocatedAbove { inner: vec![] }, @@ -87,6 +66,119 @@ impl DataTier { .add(pool_name, pool_uuid, devices, sector_size) } + /// Lookup an immutable blockdev by its Stratis UUID. + pub fn get_blockdev_by_uuid( + &self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &v1::StratBlockDev)> { + self.block_mgr + .get_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Data, bd)) + } + + /// Lookup a mutable blockdev by its Stratis UUID. + pub fn get_mut_blockdev_by_uuid( + &mut self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &mut v1::StratBlockDev)> { + self.block_mgr + .get_mut_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Data, bd)) + } + + /// Get the blockdevs belonging to this tier + pub fn blockdevs(&self) -> Vec<(DevUuid, &v1::StratBlockDev)> { + self.block_mgr.blockdevs() + } + + pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut v1::StratBlockDev)> { + self.block_mgr.blockdevs_mut() + } +} + +impl DataTier { + /// Setup a new DataTier struct from the block_mgr. + /// + /// Initially 0 segments are allocated. + /// + /// WARNING: metadata changing event + pub fn new(block_mgr: BlockDevMgr) -> DataTier { + DataTier { + block_mgr, + segments: AllocatedAbove { inner: vec![] }, + } + } + + /// Add the given paths to self. Return UUIDs of the new blockdevs + /// corresponding to the specified paths. + /// WARNING: metadata changing event + pub fn add( + &mut self, + pool_uuid: PoolUuid, + devices: UnownedDevices, + ) -> StratisResult> { + self.block_mgr.add(pool_uuid, devices) + } + + /// Lookup an immutable blockdev by its Stratis UUID. + pub fn get_blockdev_by_uuid( + &self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &v2::StratBlockDev)> { + self.block_mgr + .get_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Data, bd)) + } + + /// Lookup a mutable blockdev by its Stratis UUID. + pub fn get_mut_blockdev_by_uuid( + &mut self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &mut v2::StratBlockDev)> { + self.block_mgr + .get_mut_blockdev_by_uuid(uuid) + .map(|bd| (BlockDevTier::Data, bd)) + } + + /// Get the blockdevs belonging to this tier + pub fn blockdevs(&self) -> Vec<(DevUuid, &v2::StratBlockDev)> { + self.block_mgr.blockdevs() + } + + pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut v2::StratBlockDev)> { + self.block_mgr.blockdevs_mut() + } +} + +impl DataTier +where + B: InternalBlockDev, +{ + /// Setup a previously existing data layer from the block_mgr and + /// previously allocated segments. + pub fn setup( + block_mgr: BlockDevMgr, + data_tier_save: &DataTierSave, + ) -> BDARecordResult> { + let uuid_to_devno = block_mgr.uuid_to_devno(); + let mapper = |ld: &BaseDevSave| -> StratisResult { + metadata_to_segment(&uuid_to_devno, ld) + }; + let segments = match data_tier_save.blockdev.allocs[0] + .iter() + .map(&mapper) + .collect::>>() + { + Ok(s) => AllocatedAbove { inner: s }, + Err(e) => return Err((e, block_mgr.into_bdas())), + }; + + Ok(DataTier { + block_mgr, + segments, + }) + } + /// Allocate a region for all sector size requests from unallocated segments in /// block devices belonging to the data tier. Return true if requested /// amount or more was allocated, otherwise, false. @@ -142,39 +234,13 @@ impl DataTier { self.block_mgr.load_state() } - /// Lookup an immutable blockdev by its Stratis UUID. - pub fn get_blockdev_by_uuid(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> { - self.block_mgr - .get_blockdev_by_uuid(uuid) - .map(|bd| (BlockDevTier::Data, bd)) - } - - /// Lookup a mutable blockdev by its Stratis UUID. - pub fn get_mut_blockdev_by_uuid( - &mut self, - uuid: DevUuid, - ) -> Option<(BlockDevTier, &mut StratBlockDev)> { - self.block_mgr - .get_mut_blockdev_by_uuid(uuid) - .map(|bd| (BlockDevTier::Data, bd)) - } - - /// Get the blockdevs belonging to this tier - pub fn blockdevs(&self) -> Vec<(DevUuid, &StratBlockDev)> { - self.block_mgr.blockdevs() - } - - pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, &mut StratBlockDev)> { - self.block_mgr.blockdevs_mut() - } - pub fn grow(&mut self, dev: DevUuid) -> StratisResult { self.block_mgr.grow(dev) } /// Return the partition of the block devs that are in use and those /// that are not. - pub fn partition_by_use(&self) -> BlockDevPartition<'_> { + pub fn partition_by_use(&self) -> BlockDevPartition<'_, B> { let blockdevs = self.block_mgr.blockdevs(); let (used, unused) = blockdevs.iter().partition(|(_, bd)| bd.in_use()); BlockDevPartition { used, unused } @@ -194,7 +260,10 @@ impl DataTier { } } -impl Recordable for DataTier { +impl Recordable for DataTier +where + B: Recordable, +{ fn record(&self) -> DataTierSave { DataTierSave { blockdev: BlockDevSave { @@ -211,7 +280,10 @@ mod tests { use std::path::Path; use crate::engine::strat_engine::{ - backstore::devices::{ProcessedPathInfos, UnownedDevices}, + backstore::{ + blockdev, + devices::{ProcessedPathInfos, UnownedDevices}, + }, metadata::MDADataSize, tests::{loopbacked, real}, }; @@ -227,84 +299,169 @@ mod tests { }) } - /// Put the data tier through some paces. Make it, alloc a small amount, - /// add some more blockdevs, allocate enough that the newly added blockdevs - /// must be allocated from for success. - fn test_add_and_alloc(paths: &[&Path]) { - assert!(paths.len() > 1); + mod v1 { + use super::*; - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); + /// Put the data tier through some paces. Make it, alloc a small amount, + /// add some more blockdevs, allocate enough that the newly added blockdevs + /// must be allocated from for success. + fn test_add_and_alloc(paths: &[&Path]) { + assert!(paths.len() > 1); - let (paths1, paths2) = paths.split_at(paths.len() / 2); + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); - let devices1 = get_devices(paths1).unwrap(); - let devices2 = get_devices(paths2).unwrap(); + let (paths1, paths2) = paths.split_at(paths.len() / 2); - let mgr = BlockDevMgr::::initialize( - pool_name.clone(), - pool_uuid, - devices1, - MDADataSize::default(), - None, - None, - ) - .unwrap(); + let devices1 = get_devices(paths1).unwrap(); + let devices2 = get_devices(paths2).unwrap(); - let mut data_tier = DataTier::new(mgr); - data_tier.invariant(); + let mgr = BlockDevMgr::::initialize( + pool_name.clone(), + pool_uuid, + devices1, + MDADataSize::default(), + None, + None, + ) + .unwrap(); - // A data_tier w/ some devices but nothing allocated - let mut size = data_tier.size(); - let mut allocated = data_tier.allocated(); - assert_eq!(allocated, Sectors(0)); - assert!(size != Sectors(0)); + let mut data_tier = DataTier::::new(mgr); + data_tier.invariant(); - let last_request_amount = size; + // A data_tier w/ some devices but nothing allocated + let mut size = data_tier.size(); + let mut allocated = data_tier.allocated(); + assert_eq!(allocated, Sectors(0)); + assert!(size != Sectors(0)); - let request_amount = data_tier.block_mgr.avail_space() / 2usize; - assert!(request_amount != Sectors(0)); + let last_request_amount = size; - assert!(data_tier.alloc(&[request_amount])); - data_tier.invariant(); + let request_amount = data_tier.block_mgr.avail_space() / 2usize; + assert!(request_amount != Sectors(0)); - // A data tier w/ some amount allocated - assert!(data_tier.allocated() >= request_amount); - assert_eq!(data_tier.size(), size); - allocated = data_tier.allocated(); + assert!(data_tier.alloc(&[request_amount])); + data_tier.invariant(); - data_tier.add(pool_name, pool_uuid, devices2, None).unwrap(); - data_tier.invariant(); + // A data tier w/ some amount allocated + assert!(data_tier.allocated() >= request_amount); + assert_eq!(data_tier.size(), size); + allocated = data_tier.allocated(); - // A data tier w/ additional blockdevs added - assert!(data_tier.size() > size); - assert_eq!(data_tier.allocated(), allocated); - assert_eq!(paths.len(), data_tier.blockdevs().len()); - size = data_tier.size(); + data_tier.add(pool_name, pool_uuid, devices2, None).unwrap(); + data_tier.invariant(); - // Allocate enough to get into the newly added block devices - assert!(data_tier.alloc(&[last_request_amount])); - data_tier.invariant(); + // A data tier w/ additional blockdevs added + assert!(data_tier.size() > size); + assert_eq!(data_tier.allocated(), allocated); + assert_eq!(paths.len(), data_tier.blockdevs().len()); + size = data_tier.size(); - assert!(data_tier.allocated() >= request_amount + last_request_amount); - assert_eq!(data_tier.size(), size); + // Allocate enough to get into the newly added block devices + assert!(data_tier.alloc(&[last_request_amount])); + data_tier.invariant(); - data_tier.destroy().unwrap(); - } + assert!(data_tier.allocated() >= request_amount + last_request_amount); + assert_eq!(data_tier.size(), size); + + data_tier.destroy().unwrap(); + } + + #[test] + fn loop_test_add_and_alloc() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_add_and_alloc, + ); + } - #[test] - fn loop_test_add_and_alloc() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), - test_add_and_alloc, - ); + #[test] + fn real_test_add_and_alloc() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_add_and_alloc, + ); + } } - #[test] - fn real_test_add_and_alloc() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(2, None, None), - test_add_and_alloc, - ); + mod v2 { + use super::*; + + /// Put the data tier through some paces. Make it, alloc a small amount, + /// add some more blockdevs, allocate enough that the newly added blockdevs + /// must be allocated from for success. + fn test_add_and_alloc(paths: &[&Path]) { + assert!(paths.len() > 1); + + let pool_uuid = PoolUuid::new_v4(); + + let (paths1, paths2) = paths.split_at(paths.len() / 2); + + let devices1 = get_devices(paths1).unwrap(); + let devices2 = get_devices(paths2).unwrap(); + + let mgr = BlockDevMgr::::initialize( + pool_uuid, + devices1, + MDADataSize::default(), + ) + .unwrap(); + + let mut data_tier = DataTier::::new(mgr); + data_tier.invariant(); + + // A data_tier w/ some devices but nothing allocated + let mut size = data_tier.size(); + let mut allocated = data_tier.allocated(); + assert_eq!(allocated, Sectors(0)); + assert!(size != Sectors(0)); + + let last_request_amount = size; + + let request_amount = data_tier.block_mgr.avail_space() / 2usize; + assert!(request_amount != Sectors(0)); + + assert!(data_tier.alloc(&[request_amount])); + data_tier.invariant(); + + // A data tier w/ some amount allocated + assert!(data_tier.allocated() >= request_amount); + assert_eq!(data_tier.size(), size); + allocated = data_tier.allocated(); + + data_tier.add(pool_uuid, devices2).unwrap(); + data_tier.invariant(); + + // A data tier w/ additional blockdevs added + assert!(data_tier.size() > size); + assert_eq!(data_tier.allocated(), allocated); + assert_eq!(paths.len(), data_tier.blockdevs().len()); + size = data_tier.size(); + + // Allocate enough to get into the newly added block devices + assert!(data_tier.alloc(&[last_request_amount])); + data_tier.invariant(); + + assert!(data_tier.allocated() >= request_amount + last_request_amount); + assert_eq!(data_tier.size(), size); + + data_tier.destroy().unwrap(); + } + + #[test] + fn loop_test_add_and_alloc() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_add_and_alloc, + ); + } + + #[test] + fn real_test_add_and_alloc() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_add_and_alloc, + ); + } } } diff --git a/src/engine/strat_engine/backstore/shared.rs b/src/engine/strat_engine/backstore/shared.rs index dcd54d4dcc..67d6e55356 100644 --- a/src/engine/strat_engine/backstore/shared.rs +++ b/src/engine/strat_engine/backstore/shared.rs @@ -12,7 +12,7 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::{v1::StratBlockDev, InternalBlockDev, StratSectorSizes}, + blockdev::{InternalBlockDev, StratSectorSizes}, devices::BlockSizes, }, serde_structs::{BaseDevSave, Recordable}, @@ -150,9 +150,9 @@ impl AllocatedAbove { /// A partition of blockdevs in a BlockDevMgr between those in use by /// upper layers and those that are not. -pub struct BlockDevPartition<'a> { - pub(super) used: Vec<(DevUuid, &'a StratBlockDev)>, - pub(super) unused: Vec<(DevUuid, &'a StratBlockDev)>, +pub struct BlockDevPartition<'a, B> { + pub(super) used: Vec<(DevUuid, &'a B)>, + pub(super) unused: Vec<(DevUuid, &'a B)>, } /// A summary of block sizes for a BlockDevMgr, distinguishing between used @@ -162,8 +162,11 @@ pub struct BlockSizeSummary { pub(super) unused: HashMap>, } -impl<'a> From> for BlockSizeSummary { - fn from(pair: BlockDevPartition<'a>) -> BlockSizeSummary { +impl<'a, B> From> for BlockSizeSummary +where + B: InternalBlockDev, +{ + fn from(pair: BlockDevPartition<'a, B>) -> BlockSizeSummary { let mut used = HashMap::new(); for (u, bd) in pair.used { used.entry(bd.blksizes()) diff --git a/src/engine/strat_engine/liminal/device_info.rs b/src/engine/strat_engine/liminal/device_info.rs index f64e2abb99..84d86b9c1d 100644 --- a/src/engine/strat_engine/liminal/device_info.rs +++ b/src/engine/strat_engine/liminal/device_info.rs @@ -18,7 +18,7 @@ use crate::{ engine::{ shared::{gather_encryption_info, gather_pool_name}, strat_engine::{ - backstore::blockdev::{v1::StratBlockDev, InternalBlockDev}, + backstore::blockdev::v1::StratBlockDev, liminal::{ identify::{DeviceInfo, LuksInfo, StratisDevInfo, StratisInfo}, setup::get_name, From 2dba1f2f8a8e9bfc21273d66c1b79670bfcf1d7b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 6 Jul 2023 15:27:32 -0400 Subject: [PATCH 07/32] Diverge legacy and new backstore to reflect layering change This commit adds a linear device in between the thin pool and linear device containing all of the data devices. This will later be used when the cryptsetup layer is moved on top of the cap device. This way, the backing device for the crypt device will never change. Changing the backing device for the dm-crypt device outside of cryptsetup could result in future problems. --- .../strat_engine/backstore/backstore/mod.rs | 49 + .../{backstore.rs => backstore/v1.rs} | 134 +- .../strat_engine/backstore/backstore/v2.rs | 1446 +++++++++++++++++ src/engine/strat_engine/backstore/mod.rs | 7 +- src/engine/strat_engine/crypt/handle/v2.rs | 7 +- src/engine/strat_engine/pool.rs | 3 +- src/engine/strat_engine/thinpool/thinpool.rs | 2 +- 7 files changed, 1562 insertions(+), 86 deletions(-) create mode 100644 src/engine/strat_engine/backstore/backstore/mod.rs rename src/engine/strat_engine/backstore/{backstore.rs => backstore/v1.rs} (97%) create mode 100644 src/engine/strat_engine/backstore/backstore/v2.rs diff --git a/src/engine/strat_engine/backstore/backstore/mod.rs b/src/engine/strat_engine/backstore/backstore/mod.rs new file mode 100644 index 0000000000..ae1f48c226 --- /dev/null +++ b/src/engine/strat_engine/backstore/backstore/mod.rs @@ -0,0 +1,49 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use devicemapper::{Device, Sectors}; + +use crate::{engine::types::PoolUuid, stratis::StratisResult}; + +pub mod v1; +pub mod v2; + +pub trait InternalBackstore { + /// Return the device that this tier is currently using. + /// This may change, depending on whether the backstore is supporting a cache + /// or not. There may be no device if no data has yet been allocated from + /// the backstore. + fn device(&self) -> Option; + + /// The current size of allocated space on the blockdevs in the data tier. + fn datatier_allocated_size(&self) -> Sectors; + + /// The current usable size of all the blockdevs in the data tier. + fn datatier_usable_size(&self) -> Sectors; + + /// The total number of unallocated usable sectors in the + /// backstore. Includes both in the cap but unallocated as well as not yet + /// added to cap. + fn available_in_backstore(&self) -> Sectors; + + /// Satisfy a request for multiple segments. This request must + /// always be satisfied exactly, None is returned if this can not + /// be done. + /// + /// Precondition: self.next <= self.size() + /// Postcondition: self.next <= self.size() + /// + /// Postcondition: forall i, sizes_i == result_i.1. The second value + /// in each pair in the returned vector is therefore redundant, but is + /// retained as a convenience to the caller. + /// Postcondition: + /// forall i, result_i.0 = result_(i - 1).0 + result_(i - 1).1 + /// + /// WARNING: metadata changing event + fn alloc( + &mut self, + pool_uuid: PoolUuid, + sizes: &[Sectors], + ) -> StratisResult>>; +} diff --git a/src/engine/strat_engine/backstore/backstore.rs b/src/engine/strat_engine/backstore/backstore/v1.rs similarity index 97% rename from src/engine/strat_engine/backstore/backstore.rs rename to src/engine/strat_engine/backstore/backstore/v1.rs index f8bec491c9..fdd9ca6db1 100644 --- a/src/engine/strat_engine/backstore/backstore.rs +++ b/src/engine/strat_engine/backstore/backstore/v1.rs @@ -17,6 +17,7 @@ use crate::{ shared::gather_encryption_info, strat_engine::{ backstore::{ + backstore::InternalBackstore, blockdev::{v1::StratBlockDev, InternalBlockDev}, blockdevmgr::BlockDevMgr, cache_tier::CacheTier, @@ -111,6 +112,62 @@ pub struct Backstore { next: Sectors, } +impl InternalBackstore for Backstore { + fn device(&self) -> Option { + self.cache + .as_ref() + .map(|d| d.device()) + .or_else(|| self.linear.as_ref().map(|d| d.device())) + } + + fn datatier_allocated_size(&self) -> Sectors { + self.data_tier.allocated() + } + + fn datatier_usable_size(&self) -> Sectors { + self.data_tier.usable_size() + } + + fn available_in_backstore(&self) -> Sectors { + self.data_tier.usable_size() - self.next + } + + fn alloc( + &mut self, + pool_uuid: PoolUuid, + sizes: &[Sectors], + ) -> StratisResult>> { + let total_required = sizes.iter().cloned().sum(); + if self.available_in_backstore() < total_required { + return Ok(None); + } + + if self.data_tier.alloc(sizes) { + self.extend_cap_device(pool_uuid)?; + } else { + return Ok(None); + } + + let mut chunks = Vec::new(); + for size in sizes { + chunks.push((self.next, *size)); + self.next += *size; + } + + // Assert that the postcondition holds. + assert_eq!( + sizes, + chunks + .iter() + .map(|x| x.1) + .collect::>() + .as_slice() + ); + + Ok(Some(chunks)) + } +} + impl Backstore { /// Make a Backstore object from blockdevs that already belong to Stratis. /// Precondition: every device in datadevs and cachedevs has already been @@ -404,55 +461,6 @@ impl Backstore { Ok(()) } - /// Satisfy a request for multiple segments. This request must - /// always be satisfied exactly, None is returned if this can not - /// be done. - /// - /// Precondition: self.next <= self.size() - /// Postcondition: self.next <= self.size() - /// - /// Postcondition: forall i, sizes_i == result_i.1. The second value - /// in each pair in the returned vector is therefore redundant, but is - /// retained as a convenience to the caller. - /// Postcondition: - /// forall i, result_i.0 = result_(i - 1).0 + result_(i - 1).1 - /// - /// WARNING: metadata changing event - pub fn alloc( - &mut self, - pool_uuid: PoolUuid, - sizes: &[Sectors], - ) -> StratisResult>> { - let total_required = sizes.iter().cloned().sum(); - if self.available_in_backstore() < total_required { - return Ok(None); - } - - if self.data_tier.alloc(sizes) { - self.extend_cap_device(pool_uuid)?; - } else { - return Ok(None); - } - - let mut chunks = Vec::new(); - for size in sizes { - chunks.push((self.next, *size)); - self.next += *size; - } - - // Assert that the postcondition holds. - assert_eq!( - sizes, - chunks - .iter() - .map(|x| x.1) - .collect::>() - .as_slice() - ); - - Ok(Some(chunks)) - } - /// Get only the datadevs in the pool. pub fn datadevs(&self) -> Vec<(DevUuid, &StratBlockDev)> { self.data_tier.blockdevs() @@ -508,16 +516,6 @@ impl Backstore { self.data_tier.size() } - /// The current size of allocated space on the blockdevs in the data tier. - pub fn datatier_allocated_size(&self) -> Sectors { - self.data_tier.allocated() - } - - /// The current usable size of all the blockdevs in the data tier. - pub fn datatier_usable_size(&self) -> Sectors { - self.data_tier.usable_size() - } - /// The size of the cap device. /// /// The size of the cap device is obtained from the size of the component @@ -533,13 +531,6 @@ impl Backstore { .unwrap_or(Sectors(0)) } - /// The total number of unallocated usable sectors in the - /// backstore. Includes both in the cap but unallocated as well as not yet - /// added to cap. - pub fn available_in_backstore(&self) -> Sectors { - self.data_tier.usable_size() - self.next - } - /// Destroy the entire store. pub fn destroy(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { let devs = list_of_backstore_devices(pool_uuid); @@ -587,17 +578,6 @@ impl Backstore { bds } - /// Return the device that this tier is currently using. - /// This changes, depending on whether the backstore is supporting a cache - /// or not. There may be no device if no data has yet been allocated from - /// the backstore. - pub fn device(&self) -> Option { - self.cache - .as_ref() - .map(|d| d.device()) - .or_else(|| self.linear.as_ref().map(|d| d.device())) - } - /// Lookup an immutable blockdev by its Stratis UUID. pub fn get_blockdev_by_uuid(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> { self.data_tier.get_blockdev_by_uuid(uuid).or_else(|| { diff --git a/src/engine/strat_engine/backstore/backstore/v2.rs b/src/engine/strat_engine/backstore/backstore/v2.rs new file mode 100644 index 0000000000..2e0e10f842 --- /dev/null +++ b/src/engine/strat_engine/backstore/backstore/v2.rs @@ -0,0 +1,1446 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Code to handle the backing store of a pool. + +use std::{cmp, collections::HashMap, iter::once, path::PathBuf}; + +use chrono::{DateTime, Utc}; +use serde_json::Value; + +use devicemapper::{ + CacheDev, CacheDevTargetTable, CacheTargetParams, DevId, Device, DmDevice, DmFlags, DmOptions, + LinearDev, LinearDevTargetParams, LinearTargetParams, Sectors, TargetLine, TargetTable, +}; + +use crate::{ + engine::{ + strat_engine::{ + backstore::{ + backstore::InternalBackstore, blockdev::v2::StratBlockDev, + blockdevmgr::BlockDevMgr, cache_tier::CacheTier, data_tier::DataTier, + devices::UnownedDevices, shared::BlockSizeSummary, + }, + crypt::{crypt_metadata_size, handle::v2::CryptHandle, interpret_clevis_config}, + dm::{get_dm, list_of_backstore_devices, remove_optional_devices, DEVICEMAPPER_PATH}, + metadata::{MDADataSize, BDA}, + names::{format_backstore_ids, CacheRole}, + serde_structs::{BackstoreSave, CapSave, Recordable}, + shared::bds_to_bdas, + types::BDARecordResult, + writing::wipe_sectors, + }, + types::{ + ActionAvailability, BlockDevTier, DevUuid, EncryptionInfo, KeyDescription, PoolUuid, + UnlockMethod, + }, + }, + stratis::{StratisError, StratisResult}, +}; + +/// Use a cache block size that the kernel docs indicate is the largest +/// typical size. +const CACHE_BLOCK_SIZE: Sectors = Sectors(2048); // 1024 KiB + +/// Make a DM cache device. If the cache device is being made new, +/// take extra steps to make it clean. +fn make_cache( + pool_uuid: PoolUuid, + cache_tier: &CacheTier, + origin: LinearDev, + cap: Option, + new: bool, +) -> StratisResult { + let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::MetaSub); + let meta = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + cache_tier.meta_segments.map_to_dm(), + )?; + + if new { + // See comment in ThinPool::new() method + wipe_sectors( + meta.devnode(), + Sectors(0), + cmp::min(Sectors(8), meta.size()), + )?; + } + + let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::CacheSub); + let cache = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + cache_tier.cache_segments.map_to_dm(), + )?; + + let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::Cache); + if cap.is_some() { + let dm = get_dm(); + dm.device_suspend( + &DevId::Name(&dm_name), + DmOptions::default().set_flags(DmFlags::DM_SUSPEND), + )?; + let table = CacheDevTargetTable::new( + Sectors(0), + origin.size(), + CacheTargetParams::new( + meta.device(), + cache.device(), + origin.device(), + CACHE_BLOCK_SIZE, + vec!["writethrough".into()], + "default".to_owned(), + Vec::new(), + ), + ); + dm.table_load( + &DevId::Name(&dm_name), + &table.to_raw_table(), + DmOptions::default(), + )?; + dm.device_suspend(&DevId::Name(&dm_name), DmOptions::private())?; + }; + Ok(CacheDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + meta, + cache, + origin, + CACHE_BLOCK_SIZE, + )?) +} + +/// Set up the linear device on top of the data tier that can later be converted to a +/// cache device and serves as a placeholder for the device beneath encryption. +fn make_cap_linear_dev(pool_uuid: PoolUuid, origin: &LinearDev) -> Result { + let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::Cache); + let target = vec![TargetLine::new( + Sectors(0), + origin.size(), + LinearDevTargetParams::Linear(LinearTargetParams::new(origin.device(), Sectors(0))), + )]; + LinearDev::setup(get_dm(), &dm_name, Some(&dm_uuid), target).map_err(StratisError::from) +} + +/// This structure can allocate additional space to the upper layer, but it +/// cannot accept returned space. When it is extended to be able to accept +/// returned space the allocation algorithm will have to be revised. +#[derive(Debug)] +pub struct Backstore { + /// A cache DM Device. + cache: Option, + /// Coordinate handling of blockdevs that back the cache. Optional, since + /// this structure can operate without a cache. + cache_tier: Option>, + /// Coordinates handling of the blockdevs that form the base. + data_tier: DataTier, + /// A linear DM device. + linear: Option, + /// A placeholder device to be converted to cache. + cap_linear: Option, + /// Handle for encryption layer in backstore. + handle: Option, + /// Encryption info for encryption layer. + encryption_info: Option, + /// Index for managing allocation of cap device + next: Sectors, +} + +impl InternalBackstore for Backstore { + fn device(&self) -> Option { + self.handle.as_ref().map(|h| h.device()).or_else(|| { + self.cache + .as_ref() + .map(|d| d.device()) + .or_else(|| self.cap_linear.as_ref().map(|d| d.device())) + }) + } + + fn datatier_allocated_size(&self) -> Sectors { + self.data_tier.allocated() + } + + fn datatier_usable_size(&self) -> Sectors { + self.data_tier.usable_size() + - if self.encryption_info.is_some() { + crypt_metadata_size().sectors() + } else { + Sectors(0) + } + } + + fn available_in_backstore(&self) -> Sectors { + self.data_tier.usable_size() + - self.next + - if self.encryption_info.is_some() { + crypt_metadata_size().sectors() + } else { + Sectors(0) + } + } + + fn alloc( + &mut self, + pool_uuid: PoolUuid, + sizes: &[Sectors], + ) -> StratisResult>> { + let total_required = sizes.iter().cloned().sum(); + if self.available_in_backstore() < total_required { + return Ok(None); + } + + if self.data_tier.alloc(sizes) { + self.extend_cap_device(pool_uuid)?; + } else { + return Ok(None); + } + + let mut chunks = Vec::new(); + for size in sizes { + chunks.push((self.next, *size)); + self.next += *size; + } + + // Assert that the postcondition holds. + assert_eq!( + sizes, + chunks + .iter() + .map(|x| x.1) + .collect::>() + .as_slice() + ); + + Ok(Some(chunks)) + } +} + +impl Backstore { + /// Make a Backstore object from blockdevs that already belong to Stratis. + /// Precondition: every device in datadevs and cachedevs has already been + /// determined to belong to the pool with the specified pool_uuid. + /// + /// Precondition: backstore_save.cap.allocs[0].length <= + /// the sum of the lengths of the segments allocated + /// to the data tier cap device. + /// + /// Precondition: backstore_save.data_segments is not empty. This is a + /// consequence of the fact that metadata is saved by the pool, and if + /// a pool exists, data has been allocated to the cap device. + /// + /// Precondition: + /// * key_description.is_some() -> every StratBlockDev in datadevs has a + /// key description and that key description == key_description + /// * key_description.is_none() -> no StratBlockDev in datadevs has a + /// key description. + /// * no StratBlockDev in cachedevs has a key description + /// + /// Postcondition: + /// self.linear.is_some() XOR self.cache.is_some() + /// self.cache.is_some() <=> self.cache_tier.is_some() + pub fn setup( + pool_uuid: PoolUuid, + backstore_save: &BackstoreSave, + datadevs: Vec, + cachedevs: Vec, + last_update_time: DateTime, + ) -> BDARecordResult { + let block_mgr = BlockDevMgr::new(datadevs, Some(last_update_time)); + let data_tier = DataTier::setup(block_mgr, &backstore_save.data_tier)?; + let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::OriginSub); + let origin = match LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + data_tier.segments.map_to_dm(), + ) { + Ok(origin) => origin, + Err(e) => { + return Err(( + StratisError::from(e), + data_tier + .block_mgr + .into_bdas() + .into_iter() + .chain(bds_to_bdas(cachedevs)) + .collect::>(), + )); + } + }; + + let (cap_linear, cache_tier, cache, origin) = if !cachedevs.is_empty() { + let block_mgr = BlockDevMgr::new(cachedevs, Some(last_update_time)); + match backstore_save.cache_tier { + Some(ref cache_tier_save) => { + let cache_tier = match CacheTier::setup(block_mgr, cache_tier_save) { + Ok(ct) => ct, + Err((e, mut bdas)) => { + bdas.extend(data_tier.block_mgr.into_bdas()); + return Err((e, bdas)); + } + }; + + let cache_device = match make_cache(pool_uuid, &cache_tier, origin, None, false) + { + Ok(cd) => cd, + Err(e) => { + return Err(( + e, + data_tier + .block_mgr + .into_bdas() + .into_iter() + .chain(cache_tier.block_mgr.into_bdas()) + .collect::>(), + )); + } + }; + (None, Some(cache_tier), Some(cache_device), None) + } + None => { + let err_msg = "Cachedevs exist, but cache metadata does not exist"; + return Err(( + StratisError::Msg(err_msg.into()), + data_tier + .block_mgr + .into_bdas() + .into_iter() + .chain(block_mgr.into_bdas()) + .collect::>(), + )); + } + } + } else { + let cap_linear = match make_cap_linear_dev(pool_uuid, &origin) { + Ok(cap) => cap, + Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), + }; + (Some(cap_linear), None, None, Some(origin)) + }; + + let (encryption_info, handle) = { + let handle = match CryptHandle::setup( + &once(DEVICEMAPPER_PATH) + .chain(once( + format_backstore_ids(pool_uuid, CacheRole::Cache) + .0 + .to_string() + .as_str(), + )) + .collect::(), + pool_uuid, + UnlockMethod::Any, + ) { + Ok(opt) => opt, + Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), + }; + match handle { + Some(handle) => (Some(handle.encryption_info().clone()), Some(handle)), + None => (None, None), + } + }; + + Ok(Backstore { + data_tier, + cache_tier, + linear: origin, + cache, + cap_linear, + encryption_info, + handle, + next: backstore_save.cap.allocs[0].1, + }) + } + + /// Initialize a Backstore object, by initializing the specified devs. + /// + /// Immediately after initialization a backstore has no cap device, since + /// no segments are allocated in the data tier. + /// + /// When the backstore is initialized it may be unencrypted, or it may + /// be encrypted only with a kernel keyring and without Clevis information. + /// + /// WARNING: metadata changing event + pub fn initialize( + pool_uuid: PoolUuid, + devices: UnownedDevices, + mda_data_size: MDADataSize, + encryption_info: Option<&EncryptionInfo>, + ) -> StratisResult { + let mut data_tier = DataTier::::new( + BlockDevMgr::::initialize(pool_uuid, devices, mda_data_size)?, + ); + if encryption_info.is_some() && !data_tier.alloc(&[crypt_metadata_size().sectors()]) { + return Err(StratisError::Msg( + "There was not enough space on the device to satisfy the allocation request" + .to_string(), + )); + } + + Ok(Backstore { + data_tier, + cache_tier: None, + linear: None, + cache: None, + cap_linear: None, + handle: None, + encryption_info: encryption_info.cloned(), + next: Sectors(0), + }) + } + + /// Initialize the cache tier and add cachedevs to the backstore. + /// + /// Returns all `DevUuid`s of devices that were added to the cache on initialization. + /// + /// Precondition: Must be invoked only after some space has been allocated + /// from the backstore. This ensures that there is certainly a cap device. + // Precondition: self.cache.is_none() && self.linear.is_some() + // Postcondition: self.cache.is_some() && self.linear.is_none() + pub fn init_cache( + &mut self, + pool_uuid: PoolUuid, + devices: UnownedDevices, + ) -> StratisResult> { + match self.cache_tier { + Some(_) => unreachable!("self.cache.is_none()"), + None => { + // Note that variable length metadata is not stored on the + // cachedevs, so the mda_size can always be the minimum. + // If it is desired to change a cache dev to a data dev, it + // should be removed and then re-added in order to ensure + // that the MDA region is set to the correct size. + let bdm = BlockDevMgr::::initialize( + pool_uuid, + devices, + MDADataSize::default(), + )?; + + let cache_tier = CacheTier::new(bdm)?; + + let linear = self.linear + .take() + .expect("some space has already been allocated from the backstore => (cache_tier.is_none() <=> self.linear.is_some())"); + let cap = self.cap_linear + .take() + .expect("some space has already been allocated from the backstore => (cache_tier.is_none() <=> self.cap_linear.is_some())"); + + let cache = make_cache(pool_uuid, &cache_tier, linear, Some(cap), true)?; + + self.cache = Some(cache); + + let uuids = cache_tier + .block_mgr + .blockdevs() + .iter() + .map(|&(uuid, _)| uuid) + .collect::>(); + + self.cache_tier = Some(cache_tier); + + Ok(uuids) + } + } + } + + /// Add cachedevs to the backstore. + /// + /// If the addition of the cache devs would result in a cache with a + /// cache sub-device size greater than 32 TiB return an error. + /// FIXME: This restriction on the size of the cache sub-device is + /// expected to be removed in subsequent versions. + /// + /// Precondition: Must be invoked only after some space has been allocated + /// from the backstore. This ensures that there is certainly a cap device. + // Precondition: self.linear.is_none() && self.cache.is_some() + // Precondition: self.cache_key_desc has the desired key description + // Precondition: self.cache.is_some() && self.linear.is_none() + pub fn add_cachedevs( + &mut self, + pool_uuid: PoolUuid, + devices: UnownedDevices, + ) -> StratisResult> { + match self.cache_tier { + Some(ref mut cache_tier) => { + let cache_device = self + .cache + .as_mut() + .expect("cache_tier.is_some() <=> self.cache.is_some()"); + let (uuids, (cache_change, meta_change)) = cache_tier.add(pool_uuid, devices)?; + + if cache_change { + let table = cache_tier.cache_segments.map_to_dm(); + cache_device.set_cache_table(get_dm(), table)?; + cache_device.resume(get_dm())?; + } + + // NOTE: currently CacheTier::add() does not ever update the + // meta segments. That means that this code is dead. But, + // when CacheTier::add() is fixed, this code will become live. + if meta_change { + let table = cache_tier.meta_segments.map_to_dm(); + cache_device.set_meta_table(get_dm(), table)?; + cache_device.resume(get_dm())?; + } + + Ok(uuids) + } + None => unreachable!("self.cache.is_some()"), + } + } + + /// Add datadevs to the backstore. The data tier always exists if the + /// backstore exists at all, so there is no need to create it. + pub fn add_datadevs( + &mut self, + pool_uuid: PoolUuid, + devices: UnownedDevices, + ) -> StratisResult> { + self.data_tier.add(pool_uuid, devices) + } + + /// Extend the cap device whether it is a cache or not. Create the DM + /// device if it does not already exist. Return an error if DM + /// operations fail. Use all segments currently allocated in the data tier. + fn extend_cap_device(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { + let create = match ( + self.cache.as_mut(), + self.cap_linear + .as_mut() + .and_then(|c| self.linear.as_mut().map(|l| (c, l))), + self.handle.as_mut(), + ) { + (None, None, None) => true, + (Some(cache), None, Some(handle)) => { + let table = self.data_tier.segments.map_to_dm(); + cache.set_origin_table(get_dm(), table)?; + cache.resume(get_dm())?; + handle.resize(None)?; + false + } + (Some(cache), None, None) => { + let table = self.data_tier.segments.map_to_dm(); + cache.set_origin_table(get_dm(), table)?; + cache.resume(get_dm())?; + false + } + (None, Some((cap, linear)), Some(handle)) => { + let table = self.data_tier.segments.map_to_dm(); + linear.set_table(get_dm(), table)?; + linear.resume(get_dm())?; + let table = vec![TargetLine::new( + Sectors(0), + linear.size(), + LinearDevTargetParams::Linear(LinearTargetParams::new( + linear.device(), + Sectors(0), + )), + )]; + cap.set_table(get_dm(), table)?; + cap.resume(get_dm())?; + handle.resize(None)?; + false + } + (None, Some((cap, linear)), None) => { + let table = self.data_tier.segments.map_to_dm(); + linear.set_table(get_dm(), table)?; + linear.resume(get_dm())?; + let table = vec![TargetLine::new( + Sectors(0), + linear.size(), + LinearDevTargetParams::Linear(LinearTargetParams::new( + linear.device(), + Sectors(0), + )), + )]; + cap.set_table(get_dm(), table)?; + cap.resume(get_dm())?; + false + } + _ => panic!("NOT (self.cache().is_some() AND self.linear.is_some())"), + }; + + if create { + let table = self.data_tier.segments.map_to_dm(); + let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::OriginSub); + let origin = LinearDev::setup(get_dm(), &dm_name, Some(&dm_uuid), table)?; + let cap = make_cap_linear_dev(pool_uuid, &origin)?; + let handle = match self.encryption_info { + Some(ref einfo) => Some(CryptHandle::initialize( + &once(DEVICEMAPPER_PATH) + .chain(once( + format_backstore_ids(pool_uuid, CacheRole::Cache) + .0 + .to_string() + .as_str(), + )) + .collect::(), + pool_uuid, + einfo, + None, + )?), + None => None, + }; + self.linear = Some(origin); + self.cap_linear = Some(cap); + self.handle = handle; + } + + Ok(()) + } + + /// Get only the datadevs in the pool. + pub fn datadevs(&self) -> Vec<(DevUuid, &StratBlockDev)> { + self.data_tier.blockdevs() + } + + /// Get only the cachdevs in the pool. + pub fn cachedevs(&self) -> Vec<(DevUuid, &StratBlockDev)> { + match self.cache_tier { + Some(ref cache) => cache.blockdevs(), + None => Vec::new(), + } + } + + /// Return a reference to all the blockdevs that this pool has ownership + /// of. The blockdevs may be returned in any order. It is unsafe to assume + /// that they are grouped by tier or any other organization. + pub fn blockdevs(&self) -> Vec<(DevUuid, BlockDevTier, &StratBlockDev)> { + self.datadevs() + .into_iter() + .map(|(uuid, dev)| (uuid, BlockDevTier::Data, dev)) + .chain( + self.cachedevs() + .into_iter() + .map(|(uuid, dev)| (uuid, BlockDevTier::Cache, dev)), + ) + .collect() + } + + pub fn blockdevs_mut(&mut self) -> Vec<(DevUuid, BlockDevTier, &mut StratBlockDev)> { + match self.cache_tier { + Some(ref mut cache) => cache + .blockdevs_mut() + .into_iter() + .map(|(uuid, dev)| (uuid, BlockDevTier::Cache, dev)) + .chain( + self.data_tier + .blockdevs_mut() + .into_iter() + .map(|(uuid, dev)| (uuid, BlockDevTier::Data, dev)), + ) + .collect(), + None => self + .data_tier + .blockdevs_mut() + .into_iter() + .map(|(uuid, dev)| (uuid, BlockDevTier::Data, dev)) + .collect(), + } + } + + /// The current size of all the blockdevs in the data tier. + pub fn datatier_size(&self) -> Sectors { + self.data_tier.size() + } + + /// The size of the cap device. + /// + /// The size of the cap device is obtained from the size of the component + /// DM devices. But the devicemapper library stores the data from which + /// the size of each DM device is calculated; the result is computed and + /// no ioctl is required. + #[cfg(test)] + fn size(&self) -> Sectors { + self.linear + .as_ref() + .map(|d| d.size()) + .or_else(|| self.cache.as_ref().map(|d| d.size())) + .unwrap_or(Sectors(0)) + } + + /// Destroy the entire store. + pub fn destroy(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { + if let Some(h) = self.handle.as_mut() { + h.wipe()?; + } + let devs = list_of_backstore_devices(pool_uuid); + remove_optional_devices(devs)?; + if let Some(ref mut cache_tier) = self.cache_tier { + cache_tier.destroy()?; + } + self.data_tier.destroy() + } + + /// Teardown the DM devices in the backstore. + pub fn teardown(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { + let devs = list_of_backstore_devices(pool_uuid); + remove_optional_devices(devs)?; + if let Some(ref mut cache_tier) = self.cache_tier { + cache_tier.block_mgr.teardown()?; + } + self.data_tier.block_mgr.teardown() + } + + /// Consume the backstore and convert it into a set of BDAs representing + /// all data and cache devices. + pub fn into_bdas(self) -> HashMap { + self.data_tier + .block_mgr + .into_bdas() + .into_iter() + .chain( + self.cache_tier + .map(|ct| ct.block_mgr.into_bdas()) + .unwrap_or_default(), + ) + .collect::>() + } + + /// Drain the backstore devices into a set of all data and cache devices. + pub fn drain_bds(&mut self) -> Vec { + let mut bds = self.data_tier.block_mgr.drain_bds(); + bds.extend( + self.cache_tier + .as_mut() + .map(|ct| ct.block_mgr.drain_bds()) + .unwrap_or_default(), + ); + bds + } + + /// Lookup an immutable blockdev by its Stratis UUID. + pub fn get_blockdev_by_uuid(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> { + self.data_tier.get_blockdev_by_uuid(uuid).or_else(|| { + self.cache_tier + .as_ref() + .and_then(|c| c.get_blockdev_by_uuid(uuid)) + }) + } + + /// Lookup a mutable blockdev by its Stratis UUID. + pub fn get_mut_blockdev_by_uuid( + &mut self, + uuid: DevUuid, + ) -> Option<(BlockDevTier, &mut StratBlockDev)> { + let cache_tier = &mut self.cache_tier; + self.data_tier + .get_mut_blockdev_by_uuid(uuid) + .or_else(move || { + cache_tier + .as_mut() + .and_then(|c| c.get_mut_blockdev_by_uuid(uuid)) + }) + } + + /// The number of sectors in the backstore given up to Stratis metadata + /// on devices in the data tier. + pub fn datatier_metadata_size(&self) -> Sectors { + self.data_tier.metadata_size() + } + + /// Write the given data to the data tier's devices. + pub fn save_state(&mut self, metadata: &[u8]) -> StratisResult<()> { + self.data_tier.save_state(metadata) + } + + /// Read the currently saved state from the data tier's devices. + pub fn load_state(&self) -> StratisResult> { + self.data_tier.load_state() + } + + /// Set user info field on the specified blockdev. + /// May return an error if there is no blockdev for the given UUID. + /// + /// * Ok(Some(uuid)) provides the uuid of the changed blockdev + /// * Ok(None) is returned if the blockdev was unchanged + /// * Err(StratisError::Engine(_)) is returned if the UUID + /// does not correspond to a blockdev + pub fn set_blockdev_user_info( + &mut self, + uuid: DevUuid, + user_info: Option<&str>, + ) -> StratisResult> { + self.get_mut_blockdev_by_uuid(uuid).map_or_else( + || { + Err(StratisError::Msg(format!( + "Blockdev with a UUID of {uuid} was not found" + ))) + }, + |(_, b)| { + if b.set_user_info(user_info) { + Ok(Some(uuid)) + } else { + Ok(None) + } + }, + ) + } + + pub fn is_encrypted(&self) -> bool { + self.encryption_info.is_some() + } + + pub fn has_cache(&self) -> bool { + self.cache_tier.is_some() + } + + /// Get the encryption information for the backstore. + pub fn encryption_info(&self) -> Option<&EncryptionInfo> { + self.encryption_info.as_ref() + } + + /// Bind device in the given backstore using the given clevis + /// configuration. + /// + /// * Returns Ok(true) if the binding was performed. + /// * Returns Ok(false) if the binding had already been previously performed and + /// nothing was changed. + /// * Returns Err(_) if binding failed. + pub fn bind_clevis(&mut self, pin: &str, clevis_info: &Value) -> StratisResult { + let handle = self + .handle + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + + let mut parsed_config = clevis_info.clone(); + let yes = interpret_clevis_config(pin, &mut parsed_config)?; + + if let Some((ref existing_pin, ref existing_info)) = handle.encryption_info().clevis_info() + { + // Ignore thumbprint if stratis:tang:trust_url is set in the clevis_info + // config. + let mut config_to_check = existing_info.clone(); + if yes { + if let Value::Object(ref mut ei) = config_to_check { + ei.remove("thp"); + } + } + + if (existing_pin.as_str(), &config_to_check) == (pin, &parsed_config) { + Ok(false) + } else { + Err(StratisError::Msg(format!( + "Block devices have already been bound with pin {existing_pin} and config {existing_info}; \ + requested pin {pin} and config {parsed_config} can't be applied" + ))) + } + } else { + handle.clevis_bind(pin, clevis_info)?; + Ok(true) + } + } + + /// Unbind device in the given backstore from clevis. + /// + /// * Returns Ok(true) if the unbinding was performed. + /// * Returns Ok(false) if the unbinding had already been previously performed and + /// nothing was changed. + /// * Returns Err(_) if unbinding failed. + pub fn unbind_clevis(&mut self) -> StratisResult { + let handle = self + .handle + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + + if handle.encryption_info().clevis_info().is_some() { + handle.clevis_unbind()?; + Ok(true) + } else { + Ok(false) + } + } + + /// Bind device in the given backstore to a passphrase using the + /// given key description. + /// + /// * Returns Ok(true) if the binding was performed. + /// * Returns Ok(false) if the binding had already been previously performed and + /// nothing was changed. + /// * Returns Err(_) if binding failed. + pub fn bind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult { + let handle = self + .handle + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + + if let Some(kd) = handle.encryption_info().key_description() { + if kd == key_desc { + Ok(false) + } else { + Err(StratisError::Msg(format!( + "Block devices have already been bound with key description {}; \ + requested key description {} can't be applied", + kd.as_application_str(), + key_desc.as_application_str(), + ))) + } + } else { + handle.bind_keyring(key_desc)?; + Ok(true) + } + } + + /// Unbind device in the given backstore from the passphrase + /// associated with the key description. + /// + /// * Returns Ok(true) if the unbinding was performed. + /// * Returns Ok(false) if the unbinding had already been previously performed and + /// nothing was changed. + /// * Returns Err(_) if unbinding failed. + pub fn unbind_keyring(&mut self) -> StratisResult { + let handle = self + .handle + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + + if handle.encryption_info().key_description().is_some() { + handle.unbind_keyring()?; + Ok(true) + } else { + // is encrypted and key description is None + Ok(false) + } + } + + /// Change the keyring passphrase associated with device in this pool. + /// + /// Returns: + /// * Ok(None) if the pool is not currently bound to a keyring passphrase. + /// * Ok(Some(true)) if the pool was successfully bound to the new key description. + /// * Ok(Some(false)) if the pool is already bound to this key description. + /// * Err(_) if an operation fails while changing the passphrase. + pub fn rebind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult> { + let handle = self + .handle + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + + if handle.encryption_info().key_description() == Some(key_desc) { + Ok(Some(false)) + } else if handle.encryption_info().key_description().is_some() { + // Keys are not the same but key description is present + handle.rebind_keyring(key_desc)?; + Ok(Some(true)) + } else { + Ok(None) + } + } + + /// Regenerate the Clevis bindings with the block devices in this pool using + /// the same configuration. + /// + /// This method returns StratisResult<()> because the Clevis regen command + /// will always change the metadata when successful. The command is not idempotent + /// so this method will either fail to regenerate the bindings or it will + /// result in a metadata change. + pub fn rebind_clevis(&mut self) -> StratisResult<()> { + let handle = self + .handle + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + + if handle.encryption_info().clevis_info().is_none() { + Err(StratisError::Msg( + "Requested pool is not already bound to Clevis".to_string(), + )) + } else { + handle.rebind_clevis()?; + + Ok(()) + } + } + + pub fn grow(&mut self, dev: DevUuid) -> StratisResult { + self.data_tier.grow(dev) + } + + /// A summary of block sizes + pub fn block_size_summary(&self, tier: BlockDevTier) -> Option { + match tier { + BlockDevTier::Data => Some(self.data_tier.partition_by_use().into()), + BlockDevTier::Cache => self + .cache_tier + .as_ref() + .map(|ct| ct.partition_cache_by_use().into()), + } + } + + /// What the pool's action availability should be + pub fn action_availability(&self) -> ActionAvailability { + let data_tier_bs_summary = self + .block_size_summary(BlockDevTier::Data) + .expect("always exists"); + let cache_tier_bs_summary: Option = + self.block_size_summary(BlockDevTier::Cache); + if let Err(err) = data_tier_bs_summary.validate() { + warn!("Disabling pool changes for this pool: {}", err); + ActionAvailability::NoPoolChanges + } else if let Some(Err(err)) = cache_tier_bs_summary.map(|ct| ct.validate()) { + // NOTE: This condition should be impossible. Since the cache is + // always expanded to include all its devices, and an attempt to add + // more devices than the cache can use causes the devices to be + // rejected, there should be no unused devices in a cache. If, for + // some reason this condition fails, though, NoPoolChanges would + // be the correct state to put the pool in. + warn!("Disabling pool changes for this pool: {}", err); + ActionAvailability::NoPoolChanges + } else { + ActionAvailability::Full + } + } +} + +impl<'a> Into for &'a Backstore { + fn into(self) -> Value { + json!({ + "blockdevs": { + "datadevs": Value::Array( + self.datadevs().into_iter().map(|(_, dev)| { + dev.into() + }).collect() + ), + "cachedevs": Value::Array( + self.cachedevs().into_iter().map(|(_, dev)| { + dev.into() + }).collect() + ), + } + }) + } +} + +impl Recordable for Backstore { + fn record(&self) -> BackstoreSave { + BackstoreSave { + cache_tier: self.cache_tier.as_ref().map(|c| c.record()), + cap: CapSave { + allocs: vec![(Sectors(0), self.next)], + }, + data_tier: self.data_tier.record(), + } + } +} + +#[cfg(test)] +mod tests { + use std::{env, fs::OpenOptions, path::Path}; + + use devicemapper::{CacheDevStatus, DataBlocks, DmOptions, IEC}; + + use crate::engine::strat_engine::{ + backstore::devices::{ProcessedPathInfos, UnownedDevices}, + cmd, + crypt::crypt_metadata_size, + metadata::device_identifiers, + ns::{unshare_mount_namespace, MemoryFilesystem}, + tests::{crypt, loopbacked, real}, + }; + + use super::*; + + const INITIAL_BACKSTORE_ALLOCATION: Sectors = CACHE_BLOCK_SIZE; + + /// Assert some invariants of the backstore + /// * backstore.cache_tier.is_some() <=> backstore.cache.is_some() && + /// backstore.cache_tier.is_some() => backstore.linear.is_none() + /// * backstore's data tier allocated is equal to the size of the cap device + /// * backstore's next index is always less than the size of the cap + /// device + fn invariant(backstore: &Backstore) { + assert!( + (backstore.cache_tier.is_none() && backstore.cache.is_none()) + || (backstore.cache_tier.is_some() + && backstore.cache.is_some() + && backstore.linear.is_none()) + ); + assert_eq!( + backstore.data_tier.allocated(), + match (&backstore.linear, &backstore.cache) { + (None, None) => Sectors(0), + (&None, Some(cache)) => cache.size(), + (Some(linear), &None) => linear.size(), + _ => panic!("impossible; see first assertion"), + } + ); + assert!(backstore.next <= backstore.size()); + + backstore.data_tier.invariant(); + + if let Some(cache_tier) = &backstore.cache_tier { + cache_tier.invariant() + } + } + + fn get_devices(paths: &[&Path]) -> StratisResult { + ProcessedPathInfos::try_from(paths) + .map(|ps| ps.unpack()) + .map(|(sds, uds)| { + sds.error_on_not_empty().unwrap(); + uds + }) + } + + /// Test adding cachedevs to the backstore. + /// When cachedevs are added, cache tier, etc. must exist. + /// Nonetheless, because nothing is written or read, cache usage ought + /// to be 0. Adding some more cachedevs exercises different code path + /// from adding initial cachedevs. + fn test_add_cache_devs(paths: &[&Path]) { + assert!(paths.len() > 3); + + let meta_size = Sectors(IEC::Mi); + + let (initcachepaths, paths) = paths.split_at(1); + let (cachedevpaths, paths) = paths.split_at(1); + let (datadevpaths, initdatapaths) = paths.split_at(1); + + let pool_uuid = PoolUuid::new_v4(); + + let datadevs = get_devices(datadevpaths).unwrap(); + let cachedevs = get_devices(cachedevpaths).unwrap(); + let initdatadevs = get_devices(initdatapaths).unwrap(); + let initcachedevs = get_devices(initcachepaths).unwrap(); + + let mut backstore = + Backstore::initialize(pool_uuid, initdatadevs, MDADataSize::default(), None).unwrap(); + + invariant(&backstore); + + // Allocate space from the backstore so that the cap device is made. + backstore + .alloc(pool_uuid, &[INITIAL_BACKSTORE_ALLOCATION]) + .unwrap() + .unwrap(); + + let cache_uuids = backstore.init_cache(pool_uuid, initcachedevs).unwrap(); + + invariant(&backstore); + + assert_eq!(cache_uuids.len(), initcachepaths.len()); + assert_matches!(backstore.linear, None); + + let cache_status = backstore + .cache + .as_ref() + .map(|c| c.status(get_dm(), DmOptions::default()).unwrap()) + .unwrap(); + + match cache_status { + CacheDevStatus::Working(status) => { + let usage = &status.usage; + assert_eq!(usage.used_cache, DataBlocks(0)); + assert_eq!(usage.total_meta, meta_size.metablocks()); + assert!(usage.total_cache > DataBlocks(0)); + } + CacheDevStatus::Error => panic!("cache status could not be obtained"), + CacheDevStatus::Fail => panic!("cache is in a failed state"), + } + + let data_uuids = backstore.add_datadevs(pool_uuid, datadevs).unwrap(); + invariant(&backstore); + assert_eq!(data_uuids.len(), datadevpaths.len()); + + let cache_uuids = backstore.add_cachedevs(pool_uuid, cachedevs).unwrap(); + invariant(&backstore); + assert_eq!(cache_uuids.len(), cachedevpaths.len()); + + let cache_status = backstore + .cache + .as_ref() + .map(|c| c.status(get_dm(), DmOptions::default()).unwrap()) + .unwrap(); + + match cache_status { + CacheDevStatus::Working(status) => { + let usage = &status.usage; + assert_eq!(usage.used_cache, DataBlocks(0)); + assert_eq!(usage.total_meta, meta_size.metablocks()); + assert!(usage.total_cache > DataBlocks(0)); + } + CacheDevStatus::Error => panic!("cache status could not be obtained"), + CacheDevStatus::Fail => panic!("cache is in a failed state"), + } + + backstore.destroy(pool_uuid).unwrap(); + } + + #[test] + fn loop_test_add_cache_devs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(4, 5, None), + test_add_cache_devs, + ); + } + + #[test] + fn real_test_add_cache_devs() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(4, None, None), + test_add_cache_devs, + ); + } + + /// Create a backstore. + /// Initialize a cache and verify that there is a new device representing + /// the cache. + fn test_setup(paths: &[&Path]) { + assert!(paths.len() > 1); + + let (paths1, paths2) = paths.split_at(paths.len() / 2); + + let pool_uuid = PoolUuid::new_v4(); + + let devices1 = get_devices(paths1).unwrap(); + let devices2 = get_devices(paths2).unwrap(); + + let mut backstore = + Backstore::initialize(pool_uuid, devices1, MDADataSize::default(), None).unwrap(); + + for path in paths1 { + assert_eq!( + pool_uuid, + device_identifiers(&mut OpenOptions::new().read(true).open(path).unwrap()) + .unwrap() + .unwrap() + .pool_uuid + ); + } + + invariant(&backstore); + + // Allocate space from the backstore so that the cap device is made. + backstore + .alloc(pool_uuid, &[INITIAL_BACKSTORE_ALLOCATION]) + .unwrap() + .unwrap(); + + let old_device = backstore.device(); + + backstore.init_cache(pool_uuid, devices2).unwrap(); + + for path in paths2 { + assert_eq!( + pool_uuid, + device_identifiers(&mut OpenOptions::new().read(true).open(path).unwrap()) + .unwrap() + .unwrap() + .pool_uuid + ); + } + + invariant(&backstore); + + assert_eq!(backstore.device(), old_device); + + backstore.destroy(pool_uuid).unwrap(); + } + + #[test] + fn loop_test_setup() { + loopbacked::test_with_spec(&loopbacked::DeviceLimits::Range(2, 3, None), test_setup); + } + + #[test] + fn real_test_setup() { + real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), test_setup); + } + + fn test_clevis_initialize(paths: &[&Path]) { + unshare_mount_namespace().unwrap(); + let _memfs = MemoryFilesystem::new().unwrap(); + let pool_uuid = PoolUuid::new_v4(); + let mut backstore = Backstore::initialize( + pool_uuid, + get_devices(paths).unwrap(), + MDADataSize::default(), + Some(&EncryptionInfo::ClevisInfo(( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + ))), + ) + .unwrap(); + backstore.alloc(pool_uuid, &[Sectors(512)]).unwrap(); + cmd::udev_settle().unwrap(); + + matches!( + backstore.bind_clevis( + "tang", + &json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}) + ), + Ok(false) + ); + + invariant(&backstore); + } + + #[test] + fn clevis_real_test_initialize() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_clevis_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_real_should_fail_test_initialize() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_clevis_initialize, + ); + } + + #[test] + fn clevis_loop_test_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_clevis_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_loop_should_fail_test_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_clevis_initialize, + ); + } + + fn test_clevis_both_initialize(paths: &[&Path]) { + fn test_both(paths: &[&Path], key_desc: &KeyDescription) { + unshare_mount_namespace().unwrap(); + let _memfs = MemoryFilesystem::new().unwrap(); + let pool_uuid = PoolUuid::new_v4(); + let mut backstore = Backstore::initialize( + pool_uuid, + get_devices(paths).unwrap(), + MDADataSize::default(), + Some(&EncryptionInfo::Both( + key_desc.clone(), + ( + "tang".to_string(), + json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + ), + )), + ).unwrap(); + cmd::udev_settle().unwrap(); + + // Allocate space from the backstore so that the cap device is made. + backstore + .alloc(pool_uuid, &[2u64 * crypt_metadata_size().sectors()]) + .unwrap() + .unwrap(); + + if backstore.bind_clevis( + "tang", + &json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + ).unwrap() { + panic!( + "Clevis bind idempotence test failed" + ); + } + + invariant(&backstore); + + if backstore.bind_keyring(key_desc).unwrap() { + panic!("Keyring bind idempotence test failed") + } + + invariant(&backstore); + + if !backstore.unbind_clevis().unwrap() { + panic!("Clevis unbind test failed"); + } + + invariant(&backstore); + + if backstore.unbind_clevis().unwrap() { + panic!("Clevis unbind idempotence test failed"); + } + + invariant(&backstore); + + if backstore.unbind_keyring().is_ok() { + panic!("Keyring unbind check test failed"); + } + + invariant(&backstore); + + if !backstore.bind_clevis( + "tang", + &json!({"url": env::var("TANG_URL").expect("TANG_URL env var required"), "stratis:tang:trust_url": true}), + ).unwrap() { + panic!( + "Clevis bind test failed" + ); + } + + invariant(&backstore); + + if !backstore.unbind_keyring().unwrap() { + panic!("Keyring unbind test failed"); + } + + invariant(&backstore); + + if backstore.unbind_keyring().unwrap() { + panic!("Keyring unbind idempotence test failed"); + } + + invariant(&backstore); + + if backstore.unbind_clevis().is_ok() { + panic!("Clevis unbind check test failed"); + } + + invariant(&backstore); + + if !backstore.bind_keyring(key_desc).unwrap() { + panic!("Keyring bind test failed"); + } + } + + crypt::insert_and_cleanup_key(paths, test_both); + } + + #[test] + fn clevis_real_test_both_initialize() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_clevis_both_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_real_should_fail_test_both_initialize() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_clevis_both_initialize, + ); + } + + #[test] + fn clevis_loop_test_both_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_clevis_both_initialize, + ); + } + + #[test] + #[should_panic] + fn clevis_loop_should_fail_test_both_initialize() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_clevis_both_initialize, + ); + } +} diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index b766caad2c..d29edef726 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -3,7 +3,7 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. #[allow(clippy::module_inception)] -mod backstore; +pub mod backstore; pub mod blockdev; mod blockdevmgr; mod cache_tier; @@ -14,7 +14,6 @@ mod shared; #[cfg(test)] pub use self::devices::initialize_devices_legacy; -pub use self::{ - backstore::Backstore, - devices::{find_stratis_devs_by_uuid, get_devno_from_path, ProcessedPathInfos, UnownedDevices}, +pub use self::devices::{ + find_stratis_devs_by_uuid, get_devno_from_path, ProcessedPathInfos, UnownedDevices, }; diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs index d7743189af..b55d92fa09 100644 --- a/src/engine/strat_engine/crypt/handle/v2.rs +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -6,6 +6,7 @@ use std::{ fmt::Debug, + iter::once, path::{Path, PathBuf}, }; @@ -187,8 +188,8 @@ pub fn setup_crypt_handle( None => return Ok(None), }; - if !vec![DEVICEMAPPER_PATH, &metadata.activation_name.to_string()] - .into_iter() + if !once(DEVICEMAPPER_PATH) + .chain(once(metadata.activation_name.to_string().as_str())) .collect::() .exists() { @@ -285,7 +286,7 @@ impl CryptHandle { }; let device_path = DevicePath::new(physical_path)?; - let devno = get_devno_from_path(physical_path)?; + let devno = get_devno_from_path(&once(DEVICEMAPPER_PATH).chain(once(activation_name.to_string().as_str())).collect::())?; Ok(CryptHandle::new( device_path, pool_uuid, diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index 275a32f20f..d7506ce5bb 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -19,8 +19,9 @@ use crate::{ }, strat_engine::{ backstore::{ + backstore::{v1::Backstore, InternalBackstore}, blockdev::{v1::StratBlockDev, InternalBlockDev}, - Backstore, ProcessedPathInfos, UnownedDevices, + ProcessedPathInfos, UnownedDevices, }, liminal::DeviceSet, metadata::{MDADataSize, BDA}, diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 93bf3312af..e634400702 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -25,7 +25,7 @@ use crate::{ engine::{ engine::{DumpState, StateDiff}, strat_engine::{ - backstore::Backstore, + backstore::backstore::{v1::Backstore, InternalBackstore}, cmd::{thin_check, thin_metadata_size, thin_repair}, dm::{get_dm, list_of_thin_pool_devices, remove_optional_devices}, names::{ From b2f88a45d9de0822e46c87ddbef6cecb2d2ed93d Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 6 Jul 2023 15:34:01 -0400 Subject: [PATCH 08/32] Abstract thinpool across both backstores --- src/engine/strat_engine/pool.rs | 19 +- src/engine/strat_engine/thinpool/thinpool.rs | 4601 +++++++++++------- 2 files changed, 2837 insertions(+), 1783 deletions(-) diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool.rs index d7506ce5bb..bacb1d1889 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool.rs @@ -158,7 +158,7 @@ fn get_pool_state(info: Option, backstore: &Backstore) -> Ac #[derive(Debug)] pub struct StratPool { backstore: Backstore, - thin_pool: ThinPool, + thin_pool: ThinPool, action_avail: ActionAvailability, metadata_size: Sectors, } @@ -187,7 +187,7 @@ impl StratPool { encryption_info, )?; - let thinpool = ThinPool::new( + let thinpool = ThinPool::::new( pool_uuid, match ThinPoolSizeParams::new(backstore.datatier_usable_size()) { Ok(ref params) => params, @@ -504,13 +504,14 @@ impl<'a> Into for &'a StratPool { // Precondition: (&ThinPool).into() pattern matches Value::Object(_) // Precondition: (&Backstore).into() pattern matches Value::Object(_) fn into(self) -> Value { - let mut map: Map<_, _> = - if let Value::Object(map) = <&ThinPool as Into>::into(&self.thin_pool) { - map.into_iter() - } else { - unreachable!("ThinPool conversion returns a JSON object") - } - .collect(); + let mut map: Map<_, _> = if let Value::Object(map) = + <&ThinPool as Into>::into(&self.thin_pool) + { + map.into_iter() + } else { + unreachable!("ThinPool conversion returns a JSON object") + } + .collect(); map.extend( if let Value::Object(map) = <&Backstore as Into>::into(&self.backstore) { map.into_iter() diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index e634400702..46875e1f7d 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -8,6 +8,7 @@ use std::{ cmp::{max, min, Ordering}, collections::{HashMap, HashSet}, fmt, + marker::PhantomData, thread::scope, }; @@ -25,7 +26,7 @@ use crate::{ engine::{ engine::{DumpState, StateDiff}, strat_engine::{ - backstore::backstore::{v1::Backstore, InternalBackstore}, + backstore::backstore::{v1, v2, InternalBackstore}, cmd::{thin_check, thin_metadata_size, thin_repair}, dm::{get_dm, list_of_thin_pool_devices, remove_optional_devices}, names::{ @@ -290,7 +291,7 @@ fn calc_total_physical_used(data_used: Option, segments: &Segments) -> /// segments for its metadata device, and the filesystems and filesystem /// metadata associated with it. #[derive(Debug)] -pub struct ThinPool { +pub struct ThinPool { thin_pool: ThinPoolDev, segments: Segments, id_gen: ThinDevIdPool, @@ -305,1162 +306,1308 @@ pub struct ThinPool { fs_limit: u64, enable_overprov: bool, out_of_meta_space: bool, + backstore: PhantomData, } -impl ThinPool { - /// Make a new thin pool. - pub fn new( - pool_uuid: PoolUuid, - thin_pool_size: &ThinPoolSizeParams, - data_block_size: Sectors, - backstore: &mut Backstore, - ) -> StratisResult { - let mut segments_list = match backstore.alloc( - pool_uuid, - &[ - thin_pool_size.meta_size(), - thin_pool_size.meta_size(), - thin_pool_size.data_size(), - thin_pool_size.mdv_size(), - ], - )? { - Some(segs) => segs, - None => { - let err_msg = "Could not allocate sufficient space for thinpool devices"; - return Err(StratisError::Msg(err_msg.into())); - } - }; - - let mdv_segments = segments_list.pop().expect("len(segments_list) == 4"); - let data_segments = segments_list.pop().expect("len(segments_list) == 3"); - let spare_segments = segments_list.pop().expect("len(segments_list) == 2"); - let meta_segments = segments_list.pop().expect("len(segments_list) == 1"); - - let backstore_device = backstore.device().expect( - "Space has just been allocated from the backstore, so it must have a cap device", - ); - - // When constructing a thin-pool, Stratis reserves the first N - // sectors on a block device by creating a linear device with a - // starting offset. DM writes the super block in the first block. - // DM requires this first block to be zeros when the meta data for - // the thin-pool is initially created. If we don't zero the - // superblock DM issue error messages because it triggers code paths - // that are trying to re-adopt the device with the attributes that - // have been passed. - let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinMeta); - let meta_dev = LinearDev::setup( - get_dm(), - &dm_name, - Some(&dm_uuid), - segs_to_table(backstore_device, &[meta_segments]), - )?; - - // Wipe the first 4 KiB, i.e. 8 sectors as recommended in kernel DM - // docs: device-mapper/thin-provisioning.txt: Setting up a fresh - // pool device. - wipe_sectors( - meta_dev.devnode(), - Sectors(0), - min(Sectors(8), meta_dev.size()), - )?; - - let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinData); - let data_dev = LinearDev::setup( - get_dm(), - &dm_name, - Some(&dm_uuid), - segs_to_table(backstore_device, &[data_segments]), - )?; - - let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::MetadataVolume); - let mdv_dev = LinearDev::setup( - get_dm(), - &dm_name, - Some(&dm_uuid), - segs_to_table(backstore_device, &[mdv_segments]), - )?; - let mdv = MetadataVol::initialize(pool_uuid, mdv_dev)?; - - let (dm_name, dm_uuid) = format_thinpool_ids(pool_uuid, ThinPoolRole::Pool); - - let data_dev_size = data_dev.size(); - let thinpool_dev = ThinPoolDev::new( - get_dm(), - &dm_name, - Some(&dm_uuid), - meta_dev, - data_dev, - data_block_size, - // Either set the low water mark to the standard low water mark if - // the device is larger than DATA_LOWATER or otherwise to half of the - // capacity of the data device. - min( - DATA_LOWATER, - DataBlocks((data_dev_size / DATA_BLOCK_SIZE) / 2), - ), - vec![ - "no_discard_passdown".to_string(), - "skip_block_zeroing".to_string(), - ], - )?; +impl ThinPool { + /// Get the last cached value for the total amount of space used on the pool. + /// Stratis metadata size will be added a layer about my StratPool. + pub fn total_physical_used(&self) -> Option { + calc_total_physical_used(self.used().map(|(du, _)| du), &self.segments) + } - let thin_pool_status = thinpool_dev.status(get_dm(), DmOptions::default()).ok(); - let segments = Segments { - meta_segments: vec![meta_segments], - meta_spare_segments: vec![spare_segments], - data_segments: vec![data_segments], - mdv_segments: vec![mdv_segments], - }; - Ok(ThinPool { - thin_pool: thinpool_dev, - segments, - id_gen: ThinDevIdPool::new_from_ids(&[]), - filesystems: Table::default(), - mdv, - backstore_device, - thin_pool_status, - allocated_size: backstore.datatier_allocated_size(), - fs_limit: DEFAULT_FS_LIMIT, - enable_overprov: true, - out_of_meta_space: false, - }) + /// Get the last cached value for the total amount of space used on the + /// thin pool in the data and metadata devices. + fn used(&self) -> Option<(Sectors, MetaBlocks)> { + status_to_usage(self.thin_pool_status.as_ref()) + .map(|u| (datablocks_to_sectors(u.used_data), u.used_meta)) } - /// Set up an "existing" thin pool. - /// A thin pool must store the metadata for its thin devices, regardless of - /// whether it has an existing device node. An existing thin pool device - /// is a device where the metadata is already stored on its meta device. - /// If initial setup fails due to a thin_check failure, attempt to fix - /// the problem by running thin_repair. If failure recurs, return an - /// error. - pub fn setup( - pool_name: &str, - pool_uuid: PoolUuid, - thin_pool_save: &ThinPoolDevSave, - flex_devs: &FlexDevsSave, - backstore: &Backstore, - ) -> StratisResult { - let mdv_segments = flex_devs.meta_dev.to_vec(); - let meta_segments = flex_devs.thin_meta_dev.to_vec(); - let data_segments = flex_devs.thin_data_dev.to_vec(); - let spare_segments = flex_devs.thin_meta_dev_spare.to_vec(); + /// Sum the logical size of all filesystems on the pool. + pub fn filesystem_logical_size_sum(&self) -> StratisResult { + Ok(self + .mdv + .filesystems()? + .iter() + .map(|fssave| fssave.size) + .sum()) + } - let backstore_device = backstore.device().expect("When stratisd was running previously, space was allocated from the backstore, so backstore must have a cap device"); + /// Set the current status of the thin_pool device to thin_pool_status. + /// If there has been a change, log that change at the info or warn level + /// as appropriate. + fn set_state(&mut self, thin_pool_status: Option) { + let current_status = self.thin_pool_status.as_ref().map(|s| s.into()); + let new_status: Option = thin_pool_status.as_ref().map(|s| s.into()); - let (thinpool_name, thinpool_uuid) = format_thinpool_ids(pool_uuid, ThinPoolRole::Pool); - let (meta_dev, meta_segments, spare_segments) = setup_metadev( - pool_uuid, - &thinpool_name, - backstore_device, - meta_segments, - spare_segments, - )?; + if current_status != new_status { + let current_status_str = current_status + .map(|x| x.to_string()) + .unwrap_or_else(|| "none".to_string()); - let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinData); - let data_dev = LinearDev::setup( - get_dm(), - &dm_name, - Some(&dm_uuid), - segs_to_table(backstore_device, &data_segments), - )?; + if new_status != Some(ThinPoolStatusDigest::Good) { + warn!( + "Status of thinpool device with \"{}\" changed from \"{}\" to \"{}\"", + thin_pool_identifiers(&self.thin_pool), + current_status_str, + new_status + .map(|s| s.to_string()) + .unwrap_or_else(|| "none".to_string()), + ); + } else { + info!( + "Status of thinpool device with \"{}\" changed from \"{}\" to \"{}\"", + thin_pool_identifiers(&self.thin_pool), + current_status_str, + new_status + .map(|s| s.to_string()) + .unwrap_or_else(|| "none".to_string()), + ); + } + } - // TODO: Remove in stratisd 4.0. - let mut migrate = false; + self.thin_pool_status = thin_pool_status; + } - let data_dev_size = data_dev.size(); - let mut thinpool_dev = ThinPoolDev::setup( - get_dm(), - &thinpool_name, - Some(&thinpool_uuid), - meta_dev, - data_dev, - thin_pool_save.data_block_size, - // This is a larger amount of free space than the actual amount of free - // space currently which will cause the value to be updated when the - // thinpool's check method is invoked. - sectors_to_datablocks(data_dev_size), - thin_pool_save - .feature_args - .as_ref() - .map(|x| x.to_vec()) - .unwrap_or_else(|| { - migrate = true; - vec![ - "no_discard_passdown".to_owned(), - "skip_block_zeroing".to_owned(), - "error_if_no_space".to_owned(), - ] - }), - )?; + /// Tear down the components managed here: filesystems, the MDV, + /// and the actual thinpool device itself. + /// + /// Err(_) contains a tuple with a bool as the second element indicating whether or not + /// there are filesystems that were unable to be torn down. This distinction exists because + /// if filesystems remain, the pool could receive IO and should remain in set up pool data + /// structures. However if all filesystems were torn down, the pool can be moved to + /// the designation of partially constructed pools as no IO can be received on the pool + /// and it has been partially torn down. + pub fn teardown(&mut self, pool_uuid: PoolUuid) -> Result<(), (StratisError, bool)> { + let fs_uuids = self + .filesystems + .iter() + .map(|(_, fs_uuid, _)| *fs_uuid) + .collect::>(); - // TODO: Remove in stratisd 4.0. - if migrate { - thinpool_dev.queue_if_no_space(get_dm())?; + // Must succeed in tearing down all filesystems before the + // thinpool.. + for fs_uuid in fs_uuids { + StratFilesystem::teardown(pool_uuid, fs_uuid).map_err(|e| (e, true))?; + self.filesystems.remove_by_uuid(fs_uuid); } + let devs = list_of_thin_pool_devices(pool_uuid); + remove_optional_devices(devs).map_err(|e| (e, false))?; - let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::MetadataVolume); - let mdv_dev = LinearDev::setup( - get_dm(), - &dm_name, - Some(&dm_uuid), - segs_to_table(backstore_device, &mdv_segments), - )?; - let mdv = MetadataVol::setup(pool_uuid, mdv_dev)?; - let filesystem_metadatas = mdv.filesystems()?; + // ..but MDV has no DM dependencies with the above + self.mdv.teardown(pool_uuid).map_err(|e| (e, false))?; - let filesystems = filesystem_metadatas - .iter() - .filter_map( - |fssave| match StratFilesystem::setup(pool_uuid, &thinpool_dev, fssave) { - Ok(fs) => { - fs.udev_fs_change(pool_name, fssave.uuid, &fssave.name); - Some((Name::new(fssave.name.to_owned()), fssave.uuid, fs)) - }, - Err(err) => { - warn!( - "Filesystem specified by metadata {:?} could not be setup, reason: {:?}", - fssave, - err - ); - None - } - }, - ) - .collect::>(); + Ok(()) + } + + /// Set the pool IO mode to error on writes when out of space. + /// + /// This mode should be enabled when the pool is out of space to allocate to the + /// pool. + fn set_error_mode(&mut self) -> bool { + if !self.out_of_alloc_space() { + if let Err(e) = self.thin_pool.error_if_no_space(get_dm()) { + warn!( + "Could not put thin pool into IO error mode on out of space conditions: {}", + e + ); + false + } else { + true + } + } else { + false + } + } - let mut fs_table = Table::default(); - for (name, uuid, fs) in filesystems { - let evicted = fs_table.insert(name, uuid, fs); - if evicted.is_some() { - let err_msg = "filesystems with duplicate UUID or name specified in metadata"; - return Err(StratisError::Msg(err_msg.into())); + /// Set the pool IO mode to queue writes when out of space. + /// + /// This mode should be enabled when the pool has space to allocate to the pool. + /// This prevents unnecessary IO errors while the pools is being extended and + /// the writes can then be processed after the extension. + pub fn set_queue_mode(&mut self) -> bool { + if self.out_of_alloc_space() { + if let Err(e) = self.thin_pool.queue_if_no_space(get_dm()) { + warn!( + "Could not put thin pool into IO queue mode on out of space conditions: {}", + e + ); + false + } else { + true } + } else { + false } + } - let thin_ids: Vec = filesystem_metadatas.iter().map(|x| x.thin_id).collect(); - let thin_pool_status = thinpool_dev.status(get_dm(), DmOptions::default()).ok(); - let segments = Segments { - meta_segments, - meta_spare_segments: spare_segments, - data_segments, - mdv_segments, - }; + /// Returns true if the pool has run out of available space to allocate. + pub fn out_of_alloc_space(&self) -> bool { + self.thin_pool + .table() + .table + .params + .feature_args + .contains("error_if_no_space") + } - let fs_limit = thin_pool_save.fs_limit.unwrap_or_else(|| { - max(fs_table.len(), convert_const!(DEFAULT_FS_LIMIT, u64, usize)) as u64 - }); + pub fn get_filesystem_by_uuid(&self, uuid: FilesystemUuid) -> Option<(Name, &StratFilesystem)> { + self.filesystems.get_by_uuid(uuid) + } - Ok(ThinPool { - thin_pool: thinpool_dev, - segments, - id_gen: ThinDevIdPool::new_from_ids(&thin_ids), - filesystems: fs_table, - mdv, - backstore_device, - thin_pool_status, - allocated_size: backstore.datatier_allocated_size(), - fs_limit, - enable_overprov: thin_pool_save.enable_overprov.unwrap_or(true), - out_of_meta_space: false, - }) + pub fn get_mut_filesystem_by_uuid( + &mut self, + uuid: FilesystemUuid, + ) -> Option<(Name, &mut StratFilesystem)> { + self.filesystems.get_mut_by_uuid(uuid) } - /// Get the last cached value for the total amount of space used on the pool. - /// Stratis metadata size will be added a layer about my StratPool. - pub fn total_physical_used(&self) -> Option { - calc_total_physical_used(self.used().map(|(du, _)| du), &self.segments) + pub fn get_filesystem_by_name(&self, name: &str) -> Option<(FilesystemUuid, &StratFilesystem)> { + self.filesystems.get_by_name(name) } - /// Get the last cached value for the total amount of space used on the - /// thin pool in the data and metadata devices. - fn used(&self) -> Option<(Sectors, MetaBlocks)> { - status_to_usage(self.thin_pool_status.as_ref()) - .map(|u| (datablocks_to_sectors(u.used_data), u.used_meta)) + pub fn get_mut_filesystem_by_name( + &mut self, + name: &str, + ) -> Option<(FilesystemUuid, &mut StratFilesystem)> { + self.filesystems.get_mut_by_name(name) } - /// Run status checks and take actions on the thinpool and its components. - /// The boolean in the return value indicates if a configuration change requiring a - /// metadata save has been made. - pub fn check( + pub fn has_filesystems(&self) -> bool { + !self.filesystems.is_empty() + } + + pub fn filesystems(&self) -> Vec<(Name, FilesystemUuid, &StratFilesystem)> { + self.filesystems + .iter() + .map(|(name, uuid, x)| (name.clone(), *uuid, x)) + .collect() + } + + pub fn filesystems_mut(&mut self) -> Vec<(Name, FilesystemUuid, &mut StratFilesystem)> { + self.filesystems + .iter_mut() + .map(|(name, uuid, x)| (name.clone(), *uuid, x)) + .collect() + } + + /// Create a filesystem within the thin pool. Given name must not + /// already be in use. + pub fn create_filesystem( &mut self, + pool_name: &str, pool_uuid: PoolUuid, - backstore: &mut Backstore, - ) -> StratisResult<(bool, ThinPoolDiff)> { - assert_eq!( - backstore.device().expect( - "thinpool exists and has been allocated to, so backstore must have a cap device" - ), - self.backstore_device - ); + name: &str, + size: Sectors, + size_limit: Option, + ) -> StratisResult { + if self + .mdv + .filesystems()? + .into_iter() + .map(|fssave| fssave.name) + .collect::>() + .contains(name) + { + return Err(StratisError::Msg(format!( + "Pool {pool_name} already has a record of filesystem name {name}" + ))); + } - let mut should_save: bool = false; + let (fs_uuid, mut new_filesystem) = StratFilesystem::initialize( + pool_uuid, + &self.thin_pool, + size, + size_limit, + self.id_gen.new_id()?, + )?; + let name = Name::new(name.to_owned()); + if let Err(err) = self.mdv.save_fs(&name, fs_uuid, &new_filesystem) { + if let Err(err2) = retry_with_index(Fixed::from_millis(100).take(4), |i| { + trace!( + "Cleanup new filesystem after failed save_fs() attempt {}", + i + ); + new_filesystem.destroy(&self.thin_pool) + }) { + error!( + "When handling failed save_fs(), fs.destroy() failed: {}", + err2 + ) + } + return Err(err); + } + self.filesystems.insert(name, fs_uuid, new_filesystem); + let (name, fs) = self + .filesystems + .get_by_uuid(fs_uuid) + .expect("Inserted above"); + fs.udev_fs_change(pool_name, fs_uuid, &name); - let old_state = self.cached(); + Ok(fs_uuid) + } - // This block will only perform an extension if the check() method - // is being called when block devices have been newly added to the pool or - // the metadata low water mark has been reached. - if !self.out_of_meta_space { - match self.extend_thin_meta_device( - pool_uuid, - backstore, - None, - self.used() - .and_then(|(_, mu)| { - status_to_meta_lowater(self.thin_pool_status.as_ref()) - .map(|ml| self.thin_pool.meta_dev().size().metablocks() - mu < ml) - }) - .unwrap_or(false), - ) { - (changed, Ok(_)) => { - should_save |= changed; + /// Create a filesystem snapshot of the origin. Given origin_uuid + /// must exist. Returns the Uuid of the new filesystem. + pub fn snapshot_filesystem( + &mut self, + pool_name: &str, + pool_uuid: PoolUuid, + origin_uuid: FilesystemUuid, + snapshot_name: &str, + ) -> StratisResult<(FilesystemUuid, &mut StratFilesystem)> { + let snapshot_fs_uuid = FilesystemUuid::new_v4(); + let (snapshot_dm_name, snapshot_dm_uuid) = + format_thin_ids(pool_uuid, ThinRole::Filesystem(snapshot_fs_uuid)); + let snapshot_id = self.id_gen.new_id()?; + let new_filesystem = match self.get_filesystem_by_uuid(origin_uuid) { + Some((fs_name, filesystem)) => filesystem.snapshot( + &self.thin_pool, + snapshot_name, + &snapshot_dm_name, + Some(&snapshot_dm_uuid), + &fs_name, + snapshot_fs_uuid, + snapshot_id, + origin_uuid, + )?, + None => { + return Err(StratisError::Msg( + "snapshot_filesystem failed, filesystem not found".into(), + )); + } + }; + let new_fs_name = Name::new(snapshot_name.to_owned()); + self.mdv + .save_fs(&new_fs_name, snapshot_fs_uuid, &new_filesystem)?; + self.filesystems + .insert(new_fs_name, snapshot_fs_uuid, new_filesystem); + let (new_fs_name, fs) = self + .filesystems + .get_by_uuid(snapshot_fs_uuid) + .expect("Inserted above"); + fs.udev_fs_change(pool_name, snapshot_fs_uuid, &new_fs_name); + Ok(( + snapshot_fs_uuid, + self.filesystems + .get_mut_by_uuid(snapshot_fs_uuid) + .expect("just inserted") + .1, + )) + } + + /// Destroy a filesystem within the thin pool. Destroy metadata associated + /// with the thinpool. If there is a failure to destroy the filesystem, + /// retain it, and return an error. + /// + /// * Ok(Some(uuid)) provides the uuid of the destroyed filesystem + /// * Ok(None) is returned if the filesystem did not exist + /// * Err(_) is returned if the filesystem could not be destroyed + pub fn destroy_filesystem( + &mut self, + pool_name: &str, + uuid: FilesystemUuid, + ) -> StratisResult> { + match self.filesystems.remove_by_uuid(uuid) { + Some((fs_name, mut fs)) => match fs.destroy(&self.thin_pool) { + Ok(_) => { + self.clear_out_of_meta_flag(); + if let Err(err) = self.mdv.rm_fs(uuid) { + error!("Could not remove metadata for fs with UUID {} and name {} belonging to pool {}, reason: {:?}", + uuid, + fs_name, + pool_name, + err); + } + Ok(Some(uuid)) } - (changed, Err(e)) => { - should_save |= changed; - warn!("Device extension failed: {}", e); + Err(err) => { + self.filesystems.insert(fs_name, uuid, fs); + Err(err) } - }; + }, + None => Ok(None), } + } - if let Some((data_usage, _)) = self.used() { - if self.thin_pool.data_dev().size() - data_usage < datablocks_to_sectors(DATA_LOWATER) - && !self.out_of_alloc_space() - { - let amount_allocated = match self.extend_thin_data_device(pool_uuid, backstore) { - (changed, Ok(extend_size)) => { - should_save |= changed; - extend_size - } - (changed, Err(e)) => { - should_save |= changed; - warn!("Device extension failed: {}", e); - Sectors(0) - } - }; - should_save |= amount_allocated != Sectors(0); + #[cfg(test)] + pub fn state(&self) -> Option { + self.thin_pool_status.as_ref().map(|s| s.into()) + } - self.thin_pool.set_low_water_mark(get_dm(), DATA_LOWATER)?; - self.resume()?; + /// Rename a filesystem within the thin pool. + /// + /// * Ok(Some(true)) is returned if the filesystem was successfully renamed. + /// * Ok(Some(false)) is returned if the source and target filesystem names are the same + /// * Ok(None) is returned if the source filesystem name does not exist + /// * An error is returned if the target filesystem name already exists + pub fn rename_filesystem( + &mut self, + pool_name: &str, + uuid: FilesystemUuid, + new_name: &str, + ) -> StratisResult> { + let old_name = rename_filesystem_pre!(self; uuid; new_name); + let new_name = Name::new(new_name.to_owned()); + + let filesystem = self + .filesystems + .remove_by_uuid(uuid) + .expect("Must succeed since self.filesystems.get_by_uuid() returned a value") + .1; + + if let Err(err) = self.mdv.save_fs(&new_name, uuid, &filesystem) { + self.filesystems.insert(old_name, uuid, filesystem); + Err(err) + } else { + self.filesystems.insert(new_name, uuid, filesystem); + let (new_name, fs) = self.filesystems.get_by_uuid(uuid).expect("Inserted above"); + fs.udev_fs_change(pool_name, uuid, &new_name); + Ok(Some(true)) + } + } + + /// The names of DM devices belonging to this pool that may generate events + pub fn get_eventing_dev_names(&self, pool_uuid: PoolUuid) -> Vec { + let mut eventing = vec![ + format_flex_ids(pool_uuid, FlexRole::ThinMeta).0, + format_flex_ids(pool_uuid, FlexRole::ThinData).0, + format_flex_ids(pool_uuid, FlexRole::MetadataVolume).0, + format_thinpool_ids(pool_uuid, ThinPoolRole::Pool).0, + ]; + eventing.extend( + self.filesystems + .iter() + .map(|(_, uuid, _)| format_thin_ids(pool_uuid, ThinRole::Filesystem(*uuid)).0), + ); + eventing + } + + /// Suspend the thinpool + pub fn suspend(&mut self) -> StratisResult<()> { + // thindevs automatically suspended when thinpool is suspended + self.thin_pool.suspend(get_dm(), DmOptions::default())?; + // If MDV suspend fails, resume the thin pool and return the error + if let Err(err) = self.mdv.suspend() { + if let Err(e) = self.thin_pool.resume(get_dm()) { + Err(StratisError::Chained( + "Suspending the MDV failed and MDV suspend clean up action of resuming the thin pool also failed".to_string(), + // NOTE: This should potentially put the pool in maintenance-only + // mode. For now, this will have no effect. + Box::new(StratisError::NoActionRollbackError{ + causal_error: Box::new(err), + rollback_error: Box::new(StratisError::from(e)), + }), + )) + } else { + Err(err) } + } else { + Ok(()) } + } - let new_state = self.dump(backstore); + /// Resume the thinpool + pub fn resume(&mut self) -> StratisResult<()> { + self.mdv.resume()?; + // thindevs automatically resumed here + self.thin_pool.resume(get_dm())?; + Ok(()) + } - Ok((should_save, old_state.diff(&new_state))) + pub fn fs_limit(&self) -> u64 { + self.fs_limit } - /// Sum the logical size of all filesystems on the pool. - pub fn filesystem_logical_size_sum(&self) -> StratisResult { - Ok(self - .mdv - .filesystems()? - .iter() - .map(|fssave| fssave.size) - .sum()) + /// Returns a boolean indicating whether overprovisioning is disabled or not. + pub fn overprov_enabled(&self) -> bool { + self.enable_overprov } - /// Check all filesystems on this thin pool and return which had their sizes - /// extended, if any. This method should not need to handle thin pool status - /// because it never alters the thin pool itself. - pub fn check_fs( - &mut self, + /// Indicate to the pool that it may now have more room for metadata growth. + pub fn clear_out_of_meta_flag(&mut self) { + self.out_of_meta_space = false; + } +} + +impl ThinPool { + /// Make a new thin pool. + pub fn new( pool_uuid: PoolUuid, - backstore: &Backstore, - ) -> StratisResult> { - let mut updated = HashMap::default(); - let mut remaining_space = if !self.enable_overprov { - let sum = self.filesystem_logical_size_sum()?; - Some(Sectors( - room_for_data( - backstore.datatier_usable_size(), - self.thin_pool.meta_dev().size(), - ) - .saturating_sub(*sum), - )) - } else { - None - }; + thin_pool_size: &ThinPoolSizeParams, + data_block_size: Sectors, + backstore: &mut v1::Backstore, + ) -> StratisResult> { + let mut segments_list = backstore + .alloc( + pool_uuid, + &[ + thin_pool_size.meta_size(), + thin_pool_size.meta_size(), + thin_pool_size.data_size(), + thin_pool_size.mdv_size(), + ], + )? + .ok_or_else(|| { + let err_msg = "Could not allocate sufficient space for thinpool devices"; + StratisError::Msg(err_msg.into()) + })?; - scope(|s| { - // This collect is needed to ensure all threads are spawned in - // parallel, not each thread being spawned and immediately joined - // in the next iterator step which would result in sequential - // iteration. - #[allow(clippy::needless_collect)] - let handles = self - .filesystems - .iter_mut() - .filter_map(|(name, uuid, fs)| { - fs.visit_values(remaining_space.as_mut()) - .map(|(mt_pt, extend_size)| (name, *uuid, fs, mt_pt, extend_size)) - }) - .map(|(name, uuid, fs, mt_pt, extend_size)| { - s.spawn(move || -> StratisResult<_> { - let diff = fs.handle_fs_changes(&mt_pt, extend_size)?; - Ok((name, uuid, fs, diff)) - }) - }) - .collect::>(); + let mdv_segments = segments_list.pop().expect("len(segments_list) == 4"); + let data_segments = segments_list.pop().expect("len(segments_list) == 3"); + let spare_segments = segments_list.pop().expect("len(segments_list) == 2"); + let meta_segments = segments_list.pop().expect("len(segments_list) == 1"); - let needs_save = handles - .into_iter() - .filter_map(|h| { - h.join() - .map_err(|_| { - warn!("Failed to get status of filesystem operation"); - }) - .ok() - }) - .fold(Vec::new(), |mut acc, res| { - match res { - Ok((name, uuid, fs, diff)) => { - if diff.size.is_changed() { - acc.push((name, uuid, fs)); - } - if diff.size.is_changed() || diff.used.is_changed() { - updated.insert(uuid, diff); - } - } - Err(e) => { - warn!("Failed to extend filesystem: {}", e); - } - } - acc - }); + let backstore_device = backstore.device().expect( + "Space has just been allocated from the backstore, so it must have a cap device", + ); - let mdv = &self.mdv; - // This collect is needed to ensure all threads are spawned in - // parallel, not each thread being spawned and immediately joined - // in the next iterator step which would result in sequential - // iteration. - #[allow(clippy::needless_collect)] - let handles = needs_save.into_iter() - .map(|(name, uuid, fs)| { - s.spawn(move || { - if let Err(e) = mdv.save_fs(name, uuid, fs) { - error!("Could not save MDV for fs with UUID {} and name {} belonging to pool with UUID {}, reason: {:?}", - uuid, name, pool_uuid, e); - } - }) - }) - .collect::>(); - handles.into_iter().for_each(|h| { - if h.join().is_err() { - warn!("Failed to get status of MDV save"); - } - }); - }); + // When constructing a thin-pool, Stratis reserves the first N + // sectors on a block device by creating a linear device with a + // starting offset. DM writes the super block in the first block. + // DM requires this first block to be zeros when the meta data for + // the thin-pool is initially created. If we don't zero the + // superblock DM issue error messages because it triggers code paths + // that are trying to re-adopt the device with the attributes that + // have been passed. + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinMeta); + let meta_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &[meta_segments]), + )?; - if remaining_space == Some(Sectors(0)) { - warn!( - "Overprovisioning protection must be disabled or more space must be added to the pool to extend the filesystem further" - ); - } + // Wipe the first 4 KiB, i.e. 8 sectors as recommended in kernel DM + // docs: device-mapper/thin-provisioning.txt: Setting up a fresh + // pool device. + wipe_sectors( + meta_dev.devnode(), + Sectors(0), + min(Sectors(8), meta_dev.size()), + )?; - Ok(updated) - } + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinData); + let data_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &[data_segments]), + )?; - /// Set the current status of the thin_pool device to thin_pool_status. - /// If there has been a change, log that change at the info or warn level - /// as appropriate. - fn set_state(&mut self, thin_pool_status: Option) { - let current_status = self.thin_pool_status.as_ref().map(|s| s.into()); - let new_status: Option = thin_pool_status.as_ref().map(|s| s.into()); + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::MetadataVolume); + let mdv_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &[mdv_segments]), + )?; + let mdv = MetadataVol::initialize(pool_uuid, mdv_dev)?; - if current_status != new_status { - let current_status_str = current_status - .map(|x| x.to_string()) - .unwrap_or_else(|| "none".to_string()); + let (dm_name, dm_uuid) = format_thinpool_ids(pool_uuid, ThinPoolRole::Pool); - if new_status != Some(ThinPoolStatusDigest::Good) { - warn!( - "Status of thinpool device with \"{}\" changed from \"{}\" to \"{}\"", - thin_pool_identifiers(&self.thin_pool), - current_status_str, - new_status - .map(|s| s.to_string()) - .unwrap_or_else(|| "none".to_string()), - ); - } else { - info!( - "Status of thinpool device with \"{}\" changed from \"{}\" to \"{}\"", - thin_pool_identifiers(&self.thin_pool), - current_status_str, - new_status - .map(|s| s.to_string()) - .unwrap_or_else(|| "none".to_string()), - ); - } - } + let data_dev_size = data_dev.size(); + let thinpool_dev = ThinPoolDev::new( + get_dm(), + &dm_name, + Some(&dm_uuid), + meta_dev, + data_dev, + data_block_size, + // Either set the low water mark to the standard low water mark if + // the device is larger than DATA_LOWATER or otherwise to half of the + // capacity of the data device. + min( + DATA_LOWATER, + DataBlocks((data_dev_size / DATA_BLOCK_SIZE) / 2), + ), + vec![ + "no_discard_passdown".to_string(), + "skip_block_zeroing".to_string(), + ], + )?; - self.thin_pool_status = thin_pool_status; + let thin_pool_status = thinpool_dev.status(get_dm(), DmOptions::default()).ok(); + let segments = Segments { + meta_segments: vec![meta_segments], + meta_spare_segments: vec![spare_segments], + data_segments: vec![data_segments], + mdv_segments: vec![mdv_segments], + }; + Ok(ThinPool { + thin_pool: thinpool_dev, + segments, + id_gen: ThinDevIdPool::new_from_ids(&[]), + filesystems: Table::default(), + mdv, + backstore_device, + thin_pool_status, + allocated_size: backstore.datatier_allocated_size(), + fs_limit: DEFAULT_FS_LIMIT, + enable_overprov: true, + out_of_meta_space: false, + backstore: PhantomData, + }) } - /// Tear down the components managed here: filesystems, the MDV, - /// and the actual thinpool device itself. - /// - /// Err(_) contains a tuple with a bool as the second element indicating whether or not - /// there are filesystems that were unable to be torn down. This distinction exists because - /// if filesystems remain, the pool could receive IO and should remain in set up pool data - /// structures. However if all filesystems were torn down, the pool can be moved to - /// the designation of partially constructed pools as no IO can be received on the pool - /// and it has been partially torn down. - pub fn teardown(&mut self, pool_uuid: PoolUuid) -> Result<(), (StratisError, bool)> { - let fs_uuids = self - .filesystems - .iter() - .map(|(_, fs_uuid, _)| *fs_uuid) - .collect::>(); - - // Must succeed in tearing down all filesystems before the - // thinpool.. - for fs_uuid in fs_uuids { - StratFilesystem::teardown(pool_uuid, fs_uuid).map_err(|e| (e, true))?; - self.filesystems.remove_by_uuid(fs_uuid); + /// Set the device on all DM devices + pub fn set_device(&mut self, backstore_device: Device) -> StratisResult { + if backstore_device == self.backstore_device { + return Ok(false); } - let devs = list_of_thin_pool_devices(pool_uuid); - remove_optional_devices(devs).map_err(|e| (e, false))?; - // ..but MDV has no DM dependencies with the above - self.mdv.teardown(pool_uuid).map_err(|e| (e, false))?; + let xform_target_line = + |line: &TargetLine| -> TargetLine { + let new_params = match line.params { + LinearDevTargetParams::Linear(ref params) => LinearDevTargetParams::Linear( + LinearTargetParams::new(backstore_device, params.start_offset), + ), + LinearDevTargetParams::Flakey(ref params) => { + let feature_args = params.feature_args.iter().cloned().collect::>(); + LinearDevTargetParams::Flakey(FlakeyTargetParams::new( + backstore_device, + params.start_offset, + params.up_interval, + params.down_interval, + feature_args, + )) + } + }; - Ok(()) - } + TargetLine::new(line.start, line.length, new_params) + }; - /// Set the pool IO mode to error on writes when out of space. - /// - /// This mode should be enabled when the pool is out of space to allocate to the - /// pool. - fn set_error_mode(&mut self) -> bool { - if !self.out_of_alloc_space() { - if let Err(e) = self.thin_pool.error_if_no_space(get_dm()) { - warn!( - "Could not put thin pool into IO error mode on out of space conditions: {}", - e - ); - false - } else { - true - } - } else { - false - } - } + let meta_table = self + .thin_pool + .meta_dev() + .table() + .table + .clone() + .iter() + .map(&xform_target_line) + .collect::>(); - /// Set the pool IO mode to queue writes when out of space. - /// - /// This mode should be enabled when the pool has space to allocate to the pool. - /// This prevents unnecessary IO errors while the pools is being extended and - /// the writes can then be processed after the extension. - pub fn set_queue_mode(&mut self) -> bool { - if self.out_of_alloc_space() { - if let Err(e) = self.thin_pool.queue_if_no_space(get_dm()) { - warn!( - "Could not put thin pool into IO queue mode on out of space conditions: {}", - e - ); - false - } else { - true - } - } else { - false - } - } + let data_table = self + .thin_pool + .data_dev() + .table() + .table + .clone() + .iter() + .map(&xform_target_line) + .collect::>(); - /// Returns true if the pool has run out of available space to allocate. - pub fn out_of_alloc_space(&self) -> bool { - self.thin_pool + let mdv_table = self + .mdv + .device() .table() .table - .params - .feature_args - .contains("error_if_no_space") - } + .clone() + .iter() + .map(&xform_target_line) + .collect::>(); - /// Extend thinpool's data dev. - /// - /// This method returns the extension size as Ok(data_extension). - fn extend_thin_data_device( - &mut self, - pool_uuid: PoolUuid, - backstore: &mut Backstore, - ) -> (bool, StratisResult) { - fn do_extend( - thinpooldev: &mut ThinPoolDev, - backstore: &mut Backstore, - pool_uuid: PoolUuid, - data_existing_segments: &mut Vec<(Sectors, Sectors)>, - data_extend_size: Sectors, - ) -> StratisResult { - info!( - "Attempting to extend thinpool data sub-device belonging to pool {} by {}", - pool_uuid, data_extend_size - ); + self.thin_pool.set_meta_table(get_dm(), meta_table)?; + self.thin_pool.set_data_table(get_dm(), data_table)?; + self.mdv.set_table(mdv_table)?; - let device = backstore - .device() - .expect("If request succeeded, backstore must have cap device."); + self.backstore_device = backstore_device; - let requests = vec![data_extend_size]; - let data_index = 0; - match backstore.alloc(pool_uuid, &requests) { - Ok(Some(backstore_segs)) => { - let data_segment = backstore_segs.get(data_index).cloned(); - let data_segments = - data_segment.map(|seg| coalesce_segs(data_existing_segments, &[seg])); - if let Some(mut ds) = data_segments { - thinpooldev.suspend(get_dm(), DmOptions::default())?; - // Leaves data device suspended - let res = thinpooldev.set_data_table(get_dm(), segs_to_table(device, &ds)); + Ok(true) + } +} + +impl ThinPool { + /// Make a new thin pool. + pub fn new( + pool_uuid: PoolUuid, + thin_pool_size: &ThinPoolSizeParams, + data_block_size: Sectors, + backstore: &mut v2::Backstore, + ) -> StratisResult> { + let mut segments_list = backstore + .alloc( + pool_uuid, + &[ + thin_pool_size.meta_size(), + thin_pool_size.meta_size(), + thin_pool_size.data_size(), + thin_pool_size.mdv_size(), + ], + )? + .ok_or_else(|| { + let err_msg = "Could not allocate sufficient space for thinpool devices"; + StratisError::Msg(err_msg.into()) + })?; - if res.is_ok() { - data_existing_segments.clear(); - data_existing_segments.append(&mut ds); - } + let mdv_segments = segments_list.pop().expect("len(segments_list) == 4"); + let data_segments = segments_list.pop().expect("len(segments_list) == 3"); + let spare_segments = segments_list.pop().expect("len(segments_list) == 2"); + let meta_segments = segments_list.pop().expect("len(segments_list) == 1"); - thinpooldev.resume(get_dm())?; + let backstore_device = backstore.device().expect( + "Space has just been allocated from the backstore, so it must have a cap device", + ); - res?; - } + // When constructing a thin-pool, Stratis reserves the first N + // sectors on a block device by creating a linear device with a + // starting offset. DM writes the super block in the first block. + // DM requires this first block to be zeros when the meta data for + // the thin-pool is initially created. If we don't zero the + // superblock DM issue error messages because it triggers code paths + // that are trying to re-adopt the device with the attributes that + // have been passed. + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinMeta); + let meta_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &[meta_segments]), + )?; - if let Some(seg) = data_segment { - info!( - "Extended thinpool data sub-device belonging to pool with uuid {} by {}", - pool_uuid, seg.1 - ); - } + // Wipe the first 4 KiB, i.e. 8 sectors as recommended in kernel DM + // docs: device-mapper/thin-provisioning.txt: Setting up a fresh + // pool device. + wipe_sectors( + meta_dev.devnode(), + Sectors(0), + min(Sectors(8), meta_dev.size()), + )?; - Ok(data_segment.map(|seg| seg.1).unwrap_or(Sectors(0))) - } - Ok(None) => Ok(Sectors(0)), - Err(err) => { - error!( - "Attempted to extend a thinpool data sub-device belonging to pool with uuid {pool_uuid} but failed with error: {err:?}" - ); - Err(err) - } - } - } + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinData); + let data_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &[data_segments]), + )?; - let available_size = backstore.available_in_backstore(); - let data_ext = min(sectors_to_datablocks(available_size), DATA_ALLOC_SIZE); - if data_ext == DataBlocks(0) { - return ( - self.set_error_mode(), - Err(StratisError::OutOfSpaceError(format!( - "{DATA_ALLOC_SIZE} requested but no space is available" - ))), - ); - } + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::MetadataVolume); + let mdv_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &[mdv_segments]), + )?; + let mdv = MetadataVol::initialize(pool_uuid, mdv_dev)?; - let res = do_extend( - &mut self.thin_pool, - backstore, - pool_uuid, - &mut self.segments.data_segments, - datablocks_to_sectors(data_ext), - ); + let (dm_name, dm_uuid) = format_thinpool_ids(pool_uuid, ThinPoolRole::Pool); - match res { - Ok(Sectors(0)) | Err(_) => (false, res), - Ok(_) => (true, res), - } + let data_dev_size = data_dev.size(); + let thinpool_dev = ThinPoolDev::new( + get_dm(), + &dm_name, + Some(&dm_uuid), + meta_dev, + data_dev, + data_block_size, + // Either set the low water mark to the standard low water mark if + // the device is larger than DATA_LOWATER or otherwise to half of the + // capacity of the data device. + min( + DATA_LOWATER, + DataBlocks((data_dev_size / DATA_BLOCK_SIZE) / 2), + ), + vec![ + "no_discard_passdown".to_string(), + "skip_block_zeroing".to_string(), + ], + )?; + + let thin_pool_status = thinpool_dev.status(get_dm(), DmOptions::default()).ok(); + let segments = Segments { + meta_segments: vec![meta_segments], + meta_spare_segments: vec![spare_segments], + data_segments: vec![data_segments], + mdv_segments: vec![mdv_segments], + }; + Ok(ThinPool { + thin_pool: thinpool_dev, + segments, + id_gen: ThinDevIdPool::new_from_ids(&[]), + filesystems: Table::default(), + mdv, + backstore_device, + thin_pool_status, + allocated_size: backstore.datatier_allocated_size(), + fs_limit: DEFAULT_FS_LIMIT, + enable_overprov: true, + out_of_meta_space: false, + backstore: PhantomData, + }) } +} - /// Extend thinpool's meta dev. - /// - /// If is_lowater is true, it was determined that the low water mark has been - /// crossed for metadata and the device size should be doubled instead of - /// recalculated via thin_metadata_size. - fn extend_thin_meta_device( - &mut self, +impl ThinPool +where + B: 'static + InternalBackstore, +{ + /// Set up an "existing" thin pool. + /// A thin pool must store the metadata for its thin devices, regardless of + /// whether it has an existing device node. An existing thin pool device + /// is a device where the metadata is already stored on its meta device. + /// If initial setup fails due to a thin_check failure, attempt to fix + /// the problem by running thin_repair. If failure recurs, return an + /// error. + pub fn setup( + pool_name: &str, pool_uuid: PoolUuid, - backstore: &mut Backstore, - new_thin_limit: Option, - is_lowater: bool, - ) -> (bool, StratisResult) { - fn do_extend( - thinpooldev: &mut ThinPoolDev, - backstore: &mut Backstore, - pool_uuid: PoolUuid, - meta_existing_segments: &mut Vec<(Sectors, Sectors)>, - spare_meta_existing_segments: &mut Vec<(Sectors, Sectors)>, - meta_extend_size: Sectors, - ) -> StratisResult { - info!( - "Attempting to extend thinpool meta sub-device belonging to pool {} by {}", - pool_uuid, meta_extend_size - ); - - let device = backstore - .device() - .expect("If request succeeded, backstore must have cap device."); + thin_pool_save: &ThinPoolDevSave, + flex_devs: &FlexDevsSave, + backstore: &B, + ) -> StratisResult> { + let mdv_segments = flex_devs.meta_dev.to_vec(); + let meta_segments = flex_devs.thin_meta_dev.to_vec(); + let data_segments = flex_devs.thin_data_dev.to_vec(); + let spare_segments = flex_devs.thin_meta_dev_spare.to_vec(); - let requests = vec![meta_extend_size, meta_extend_size]; - let meta_index = 0; - let spare_index = 1; - match backstore.alloc(pool_uuid, &requests) { - Ok(Some(backstore_segs)) => { - let meta_and_spare_segment = backstore_segs.get(meta_index).and_then(|seg| { - backstore_segs.get(spare_index).map(|seg_s| (*seg, *seg_s)) - }); - let meta_and_spare_segments = meta_and_spare_segment.map(|(seg, seg_s)| { - ( - coalesce_segs(meta_existing_segments, &[seg]), - coalesce_segs(spare_meta_existing_segments, &[seg_s]), - ) - }); + let backstore_device = backstore.device().expect("When stratisd was running previously, space was allocated from the backstore, so backstore must have a cap device"); - if let Some((mut ms, mut sms)) = meta_and_spare_segments { - thinpooldev.suspend(get_dm(), DmOptions::default())?; + let (thinpool_name, thinpool_uuid) = format_thinpool_ids(pool_uuid, ThinPoolRole::Pool); + let (meta_dev, meta_segments, spare_segments) = setup_metadev( + pool_uuid, + &thinpool_name, + backstore_device, + meta_segments, + spare_segments, + )?; - // Leaves meta device suspended - let res = thinpooldev.set_meta_table(get_dm(), segs_to_table(device, &ms)); + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::ThinData); + let data_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &data_segments), + )?; - if res.is_ok() { - meta_existing_segments.clear(); - meta_existing_segments.append(&mut ms); + // TODO: Remove in stratisd 4.0. + let mut migrate = false; - spare_meta_existing_segments.clear(); - spare_meta_existing_segments.append(&mut sms); - } + let data_dev_size = data_dev.size(); + let mut thinpool_dev = ThinPoolDev::setup( + get_dm(), + &thinpool_name, + Some(&thinpool_uuid), + meta_dev, + data_dev, + thin_pool_save.data_block_size, + // This is a larger amount of free space than the actual amount of free + // space currently which will cause the value to be updated when the + // thinpool's check method is invoked. + sectors_to_datablocks(data_dev_size), + thin_pool_save + .feature_args + .as_ref() + .map(|hs| hs.to_vec()) + .unwrap_or_else(|| { + migrate = true; + vec![ + "no_discard_passdown".to_owned(), + "skip_block_zeroing".to_owned(), + "error_if_no_space".to_owned(), + ] + }), + )?; - thinpooldev.resume(get_dm())?; + // TODO: Remove in stratisd 4.0. + if migrate { + thinpool_dev.queue_if_no_space(get_dm())?; + } - res?; - } + let (dm_name, dm_uuid) = format_flex_ids(pool_uuid, FlexRole::MetadataVolume); + let mdv_dev = LinearDev::setup( + get_dm(), + &dm_name, + Some(&dm_uuid), + segs_to_table(backstore_device, &mdv_segments), + )?; + let mdv = MetadataVol::setup(pool_uuid, mdv_dev)?; + let filesystem_metadatas = mdv.filesystems()?; - if let Some((seg, _)) = meta_and_spare_segment { - info!( - "Extended thinpool meta sub-device belonging to pool with uuid {} by {}", - pool_uuid, seg.1 + let filesystems = filesystem_metadatas + .iter() + .filter_map( + |fssave| match StratFilesystem::setup(pool_uuid, &thinpool_dev, fssave) { + Ok(fs) => { + fs.udev_fs_change(pool_name, fssave.uuid, &fssave.name); + Some((Name::new(fssave.name.to_owned()), fssave.uuid, fs)) + }, + Err(err) => { + warn!( + "Filesystem specified by metadata {:?} could not be setup, reason: {:?}", + fssave, + err ); + None } - - Ok(meta_and_spare_segment - .map(|(seg, _)| seg.1) - .unwrap_or(Sectors(0))) - } - Ok(None) => Ok(Sectors(0)), - Err(err) => { - error!( - "Attempted to extend a thinpool meta sub-device belonging to pool with uuid {} but failed with error: {:?}", - pool_uuid, - err - ); - Err(err) - } - } - } - - let new_meta_size = if is_lowater { - min( - 2u64 * self.thin_pool.meta_dev().size(), - backstore.datatier_usable_size(), + }, ) - } else { - match thin_metadata_size( - DATA_BLOCK_SIZE, - backstore.datatier_usable_size(), - new_thin_limit.unwrap_or(self.fs_limit), - ) { - Ok(nms) => nms, - Err(e) => return (false, Err(e)), - } - }; - let current_meta_size = self.thin_pool.meta_dev().size(); - let meta_growth = Sectors(new_meta_size.saturating_sub(*current_meta_size)); + .collect::>(); - if !self.overprov_enabled() && meta_growth > Sectors(0) { - let sum = match self.filesystem_logical_size_sum() { - Ok(s) => s, - Err(e) => { - return (false, Err(e)); - } - }; - let total: Sectors = sum + INITIAL_MDV_SIZE + 2u64 * current_meta_size; - match total.cmp(&backstore.datatier_usable_size()) { - Ordering::Less => (), - Ordering::Equal => { - self.out_of_meta_space = true; - return (false, Err(StratisError::Msg( - "Metadata cannot be extended any further without adding more space or enabling overprovisioning; the sum of filesystem sizes is as large as all space not used for metadata".to_string() - ))); - } - Ordering::Greater => { - self.out_of_meta_space = true; - return (false, Err(StratisError::Msg( - "Detected a size of MDV, filesystem sizes and metadata size that is greater than available space in the pool while overprovisioning is disabled; please file a bug report".to_string() - ))); - } + let mut fs_table = Table::default(); + for (name, uuid, fs) in filesystems { + let evicted = fs_table.insert(name, uuid, fs); + if evicted.is_some() { + let err_msg = "filesystems with duplicate UUID or name specified in metadata"; + return Err(StratisError::Msg(err_msg.into())); } } - if 2u64 * meta_growth > backstore.available_in_backstore() { - self.out_of_meta_space = true; - ( - self.set_error_mode(), - Err(StratisError::Msg( - "Not enough unallocated space available on the pool to extend metadata device" - .to_string(), - )), - ) - } else if meta_growth > Sectors(0) { - let ext = do_extend( - &mut self.thin_pool, - backstore, - pool_uuid, - &mut self.segments.meta_segments, - &mut self.segments.meta_spare_segments, - meta_growth, - ); - - (ext.is_ok(), ext) - } else { - (false, Ok(Sectors(0))) - } - } - - pub fn get_filesystem_by_uuid(&self, uuid: FilesystemUuid) -> Option<(Name, &StratFilesystem)> { - self.filesystems.get_by_uuid(uuid) - } + let thin_ids: Vec = filesystem_metadatas.iter().map(|x| x.thin_id).collect(); + let thin_pool_status = thinpool_dev.status(get_dm(), DmOptions::default()).ok(); + let segments = Segments { + meta_segments, + meta_spare_segments: spare_segments, + data_segments, + mdv_segments, + }; - pub fn get_mut_filesystem_by_uuid( - &mut self, - uuid: FilesystemUuid, - ) -> Option<(Name, &mut StratFilesystem)> { - self.filesystems.get_mut_by_uuid(uuid) - } + let fs_limit = thin_pool_save.fs_limit.unwrap_or_else(|| { + max(fs_table.len(), convert_const!(DEFAULT_FS_LIMIT, u64, usize)) as u64 + }); - pub fn get_filesystem_by_name(&self, name: &str) -> Option<(FilesystemUuid, &StratFilesystem)> { - self.filesystems.get_by_name(name) + Ok(ThinPool { + thin_pool: thinpool_dev, + segments, + id_gen: ThinDevIdPool::new_from_ids(&thin_ids), + filesystems: fs_table, + mdv, + backstore_device, + thin_pool_status, + allocated_size: backstore.datatier_allocated_size(), + fs_limit, + enable_overprov: thin_pool_save.enable_overprov.unwrap_or(true), + out_of_meta_space: false, + backstore: PhantomData, + }) } - pub fn get_mut_filesystem_by_name( + /// Run status checks and take actions on the thinpool and its components. + /// The boolean in the return value indicates if a configuration change requiring a + /// metadata save has been made. + pub fn check( &mut self, - name: &str, - ) -> Option<(FilesystemUuid, &mut StratFilesystem)> { - self.filesystems.get_mut_by_name(name) - } - - pub fn has_filesystems(&self) -> bool { - !self.filesystems.is_empty() - } + pool_uuid: PoolUuid, + backstore: &mut B, + ) -> StratisResult<(bool, ThinPoolDiff)> { + assert_eq!( + backstore.device().expect( + "thinpool exists and has been allocated to, so backstore must have a cap device" + ), + self.backstore_device + ); - pub fn filesystems(&self) -> Vec<(Name, FilesystemUuid, &StratFilesystem)> { - self.filesystems - .iter() - .map(|(name, uuid, x)| (name.clone(), *uuid, x)) - .collect() - } + let mut should_save: bool = false; - pub fn filesystems_mut(&mut self) -> Vec<(Name, FilesystemUuid, &mut StratFilesystem)> { - self.filesystems - .iter_mut() - .map(|(name, uuid, x)| (name.clone(), *uuid, x)) - .collect() - } + let old_state = self.cached(); - /// Create a filesystem within the thin pool. Given name must not - /// already be in use. - pub fn create_filesystem( - &mut self, - pool_name: &str, - pool_uuid: PoolUuid, - name: &str, - size: Sectors, - size_limit: Option, - ) -> StratisResult { - if self - .mdv - .filesystems()? - .into_iter() - .map(|fssave| fssave.name) - .collect::>() - .contains(name) - { - return Err(StratisError::Msg(format!( - "Pool {pool_name} already has a record of filesystem name {name}" - ))); + // This block will only perform an extension if the check() method + // is being called when block devices have been newly added to the pool or + // the metadata low water mark has been reached. + if !self.out_of_meta_space { + match self.extend_thin_meta_device( + pool_uuid, + backstore, + None, + self.used() + .and_then(|(_, mu)| { + status_to_meta_lowater(self.thin_pool_status.as_ref()) + .map(|ml| self.thin_pool.meta_dev().size().metablocks() - mu < ml) + }) + .unwrap_or(false), + ) { + (changed, Ok(_)) => { + should_save |= changed; + } + (changed, Err(e)) => { + should_save |= changed; + warn!("Device extension failed: {}", e); + } + }; } - let (fs_uuid, mut new_filesystem) = StratFilesystem::initialize( - pool_uuid, - &self.thin_pool, - size, - size_limit, - self.id_gen.new_id()?, - )?; - let name = Name::new(name.to_owned()); - if let Err(err) = self.mdv.save_fs(&name, fs_uuid, &new_filesystem) { - if let Err(err2) = retry_with_index(Fixed::from_millis(100).take(4), |i| { - trace!( - "Cleanup new filesystem after failed save_fs() attempt {}", - i - ); - new_filesystem.destroy(&self.thin_pool) - }) { - error!( - "When handling failed save_fs(), fs.destroy() failed: {}", - err2 - ) + if let Some((data_usage, _)) = self.used() { + if self.thin_pool.data_dev().size() - data_usage < datablocks_to_sectors(DATA_LOWATER) + && !self.out_of_alloc_space() + { + let amount_allocated = match self.extend_thin_data_device(pool_uuid, backstore) { + (changed, Ok(extend_size)) => { + should_save |= changed; + extend_size + } + (changed, Err(e)) => { + should_save |= changed; + warn!("Device extension failed: {}", e); + Sectors(0) + } + }; + should_save |= amount_allocated != Sectors(0); + + self.thin_pool.set_low_water_mark(get_dm(), DATA_LOWATER)?; + self.resume()?; } - return Err(err); } - self.filesystems.insert(name, fs_uuid, new_filesystem); - let (name, fs) = self - .filesystems - .get_by_uuid(fs_uuid) - .expect("Inserted above"); - fs.udev_fs_change(pool_name, fs_uuid, &name); - Ok(fs_uuid) + let new_state = self.dump(backstore); + + Ok((should_save, old_state.diff(&new_state))) } - /// Create a filesystem snapshot of the origin. Given origin_uuid - /// must exist. Returns the Uuid of the new filesystem. - pub fn snapshot_filesystem( + /// Check all filesystems on this thin pool and return which had their sizes + /// extended, if any. This method should not need to handle thin pool status + /// because it never alters the thin pool itself. + pub fn check_fs( &mut self, - pool_name: &str, pool_uuid: PoolUuid, - origin_uuid: FilesystemUuid, - snapshot_name: &str, - ) -> StratisResult<(FilesystemUuid, &mut StratFilesystem)> { - let snapshot_fs_uuid = FilesystemUuid::new_v4(); - let (snapshot_dm_name, snapshot_dm_uuid) = - format_thin_ids(pool_uuid, ThinRole::Filesystem(snapshot_fs_uuid)); - let snapshot_id = self.id_gen.new_id()?; - let new_filesystem = match self.get_filesystem_by_uuid(origin_uuid) { - Some((fs_name, filesystem)) => filesystem.snapshot( - &self.thin_pool, - snapshot_name, - &snapshot_dm_name, - Some(&snapshot_dm_uuid), - &fs_name, - snapshot_fs_uuid, - snapshot_id, - origin_uuid, - )?, - None => { - return Err(StratisError::Msg( - "snapshot_filesystem failed, filesystem not found".into(), - )); - } + backstore: &B, + ) -> StratisResult> { + let mut updated = HashMap::default(); + let mut remaining_space = if !self.enable_overprov { + let sum = self.filesystem_logical_size_sum()?; + Some(Sectors( + room_for_data( + backstore.datatier_usable_size(), + self.thin_pool.meta_dev().size(), + ) + .saturating_sub(*sum), + )) + } else { + None }; - let new_fs_name = Name::new(snapshot_name.to_owned()); - self.mdv - .save_fs(&new_fs_name, snapshot_fs_uuid, &new_filesystem)?; - self.filesystems - .insert(new_fs_name, snapshot_fs_uuid, new_filesystem); - let (new_fs_name, fs) = self - .filesystems - .get_by_uuid(snapshot_fs_uuid) - .expect("Inserted above"); - fs.udev_fs_change(pool_name, snapshot_fs_uuid, &new_fs_name); - Ok(( - snapshot_fs_uuid, - self.filesystems - .get_mut_by_uuid(snapshot_fs_uuid) - .expect("just inserted") - .1, - )) - } - /// Destroy a filesystem within the thin pool. Destroy metadata associated - /// with the thinpool. If there is a failure to destroy the filesystem, - /// retain it, and return an error. - /// - /// * Ok(Some(uuid)) provides the uuid of the destroyed filesystem - /// * Ok(None) is returned if the filesystem did not exist - /// * Err(_) is returned if the filesystem could not be destroyed - pub fn destroy_filesystem( - &mut self, - pool_name: &str, - uuid: FilesystemUuid, - ) -> StratisResult> { - match self.filesystems.remove_by_uuid(uuid) { - Some((fs_name, mut fs)) => match fs.destroy(&self.thin_pool) { - Ok(_) => { - self.clear_out_of_meta_flag(); - if let Err(err) = self.mdv.rm_fs(uuid) { - error!("Could not remove metadata for fs with UUID {} and name {} belonging to pool {}, reason: {:?}", - uuid, - fs_name, - pool_name, - err); + scope(|s| { + // This collect is needed to ensure all threads are spawned in + // parallel, not each thread being spawned and immediately joined + // in the next iterator step which would result in sequential + // iteration. + #[allow(clippy::needless_collect)] + let handles = self + .filesystems + .iter_mut() + .filter_map(|(name, uuid, fs)| { + fs.visit_values(remaining_space.as_mut()) + .map(|(mt_pt, extend_size)| (name, *uuid, fs, mt_pt, extend_size)) + }) + .map(|(name, uuid, fs, mt_pt, extend_size)| { + s.spawn(move || -> StratisResult<_> { + let diff = fs.handle_fs_changes(&mt_pt, extend_size)?; + Ok((name, uuid, fs, diff)) + }) + }) + .collect::>(); + + let needs_save = handles + .into_iter() + .filter_map(|h| { + h.join() + .map_err(|_| { + warn!("Failed to get status of filesystem operation"); + }) + .ok() + }) + .fold(Vec::new(), |mut acc, res| { + match res { + Ok((name, uuid, fs, diff)) => { + if diff.size.is_changed() { + acc.push((name, uuid, fs)); + } + if diff.size.is_changed() || diff.used.is_changed() { + updated.insert(uuid, diff); + } + } + Err(e) => { + warn!("Failed to extend filesystem: {}", e); + } } - Ok(Some(uuid)) - } - Err(err) => { - self.filesystems.insert(fs_name, uuid, fs); - Err(err) + acc + }); + + let mdv = &self.mdv; + // This collect is needed to ensure all threads are spawned in + // parallel, not each thread being spawned and immediately joined + // in the next iterator step which would result in sequential + // iteration. + #[allow(clippy::needless_collect)] + let handles = needs_save.into_iter() + .map(|(name, uuid, fs)| { + s.spawn(move || { + if let Err(e) = mdv.save_fs(name, uuid, fs) { + error!("Could not save MDV for fs with UUID {} and name {} belonging to pool with UUID {}, reason: {:?}", + uuid, name, pool_uuid, e); + } + }) + }) + .collect::>(); + handles.into_iter().for_each(|h| { + if h.join().is_err() { + warn!("Failed to get status of MDV save"); } - }, - None => Ok(None), + }); + }); + + if remaining_space == Some(Sectors(0)) { + warn!( + "Overprovisioning protection must be disabled or more space must be added to the pool to extend the filesystem further" + ); } - } - #[cfg(test)] - pub fn state(&self) -> Option { - self.thin_pool_status.as_ref().map(|s| s.into()) + Ok(updated) } - /// Rename a filesystem within the thin pool. + /// Extend thinpool's data dev. /// - /// * Ok(Some(true)) is returned if the filesystem was successfully renamed. - /// * Ok(Some(false)) is returned if the source and target filesystem names are the same - /// * Ok(None) is returned if the source filesystem name does not exist - /// * An error is returned if the target filesystem name already exists - pub fn rename_filesystem( + /// This method returns the extension size as Ok(data_extension). + fn extend_thin_data_device( &mut self, - pool_name: &str, - uuid: FilesystemUuid, - new_name: &str, - ) -> StratisResult> { - let old_name = rename_filesystem_pre!(self; uuid; new_name); - let new_name = Name::new(new_name.to_owned()); + pool_uuid: PoolUuid, + backstore: &mut B, + ) -> (bool, StratisResult) { + fn do_extend( + thinpooldev: &mut ThinPoolDev, + backstore: &mut B, + pool_uuid: PoolUuid, + data_existing_segments: &mut Vec<(Sectors, Sectors)>, + data_extend_size: Sectors, + ) -> StratisResult + where + B: InternalBackstore, + { + info!( + "Attempting to extend thinpool data sub-device belonging to pool {} by {}", + pool_uuid, data_extend_size + ); - let filesystem = self - .filesystems - .remove_by_uuid(uuid) - .expect("Must succeed since self.filesystems.get_by_uuid() returned a value") - .1; + let device = backstore + .device() + .expect("If request succeeded, backstore must have cap device."); + + let requests = vec![data_extend_size]; + let data_index = 0; + match backstore.alloc(pool_uuid, &requests) { + Ok(Some(backstore_segs)) => { + let data_segment = backstore_segs.get(data_index).cloned(); + let data_segments = + data_segment.map(|seg| coalesce_segs(data_existing_segments, &[seg])); + if let Some(mut ds) = data_segments { + thinpooldev.suspend(get_dm(), DmOptions::default())?; + // Leaves data device suspended + let res = thinpooldev.set_data_table(get_dm(), segs_to_table(device, &ds)); + + if res.is_ok() { + data_existing_segments.clear(); + data_existing_segments.append(&mut ds); + } + + thinpooldev.resume(get_dm())?; + + res?; + } + + if let Some(seg) = data_segment { + info!( + "Extended thinpool data sub-device belonging to pool with uuid {} by {}", + pool_uuid, seg.1 + ); + } + + Ok(data_segment.map(|seg| seg.1).unwrap_or(Sectors(0))) + } + Ok(None) => Ok(Sectors(0)), + Err(err) => { + error!( + "Attempted to extend a thinpool data sub-device belonging to pool with uuid {pool_uuid} but failed with error: {err:?}" + ); + Err(err) + } + } + } - if let Err(err) = self.mdv.save_fs(&new_name, uuid, &filesystem) { - self.filesystems.insert(old_name, uuid, filesystem); - Err(err) - } else { - self.filesystems.insert(new_name, uuid, filesystem); - let (new_name, fs) = self.filesystems.get_by_uuid(uuid).expect("Inserted above"); - fs.udev_fs_change(pool_name, uuid, &new_name); - Ok(Some(true)) + let available_size = backstore.available_in_backstore(); + let data_ext = min(sectors_to_datablocks(available_size), DATA_ALLOC_SIZE); + if data_ext == DataBlocks(0) { + return ( + self.set_error_mode(), + Err(StratisError::OutOfSpaceError(format!( + "{DATA_ALLOC_SIZE} requested but no space is available" + ))), + ); } - } - /// The names of DM devices belonging to this pool that may generate events - pub fn get_eventing_dev_names(&self, pool_uuid: PoolUuid) -> Vec { - let mut eventing = vec![ - format_flex_ids(pool_uuid, FlexRole::ThinMeta).0, - format_flex_ids(pool_uuid, FlexRole::ThinData).0, - format_flex_ids(pool_uuid, FlexRole::MetadataVolume).0, - format_thinpool_ids(pool_uuid, ThinPoolRole::Pool).0, - ]; - eventing.extend( - self.filesystems - .iter() - .map(|(_, uuid, _)| format_thin_ids(pool_uuid, ThinRole::Filesystem(*uuid)).0), + let res = do_extend( + &mut self.thin_pool, + backstore, + pool_uuid, + &mut self.segments.data_segments, + datablocks_to_sectors(data_ext), ); - eventing - } - /// Suspend the thinpool - pub fn suspend(&mut self) -> StratisResult<()> { - // thindevs automatically suspended when thinpool is suspended - self.thin_pool.suspend(get_dm(), DmOptions::default())?; - // If MDV suspend fails, resume the thin pool and return the error - if let Err(err) = self.mdv.suspend() { - if let Err(e) = self.thin_pool.resume(get_dm()) { - Err(StratisError::Chained( - "Suspending the MDV failed and MDV suspend clean up action of resuming the thin pool also failed".to_string(), - // NOTE: This should potentially put the pool in maintenance-only - // mode. For now, this will have no effect. - Box::new(StratisError::NoActionRollbackError{ - causal_error: Box::new(err), - rollback_error: Box::new(StratisError::from(e)), - }), - )) - } else { - Err(err) - } - } else { - Ok(()) + match res { + Ok(Sectors(0)) | Err(_) => (false, res), + Ok(_) => (true, res), } } - /// Resume the thinpool - pub fn resume(&mut self) -> StratisResult<()> { - self.mdv.resume()?; - // thindevs automatically resumed here - self.thin_pool.resume(get_dm())?; - Ok(()) - } + /// Extend thinpool's meta dev. + /// + /// If is_lowater is true, it was determined that the low water mark has been + /// crossed for metadata and the device size should be doubled instead of + /// recalculated via thin_metadata_size. + fn extend_thin_meta_device( + &mut self, + pool_uuid: PoolUuid, + backstore: &mut B, + new_thin_limit: Option, + is_lowater: bool, + ) -> (bool, StratisResult) { + fn do_extend( + thinpooldev: &mut ThinPoolDev, + backstore: &mut B, + pool_uuid: PoolUuid, + meta_existing_segments: &mut Vec<(Sectors, Sectors)>, + spare_meta_existing_segments: &mut Vec<(Sectors, Sectors)>, + meta_extend_size: Sectors, + ) -> StratisResult + where + B: InternalBackstore, + { + info!( + "Attempting to extend thinpool meta sub-device belonging to pool {} by {}", + pool_uuid, meta_extend_size + ); - /// Set the device on all DM devices - pub fn set_device(&mut self, backstore_device: Device) -> StratisResult { - if backstore_device == self.backstore_device { - return Ok(false); - } + let device = backstore + .device() + .expect("If request succeeded, backstore must have cap device."); - let xform_target_line = - |line: &TargetLine| -> TargetLine { - let new_params = match line.params { - LinearDevTargetParams::Linear(ref params) => LinearDevTargetParams::Linear( - LinearTargetParams::new(backstore_device, params.start_offset), - ), - LinearDevTargetParams::Flakey(ref params) => { - let feature_args = params.feature_args.iter().cloned().collect::>(); - LinearDevTargetParams::Flakey(FlakeyTargetParams::new( - backstore_device, - params.start_offset, - params.up_interval, - params.down_interval, - feature_args, - )) - } - }; + let requests = vec![meta_extend_size, meta_extend_size]; + let meta_index = 0; + let spare_index = 1; + match backstore.alloc(pool_uuid, &requests) { + Ok(Some(backstore_segs)) => { + let meta_and_spare_segment = backstore_segs.get(meta_index).and_then(|seg| { + backstore_segs.get(spare_index).map(|seg_s| (*seg, *seg_s)) + }); + let meta_and_spare_segments = meta_and_spare_segment.map(|(seg, seg_s)| { + ( + coalesce_segs(meta_existing_segments, &[seg]), + coalesce_segs(spare_meta_existing_segments, &[seg_s]), + ) + }); - TargetLine::new(line.start, line.length, new_params) - }; + if let Some((mut ms, mut sms)) = meta_and_spare_segments { + thinpooldev.suspend(get_dm(), DmOptions::default())?; - let meta_table = self - .thin_pool - .meta_dev() - .table() - .table - .clone() - .iter() - .map(&xform_target_line) - .collect::>(); + // Leaves meta device suspended + let res = thinpooldev.set_meta_table(get_dm(), segs_to_table(device, &ms)); - let data_table = self - .thin_pool - .data_dev() - .table() - .table - .clone() - .iter() - .map(&xform_target_line) - .collect::>(); + if res.is_ok() { + meta_existing_segments.clear(); + meta_existing_segments.append(&mut ms); - let mdv_table = self - .mdv - .device() - .table() - .table - .clone() - .iter() - .map(&xform_target_line) - .collect::>(); + spare_meta_existing_segments.clear(); + spare_meta_existing_segments.append(&mut sms); + } - self.thin_pool.set_meta_table(get_dm(), meta_table)?; - self.thin_pool.set_data_table(get_dm(), data_table)?; - self.mdv.set_table(mdv_table)?; + thinpooldev.resume(get_dm())?; - self.backstore_device = backstore_device; + res?; + } - Ok(true) - } + if let Some((seg, _)) = meta_and_spare_segment { + info!( + "Extended thinpool meta sub-device belonging to pool with uuid {} by {}", + pool_uuid, seg.1 + ); + } + + Ok(meta_and_spare_segment + .map(|(seg, _)| seg.1) + .unwrap_or(Sectors(0))) + } + Ok(None) => Ok(Sectors(0)), + Err(err) => { + error!( + "Attempted to extend a thinpool meta sub-device belonging to pool with uuid {} but failed with error: {:?}", + pool_uuid, + err + ); + Err(err) + } + } + } + + let new_meta_size = if is_lowater { + min( + 2u64 * self.thin_pool.meta_dev().size(), + backstore.datatier_usable_size(), + ) + } else { + match thin_metadata_size( + DATA_BLOCK_SIZE, + backstore.datatier_usable_size(), + new_thin_limit.unwrap_or(self.fs_limit), + ) { + Ok(nms) => nms, + Err(e) => return (false, Err(e)), + } + }; + let current_meta_size = self.thin_pool.meta_dev().size(); + let meta_growth = Sectors(new_meta_size.saturating_sub(*current_meta_size)); + + if !self.overprov_enabled() && meta_growth > Sectors(0) { + let sum = match self.filesystem_logical_size_sum() { + Ok(s) => s, + Err(e) => { + return (false, Err(e)); + } + }; + let total: Sectors = sum + INITIAL_MDV_SIZE + 2u64 * current_meta_size; + match total.cmp(&backstore.datatier_usable_size()) { + Ordering::Less => (), + Ordering::Equal => { + self.out_of_meta_space = true; + return (false, Err(StratisError::Msg( + "Metadata cannot be extended any further without adding more space or enabling overprovisioning; the sum of filesystem sizes is as large as all space not used for metadata".to_string() + ))); + } + Ordering::Greater => { + self.out_of_meta_space = true; + return (false, Err(StratisError::Msg( + "Detected a size of MDV, filesystem sizes and metadata size that is greater than available space in the pool while overprovisioning is disabled; please file a bug report".to_string() + ))); + } + } + } + + if 2u64 * meta_growth > backstore.available_in_backstore() { + self.out_of_meta_space = true; + ( + self.set_error_mode(), + Err(StratisError::Msg( + "Not enough unallocated space available on the pool to extend metadata device" + .to_string(), + )), + ) + } else if meta_growth > Sectors(0) { + let ext = do_extend( + &mut self.thin_pool, + backstore, + pool_uuid, + &mut self.segments.meta_segments, + &mut self.segments.meta_spare_segments, + meta_growth, + ); - pub fn fs_limit(&self) -> u64 { - self.fs_limit + (ext.is_ok(), ext) + } else { + (false, Ok(Sectors(0))) + } } pub fn set_fs_limit( &mut self, pool_uuid: PoolUuid, - backstore: &mut Backstore, + backstore: &mut B, new_limit: u64, ) -> (bool, StratisResult<()>) { if self.fs_limit >= new_limit { @@ -1495,25 +1642,16 @@ impl ThinPool { /// Return the limit for total size of all filesystems when overprovisioning /// is disabled. - pub fn total_fs_limit(&self, backstore: &Backstore) -> Sectors { + pub fn total_fs_limit(&self, backstore: &B) -> Sectors { room_for_data( backstore.datatier_usable_size(), self.thin_pool.meta_dev().size(), ) } - /// Returns a boolean indicating whether overprovisioning is disabled or not. - pub fn overprov_enabled(&self) -> bool { - self.enable_overprov - } - /// Set the overprovisioning mode to either enabled or disabled based on the boolean /// provided as an input and return an error if changing this property fails. - pub fn set_overprov_mode( - &mut self, - backstore: &Backstore, - enabled: bool, - ) -> (bool, StratisResult<()>) { + pub fn set_overprov_mode(&mut self, backstore: &B, enabled: bool) -> (bool, StratisResult<()>) { if self.enable_overprov && !enabled { let data_limit = self.total_fs_limit(backstore); @@ -1540,11 +1678,6 @@ impl ThinPool { } } - /// Indicate to the pool that it may now have more room for metadata growth. - pub fn clear_out_of_meta_flag(&mut self) { - self.out_of_meta_space = false; - } - /// Set the filesystem size limit for filesystem with given UUID. pub fn set_fs_size_limit( &mut self, @@ -1583,7 +1716,7 @@ impl ThinPool { } } -impl<'a> Into for &'a ThinPool { +impl<'a, B> Into for &'a ThinPool { fn into(self) -> Value { json!({ "filesystems": Value::Array( @@ -1629,9 +1762,12 @@ impl StateDiff for ThinPoolState { } } -impl<'a> DumpState<'a> for ThinPool { +impl<'a, B> DumpState<'a> for ThinPool +where + B: 'static + InternalBackstore, +{ type State = ThinPoolState; - type DumpInput = &'a Backstore; + type DumpInput = &'a B; fn cached(&self) -> Self::State { ThinPoolState { @@ -1662,13 +1798,13 @@ impl Recordable for Segments { } } -impl Recordable for ThinPool { +impl Recordable for ThinPool { fn record(&self) -> FlexDevsSave { self.segments.record() } } -impl Recordable for ThinPool { +impl Recordable for ThinPool { fn record(&self) -> ThinPoolDevSave { ThinPoolDevSave { data_block_size: self.thin_pool.data_block_size(), @@ -1767,7 +1903,7 @@ mod tests { engine::Filesystem, shared::DEFAULT_THIN_DEV_SIZE, strat_engine::{ - backstore::{ProcessedPathInfos, UnownedDevices}, + backstore::{backstore, ProcessedPathInfos, UnownedDevices}, cmd, metadata::MDADataSize, tests::{loopbacked, real}, @@ -1789,307 +1925,1093 @@ mod tests { }) } - /// Test lazy allocation. - /// Verify that ThinPool::new() succeeds. - /// Verify that the starting size is equal to the calculated initial size params. - /// Verify that check on an empty pool does not increase the allocation size. - /// Create filesystems on the thin pool until the low water mark is passed. - /// Verify that the data and metadata devices have been extended by the calculated - /// increase amount. - /// Verify that the total allocated size is equal to the size of all flex devices - /// added together. - /// Verify that the metadata device is the size equal to the output of - /// thin_metadata_size. - fn test_lazy_allocation(paths: &[&Path]) { - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - - let devices = get_devices(paths).unwrap(); - - let mut backstore = - Backstore::initialize(pool_name, pool_uuid, devices, MDADataSize::default(), None) + mod v1 { + use super::*; + + /// Test lazy allocation. + /// Verify that ThinPool::new() succeeds. + /// Verify that the starting size is equal to the calculated initial size params. + /// Verify that check on an empty pool does not increase the allocation size. + /// Create filesystems on the thin pool until the low water mark is passed. + /// Verify that the data and metadata devices have been extended by the calculated + /// increase amount. + /// Verify that the total allocated size is equal to the size of all flex devices + /// added together. + /// Verify that the metadata device is the size equal to the output of + /// thin_metadata_size. + fn test_lazy_allocation(paths: &[&Path]) { + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + pool_name, + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let size = ThinPoolSizeParams::new(backstore.datatier_usable_size()).unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &size, + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let init_data_size = size.data_size(); + let init_meta_size = size.meta_size(); + let available_on_start = backstore.available_in_backstore(); + + assert_eq!(init_data_size, pool.thin_pool.data_dev().size()); + assert_eq!(init_meta_size, pool.thin_pool.meta_dev().size()); + + // This confirms that the check method does not increase the size until + // the data low water mark is hit. + pool.check(pool_uuid, &mut backstore).unwrap(); + + assert_eq!(init_data_size, pool.thin_pool.data_dev().size()); + assert_eq!(init_meta_size, pool.thin_pool.meta_dev().size()); + + let mut i = 0; + loop { + pool.create_filesystem( + "testpool", + pool_uuid, + format!("testfs{i}").as_str(), + Sectors(2 * IEC::Gi), + None, + ) + .unwrap(); + i += 1; + + let init_used = pool.used().unwrap().0; + let init_size = pool.thin_pool.data_dev().size(); + let (changed, diff) = pool.check(pool_uuid, &mut backstore).unwrap(); + if init_size - init_used < datablocks_to_sectors(DATA_LOWATER) { + assert!(changed); + assert!(diff.allocated_size.is_changed()); + break; + } + } + + assert_eq!( + init_data_size + + datablocks_to_sectors(min( + DATA_ALLOC_SIZE, + sectors_to_datablocks(available_on_start), + )), + pool.thin_pool.data_dev().size(), + ); + assert_eq!( + pool.thin_pool.meta_dev().size(), + thin_metadata_size( + DATA_BLOCK_SIZE, + backstore.datatier_usable_size(), + DEFAULT_FS_LIMIT, + ) + .unwrap() + ); + assert_eq!( + backstore.datatier_allocated_size(), + pool.thin_pool.data_dev().size() + + pool.thin_pool.meta_dev().size() * 2u64 + + pool.mdv.device().size() + ); + } + + #[test] + fn loop_test_lazy_allocation() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, Some(Sectors(10 * IEC::Mi))), + test_lazy_allocation, + ); + } + + #[test] + fn real_test_lazy_allocation() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, Some(Sectors(10 * IEC::Mi)), None), + test_lazy_allocation, + ); + } + + /// Verify that a full pool extends properly when additional space is added. + fn test_full_pool(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); + let (first_path, remaining_paths) = paths.split_at(1); + + let first_devices = get_devices(first_path).unwrap(); + let remaining_devices = get_devices(remaining_paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + Name::new(pool_name.to_string()), + pool_uuid, + first_devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); + + let write_buf = &vec![8u8; BYTES_PER_WRITE].into_boxed_slice(); + let source_tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + { + // to allow mutable borrow of pool + let (_, filesystem) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&filesystem.devnode()), + source_tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + let file_path = source_tmp_dir.path().join("stratis_test.txt"); + let mut f = BufWriter::with_capacity( + convert_test!(IEC::Mi, u64, usize), + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(file_path) + .unwrap(), + ); + // Write the write_buf until the pool is full + loop { + match pool + .thin_pool + .status(get_dm(), DmOptions::default()) + .unwrap() + { + ThinPoolStatus::Working(_) => { + f.write_all(write_buf).unwrap(); + if f.sync_all().is_err() { + break; + } + } + ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), + ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail Expected working."), + } + } + } + match pool + .thin_pool + .status(get_dm(), DmOptions::default()) + .unwrap() + { + ThinPoolStatus::Working(ref status) => { + assert_eq!( + status.summary, + ThinPoolStatusSummary::OutOfSpace, + "Expected full pool" + ); + } + ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), + ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail Expected working/full."), + }; + + // Add block devices to the pool and run check() to extend + backstore + .add_datadevs( + Name::new(pool_name.to_string()), + pool_uuid, + remaining_devices, + None, + ) + .unwrap(); + pool.check(pool_uuid, &mut backstore).unwrap(); + // Verify the pool is back in a Good state + match pool + .thin_pool + .status(get_dm(), DmOptions::default()) + .unwrap() + { + ThinPoolStatus::Working(ref status) => { + assert_eq!( + status.summary, + ThinPoolStatusSummary::Good, + "Expected pool to be restored to good state" + ); + } + ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), + ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail. Expected working/good."), + }; + } + + #[test] + fn loop_test_full_pool() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(2, Some(Bytes::from(IEC::Gi * 2).sectors())), + test_full_pool, + ); + } + + #[test] + fn real_test_full_pool() { + real::test_with_spec( + &real::DeviceLimits::Exactly( + 2, + Some(Bytes::from(IEC::Gi * 2).sectors()), + Some(Bytes::from(IEC::Gi * 4).sectors()), + ), + test_full_pool, + ); + } + + /// Verify a snapshot has the same files and same contents as the origin. + fn test_filesystem_snapshot(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + Name::new(pool_name.to_string()), + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let filesystem_name = "stratis_test_filesystem"; + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + filesystem_name, + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); + + cmd::udev_settle().unwrap(); + + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{filesystem_name}")).exists()); + + let write_buf = &[8u8; SECTOR_SIZE]; + let file_count = 10; + + let source_tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + { + // to allow mutable borrow of pool + let (_, filesystem) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&filesystem.devnode()), + source_tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + for i in 0..file_count { + let file_path = source_tmp_dir.path().join(format!("stratis_test{i}.txt")); + let mut f = BufWriter::with_capacity( + convert_test!(IEC::Mi, u64, usize), + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(file_path) + .unwrap(), + ); + f.write_all(write_buf).unwrap(); + f.sync_all().unwrap(); + } + } + + let snapshot_name = "test_snapshot"; + let (_, snapshot_filesystem) = pool + .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, snapshot_name) + .unwrap(); + + cmd::udev_settle().unwrap(); + + // Assert both symlinks are still present. + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{filesystem_name}")).exists()); + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{snapshot_name}")).exists()); + + let mut read_buf = [0u8; SECTOR_SIZE]; + let snapshot_tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + { + mount( + Some(&snapshot_filesystem.devnode()), + snapshot_tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + for i in 0..file_count { + let file_path = snapshot_tmp_dir.path().join(format!("stratis_test{i}.txt")); + let mut f = OpenOptions::new().read(true).open(file_path).unwrap(); + f.read_exact(&mut read_buf).unwrap(); + assert_eq!(read_buf[0..SECTOR_SIZE], write_buf[0..SECTOR_SIZE]); + } + } + } + + #[test] + fn loop_test_filesystem_snapshot() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_filesystem_snapshot, + ); + } + + #[test] + fn real_test_filesystem_snapshot() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_filesystem_snapshot, + ); + } + + /// Verify that a filesystem rename causes the filesystem metadata to be + /// updated. + fn test_filesystem_rename(paths: &[&Path]) { + let pool_name = Name::new("pool_name".to_string()); + let name1 = "name1"; + let name2 = "name2"; + + let pool_uuid = PoolUuid::new_v4(); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + pool_name, + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let pool_name = "stratis_test_pool"; + let fs_uuid = pool + .create_filesystem(pool_name, pool_uuid, name1, DEFAULT_THIN_DEV_SIZE, None) + .unwrap(); + + cmd::udev_settle().unwrap(); + + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{name1}")).exists()); + + let action = pool.rename_filesystem(pool_name, fs_uuid, name2).unwrap(); + + cmd::udev_settle().unwrap(); + + // Check that the symlink has been renamed. + assert!(!Path::new(&format!("/dev/stratis/{pool_name}/{name1}")).exists()); + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{name2}")).exists()); + + assert_eq!(action, Some(true)); + let flexdevs: FlexDevsSave = pool.record(); + let thinpoolsave: ThinPoolDevSave = pool.record(); + + retry_operation!(pool.teardown(pool_uuid)); + + let pool = ThinPool::setup(pool_name, pool_uuid, &thinpoolsave, &flexdevs, &backstore) + .unwrap(); + + assert_eq!(&*pool.get_filesystem_by_uuid(fs_uuid).unwrap().0, name2); + } + + #[test] + fn loop_test_filesystem_rename() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_filesystem_rename, + ); + } + + #[test] + fn real_test_filesystem_rename() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_filesystem_rename, + ); + } + + /// Verify that setting up a pool when the pool has not been previously torn + /// down does not fail. Clutter the original pool with a filesystem with + /// some data on it. + fn test_pool_setup(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + Name::new(pool_name.to_string()), + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let fs_uuid = pool + .create_filesystem(pool_name, pool_uuid, "fsname", DEFAULT_THIN_DEV_SIZE, None) + .unwrap(); + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + { + let (_, fs) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + writeln!( + &OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(new_file) + .unwrap(), + "data" + ) + .unwrap(); + } + let thinpooldevsave: ThinPoolDevSave = pool.record(); + + let new_pool = ThinPool::setup( + pool_name, + pool_uuid, + &thinpooldevsave, + &pool.record(), + &backstore, + ) + .unwrap(); + + assert!(new_pool.get_filesystem_by_uuid(fs_uuid).is_some()); + } + + #[test] + fn loop_test_pool_setup() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_pool_setup, + ); + } + + #[test] + fn real_test_pool_setup() { + real::test_with_spec(&real::DeviceLimits::AtLeast(1, None, None), test_pool_setup); + } + /// Verify that destroy_filesystems actually deallocates the space + /// from the thinpool, by attempting to reinstantiate it using the + /// same thin id and verifying that it fails. + fn test_thindev_destroy(paths: &[&Path]) { + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + pool_name, + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + let pool_name = "stratis_test_pool"; + let fs_name = "stratis_test_filesystem"; + let fs_uuid = pool + .create_filesystem(pool_name, pool_uuid, fs_name, DEFAULT_THIN_DEV_SIZE, None) .unwrap(); - let size = ThinPoolSizeParams::new(backstore.datatier_usable_size()).unwrap(); - let mut pool = ThinPool::new(pool_uuid, &size, DATA_BLOCK_SIZE, &mut backstore).unwrap(); - let init_data_size = size.data_size(); - let init_meta_size = size.meta_size(); - let available_on_start = backstore.available_in_backstore(); + retry_operation!(pool.destroy_filesystem(pool_name, fs_uuid)); + let flexdevs: FlexDevsSave = pool.record(); + let thinpooldevsave: ThinPoolDevSave = pool.record(); + pool.teardown(pool_uuid).unwrap(); - assert_eq!(init_data_size, pool.thin_pool.data_dev().size()); - assert_eq!(init_meta_size, pool.thin_pool.meta_dev().size()); + // Check that destroyed fs is not present in MDV. If the record + // had been left on the MDV that didn't match a thin_id in the + // thinpool, ::setup() will fail. + let pool = ThinPool::setup( + pool_name, + pool_uuid, + &thinpooldevsave, + &flexdevs, + &backstore, + ) + .unwrap(); + + assert_matches!(pool.get_filesystem_by_uuid(fs_uuid), None); + } + + #[test] + fn loop_test_thindev_destroy() { + // This test requires more than 1 GiB. + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_thindev_destroy, + ); + } + + #[test] + fn real_test_thindev_destroy() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_thindev_destroy, + ); + } - // This confirms that the check method does not increase the size until - // the data low water mark is hit. - pool.check(pool_uuid, &mut backstore).unwrap(); + /// Just suspend and resume the device and make sure it doesn't crash. + /// Suspend twice in succession and then resume twice in succession + /// to check idempotency. + fn test_suspend_resume(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); - assert_eq!(init_data_size, pool.thin_pool.data_dev().size()); - assert_eq!(init_meta_size, pool.thin_pool.meta_dev().size()); + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( + Name::new(pool_name.to_string()), + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); - let mut i = 0; - loop { pool.create_filesystem( - "testpool", + pool_name, pool_uuid, - format!("testfs{i}").as_str(), - Sectors(2 * IEC::Gi), + "stratis_test_filesystem", + DEFAULT_THIN_DEV_SIZE, None, ) .unwrap(); - i += 1; - - let init_used = pool.used().unwrap().0; - let init_size = pool.thin_pool.data_dev().size(); - let (changed, diff) = pool.check(pool_uuid, &mut backstore).unwrap(); - if init_size - init_used < datablocks_to_sectors(DATA_LOWATER) { - assert!(changed); - assert!(diff.allocated_size.is_changed()); - break; - } + + pool.suspend().unwrap(); + pool.suspend().unwrap(); + pool.resume().unwrap(); + pool.resume().unwrap(); } - assert_eq!( - init_data_size - + datablocks_to_sectors(min( - DATA_ALLOC_SIZE, - sectors_to_datablocks(available_on_start), - )), - pool.thin_pool.data_dev().size(), - ); - assert_eq!( - pool.thin_pool.meta_dev().size(), - thin_metadata_size( - DATA_BLOCK_SIZE, - backstore.datatier_usable_size(), - DEFAULT_FS_LIMIT, - ) - .unwrap() - ); - assert_eq!( - backstore.datatier_allocated_size(), - pool.thin_pool.data_dev().size() - + pool.thin_pool.meta_dev().size() * 2u64 - + pool.mdv.device().size() - ); - } + #[test] + fn loop_test_suspend_resume() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_suspend_resume, + ); + } - #[test] - fn loop_test_lazy_allocation() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, Some(Sectors(10 * IEC::Mi))), - test_lazy_allocation, - ); - } + #[test] + fn real_test_suspend_resume() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_suspend_resume, + ); + } - #[test] - fn real_test_lazy_allocation() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(2, Some(Sectors(10 * IEC::Mi)), None), - test_lazy_allocation, - ); - } + /// Set up thinpool and backstore. Set up filesystem and write to it. + /// Add cachedev to backstore, causing cache to be built. + /// Update device on self. Read written bits from filesystem + /// presented on cache device. + fn test_set_device(paths: &[&Path]) { + assert!(paths.len() > 1); - /// Verify that a full pool extends properly when additional space is added. - fn test_full_pool(paths: &[&Path]) { - let pool_name = "pool"; - let pool_uuid = PoolUuid::new_v4(); - let (first_path, remaining_paths) = paths.split_at(1); + let (paths1, paths2) = paths.split_at(paths.len() / 2); - let first_devices = get_devices(first_path).unwrap(); - let remaining_devices = get_devices(remaining_paths).unwrap(); + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); - let mut backstore = Backstore::initialize( - Name::new(pool_name.to_string()), - pool_uuid, - first_devices, - MDADataSize::default(), - None, - ) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); + let devices1 = get_devices(paths1).unwrap(); + let devices = get_devices(paths2).unwrap(); - let fs_uuid = pool - .create_filesystem( - pool_name, + let mut backstore = backstore::v1::Backstore::initialize( + Name::new(pool_name.to_string()), pool_uuid, - "stratis_test_filesystem", - DEFAULT_THIN_DEV_SIZE, + devices, + MDADataSize::default(), None, ) .unwrap(); - - let write_buf = &vec![8u8; BYTES_PER_WRITE].into_boxed_slice(); - let source_tmp_dir = tempfile::Builder::new() - .prefix("stratis_testing") - .tempdir() - .unwrap(); - { - // to allow mutable borrow of pool - let (_, filesystem) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); - mount( - Some(&filesystem.devnode()), - source_tmp_dir.path(), - Some("xfs"), - MsFlags::empty(), - None as Option<&str>, + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, ) .unwrap(); - let file_path = source_tmp_dir.path().join("stratis_test.txt"); - let mut f = BufWriter::with_capacity( - convert_test!(IEC::Mi, u64, usize), + + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + let bytestring = b"some bytes"; + { + let (_, fs) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); OpenOptions::new() .create(true) .truncate(true) .write(true) - .open(file_path) - .unwrap(), + .open(&new_file) + .unwrap() + .write_all(bytestring) + .unwrap(); + } + let filesystem_saves = pool.mdv.filesystems().unwrap(); + assert_eq!(filesystem_saves.len(), 1); + assert_eq!( + filesystem_saves + .first() + .expect("filesystem_saves().len == 1") + .uuid, + fs_uuid ); - // Write the write_buf until the pool is full - loop { - match pool - .thin_pool - .status(get_dm(), DmOptions::default()) + + pool.suspend().unwrap(); + let old_device = backstore + .device() + .expect("Space already allocated from backstore, backstore must have device"); + backstore + .init_cache(Name::new(pool_name.to_string()), pool_uuid, devices1, None) + .unwrap(); + let new_device = backstore + .device() + .expect("Space already allocated from backstore, backstore must have device"); + assert_ne!(old_device, new_device); + pool.set_device(new_device).unwrap(); + pool.resume().unwrap(); + + let mut buf = [0u8; 10]; + { + OpenOptions::new() + .read(true) + .open(&new_file) .unwrap() - { - ThinPoolStatus::Working(_) => { - f.write_all(write_buf).unwrap(); - if f.sync_all().is_err() { - break; - } - } - ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), - ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail Expected working."), - } + .read_exact(&mut buf) + .unwrap(); } + assert_eq!(&buf, bytestring); + + let filesystem_saves = pool.mdv.filesystems().unwrap(); + assert_eq!(filesystem_saves.len(), 1); + assert_eq!( + filesystem_saves + .first() + .expect("filesystem_saves().len == 1") + .uuid, + fs_uuid + ); } - match pool - .thin_pool - .status(get_dm(), DmOptions::default()) - .unwrap() - { - ThinPoolStatus::Working(ref status) => { - assert_eq!( - status.summary, - ThinPoolStatusSummary::OutOfSpace, - "Expected full pool" - ); - } - ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), - ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail Expected working/full."), - }; - // Add block devices to the pool and run check() to extend - backstore - .add_datadevs( + #[test] + fn loop_test_set_device() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(3, 4, None), + test_set_device, + ); + } + + #[test] + fn real_test_set_device() { + real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), test_set_device); + } + + /// Set up thinpool and backstore. Set up filesystem and set size limit. + /// Write past the halfway mark of the filesystem and check that the filesystem + /// size limit is respected. Increase the filesystem size limit and check that + /// it is respected. Remove the filesystem size limit and verify that the + /// filesystem size doubles. Verify that the filesystem size limit cannot be set + /// below the current filesystem size. + fn test_fs_size_limit(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v1::Backstore::initialize( Name::new(pool_name.to_string()), pool_uuid, - remaining_devices, + devices, + MDADataSize::default(), None, ) .unwrap(); - pool.check(pool_uuid, &mut backstore).unwrap(); - // Verify the pool is back in a Good state - match pool - .thin_pool - .status(get_dm(), DmOptions::default()) - .unwrap() - { - ThinPoolStatus::Working(ref status) => { - assert_eq!( - status.summary, - ThinPoolStatusSummary::Good, - "Expected pool to be restored to good state" - ); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + Sectors::from(2400 * IEC::Ki), + // 1400 * IEC::Mi + Some(Sectors(2800 * IEC::Ki)), + ) + .unwrap(); + let devnode = { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(Sectors(2800 * IEC::Ki))); + fs.devnode() + }; + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + mount( + Some(&devnode), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(new_file) + .unwrap(); + let mut bytes_written = Bytes(0); + // Write 800 * IEC::Mi + while bytes_written < Bytes::from(800 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); } - ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), - ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail. Expected working/good."), - }; - } + file.sync_all().unwrap(); + pool.check_fs(pool_uuid, &backstore).unwrap(); - #[test] - fn loop_test_full_pool() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(2, Some(Bytes::from(IEC::Gi * 2).sectors())), - test_full_pool, - ); - } + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(fs.size().sectors())); + } - #[test] - fn real_test_full_pool() { - real::test_with_spec( - &real::DeviceLimits::Exactly( - 2, - Some(Bytes::from(IEC::Gi * 2).sectors()), - Some(Bytes::from(IEC::Gi * 4).sectors()), - ), - test_full_pool, - ); + // 1600 * IEC::Mi + pool.set_fs_size_limit(fs_uuid, Some(Sectors(3200 * IEC::Ki))) + .unwrap(); + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); + } + let mut bytes_written = Bytes(0); + // Write 200 * IEC::Mi + while bytes_written < Bytes::from(200 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); + } + file.sync_all().unwrap(); + pool.check_fs(pool_uuid, &backstore).unwrap(); + + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(fs.size().sectors())); + } + + { + let (_, fs) = pool + .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, "snapshot") + .unwrap(); + assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); + } + + pool.set_fs_size_limit(fs_uuid, None).unwrap(); + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), None); + } + let mut bytes_written = Bytes(0); + // Write 400 * IEC::Mi + while bytes_written < Bytes::from(400 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); + } + file.sync_all().unwrap(); + pool.check_fs(pool_uuid, &backstore).unwrap(); + + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size().sectors(), Sectors(6400 * IEC::Ki)); + } + + assert!(pool.set_fs_size_limit(fs_uuid, Some(Sectors(50))).is_err()); + } + + #[test] + fn loop_test_fs_size_limit() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, Some(Sectors(10 * IEC::Mi))), + test_fs_size_limit, + ); + } + + #[test] + fn real_test_fs_size_limit() { + real::test_with_spec( + &real::DeviceLimits::Range(1, 3, Some(Sectors(10 * IEC::Mi)), None), + test_fs_size_limit, + ); + } } - /// Verify a snapshot has the same files and same contents as the origin. - fn test_filesystem_snapshot(paths: &[&Path]) { - let pool_name = "pool"; - let pool_uuid = PoolUuid::new_v4(); + mod v2 { + use super::*; - let devices = get_devices(paths).unwrap(); + /// Test lazy allocation. + /// Verify that ThinPool::new() succeeds. + /// Verify that the starting size is equal to the calculated initial size params. + /// Verify that check on an empty pool does not increase the allocation size. + /// Create filesystems on the thin pool until the low water mark is passed. + /// Verify that the data and metadata devices have been extended by the calculated + /// increase amount. + /// Verify that the total allocated size is equal to the size of all flex devices + /// added together. + /// Verify that the metadata device is the size equal to the output of + /// thin_metadata_size. + fn test_lazy_allocation(paths: &[&Path]) { + let pool_uuid = PoolUuid::new_v4(); - let mut backstore = Backstore::initialize( - Name::new(pool_name.to_string()), - pool_uuid, - devices, - MDADataSize::default(), - None, - ) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); + let devices = get_devices(paths).unwrap(); - let filesystem_name = "stratis_test_filesystem"; - let fs_uuid = pool - .create_filesystem( - pool_name, + let mut backstore = backstore::v2::Backstore::initialize( pool_uuid, - filesystem_name, - DEFAULT_THIN_DEV_SIZE, + devices, + MDADataSize::default(), None, ) .unwrap(); + let size = ThinPoolSizeParams::new(backstore.datatier_usable_size()).unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &size, + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let init_data_size = size.data_size(); + let init_meta_size = size.meta_size(); + let available_on_start = backstore.available_in_backstore(); + + assert_eq!(init_data_size, pool.thin_pool.data_dev().size()); + assert_eq!(init_meta_size, pool.thin_pool.meta_dev().size()); + + // This confirms that the check method does not increase the size until + // the data low water mark is hit. + pool.check(pool_uuid, &mut backstore).unwrap(); + + assert_eq!(init_data_size, pool.thin_pool.data_dev().size()); + assert_eq!(init_meta_size, pool.thin_pool.meta_dev().size()); + + let mut i = 0; + loop { + pool.create_filesystem( + "testpool", + pool_uuid, + format!("testfs{i}").as_str(), + Sectors(2 * IEC::Gi), + None, + ) + .unwrap(); + i += 1; + + let init_used = pool.used().unwrap().0; + let init_size = pool.thin_pool.data_dev().size(); + let (changed, diff) = pool.check(pool_uuid, &mut backstore).unwrap(); + if init_size - init_used < datablocks_to_sectors(DATA_LOWATER) { + assert!(changed); + assert!(diff.allocated_size.is_changed()); + break; + } + } + + assert_eq!( + init_data_size + + datablocks_to_sectors(min( + DATA_ALLOC_SIZE, + sectors_to_datablocks(available_on_start), + )), + pool.thin_pool.data_dev().size(), + ); + assert_eq!( + pool.thin_pool.meta_dev().size(), + thin_metadata_size( + DATA_BLOCK_SIZE, + backstore.datatier_usable_size(), + DEFAULT_FS_LIMIT, + ) + .unwrap() + ); + assert_eq!( + backstore.datatier_allocated_size(), + pool.thin_pool.data_dev().size() + + pool.thin_pool.meta_dev().size() * 2u64 + + pool.mdv.device().size() + ); + } + + #[test] + fn loop_test_lazy_allocation() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, Some(Sectors(10 * IEC::Mi))), + test_lazy_allocation, + ); + } - cmd::udev_settle().unwrap(); + #[test] + fn real_test_lazy_allocation() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, Some(Sectors(10 * IEC::Mi)), None), + test_lazy_allocation, + ); + } - assert!(Path::new(&format!("/dev/stratis/{pool_name}/{filesystem_name}")).exists()); + /// Verify that a full pool extends properly when additional space is added. + fn test_full_pool(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); + let (first_path, remaining_paths) = paths.split_at(1); - let write_buf = &[8u8; SECTOR_SIZE]; - let file_count = 10; + let first_devices = get_devices(first_path).unwrap(); + let remaining_devices = get_devices(remaining_paths).unwrap(); - let source_tmp_dir = tempfile::Builder::new() - .prefix("stratis_testing") - .tempdir() + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + first_devices, + MDADataSize::default(), + None, + ) .unwrap(); - { - // to allow mutable borrow of pool - let (_, filesystem) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); - mount( - Some(&filesystem.devnode()), - source_tmp_dir.path(), - Some("xfs"), - MsFlags::empty(), - None as Option<&str>, + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, ) .unwrap(); - for i in 0..file_count { - let file_path = source_tmp_dir.path().join(format!("stratis_test{i}.txt")); + + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); + + let write_buf = &vec![8u8; BYTES_PER_WRITE].into_boxed_slice(); + let source_tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + { + // to allow mutable borrow of pool + let (_, filesystem) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&filesystem.devnode()), + source_tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + let file_path = source_tmp_dir.path().join("stratis_test.txt"); let mut f = BufWriter::with_capacity( convert_test!(IEC::Mi, u64, usize), OpenOptions::new() @@ -2099,592 +3021,723 @@ mod tests { .open(file_path) .unwrap(), ); - f.write_all(write_buf).unwrap(); - f.sync_all().unwrap(); + // Write the write_buf until the pool is full + loop { + match pool + .thin_pool + .status(get_dm(), DmOptions::default()) + .unwrap() + { + ThinPoolStatus::Working(_) => { + f.write_all(write_buf).unwrap(); + if f.sync_all().is_err() { + break; + } + } + ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), + ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail Expected working."), + } + } } + match pool + .thin_pool + .status(get_dm(), DmOptions::default()) + .unwrap() + { + ThinPoolStatus::Working(ref status) => { + assert_eq!( + status.summary, + ThinPoolStatusSummary::OutOfSpace, + "Expected full pool" + ); + } + ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), + ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail Expected working/full."), + }; + + // Add block devices to the pool and run check() to extend + backstore + .add_datadevs(pool_uuid, remaining_devices) + .unwrap(); + pool.check(pool_uuid, &mut backstore).unwrap(); + // Verify the pool is back in a Good state + match pool + .thin_pool + .status(get_dm(), DmOptions::default()) + .unwrap() + { + ThinPoolStatus::Working(ref status) => { + assert_eq!( + status.summary, + ThinPoolStatusSummary::Good, + "Expected pool to be restored to good state" + ); + } + ThinPoolStatus::Error => panic!("Could not obtain status for thinpool."), + ThinPoolStatus::Fail => panic!("ThinPoolStatus::Fail. Expected working/good."), + }; + } + + #[test] + fn loop_test_full_pool() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(2, Some(Bytes::from(IEC::Gi * 2).sectors())), + test_full_pool, + ); } - let snapshot_name = "test_snapshot"; - let (_, snapshot_filesystem) = pool - .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, snapshot_name) + #[test] + fn real_test_full_pool() { + real::test_with_spec( + &real::DeviceLimits::Exactly( + 2, + Some(Bytes::from(IEC::Gi * 2).sectors()), + Some(Bytes::from(IEC::Gi * 4).sectors()), + ), + test_full_pool, + ); + } + + /// Verify a snapshot has the same files and same contents as the origin. + fn test_filesystem_snapshot(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + devices, + MDADataSize::default(), + None, + ) .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); + + let filesystem_name = "stratis_test_filesystem"; + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + filesystem_name, + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); + + cmd::udev_settle().unwrap(); - cmd::udev_settle().unwrap(); + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{filesystem_name}")).exists()); - // Assert both symlinks are still present. - assert!(Path::new(&format!("/dev/stratis/{pool_name}/{filesystem_name}")).exists()); - assert!(Path::new(&format!("/dev/stratis/{pool_name}/{snapshot_name}")).exists()); + let write_buf = &[8u8; SECTOR_SIZE]; + let file_count = 10; - let mut read_buf = [0u8; SECTOR_SIZE]; - let snapshot_tmp_dir = tempfile::Builder::new() - .prefix("stratis_testing") - .tempdir() + let source_tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + { + // to allow mutable borrow of pool + let (_, filesystem) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&filesystem.devnode()), + source_tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + for i in 0..file_count { + let file_path = source_tmp_dir.path().join(format!("stratis_test{i}.txt")); + let mut f = BufWriter::with_capacity( + convert_test!(IEC::Mi, u64, usize), + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(file_path) + .unwrap(), + ); + f.write_all(write_buf).unwrap(); + f.sync_all().unwrap(); + } + } + + let snapshot_name = "test_snapshot"; + let (_, snapshot_filesystem) = pool + .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, snapshot_name) + .unwrap(); + + cmd::udev_settle().unwrap(); + + // Assert both symlinks are still present. + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{filesystem_name}")).exists()); + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{snapshot_name}")).exists()); + + let mut read_buf = [0u8; SECTOR_SIZE]; + let snapshot_tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + { + mount( + Some(&snapshot_filesystem.devnode()), + snapshot_tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + for i in 0..file_count { + let file_path = snapshot_tmp_dir.path().join(format!("stratis_test{i}.txt")); + let mut f = OpenOptions::new().read(true).open(file_path).unwrap(); + f.read_exact(&mut read_buf).unwrap(); + assert_eq!(read_buf[0..SECTOR_SIZE], write_buf[0..SECTOR_SIZE]); + } + } + } + + #[test] + fn loop_test_filesystem_snapshot() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_filesystem_snapshot, + ); + } + + #[test] + fn real_test_filesystem_snapshot() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_filesystem_snapshot, + ); + } + + /// Verify that a filesystem rename causes the filesystem metadata to be + /// updated. + fn test_filesystem_rename(paths: &[&Path]) { + let name1 = "name1"; + let name2 = "name2"; + + let pool_uuid = PoolUuid::new_v4(); + + let devices = get_devices(paths).unwrap(); + + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + devices, + MDADataSize::default(), + None, + ) .unwrap(); - { - mount( - Some(&snapshot_filesystem.devnode()), - snapshot_tmp_dir.path(), - Some("xfs"), - MsFlags::empty(), - None as Option<&str>, + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, ) .unwrap(); - for i in 0..file_count { - let file_path = snapshot_tmp_dir.path().join(format!("stratis_test{i}.txt")); - let mut f = OpenOptions::new().read(true).open(file_path).unwrap(); - f.read_exact(&mut read_buf).unwrap(); - assert_eq!(read_buf[0..SECTOR_SIZE], write_buf[0..SECTOR_SIZE]); - } - } - } - #[test] - fn loop_test_filesystem_snapshot() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), - test_filesystem_snapshot, - ); - } + let pool_name = "stratis_test_pool"; + let fs_uuid = pool + .create_filesystem(pool_name, pool_uuid, name1, DEFAULT_THIN_DEV_SIZE, None) + .unwrap(); - #[test] - fn real_test_filesystem_snapshot() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(2, None, None), - test_filesystem_snapshot, - ); - } + cmd::udev_settle().unwrap(); - /// Verify that a filesystem rename causes the filesystem metadata to be - /// updated. - fn test_filesystem_rename(paths: &[&Path]) { - let pool_name = Name::new("pool_name".to_string()); - let name1 = "name1"; - let name2 = "name2"; + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{name1}")).exists()); - let pool_uuid = PoolUuid::new_v4(); + let action = pool.rename_filesystem(pool_name, fs_uuid, name2).unwrap(); - let devices = get_devices(paths).unwrap(); + cmd::udev_settle().unwrap(); - let mut backstore = - Backstore::initialize(pool_name, pool_uuid, devices, MDADataSize::default(), None) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); + // Check that the symlink has been renamed. + assert!(!Path::new(&format!("/dev/stratis/{pool_name}/{name1}")).exists()); + assert!(Path::new(&format!("/dev/stratis/{pool_name}/{name2}")).exists()); - let pool_name = "stratis_test_pool"; - let fs_uuid = pool - .create_filesystem(pool_name, pool_uuid, name1, DEFAULT_THIN_DEV_SIZE, None) - .unwrap(); + assert_eq!(action, Some(true)); + let flexdevs: FlexDevsSave = pool.record(); + let thinpoolsave: ThinPoolDevSave = pool.record(); - cmd::udev_settle().unwrap(); + retry_operation!(pool.teardown(pool_uuid)); - assert!(Path::new(&format!("/dev/stratis/{pool_name}/{name1}")).exists()); + let pool = ThinPool::setup(pool_name, pool_uuid, &thinpoolsave, &flexdevs, &backstore) + .unwrap(); - let action = pool.rename_filesystem(pool_name, fs_uuid, name2).unwrap(); + assert_eq!(&*pool.get_filesystem_by_uuid(fs_uuid).unwrap().0, name2); + } - cmd::udev_settle().unwrap(); + #[test] + fn loop_test_filesystem_rename() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_filesystem_rename, + ); + } - // Check that the symlink has been renamed. - assert!(!Path::new(&format!("/dev/stratis/{pool_name}/{name1}")).exists()); - assert!(Path::new(&format!("/dev/stratis/{pool_name}/{name2}")).exists()); + #[test] + fn real_test_filesystem_rename() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_filesystem_rename, + ); + } - assert_eq!(action, Some(true)); - let flexdevs: FlexDevsSave = pool.record(); - let thinpoolsave: ThinPoolDevSave = pool.record(); + /// Verify that setting up a pool when the pool has not been previously torn + /// down does not fail. Clutter the original pool with a filesystem with + /// some data on it. + fn test_pool_setup(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); - retry_operation!(pool.teardown(pool_uuid)); + let devices = get_devices(paths).unwrap(); - let pool = - ThinPool::setup(pool_name, pool_uuid, &thinpoolsave, &flexdevs, &backstore).unwrap(); + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); - assert_eq!(&*pool.get_filesystem_by_uuid(fs_uuid).unwrap().0, name2); - } + let fs_uuid = pool + .create_filesystem(pool_name, pool_uuid, "fsname", DEFAULT_THIN_DEV_SIZE, None) + .unwrap(); - #[test] - fn loop_test_filesystem_rename() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 4, None), - test_filesystem_rename, - ); - } + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + { + let (_, fs) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + writeln!( + &OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(new_file) + .unwrap(), + "data" + ) + .unwrap(); + } + let thinpooldevsave: ThinPoolDevSave = pool.record(); - #[test] - fn real_test_filesystem_rename() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_filesystem_rename, - ); - } + let new_pool = ThinPool::setup( + pool_name, + pool_uuid, + &thinpooldevsave, + &pool.record(), + &backstore, + ) + .unwrap(); - /// Verify that setting up a pool when the pool has not been previously torn - /// down does not fail. Clutter the original pool with a filesystem with - /// some data on it. - fn test_pool_setup(paths: &[&Path]) { - let pool_name = "pool"; - let pool_uuid = PoolUuid::new_v4(); + assert!(new_pool.get_filesystem_by_uuid(fs_uuid).is_some()); + } - let devices = get_devices(paths).unwrap(); + #[test] + fn loop_test_pool_setup() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_pool_setup, + ); + } - let mut backstore = Backstore::initialize( - Name::new(pool_name.to_string()), - pool_uuid, - devices, - MDADataSize::default(), - None, - ) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); + #[test] + fn real_test_pool_setup() { + real::test_with_spec(&real::DeviceLimits::AtLeast(1, None, None), test_pool_setup); + } + /// Verify that destroy_filesystems actually deallocates the space + /// from the thinpool, by attempting to reinstantiate it using the + /// same thin id and verifying that it fails. + fn test_thindev_destroy(paths: &[&Path]) { + let pool_uuid = PoolUuid::new_v4(); - let fs_uuid = pool - .create_filesystem(pool_name, pool_uuid, "fsname", DEFAULT_THIN_DEV_SIZE, None) - .unwrap(); + let devices = get_devices(paths).unwrap(); - let tmp_dir = tempfile::Builder::new() - .prefix("stratis_testing") - .tempdir() + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + devices, + MDADataSize::default(), + None, + ) .unwrap(); - let new_file = tmp_dir.path().join("stratis_test.txt"); - { - let (_, fs) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); - mount( - Some(&fs.devnode()), - tmp_dir.path(), - Some("xfs"), - MsFlags::empty(), - None as Option<&str>, + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, ) .unwrap(); - writeln!( - &OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(new_file) - .unwrap(), - "data" + let pool_name = "stratis_test_pool"; + let fs_name = "stratis_test_filesystem"; + let fs_uuid = pool + .create_filesystem(pool_name, pool_uuid, fs_name, DEFAULT_THIN_DEV_SIZE, None) + .unwrap(); + + retry_operation!(pool.destroy_filesystem(pool_name, fs_uuid)); + let flexdevs: FlexDevsSave = pool.record(); + let thinpooldevsave: ThinPoolDevSave = pool.record(); + pool.teardown(pool_uuid).unwrap(); + + // Check that destroyed fs is not present in MDV. If the record + // had been left on the MDV that didn't match a thin_id in the + // thinpool, ::setup() will fail. + let pool = ThinPool::setup( + pool_name, + pool_uuid, + &thinpooldevsave, + &flexdevs, + &backstore, ) .unwrap(); - } - let thinpooldevsave: ThinPoolDevSave = pool.record(); - let new_pool = ThinPool::setup( - pool_name, - pool_uuid, - &thinpooldevsave, - &pool.record(), - &backstore, - ) - .unwrap(); + assert_matches!(pool.get_filesystem_by_uuid(fs_uuid), None); + } - assert!(new_pool.get_filesystem_by_uuid(fs_uuid).is_some()); - } + #[test] + fn loop_test_thindev_destroy() { + // This test requires more than 1 GiB. + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, None), + test_thindev_destroy, + ); + } - #[test] - fn loop_test_pool_setup() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 4, None), - test_pool_setup, - ); - } + #[test] + fn real_test_thindev_destroy() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_thindev_destroy, + ); + } - #[test] - fn real_test_pool_setup() { - real::test_with_spec(&real::DeviceLimits::AtLeast(1, None, None), test_pool_setup); - } - /// Verify that destroy_filesystems actually deallocates the space - /// from the thinpool, by attempting to reinstantiate it using the - /// same thin id and verifying that it fails. - fn test_thindev_destroy(paths: &[&Path]) { - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); + /// Just suspend and resume the device and make sure it doesn't crash. + /// Suspend twice in succession and then resume twice in succession + /// to check idempotency. + fn test_suspend_resume(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); - let devices = get_devices(paths).unwrap(); + let devices = get_devices(paths).unwrap(); - let mut backstore = - Backstore::initialize(pool_name, pool_uuid, devices, MDADataSize::default(), None) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); - let pool_name = "stratis_test_pool"; - let fs_name = "stratis_test_filesystem"; - let fs_uuid = pool - .create_filesystem(pool_name, pool_uuid, fs_name, DEFAULT_THIN_DEV_SIZE, None) + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) .unwrap(); - retry_operation!(pool.destroy_filesystem(pool_name, fs_uuid)); - let flexdevs: FlexDevsSave = pool.record(); - let thinpooldevsave: ThinPoolDevSave = pool.record(); - pool.teardown(pool_uuid).unwrap(); + pool.create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); - // Check that destroyed fs is not present in MDV. If the record - // had been left on the MDV that didn't match a thin_id in the - // thinpool, ::setup() will fail. - let pool = ThinPool::setup( - pool_name, - pool_uuid, - &thinpooldevsave, - &flexdevs, - &backstore, - ) - .unwrap(); + pool.suspend().unwrap(); + pool.suspend().unwrap(); + pool.resume().unwrap(); + pool.resume().unwrap(); + } - assert_matches!(pool.get_filesystem_by_uuid(fs_uuid), None); - } + #[test] + fn loop_test_suspend_resume() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 4, None), + test_suspend_resume, + ); + } - #[test] - fn loop_test_thindev_destroy() { - // This test requires more than 1 GiB. - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), - test_thindev_destroy, - ); - } + #[test] + fn real_test_suspend_resume() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(1, None, None), + test_suspend_resume, + ); + } - #[test] - fn real_test_thindev_destroy() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_thindev_destroy, - ); - } + /// Set up thinpool and backstore. Set up filesystem and write to it. + /// Add cachedev to backstore, causing cache to be built. + /// Update device on self. Read written bits from filesystem + /// presented on cache device. + fn test_cache(paths: &[&Path]) { + assert!(paths.len() > 1); - /// Just suspend and resume the device and make sure it doesn't crash. - /// Suspend twice in succession and then resume twice in succession - /// to check idempotency. - fn test_suspend_resume(paths: &[&Path]) { - let pool_name = "pool"; - let pool_uuid = PoolUuid::new_v4(); + let (paths1, paths2) = paths.split_at(paths.len() / 2); - let devices = get_devices(paths).unwrap(); + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); - let mut backstore = Backstore::initialize( - Name::new(pool_name.to_string()), - pool_uuid, - devices, - MDADataSize::default(), - None, - ) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); + let devices1 = get_devices(paths1).unwrap(); + let devices = get_devices(paths2).unwrap(); - pool.create_filesystem( - pool_name, - pool_uuid, - "stratis_test_filesystem", - DEFAULT_THIN_DEV_SIZE, - None, - ) - .unwrap(); + let mut backstore = backstore::v2::Backstore::initialize( + pool_uuid, + devices, + MDADataSize::default(), + None, + ) + .unwrap(); + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) + .unwrap(); - pool.suspend().unwrap(); - pool.suspend().unwrap(); - pool.resume().unwrap(); - pool.resume().unwrap(); - } + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + DEFAULT_THIN_DEV_SIZE, + None, + ) + .unwrap(); - #[test] - fn loop_test_suspend_resume() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 4, None), - test_suspend_resume, - ); - } + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + let bytestring = b"some bytes"; + { + let (_, fs) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&new_file) + .unwrap() + .write_all(bytestring) + .unwrap(); + } + let filesystem_saves = pool.mdv.filesystems().unwrap(); + assert_eq!(filesystem_saves.len(), 1); + assert_eq!( + filesystem_saves + .first() + .expect("filesystem_saves().len == 1") + .uuid, + fs_uuid + ); - #[test] - fn real_test_suspend_resume() { - real::test_with_spec( - &real::DeviceLimits::AtLeast(1, None, None), - test_suspend_resume, - ); - } + backstore.init_cache(pool_uuid, devices1).unwrap(); - /// Set up thinpool and backstore. Set up filesystem and write to it. - /// Add cachedev to backstore, causing cache to be built. - /// Update device on self. Read written bits from filesystem - /// presented on cache device. - fn test_set_device(paths: &[&Path]) { - assert!(paths.len() > 1); + let mut buf = [0u8; 10]; + { + OpenOptions::new() + .read(true) + .open(&new_file) + .unwrap() + .read_exact(&mut buf) + .unwrap(); + } + assert_eq!(&buf, bytestring); + + let filesystem_saves = pool.mdv.filesystems().unwrap(); + assert_eq!(filesystem_saves.len(), 1); + assert_eq!( + filesystem_saves + .first() + .expect("filesystem_saves().len == 1") + .uuid, + fs_uuid + ); + } - let (paths1, paths2) = paths.split_at(paths.len() / 2); + #[test] + fn loop_test_cache() { + loopbacked::test_with_spec(&loopbacked::DeviceLimits::Range(3, 4, None), test_cache); + } - let pool_name = "pool"; - let pool_uuid = PoolUuid::new_v4(); + #[test] + fn real_test_cache() { + real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), test_cache); + } - let devices1 = get_devices(paths1).unwrap(); - let devices = get_devices(paths2).unwrap(); + /// Set up thinpool and backstore. Set up filesystem and set size limit. + /// Write past the halfway mark of the filesystem and check that the filesystem + /// size limit is respected. Increase the filesystem size limit and check that + /// it is respected. Remove the filesystem size limit and verify that the + /// filesystem size doubles. Verify that the filesystem size limit cannot be set + /// below the current filesystem size. + fn test_fs_size_limit(paths: &[&Path]) { + let pool_name = "pool"; + let pool_uuid = PoolUuid::new_v4(); - let mut backstore = Backstore::initialize( - Name::new(pool_name.to_string()), - pool_uuid, - devices, - MDADataSize::default(), - None, - ) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); + let devices = get_devices(paths).unwrap(); - let fs_uuid = pool - .create_filesystem( - pool_name, + let mut backstore = backstore::v2::Backstore::initialize( pool_uuid, - "stratis_test_filesystem", - DEFAULT_THIN_DEV_SIZE, + devices, + MDADataSize::default(), None, ) .unwrap(); - - let tmp_dir = tempfile::Builder::new() - .prefix("stratis_testing") - .tempdir() + let mut pool = ThinPool::::new( + pool_uuid, + &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), + DATA_BLOCK_SIZE, + &mut backstore, + ) .unwrap(); - let new_file = tmp_dir.path().join("stratis_test.txt"); - let bytestring = b"some bytes"; - { - let (_, fs) = pool.get_filesystem_by_uuid(fs_uuid).unwrap(); + + let fs_uuid = pool + .create_filesystem( + pool_name, + pool_uuid, + "stratis_test_filesystem", + Sectors::from(2400 * IEC::Ki), + // 1400 * IEC::Mi + Some(Sectors(2800 * IEC::Ki)), + ) + .unwrap(); + let devnode = { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(Sectors(2800 * IEC::Ki))); + fs.devnode() + }; + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); mount( - Some(&fs.devnode()), + Some(&devnode), tmp_dir.path(), Some("xfs"), MsFlags::empty(), None as Option<&str>, ) .unwrap(); - OpenOptions::new() + let mut file = OpenOptions::new() .create(true) .truncate(true) .write(true) - .open(&new_file) - .unwrap() - .write_all(bytestring) + .open(new_file) .unwrap(); - } - let filesystem_saves = pool.mdv.filesystems().unwrap(); - assert_eq!(filesystem_saves.len(), 1); - assert_eq!( - filesystem_saves - .first() - .expect("filesystem_saves().len == 1") - .uuid, - fs_uuid - ); + let mut bytes_written = Bytes(0); + // Write 800 * IEC::Mi + while bytes_written < Bytes::from(800 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); + } + file.sync_all().unwrap(); + pool.check_fs(pool_uuid, &backstore).unwrap(); - pool.suspend().unwrap(); - let old_device = backstore - .device() - .expect("Space already allocated from backstore, backstore must have device"); - backstore - .init_cache(Name::new(pool_name.to_string()), pool_uuid, devices1, None) - .unwrap(); - let new_device = backstore - .device() - .expect("Space already allocated from backstore, backstore must have device"); - assert_ne!(old_device, new_device); - pool.set_device(new_device).unwrap(); - pool.resume().unwrap(); + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(fs.size().sectors())); + } - let mut buf = [0u8; 10]; - { - OpenOptions::new() - .read(true) - .open(&new_file) - .unwrap() - .read_exact(&mut buf) + // 1600 * IEC::Mi + pool.set_fs_size_limit(fs_uuid, Some(Sectors(3200 * IEC::Ki))) .unwrap(); - } - assert_eq!(&buf, bytestring); - - let filesystem_saves = pool.mdv.filesystems().unwrap(); - assert_eq!(filesystem_saves.len(), 1); - assert_eq!( - filesystem_saves - .first() - .expect("filesystem_saves().len == 1") - .uuid, - fs_uuid - ); - } - - #[test] - fn loop_test_set_device() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(3, 4, None), - test_set_device, - ); - } - - #[test] - fn real_test_set_device() { - real::test_with_spec(&real::DeviceLimits::AtLeast(2, None, None), test_set_device); - } - - /// Set up thinpool and backstore. Set up filesystem and set size limit. - /// Write past the halfway mark of the filesystem and check that the filesystem - /// size limit is respected. Increase the filesystem size limit and check that - /// it is respected. Remove the filesystem size limit and verify that the - /// filesystem size doubles. Verify that the filesystem size limit cannot be set - /// below the current filesystem size. - fn test_fs_size_limit(paths: &[&Path]) { - let pool_name = "pool"; - let pool_uuid = PoolUuid::new_v4(); - - let devices = get_devices(paths).unwrap(); - - let mut backstore = Backstore::initialize( - Name::new(pool_name.to_string()), - pool_uuid, - devices, - MDADataSize::default(), - None, - ) - .unwrap(); - let mut pool = ThinPool::new( - pool_uuid, - &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), - DATA_BLOCK_SIZE, - &mut backstore, - ) - .unwrap(); - - let fs_uuid = pool - .create_filesystem( - pool_name, - pool_uuid, - "stratis_test_filesystem", - Sectors::from(2400 * IEC::Ki), - // 1400 * IEC::Mi - Some(Sectors(2800 * IEC::Ki)), - ) - .unwrap(); - let devnode = { - let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), Some(Sectors(2800 * IEC::Ki))); - fs.devnode() - }; + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); + } + let mut bytes_written = Bytes(0); + // Write 200 * IEC::Mi + while bytes_written < Bytes::from(200 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); + } + file.sync_all().unwrap(); + pool.check_fs(pool_uuid, &backstore).unwrap(); - let tmp_dir = tempfile::Builder::new() - .prefix("stratis_testing") - .tempdir() - .unwrap(); - let new_file = tmp_dir.path().join("stratis_test.txt"); - mount( - Some(&devnode), - tmp_dir.path(), - Some("xfs"), - MsFlags::empty(), - None as Option<&str>, - ) - .unwrap(); - let mut file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(new_file) - .unwrap(); - let mut bytes_written = Bytes(0); - // Write 800 * IEC::Mi - while bytes_written < Bytes::from(800 * IEC::Mi) { - file.write_all(&[1; 4096]).unwrap(); - bytes_written += Bytes(4096); - } - file.sync_all().unwrap(); - pool.check_fs(pool_uuid, &backstore).unwrap(); + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), Some(fs.size().sectors())); + } - { - let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), Some(fs.size().sectors())); - } + { + let (_, fs) = pool + .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, "snapshot") + .unwrap(); + assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); + } - // 1600 * IEC::Mi - pool.set_fs_size_limit(fs_uuid, Some(Sectors(3200 * IEC::Ki))) - .unwrap(); - { - let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); - } - let mut bytes_written = Bytes(0); - // Write 200 * IEC::Mi - while bytes_written < Bytes::from(200 * IEC::Mi) { - file.write_all(&[1; 4096]).unwrap(); - bytes_written += Bytes(4096); - } - file.sync_all().unwrap(); - pool.check_fs(pool_uuid, &backstore).unwrap(); + pool.set_fs_size_limit(fs_uuid, None).unwrap(); + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size_limit(), None); + } + let mut bytes_written = Bytes(0); + // Write 400 * IEC::Mi + while bytes_written < Bytes::from(400 * IEC::Mi) { + file.write_all(&[1; 4096]).unwrap(); + bytes_written += Bytes(4096); + } + file.sync_all().unwrap(); + pool.check_fs(pool_uuid, &backstore).unwrap(); - { - let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), Some(fs.size().sectors())); - } + { + let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); + assert_eq!(fs.size().sectors(), Sectors(6400 * IEC::Ki)); + } - { - let (_, fs) = pool - .snapshot_filesystem(pool_name, pool_uuid, fs_uuid, "snapshot") - .unwrap(); - assert_eq!(fs.size_limit(), Some(Sectors(3200 * IEC::Ki))); + assert!(pool.set_fs_size_limit(fs_uuid, Some(Sectors(50))).is_err()); } - pool.set_fs_size_limit(fs_uuid, None).unwrap(); - { - let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size_limit(), None); - } - let mut bytes_written = Bytes(0); - // Write 400 * IEC::Mi - while bytes_written < Bytes::from(400 * IEC::Mi) { - file.write_all(&[1; 4096]).unwrap(); - bytes_written += Bytes(4096); + #[test] + fn loop_test_fs_size_limit() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(1, 3, Some(Sectors(10 * IEC::Mi))), + test_fs_size_limit, + ); } - file.sync_all().unwrap(); - pool.check_fs(pool_uuid, &backstore).unwrap(); - { - let (_, fs) = pool.get_mut_filesystem_by_uuid(fs_uuid).unwrap(); - assert_eq!(fs.size().sectors(), Sectors(6400 * IEC::Ki)); + #[test] + fn real_test_fs_size_limit() { + real::test_with_spec( + &real::DeviceLimits::Range(1, 3, Some(Sectors(10 * IEC::Mi)), None), + test_fs_size_limit, + ); } - - assert!(pool.set_fs_size_limit(fs_uuid, Some(Sectors(50))).is_err()); - } - - #[test] - fn loop_test_fs_size_limit() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(1, 3, Some(Sectors(10 * IEC::Mi))), - test_fs_size_limit, - ); - } - - #[test] - fn real_test_fs_size_limit() { - real::test_with_spec( - &real::DeviceLimits::Range(1, 3, Some(Sectors(10 * IEC::Mi)), None), - test_fs_size_limit, - ); } } From 87752a3d3932f3e801ac8c7b2e806ffd1d38d00c Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 6 Jul 2023 12:46:22 -0400 Subject: [PATCH 09/32] Diverge new pool from legacy pool and update liminal device code --- src/engine/strat_engine/backstore/mod.rs | 4 +- src/engine/strat_engine/dm.rs | 88 +- src/engine/strat_engine/engine.rs | 197 +- .../strat_engine/liminal/device_info.rs | 41 +- src/engine/strat_engine/liminal/identify.rs | 399 ++-- src/engine/strat_engine/liminal/liminal.rs | 423 ++-- src/engine/strat_engine/liminal/setup.rs | 477 +++-- src/engine/strat_engine/pool/mod.rs | 9 + src/engine/strat_engine/pool/shared.rs | 344 ++++ .../strat_engine/{pool.rs => pool/v1.rs} | 24 +- src/engine/strat_engine/pool/v2.rs | 1767 +++++++++++++++++ 11 files changed, 3264 insertions(+), 509 deletions(-) create mode 100644 src/engine/strat_engine/pool/mod.rs create mode 100644 src/engine/strat_engine/pool/shared.rs rename src/engine/strat_engine/{pool.rs => pool/v1.rs} (99%) create mode 100644 src/engine/strat_engine/pool/v2.rs diff --git a/src/engine/strat_engine/backstore/mod.rs b/src/engine/strat_engine/backstore/mod.rs index d29edef726..b3a39f04e4 100644 --- a/src/engine/strat_engine/backstore/mod.rs +++ b/src/engine/strat_engine/backstore/mod.rs @@ -12,8 +12,8 @@ mod devices; mod range_alloc; mod shared; -#[cfg(test)] -pub use self::devices::initialize_devices_legacy; pub use self::devices::{ find_stratis_devs_by_uuid, get_devno_from_path, ProcessedPathInfos, UnownedDevices, }; +#[cfg(test)] +pub use self::devices::{initialize_devices, initialize_devices_legacy}; diff --git a/src/engine/strat_engine/dm.rs b/src/engine/strat_engine/dm.rs index f52aeac67d..7c3543625f 100644 --- a/src/engine/strat_engine/dm.rs +++ b/src/engine/strat_engine/dm.rs @@ -13,8 +13,8 @@ use devicemapper::{DevId, DmError, DmNameBuf, DmOptions, DM}; use crate::{ engine::{ strat_engine::names::{ - format_backstore_ids, format_crypt_name, format_flex_ids, format_thin_ids, - format_thinpool_ids, CacheRole, FlexRole, ThinPoolRole, ThinRole, + format_backstore_ids, format_crypt_backstore_name, format_crypt_name, format_flex_ids, + format_thin_ids, format_thinpool_ids, CacheRole, FlexRole, ThinPoolRole, ThinRole, }, types::{DevUuid, FilesystemUuid, PoolUuid}, }, @@ -42,8 +42,7 @@ pub fn get_dm() -> &'static DM { ) } -pub fn remove_optional_devices(devs: Vec) -> StratisResult { - let mut did_something = false; +pub fn remove_optional_devices(devs: Vec) -> StratisResult<()> { let devices = get_dm() .list_devices()? .into_iter() @@ -51,18 +50,22 @@ pub fn remove_optional_devices(devs: Vec) -> StratisResult { .collect::>(); for device in devs { if devices.contains(&device) { - did_something = true; get_dm().device_remove(&DevId::Name(&device), DmOptions::default())?; } } - Ok(did_something) + Ok(()) } -pub fn stop_partially_constructed_pool( +pub fn stop_partially_constructed_pool_legacy( pool_uuid: PoolUuid, dev_uuids: &[DevUuid], -) -> StratisResult { - let devs = list_of_partial_pool_devices(pool_uuid, dev_uuids); +) -> StratisResult<()> { + let devs = list_of_partial_pool_devices_legacy(pool_uuid, dev_uuids); + remove_optional_devices(devs) +} + +pub fn stop_partially_constructed_pool(pool_uuid: PoolUuid) -> StratisResult<()> { + let devs = list_of_partial_pool_devices(pool_uuid); remove_optional_devices(devs) } @@ -91,7 +94,7 @@ pub fn mdv_device(pool_uuid: PoolUuid) -> DmNameBuf { thin_mdv } -pub fn list_of_backstore_devices(pool_uuid: PoolUuid) -> Vec { +pub fn list_of_backstore_devices_legacy(pool_uuid: PoolUuid) -> Vec { let mut devs = Vec::new(); let (cache, _) = format_backstore_ids(pool_uuid, CacheRole::Cache); @@ -106,6 +109,17 @@ pub fn list_of_backstore_devices(pool_uuid: PoolUuid) -> Vec { devs } +pub fn list_of_backstore_devices(pool_uuid: PoolUuid) -> Vec { + let mut devs = Vec::new(); + + let crypt = format_crypt_backstore_name(&pool_uuid); + devs.push(crypt); + + devs.extend(list_of_backstore_devices_legacy(pool_uuid)); + + devs +} + pub fn list_of_crypt_devices(dev_uuids: &[DevUuid]) -> Vec { let mut devs = Vec::new(); @@ -120,24 +134,70 @@ pub fn list_of_crypt_devices(dev_uuids: &[DevUuid]) -> Vec { /// List of device names for removal on partially constructed pool stop. Does not have /// filesystem names because partially constructed pools are guaranteed not to have any /// active filesystems. -fn list_of_partial_pool_devices(pool_uuid: PoolUuid, dev_uuids: &[DevUuid]) -> Vec { +fn list_of_partial_pool_devices_legacy( + pool_uuid: PoolUuid, + dev_uuids: &[DevUuid], +) -> Vec { let mut devs = Vec::new(); devs.extend(list_of_thin_pool_devices(pool_uuid)); devs.push(mdv_device(pool_uuid)); - devs.extend(list_of_backstore_devices(pool_uuid)); + devs.extend(list_of_backstore_devices_legacy(pool_uuid)); devs.extend(list_of_crypt_devices(dev_uuids)); devs } +/// List of device names for removal on partially constructed pool stop. Does not have +/// filesystem names because partially constructed pools are guaranteed not to have any +/// active filesystems. +fn list_of_partial_pool_devices(pool_uuid: PoolUuid) -> Vec { + let mut devs = Vec::new(); + + devs.extend(list_of_thin_pool_devices(pool_uuid)); + + devs.push(mdv_device(pool_uuid)); + + devs.extend(list_of_backstore_devices(pool_uuid)); + + devs +} + +/// Check whether there are any leftover devicemapper devices from the pool. +pub fn has_leftover_devices_legacy(pool_uuid: PoolUuid, dev_uuids: &[DevUuid]) -> bool { + let mut has_leftover = false; + let devices = list_of_partial_pool_devices_legacy(pool_uuid, dev_uuids); + match get_dm().list_devices() { + Ok(l) => { + let listed_devices = l + .into_iter() + .map(|(dm_name, _, _)| dm_name) + .collect::>(); + for device in devices { + if listed_devices.contains(&device) { + has_leftover |= true; + } + } + } + Err(_) => { + for device in devices { + if Path::new(&format!("/dev/mapper/{}", &*device)).exists() { + has_leftover |= true; + } + } + } + } + + has_leftover +} + /// Check whether there are any leftover devicemapper devices from the pool. -pub fn has_leftover_devices(pool_uuid: PoolUuid, dev_uuids: &[DevUuid]) -> bool { +pub fn has_leftover_devices(pool_uuid: PoolUuid) -> bool { let mut has_leftover = false; - let devices = list_of_partial_pool_devices(pool_uuid, dev_uuids); + let devices = list_of_partial_pool_devices(pool_uuid); match get_dm().list_devices() { Ok(l) => { let listed_devices = l diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index a605d19b31..442d44218f 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -29,7 +29,7 @@ use crate::{ keys::StratKeyActions, liminal::{find_all, DeviceSet, LiminalDevices}, ns::MemoryFilesystem, - pool::StratPool, + pool::{v1, v2, AnyPool}, }, structures::{ AllLockReadGuard, AllLockWriteGuard, AllOrSomeLock, Lockable, SomeLockReadGuard, @@ -50,7 +50,7 @@ type PoolJoinHandles = Vec>>; #[derive(Debug)] pub struct StratEngine { - pools: AllOrSomeLock, + pools: AllOrSomeLock, // Maps pool UUIDs to information about sets of devices that are // associated with that UUID but have not been converted into a pool. @@ -97,45 +97,134 @@ impl StratEngine { }) } + #[cfg(test)] + async fn create_pool_legacy( + &self, + name: &str, + blockdev_paths: &[&Path], + encryption_info: Option<&EncryptionInfo>, + ) -> StratisResult> { + validate_name(name)?; + let name = Name::new(name.to_owned()); + + validate_paths(blockdev_paths)?; + + let cloned_paths = blockdev_paths + .iter() + .map(|p| p.to_path_buf()) + .collect::>(); + + let devices = spawn_blocking!({ + let borrowed_paths = cloned_paths.iter().map(|p| p.as_path()).collect::>(); + ProcessedPathInfos::try_from(borrowed_paths.as_slice()) + })??; + + let (stratis_devices, unowned_devices) = devices.unpack(); + + let maybe_guard = self.pools.read(PoolIdentifier::Name(name.clone())).await; + if let Some(guard) = maybe_guard { + let (name, uuid, pool) = guard.as_tuple(); + + let (this_pool, other_pools) = stratis_devices.partition(uuid); + other_pools.error_on_not_empty()?; + + create_pool_idempotent_or_err( + pool, + &name, + &this_pool + .values() + .map(|info| info.devnode.as_path()) + .chain( + unowned_devices + .unpack() + .iter() + .map(|info| info.devnode.as_path()), + ) + .collect::>(), + ) + } else { + stratis_devices.error_on_not_empty()?; + + if unowned_devices.is_empty() { + return Err(StratisError::Msg( + "At least one blockdev is required to create a pool.".to_string(), + )); + } + + let block_size_summary = unowned_devices.blocksizes(); + if block_size_summary.len() > 1 { + let err_str = "The devices specified for initializing the pool do not all have the same physical sector size or do not all have the same logical sector size".into(); + return Err(StratisError::Msg(err_str)); + } + + let cloned_name = name.clone(); + let cloned_enc_info = encryption_info.cloned(); + + let pool_uuid = { + let mut pools = self.pools.modify_all().await; + let (pool_uuid, pool) = spawn_blocking!({ + v1::StratPool::initialize( + &cloned_name, + unowned_devices, + cloned_enc_info.as_ref(), + ) + })??; + pools.insert(Name::new(name.to_string()), pool_uuid, AnyPool::V1(pool)); + pool_uuid + }; + + Ok(CreateAction::Created(pool_uuid)) + } + } + pub async fn get_pool( &self, key: PoolIdentifier, - ) -> Option> { + ) -> Option> { get_pool!(self; key) } pub async fn get_mut_pool( &self, key: PoolIdentifier, - ) -> Option> { + ) -> Option> { get_mut_pool!(self; key) } - pub async fn pools(&self) -> AllLockReadGuard { + pub async fn pools(&self) -> AllLockReadGuard { self.pools.read_all().await } - pub async fn pools_mut(&self) -> AllLockWriteGuard { + pub async fn pools_mut(&self) -> AllLockWriteGuard { self.pools.write_all().await } fn spawn_pool_check_handling( joins: &mut PoolJoinHandles, - mut guard: SomeLockWriteGuard, + mut guard: SomeLockWriteGuard, ) { joins.push(spawn_blocking(move || { let (name, uuid, pool) = guard.as_mut_tuple(); - Ok((uuid, pool.event_on(uuid, &name)?)) + Ok(( + uuid, + match pool { + AnyPool::V1(p) => p.event_on(uuid, &name)?, + AnyPool::V2(p) => p.event_on(uuid, &name)?, + }, + )) })); } fn spawn_fs_check_handling( joins: &mut Vec>>>, - mut guard: SomeLockWriteGuard, + mut guard: SomeLockWriteGuard, ) { joins.push(spawn_blocking(move || { let (_, uuid, pool) = guard.as_mut_tuple(); - pool.fs_event_on(uuid) + match pool { + AnyPool::V1(p) => p.fs_event_on(uuid), + AnyPool::V2(p) => p.fs_event_on(uuid), + } })); } @@ -207,7 +296,7 @@ impl StratEngine { /// The implementation for pool_evented when called by the timer thread. async fn pool_evented_timer(&self) -> HashMap { let mut joins = Vec::new(); - let guards: Vec> = + let guards: Vec> = self.pools.write_all().await.into(); for guard in guards { Self::spawn_pool_check_handling(&mut joins, guard); @@ -239,7 +328,7 @@ impl StratEngine { /// The implementation for fs_evented when called by the timer thread. async fn fs_evented_timer(&self) -> HashMap { let mut joins = Vec::new(); - let guards: Vec> = + let guards: Vec> = self.pools.write_all().await.into(); for guard in guards { Self::spawn_fs_check_handling(&mut joins, guard); @@ -255,8 +344,11 @@ impl StratEngine { let mut untorndown_pools = Vec::new(); let mut write_all = block_on(self.pools.write_all()); for (_, uuid, pool) in write_all.iter_mut() { - pool.teardown(*uuid) - .unwrap_or_else(|_| untorndown_pools.push(uuid)); + match pool { + AnyPool::V1(p) => p.teardown(*uuid), + AnyPool::V2(p) => p.teardown(*uuid), + } + .unwrap_or_else(|_| untorndown_pools.push(uuid)); } if untorndown_pools.is_empty() { Ok(()) @@ -280,13 +372,26 @@ impl<'a> Into for &'a StratEngine { "name": Value::from(name.to_string()), }); if let Value::Object(ref mut map) = json { - map.extend( - if let Value::Object(map) = <&StratPool as Into>::into(pool) { - map.into_iter() - } else { - unreachable!("StratPool conversion returns a JSON object"); + match pool { + AnyPool::V1(p) => { + map.extend( + if let Value::Object(map) = <&v1::StratPool as Into>::into(p) { + map.into_iter() + } else { + unreachable!("StratPool conversion returns a JSON object"); + } + ); } - ); + AnyPool::V2(p) => { + map.extend( + if let Value::Object(map) = <&v2::StratPool as Into>::into(p) { + map.into_iter() + } else { + unreachable!("StratPool conversion returns a JSON object"); + } + ); + } + } } else { unreachable!("json!() always creates a JSON object") } @@ -448,9 +553,13 @@ impl Engine for StratEngine { let pool_uuid = { let mut pools = self.pools.modify_all().await; let (pool_uuid, pool) = spawn_blocking!({ - StratPool::initialize(&cloned_name, unowned_devices, cloned_enc_info.as_ref()) + v2::StratPool::initialize( + &cloned_name, + unowned_devices, + cloned_enc_info.as_ref(), + ) })??; - pools.insert(Name::new(name.to_string()), pool_uuid, pool); + pools.insert(Name::new(name.to_string()), pool_uuid, AnyPool::V2(pool)); pool_uuid }; @@ -460,9 +569,12 @@ impl Engine for StratEngine { async fn destroy_pool(&self, uuid: PoolUuid) -> StratisResult> { if let Some(pool) = self.pools.read(PoolIdentifier::Uuid(uuid)).await { - if pool.has_filesystems() { + if match &*pool { + AnyPool::V1(p) => p.has_filesystems(), + AnyPool::V2(p) => p.has_filesystems(), + } { return Err(StratisError::Msg("filesystems remaining on pool".into())); - }; + } } else { return Ok(DeleteAction::Identity); } @@ -472,7 +584,13 @@ impl Engine for StratEngine { .remove_by_uuid(uuid) .expect("Must succeed since self.pools.get_by_uuid() returned a value"); - let (res, mut pool) = spawn_blocking!((pool.destroy(uuid), pool))?; + let (res, mut pool) = spawn_blocking!(( + match pool { + AnyPool::V1(ref mut p) => p.destroy(uuid), + AnyPool::V2(ref mut p) => p.destroy(uuid), + }, + pool + ))?; if let Err((err, true)) = res { guard.insert(pool_name, uuid, pool); Err(err) @@ -481,7 +599,10 @@ impl Engine for StratEngine { // because some of the block devices could have been destroyed above. Using the // cached data structures alone could result in phantom devices that have already // been destroyed but are still recorded in the stopped pool. - let device_set = DeviceSet::from(pool.drain_bds()); + let device_set = match pool { + AnyPool::V1(ref mut p) => DeviceSet::from(p.drain_bds()), + AnyPool::V2(ref mut p) => DeviceSet::from(p.drain_bds()), + }; self.liminal_devices .write() .await @@ -509,9 +630,15 @@ impl Engine for StratEngine { let cloned_new_name = new_name.clone(); let (res, pool) = spawn_blocking!({ - let res = pool.rename_pool(&cloned_new_name); + let res = match pool { + AnyPool::V1(ref mut p) => p.rename_pool(&cloned_new_name), + AnyPool::V2(_) => Ok(()), + }; ( - res.and_then(|_| pool.write_metadata(&cloned_new_name)), + res.and_then(|_| match pool { + AnyPool::V1(ref mut p) => p.write_metadata(&cloned_new_name), + AnyPool::V2(ref mut p) => p.write_metadata(&cloned_new_name), + }), pool, ) })?; @@ -521,7 +648,10 @@ impl Engine for StratEngine { } else { guard.insert(new_name, uuid, pool); let (new_name, pool) = guard.get_by_uuid(uuid).expect("Inserted above"); - pool.udev_pool_change(&new_name); + match pool { + AnyPool::V1(p) => p.udev_pool_change(&new_name), + AnyPool::V2(p) => p.udev_pool_change(&new_name), + }; Ok(RenameAction::Renamed(uuid)) } } @@ -603,7 +733,10 @@ impl Engine for StratEngine { let read_pools = self.pools.read_all().await; let mut write_event_nrs = self.watched_dev_last_event_nrs.write().await; for (_, pool_uuid, pool) in read_pools.iter() { - let dev_names = pool.get_eventing_dev_names(*pool_uuid); + let dev_names = match pool { + AnyPool::V1(p) => p.get_eventing_dev_names(*pool_uuid), + AnyPool::V2(p) => p.get_eventing_dev_names(*pool_uuid), + }; let event_nrs = device_list .iter() .filter_map(|(dm_name, event_nr)| { @@ -928,11 +1061,11 @@ mod test { operation: F, unlock_method: UnlockMethod, ) where - F: FnOnce(&mut StratPool) + UnwindSafe, + F: FnOnce(&mut AnyPool) + UnwindSafe, { unshare_mount_namespace().unwrap(); let engine = StratEngine::initialize().unwrap(); - let uuid = test_async!(engine.create_pool(name, data_paths, Some(encryption_info))) + let uuid = test_async!(engine.create_pool_legacy(name, data_paths, Some(encryption_info))) .unwrap() .changed() .expect("Pool should be newly created"); diff --git a/src/engine/strat_engine/liminal/device_info.rs b/src/engine/strat_engine/liminal/device_info.rs index 84d86b9c1d..561c188d53 100644 --- a/src/engine/strat_engine/liminal/device_info.rs +++ b/src/engine/strat_engine/liminal/device_info.rs @@ -18,7 +18,7 @@ use crate::{ engine::{ shared::{gather_encryption_info, gather_pool_name}, strat_engine::{ - backstore::blockdev::v1::StratBlockDev, + backstore::blockdev::{v1, v2}, liminal::{ identify::{DeviceInfo, LuksInfo, StratisDevInfo, StratisInfo}, setup::get_name, @@ -27,10 +27,10 @@ use crate::{ }, types::{ DevUuid, EncryptionInfo, LockedPoolInfo, MaybeInconsistent, Name, PoolDevice, - PoolEncryptionInfo, PoolUuid, StoppedPoolInfo, + PoolEncryptionInfo, PoolUuid, StoppedPoolInfo, StratSigblockVersion, }, }, - stratis::StratisResult, + stratis::{StratisError, StratisResult}, }; /// Info for a discovered LUKS device belonging to Stratis. @@ -516,8 +516,20 @@ impl IntoIterator for DeviceSet { } } -impl From> for DeviceSet { - fn from(vec: Vec) -> Self { +impl From> for DeviceSet { + fn from(vec: Vec) -> Self { + vec.into_iter() + .fold(DeviceSet::default(), |mut device_set, bd| { + for info in Vec::::from(bd) { + device_set.process_info_add(info); + } + device_set + }) + } +} + +impl From> for DeviceSet { + fn from(vec: Vec) -> Self { vec.into_iter() .fold(DeviceSet::default(), |mut device_set, bd| { for info in Vec::::from(bd) { @@ -762,6 +774,25 @@ impl DeviceSet { } } + pub fn metadata_version(&self) -> StratisResult { + let metadata_version = self.iter().fold(HashSet::new(), |mut set, (_, info)| { + match info { + LInfo::Luks(_) => set.insert(StratSigblockVersion::V1), + LInfo::Stratis(info) => set.insert(info.bda.sigblock_version()), + }; + set + }); + if metadata_version.len() > 1 { + return Err(StratisError::Msg( + "Found two versions of metadata for given device set".to_string(), + )); + } + Ok(*metadata_version + .iter() + .next() + .expect("No empty device sets")) + } + /// Returns a boolean indicating whether the data structure has any devices /// registered. pub fn is_empty(&self) -> bool { diff --git a/src/engine/strat_engine/liminal/identify.rs b/src/engine/strat_engine/liminal/identify.rs index 0dea8acda7..9833d2db68 100644 --- a/src/engine/strat_engine/liminal/identify.rs +++ b/src/engine/strat_engine/liminal/identify.rs @@ -52,7 +52,7 @@ use devicemapper::Device; use crate::engine::{ strat_engine::{ - backstore::blockdev::{v1::StratBlockDev, InternalBlockDev}, + backstore::blockdev::{v1, v2, InternalBlockDev}, crypt::handle::v1::CryptHandle, metadata::{static_header, StratisIdentifiers, BDA}, udev::{ @@ -173,8 +173,8 @@ impl DeviceInfo { } } -impl From for Vec { - fn from(bd: StratBlockDev) -> Self { +impl From for Vec { + fn from(bd: v1::StratBlockDev) -> Self { let mut device_infos = Vec::new(); match (bd.encryption_info(), bd.pool_name(), bd.luks_device()) { (Some(ei), Some(pname), Some(dev)) => { @@ -215,6 +215,18 @@ impl From for Vec { } } +impl From for Vec { + fn from(bd: v2::StratBlockDev) -> Self { + vec![DeviceInfo::Stratis(StratisInfo { + dev_info: StratisDevInfo { + device_number: *bd.device(), + devnode: bd.physical_path().to_owned(), + }, + bda: bd.into_bda(), + })] + } +} + impl fmt::Display for DeviceInfo { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -492,7 +504,10 @@ mod tests { use crate::{ engine::{ strat_engine::{ - backstore::{initialize_devices_legacy, ProcessedPathInfos, UnownedDevices}, + backstore::{ + initialize_devices, initialize_devices_legacy, ProcessedPathInfos, + UnownedDevices, + }, cmd::create_fs, metadata::MDADataSize, tests::{crypt, loopbacked, real}, @@ -514,165 +529,6 @@ mod tests { }) } - /// Test that an encrypted device initialized by stratisd is properly - /// recognized. - /// - /// * Verify that the physical paths are recognized as LUKS devices - /// belonging to Stratis. - /// * Verify that the physical paths are not recognized as Stratis devices. - /// * Verify that the metadata paths are recognized as Stratis devices. - fn test_process_luks_device_initialized(paths: &[&Path]) { - assert!(!paths.is_empty()); - - fn luks_device_test(paths: &[&Path], key_description: &KeyDescription) { - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - - let devices = initialize_devices_legacy( - get_devices(paths).unwrap(), - pool_name, - pool_uuid, - MDADataSize::default(), - Some(&EncryptionInfo::KeyDesc(key_description.clone())), - None, - ) - .unwrap(); - - for dev in devices { - let info = - block_device_apply(&DevicePath::new(dev.physical_path()).unwrap(), |dev| { - process_luks_device(dev) - }) - .unwrap() - .expect("No device with specified devnode found in udev database") - .expect("No LUKS information for Stratis found on specified device"); - - if info.identifiers.pool_uuid != pool_uuid { - panic!( - "Discovered pool UUID {} != expected pool UUID {}", - info.identifiers.pool_uuid, pool_uuid - ); - } - - if info.dev_info.devnode != dev.physical_path() { - panic!( - "Discovered device node {} != expected device node {}", - info.dev_info.devnode.display(), - dev.physical_path().display() - ); - } - - if info.encryption_info.key_description() != Some(key_description) { - panic!( - "Discovered key description {:?} != expected key description {:?}", - info.encryption_info.key_description(), - Some(key_description.as_application_str()) - ); - } - - let info = - block_device_apply(&DevicePath::new(dev.physical_path()).unwrap(), |dev| { - process_stratis_device(dev) - }) - .unwrap() - .expect("No device with specified devnode found in udev database"); - if info.is_some() { - panic!("Encrypted block device was incorrectly identified as a Stratis device"); - } - - let info = - block_device_apply(&DevicePath::new(dev.metadata_path()).unwrap(), |dev| { - process_stratis_device(dev) - }) - .unwrap() - .expect("No device with specified devnode found in udev database") - .expect("No Stratis metadata found on specified device"); - - if info.bda.identifiers().pool_uuid != pool_uuid - || info.dev_info.devnode != dev.metadata_path() - { - panic!( - "Wrong identifiers and devnode found on Stratis block device: found: pool UUID: {}, device node; {} != expected: pool UUID: {}, device node: {}", - info.bda.identifiers().pool_uuid, - info.dev_info.devnode.display(), - pool_uuid, - dev.metadata_path().display(), - ); - } - } - } - - crypt::insert_and_cleanup_key(paths, luks_device_test); - } - - #[test] - fn loop_test_process_luks_device_initialized() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_process_luks_device_initialized, - ); - } - - #[test] - fn real_test_process_luks_device_initialized() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, None), - test_process_luks_device_initialized, - ); - } - - /// Test that the process_*_device methods return the expected - /// pool UUID and device node for initialized paths. - fn test_process_device_initialized(paths: &[&Path]) { - assert!(!paths.is_empty()); - - let pool_uuid = PoolUuid::new_v4(); - let pool_name = Name::new("pool_name".to_string()); - - initialize_devices_legacy( - get_devices(paths).unwrap(), - pool_name, - pool_uuid, - MDADataSize::default(), - None, - None, - ) - .unwrap(); - - for path in paths { - let device_path = DevicePath::new(path).expect("our test path"); - let info = block_device_apply(&device_path, process_stratis_device) - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(info.bda.identifiers().pool_uuid, pool_uuid); - assert_eq!(&&info.dev_info.devnode, path); - - assert_eq!( - block_device_apply(&device_path, process_luks_device) - .unwrap() - .unwrap(), - None - ); - } - } - - #[test] - fn loop_test_process_device_initialized() { - loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Exactly(1, None), - test_process_device_initialized, - ); - } - - #[test] - fn real_test_process_device_initialized() { - real::test_with_spec( - &real::DeviceLimits::Exactly(1, None, None), - test_process_device_initialized, - ); - } - /// Test that the process_*_device methods return None if the device is /// not a Stratis device. Strictly speaking, the methods are only supposed /// to be called in particular contexts, the situation where the device @@ -733,4 +589,221 @@ mod tests { test_process_device_uninitialized, ); } + + mod v1 { + use super::*; + + /// Test that an encrypted device initialized by stratisd is properly + /// recognized. + /// + /// * Verify that the physical paths are recognized as LUKS devices + /// belonging to Stratis. + /// * Verify that the physical paths are not recognized as Stratis devices. + /// * Verify that the metadata paths are recognized as Stratis devices. + fn test_process_luks_device_initialized(paths: &[&Path]) { + assert!(!paths.is_empty()); + + fn luks_device_test(paths: &[&Path], key_description: &KeyDescription) { + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + + let devices = initialize_devices_legacy( + get_devices(paths).unwrap(), + pool_name, + pool_uuid, + MDADataSize::default(), + Some(&EncryptionInfo::KeyDesc(key_description.clone())), + None, + ) + .unwrap(); + + for dev in devices { + let info = block_device_apply( + &DevicePath::new(dev.physical_path()).unwrap(), + process_luks_device, + ) + .unwrap() + .expect("No device with specified devnode found in udev database") + .expect("No LUKS information for Stratis found on specified device"); + + if info.identifiers.pool_uuid != pool_uuid { + panic!( + "Discovered pool UUID {} != expected pool UUID {}", + info.identifiers.pool_uuid, pool_uuid + ); + } + + if info.dev_info.devnode != dev.physical_path() { + panic!( + "Discovered device node {} != expected device node {}", + info.dev_info.devnode.display(), + dev.physical_path().display() + ); + } + + if info.encryption_info.key_description() != Some(key_description) { + panic!( + "Discovered key description {:?} != expected key description {:?}", + info.encryption_info.key_description(), + Some(key_description.as_application_str()) + ); + } + + let info = block_device_apply( + &DevicePath::new(dev.physical_path()).unwrap(), + process_stratis_device, + ) + .unwrap() + .expect("No device with specified devnode found in udev database"); + if info.is_some() { + panic!( + "Encrypted block device was incorrectly identified as a Stratis device" + ); + } + + let info = block_device_apply( + &DevicePath::new(dev.metadata_path()).unwrap(), + process_stratis_device, + ) + .unwrap() + .expect("No device with specified devnode found in udev database") + .expect("No Stratis metadata found on specified device"); + + if info.bda.identifiers().pool_uuid != pool_uuid + || info.dev_info.devnode != dev.metadata_path() + { + panic!( + "Wrong identifiers and devnode found on Stratis block device: found: pool UUID: {}, device node; {} != expected: pool UUID: {}, device node: {}", + info.bda.identifiers().pool_uuid, + info.dev_info.devnode.display(), + pool_uuid, + dev.metadata_path().display(), + ); + } + } + } + + crypt::insert_and_cleanup_key(paths, luks_device_test); + } + + #[test] + fn loop_test_process_luks_device_initialized() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_process_luks_device_initialized, + ); + } + + #[test] + fn real_test_process_luks_device_initialized() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_process_luks_device_initialized, + ); + } + + /// Test that the process_*_device methods return the expected + /// pool UUID and device node for initialized paths. + fn test_process_device_initialized(paths: &[&Path]) { + assert!(!paths.is_empty()); + + let pool_uuid = PoolUuid::new_v4(); + let pool_name = Name::new("pool_name".to_string()); + + initialize_devices_legacy( + get_devices(paths).unwrap(), + pool_name, + pool_uuid, + MDADataSize::default(), + None, + None, + ) + .unwrap(); + + for path in paths { + let device_path = DevicePath::new(path).expect("our test path"); + let info = block_device_apply(&device_path, process_stratis_device) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(info.bda.identifiers().pool_uuid, pool_uuid); + assert_eq!(&&info.dev_info.devnode, path); + + assert_eq!( + block_device_apply(&device_path, process_luks_device) + .unwrap() + .unwrap(), + None + ); + } + } + + #[test] + fn loop_test_process_device_initialized() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_process_device_initialized, + ); + } + + #[test] + fn real_test_process_device_initialized() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_process_device_initialized, + ); + } + } + + mod v2 { + use super::*; + + /// Test that the process_*_device methods return the expected + /// pool UUID and device node for initialized paths. + fn test_process_device_initialized(paths: &[&Path]) { + assert!(!paths.is_empty()); + + let pool_uuid = PoolUuid::new_v4(); + + initialize_devices( + get_devices(paths).unwrap(), + pool_uuid, + MDADataSize::default(), + ) + .unwrap(); + + for path in paths { + let device_path = DevicePath::new(path).expect("our test path"); + let info = block_device_apply(&device_path, process_stratis_device) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(info.bda.identifiers().pool_uuid, pool_uuid); + assert_eq!(&&info.dev_info.devnode, path); + + assert_eq!( + block_device_apply(&device_path, process_luks_device) + .unwrap() + .unwrap(), + None + ); + } + } + + #[test] + fn loop_test_process_device_initialized() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_process_device_initialized, + ); + } + + #[test] + fn real_test_process_device_initialized() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_process_device_initialized, + ); + } + } } diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index 1f4079650c..7e2a16e9d7 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -13,16 +13,18 @@ use chrono::{DateTime, Utc}; use either::Either; use serde_json::{Map, Value}; +use devicemapper::Sectors; + use crate::{ engine::{ engine::{DumpState, Pool, StateDiff}, strat_engine::{ - backstore::{ - blockdev::{v1::StratBlockDev, InternalBlockDev}, - find_stratis_devs_by_uuid, - }, + backstore::{blockdev::InternalBlockDev, find_stratis_devs_by_uuid}, crypt::handle::v1::CryptHandle, - dm::{has_leftover_devices, stop_partially_constructed_pool}, + dm::{ + has_leftover_devices, has_leftover_devices_legacy, stop_partially_constructed_pool, + stop_partially_constructed_pool_legacy, + }, liminal::{ device_info::{ reconstruct_stratis_infos, split_stratis_infos, stratis_infos_ref, DeviceSet, @@ -32,10 +34,10 @@ use crate::{ bda_wrapper, identify_block_device, DeviceInfo, LuksInfo, StratisDevInfo, StratisInfo, }, - setup::{get_blockdevs, get_metadata}, + setup::{get_blockdevs, get_blockdevs_legacy, get_metadata}, }, metadata::{StratisIdentifiers, BDA}, - pool::StratPool, + pool::{v1, v2, AnyPool}, serde_structs::PoolSave, shared::tiers_to_bdas, types::BDARecordResult, @@ -43,8 +45,8 @@ use crate::{ structures::Table, types::{ DevUuid, LockedPoolsInfo, MaybeInconsistent, Name, PoolEncryptionInfo, PoolIdentifier, - PoolUuid, StoppedPoolsInfo, StratBlockDevDiff, UdevEngineEvent, UnlockMethod, - UuidOrConflict, + PoolUuid, StoppedPoolsInfo, StratBlockDevDiff, StratSigblockVersion, UdevEngineEvent, + UnlockMethod, UuidOrConflict, }, BlockDevTier, }, @@ -90,7 +92,7 @@ impl LiminalDevices { // Unlock the liminal encrypted devices that correspond to the given pool UUID. fn unlock_pool( &mut self, - pools: &Table, + pools: &Table, pool_uuid: PoolUuid, unlock_method: UnlockMethod, ) -> StratisResult> { @@ -165,10 +167,10 @@ impl LiminalDevices { /// pool. pub fn start_pool( &mut self, - pools: &Table, + pools: &Table, id: PoolIdentifier, unlock_method: Option, - ) -> StratisResult<(Name, PoolUuid, StratPool, Vec)> { + ) -> StratisResult<(Name, PoolUuid, AnyPool, Vec)> { let pool_uuid = match id { PoolIdentifier::Uuid(u) => u, PoolIdentifier::Name(n) => self @@ -250,12 +252,16 @@ impl LiminalDevices { /// filesystems, as in that case the pool needs to be administered. pub fn stop_pool( &mut self, - pools: &mut Table, + pools: &mut Table, pool_name: Name, pool_uuid: PoolUuid, - mut pool: StratPool, + mut pool: AnyPool, ) -> StratisResult { - let (devices, err) = match pool.stop(&pool_name, pool_uuid) { + let res = match pool { + AnyPool::V1(ref mut p) => p.stop(&pool_name, pool_uuid), + AnyPool::V2(ref mut p) => p.stop(&pool_name, pool_uuid), + }; + let (devices, err) = match res { Ok(devs) => (devs, None), Err((e, true)) => { pools.insert(pool_name, pool_uuid, pool); @@ -263,7 +269,13 @@ impl LiminalDevices { } Err((e, false)) => { warn!("Failed to stop pool; placing in partially constructed pools"); - (DeviceSet::from(pool.drain_bds()), Some(e)) + ( + match pool { + AnyPool::V1(ref mut p) => DeviceSet::from(p.drain_bds()), + AnyPool::V2(ref mut p) => DeviceSet::from(p.drain_bds()), + }, + Some(e), + ) } }; for (_, device) in devices.iter() { @@ -305,23 +317,40 @@ impl LiminalDevices { /// Tear down a partially constructed pool. pub fn stop_partially_constructed_pool(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { if let Some(device_set) = self.partially_constructed_pools.remove(&pool_uuid) { - match stop_partially_constructed_pool( - pool_uuid, - &device_set - .iter() - .map(|(dev_uuid, _)| *dev_uuid) - .collect::>(), - ) { - Ok(_) => { - self.stopped_pools.insert(pool_uuid, device_set); - Ok(()) - } - Err(e) => { - warn!("Failed to stop partially constructed pool: {}", e); - self.partially_constructed_pools - .insert(pool_uuid, device_set); - Err(e) + let metadata_version = device_set.metadata_version()?; + match metadata_version { + StratSigblockVersion::V1 => { + match stop_partially_constructed_pool_legacy( + pool_uuid, + &device_set + .iter() + .map(|(dev_uuid, _)| *dev_uuid) + .collect::>(), + ) { + Ok(_) => { + self.stopped_pools.insert(pool_uuid, device_set); + Ok(()) + } + Err(e) => { + warn!("Failed to stop partially constructed pool: {}", e); + self.partially_constructed_pools + .insert(pool_uuid, device_set); + Err(e) + } + } } + StratSigblockVersion::V2 => match stop_partially_constructed_pool(pool_uuid) { + Ok(_) => { + self.stopped_pools.insert(pool_uuid, device_set); + Ok(()) + } + Err(e) => { + warn!("Failed to stop partially constructed pool: {}", e); + self.partially_constructed_pools + .insert(pool_uuid, device_set); + Err(e) + } + }, } } else { Ok(()) @@ -403,11 +432,14 @@ impl LiminalDevices { } /// Calculate whether block device size has changed. - fn handle_size_change( + fn handle_size_change<'a, B>( tier: BlockDevTier, dev_uuid: DevUuid, - dev: &mut StratBlockDev, - ) -> Option<(DevUuid, StratBlockDevDiff)> { + dev: &mut B, + ) -> Option<(DevUuid, <>::State as StateDiff>::Diff)> + where + B: DumpState<'a, DumpInput = Sectors> + InternalBlockDev, + { if tier == BlockDevTier::Data { let orig = dev.cached(); match dev.calc_new_size() { @@ -415,7 +447,7 @@ impl LiminalDevices { Err(e) => { warn!( "Failed to determine device size for {}: {}", - dev.devnode().display(), + dev.physical_path().display(), e ); None @@ -439,7 +471,7 @@ impl LiminalDevices { HashMap>, HashMap>, ), - ) -> Vec<(Name, PoolUuid, StratPool)> { + ) -> Vec<(Name, PoolUuid, AnyPool)> { let table = Table::default(); let (mut luks_devices, mut stratis_devices) = all_devices; @@ -508,26 +540,49 @@ impl LiminalDevices { self.try_setup_started_pool(&table, *pool_uuid, info_map) .map(|(pool_name, mut pool)| { - match pool.blockdevs_mut() { - Ok(blockdevs) => { - for (dev_uuid, tier, blockdev) in blockdevs { - if let Some(size) = - Self::handle_size_change(tier, dev_uuid, blockdev) - .and_then(|(_, d)| d.size.changed()) - .and_then(|c| c) - { - blockdev.set_new_size(size); + match pool { + AnyPool::V1(ref mut p) => { + match p.blockdevs_mut() { + Ok(blockdevs) => { + for (dev_uuid, tier, blockdev) in blockdevs { + if let Some(size) = + Self::handle_size_change(tier, dev_uuid, blockdev) + .and_then(|(_, d)| d.size.changed()) + .and_then(|c| c) + { + blockdev.set_new_size(size); + } + } + } + Err(e) => { + warn!("Failed to check size of block devices in newly set up pool: {}", e); } } - } - Err(e) => { - warn!("Failed to check size of block devices in newly set up pool: {}", e); - } + (pool_name, *pool_uuid, pool) + }, + AnyPool::V2(ref mut p) => { + match p.blockdevs_mut() { + Ok(blockdevs) => { + for (dev_uuid, tier, blockdev) in blockdevs { + if let Some(size) = + Self::handle_size_change(tier, dev_uuid, blockdev) + .and_then(|(_, d)| d.size.changed()) + .and_then(|c| c) + { + blockdev.set_new_size(size); + } + } + } + Err(e) => { + warn!("Failed to check size of block devices in newly set up pool: {}", e); + } + } + (pool_name, *pool_uuid, pool) + }, } - (pool_name, *pool_uuid, pool) }) }) - .collect::>() + .collect::>() } /// Attempt to set up a pool, starting it if it is not already started. @@ -538,18 +593,18 @@ impl LiminalDevices { /// self.stopped_pools.get(pool_uuid).is_none() fn try_setup_pool( &mut self, - pools: &Table, + pools: &Table, pool_uuid: PoolUuid, device_set: DeviceSet, - ) -> StratisResult<(Name, StratPool)> { + ) -> StratisResult<(Name, AnyPool)> { fn try_setup_pool_failure( - pools: &Table, + pools: &Table, pool_uuid: PoolUuid, luks_info: StratisResult<(Option, MaybeInconsistent>)>, infos: &HashMap, bdas: HashMap, meta_res: StratisResult<(DateTime, PoolSave)>, - ) -> BDARecordResult<(Name, StratPool)> { + ) -> BDARecordResult<(Name, AnyPool)> { let (timestamp, metadata) = match meta_res { Ok(o) => o, Err(e) => return Err((e, bdas)), @@ -620,18 +675,18 @@ impl LiminalDevices { /// self.stopped_pools.get(pool_uuid).is_none() fn try_setup_started_pool( &mut self, - pools: &Table, + pools: &Table, pool_uuid: PoolUuid, device_set: DeviceSet, - ) -> Option<(Name, StratPool)> { + ) -> Option<(Name, AnyPool)> { fn try_setup_started_pool_failure( - pools: &Table, + pools: &Table, pool_uuid: PoolUuid, luks_info: StratisResult<(Option, MaybeInconsistent>)>, infos: &HashMap, bdas: HashMap, meta_res: StratisResult<(DateTime, PoolSave)>, - ) -> BDARecordResult>> { + ) -> BDARecordResult>> { let (timestamp, metadata) = match meta_res { Ok(o) => o, Err(e) => return Err((e, bdas)), @@ -710,7 +765,7 @@ impl LiminalDevices { /// to determine whether the size has indeed changed so we can update it in /// our internal data structures. pub fn block_evaluate_size( - pools: &mut Table, + pools: &mut Table, event: &UdevEngineEvent, ) -> StratisResult> { let mut ret = None; @@ -736,8 +791,17 @@ impl LiminalDevices { let pool_uuid = di.stratis_identifiers().pool_uuid; let dev_uuid = di.stratis_identifiers().device_uuid; if let Some((_, pool)) = pools.get_mut_by_uuid(pool_uuid) { - if let Some((tier, dev)) = pool.get_mut_strat_blockdev(dev_uuid)? { - ret = Self::handle_size_change(tier, dev_uuid, dev); + match pool { + AnyPool::V1(p) => { + if let Some((tier, dev)) = p.get_mut_strat_blockdev(dev_uuid)? { + ret = Self::handle_size_change(tier, dev_uuid, dev); + } + } + AnyPool::V2(p) => { + if let Some((tier, dev)) = p.get_mut_strat_blockdev(dev_uuid)? { + ret = Self::handle_size_change(tier, dev_uuid, dev); + } + } } } } @@ -754,9 +818,9 @@ impl LiminalDevices { /// constructing the pool, retain the set of devices. pub fn block_evaluate( &mut self, - pools: &Table, + pools: &Table, event: &UdevEngineEvent, - ) -> Option<(Name, PoolUuid, StratPool)> { + ) -> Option<(Name, PoolUuid, AnyPool)> { let event_type = event.event_type(); let device_path = match event.device().devnode() { Some(d) => d, @@ -781,10 +845,21 @@ impl LiminalDevices { let pool_uuid = stratis_identifiers.pool_uuid; let device_uuid = stratis_identifiers.device_uuid; if let Some((_, pool)) = pools.get_by_uuid(pool_uuid) { - if pool.get_strat_blockdev(device_uuid).is_none() { - warn!("Found a device with {} that identifies itself as belonging to pool with UUID {}, but that pool is already up and running and does not appear to contain the device", - info, - pool_uuid); + match pool { + AnyPool::V1(p) => { + if p.get_strat_blockdev(device_uuid).is_none() { + warn!("Found a device with {} that identifies itself as belonging to pool with UUID {}, but that pool is already up and running and does not appear to contain the device", + info, + pool_uuid); + } + } + AnyPool::V2(p) => { + if p.get_strat_blockdev(device_uuid).is_none() { + warn!("Found a device with {} that identifies itself as belonging to pool with UUID {}, but that pool is already up and running and does not appear to contain the device", + info, + pool_uuid); + } + } } // FIXME: There might be something to check if the device is // included in the pool, but that is less clear. @@ -876,15 +951,38 @@ impl LiminalDevices { pub fn handle_stopped_pool(&mut self, pool_uuid: PoolUuid, device_set: DeviceSet) { if !device_set.is_empty() { - let device_uuids = device_set - .iter() - .map(|(dev_uuid, _)| *dev_uuid) - .collect::>(); - if has_leftover_devices(pool_uuid, &device_uuids) { - self.partially_constructed_pools - .insert(pool_uuid, device_set); - } else { - self.stopped_pools.insert(pool_uuid, device_set); + match device_set.metadata_version() { + Ok(mv) => { + match mv { + StratSigblockVersion::V1 => { + let dev_uuids = device_set + .iter() + .map(|(dev_uuid, _)| *dev_uuid) + .collect::>(); + if has_leftover_devices_legacy(pool_uuid, &dev_uuids) { + self.partially_constructed_pools + .insert(pool_uuid, device_set); + } else { + self.stopped_pools.insert(pool_uuid, device_set); + } + } + StratSigblockVersion::V2 => { + if has_leftover_devices(pool_uuid) { + self.partially_constructed_pools + .insert(pool_uuid, device_set); + } else { + self.stopped_pools.insert(pool_uuid, device_set); + } + } + }; + } + Err(e) => { + warn!( + "Unable to detect leftover devices: {}; putting in stopped pools", + e + ); + self.stopped_pools.insert(pool_uuid, device_set); + } } } } @@ -988,14 +1086,14 @@ fn load_stratis_metadata( /// If there is a name conflict between the set of devices in devices /// and some existing pool, return an error. fn setup_pool( - pools: &Table, + pools: &Table, pool_uuid: PoolUuid, luks_info: StratisResult<(Option, MaybeInconsistent>)>, infos: &HashMap, bdas: HashMap, timestamp: DateTime, metadata: PoolSave, -) -> BDARecordResult<(Name, StratPool)> { +) -> BDARecordResult<(Name, AnyPool)> { if let Some((uuid, _)) = pools.get_by_name(&metadata.name) { return Err(( StratisError::Msg(format!( @@ -1006,74 +1104,109 @@ fn setup_pool( )), bdas)); } - let (datadevs, cachedevs) = match get_blockdevs(&metadata.backstore, infos, bdas) { - Err((err, bdas)) => return Err( - (StratisError::Chained( - format!( - "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", - pool_uuid, - &metadata.name, - ), - Box::new(err) - ), bdas)), - Ok((datadevs, cachedevs)) => (datadevs, cachedevs), - }; + let bda = bdas.values().next(); + let metadata_version = bda.expect("Must have at least one BDA").sigblock_version(); - if datadevs.first().is_none() { - return Err(( - StratisError::Msg(format!( - "There do not appear to be any data devices in the set with pool UUID {pool_uuid}" - )), - tiers_to_bdas(datadevs, cachedevs, None), - )); - } + match metadata_version { + StratSigblockVersion::V1 => { + let (datadevs, cachedevs) = match get_blockdevs_legacy(&metadata.backstore, infos, bdas) { + Err((err, bdas)) => return Err( + (StratisError::Chained( + format!( + "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", + pool_uuid, + &metadata.name, + ), + Box::new(err) + ), bdas)), + Ok((datadevs, cachedevs)) => (datadevs, cachedevs), + }; - let (pool_einfo, pool_name) = match luks_info { - Ok(inner) => inner, - Err(_) => { - // NOTE: This is not actually a hopeless situation. It may be - // that a LUKS device owned by Stratis corresponding to a - // Stratis device has just not been discovered yet. If it - // is, the appropriate info will be updated, and setup may - // yet succeed. - return Err(( - StratisError::Msg(format!( - "Some data devices in the set belonging to pool with UUID {} and name {} appear to be encrypted devices managed by Stratis, and some do not", - pool_uuid, - &metadata.name - )), tiers_to_bdas(datadevs, cachedevs, None))); - } - }; + let (pool_einfo, pool_name) = match luks_info { + Ok(inner) => inner, + Err(_) => { + // NOTE: This is not actually a hopeless situation. It may be + // that a LUKS device owned by Stratis corresponding to a + // Stratis device has just not been discovered yet. If it + // is, the appropriate info will be updated, and setup may + // yet succeed. + return Err(( + StratisError::Msg(format!( + "Some data devices in the set belonging to pool with UUID {} and name {} appear to be encrypted devices managed by Stratis, and some do not", + pool_uuid, + &metadata.name + )), tiers_to_bdas(datadevs, cachedevs, None))); + } + }; - StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata, pool_einfo) - .map(|(name, mut pool)| { - if matches!(pool_name, MaybeInconsistent::Yes | MaybeInconsistent::No(None)) || MaybeInconsistent::No(Some(&name)) != pool_name.as_ref() || pool.blockdevs().iter().map(|(_, _, bd)| { - bd.pool_name() - }).fold(false, |acc, next| { - match next { - Some(Some(name)) => { - if MaybeInconsistent::No(Some(name)) == pool_name.as_ref() { - acc - } else { - true + v1::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata, pool_einfo) + .map(|(name, mut pool)| { + if matches!(pool_name, MaybeInconsistent::Yes | MaybeInconsistent::No(None)) || MaybeInconsistent::No(Some(&name)) != pool_name.as_ref() || pool.blockdevs().iter().map(|(_, _, bd)| { + bd.pool_name() + }).fold(false, |acc, next| { + match next { + Some(Some(name)) => { + if MaybeInconsistent::No(Some(name)) == pool_name.as_ref() { + acc + } else { + true + } + }, + Some(None) => true, + None => false, } - }, - Some(None) => true, - None => false, - } - }) { - if let Err(e) = pool.rename_pool(&name) { - warn!("Pool will not be able to be started by name; pool name metadata in LUKS2 token is not consistent across all devices: {}", e); - } + }) { + if let Err(e) = pool.rename_pool(&name) { + warn!("Pool will not be able to be started by name; pool name metadata in LUKS2 token is not consistent across all devices: {}", e); + } + } + (name, AnyPool::V1(pool)) + }) + .map_err(|(err, bdas)| { + (StratisError::Chained( + format!( + "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" + ), + Box::new(err), + ), bdas) + }) + } + StratSigblockVersion::V2 => { + let (datadevs, cachedevs) = match get_blockdevs(&metadata.backstore, infos, bdas) { + Err((err, bdas)) => return Err( + (StratisError::Chained( + format!( + "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", + pool_uuid, + &metadata.name, + ), + Box::new(err) + ), bdas)), + Ok((datadevs, cachedevs)) => (datadevs, cachedevs), + }; + + let dev = datadevs.first(); + if dev.is_none() { + return Err(( + StratisError::Msg(format!( + "There do not appear to be any data devices in the set with pool UUID {pool_uuid}" + )), + tiers_to_bdas(datadevs, cachedevs, None), + )); } - (name, pool) - }) - .map_err(|(err, bdas)| { - (StratisError::Chained( - format!( - "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" - ), - Box::new(err), - ), bdas) - }) + + v2::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata) + .map(|(name, pool)| { + (name, AnyPool::V2(pool)) + }) + .map_err(|(err, bdas)| { + (StratisError::Chained( + format!( + "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" + ), + Box::new(err), + ), bdas) + }) + } + } } diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index 143f90a815..8ac4f05334 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -19,8 +19,8 @@ use crate::{ engine::{ strat_engine::{ backstore::blockdev::{ - v1::{StratBlockDev, UnderlyingDevice}, - InternalBlockDev, + v1::{self, UnderlyingDevice}, + v2, InternalBlockDev, }, crypt::handle::v1::CryptHandle, device::blkdev_size, @@ -137,11 +137,11 @@ pub fn get_name(infos: HashMap) -> StratisResult, mut bdas: HashMap, -) -> BDARecordResult<(Vec, Vec)> { +) -> BDARecordResult<(Vec, Vec)> { let recorded_data_map: HashMap = backstore_save .data_tier .blockdev @@ -180,105 +180,11 @@ pub fn get_blockdevs( } } - // Construct a single StratBlockDev. Return the tier to which the - // blockdev has been found to belong. Returns an error if the block - // device has shrunk, no metadata can be found for the block device, - // or it is impossible to set up the device because the recorded - // allocation information is impossible. - fn get_blockdev( - info: &LStratisDevInfo, - bda: BDA, - data_map: &HashMap, - cache_map: &HashMap, - segment_table: &HashMap>, - ) -> BDAResult<(BlockDevTier, StratBlockDev)> { - let actual_size = match OpenOptions::new() - .read(true) - .open(&info.dev_info.devnode) - .map_err(StratisError::from) - .and_then(|f| blkdev_size(&f)) - { - Ok(actual_size) => actual_size, - Err(err) => return Err((err, bda)), - }; - - // Return an error if apparent size of Stratis block device appears to - // have decreased since metadata was recorded or if size of block - // device could not be obtained. - let actual_size_sectors = actual_size.sectors(); - let recorded_size = bda.dev_size().sectors(); - if actual_size_sectors < recorded_size { - let err_msg = format!( - "Stratis device with {}, {} had recorded size {}, but actual size is less at {}", - info.dev_info, - bda.identifiers(), - recorded_size, - actual_size_sectors - ); - return Err((StratisError::Msg(err_msg), bda)); - } - - let dev_uuid = bda.dev_uuid(); - - // Locate the device in the metadata using its uuid. Return the device - // metadata and whether it was a cache or a datadev. - let (tier, &(_, bd_save)) = match data_map - .get(&dev_uuid) - .map(|bd_save| (BlockDevTier::Data, bd_save)) - .or_else(|| { - cache_map - .get(&dev_uuid) - .map(|bd_save| (BlockDevTier::Cache, bd_save)) - }) { - Some(s) => s, - None => { - let err_msg = format!( - "Stratis device with {}, {} had no record in pool metadata", - bda.identifiers(), - info.dev_info - ); - return Err((StratisError::Msg(err_msg), bda)); - } - }; - - // This should always succeed since the actual size is at - // least the recorded size, so all segments should be - // available to be allocated. If this fails, the most likely - // conclusion is metadata corruption. - let segments = segment_table.get(&dev_uuid); - - let physical_path = match &info.luks { - Some(luks) => &luks.dev_info.devnode, - None => &info.dev_info.devnode, - }; - let handle = match CryptHandle::setup(physical_path, None) { - Ok(h) => h, - Err(e) => return Err((e, bda)), - }; - let underlying_device = match handle { - Some(handle) => UnderlyingDevice::Encrypted(handle), - None => UnderlyingDevice::Unencrypted(match DevicePath::new(physical_path) { - Ok(d) => d, - Err(e) => return Err((e, bda)), - }), - }; - Ok(( - tier, - StratBlockDev::new( - info.dev_info.device_number, - bda, - segments.unwrap_or(&vec![]), - bd_save.user_info.clone(), - bd_save.hardware_info.clone(), - underlying_device, - )?, - )) - } - - let (mut datadevs, mut cachedevs): (Vec, Vec) = (vec![], vec![]); + let (mut datadevs, mut cachedevs): (Vec, Vec) = + (vec![], vec![]); let dev_uuids = infos.keys().collect::>(); for dev_uuid in dev_uuids { - match get_blockdev( + match get_blockdev_legacy( infos.get(dev_uuid).expect("bdas.keys() == infos.keys()"), bdas.remove(dev_uuid).expect("bdas.keys() == infos.keys()"), &recorded_data_map, @@ -294,58 +200,105 @@ pub fn get_blockdevs( } } - // Verify that devices located are consistent with the metadata recorded - // and generally consistent with expectations. If all seems correct, - // sort the devices according to their order in the metadata. - fn check_and_sort_devs( - mut devs: Vec, - dev_map: &HashMap, - ) -> BDARecordResult> { - let mut uuids = HashSet::new(); - let mut duplicate_uuids = Vec::new(); - let mut metadata_version = HashSet::new(); - for dev in &devs { - let dev_uuid = dev.uuid(); - if !uuids.insert(dev_uuid) { - duplicate_uuids.push(dev_uuid); - } - metadata_version.insert(dev.metadata_version()); - } - - if !duplicate_uuids.is_empty() { - let err_msg = format!( - "The following list of Stratis UUIDs were each claimed by more than one Stratis device: {}", - duplicate_uuids.iter().map(|u| u.to_string()).collect::>().join(", ") - ); - return Err((StratisError::Msg(err_msg), bds_to_bdas(devs))); + let datadevs = match check_and_sort_devs(datadevs, &recorded_data_map) { + Ok(dd) => dd, + Err((err, mut bdas)) => { + bdas.extend(bds_to_bdas(cachedevs)); + return Err(( + StratisError::Msg(format!( + "Data devices did not appear consistent with metadata: {err}" + )), + bdas, + )); } + }; - if metadata_version.len() > 1 { + let cachedevs = match check_and_sort_devs(cachedevs, &recorded_cache_map) { + Ok(cd) => cd, + Err((err, mut bdas)) => { + bdas.extend(bds_to_bdas(datadevs)); return Err(( StratisError::Msg(format!( - "Found mismatching metadata versions across block devices: {:?}", - metadata_version, + "Cache devices did not appear consistent with metadata: {err}" )), - bds_to_bdas(devs), + bdas, )); } + }; - let recorded_uuids: HashSet<_> = dev_map.keys().cloned().collect(); - if uuids != recorded_uuids { - let err_msg = format!( - "UUIDs of devices found ({}) did not correspond with UUIDs specified in the metadata for this group of devices ({})", - uuids.iter().map(|u| u.to_string()).collect::>().join(", "), - recorded_uuids.iter().map(|u| u.to_string()).collect::>().join(", "), - ); - return Err((StratisError::Msg(err_msg), bds_to_bdas(devs))); + Ok((datadevs, cachedevs)) +} + +/// Get all the blockdevs corresponding to this pool that can be obtained from +/// the given devices. Sort the blockdevs in the order in which they were +/// recorded in the metadata. +/// Returns an error if the blockdevs obtained do not match the metadata. +/// Returns a tuple, of which the first are the data devs, and the second +/// are the devs that support the cache tier. +/// Precondition: Every device in infos has already been determined to +/// belong to one pool; all BDAs agree on their pool UUID, set of keys in +/// infos and bdas are identical. +pub fn get_blockdevs( + backstore_save: &BackstoreSave, + infos: &HashMap, + mut bdas: HashMap, +) -> BDARecordResult<(Vec, Vec)> { + let recorded_data_map: HashMap = backstore_save + .data_tier + .blockdev + .devs + .iter() + .enumerate() + .map(|(i, bds)| (bds.uuid, (i, bds))) + .collect(); + + let recorded_cache_map: HashMap = + match backstore_save.cache_tier { + Some(ref cache_tier) => cache_tier + .blockdev + .devs + .iter() + .enumerate() + .map(|(i, bds)| (bds.uuid, (i, bds))) + .collect(), + None => HashMap::new(), + }; + + let mut segment_table: HashMap> = HashMap::new(); + for seg in &backstore_save.data_tier.blockdev.allocs[0] { + segment_table + .entry(seg.parent) + .or_default() + .push((seg.start, seg.length)) + } + + if let Some(ref cache_tier) = backstore_save.cache_tier { + for seg in cache_tier.blockdev.allocs.iter().flat_map(|i| i.iter()) { + segment_table + .entry(seg.parent) + .or_default() + .push((seg.start, seg.length)) } + } - // Sort the devices according to their original location in the - // metadata. Use a faster unstable sort, because the order of - // devs before the sort is arbitrary and does not need to be - // preserved. - devs.sort_unstable_by_key(|dev| dev_map[&dev.uuid()].0); - Ok(devs) + let (mut datadevs, mut cachedevs): (Vec, Vec) = + (vec![], vec![]); + let dev_uuids = infos.keys().collect::>(); + for dev_uuid in dev_uuids { + match get_blockdev( + infos.get(dev_uuid).expect("bdas.keys() == infos.keys()"), + bdas.remove(dev_uuid).expect("bdas.keys() == infos.keys()"), + &recorded_data_map, + &recorded_cache_map, + &segment_table, + ) { + Ok((tier, blockdev)) => match tier { + BlockDevTier::Data => &mut datadevs, + BlockDevTier::Cache => &mut cachedevs, + } + .push(blockdev), + Err((e, bda)) => return Err((e, tiers_to_bdas(datadevs, cachedevs, Some(bda)))), + } } let datadevs = match check_and_sort_devs(datadevs, &recorded_data_map) { @@ -376,3 +329,241 @@ pub fn get_blockdevs( Ok((datadevs, cachedevs)) } + +// Construct a single legacy StratBlockDev. Return the tier to which the +// blockdev has been found to belong. Returns an error if the block +// device has shrunk, no metadata can be found for the block device, +// or it is impossible to set up the device because the recorded +// allocation information is impossible. +fn get_blockdev_legacy( + info: &LStratisDevInfo, + bda: BDA, + data_map: &HashMap, + cache_map: &HashMap, + segment_table: &HashMap>, +) -> BDAResult<(BlockDevTier, v1::StratBlockDev)> { + let actual_size = match OpenOptions::new() + .read(true) + .open(&info.dev_info.devnode) + .map_err(StratisError::from) + .and_then(|f| blkdev_size(&f)) + { + Ok(actual_size) => actual_size, + Err(err) => return Err((err, bda)), + }; + + // Return an error if apparent size of Stratis block device appears to + // have decreased since metadata was recorded or if size of block + // device could not be obtained. + let actual_size_sectors = actual_size.sectors(); + let recorded_size = bda.dev_size().sectors(); + if actual_size_sectors < recorded_size { + let err_msg = format!( + "Stratis device with {}, {} had recorded size {}, but actual size is less at {}", + info.dev_info, + bda.identifiers(), + recorded_size, + actual_size_sectors + ); + return Err((StratisError::Msg(err_msg), bda)); + } + + let dev_uuid = bda.dev_uuid(); + + // Locate the device in the metadata using its uuid. Return the device + // metadata and whether it was a cache or a datadev. + let (tier, &(_, bd_save)) = match data_map + .get(&dev_uuid) + .map(|bd_save| (BlockDevTier::Data, bd_save)) + .or_else(|| { + cache_map + .get(&dev_uuid) + .map(|bd_save| (BlockDevTier::Cache, bd_save)) + }) { + Some(s) => s, + None => { + let err_msg = format!( + "Stratis device with {}, {} had no record in pool metadata", + bda.identifiers(), + info.dev_info + ); + return Err((StratisError::Msg(err_msg), bda)); + } + }; + + // This should always succeed since the actual size is at + // least the recorded size, so all segments should be + // available to be allocated. If this fails, the most likely + // conclusion is metadata corruption. + let segments = segment_table.get(&dev_uuid); + + let physical_path = match &info.luks { + Some(luks) => &luks.dev_info.devnode, + None => &info.dev_info.devnode, + }; + let handle = match CryptHandle::setup(physical_path, None) { + Ok(h) => h, + Err(e) => return Err((e, bda)), + }; + let underlying_device = match handle { + Some(handle) => UnderlyingDevice::Encrypted(handle), + None => UnderlyingDevice::Unencrypted(match DevicePath::new(physical_path) { + Ok(d) => d, + Err(e) => return Err((e, bda)), + }), + }; + Ok(( + tier, + v1::StratBlockDev::new( + info.dev_info.device_number, + bda, + segments.unwrap_or(&vec![]), + bd_save.user_info.clone(), + bd_save.hardware_info.clone(), + underlying_device, + )?, + )) +} + +// Construct a single StratBlockDev. Return the tier to which the +// blockdev has been found to belong. Returns an error if the block +// device has shrunk, no metadata can be found for the block device, +// or it is impossible to set up the device because the recorded +// allocation information is impossible. +fn get_blockdev( + info: &LStratisDevInfo, + bda: BDA, + data_map: &HashMap, + cache_map: &HashMap, + segment_table: &HashMap>, +) -> BDAResult<(BlockDevTier, v2::StratBlockDev)> { + let actual_size = match OpenOptions::new() + .read(true) + .open(&info.dev_info.devnode) + .map_err(StratisError::from) + .and_then(|f| blkdev_size(&f)) + { + Ok(actual_size) => actual_size, + Err(err) => return Err((err, bda)), + }; + + // Return an error if apparent size of Stratis block device appears to + // have decreased since metadata was recorded or if size of block + // device could not be obtained. + let actual_size_sectors = actual_size.sectors(); + let recorded_size = bda.dev_size().sectors(); + if actual_size_sectors < recorded_size { + let err_msg = format!( + "Stratis device with {}, {} had recorded size {}, but actual size is less at {}", + info.dev_info, + bda.identifiers(), + recorded_size, + actual_size_sectors + ); + return Err((StratisError::Msg(err_msg), bda)); + } + + let dev_uuid = bda.dev_uuid(); + + // Locate the device in the metadata using its uuid. Return the device + // metadata and whether it was a cache or a datadev. + let (tier, &(_, bd_save)) = match data_map + .get(&dev_uuid) + .map(|bd_save| (BlockDevTier::Data, bd_save)) + .or_else(|| { + cache_map + .get(&dev_uuid) + .map(|bd_save| (BlockDevTier::Cache, bd_save)) + }) { + Some(s) => s, + None => { + let err_msg = format!( + "Stratis device with {}, {} had no record in pool metadata", + bda.identifiers(), + info.dev_info + ); + return Err((StratisError::Msg(err_msg), bda)); + } + }; + + let devnode = match DevicePath::new(&info.dev_info.devnode) { + Ok(d) => d, + Err(e) => return Err((e, bda)), + }; + + // This should always succeed since the actual size is at + // least the recorded size, so all segments should be + // available to be allocated. If this fails, the most likely + // conclusion is metadata corruption. + let segments = segment_table.get(&dev_uuid); + + assert_eq!(info.luks, None); + Ok(( + tier, + v2::StratBlockDev::new( + info.dev_info.device_number, + bda, + segments.unwrap_or(&vec![]), + bd_save.user_info.clone(), + bd_save.hardware_info.clone(), + devnode, + )?, + )) +} + +// Verify that devices located are consistent with the metadata recorded +// and generally consistent with expectations. If all seems correct, +// sort the devices according to their order in the metadata. +fn check_and_sort_devs( + mut devs: Vec, + dev_map: &HashMap, +) -> BDARecordResult> +where + B: InternalBlockDev, +{ + let mut uuids = HashSet::new(); + let mut duplicate_uuids = Vec::new(); + let mut metadata_version = HashSet::new(); + for dev in &devs { + let dev_uuid = dev.uuid(); + if !uuids.insert(dev_uuid) { + duplicate_uuids.push(dev_uuid); + } + metadata_version.insert(dev.metadata_version()); + } + + if !duplicate_uuids.is_empty() { + let err_msg = format!( + "The following list of Stratis UUIDs were each claimed by more than one Stratis device: {}", + duplicate_uuids.iter().map(|u| u.to_string()).collect::>().join(", ") + ); + return Err((StratisError::Msg(err_msg), bds_to_bdas(devs))); + } + + if metadata_version.len() > 1 { + return Err(( + StratisError::Msg(format!( + "Found mismatching metadata versions across block devices: {:?}", + metadata_version, + )), + bds_to_bdas(devs), + )); + } + + let recorded_uuids: HashSet<_> = dev_map.keys().cloned().collect(); + if uuids != recorded_uuids { + let err_msg = format!( + "UUIDs of devices found ({}) did not correspond with UUIDs specified in the metadata for this group of devices ({})", + uuids.iter().map(|u| u.to_string()).collect::>().join(", "), + recorded_uuids.iter().map(|u| u.to_string()).collect::>().join(", "), + ); + return Err((StratisError::Msg(err_msg), bds_to_bdas(devs))); + } + + // Sort the devices according to their original location in the + // metadata. Use a faster unstable sort, because the order of + // devs before the sort is arbitrary and does not need to be + // preserved. + devs.sort_unstable_by_key(|dev| dev_map[&dev.uuid()].0); + Ok(devs) +} diff --git a/src/engine/strat_engine/pool/mod.rs b/src/engine/strat_engine/pool/mod.rs new file mode 100644 index 0000000000..195e618267 --- /dev/null +++ b/src/engine/strat_engine/pool/mod.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +mod shared; +pub mod v1; +pub mod v2; + +pub use shared::AnyPool; diff --git a/src/engine/strat_engine/pool/shared.rs b/src/engine/strat_engine/pool/shared.rs new file mode 100644 index 0000000000..46099a7470 --- /dev/null +++ b/src/engine/strat_engine/pool/shared.rs @@ -0,0 +1,344 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::path::Path; + +use serde_json::Value; + +use devicemapper::{Bytes, Sectors}; + +use crate::{ + engine::{ + engine::{BlockDev, Filesystem, Pool}, + strat_engine::pool::{v1, v2}, + types::{ + ActionAvailability, BlockDevTier, Clevis, CreateAction, DeleteAction, DevUuid, + FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, + PoolUuid, PropChangeAction, RegenAction, RenameAction, SetCreateAction, + SetDeleteAction, + }, + }, + stratis::StratisResult, +}; + +#[derive(Debug)] +pub enum AnyPool { + V1(v1::StratPool), + V2(v2::StratPool), +} + +impl Pool for AnyPool { + fn init_cache( + &mut self, + pool_uuid: PoolUuid, + pool_name: &str, + blockdevs: &[&Path], + supports_encrypted: bool, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.init_cache(pool_uuid, pool_name, blockdevs, supports_encrypted), + AnyPool::V2(p) => p.init_cache(pool_uuid, pool_name, blockdevs, supports_encrypted), + } + } + + fn bind_clevis( + &mut self, + pin: &str, + clevis_info: &Value, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.bind_clevis(pin, clevis_info), + AnyPool::V2(p) => p.bind_clevis(pin, clevis_info), + } + } + + fn unbind_clevis(&mut self) -> StratisResult> { + match self { + AnyPool::V1(p) => p.unbind_clevis(), + AnyPool::V2(p) => p.unbind_clevis(), + } + } + + fn bind_keyring( + &mut self, + key_description: &KeyDescription, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.bind_keyring(key_description), + AnyPool::V2(p) => p.bind_keyring(key_description), + } + } + + fn unbind_keyring(&mut self) -> StratisResult> { + match self { + AnyPool::V1(p) => p.unbind_keyring(), + AnyPool::V2(p) => p.unbind_keyring(), + } + } + + fn rebind_keyring( + &mut self, + new_key_desc: &KeyDescription, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.rebind_keyring(new_key_desc), + AnyPool::V2(p) => p.rebind_keyring(new_key_desc), + } + } + + fn rebind_clevis(&mut self) -> StratisResult { + match self { + AnyPool::V1(p) => p.rebind_clevis(), + AnyPool::V2(p) => p.rebind_clevis(), + } + } + + fn create_filesystems<'a>( + &mut self, + pool_name: &str, + pool_uuid: PoolUuid, + specs: &[(&'a str, Option, Option)], + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.create_filesystems(pool_name, pool_uuid, specs), + AnyPool::V2(p) => p.create_filesystems(pool_name, pool_uuid, specs), + } + } + + fn add_blockdevs( + &mut self, + pool_uuid: PoolUuid, + pool_name: &str, + paths: &[&Path], + tier: BlockDevTier, + ) -> StratisResult<(SetCreateAction, Option)> { + match self { + AnyPool::V1(p) => p.add_blockdevs(pool_uuid, pool_name, paths, tier), + AnyPool::V2(p) => p.add_blockdevs(pool_uuid, pool_name, paths, tier), + } + } + + fn destroy_filesystems( + &mut self, + pool_name: &str, + fs_uuids: &[FilesystemUuid], + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.destroy_filesystems(pool_name, fs_uuids), + AnyPool::V2(p) => p.destroy_filesystems(pool_name, fs_uuids), + } + } + + fn rename_filesystem( + &mut self, + pool_name: &str, + uuid: FilesystemUuid, + new_name: &str, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.rename_filesystem(pool_name, uuid, new_name), + AnyPool::V2(p) => p.rename_filesystem(pool_name, uuid, new_name), + } + } + + fn snapshot_filesystem<'a>( + &'a mut self, + pool_name: &str, + pool_uuid: PoolUuid, + origin_uuid: FilesystemUuid, + snapshot_name: &str, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => { + p.snapshot_filesystem(pool_name, pool_uuid, origin_uuid, snapshot_name) + } + AnyPool::V2(p) => { + p.snapshot_filesystem(pool_name, pool_uuid, origin_uuid, snapshot_name) + } + } + } + + fn total_physical_size(&self) -> Sectors { + match self { + AnyPool::V1(p) => p.total_physical_size(), + AnyPool::V2(p) => p.total_physical_size(), + } + } + + fn total_allocated_size(&self) -> Sectors { + match self { + AnyPool::V1(p) => p.total_allocated_size(), + AnyPool::V2(p) => p.total_allocated_size(), + } + } + + fn total_physical_used(&self) -> Option { + match self { + AnyPool::V1(p) => p.total_physical_used(), + AnyPool::V2(p) => p.total_physical_used(), + } + } + + fn filesystems(&self) -> Vec<(Name, FilesystemUuid, &dyn Filesystem)> { + match self { + AnyPool::V1(p) => p.filesystems(), + AnyPool::V2(p) => p.filesystems(), + } + } + + fn get_filesystem(&self, uuid: FilesystemUuid) -> Option<(Name, &dyn Filesystem)> { + match self { + AnyPool::V1(p) => p.get_filesystem(uuid), + AnyPool::V2(p) => p.get_filesystem(uuid), + } + } + + fn get_filesystem_by_name(&self, fs_name: &Name) -> Option<(FilesystemUuid, &dyn Filesystem)> { + match self { + AnyPool::V1(p) => p.get_filesystem_by_name(fs_name), + AnyPool::V2(p) => p.get_filesystem_by_name(fs_name), + } + } + + fn blockdevs(&self) -> Vec<(DevUuid, BlockDevTier, &dyn BlockDev)> { + match self { + AnyPool::V1(p) => ::blockdevs(p), + AnyPool::V2(p) => ::blockdevs(p), + } + } + + fn get_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &dyn BlockDev)> { + match self { + AnyPool::V1(p) => p.get_blockdev(uuid), + AnyPool::V2(p) => p.get_blockdev(uuid), + } + } + + fn get_mut_blockdev( + &mut self, + uuid: DevUuid, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.get_mut_blockdev(uuid), + AnyPool::V2(p) => p.get_mut_blockdev(uuid), + } + } + + fn set_blockdev_user_info( + &mut self, + pool_name: &str, + uuid: DevUuid, + user_info: Option<&str>, + ) -> StratisResult> { + match self { + AnyPool::V1(p) => p.set_blockdev_user_info(pool_name, uuid, user_info), + AnyPool::V2(p) => p.set_blockdev_user_info(pool_name, uuid, user_info), + } + } + + fn has_cache(&self) -> bool { + match self { + AnyPool::V1(p) => p.has_cache(), + AnyPool::V2(p) => p.has_cache(), + } + } + + fn is_encrypted(&self) -> bool { + match self { + AnyPool::V1(p) => p.is_encrypted(), + AnyPool::V2(p) => p.is_encrypted(), + } + } + + fn encryption_info(&self) -> Option { + match self { + AnyPool::V1(p) => p.encryption_info(), + AnyPool::V2(p) => p.encryption_info(), + } + } + + fn avail_actions(&self) -> ActionAvailability { + match self { + AnyPool::V1(p) => p.avail_actions(), + AnyPool::V2(p) => p.avail_actions(), + } + } + + fn fs_limit(&self) -> u64 { + match self { + AnyPool::V1(p) => p.fs_limit(), + AnyPool::V2(p) => p.fs_limit(), + } + } + + fn set_fs_limit( + &mut self, + pool_name: &Name, + pool_uuid: PoolUuid, + new_limit: u64, + ) -> StratisResult<()> { + match self { + AnyPool::V1(p) => p.set_fs_limit(pool_name, pool_uuid, new_limit), + AnyPool::V2(p) => p.set_fs_limit(pool_name, pool_uuid, new_limit), + } + } + + fn overprov_enabled(&self) -> bool { + match self { + AnyPool::V1(p) => p.overprov_enabled(), + AnyPool::V2(p) => p.overprov_enabled(), + } + } + + fn set_overprov_mode(&mut self, pool_name: &Name, enabled: bool) -> StratisResult<()> { + match self { + AnyPool::V1(p) => p.set_overprov_mode(pool_name, enabled), + AnyPool::V2(p) => p.set_overprov_mode(pool_name, enabled), + } + } + + fn out_of_alloc_space(&self) -> bool { + match self { + AnyPool::V1(p) => p.out_of_alloc_space(), + AnyPool::V2(p) => p.out_of_alloc_space(), + } + } + + fn grow_physical( + &mut self, + name: &Name, + pool_uuid: PoolUuid, + device: DevUuid, + ) -> StratisResult<(GrowAction<(PoolUuid, DevUuid)>, Option)> { + match self { + AnyPool::V1(p) => p.grow_physical(name, pool_uuid, device), + AnyPool::V2(p) => p.grow_physical(name, pool_uuid, device), + } + } + + fn set_fs_size_limit( + &mut self, + fs: FilesystemUuid, + limit: Option, + ) -> StratisResult>> { + match self { + AnyPool::V1(p) => p.set_fs_size_limit(fs, limit), + AnyPool::V2(p) => p.set_fs_size_limit(fs, limit), + } + } + + fn current_metadata(&self, pool_name: &Name) -> StratisResult { + match self { + AnyPool::V1(p) => p.current_metadata(pool_name), + AnyPool::V2(p) => p.current_metadata(pool_name), + } + } + + fn last_metadata(&self) -> StratisResult { + match self { + AnyPool::V1(p) => p.last_metadata(), + AnyPool::V2(p) => p.last_metadata(), + } + } +} diff --git a/src/engine/strat_engine/pool.rs b/src/engine/strat_engine/pool/v1.rs similarity index 99% rename from src/engine/strat_engine/pool.rs rename to src/engine/strat_engine/pool/v1.rs index bacb1d1889..4835071316 100644 --- a/src/engine/strat_engine/pool.rs +++ b/src/engine/strat_engine/pool/v1.rs @@ -10,6 +10,15 @@ use serde_json::{Map, Value}; use devicemapper::{Bytes, DmNameBuf, Sectors}; use stratisd_proc_macros::strat_pool_impl_gen; +#[cfg(test)] +use crate::engine::{ + strat_engine::{ + backstore::UnownedDevices, + metadata::MDADataSize, + thinpool::{ThinPoolSizeParams, DATA_BLOCK_SIZE}, + }, + types::EncryptionInfo, +}; use crate::{ engine::{ engine::{BlockDev, DumpState, Filesystem, Pool, StateDiff}, @@ -21,18 +30,18 @@ use crate::{ backstore::{ backstore::{v1::Backstore, InternalBackstore}, blockdev::{v1::StratBlockDev, InternalBlockDev}, - ProcessedPathInfos, UnownedDevices, + ProcessedPathInfos, }, liminal::DeviceSet, - metadata::{MDADataSize, BDA}, + metadata::BDA, serde_structs::{FlexDevsSave, PoolSave, Recordable}, shared::tiers_to_bdas, - thinpool::{StratFilesystem, ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE}, + thinpool::{StratFilesystem, ThinPool}, types::BDARecordResult, }, types::{ ActionAvailability, BlockDevTier, Clevis, Compare, CreateAction, DeleteAction, DevUuid, - Diff, EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, + Diff, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, PoolUuid, RegenAction, RenameAction, SetCreateAction, SetDeleteAction, StratFilesystemDiff, StratPoolDiff, }, @@ -169,6 +178,7 @@ impl StratPool { /// 1. Initialize the block devices specified by paths. /// 2. Set up thinpool device to back filesystems. /// Precondition: p.is_absolute() is true for all p in paths + #[cfg(test)] pub fn initialize( name: &str, devices: UnownedDevices, @@ -1337,6 +1347,7 @@ mod tests { use crate::engine::{ strat_engine::{ cmd::udev_settle, + pool::AnyPool, tests::{loopbacked, real}, thinpool::ThinPoolStatusDigest, }, @@ -1807,7 +1818,10 @@ mod tests { while !pool.out_of_alloc_space() { f.write_all(write_block).unwrap(); f.sync_all().unwrap(); - pool.event_on(pool_uuid, &pool_name).unwrap(); + match pool { + AnyPool::V1(p) => p.event_on(pool_uuid, &pool_name).unwrap(), + AnyPool::V2(p) => p.event_on(pool_uuid, &pool_name).unwrap(), + }; } } diff --git a/src/engine/strat_engine/pool/v2.rs b/src/engine/strat_engine/pool/v2.rs new file mode 100644 index 0000000000..01c98ac1d8 --- /dev/null +++ b/src/engine/strat_engine/pool/v2.rs @@ -0,0 +1,1767 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{collections::HashMap, path::Path, vec::Vec}; + +use chrono::{DateTime, Utc}; +use serde_json::{Map, Value}; + +use devicemapper::{Bytes, DmNameBuf, Sectors}; +use stratisd_proc_macros::strat_pool_impl_gen; + +use crate::{ + engine::{ + engine::{BlockDev, DumpState, Filesystem, Pool, StateDiff}, + shared::{ + init_cache_idempotent_or_err, validate_filesystem_size, validate_filesystem_size_specs, + validate_name, validate_paths, + }, + strat_engine::{ + backstore::{ + backstore::{v2::Backstore, InternalBackstore}, + blockdev::{v2::StratBlockDev, InternalBlockDev}, + ProcessedPathInfos, UnownedDevices, + }, + liminal::DeviceSet, + metadata::{MDADataSize, BDA}, + serde_structs::{FlexDevsSave, PoolSave, Recordable}, + shared::tiers_to_bdas, + thinpool::{StratFilesystem, ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE}, + types::BDARecordResult, + }, + types::{ + ActionAvailability, BlockDevTier, Clevis, Compare, CreateAction, DeleteAction, DevUuid, + Diff, EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, + PoolEncryptionInfo, PoolUuid, PropChangeAction, RegenAction, RenameAction, + SetCreateAction, SetDeleteAction, StratFilesystemDiff, StratPoolDiff, + }, + }, + stratis::{StratisError, StratisResult}, +}; + +/// Get the index which indicates the start of unallocated space in the cap +/// device. +/// NOTE: Since segments are always allocated to each flex dev in order, the +/// last segment for each is the highest. This allows avoiding sorting all the +/// segments and just sorting the set consisting of the last segment from +/// each list of segments. +/// Precondition: This method is called only when setting up a pool, which +/// ensures that the flex devs metadata lists are all non-empty. +fn next_index(flex_devs: &FlexDevsSave) -> Sectors { + let expect_msg = "Setting up rather than initializing a pool, so each flex dev must have been allocated at least some segments."; + [ + flex_devs + .meta_dev + .last() + .unwrap_or_else(|| panic!("{}", expect_msg)), + flex_devs + .thin_meta_dev + .last() + .unwrap_or_else(|| panic!("{}", expect_msg)), + flex_devs + .thin_data_dev + .last() + .unwrap_or_else(|| panic!("{}", expect_msg)), + flex_devs + .thin_meta_dev_spare + .last() + .unwrap_or_else(|| panic!("{}", expect_msg)), + ] + .iter() + .max_by_key(|x| x.0) + .map(|&&(start, length)| start + length) + .expect("iterator is non-empty") +} + +/// Check the metadata of an individual pool for consistency. +/// Precondition: This method is called only when setting up a pool, which +/// ensures that the flex devs metadata lists are all non-empty. +fn check_metadata(metadata: &PoolSave) -> StratisResult<()> { + let flex_devs = &metadata.flex_devs; + let next = next_index(flex_devs); + let allocated_from_cap = metadata.backstore.cap.allocs[0].1; + + if allocated_from_cap != next { + let err_msg = format!( + "{next} used in thinpool, but {allocated_from_cap} allocated from backstore cap device" + ); + return Err(StratisError::Msg(err_msg)); + } + + // If the total length of the allocations in the flex devs, does not + // equal next, consider the situation an error. + { + let total_allocated = flex_devs + .meta_dev + .iter() + .chain(flex_devs.thin_meta_dev.iter()) + .chain(flex_devs.thin_data_dev.iter()) + .chain(flex_devs.thin_meta_dev_spare.iter()) + .map(|x| x.1) + .sum::(); + if total_allocated != next { + let err_msg = format!( + "{} used in thinpool, but {} given up by cache for pool {}", + total_allocated, next, metadata.name + ); + return Err(StratisError::Msg(err_msg)); + } + } + + // If the amount allocated to the cap device is less than the amount + // allocated to the flex devices, consider the situation an error. + // Consider it an error if the amount allocated to the cap device is 0. + // If this is the case, then the thin pool can not exist. + { + let total_allocated = metadata.backstore.data_tier.blockdev.allocs[0] + .iter() + .map(|x| x.length) + .sum::(); + + if total_allocated == Sectors(0) { + let err_msg = format!( + "no segments allocated to the cap device for pool {}", + metadata.name + ); + return Err(StratisError::Msg(err_msg)); + } + + if next > total_allocated { + let err_msg = format!( + "{next} allocated to cap device, but {total_allocated} allocated to flex devs" + ); + return Err(StratisError::Msg(err_msg)); + } + } + + Ok(()) +} + +#[derive(Debug)] +pub struct StratPool { + backstore: Backstore, + thin_pool: ThinPool, + action_avail: ActionAvailability, + metadata_size: Sectors, +} + +#[strat_pool_impl_gen] +impl StratPool { + /// Initialize a Stratis Pool. + /// 1. Initialize the block devices specified by paths. + /// 2. Set up thinpool device to back filesystems. + /// Precondition: p.is_absolute() is true for all p in paths + pub fn initialize( + name: &str, + devices: UnownedDevices, + encryption_info: Option<&EncryptionInfo>, + ) -> StratisResult<(PoolUuid, StratPool)> { + let pool_uuid = PoolUuid::new_v4(); + + // FIXME: Initializing with the minimum MDA size is not necessarily + // enough. If there are enough devices specified, more space will be + // required. + let mut backstore = + Backstore::initialize(pool_uuid, devices, MDADataSize::default(), encryption_info)?; + + let thinpool = ThinPool::::new( + pool_uuid, + match ThinPoolSizeParams::new(backstore.datatier_usable_size()) { + Ok(ref params) => params, + Err(causal_error) => { + if let Err(cleanup_err) = backstore.destroy(pool_uuid) { + warn!("Failed to clean up Stratis metadata for incompletely set up pool with UUID {}: {}.", pool_uuid, cleanup_err); + return Err(StratisError::NoActionRollbackError { + causal_error: Box::new(causal_error), + rollback_error: Box::new(cleanup_err), + }); + } + return Err(causal_error); + } + }, + DATA_BLOCK_SIZE, + &mut backstore, + ); + + let thinpool = match thinpool { + Ok(thinpool) => thinpool, + Err(causal_error) => { + if let Err(cleanup_err) = backstore.destroy(pool_uuid) { + warn!("Failed to clean up Stratis metadata for incompletely set up pool with UUID {}: {}.", pool_uuid, cleanup_err); + return Err(StratisError::NoActionRollbackError { + causal_error: Box::new(causal_error), + rollback_error: Box::new(cleanup_err), + }); + } + return Err(causal_error); + } + }; + + let metadata_size = backstore.datatier_metadata_size(); + let mut pool = StratPool { + backstore, + thin_pool: thinpool, + action_avail: ActionAvailability::Full, + metadata_size, + }; + + pool.write_metadata(&Name::new(name.to_owned()))?; + + Ok((pool_uuid, pool)) + } + + /// Setup a StratPool using its UUID and the list of devnodes it has. + /// Precondition: every device in devnodes has already been determined + /// to belong to the pool with the specified uuid. + /// Precondition: A metadata verification step has already been run. + /// + /// Precondition: + /// * key_description.is_some() -> every StratBlockDev in datadevs has a + /// key description and that key description == key_description + /// * key_description.is_none() -> no StratBlockDev in datadevs has a + /// key description. + /// * no StratBlockDev in cachdevs has a key description + pub fn setup( + uuid: PoolUuid, + datadevs: Vec, + cachedevs: Vec, + timestamp: DateTime, + metadata: &PoolSave, + ) -> BDARecordResult<(Name, StratPool)> { + if let Err(e) = check_metadata(metadata) { + return Err((e, tiers_to_bdas(datadevs, cachedevs, None))); + } + + let backstore = + Backstore::setup(uuid, &metadata.backstore, datadevs, cachedevs, timestamp)?; + let action_avail = backstore.action_availability(); + + let pool_name = &metadata.name; + + if action_avail != ActionAvailability::Full { + warn!( + "Disabling some actions for pool {} with UUID {}; pool is designated {}", + pool_name, uuid, action_avail + ); + } + + let thinpool = match ThinPool::setup( + pool_name, + uuid, + &metadata.thinpool_dev, + &metadata.flex_devs, + &backstore, + ) { + Ok(tp) => tp, + Err(e) => return Err((e, backstore.into_bdas())), + }; + + // TODO: Remove in stratisd 4.0 + let mut needs_save = metadata.thinpool_dev.fs_limit.is_none() + || metadata.thinpool_dev.feature_args.is_none(); + + let metadata_size = backstore.datatier_metadata_size(); + let mut pool = StratPool { + backstore, + thin_pool: thinpool, + action_avail, + metadata_size, + }; + + // The value of the started field in the pool metadata needs to be + // updated unless the value is already present in the metadata and has + // value true. + needs_save |= !metadata.started.unwrap_or(false); + + if needs_save { + if let Err(err) = pool.write_metadata(pool_name) { + if let StratisError::ActionDisabled(avail) = err { + warn!("Pool-level metadata could not be written for pool with name {} and UUID {} because pool is in a limited availability state, {}, which prevents any pool actions; pool will remain set up", pool_name, uuid, avail); + } else { + return Err((err, pool.backstore.into_bdas())); + } + } + } + + Ok((Name::new(pool_name.to_owned()), pool)) + } + + fn get_filesystem(&self, uuid: FilesystemUuid) -> Option<(Name, &StratFilesystem)> { + self.thin_pool.get_filesystem_by_uuid(uuid) + } + + fn get_filesystem_by_name(&self, fs_name: &Name) -> Option<(FilesystemUuid, &StratFilesystem)> { + self.thin_pool.get_filesystem_by_name(fs_name) + } + + /// Send a synthetic udev change event to every filesystem on the given pool. + pub fn udev_pool_change(&self, pool_name: &str) { + for (name, uuid, fs) in self.thin_pool.filesystems() { + fs.udev_fs_change(pool_name, uuid, &name); + } + } + + /// Write current metadata to pool members. + #[pool_mutating_action("NoPoolChanges")] + pub fn write_metadata(&mut self, name: &str) -> StratisResult<()> { + let data = serde_json::to_string(&self.record(name))?; + self.backstore.save_state(data.as_bytes()) + } + + /// Teardown a pool. + #[cfg(test)] + pub fn teardown(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { + self.thin_pool.teardown(pool_uuid).map_err(|(e, _)| e)?; + self.backstore.teardown(pool_uuid)?; + Ok(()) + } + + pub fn has_filesystems(&self) -> bool { + self.thin_pool.has_filesystems() + } + + /// The names of DM devices belonging to this pool that may generate events + pub fn get_eventing_dev_names(&self, pool_uuid: PoolUuid) -> Vec { + self.thin_pool.get_eventing_dev_names(pool_uuid) + } + + /// Called when a DM device in this pool has generated an event. This method + /// handles checking pools. + #[pool_mutating_action("NoPoolChanges")] + pub fn event_on(&mut self, pool_uuid: PoolUuid, pool_name: &Name) -> StratisResult { + let cached = self.cached(); + let (changed, thin_pool) = self.thin_pool.check(pool_uuid, &mut self.backstore)?; + let pool = cached.diff(&self.dump(())); + if changed { + self.write_metadata(pool_name)?; + } + Ok(PoolDiff { thin_pool, pool }) + } + + /// Called when a DM device in this pool has generated an event. This method + /// handles checking filesystems. + #[pool_mutating_action("NoPoolChanges")] + pub fn fs_event_on( + &mut self, + pool_uuid: PoolUuid, + ) -> StratisResult> { + self.thin_pool.check_fs(pool_uuid, &self.backstore) + } + + pub fn record(&self, name: &str) -> PoolSave { + PoolSave { + name: name.to_owned(), + backstore: self.backstore.record(), + flex_devs: self.thin_pool.record(), + thinpool_dev: self.thin_pool.record(), + started: Some(true), + } + } + + pub fn get_strat_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &StratBlockDev)> { + self.backstore.get_blockdev_by_uuid(uuid) + } + + #[pool_mutating_action("NoPoolChanges")] + pub fn get_mut_strat_blockdev( + &mut self, + uuid: DevUuid, + ) -> StratisResult> { + Ok(self.backstore.get_mut_blockdev_by_uuid(uuid)) + } + + pub fn blockdevs(&self) -> Vec<(DevUuid, BlockDevTier, &StratBlockDev)> { + self.backstore.blockdevs() + } + + #[pool_mutating_action("NoPoolChanges")] + pub fn blockdevs_mut( + &mut self, + ) -> StratisResult> { + Ok(self.backstore.blockdevs_mut()) + } + + /// Destroy the pool. + /// Precondition: All filesystems belonging to this pool must be + /// unmounted. + /// + /// This method is not a mutating action as the pool should be allowed + /// to be destroyed even if the metadata is inconsistent. + pub fn destroy(&mut self, pool_uuid: PoolUuid) -> Result<(), (StratisError, bool)> { + self.thin_pool.teardown(pool_uuid)?; + self.backstore.destroy(pool_uuid).map_err(|e| (e, false))?; + Ok(()) + } + + /// Check the limit of filesystems on a pool and return an error if it has been passed. + fn check_fs_limit(&self, new_fs: usize) -> StratisResult<()> { + let fs_limit = self.fs_limit(); + if convert_int!(fs_limit, u64, usize)? < self.filesystems().len() + new_fs { + Err(StratisError::Msg(format!("The pool limit of {fs_limit} filesystems has already been reached; increase the filesystem limit on the pool to continue"))) + } else { + Ok(()) + } + } + + /// Stop a pool, consuming it and converting it into a set of devices to be + /// set up again later. + pub fn stop( + &mut self, + pool_name: &Name, + pool_uuid: PoolUuid, + ) -> Result { + self.thin_pool.teardown(pool_uuid)?; + let mut data = self.record(pool_name); + data.started = Some(false); + let json = serde_json::to_string(&data).map_err(|e| (StratisError::from(e), false))?; + self.backstore + .save_state(json.as_bytes()) + .map_err(|e| (e, false))?; + self.backstore.teardown(pool_uuid).map_err(|e| (e, false))?; + let bds = self.backstore.drain_bds(); + Ok(DeviceSet::from(bds)) + } + + /// Convert a pool into a record of BDAs for the given block devices in the pool. + pub fn into_bdas(self) -> HashMap { + self.backstore.into_bdas() + } + + /// Drain pool block devices into a record of block devices in the pool. + pub fn drain_bds(&mut self) -> Vec { + self.backstore.drain_bds() + } + + #[cfg(test)] + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + pub fn return_rollback_failure(&mut self) -> StratisResult<()> { + Err(StratisError::RollbackError { + causal_error: Box::new(StratisError::Msg("Causal error".to_string())), + rollback_error: Box::new(StratisError::Msg("Rollback error".to_string())), + level: ActionAvailability::NoRequests, + }) + } + + #[cfg(test)] + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + pub fn return_rollback_failure_chain(&mut self) -> StratisResult<()> { + Err(StratisError::Chained( + "Chained error".to_string(), + Box::new(StratisError::RollbackError { + causal_error: Box::new(StratisError::Msg("Causal error".to_string())), + rollback_error: Box::new(StratisError::Msg("Rollback error".to_string())), + level: ActionAvailability::NoRequests, + }), + )) + } + + /// Verifies that the filesystem operation to be performed is allowed to perform + /// overprovisioning if it is determined to be the end result. + fn check_overprov(&self, increase: Sectors) -> StratisResult<()> { + let cur_filesystem_size_sum = self.thin_pool.filesystem_logical_size_sum()?; + if !self.thin_pool.overprov_enabled() + && cur_filesystem_size_sum + increase > self.thin_pool.total_fs_limit(&self.backstore) + { + Err(StratisError::Msg(format!( + "Overprovisioning is disabled on this pool and increasing total filesystem size ({cur_filesystem_size_sum}) by {increase} would result in overprovisioning" + ))) + } else { + Ok(()) + } + } +} + +impl<'a> Into for &'a StratPool { + // Precondition: (&ThinPool).into() pattern matches Value::Object(_) + // Precondition: (&Backstore).into() pattern matches Value::Object(_) + fn into(self) -> Value { + let mut map: Map<_, _> = if let Value::Object(map) = + <&ThinPool as Into>::into(&self.thin_pool) + { + map.into_iter() + } else { + unreachable!("ThinPool conversion returns a JSON object") + } + .collect(); + map.extend( + if let Value::Object(map) = <&Backstore as Into>::into(&self.backstore) { + map.into_iter() + } else { + unreachable!("Backstore conversion returns a JSON object") + }, + ); + map.insert( + "available_actions".to_string(), + Value::from(self.action_avail.to_string()), + ); + map.insert("fs_limit".to_string(), Value::from(self.fs_limit())); + Value::from(map) + } +} + +#[strat_pool_impl_gen] +impl Pool for StratPool { + #[pool_mutating_action("NoRequests")] + fn init_cache( + &mut self, + pool_uuid: PoolUuid, + pool_name: &str, + blockdevs: &[&Path], + supports_encrypted: bool, + ) -> StratisResult> { + validate_paths(blockdevs)?; + + if self.is_encrypted() && !supports_encrypted { + return Err(StratisError::Msg( + "Use of a cache is not supported with an encrypted pool".to_string(), + )); + } + + let devices = ProcessedPathInfos::try_from(blockdevs)?; + let (stratis_devices, unowned_devices) = devices.unpack(); + let (this_pool, other_pools) = stratis_devices.partition(pool_uuid); + + other_pools.error_on_not_empty()?; + + let (in_pool, out_pool): (Vec<_>, Vec<_>) = this_pool + .keys() + .map(|dev_uuid| { + self.backstore + .get_blockdev_by_uuid(*dev_uuid) + .map(|(tier, _)| (*dev_uuid, tier)) + }) + .partition(|v| v.is_some()); + + if !out_pool.is_empty() { + let error_message = format!( + "Devices ({}) appear to be already in use by this pool which has UUID {} but this pool has no record of them", + out_pool + .iter() + .map(|opt| this_pool.get(&opt.expect("was looked up").0).expect("partitioned from this_pool").devnode.display().to_string()) + .collect::>() + .join(", "), + pool_uuid + ); + return Err(StratisError::Msg(error_message)); + }; + + let (datadevs, cachedevs): (Vec<_>, Vec<_>) = in_pool + .iter() + .map(|opt| opt.expect("in_pool devices are Some")) + .partition(|(_, tier)| *tier == BlockDevTier::Data); + + if !datadevs.is_empty() { + let error_message = format!( + "Devices ({}) appear to be already in use by this pool which has UUID {} in the data tier", + datadevs + .iter() + .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string()) + .collect::>() + .join(", "), + pool_uuid + ); + return Err(StratisError::Msg(error_message)); + }; + + if !self.has_cache() { + if unowned_devices.is_empty() { + return Err(StratisError::Msg( + "At least one device is required to initialize a cache.".to_string(), + )); + } + + let block_size_summary = unowned_devices.blocksizes(); + if block_size_summary.len() > 1 { + let err_str = "The devices specified for the cache tier do not all have the same physical sector size or do not all have the same logical sector size.".into(); + return Err(StratisError::Msg(err_str)); + } + + let cache_sector_sizes = block_size_summary + .keys() + .next() + .expect("unowned_devices is not empty"); + + let current_data_sector_sizes = self + .backstore + .block_size_summary(BlockDevTier::Data) + .expect("always exists for data tier") + .validate() + .expect("All operations prevented if validate() function returns an error"); + + if cache_sector_sizes.logical_sector_size + != current_data_sector_sizes.base.logical_sector_size + { + let err_str = "The logical sector size of the devices proposed for the cache tier does not match the effective logical sector size of the data tier".to_string(); + return Err(StratisError::Msg(err_str)); + } + + self.thin_pool.suspend()?; + let devices_result = self.backstore.init_cache(pool_uuid, unowned_devices); + self.thin_pool.resume()?; + let devices = devices_result?; + self.write_metadata(pool_name)?; + Ok(SetCreateAction::new(devices)) + } else { + init_cache_idempotent_or_err( + &cachedevs + .iter() + .map(|(uuid, _)| { + this_pool + .get(uuid) + .expect("partitioned from this_pool") + .devnode + .as_path() + }) + .chain( + unowned_devices + .unpack() + .iter() + .map(|info| info.devnode.as_path()), + ) + .collect::>(), + self.backstore + .cachedevs() + .into_iter() + .map(|(_, bd)| bd.physical_path().to_owned()), + ) + } + } + + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + fn bind_clevis( + &mut self, + pin: &str, + clevis_info: &Value, + ) -> StratisResult> { + let changed = self.backstore.bind_clevis(pin, clevis_info)?; + if changed { + Ok(CreateAction::Created(Clevis)) + } else { + Ok(CreateAction::Identity) + } + } + + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + fn unbind_clevis(&mut self) -> StratisResult> { + let changed = self.backstore.unbind_clevis()?; + if changed { + Ok(DeleteAction::Deleted(Clevis)) + } else { + Ok(DeleteAction::Identity) + } + } + + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + fn bind_keyring( + &mut self, + key_description: &KeyDescription, + ) -> StratisResult> { + let changed = self.backstore.bind_keyring(key_description)?; + if changed { + Ok(CreateAction::Created(Key)) + } else { + Ok(CreateAction::Identity) + } + } + + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + fn unbind_keyring(&mut self) -> StratisResult> { + let changed = self.backstore.unbind_keyring()?; + if changed { + Ok(DeleteAction::Deleted(Key)) + } else { + Ok(DeleteAction::Identity) + } + } + + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + fn rebind_keyring( + &mut self, + new_key_desc: &KeyDescription, + ) -> StratisResult> { + match self.backstore.rebind_keyring(new_key_desc)? { + Some(true) => Ok(RenameAction::Renamed(Key)), + Some(false) => Ok(RenameAction::Identity), + None => Ok(RenameAction::NoSource), + } + } + + #[pool_mutating_action("NoRequests")] + #[pool_rollback] + fn rebind_clevis(&mut self) -> StratisResult { + self.backstore.rebind_clevis().map(|_| RegenAction) + } + + #[pool_mutating_action("NoRequests")] + fn create_filesystems<'a>( + &mut self, + pool_name: &str, + pool_uuid: PoolUuid, + specs: &[(&'a str, Option, Option)], + ) -> StratisResult> { + self.check_fs_limit(specs.len())?; + + let spec_map = validate_filesystem_size_specs(specs)?; + + let increase = spec_map + .values() + .map(|(size, _)| size) + .copied() + .sum::(); + self.check_overprov(increase)?; + + spec_map.iter().try_fold((), |_, (name, (size, _))| { + validate_name(name) + .and_then(|()| { + if let Some((_, fs)) = self.thin_pool.get_filesystem_by_name(name) { + if fs.thindev_size() == *size { + Ok(()) + } else { + Err(StratisError::Msg(format!( + "Size {} of filesystem {} to be created conflicts with size {} for existing filesystem", + size, + name, + fs.thindev_size() + ))) + } + } else { + Ok(()) + } + }) + })?; + + // TODO: Roll back on filesystem initialization failure. + let mut result = Vec::new(); + for (name, (size, size_limit)) in spec_map { + if self.thin_pool.get_mut_filesystem_by_name(name).is_none() { + let fs_uuid = self + .thin_pool + .create_filesystem(pool_name, pool_uuid, name, size, size_limit)?; + result.push((name, fs_uuid, size)); + } + } + + Ok(SetCreateAction::new(result)) + } + + #[pool_mutating_action("NoRequests")] + fn add_blockdevs( + &mut self, + pool_uuid: PoolUuid, + pool_name: &str, + paths: &[&Path], + tier: BlockDevTier, + ) -> StratisResult<(SetCreateAction, Option)> { + validate_paths(paths)?; + + let bdev_info = if tier == BlockDevTier::Cache && !self.has_cache() { + return Err(StratisError::Msg( + format!( + "No cache has been initialized for pool with UUID {pool_uuid} and name {pool_name}; it is therefore impossible to add additional devices to the cache" + ) + )); + } else { + let devices = ProcessedPathInfos::try_from(paths)?; + let (stratis_devices, unowned_devices) = devices.unpack(); + let (this_pool, other_pools) = stratis_devices.partition(pool_uuid); + + other_pools.error_on_not_empty()?; + + let (in_pool, out_pool): (Vec<_>, Vec<_>) = this_pool + .keys() + .map(|dev_uuid| { + self.backstore + .get_blockdev_by_uuid(*dev_uuid) + .map(|(tier, _)| (*dev_uuid, tier)) + }) + .partition(|v| v.is_some()); + + if !out_pool.is_empty() { + let error_message = format!( + "Devices ({}) appear to be already in use by this pool which has UUID {} but this pool has no record of them", + out_pool + .iter() + .map(|opt| this_pool.get(&opt.expect("was looked up").0).expect("partitioned from this_pool").devnode.display().to_string()) + .collect::>() + .join(", "), + pool_uuid + ); + return Err(StratisError::Msg(error_message)); + }; + + let (datadevs, cachedevs): (Vec<_>, Vec<_>) = in_pool + .iter() + .map(|opt| opt.expect("in_pool devices are Some")) + .partition(|(_, tier)| *tier == BlockDevTier::Data); + + if tier == BlockDevTier::Cache { + // If adding cache devices, must suspend the pool; the cache + // must be augmented with the new devices. + if !datadevs.is_empty() { + let error_message = format!( + "Devices ({}) appear to be already in use by this pool which has UUID {}, but in the data tier not the cache tier", + datadevs + .iter() + .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string()) + .collect::>() + .join(", "), + pool_uuid + ); + return Err(StratisError::Msg(error_message)); + }; + + if unowned_devices.is_empty() { + return Ok((SetCreateAction::new(vec![]), None)); + } + + let block_size_summary = unowned_devices.blocksizes(); + if block_size_summary.len() > 1 { + let err_str = "The devices specified to be added to the cache tier do not all have the same physical sector size or do not all have the same logical sector size.".into(); + return Err(StratisError::Msg(err_str)); + } + let added_sector_sizes = block_size_summary + .keys() + .next() + .expect("unowned devices is not empty"); + + let current_sector_sizes = self + .backstore + .block_size_summary(BlockDevTier::Cache) + .expect("already returned if no cache tier") + .validate() + .expect("All devices of the cache tier must be in use, so there can only be one representative logical sector size."); + + if !(¤t_sector_sizes.base == added_sector_sizes) { + let err_str = format!("The sector sizes of the devices proposed for extending the cache tier, {added_sector_sizes}, do not match the effective sector sizes of the existing cache devices, {0}", current_sector_sizes.base); + return Err(StratisError::Msg(err_str)); + } + + self.thin_pool.suspend()?; + let bdev_info_res = self.backstore.add_cachedevs(pool_uuid, unowned_devices); + self.thin_pool.resume()?; + let bdev_info = bdev_info_res?; + Ok((SetCreateAction::new(bdev_info), None)) + } else { + if !cachedevs.is_empty() { + let error_message = format!( + "Devices ({}) appear to be already in use by this pool which has UUID {}, but in the cache tier not the data tier", + cachedevs + .iter() + .map(|(uuid, _)| this_pool.get(uuid).expect("partitioned from this_pool").devnode.display().to_string()) + .collect::>() + .join(", "), + pool_uuid + ); + return Err(StratisError::Msg(error_message)); + }; + + if unowned_devices.is_empty() { + return Ok((SetCreateAction::new(vec![]), None)); + } + + let block_size_summary = unowned_devices.blocksizes(); + if block_size_summary.len() > 1 { + let err_str = "The devices specified to be added to the data tier do not have uniform physical and logical sector sizes.".into(); + return Err(StratisError::Msg(err_str)); + } + + let added_sector_sizes = block_size_summary + .keys() + .next() + .expect("unowned devices is not empty"); + + let current_sector_sizes = self + .backstore + .block_size_summary(BlockDevTier::Data) + .expect("always exists") + .validate() + .expect("All operations prevented if validate() function on data tier block size summary returns an error"); + + if !(¤t_sector_sizes.base == added_sector_sizes) { + let err_str = format!("The sector sizes of the devices proposed for extending the data tier, {added_sector_sizes}, do not match the effective sector sizes of the existing data devices, {0}", current_sector_sizes.base); + return Err(StratisError::Msg(err_str)); + } + + let cached = self.cached(); + + // If just adding data devices, no need to suspend the pool. + // No action will be taken on the DM devices. + let bdev_info = self.backstore.add_datadevs(pool_uuid, unowned_devices)?; + self.thin_pool.set_queue_mode(); + self.thin_pool.clear_out_of_meta_flag(); + + Ok(( + SetCreateAction::new(bdev_info), + Some(PoolDiff { + thin_pool: self.thin_pool.cached().unchanged(), + pool: cached.diff(&self.dump(())), + }), + )) + } + }; + self.write_metadata(pool_name)?; + bdev_info + } + + #[pool_mutating_action("NoRequests")] + fn destroy_filesystems( + &mut self, + pool_name: &str, + fs_uuids: &[FilesystemUuid], + ) -> StratisResult> { + let mut removed = Vec::new(); + for &uuid in fs_uuids { + if let Some(uuid) = self.thin_pool.destroy_filesystem(pool_name, uuid)? { + removed.push(uuid); + } + } + + let snapshots: Vec = self + .thin_pool + .filesystems() + .iter() + .filter_map(|(_, u, fs)| { + fs.origin() + .and_then(|x| if removed.contains(&x) { Some(*u) } else { None }) + }) + .collect(); + + let mut updated_origins = vec![]; + for sn_uuid in snapshots { + if let Err(err) = self.thin_pool.unset_fs_origin(sn_uuid) { + warn!( + "Failed to write null origin to metadata for filesystem with UUID {}: {}", + sn_uuid, err + ); + } else { + updated_origins.push(sn_uuid); + } + } + + Ok(SetDeleteAction::new(removed, updated_origins)) + } + + #[pool_mutating_action("NoRequests")] + fn rename_filesystem( + &mut self, + pool_name: &str, + uuid: FilesystemUuid, + new_name: &str, + ) -> StratisResult> { + validate_name(new_name)?; + match self + .thin_pool + .rename_filesystem(pool_name, uuid, new_name)? + { + Some(true) => Ok(RenameAction::Renamed(uuid)), + Some(false) => Ok(RenameAction::Identity), + None => Ok(RenameAction::NoSource), + } + } + + #[pool_mutating_action("NoRequests")] + fn snapshot_filesystem<'a>( + &'a mut self, + pool_name: &str, + pool_uuid: PoolUuid, + origin_uuid: FilesystemUuid, + snapshot_name: &str, + ) -> StratisResult> { + self.check_fs_limit(1)?; + + validate_name(snapshot_name)?; + self.check_overprov( + self.thin_pool + .get_filesystem_by_uuid(origin_uuid) + .ok_or_else(|| { + StratisError::Msg(format!( + "Filesystem with UUID {origin_uuid} could not be found" + )) + })? + .1 + .thindev_size(), + )?; + + if self + .thin_pool + .get_filesystem_by_name(snapshot_name) + .is_some() + { + return Ok(CreateAction::Identity); + } + + self.thin_pool + .snapshot_filesystem(pool_name, pool_uuid, origin_uuid, snapshot_name) + .map(|(uuid, fs)| CreateAction::Created((uuid, fs as &mut dyn Filesystem))) + } + + fn total_physical_size(&self) -> Sectors { + self.backstore.datatier_size() + } + + fn total_allocated_size(&self) -> Sectors { + self.backstore.datatier_allocated_size() + self.metadata_size + } + + fn total_physical_used(&self) -> Option { + // TODO: note that with the addition of another layer, the + // calculation of the amount of physical spaced used by means + // of adding the amount used by Stratis metadata to the amount used + // by the pool abstraction will be invalid. In the event of, e.g., + // software RAID, the amount will be far too low to be useful, in the + // event of, e.g, VDO, the amount will be far too large to be useful. + self.thin_pool + .total_physical_used() + .map(|u| u + self.metadata_size) + } + + fn filesystems(&self) -> Vec<(Name, FilesystemUuid, &dyn Filesystem)> { + self.thin_pool + .filesystems() + .into_iter() + .map(|(n, u, f)| (n, u, f as &dyn Filesystem)) + .collect() + } + + fn get_filesystem(&self, uuid: FilesystemUuid) -> Option<(Name, &dyn Filesystem)> { + self.get_filesystem(uuid) + .map(|(name, fs)| (name, fs as &dyn Filesystem)) + } + + fn get_filesystem_by_name(&self, fs_name: &Name) -> Option<(FilesystemUuid, &dyn Filesystem)> { + self.get_filesystem_by_name(fs_name) + .map(|(uuid, fs)| (uuid, fs as &dyn Filesystem)) + } + + fn blockdevs(&self) -> Vec<(DevUuid, BlockDevTier, &dyn BlockDev)> { + self.backstore + .blockdevs() + .into_iter() + .map(|(uuid, tier, bd)| (uuid, tier, bd as &dyn BlockDev)) + .collect::>() + } + + fn get_blockdev(&self, uuid: DevUuid) -> Option<(BlockDevTier, &dyn BlockDev)> { + self.get_strat_blockdev(uuid) + .map(|(t, bd)| (t, bd as &dyn BlockDev)) + } + + #[pool_mutating_action("NoRequests")] + fn get_mut_blockdev( + &mut self, + uuid: DevUuid, + ) -> StratisResult> { + self.get_mut_strat_blockdev(uuid) + .map(|opt| opt.map(|(t, bd)| (t, bd as &mut dyn BlockDev))) + } + + #[pool_mutating_action("NoRequests")] + fn set_blockdev_user_info( + &mut self, + pool_name: &str, + uuid: DevUuid, + user_info: Option<&str>, + ) -> StratisResult> { + let result = self.backstore.set_blockdev_user_info(uuid, user_info); + match result { + Ok(Some(uuid)) => { + self.write_metadata(pool_name)?; + Ok(RenameAction::Renamed(uuid)) + } + Ok(None) => Ok(RenameAction::Identity), + Err(_) => Ok(RenameAction::NoSource), + } + } + + fn has_cache(&self) -> bool { + self.backstore.has_cache() + } + + fn is_encrypted(&self) -> bool { + self.backstore.is_encrypted() + } + + fn encryption_info(&self) -> Option { + self.backstore + .encryption_info() + .map(PoolEncryptionInfo::from) + } + + fn avail_actions(&self) -> ActionAvailability { + self.action_avail.clone() + } + + fn fs_limit(&self) -> u64 { + self.thin_pool.fs_limit() + } + + #[pool_mutating_action("NoPoolChanges")] + fn set_fs_limit( + &mut self, + pool_name: &Name, + pool_uuid: PoolUuid, + new_limit: u64, + ) -> StratisResult<()> { + let (should_save, res) = + self.thin_pool + .set_fs_limit(pool_uuid, &mut self.backstore, new_limit); + if should_save { + self.write_metadata(pool_name)?; + } + res + } + + fn overprov_enabled(&self) -> bool { + self.thin_pool.overprov_enabled() + } + + #[pool_mutating_action("NoPoolChanges")] + fn set_overprov_mode(&mut self, pool_name: &Name, enabled: bool) -> StratisResult<()> { + let (should_save, res) = self.thin_pool.set_overprov_mode(&self.backstore, enabled); + if should_save { + self.write_metadata(pool_name)?; + } + res + } + + fn out_of_alloc_space(&self) -> bool { + self.thin_pool.out_of_alloc_space() + } + + #[pool_mutating_action("NoRequests")] + fn grow_physical( + &mut self, + name: &Name, + pool_uuid: PoolUuid, + device: DevUuid, + ) -> StratisResult<(GrowAction<(PoolUuid, DevUuid)>, Option)> { + let cached = self.cached(); + + let changed = self.backstore.grow(device)?; + if changed { + if self.thin_pool.set_queue_mode() { + self.write_metadata(name)?; + } + Ok(( + GrowAction::Grown((pool_uuid, device)), + Some(PoolDiff { + thin_pool: self.thin_pool.cached().unchanged(), + pool: cached.diff(&self.dump(())), + }), + )) + } else { + Ok((GrowAction::Identity, None)) + } + } + + #[pool_mutating_action("NoRequests")] + fn set_fs_size_limit( + &mut self, + fs_uuid: FilesystemUuid, + limit: Option, + ) -> StratisResult>> { + let (name, _) = self.get_filesystem(fs_uuid).ok_or_else(|| { + StratisError::Msg(format!("Filesystem with UUID {fs_uuid} not found")) + })?; + let limit = validate_filesystem_size(&name, limit)?; + if self.thin_pool.set_fs_size_limit(fs_uuid, limit)? { + Ok(PropChangeAction::NewValue(limit)) + } else { + Ok(PropChangeAction::Identity) + } + } + + fn current_metadata(&self, pool_name: &Name) -> StratisResult { + serde_json::to_string(&self.record(pool_name)).map_err(|e| e.into()) + } + + fn last_metadata(&self) -> StratisResult { + self.backstore.load_state().and_then(|v| { + String::from_utf8(v) + .map_err(|_| StratisError::Msg("metadata byte array is not utf-8".into())) + }) + } +} + +pub struct StratPoolState { + metadata_size: Bytes, + out_of_alloc_space: bool, +} + +impl StateDiff for StratPoolState { + type Diff = StratPoolDiff; + + fn diff(&self, other: &Self) -> Self::Diff { + StratPoolDiff { + metadata_size: self.metadata_size.compare(&other.metadata_size), + out_of_alloc_space: self.out_of_alloc_space.compare(&other.out_of_alloc_space), + } + } + + fn unchanged(&self) -> Self::Diff { + StratPoolDiff { + metadata_size: Diff::Unchanged(self.metadata_size), + out_of_alloc_space: Diff::Unchanged(self.out_of_alloc_space), + } + } +} + +impl<'a> DumpState<'a> for StratPool { + type State = StratPoolState; + type DumpInput = (); + + fn cached(&self) -> Self::State { + StratPoolState { + metadata_size: self.metadata_size.bytes(), + out_of_alloc_space: self.thin_pool.out_of_alloc_space(), + } + } + + fn dump(&mut self, _: Self::DumpInput) -> Self::State { + self.metadata_size = self.backstore.datatier_metadata_size(); + StratPoolState { + metadata_size: self.metadata_size.bytes(), + out_of_alloc_space: self.thin_pool.out_of_alloc_space(), + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + fs::OpenOptions, + io::{BufWriter, Read, Write}, + }; + + use nix::mount::{mount, umount, MsFlags}; + + use devicemapper::{Bytes, IEC, SECTOR_SIZE}; + + use crate::engine::{ + strat_engine::{ + cmd::udev_settle, + pool::AnyPool, + tests::{loopbacked, real}, + thinpool::ThinPoolStatusDigest, + }, + types::{EngineAction, PoolIdentifier}, + Engine, StratEngine, + }; + + use super::*; + + fn invariant(pool: &StratPool, pool_name: &str) { + check_metadata(&pool.record(&Name::new(pool_name.into()))).unwrap(); + assert!(!(pool.is_encrypted() && pool.backstore.has_cache())); + if pool.avail_actions() == ActionAvailability::NoRequests { + assert!( + pool.encryption_info().is_some() + && pool + .encryption_info() + .map(|ei| { ei.is_inconsistent() }) + .unwrap_or(false) + ); + } else if pool.avail_actions() == ActionAvailability::Full { + assert!(!pool + .encryption_info() + .map(|ei| ei.is_inconsistent()) + .unwrap_or(false)); + } + assert!(pool + .backstore + .blockdevs() + .iter() + .all(|(_, _, bd)| bd.metadata_path().is_absolute())) + } + + /// Test that initializing a cache causes metadata to be updated. Verify + /// that data written before the cache was initialized can be read + /// afterwards. + fn test_add_cachedevs(paths: &[&Path]) { + assert!(paths.len() > 1); + + let (paths1, paths2) = paths.split_at(paths.len() / 2); + + let devices2 = ProcessedPathInfos::try_from(paths2).unwrap(); + let (stratis_devices, unowned_devices2) = devices2.unpack(); + stratis_devices.error_on_not_empty().unwrap(); + + let name = "stratis-test-pool"; + let (uuid, mut pool) = StratPool::initialize(name, unowned_devices2, None).unwrap(); + invariant(&pool, name); + + let metadata1 = pool.record(name); + assert_matches!(metadata1.backstore.cache_tier, None); + + let (_, fs_uuid, _) = pool + .create_filesystems(name, uuid, &[("stratis-filesystem", None, None)]) + .unwrap() + .changed() + .and_then(|mut fs| fs.pop()) + .unwrap(); + invariant(&pool, name); + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + let bytestring = b"some bytes"; + { + let (_, fs) = pool.get_filesystem(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&new_file) + .unwrap() + .write_all(bytestring) + .unwrap(); + } + + pool.init_cache(uuid, name, paths1, true).unwrap(); + invariant(&pool, name); + + let metadata2 = pool.record(name); + assert!(metadata2.backstore.cache_tier.is_some()); + + let mut buf = [0u8; 10]; + { + OpenOptions::new() + .read(true) + .open(&new_file) + .unwrap() + .read_exact(&mut buf) + .unwrap(); + } + assert_eq!(&buf, bytestring); + umount(tmp_dir.path()).unwrap(); + pool.teardown(uuid).unwrap(); + } + + #[test] + fn loop_test_add_cachedevs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(3, 4, None), + test_add_cachedevs, + ); + } + + #[test] + fn real_test_add_cachedevs() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(2, None, None), + test_add_cachedevs, + ); + } + + // Verify that it is possible to add datadevs after a cache is initialized. + fn test_add_cachedevs_and_datadevs(paths: &[&Path]) { + assert!(paths.len() > 2); + + let (cache_path, data_paths) = paths.split_at(1); + let (data_path, data_paths) = data_paths.split_at(1); + + let devices = ProcessedPathInfos::try_from(data_path).unwrap(); + let (stratis_devices, unowned_devices) = devices.unpack(); + stratis_devices.error_on_not_empty().unwrap(); + + let name = "stratis-test-pool"; + let (uuid, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap(); + invariant(&pool, name); + + pool.init_cache(uuid, name, cache_path, true).unwrap(); + invariant(&pool, name); + + pool.add_blockdevs(uuid, name, data_paths, BlockDevTier::Data) + .unwrap(); + + pool.teardown(uuid).unwrap(); + } + + #[test] + fn loop_test_add_cachedevs_and_datadevs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(3, 4, None), + test_add_cachedevs_and_datadevs, + ); + } + + #[test] + fn real_test_add_cachedevs_and_datadevs() { + real::test_with_spec( + &real::DeviceLimits::AtLeast(3, None, None), + test_add_cachedevs_and_datadevs, + ); + } + + /// Verify that adding additional blockdevs will cause a pool that is + /// out of space to be extended. + fn test_add_datadevs(paths: &[&Path]) { + assert!(paths.len() > 1); + + let (paths1, paths2) = paths.split_at(1); + + let devices1 = ProcessedPathInfos::try_from(paths1).unwrap(); + let (stratis_devices, unowned_devices1) = devices1.unpack(); + stratis_devices.error_on_not_empty().unwrap(); + + let name = "stratis-test-pool"; + let (pool_uuid, mut pool) = StratPool::initialize(name, unowned_devices1, None).unwrap(); + invariant(&pool, name); + + let fs_name = "stratis_test_filesystem"; + let (_, fs_uuid, _) = pool + .create_filesystems(name, pool_uuid, &[(fs_name, None, None)]) + .unwrap() + .changed() + .and_then(|mut fs| fs.pop()) + .expect("just created one"); + + let devnode = pool.get_filesystem(fs_uuid).unwrap().1.devnode(); + + { + let buffer_length = IEC::Mi; + let mut f = BufWriter::with_capacity( + convert_test!(buffer_length, u64, usize), + OpenOptions::new().write(true).open(devnode).unwrap(), + ); + + let buf = &[1u8; SECTOR_SIZE]; + + let mut amount_written = Sectors(0); + let buffer_length = Bytes::from(buffer_length).sectors(); + while matches!(pool.thin_pool.state(), Some(ThinPoolStatusDigest::Good)) { + f.write_all(buf).unwrap(); + amount_written += Sectors(1); + // Run check roughly every time the buffer is cleared. + // Running it more often is pointless as the pool is guaranteed + // not to see any effects unless the buffer is cleared. + if amount_written % buffer_length == Sectors(1) { + pool.event_on(pool_uuid, &Name::new(name.to_string())) + .unwrap(); + } + } + + pool.add_blockdevs(pool_uuid, name, paths2, BlockDevTier::Data) + .unwrap(); + + let pool_diff = pool + .event_on(pool_uuid, &Name::new(name.to_string())) + .unwrap(); + + assert!(pool_diff.thin_pool.allocated_size.is_changed()); + + match pool.thin_pool.state() { + Some(ThinPoolStatusDigest::Good) => (), + _ => panic!("thin pool status should be back to working"), + } + } + udev_settle().unwrap(); + } + + #[test] + fn loop_test_add_datadevs() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(IEC::Gi * 4).sectors())), + test_add_datadevs, + ); + } + + #[test] + fn real_test_add_datadevs() { + real::test_with_spec( + &real::DeviceLimits::AtLeast( + 2, + Some(Bytes::from(IEC::Gi * 2).sectors()), + Some(Bytes::from(IEC::Gi * 4).sectors()), + ), + test_add_datadevs, + ); + } + + /// Test that rollback errors are properly detected an maintenance mode + /// is set accordingly. + fn test_maintenance_mode(paths: &[&Path]) { + assert!(paths.len() > 1); + + let name = "stratis-test-pool"; + + let devices = ProcessedPathInfos::try_from(paths).unwrap(); + let (stratis_devices, unowned_devices) = devices.unpack(); + stratis_devices.error_on_not_empty().unwrap(); + + let (uuid, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap(); + invariant(&pool, name); + + assert_eq!(pool.action_avail, ActionAvailability::Full); + assert!(pool.return_rollback_failure().is_err()); + assert_eq!(pool.action_avail, ActionAvailability::NoRequests); + + pool.destroy(uuid).unwrap(); + udev_settle().unwrap(); + + let name = "stratis-test-pool"; + + let devices = ProcessedPathInfos::try_from(paths).unwrap(); + let (stratis_devices, unowned_devices) = devices.unpack(); + stratis_devices.error_on_not_empty().unwrap(); + + let (_, mut pool) = StratPool::initialize(name, unowned_devices, None).unwrap(); + invariant(&pool, name); + + assert_eq!(pool.action_avail, ActionAvailability::Full); + assert!(pool.return_rollback_failure_chain().is_err()); + assert_eq!(pool.action_avail, ActionAvailability::NoRequests); + } + + #[test] + fn loop_test_maintenance_mode() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(IEC::Gi * 4).sectors())), + test_maintenance_mode, + ); + } + + #[test] + fn real_test_maintenance_mode() { + real::test_with_spec( + &real::DeviceLimits::AtLeast( + 2, + Some(Bytes::from(IEC::Gi * 2).sectors()), + Some(Bytes::from(IEC::Gi * 4).sectors()), + ), + test_maintenance_mode, + ); + } + + /// Test overprovisioning mode disabled and enabled assuring that the appropriate + /// checks and behavior are in place. + fn test_overprov(paths: &[&Path]) { + assert!(paths.len() == 1); + + let pool_name = "pool"; + + let devices = ProcessedPathInfos::try_from(paths).unwrap(); + let (stratis_devices, unowned_devices) = devices.unpack(); + stratis_devices.error_on_not_empty().unwrap(); + + let (pool_uuid, mut pool) = + StratPool::initialize(pool_name, unowned_devices, None).unwrap(); + + let (_, fs_uuid, _) = pool + .create_filesystems( + pool_name, + pool_uuid, + &[( + "stratis_test_filesystem", + Some(pool.backstore.datatier_usable_size().bytes() * 2u64), + None, + )], + ) + .unwrap() + .changed() + .unwrap() + .pop() + .unwrap(); + udev_settle().unwrap(); + assert!(pool + .set_overprov_mode(&Name::new(pool_name.to_string()), false) + .is_err()); + pool.destroy_filesystems(pool_name, &[fs_uuid]).unwrap(); + + pool.set_overprov_mode(&Name::new(pool_name.to_string()), false) + .unwrap(); + assert!(pool + .create_filesystems( + pool_name, + pool_uuid, + &[( + "stratis_test_filesystem", + Some(pool.backstore.datatier_usable_size().bytes() * 2u64), + None, + )], + ) + .is_err()); + + let mut initial_fs_size = pool.backstore.datatier_usable_size().bytes() * 2u64 / 3u64; + initial_fs_size = initial_fs_size.sectors().bytes(); + let half_init_size = initial_fs_size / 2u64 + Bytes(1); + let (_, fs_uuid, _) = pool + .create_filesystems( + pool_name, + pool_uuid, + &[("stratis_test_filesystem", Some(initial_fs_size), None)], + ) + .unwrap() + .changed() + .unwrap() + .pop() + .unwrap(); + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + let sector = &[0; 512]; + + { + let (_, fs) = pool.get_filesystem(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + } + + let mut f = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(new_file) + .unwrap(); + let mut written = Sectors(0); + while written.bytes() < half_init_size { + f.write_all(sector).unwrap(); + written += Sectors(1); + } + let diffs = pool.fs_event_on(pool_uuid).unwrap(); + assert!(diffs.get(&fs_uuid).unwrap().size.is_changed()); + + let (_, fs) = pool.get_filesystem(fs_uuid).unwrap(); + assert!(fs.thindev_size() < initial_fs_size.sectors() * 2u64); + } + + #[test] + fn loop_test_overprov() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, Some(Sectors(10 * IEC::Mi))), + test_overprov, + ); + } + + #[test] + fn real_test_overprov() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, Some(Sectors(10 * IEC::Mi)), None), + test_overprov, + ); + } + + /// Set up for testing physical device growth. + fn test_grow_physical_pre_grow(paths: &[&Path]) { + let pool_name = Name::new("pool".to_string()); + let engine = StratEngine::initialize().unwrap(); + let pool_uuid = test_async!(engine.create_pool(&pool_name, paths, None)) + .unwrap() + .changed() + .unwrap(); + let mut guard = test_async!(engine.get_mut_pool(PoolIdentifier::Uuid(pool_uuid))).unwrap(); + let (_, _, pool) = guard.as_mut_tuple(); + + let (_, fs_uuid, _) = pool + .create_filesystems( + &pool_name, + pool_uuid, + &[("stratis_test_filesystem", None, None)], + ) + .unwrap() + .changed() + .unwrap() + .pop() + .unwrap(); + + let tmp_dir = tempfile::Builder::new() + .prefix("stratis_testing") + .tempdir() + .unwrap(); + let new_file = tmp_dir.path().join("stratis_test.txt"); + let write_block = &[0; 512_000]; + + { + let (_, fs) = pool.get_filesystem(fs_uuid).unwrap(); + mount( + Some(&fs.devnode()), + tmp_dir.path(), + Some("xfs"), + MsFlags::empty(), + None as Option<&str>, + ) + .unwrap(); + } + + let mut f = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(new_file) + .unwrap(); + while !pool.out_of_alloc_space() { + f.write_all(write_block).unwrap(); + f.sync_all().unwrap(); + match pool { + AnyPool::V1(p) => p.event_on(pool_uuid, &pool_name).unwrap(), + AnyPool::V2(p) => p.event_on(pool_uuid, &pool_name).unwrap(), + }; + } + } + + /// Test that growing a physical device succeeds, the device has doubled in size, + /// and that the pool registers new available allocation space if it is out of space + /// at the time of device growth. + fn test_grow_physical_post_grow(_: &[&Path]) { + let engine = StratEngine::initialize().unwrap(); + + let mut pools = test_async!(engine.pools_mut()); + assert!(pools.len() == 1); + let (pool_name, pool_uuid, pool) = pools.iter_mut().next().unwrap(); + + let (dev_uuid, size) = { + let blockdevs = pool.blockdevs(); + let (dev_uuid, _, dev) = blockdevs.first().unwrap(); + (*dev_uuid, dev.size()) + }; + + assert!(pool.out_of_alloc_space()); + let (act, pool_diff) = pool.grow_physical(pool_name, *pool_uuid, dev_uuid).unwrap(); + assert!(act.is_changed()); + let (_, dev) = pool.get_blockdev(dev_uuid).unwrap(); + assert_eq!(dev.size(), 2u64 * size); + assert!(!pool.out_of_alloc_space()); + assert!(!pool_diff + .unwrap() + .pool + .out_of_alloc_space + .changed() + .unwrap()); + } + + #[test] + fn loop_test_grow_physical() { + loopbacked::test_device_grow_with_spec( + &loopbacked::DeviceLimits::Exactly(2, Some(Sectors(10 * IEC::Mi))), + test_grow_physical_pre_grow, + test_grow_physical_post_grow, + ); + } +} From e806e0748c887873057506c6b605fbbbbc2e384e Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 30 Jun 2023 18:24:39 -0400 Subject: [PATCH 10/32] Add and remove attributes --- src/engine/strat_engine/backstore/backstore/v1.rs | 1 + src/engine/strat_engine/backstore/blockdevmgr.rs | 2 -- src/engine/strat_engine/backstore/cache_tier.rs | 2 -- src/engine/strat_engine/backstore/data_tier.rs | 3 +-- src/engine/strat_engine/crypt/handle/v2.rs | 2 -- src/engine/strat_engine/thinpool/thinpool.rs | 1 + 6 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/engine/strat_engine/backstore/backstore/v1.rs b/src/engine/strat_engine/backstore/backstore/v1.rs index fdd9ca6db1..725261c26f 100644 --- a/src/engine/strat_engine/backstore/backstore/v1.rs +++ b/src/engine/strat_engine/backstore/backstore/v1.rs @@ -284,6 +284,7 @@ impl Backstore { /// be encrypted only with a kernel keyring and without Clevis information. /// /// WARNING: metadata changing event + #[cfg(test)] pub fn initialize( pool_name: Name, pool_uuid: PoolUuid, diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index 0cb513b17c..f36ee5846f 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -4,8 +4,6 @@ // Code to handle a collection of block devices. -#![allow(dead_code)] - use std::collections::HashMap; #[cfg(test)] use std::collections::HashSet; diff --git a/src/engine/strat_engine/backstore/cache_tier.rs b/src/engine/strat_engine/backstore/cache_tier.rs index adf2d38614..d25b333546 100644 --- a/src/engine/strat_engine/backstore/cache_tier.rs +++ b/src/engine/strat_engine/backstore/cache_tier.rs @@ -4,8 +4,6 @@ // Code to handle the backing store of a pool. -#![allow(dead_code)] - #[cfg(test)] use std::collections::HashSet; diff --git a/src/engine/strat_engine/backstore/data_tier.rs b/src/engine/strat_engine/backstore/data_tier.rs index 592b725fcf..10ce98b93f 100644 --- a/src/engine/strat_engine/backstore/data_tier.rs +++ b/src/engine/strat_engine/backstore/data_tier.rs @@ -4,8 +4,6 @@ // Code to handle the backing store of a pool. -#![allow(dead_code)] - #[cfg(test)] use std::collections::HashSet; @@ -45,6 +43,7 @@ impl DataTier { /// Initially 0 segments are allocated. /// /// WARNING: metadata changing event + #[cfg(test)] pub fn new(block_mgr: BlockDevMgr) -> DataTier { DataTier { block_mgr, diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs index b55d92fa09..4ea46e0f60 100644 --- a/src/engine/strat_engine/crypt/handle/v2.rs +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -#![allow(dead_code)] - use std::{ fmt::Debug, iter::once, diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 46875e1f7d..348d736e7a 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -732,6 +732,7 @@ impl ThinPool { impl ThinPool { /// Make a new thin pool. + #[cfg(test)] pub fn new( pool_uuid: PoolUuid, thin_pool_size: &ThinPoolSizeParams, From e96dbab0f432ac1863d7e4717dec5bddd65883d9 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Tue, 11 Jul 2023 10:35:30 -0400 Subject: [PATCH 11/32] Add test executable to build legacy pools --- .githooks/pre-commit | 1 + .github/workflows/fedora.yml | 6 + Cargo.toml | 5 + Makefile | 17 ++- src/bin/stratis-legacy-pool.rs | 130 ++++++++++++++++++ src/engine/mod.rs | 2 + .../strat_engine/backstore/backstore/v1.rs | 2 +- .../strat_engine/backstore/backstore/v2.rs | 7 +- .../strat_engine/backstore/data_tier.rs | 2 +- src/engine/strat_engine/mod.rs | 2 + src/engine/strat_engine/pool/v1.rs | 4 +- src/engine/strat_engine/thinpool/mdv.rs | 37 +++-- src/engine/strat_engine/thinpool/thinpool.rs | 2 +- tests/client-dbus/tests/udev/_utils.py | 80 ++++++++--- tests/client-dbus/tests/udev/test_bind.py | 11 +- 15 files changed, 263 insertions(+), 45 deletions(-) create mode 100644 src/bin/stratis-legacy-pool.rs diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 4d4cda004f..45f5a07ec6 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -5,6 +5,7 @@ export PROFILEDIR=debug make fmt-ci && make build && make stratisd-tools && + make build-test-extras && make build-min && make build-no-ipc && make test && diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index 264f6a198d..a8f57c38e1 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -45,6 +45,9 @@ jobs: - task: PROFILEDIR=debug make -f Makefile build toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo + - task: PROFILEDIR=debug make -f Makefile build-test-extras + toolchain: 1.72.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + components: cargo - task: PROFILEDIR=debug make -f Makefile build-min toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo @@ -74,6 +77,9 @@ jobs: - task: make -f Makefile build toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo + - task: make -f Makefile build-test-extras + toolchain: 1.72.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + components: cargo - task: make -f Makefile build-min toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo diff --git a/Cargo.toml b/Cargo.toml index 69091f5438..389f170762 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,10 @@ required-features = ["udev_scripts"] name = "stratis-utils" required-features = ["engine"] +[[bin]] +name = "stratis-legacy-pool" +required-features = ["test_extras"] + [dependencies.async-trait] version = "0.1.51" optional = true @@ -285,6 +289,7 @@ extras = ["pretty-hex"] min = ["termios"] systemd_compat = ["bindgen"] udev_scripts = ["data-encoding"] +test_extras = ["engine"] [package.metadata.vendor-filter] platforms = ["*-unknown-linux-gnu"] diff --git a/Makefile b/Makefile index 3ab4269f83..fb0c274d28 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ MIN_FEATURES = --no-default-features --features engine,min NO_IPC_FEATURES = --no-default-features --features engine SYSTEMD_FEATURES = --no-default-features --features engine,min,systemd_compat EXTRAS_FEATURES = --no-default-features --features engine,extras,min +TEST_EXTRAS_FEATURES = --no-default-features --features test_extras UDEV_FEATURES = --no-default-features --features udev_scripts UTILS_FEATURES = --no-default-features --features engine,systemd_compat @@ -293,6 +294,14 @@ stratisd-tools: cargo ${BUILD} ${RELEASE_FLAG} \ --bin=stratisd-tools ${EXTRAS_FEATURES} ${TARGET_ARGS} +## Build the test extras +build-test-extras: + PKG_CONFIG_ALLOW_CROSS=1 \ + RUSTFLAGS="${DENY}" \ + cargo build ${RELEASE_FLAG} \ + --bin=stratis-legacy-pool ${TEST_EXTRAS_FEATURES} ${TARGET_ARGS} + +## Build the stratis-dumpmetadata program ## Build stratis-min for early userspace stratis-min: PKG_CONFIG_ALLOW_CROSS=1 \ @@ -514,8 +523,12 @@ clippy-utils: clippy-no-ipc: RUSTFLAGS="${DENY}" cargo clippy ${CLIPPY_OPTS} ${NO_IPC_FEATURES} -- ${CLIPPY_DENY} ${CLIPPY_PEDANTIC} ${CLIPPY_PEDANTIC_USELESS} +## Run clippy on no-ipc-build +clippy-test-extras: + RUSTFLAGS="${DENY}" cargo clippy ${CLIPPY_OPTS} ${TEST_EXTRAS_FEATURES} -- ${CLIPPY_DENY} ${CLIPPY_PEDANTIC} ${CLIPPY_PEDANTIC_USELESS} + ## Run clippy on the current source tree -clippy: clippy-macros clippy-min clippy-udev-utils clippy-no-ipc clippy-utils +clippy: clippy-macros clippy-min clippy-udev-utils clippy-no-ipc clippy-utils clippy-test-extras RUSTFLAGS="${DENY}" cargo clippy ${CLIPPY_OPTS} -- ${CLIPPY_DENY} ${CLIPPY_PEDANTIC} ${CLIPPY_PEDANTIC_USELESS} ## Lint Python parts of the source code @@ -530,6 +543,7 @@ pylint: build-all-man build-all-rust build-min + build-test-extras build-udev-utils build-stratis-base32-decode build-stratis-str-cmp @@ -542,6 +556,7 @@ pylint: clippy-macros clippy-min clippy-no-ipc + clippy-test-extras clippy-udev-utils docs-ci docs-rust diff --git a/src/bin/stratis-legacy-pool.rs b/src/bin/stratis-legacy-pool.rs new file mode 100644 index 0000000000..5b7081240f --- /dev/null +++ b/src/bin/stratis-legacy-pool.rs @@ -0,0 +1,130 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use std::{env, path::PathBuf}; + +use clap::{Arg, ArgAction, ArgGroup, Command}; +use serde_json::{json, Map, Value}; + +use stratisd::{ + engine::{ + register_clevis_token, EncryptionInfo, KeyDescription, ProcessedPathInfos, StratPool, + CLEVIS_TANG_TRUST_URL, + }, + stratis::StratisResult, +}; + +fn stratis_legacy_pool_args() -> Command { + Command::new("stratis-legacy-pool") + .arg(Arg::new("pool_name").num_args(1).required(true)) + .arg( + Arg::new("blockdevs") + .action(ArgAction::Append) + .required(true), + ) + .arg( + Arg::new("key_desc") + .long("key-desc") + .num_args(1) + .required(false), + ) + .arg( + Arg::new("clevis") + .long("clevis") + .num_args(1) + .required(false) + .value_parser(["nbde", "tang", "tpm2"]) + .requires_if("nbde", "tang_args") + .requires_if("tang", "tang_args"), + ) + .arg( + Arg::new("tang_url") + .long("tang-url") + .num_args(1) + .required_if_eq("clevis", "nbde") + .required_if_eq("clevis", "tang"), + ) + .arg(Arg::new("thumbprint").long("thumbprint").num_args(1)) + .arg(Arg::new("trust_url").long("trust-url").num_args(0)) + .group( + ArgGroup::new("tang_args") + .arg("thumbprint") + .arg("trust_url"), + ) +} + +type ParseReturn = StratisResult<( + String, + Vec, + Option, + Option<(String, Value)>, +)>; + +fn parse_args() -> ParseReturn { + let args = env::args().collect::>(); + let parser = stratis_legacy_pool_args(); + let matches = parser.get_matches_from(args); + + let pool_name = matches + .get_one::("pool_name") + .expect("required") + .clone(); + let blockdevs = matches + .get_many::("blockdevs") + .expect("required") + .map(PathBuf::from) + .collect::>(); + let key_desc = match matches.get_one::("key_desc") { + Some(kd) => Some(KeyDescription::try_from(kd)?), + None => None, + }; + let pin = matches.get_one::("clevis"); + let clevis_info = match pin.map(|s| s.as_str()) { + Some("nbde" | "tang") => { + let mut json = Map::new(); + json.insert( + "url".to_string(), + Value::from( + matches + .get_one::("tang_url") + .expect("Required") + .clone(), + ), + ); + if matches.get_flag("trust_url") { + json.insert(CLEVIS_TANG_TRUST_URL.to_string(), Value::from(true)); + } else if let Some(thp) = matches.get_one::("thumbprint") { + json.insert("thp".to_string(), Value::from(thp.clone())); + } + pin.map(|p| (p.to_string(), Value::from(json))) + } + Some("tpm2") => Some(("tpm2".to_string(), json!({}))), + Some(_) => unreachable!("Validated by parser"), + None => None, + }; + + Ok((pool_name, blockdevs, key_desc, clevis_info)) +} + +fn main() -> StratisResult<()> { + let (name, devices, key_desc, clevis_info) = parse_args()?; + let unowned = ProcessedPathInfos::try_from( + devices + .iter() + .map(|p| p.as_path()) + .collect::>() + .as_slice(), + )? + .unpack() + .1; + let encryption_info = match (key_desc, clevis_info) { + (Some(kd), Some(ci)) => Some(EncryptionInfo::Both(kd, ci)), + (Some(kd), _) => Some(EncryptionInfo::KeyDesc(kd)), + (_, Some(ci)) => Some(EncryptionInfo::ClevisInfo(ci)), + (_, _) => None, + }; + register_clevis_token()?; + StratPool::initialize(name.as_str(), unowned, encryption_info.as_ref())?; + Ok(()) +} diff --git a/src/engine/mod.rs b/src/engine/mod.rs index ba41f7fea9..3770967ab8 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -2,6 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. +#[cfg(feature = "test_extras")] +pub use self::strat_engine::{ProcessedPathInfos, StratPool}; pub use self::{ engine::{BlockDev, Engine, Filesystem, KeyActions, Pool, Report}, shared::{total_allocated, total_used}, diff --git a/src/engine/strat_engine/backstore/backstore/v1.rs b/src/engine/strat_engine/backstore/backstore/v1.rs index 725261c26f..de50a456ee 100644 --- a/src/engine/strat_engine/backstore/backstore/v1.rs +++ b/src/engine/strat_engine/backstore/backstore/v1.rs @@ -284,7 +284,7 @@ impl Backstore { /// be encrypted only with a kernel keyring and without Clevis information. /// /// WARNING: metadata changing event - #[cfg(test)] + #[cfg(any(test, feature = "test_extras"))] pub fn initialize( pool_name: Name, pool_uuid: PoolUuid, diff --git a/src/engine/strat_engine/backstore/backstore/v2.rs b/src/engine/strat_engine/backstore/backstore/v2.rs index 2e0e10f842..f05f2283e2 100644 --- a/src/engine/strat_engine/backstore/backstore/v2.rs +++ b/src/engine/strat_engine/backstore/backstore/v2.rs @@ -1062,7 +1062,12 @@ mod tests { assert_eq!( backstore.data_tier.allocated(), match (&backstore.linear, &backstore.cache) { - (None, None) => Sectors(0), + (None, None) => + if backstore.is_encrypted() { + crypt_metadata_size().sectors() + } else { + Sectors(0) + }, (&None, Some(cache)) => cache.size(), (Some(linear), &None) => linear.size(), _ => panic!("impossible; see first assertion"), diff --git a/src/engine/strat_engine/backstore/data_tier.rs b/src/engine/strat_engine/backstore/data_tier.rs index 10ce98b93f..c8f6f7c281 100644 --- a/src/engine/strat_engine/backstore/data_tier.rs +++ b/src/engine/strat_engine/backstore/data_tier.rs @@ -43,7 +43,7 @@ impl DataTier { /// Initially 0 segments are allocated. /// /// WARNING: metadata changing event - #[cfg(test)] + #[cfg(any(test, feature = "test_extras"))] pub fn new(block_mgr: BlockDevMgr) -> DataTier { DataTier { block_mgr, diff --git a/src/engine/strat_engine/mod.rs b/src/engine/strat_engine/mod.rs index e4063d7c0d..61243db143 100644 --- a/src/engine/strat_engine/mod.rs +++ b/src/engine/strat_engine/mod.rs @@ -22,6 +22,8 @@ mod types; mod udev; mod writing; +#[cfg(feature = "test_extras")] +pub use self::{backstore::ProcessedPathInfos, pool::v1::StratPool}; pub use self::{ crypt::{ crypt_metadata_size, register_clevis_token, set_up_crypt_logging, CLEVIS_TANG_TRUST_URL, diff --git a/src/engine/strat_engine/pool/v1.rs b/src/engine/strat_engine/pool/v1.rs index 4835071316..e147d79db1 100644 --- a/src/engine/strat_engine/pool/v1.rs +++ b/src/engine/strat_engine/pool/v1.rs @@ -10,7 +10,7 @@ use serde_json::{Map, Value}; use devicemapper::{Bytes, DmNameBuf, Sectors}; use stratisd_proc_macros::strat_pool_impl_gen; -#[cfg(test)] +#[cfg(any(test, feature = "test_extras"))] use crate::engine::{ strat_engine::{ backstore::UnownedDevices, @@ -178,7 +178,7 @@ impl StratPool { /// 1. Initialize the block devices specified by paths. /// 2. Set up thinpool device to back filesystems. /// Precondition: p.is_absolute() is true for all p in paths - #[cfg(test)] + #[cfg(any(test, feature = "test_extras"))] pub fn initialize( name: &str, devices: UnownedDevices, diff --git a/src/engine/strat_engine/thinpool/mdv.rs b/src/engine/strat_engine/thinpool/mdv.rs index 70da2b8825..d1332da573 100644 --- a/src/engine/strat_engine/thinpool/mdv.rs +++ b/src/engine/strat_engine/thinpool/mdv.rs @@ -192,16 +192,33 @@ impl MetadataVol { /// Tear down a Metadata Volume. pub fn teardown(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { - if let Err(retry::Error { error, .. }) = - retry_with_index(Fixed::from_millis(100).take(2), |i| { - trace!("MDV unmount attempt {}", i); - umount(&self.mount_pt) - }) - { - return Err(StratisError::Chained( - "Failed to unmount MDV".to_string(), - Box::new(StratisError::from(error)), - )); + let mtpt_stat = match stat(&self.mount_pt) { + Ok(s) => s, + Err(e) => match e { + nix::errno::Errno::ENOENT => return Ok(()), + e => return Err(StratisError::Nix(e)), + }, + }; + let parent_stat = match stat(&self.mount_pt.join("..")) { + Ok(s) => s, + Err(e) => match e { + nix::errno::Errno::ENOENT => return Ok(()), + e => return Err(StratisError::Nix(e)), + }, + }; + + if mtpt_stat.st_dev != parent_stat.st_dev { + if let Err(retry::Error { error, .. }) = + retry_with_index(Fixed::from_millis(100).take(2), |i| { + trace!("MDV unmount attempt {}", i); + umount(&self.mount_pt) + }) + { + return Err(StratisError::Chained( + "Failed to unmount MDV".to_string(), + Box::new(StratisError::from(error)), + )); + } } if let Err(err) = remove_dir(&self.mount_pt) { diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 348d736e7a..3dc35d4d9a 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -732,7 +732,7 @@ impl ThinPool { impl ThinPool { /// Make a new thin pool. - #[cfg(test)] + #[cfg(any(test, feature = "test_extras"))] pub fn new( pool_uuid: PoolUuid, thin_pool_size: &ThinPoolSizeParams, diff --git a/tests/client-dbus/tests/udev/_utils.py b/tests/client-dbus/tests/udev/_utils.py index cc8548f9f8..40f23a515b 100644 --- a/tests/client-dbus/tests/udev/_utils.py +++ b/tests/client-dbus/tests/udev/_utils.py @@ -35,10 +35,12 @@ from stratisd_client_dbus import ( Blockdev, Manager, + MOBlockDev, MOPool, ObjectManager, Pool, StratisdErrors, + blockdevs, get_object, pools, ) @@ -48,6 +50,7 @@ from ._loopback import LoopBackDevices _STRATISD = os.environ["STRATISD"] +_LEGACY_POOL = os.environ["LEGACY_POOL"] CRYPTO_LUKS_FS_TYPE = "crypto_LUKS" STRATIS_FS_TYPE = "stratis" @@ -72,36 +75,50 @@ def create_pool( :param key_description: optional key description :type key_description: str or NoneType :param clevis_info: clevis information, pin and config - :type clevis_info: pair of str * str - :return: result of the CreatePool D-Bus method call if it succeeds + :type clevis_info: pair of str * (bool, str) + :return: result of pool create if operation succeeds :rtype: bool * str * list of str :raises RuntimeError: if pool is not created """ - (result, exit_code, error_str) = Manager.Methods.CreatePool( - get_object(TOP_OBJECT), - { - "name": name, - "devices": devices, - "key_desc": ( - (False, "") if key_description is None else (True, key_description) - ), - "clevis_info": ( - (False, ("", "")) if clevis_info is None else (True, clevis_info) - ), - }, - ) + newly_created = False + + if len(get_pools(name)) == 0: + cmdline = [_LEGACY_POOL, name] + devices + if key_description is not None: + cmdline.extend(["--key-desc", key_description]) + if clevis_info is not None: + (pin, (tang_url, thp)) = clevis_info + cmdline.extend(["--clevis", pin]) + if pin == "tang": + cmdline.extend(["--tang-url", tang_url]) + if thp is None: + cmdline.append("--trust-url") + else: + cmdline.extend(["--thumbprint", thp]) + + with subprocess.Popen( + cmdline, + text=True, + ) as output: + output.wait() + if output.returncode != 0: + raise RuntimeError( + f"Unable to create a pool {name} with devices {devices}: {output.stderr}" + ) - if exit_code != StratisdErrors.OK: - raise RuntimeError( - f"Unable to create a pool {name} with devices {devices}: {error_str}" - ) + newly_created = True - (_, (pool_object_path, _)) = result + i = 0 + while get_pools(name) == [] and i < 5: + i += 1 + time.sleep(1) + (pool_object_path, _) = next(iter(get_pools(name))) + bd_object_paths = [op for op, _ in get_blockdevs(pool_object_path)] if not overprovision: Pool.Properties.Overprovisioning.Set(get_object(pool_object_path), False) - return result + return (newly_created, (pool_object_path, bd_object_paths)) def get_pools(name=None): @@ -125,6 +142,27 @@ def get_pools(name=None): ] +def get_blockdevs(pool=None): + """ + Get the device nodes belonging to the pool indicated by parent. + + :param parent: list of object paths representing blockdevs + :type blockdev_object_paths: list of str + :return: list of blockdev information found + :rtype: list of (str * MOBlockdev) + """ + managed_objects = ObjectManager.Methods.GetManagedObjects( + get_object(TOP_OBJECT), {} + ) + + return [ + (op, MOBlockDev(info)) + for op, info in blockdevs(props={} if pool is None else {"Pool": pool}).search( + managed_objects + ) + ] + + def get_devnodes(device_object_paths): """ Get the device nodes belonging to these object paths. diff --git a/tests/client-dbus/tests/udev/test_bind.py b/tests/client-dbus/tests/udev/test_bind.py index 87b9bc55ad..c35e47747b 100644 --- a/tests/client-dbus/tests/udev/test_bind.py +++ b/tests/client-dbus/tests/udev/test_bind.py @@ -31,6 +31,8 @@ random_string, ) +_TANG_URL = os.getenv("TANG_URL") + class TestBindingAndAddingTrustedUrl(UdevTest): """ @@ -38,7 +40,6 @@ class TestBindingAndAddingTrustedUrl(UdevTest): adding data devices in various orders. """ - _TANG_URL = os.getenv("TANG_URL") _CLEVIS_CONFIG = {"url": _TANG_URL, "stratis:tang:trust_url": True} _CLEVIS_CONFIG_STR = json.dumps(_CLEVIS_CONFIG) @@ -161,11 +162,9 @@ def test_swap_binding_2(self): (key_description, key) = ("key_spec", "data") with OptionalKeyServiceContextManager(key_spec=[(key_description, key)]): - clevis_info = ("tang", self._CLEVIS_CONFIG_STR) - pool_name = random_string(5) (_, (pool_object_path, _)) = create_pool( - pool_name, initial_devnodes, clevis_info=clevis_info + pool_name, initial_devnodes, clevis_info=("tang", (_TANG_URL, None)) ) self.wait_for_pools(1) @@ -201,10 +200,8 @@ def test_rebind_with_clevis(self): with ServiceContextManager(): pool_name = random_string(5) - clevis_info = ("tang", self._CLEVIS_CONFIG_STR) - (_, (pool_object_path, _)) = create_pool( - pool_name, initial_devnodes, clevis_info=clevis_info + pool_name, initial_devnodes, clevis_info=("tang", (_TANG_URL, None)) ) self.wait_for_pools(1) From 1143b24381e4b090323ffb19d1c46731c48a587a Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 6 Sep 2023 23:44:45 -0400 Subject: [PATCH 12/32] Bump minimum dependency for stratisd_proc_macros --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 389f170762..973e494e3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -209,7 +209,7 @@ version = "0.10.1" optional = true [dependencies.stratisd_proc_macros] -version = "0.2.0" +version = "0.2.1" optional = true path = "./stratisd_proc_macros" From 0e24178450a74527936a2cb3cf99ba475125d0cd Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 8 Sep 2023 09:03:49 -0400 Subject: [PATCH 13/32] Expose metadata version on D-Bus --- src/dbus_api/consts.rs | 1 + src/dbus_api/pool/mod.rs | 6 ++++-- src/dbus_api/pool/pool_3_7/api.rs | 17 +++++++++++++++-- src/dbus_api/pool/pool_3_7/mod.rs | 3 ++- src/dbus_api/pool/pool_3_7/props.rs | 18 ++++++++++++++++++ src/dbus_api/pool/shared.rs | 6 ++++++ src/engine/engine.rs | 3 +++ src/engine/mod.rs | 4 ++-- src/engine/sim_engine/pool.rs | 6 +++++- src/engine/strat_engine/pool/shared.rs | 9 ++++++++- src/engine/strat_engine/pool/v1.rs | 6 +++++- src/engine/strat_engine/pool/v2.rs | 5 +++++ 12 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 src/dbus_api/pool/pool_3_7/props.rs diff --git a/src/dbus_api/consts.rs b/src/dbus_api/consts.rs index 6fe299ef21..16ea2377d5 100644 --- a/src/dbus_api/consts.rs +++ b/src/dbus_api/consts.rs @@ -48,6 +48,7 @@ pub const POOL_ALLOC_SIZE_PROP: &str = "AllocatedSize"; pub const POOL_FS_LIMIT_PROP: &str = "FsLimit"; pub const POOL_OVERPROV_PROP: &str = "Overprovisioning"; pub const POOL_NO_ALLOCABLE_SPACE_PROP: &str = "NoAllocSpace"; +pub const POOL_METADATA_VERSION_PROP: &str = "MetadataVersion"; pub const FILESYSTEM_INTERFACE_NAME_3_0: &str = "org.storage.stratis3.filesystem.r0"; pub const FILESYSTEM_INTERFACE_NAME_3_1: &str = "org.storage.stratis3.filesystem.r1"; diff --git a/src/dbus_api/pool/mod.rs b/src/dbus_api/pool/mod.rs index 3fb20d179d..9bce858b2a 100644 --- a/src/dbus_api/pool/mod.rs +++ b/src/dbus_api/pool/mod.rs @@ -272,7 +272,8 @@ pub fn create_dbus_pool<'a>( .add_p(pool_3_0::total_size_property(&f)) .add_p(pool_3_1::fs_limit_property(&f)) .add_p(pool_3_1::enable_overprov_property(&f)) - .add_p(pool_3_1::no_alloc_space_property(&f)), + .add_p(pool_3_1::no_alloc_space_property(&f)) + .add_p(pool_3_7::metadata_version_property(&f)), ); let path = object_path.get_name().to_owned(); @@ -403,7 +404,8 @@ pub fn get_pool_properties( consts::POOL_TOTAL_SIZE_PROP => shared::pool_total_size(pool), consts::POOL_FS_LIMIT_PROP => shared::pool_fs_limit(pool), consts::POOL_OVERPROV_PROP => shared::pool_overprov_enabled(pool), - consts::POOL_NO_ALLOCABLE_SPACE_PROP => shared::pool_no_alloc_space(pool) + consts::POOL_NO_ALLOCABLE_SPACE_PROP => shared::pool_no_alloc_space(pool), + consts::POOL_METADATA_VERSION_PROP => shared::pool_metadata_version(pool) } } } diff --git a/src/dbus_api/pool/pool_3_7/api.rs b/src/dbus_api/pool/pool_3_7/api.rs index 910c0c40e5..59603f4e34 100644 --- a/src/dbus_api/pool/pool_3_7/api.rs +++ b/src/dbus_api/pool/pool_3_7/api.rs @@ -2,10 +2,14 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use dbus_tree::{Factory, MTSync, Method}; +use dbus_tree::{Access, EmitsChangedSignal, Factory, MTSync, Method, Property}; use crate::dbus_api::{ - pool::pool_3_7::methods::{destroy_filesystems, metadata}, + consts, + pool::pool_3_7::{ + methods::{destroy_filesystems, metadata}, + props::get_pool_metadata_version, + }, types::TData, }; @@ -34,3 +38,12 @@ pub fn get_metadata_method(f: &Factory, TData>) -> Method, TData>, +) -> Property, TData> { + f.property::(consts::POOL_METADATA_VERSION_PROP, ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::Const) + .on_get(get_pool_metadata_version) +} diff --git a/src/dbus_api/pool/pool_3_7/mod.rs b/src/dbus_api/pool/pool_3_7/mod.rs index 063e004350..834df4053d 100644 --- a/src/dbus_api/pool/pool_3_7/mod.rs +++ b/src/dbus_api/pool/pool_3_7/mod.rs @@ -4,5 +4,6 @@ mod api; mod methods; +mod props; -pub use api::{destroy_filesystems_method, get_metadata_method}; +pub use api::{destroy_filesystems_method, get_metadata_method, metadata_version_property}; diff --git a/src/dbus_api/pool/pool_3_7/props.rs b/src/dbus_api/pool/pool_3_7/props.rs new file mode 100644 index 0000000000..50fe52c3df --- /dev/null +++ b/src/dbus_api/pool/pool_3_7/props.rs @@ -0,0 +1,18 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::arg::IterAppend; +use dbus_tree::{MTSync, MethodErr, PropInfo}; + +use crate::dbus_api::{ + pool::shared::{self, get_pool_property}, + types::TData, +}; + +pub fn get_pool_metadata_version( + i: &mut IterAppend<'_>, + p: &PropInfo<'_, MTSync, TData>, +) -> Result<(), MethodErr> { + get_pool_property(i, p, |(_, _, pool)| Ok(shared::pool_metadata_version(pool))) +} diff --git a/src/dbus_api/pool/shared.rs b/src/dbus_api/pool/shared.rs index a22c6661dd..0da78b0867 100644 --- a/src/dbus_api/pool/shared.rs +++ b/src/dbus_api/pool/shared.rs @@ -321,6 +321,12 @@ pub fn pool_fs_limit(pool: &dyn Pool) -> u64 { pool.fs_limit() } +/// Generate a D-Bus representation of the filesystem limit on the pool. +#[inline] +pub fn pool_metadata_version(pool: &dyn Pool) -> u64 { + pool.metadata_version() as u64 +} + /// Set the filesystem limit on a pool. #[inline] pub fn set_pool_fs_limit( diff --git a/src/engine/engine.rs b/src/engine/engine.rs index 5002ef741a..ddcc1b405b 100644 --- a/src/engine/engine.rs +++ b/src/engine/engine.rs @@ -347,6 +347,9 @@ pub trait Pool: Debug + Send + Sync { /// Return the metadata that was last written to pool devices. fn last_metadata(&self) -> StratisResult; + + /// Get the metadata version for a given pool. + fn metadata_version(&self) -> StratSigblockVersion; } pub type HandleEvents

= ( diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 3770967ab8..ad9a0eb685 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -21,8 +21,8 @@ pub use self::{ MaybeInconsistent, Name, PoolDiff, PoolEncryptionInfo, PoolIdentifier, PoolUuid, PropChangeAction, RenameAction, ReportType, SetCreateAction, SetDeleteAction, SetUnlockAction, StartAction, StopAction, StoppedPoolInfo, StoppedPoolsInfo, - StratBlockDevDiff, StratFilesystemDiff, StratPoolDiff, StratisUuid, ThinPoolDiff, - ToDisplay, UdevEngineEvent, UnlockMethod, + StratBlockDevDiff, StratFilesystemDiff, StratPoolDiff, StratSigblockVersion, StratisUuid, + ThinPoolDiff, ToDisplay, UdevEngineEvent, UnlockMethod, }, }; diff --git a/src/engine/sim_engine/pool.rs b/src/engine/sim_engine/pool.rs index 6b8441ffec..1e9a017b29 100644 --- a/src/engine/sim_engine/pool.rs +++ b/src/engine/sim_engine/pool.rs @@ -25,7 +25,7 @@ use crate::{ ActionAvailability, BlockDevTier, Clevis, CreateAction, DeleteAction, DevUuid, EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, PoolUuid, RegenAction, RenameAction, SetCreateAction, - SetDeleteAction, + SetDeleteAction, StratSigblockVersion, }, PropChangeAction, }, @@ -764,6 +764,10 @@ impl Pool for SimPool { // Just invent a name for the pool; a sim pool has no real metadata serde_json::to_string(&self.record("")).map_err(|e| e.into()) } + + fn metadata_version(&self) -> StratSigblockVersion { + StratSigblockVersion::V2 + } } #[cfg(test)] diff --git a/src/engine/strat_engine/pool/shared.rs b/src/engine/strat_engine/pool/shared.rs index 46099a7470..3c17b2e361 100644 --- a/src/engine/strat_engine/pool/shared.rs +++ b/src/engine/strat_engine/pool/shared.rs @@ -16,7 +16,7 @@ use crate::{ ActionAvailability, BlockDevTier, Clevis, CreateAction, DeleteAction, DevUuid, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, PoolUuid, PropChangeAction, RegenAction, RenameAction, SetCreateAction, - SetDeleteAction, + SetDeleteAction, StratSigblockVersion, }, }, stratis::StratisResult, @@ -341,4 +341,11 @@ impl Pool for AnyPool { AnyPool::V2(p) => p.last_metadata(), } } + + fn metadata_version(&self) -> StratSigblockVersion { + match self { + AnyPool::V1(p) => p.metadata_version(), + AnyPool::V2(p) => p.metadata_version(), + } + } } diff --git a/src/engine/strat_engine/pool/v1.rs b/src/engine/strat_engine/pool/v1.rs index e147d79db1..0fd9aba521 100644 --- a/src/engine/strat_engine/pool/v1.rs +++ b/src/engine/strat_engine/pool/v1.rs @@ -43,7 +43,7 @@ use crate::{ ActionAvailability, BlockDevTier, Clevis, Compare, CreateAction, DeleteAction, DevUuid, Diff, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, PoolUuid, RegenAction, RenameAction, SetCreateAction, - SetDeleteAction, StratFilesystemDiff, StratPoolDiff, + SetDeleteAction, StratFilesystemDiff, StratPoolDiff, StratSigblockVersion, }, PropChangeAction, }, @@ -1288,6 +1288,10 @@ impl Pool for StratPool { .map_err(|_| StratisError::Msg("metadata byte array is not utf-8".into())) }) } + + fn metadata_version(&self) -> StratSigblockVersion { + StratSigblockVersion::V1 + } } pub struct StratPoolState { diff --git a/src/engine/strat_engine/pool/v2.rs b/src/engine/strat_engine/pool/v2.rs index 01c98ac1d8..4f1eec0d8a 100644 --- a/src/engine/strat_engine/pool/v2.rs +++ b/src/engine/strat_engine/pool/v2.rs @@ -35,6 +35,7 @@ use crate::{ Diff, EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, PoolUuid, PropChangeAction, RegenAction, RenameAction, SetCreateAction, SetDeleteAction, StratFilesystemDiff, StratPoolDiff, + StratSigblockVersion, }, }, stratis::{StratisError, StratisResult}, @@ -1189,6 +1190,10 @@ impl Pool for StratPool { .map_err(|_| StratisError::Msg("metadata byte array is not utf-8".into())) }) } + + fn metadata_version(&self) -> StratSigblockVersion { + StratSigblockVersion::V2 + } } pub struct StratPoolState { From 8b49e4a8aeea0923888910035058c41ca0a03e7a Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 22 Sep 2023 16:06:14 -0400 Subject: [PATCH 14/32] Fix up tests plan and tests for legacy pool script --- plans/all.fmf | 36 ++++++++++++++++++++++++++++++++---- tests-fmf/python.fmf | 2 ++ tests-fmf/rust.fmf | 1 + 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/plans/all.fmf b/plans/all.fmf index aa6c4c4a7b..0dbc96eb9f 100644 --- a/plans/all.fmf +++ b/plans/all.fmf @@ -1,5 +1,6 @@ summary: top level management +enabled: true adjust: when: plan == cockpit enabled: false @@ -11,28 +12,55 @@ prepare: - name: Install packages how: install package: - - tang + - cargo + - clang + - cryptsetup-devel + - curl + - dbus-devel + - device-mapper-devel + - libblkid-devel + - make + - ncurses + - rust - systemd - swtpm - swtpm-tools - tpm2-tools + - systemd-devel + - tang - name: Start TPM2 emulation how: shell script: mkdir /var/tmp/swtpm; swtpm_setup --tpm-state /var/tmp/swtpm --tpm2; swtpm chardev --vtpm-proxy --tpmstate dir=/var/tmp/swtpm --tpm2 &> /var/log/swtpm & - name: Start tang server how: shell script: systemctl enable tangd.socket --now - - name: Reload udev - how: shell - script: udevadm control --reload - name: Show test system information how: shell script: free -m; lsblk -i; lscpu; cat /proc/1/sched - name: Record mkfs.xfs version how: shell script: mkfs.xfs -V + discover: how: fmf + execute: how: tmt exit-first: false + +/python: + prepare+: + - name: Build and install legacy pool script + how: shell + script: + - PROFILEDIR=debug make build-test-extras + - mv target/debug/stratis-legacy-pool /usr/local/bin + discover+: + filter: "tag:python" + +/rust: + discover+: + filter: "tag:rust" + execute: + how: tmt + exit-first: false diff --git a/tests-fmf/python.fmf b/tests-fmf/python.fmf index b611347b74..3d2b37c06b 100644 --- a/tests-fmf/python.fmf +++ b/tests-fmf/python.fmf @@ -1,5 +1,6 @@ path: /tests/client-dbus duration: 20m +tag: python require: - clevis-luks @@ -16,6 +17,7 @@ environment: STRATISD: /usr/libexec/stratisd STRATIS_DUMPMETADATA: /usr/bin/stratis-dumpmetadata PYTHONPATH: ./src + LEGACY_POOL: /usr/local/bin/stratis-legacy-pool /udev: summary: Run Python udev tests diff --git a/tests-fmf/rust.fmf b/tests-fmf/rust.fmf index e5595a7714..06855ff3d1 100644 --- a/tests-fmf/rust.fmf +++ b/tests-fmf/rust.fmf @@ -1,5 +1,6 @@ path: / duration: 20m +tag: rust require: - cargo From 696554ef116282d84a2552d2192419e9970b3dd1 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 1 Nov 2023 14:40:41 -0400 Subject: [PATCH 15/32] Merge encryption info and crypt handle into one field --- .../strat_engine/backstore/backstore/v2.rs | 246 ++++++++++-------- src/engine/strat_engine/crypt/handle/v2.rs | 22 +- 2 files changed, 161 insertions(+), 107 deletions(-) diff --git a/src/engine/strat_engine/backstore/backstore/v2.rs b/src/engine/strat_engine/backstore/backstore/v2.rs index f05f2283e2..6b478b887c 100644 --- a/src/engine/strat_engine/backstore/backstore/v2.rs +++ b/src/engine/strat_engine/backstore/backstore/v2.rs @@ -7,6 +7,7 @@ use std::{cmp, collections::HashMap, iter::once, path::PathBuf}; use chrono::{DateTime, Utc}; +use either::Either; use serde_json::Value; use devicemapper::{ @@ -117,7 +118,10 @@ fn make_cache( /// Set up the linear device on top of the data tier that can later be converted to a /// cache device and serves as a placeholder for the device beneath encryption. -fn make_cap_linear_dev(pool_uuid: PoolUuid, origin: &LinearDev) -> Result { +fn make_placeholder_dev( + pool_uuid: PoolUuid, + origin: &LinearDev, +) -> Result { let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::Cache); let target = vec![TargetLine::new( Sectors(0), @@ -132,33 +136,35 @@ fn make_cap_linear_dev(pool_uuid: PoolUuid, origin: &LinearDev) -> Result, /// Coordinate handling of blockdevs that back the cache. Optional, since /// this structure can operate without a cache. cache_tier: Option>, /// Coordinates handling of the blockdevs that form the base. data_tier: DataTier, /// A linear DM device. - linear: Option, - /// A placeholder device to be converted to cache. - cap_linear: Option, - /// Handle for encryption layer in backstore. - handle: Option, - /// Encryption info for encryption layer. - encryption_info: Option, + origin: Option, + /// A placeholder device to be converted to cache or a cache device. + cache: Option, + /// A placeholder device to be converted to cache; necessary for reencryption support. + placeholder: Option, + /// Either encryption information for a handle to be created at a later time or + /// handle for encryption layer in backstore. + enc: Option>, /// Index for managing allocation of cap device next: Sectors, } impl InternalBackstore for Backstore { fn device(&self) -> Option { - self.handle.as_ref().map(|h| h.device()).or_else(|| { - self.cache - .as_ref() - .map(|d| d.device()) - .or_else(|| self.cap_linear.as_ref().map(|d| d.device())) - }) + self.enc + .as_ref() + .and_then(|either| either.as_ref().right().map(|h| h.device())) + .or_else(|| { + self.cache + .as_ref() + .map(|c| c.device()) + .or_else(|| self.placeholder.as_ref().map(|lin| lin.device())) + }) } fn datatier_allocated_size(&self) -> Sectors { @@ -167,7 +173,7 @@ impl InternalBackstore for Backstore { fn datatier_usable_size(&self) -> Sectors { self.data_tier.usable_size() - - if self.encryption_info.is_some() { + - if self.enc.is_some() { crypt_metadata_size().sectors() } else { Sectors(0) @@ -177,7 +183,7 @@ impl InternalBackstore for Backstore { fn available_in_backstore(&self) -> Sectors { self.data_tier.usable_size() - self.next - - if self.encryption_info.is_some() { + - if self.enc.is_some() { crypt_metadata_size().sectors() } else { Sectors(0) @@ -241,7 +247,7 @@ impl Backstore { /// * no StratBlockDev in cachedevs has a key description /// /// Postcondition: - /// self.linear.is_some() XOR self.cache.is_some() + /// self.origin.is_some() XOR self.cache.is_some() /// self.cache.is_some() <=> self.cache_tier.is_some() pub fn setup( pool_uuid: PoolUuid, @@ -273,7 +279,7 @@ impl Backstore { } }; - let (cap_linear, cache_tier, cache, origin) = if !cachedevs.is_empty() { + let (placeholder, cache, cache_tier, origin) = if !cachedevs.is_empty() { let block_mgr = BlockDevMgr::new(cachedevs, Some(last_update_time)); match backstore_save.cache_tier { Some(ref cache_tier_save) => { @@ -300,7 +306,7 @@ impl Backstore { )); } }; - (None, Some(cache_tier), Some(cache_device), None) + (None, Some(cache_device), Some(cache_tier), None) } None => { let err_msg = "Cachedevs exist, but cache metadata does not exist"; @@ -316,43 +322,36 @@ impl Backstore { } } } else { - let cap_linear = match make_cap_linear_dev(pool_uuid, &origin) { - Ok(cap) => cap, + let placeholder = match make_placeholder_dev(pool_uuid, &origin) { + Ok(pl) => pl, Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), }; - (Some(cap_linear), None, None, Some(origin)) + (Some(placeholder), None, None, Some(origin)) }; - let (encryption_info, handle) = { - let handle = match CryptHandle::setup( - &once(DEVICEMAPPER_PATH) - .chain(once( - format_backstore_ids(pool_uuid, CacheRole::Cache) - .0 - .to_string() - .as_str(), - )) - .collect::(), - pool_uuid, - UnlockMethod::Any, - ) { - Ok(opt) => opt, - Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), - }; - match handle { - Some(handle) => (Some(handle.encryption_info().clone()), Some(handle)), - None => (None, None), - } + let enc = match CryptHandle::setup( + &once(DEVICEMAPPER_PATH) + .chain(once( + format_backstore_ids(pool_uuid, CacheRole::Cache) + .0 + .to_string() + .as_str(), + )) + .collect::(), + pool_uuid, + UnlockMethod::Any, + ) { + Ok(opt) => opt.map(Either::Right), + Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), }; Ok(Backstore { data_tier, cache_tier, - linear: origin, + origin, cache, - cap_linear, - encryption_info, - handle, + placeholder, + enc, next: backstore_save.cap.allocs[0].1, }) } @@ -372,26 +371,28 @@ impl Backstore { mda_data_size: MDADataSize, encryption_info: Option<&EncryptionInfo>, ) -> StratisResult { - let mut data_tier = DataTier::::new( - BlockDevMgr::::initialize(pool_uuid, devices, mda_data_size)?, - ); - if encryption_info.is_some() && !data_tier.alloc(&[crypt_metadata_size().sectors()]) { - return Err(StratisError::Msg( - "There was not enough space on the device to satisfy the allocation request" - .to_string(), - )); - } + let data_tier = DataTier::::new(BlockDevMgr::::initialize( + pool_uuid, + devices, + mda_data_size, + )?); - Ok(Backstore { + let mut backstore = Backstore { data_tier, + placeholder: None, cache_tier: None, - linear: None, cache: None, - cap_linear: None, - handle: None, - encryption_info: encryption_info.cloned(), + origin: None, + enc: encryption_info.cloned().map(Either::Left), next: Sectors(0), - }) + }; + + let size = crypt_metadata_size().sectors(); + backstore.alloc(pool_uuid, &[size])?.ok_or_else(|| { + StratisError::Msg(format!("Failed to satisfy request in backstore for {size}")) + })?; + + Ok(backstore) } /// Initialize the cache tier and add cachedevs to the backstore. @@ -400,8 +401,8 @@ impl Backstore { /// /// Precondition: Must be invoked only after some space has been allocated /// from the backstore. This ensures that there is certainly a cap device. - // Precondition: self.cache.is_none() && self.linear.is_some() - // Postcondition: self.cache.is_some() && self.linear.is_none() + // Precondition: self.cache.is_none() && self.placeholder.is_some() + // Postcondition: self.cache.is_some() && self.placeholder.is_none() pub fn init_cache( &mut self, pool_uuid: PoolUuid, @@ -423,14 +424,14 @@ impl Backstore { let cache_tier = CacheTier::new(bdm)?; - let linear = self.linear + let origin = self.origin .take() - .expect("some space has already been allocated from the backstore => (cache_tier.is_none() <=> self.linear.is_some())"); - let cap = self.cap_linear + .expect("some space has already been allocated from the backstore => (cache_tier.is_none() <=> self.origin.is_some())"); + let placeholder = self.placeholder .take() - .expect("some space has already been allocated from the backstore => (cache_tier.is_none() <=> self.cap_linear.is_some())"); + .expect("some space has already been allocated from the backstore => (cache_tier.is_none() <=> self.placeholder.is_some())"); - let cache = make_cache(pool_uuid, &cache_tier, linear, Some(cap), true)?; + let cache = make_cache(pool_uuid, &cache_tier, origin, Some(placeholder), true)?; self.cache = Some(cache); @@ -457,9 +458,9 @@ impl Backstore { /// /// Precondition: Must be invoked only after some space has been allocated /// from the backstore. This ensures that there is certainly a cap device. - // Precondition: self.linear.is_none() && self.cache.is_some() + // Precondition: self.origin.is_none() && self.cache.is_some() // Precondition: self.cache_key_desc has the desired key description - // Precondition: self.cache.is_some() && self.linear.is_none() + // Precondition: self.cache.is_some() && self.origin.is_none() pub fn add_cachedevs( &mut self, pool_uuid: PoolUuid, @@ -510,13 +511,14 @@ impl Backstore { fn extend_cap_device(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { let create = match ( self.cache.as_mut(), - self.cap_linear + self.placeholder .as_mut() - .and_then(|c| self.linear.as_mut().map(|l| (c, l))), - self.handle.as_mut(), + .and_then(|c| self.origin.as_mut().map(|l| (c, l))), + self.enc.as_mut(), ) { (None, None, None) => true, - (Some(cache), None, Some(handle)) => { + (_, _, Some(Either::Left(_))) => true, + (Some(cache), None, Some(Either::Right(handle))) => { let table = self.data_tier.segments.map_to_dm(); cache.set_origin_table(get_dm(), table)?; cache.resume(get_dm())?; @@ -529,7 +531,7 @@ impl Backstore { cache.resume(get_dm())?; false } - (None, Some((cap, linear)), Some(handle)) => { + (None, Some((cap, linear)), Some(Either::Right(handle))) => { let table = self.data_tier.segments.map_to_dm(); linear.set_table(get_dm(), table)?; linear.resume(get_dm())?; @@ -562,16 +564,16 @@ impl Backstore { cap.resume(get_dm())?; false } - _ => panic!("NOT (self.cache().is_some() AND self.linear.is_some())"), + _ => panic!("NOT (self.cache().is_some() AND self.origin.is_some())"), }; if create { let table = self.data_tier.segments.map_to_dm(); let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::OriginSub); let origin = LinearDev::setup(get_dm(), &dm_name, Some(&dm_uuid), table)?; - let cap = make_cap_linear_dev(pool_uuid, &origin)?; - let handle = match self.encryption_info { - Some(ref einfo) => Some(CryptHandle::initialize( + let placeholder = make_placeholder_dev(pool_uuid, &origin)?; + let handle = match self.enc { + Some(Either::Left(ref einfo)) => Some(CryptHandle::initialize( &once(DEVICEMAPPER_PATH) .chain(once( format_backstore_ids(pool_uuid, CacheRole::Cache) @@ -584,11 +586,12 @@ impl Backstore { einfo, None, )?), + Some(Either::Right(_)) => unreachable!("Checked above"), None => None, }; - self.linear = Some(origin); - self.cap_linear = Some(cap); - self.handle = handle; + self.origin = Some(origin); + self.placeholder = Some(placeholder); + self.enc = handle.map(Either::Right); } Ok(()) @@ -657,16 +660,17 @@ impl Backstore { /// no ioctl is required. #[cfg(test)] fn size(&self) -> Sectors { - self.linear + self.enc .as_ref() - .map(|d| d.size()) + .and_then(|either| either.as_ref().right().map(|handle| handle.size())) + .or_else(|| self.placeholder.as_ref().map(|d| d.size())) .or_else(|| self.cache.as_ref().map(|d| d.size())) .unwrap_or(Sectors(0)) } /// Destroy the entire store. pub fn destroy(&mut self, pool_uuid: PoolUuid) -> StratisResult<()> { - if let Some(h) = self.handle.as_mut() { + if let Some(h) = self.enc.as_mut().and_then(|either| either.as_ref().right()) { h.wipe()?; } let devs = list_of_backstore_devices(pool_uuid); @@ -783,7 +787,7 @@ impl Backstore { } pub fn is_encrypted(&self) -> bool { - self.encryption_info.is_some() + self.enc.is_some() } pub fn has_cache(&self) -> bool { @@ -792,7 +796,9 @@ impl Backstore { /// Get the encryption information for the backstore. pub fn encryption_info(&self) -> Option<&EncryptionInfo> { - self.encryption_info.as_ref() + self.enc + .as_ref() + .map(|either| either.as_ref().either(|e| e, |h| h.encryption_info())) } /// Bind device in the given backstore using the given clevis @@ -804,9 +810,14 @@ impl Backstore { /// * Returns Err(_) if binding failed. pub fn bind_clevis(&mut self, pin: &str, clevis_info: &Value) -> StratisResult { let handle = self - .handle + .enc + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))? .as_mut() - .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + .right() + .ok_or_else(|| { + StratisError::Msg("No space has been allocated from the backstore".to_string()) + })?; let mut parsed_config = clevis_info.clone(); let yes = interpret_clevis_config(pin, &mut parsed_config)?; @@ -844,9 +855,14 @@ impl Backstore { /// * Returns Err(_) if unbinding failed. pub fn unbind_clevis(&mut self) -> StratisResult { let handle = self - .handle + .enc .as_mut() - .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))? + .as_mut() + .right() + .ok_or_else(|| { + StratisError::Msg("No space has been allocated from the backstore".to_string()) + })?; if handle.encryption_info().clevis_info().is_some() { handle.clevis_unbind()?; @@ -865,9 +881,14 @@ impl Backstore { /// * Returns Err(_) if binding failed. pub fn bind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult { let handle = self - .handle + .enc + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))? .as_mut() - .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + .right() + .ok_or_else(|| { + StratisError::Msg("No space has been allocated from the backstore".to_string()) + })?; if let Some(kd) = handle.encryption_info().key_description() { if kd == key_desc { @@ -895,9 +916,14 @@ impl Backstore { /// * Returns Err(_) if unbinding failed. pub fn unbind_keyring(&mut self) -> StratisResult { let handle = self - .handle + .enc .as_mut() - .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))? + .as_mut() + .right() + .ok_or_else(|| { + StratisError::Msg("No space has been allocated from the backstore".to_string()) + })?; if handle.encryption_info().key_description().is_some() { handle.unbind_keyring()?; @@ -917,9 +943,14 @@ impl Backstore { /// * Err(_) if an operation fails while changing the passphrase. pub fn rebind_keyring(&mut self, key_desc: &KeyDescription) -> StratisResult> { let handle = self - .handle + .enc + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))? .as_mut() - .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + .right() + .ok_or_else(|| { + StratisError::Msg("No space has been allocated from the backstore".to_string()) + })?; if handle.encryption_info().key_description() == Some(key_desc) { Ok(Some(false)) @@ -941,9 +972,14 @@ impl Backstore { /// result in a metadata change. pub fn rebind_clevis(&mut self) -> StratisResult<()> { let handle = self - .handle + .enc + .as_mut() + .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))? .as_mut() - .ok_or_else(|| StratisError::Msg("Pool is not encrypted".to_string()))?; + .right() + .ok_or_else(|| { + StratisError::Msg("No space has been allocated from the backstore".to_string()) + })?; if handle.encryption_info().clevis_info().is_none() { Err(StratisError::Msg( @@ -1048,7 +1084,7 @@ mod tests { /// Assert some invariants of the backstore /// * backstore.cache_tier.is_some() <=> backstore.cache.is_some() && - /// backstore.cache_tier.is_some() => backstore.linear.is_none() + /// backstore.cache_tier.is_some() => backstore.origin.is_none() /// * backstore's data tier allocated is equal to the size of the cap device /// * backstore's next index is always less than the size of the cap /// device @@ -1057,11 +1093,11 @@ mod tests { (backstore.cache_tier.is_none() && backstore.cache.is_none()) || (backstore.cache_tier.is_some() && backstore.cache.is_some() - && backstore.linear.is_none()) + && backstore.origin.is_none()) ); assert_eq!( backstore.data_tier.allocated(), - match (&backstore.linear, &backstore.cache) { + match (&backstore.origin, &backstore.cache) { (None, None) => if backstore.is_encrypted() { crypt_metadata_size().sectors() @@ -1128,7 +1164,7 @@ mod tests { invariant(&backstore); assert_eq!(cache_uuids.len(), initcachepaths.len()); - assert_matches!(backstore.linear, None); + assert_matches!(backstore.origin, None); let cache_status = backstore .cache diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs index 4ea46e0f60..23d4d16b9e 100644 --- a/src/engine/strat_engine/crypt/handle/v2.rs +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -4,6 +4,7 @@ use std::{ fmt::Debug, + fs::File, iter::once, path::{Path, PathBuf}, }; @@ -42,6 +43,7 @@ use crate::{ wipe_fallback, }, }, + device::blkdev_size, dm::DEVICEMAPPER_PATH, names::format_crypt_backstore_name, }, @@ -99,6 +101,7 @@ pub fn load_crypt_metadata( .collect::(); let activated_path = path.canonicalize().unwrap_or(path); let devno = get_devno_from_path(&activated_path)?; + let size = blkdev_size(&File::open(&activated_path)?)?.sectors(); Ok(Some(CryptMetadata { physical_path: physical, pool_uuid, @@ -106,6 +109,7 @@ pub fn load_crypt_metadata( activation_name, activated_path, device: devno, + size, })) } @@ -117,6 +121,7 @@ pub struct CryptMetadata { pub activation_name: DmNameBuf, pub activated_path: PathBuf, pub device: Device, + pub size: Sectors, } /// Check whether the physical device path corresponds to an encrypted @@ -204,6 +209,7 @@ pub fn setup_crypt_handle( metadata.pool_uuid, metadata.encryption_info, metadata.device, + metadata.size, ))) } @@ -222,6 +228,7 @@ impl CryptHandle { pool_uuid: PoolUuid, encryption_info: EncryptionInfo, devno: Device, + size: Sectors, ) -> CryptHandle { let activation_name = format_crypt_backstore_name(&pool_uuid); let path = vec![DEVICEMAPPER_PATH, &activation_name.to_string()] @@ -236,6 +243,7 @@ impl CryptHandle { activation_name, device: devno, activated_path, + size, }, } } @@ -285,11 +293,13 @@ impl CryptHandle { let device_path = DevicePath::new(physical_path)?; let devno = get_devno_from_path(&once(DEVICEMAPPER_PATH).chain(once(activation_name.to_string().as_str())).collect::())?; + let size = blkdev_size(&File::open(["/dev", "mapper", &activation_name.to_string()].iter().collect::())?)?.sectors(); Ok(CryptHandle::new( device_path, pool_uuid, encryption_info, devno, + size, )) }) .map_err(|e| { @@ -488,6 +498,12 @@ impl CryptHandle { } } + /// Get the device size for this encrypted device. + #[cfg(test)] + pub fn size(&self) -> Sectors { + self.metadata.size + } + /// Get the encryption info for this encrypted device. pub fn encryption_info(&self) -> &EncryptionInfo { &self.metadata.encryption_info @@ -731,7 +747,7 @@ impl CryptHandle { /// Changed the encrypted device size /// `None` will fill up the entire underlying physical device. /// `Some(_)` will resize the device to the given number of sectors. - pub fn resize(&self, size: Option) -> StratisResult<()> { + pub fn resize(&mut self, size: Option) -> StratisResult<()> { let processed_size = match size { Some(s) => { if s == Sectors(0) { @@ -754,7 +770,9 @@ impl CryptHandle { crypt .context_handle() .resize(&self.activation_name().to_string(), processed_size) - .map_err(StratisError::Crypt) + .map_err(StratisError::Crypt)?; + self.metadata.size = blkdev_size(&File::open(&self.metadata.activated_path)?)?.sectors(); + Ok(()) } } From 67d832ddb0e9a95acde290955912dbb6a0827f32 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 9 Nov 2023 09:35:02 -0500 Subject: [PATCH 16/32] Fix for case where metadata needs to be read but device is not activated --- src/engine/strat_engine/crypt/handle/v2.rs | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs index 23d4d16b9e..a7ee3d3425 100644 --- a/src/engine/strat_engine/crypt/handle/v2.rs +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -100,16 +100,12 @@ pub fn load_crypt_metadata( .into_iter() .collect::(); let activated_path = path.canonicalize().unwrap_or(path); - let devno = get_devno_from_path(&activated_path)?; - let size = blkdev_size(&File::open(&activated_path)?)?.sectors(); Ok(Some(CryptMetadata { physical_path: physical, pool_uuid, encryption_info, activation_name, activated_path, - device: devno, - size, })) } @@ -120,8 +116,6 @@ pub struct CryptMetadata { pub encryption_info: EncryptionInfo, pub activation_name: DmNameBuf, pub activated_path: PathBuf, - pub device: Device, - pub size: Sectors, } /// Check whether the physical device path corresponds to an encrypted @@ -204,12 +198,15 @@ pub fn setup_crypt_handle( )? } + let device = get_devno_from_path(&metadata.activated_path)?; + let size = blkdev_size(&File::open(&metadata.activated_path)?)?.sectors(); + Ok(Some(CryptHandle::new( metadata.physical_path, metadata.pool_uuid, metadata.encryption_info, - metadata.device, - metadata.size, + device, + size, ))) } @@ -220,6 +217,8 @@ pub fn setup_crypt_handle( #[derive(Debug, Clone)] pub struct CryptHandle { metadata: CryptMetadata, + device: Device, + size: Sectors, } impl CryptHandle { @@ -241,10 +240,10 @@ impl CryptHandle { pool_uuid, encryption_info, activation_name, - device: devno, activated_path, - size, }, + device: devno, + size, } } @@ -501,7 +500,7 @@ impl CryptHandle { /// Get the device size for this encrypted device. #[cfg(test)] pub fn size(&self) -> Sectors { - self.metadata.size + self.size } /// Get the encryption info for this encrypted device. @@ -528,7 +527,7 @@ impl CryptHandle { /// Device number for the LUKS2 encrypted device. pub fn device(&self) -> Device { - self.metadata.device + self.device } /// Get the keyslot associated with the given token ID. @@ -771,7 +770,7 @@ impl CryptHandle { .context_handle() .resize(&self.activation_name().to_string(), processed_size) .map_err(StratisError::Crypt)?; - self.metadata.size = blkdev_size(&File::open(&self.metadata.activated_path)?)?.sectors(); + self.size = blkdev_size(&File::open(&self.metadata.activated_path)?)?.sectors(); Ok(()) } } @@ -1034,8 +1033,6 @@ mod tests { fn test_both_initialize(paths: &[&Path]) { fn both_initialize(paths: &[&Path], key_desc: &KeyDescription, pool_uuid: PoolUuid) { - unshare_mount_namespace().unwrap(); - let _memfs = MemoryFilesystem::new().unwrap(); let path = paths.first().copied().expect("Expected exactly one path"); let handle = CryptHandle::initialize( path, From 3caa9e7267920b2d6012d1b68cccc9263bd427a4 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 6 Nov 2023 18:16:00 -0500 Subject: [PATCH 17/32] Add space for metadata in unencrypted use case --- src/bin/stratis-legacy-pool.rs | 2 + .../strat_engine/backstore/backstore/v1.rs | 1 + .../strat_engine/backstore/backstore/v2.rs | 176 +++++++++++++----- src/engine/strat_engine/engine.rs | 2 +- src/engine/strat_engine/pool/v1.rs | 4 +- src/engine/strat_engine/pool/v2.rs | 36 ++-- src/engine/strat_engine/serde_structs.rs | 3 + src/engine/strat_engine/thinpool/thinpool.rs | 2 +- 8 files changed, 157 insertions(+), 69 deletions(-) diff --git a/src/bin/stratis-legacy-pool.rs b/src/bin/stratis-legacy-pool.rs index 5b7081240f..fb5dbaeea0 100644 --- a/src/bin/stratis-legacy-pool.rs +++ b/src/bin/stratis-legacy-pool.rs @@ -108,6 +108,8 @@ fn parse_args() -> ParseReturn { } fn main() -> StratisResult<()> { + env_logger::init(); + let (name, devices, key_desc, clevis_info) = parse_args()?; let unowned = ProcessedPathInfos::try_from( devices diff --git a/src/engine/strat_engine/backstore/backstore/v1.rs b/src/engine/strat_engine/backstore/backstore/v1.rs index de50a456ee..4131b2fc16 100644 --- a/src/engine/strat_engine/backstore/backstore/v1.rs +++ b/src/engine/strat_engine/backstore/backstore/v1.rs @@ -984,6 +984,7 @@ impl Recordable for Backstore { cache_tier: self.cache_tier.as_ref().map(|c| c.record()), cap: CapSave { allocs: vec![(Sectors(0), self.next)], + crypt_meta_allocs: Vec::new(), }, data_tier: self.data_tier.record(), } diff --git a/src/engine/strat_engine/backstore/backstore/v2.rs b/src/engine/strat_engine/backstore/backstore/v2.rs index 6b478b887c..c47c652e07 100644 --- a/src/engine/strat_engine/backstore/backstore/v2.rs +++ b/src/engine/strat_engine/backstore/backstore/v2.rs @@ -150,8 +150,10 @@ pub struct Backstore { /// Either encryption information for a handle to be created at a later time or /// handle for encryption layer in backstore. enc: Option>, - /// Index for managing allocation of cap device - next: Sectors, + /// Data allocations on the cap device, + allocs: Vec<(Sectors, Sectors)>, + /// Metadata allocations on the cache or placeholder device. + crypt_meta_allocs: Vec<(Sectors, Sectors)>, } impl InternalBackstore for Backstore { @@ -159,12 +161,8 @@ impl InternalBackstore for Backstore { self.enc .as_ref() .and_then(|either| either.as_ref().right().map(|h| h.device())) - .or_else(|| { - self.cache - .as_ref() - .map(|c| c.device()) - .or_else(|| self.placeholder.as_ref().map(|lin| lin.device())) - }) + .or_else(|| self.cache.as_ref().map(|c| c.device())) + .or_else(|| self.placeholder.as_ref().map(|lin| lin.device())) } fn datatier_allocated_size(&self) -> Sectors { @@ -172,22 +170,13 @@ impl InternalBackstore for Backstore { } fn datatier_usable_size(&self) -> Sectors { - self.data_tier.usable_size() - - if self.enc.is_some() { - crypt_metadata_size().sectors() - } else { - Sectors(0) - } + self.data_tier.usable_size() - self.crypt_meta_allocs.iter().map(|(_, len)| *len).sum() } fn available_in_backstore(&self) -> Sectors { self.data_tier.usable_size() - - self.next - - if self.enc.is_some() { - crypt_metadata_size().sectors() - } else { - Sectors(0) - } + - self.allocs.iter().map(|(_, len)| *len).sum() + - self.crypt_meta_allocs.iter().map(|(_, len)| *len).sum() } fn alloc( @@ -208,8 +197,10 @@ impl InternalBackstore for Backstore { let mut chunks = Vec::new(); for size in sizes { - chunks.push((self.next, *size)); - self.next += *size; + let next = self.calc_next_cap(); + let seg = (next, *size); + chunks.push(seg); + self.allocs.push(seg); } // Assert that the postcondition holds. @@ -227,6 +218,66 @@ impl InternalBackstore for Backstore { } impl Backstore { + /// Calculate size allocated to data and not metadata in the backstore. + #[cfg(test)] + pub fn data_alloc_size(&self) -> Sectors { + self.allocs.iter().map(|(_, length)| *length).sum() + } + + /// Calculate next from all of the metadata and data allocations present in the backstore. + fn calc_next_cache(&self) -> StratisResult { + let mut all_allocs = if self.allocs.is_empty() { + if matches!(self.enc, Some(Either::Right(_))) { + return Err(StratisError::Msg( + "Metadata can only be allocated at the beginning of the cache device before the encryption device".to_string() + )); + } else { + self.crypt_meta_allocs.clone() + } + } else { + return Err(StratisError::Msg( + "Metadata can only be allocated at the beginning of the cache device before the encryption device".to_string() + )); + }; + all_allocs.sort(); + + for window in all_allocs.windows(2) { + let (start, length) = (window[0].0, window[0].1); + let start_next = window[1].0; + assert_eq!(start + length, start_next); + } + + Ok(all_allocs + .last() + .map(|(offset, len)| *offset + *len) + .unwrap_or(Sectors(0))) + } + + /// Calculate next from all of the metadata and data allocations present in the backstore. + fn calc_next_cap(&self) -> Sectors { + let mut all_allocs = if self.is_encrypted() { + self.allocs.clone() + } else { + self.allocs + .iter() + .cloned() + .chain(self.crypt_meta_allocs.iter().cloned()) + .collect::>() + }; + all_allocs.sort(); + + for window in all_allocs.windows(2) { + let (start, length) = (window[0].0, window[0].1); + let start_next = window[1].0; + assert_eq!(start + length, start_next); + } + + all_allocs + .last() + .map(|(offset, len)| *offset + *len) + .unwrap_or(Sectors(0)) + } + /// Make a Backstore object from blockdevs that already belong to Stratis. /// Precondition: every device in datadevs and cachedevs has already been /// determined to belong to the pool with the specified pool_uuid. @@ -352,7 +403,8 @@ impl Backstore { cache, placeholder, enc, - next: backstore_save.cap.allocs[0].1, + allocs: backstore_save.cap.allocs.clone(), + crypt_meta_allocs: backstore_save.cap.crypt_meta_allocs.clone(), }) } @@ -384,17 +436,52 @@ impl Backstore { cache: None, origin: None, enc: encryption_info.cloned().map(Either::Left), - next: Sectors(0), + allocs: Vec::new(), + crypt_meta_allocs: Vec::new(), }; let size = crypt_metadata_size().sectors(); - backstore.alloc(pool_uuid, &[size])?.ok_or_else(|| { - StratisError::Msg(format!("Failed to satisfy request in backstore for {size}")) - })?; + if !backstore.meta_alloc_cache(&[size])? { + return Err(StratisError::Msg(format!( + "Failed to satisfy request in backstore for {size}" + ))); + } Ok(backstore) } + fn meta_alloc_cache(&mut self, sizes: &[Sectors]) -> StratisResult { + let total_required = sizes.iter().cloned().sum(); + let available = self.available_in_backstore(); + if available < total_required { + return Ok(false); + } + + if !self.data_tier.alloc(sizes) { + return Ok(false); + } + + let mut chunks = Vec::new(); + for size in sizes { + let next = self.calc_next_cache()?; + let seg = (next, *size); + chunks.push(seg); + self.crypt_meta_allocs.push(seg); + } + + // Assert that the postcondition holds. + assert_eq!( + sizes, + chunks + .iter() + .map(|x| x.1) + .collect::>() + .as_slice() + ); + + Ok(true) + } + /// Initialize the cache tier and add cachedevs to the backstore. /// /// Returns all `DevUuid`s of devices that were added to the cache on initialization. @@ -513,7 +600,7 @@ impl Backstore { self.cache.as_mut(), self.placeholder .as_mut() - .and_then(|c| self.origin.as_mut().map(|l| (c, l))), + .and_then(|p| self.origin.as_mut().map(|o| (p, o))), self.enc.as_mut(), ) { (None, None, None) => true, @@ -531,20 +618,20 @@ impl Backstore { cache.resume(get_dm())?; false } - (None, Some((cap, linear)), Some(Either::Right(handle))) => { + (None, Some((placeholder, origin)), Some(Either::Right(handle))) => { let table = self.data_tier.segments.map_to_dm(); - linear.set_table(get_dm(), table)?; - linear.resume(get_dm())?; + origin.set_table(get_dm(), table)?; + origin.resume(get_dm())?; let table = vec![TargetLine::new( Sectors(0), - linear.size(), + origin.size(), LinearDevTargetParams::Linear(LinearTargetParams::new( - linear.device(), + origin.device(), Sectors(0), )), )]; - cap.set_table(get_dm(), table)?; - cap.resume(get_dm())?; + placeholder.set_table(get_dm(), table)?; + placeholder.resume(get_dm())?; handle.resize(None)?; false } @@ -1056,7 +1143,8 @@ impl Recordable for Backstore { BackstoreSave { cache_tier: self.cache_tier.as_ref().map(|c| c.record()), cap: CapSave { - allocs: vec![(Sectors(0), self.next)], + allocs: self.allocs.clone(), + crypt_meta_allocs: self.crypt_meta_allocs.clone(), }, data_tier: self.data_tier.record(), } @@ -1098,18 +1186,20 @@ mod tests { assert_eq!( backstore.data_tier.allocated(), match (&backstore.origin, &backstore.cache) { - (None, None) => - if backstore.is_encrypted() { - crypt_metadata_size().sectors() - } else { - Sectors(0) - }, + (None, None) => crypt_metadata_size().sectors(), (&None, Some(cache)) => cache.size(), (Some(linear), &None) => linear.size(), _ => panic!("impossible; see first assertion"), } ); - assert!(backstore.next <= backstore.size()); + assert!( + backstore + .allocs + .iter() + .map(|(_, len)| *len) + .sum::() + <= backstore.size() + ); backstore.data_tier.invariant(); diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 442d44218f..5546faabe7 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -98,7 +98,7 @@ impl StratEngine { } #[cfg(test)] - async fn create_pool_legacy( + pub(crate) async fn create_pool_legacy( &self, name: &str, blockdev_paths: &[&Path], diff --git a/src/engine/strat_engine/pool/v1.rs b/src/engine/strat_engine/pool/v1.rs index 0fd9aba521..f739eb7ee4 100644 --- a/src/engine/strat_engine/pool/v1.rs +++ b/src/engine/strat_engine/pool/v1.rs @@ -1356,7 +1356,7 @@ mod tests { thinpool::ThinPoolStatusDigest, }, types::{EngineAction, PoolIdentifier}, - Engine, StratEngine, + StratEngine, }; use super::*; @@ -1775,7 +1775,7 @@ mod tests { fn test_grow_physical_pre_grow(paths: &[&Path]) { let pool_name = Name::new("pool".to_string()); let engine = StratEngine::initialize().unwrap(); - let pool_uuid = test_async!(engine.create_pool(&pool_name, paths, None)) + let pool_uuid = test_async!(engine.create_pool_legacy(&pool_name, paths, None)) .unwrap() .changed() .unwrap(); diff --git a/src/engine/strat_engine/pool/v2.rs b/src/engine/strat_engine/pool/v2.rs index 4f1eec0d8a..a77c6ec106 100644 --- a/src/engine/strat_engine/pool/v2.rs +++ b/src/engine/strat_engine/pool/v2.rs @@ -50,29 +50,15 @@ use crate::{ /// Precondition: This method is called only when setting up a pool, which /// ensures that the flex devs metadata lists are all non-empty. fn next_index(flex_devs: &FlexDevsSave) -> Sectors { - let expect_msg = "Setting up rather than initializing a pool, so each flex dev must have been allocated at least some segments."; [ - flex_devs - .meta_dev - .last() - .unwrap_or_else(|| panic!("{}", expect_msg)), - flex_devs - .thin_meta_dev - .last() - .unwrap_or_else(|| panic!("{}", expect_msg)), - flex_devs - .thin_data_dev - .last() - .unwrap_or_else(|| panic!("{}", expect_msg)), - flex_devs - .thin_meta_dev_spare - .last() - .unwrap_or_else(|| panic!("{}", expect_msg)), + &flex_devs.meta_dev, + &flex_devs.thin_meta_dev, + &flex_devs.thin_data_dev, + &flex_devs.thin_meta_dev_spare, ] .iter() - .max_by_key(|x| x.0) - .map(|&&(start, length)| start + length) - .expect("iterator is non-empty") + .flat_map(|vec| vec.iter().map(|(_, length)| *length)) + .sum() } /// Check the metadata of an individual pool for consistency. @@ -81,7 +67,13 @@ fn next_index(flex_devs: &FlexDevsSave) -> Sectors { fn check_metadata(metadata: &PoolSave) -> StratisResult<()> { let flex_devs = &metadata.flex_devs; let next = next_index(flex_devs); - let allocated_from_cap = metadata.backstore.cap.allocs[0].1; + let allocated_from_cap = metadata + .backstore + .cap + .allocs + .iter() + .map(|(_, size)| *size) + .sum::(); if allocated_from_cap != next { let err_msg = format!( @@ -168,7 +160,7 @@ impl StratPool { let thinpool = ThinPool::::new( pool_uuid, - match ThinPoolSizeParams::new(backstore.datatier_usable_size()) { + match ThinPoolSizeParams::new(backstore.available_in_backstore()) { Ok(ref params) => params, Err(causal_error) => { if let Err(cleanup_err) = backstore.destroy(pool_uuid) { diff --git a/src/engine/strat_engine/serde_structs.rs b/src/engine/strat_engine/serde_structs.rs index 66c2c813e9..1820425e65 100644 --- a/src/engine/strat_engine/serde_structs.rs +++ b/src/engine/strat_engine/serde_structs.rs @@ -127,6 +127,9 @@ pub struct BaseBlockDevSave { #[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct CapSave { pub allocs: Vec<(Sectors, Sectors)>, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub crypt_meta_allocs: Vec<(Sectors, Sectors)>, } #[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 3dc35d4d9a..040590700d 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -2939,7 +2939,7 @@ mod tests { .unwrap() ); assert_eq!( - backstore.datatier_allocated_size(), + backstore.data_alloc_size(), pool.thin_pool.data_dev().size() + pool.thin_pool.meta_dev().size() * 2u64 + pool.mdv.device().size() From 4d89fab217118592368a8078b39a64c99b7618f1 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 20 Nov 2023 14:17:41 -0500 Subject: [PATCH 18/32] Add infrastructure for allocating from either end of device This commit adds the ability to allocate both data and metadata from either the front of the back of the device. This will be necessary for reserving integrity space at the end of the device. --- .../strat_engine/backstore/blockdev/v1.rs | 4 +- .../strat_engine/backstore/blockdev/v2.rs | 20 ++++++++- .../strat_engine/backstore/range_alloc.rs | 41 ++++++++++++++++--- src/engine/strat_engine/liminal/setup.rs | 11 ++++- src/engine/strat_engine/serde_structs.rs | 6 +++ 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdev/v1.rs b/src/engine/strat_engine/backstore/blockdev/v1.rs index c7a6be205b..10b717c097 100644 --- a/src/engine/strat_engine/backstore/blockdev/v1.rs +++ b/src/engine/strat_engine/backstore/blockdev/v1.rs @@ -374,7 +374,7 @@ impl InternalBlockDev for StratBlockDev { } fn alloc(&mut self, size: Sectors) -> PerDevSegments { - self.used.alloc(size) + self.used.alloc_front(size) } fn calc_new_size(&self) -> StratisResult> { @@ -595,6 +595,8 @@ impl Recordable for StratBlockDev { uuid: self.uuid(), user_info: self.user_info.clone(), hardware_info: self.hardware_info.clone(), + raid_meta_allocs: Vec::new(), + integrity_meta_allocs: Vec::new(), } } } diff --git a/src/engine/strat_engine/backstore/blockdev/v2.rs b/src/engine/strat_engine/backstore/blockdev/v2.rs index 5b5a3451e3..6640da8f36 100644 --- a/src/engine/strat_engine/backstore/blockdev/v2.rs +++ b/src/engine/strat_engine/backstore/blockdev/v2.rs @@ -51,6 +51,8 @@ pub struct StratBlockDev { devnode: DevicePath, new_size: Option, blksizes: StratSectorSizes, + raid_meta_allocs: Vec<(Sectors, Sectors)>, + integrity_meta_allocs: Vec<(Sectors, Sectors)>, } impl StratBlockDev { @@ -115,6 +117,8 @@ impl StratBlockDev { devnode, new_size: None, blksizes, + raid_meta_allocs: Vec::new(), + integrity_meta_allocs: Vec::new(), }) } @@ -160,6 +164,18 @@ impl StratBlockDev { Ok(blkdev_size(&File::open(physical_path)?)?.sectors()) } + /// Allocate room for metadata from the front of the device. + #[allow(dead_code)] + fn alloc_meta_front(&mut self, size: Sectors) -> PerDevSegments { + self.used.alloc_front(size) + } + + /// Allocate room for metadata from the back of the device. + #[allow(dead_code)] + fn alloc_meta_back(&mut self, size: Sectors) -> PerDevSegments { + self.used.alloc_back(size) + } + /// Set the newly detected size of a block device. pub fn set_new_size(&mut self, new_size: Sectors) { match self.bda.dev_size().cmp(&BlockdevSize::new(new_size)) { @@ -228,7 +244,7 @@ impl InternalBlockDev for StratBlockDev { } fn alloc(&mut self, size: Sectors) -> PerDevSegments { - self.used.alloc(size) + self.used.alloc_front(size) } fn calc_new_size(&self) -> StratisResult> { @@ -376,6 +392,8 @@ impl Recordable for StratBlockDev { uuid: self.uuid(), user_info: self.user_info.clone(), hardware_info: self.hardware_info.clone(), + raid_meta_allocs: self.raid_meta_allocs.clone(), + integrity_meta_allocs: self.integrity_meta_allocs.clone(), } } } diff --git a/src/engine/strat_engine/backstore/range_alloc.rs b/src/engine/strat_engine/backstore/range_alloc.rs index 906b2e814f..1634d8d0f0 100644 --- a/src/engine/strat_engine/backstore/range_alloc.rs +++ b/src/engine/strat_engine/backstore/range_alloc.rs @@ -315,6 +315,12 @@ impl<'a> Iterator for Iter<'a> { } } +impl<'a> DoubleEndedIterator for Iter<'a> { + fn next_back(&mut self) -> Option { + self.items.next_back() + } +} + #[derive(Debug)] pub struct RangeAllocator { segments: PerDevSegments, @@ -348,9 +354,9 @@ impl RangeAllocator { self.segments.sum() } - /// Attempt to allocate. + /// Attempt to allocate from the front of the device. /// Returns a PerDevSegments object containing the allocated ranges. - pub fn alloc(&mut self, amount: Sectors) -> PerDevSegments { + pub fn alloc_front(&mut self, amount: Sectors) -> PerDevSegments { let mut segs = PerDevSegments::new(self.segments.limit()); let mut needed = amount; @@ -372,6 +378,31 @@ impl RangeAllocator { segs } + /// Attempt to allocate from the back of the device. + /// Returns a PerDevSegments object containing the allocated ranges. + #[allow(dead_code)] + pub fn alloc_back(&mut self, amount: Sectors) -> PerDevSegments { + let mut segs = PerDevSegments::new(self.segments.limit()); + let mut needed = amount; + + for (&start, &len) in self.segments.complement().iter().rev() { + if needed == Sectors(0) { + break; + } + let to_use = min(needed, len); + let used_range = (start + len - to_use, to_use); + segs.insert(&used_range) + .expect("wholly disjoint from other elements in segs"); + needed -= to_use; + } + self.segments = self + .segments + .union(&segs) + .expect("all segments verified to be in available ranges"); + + segs + } + /// Increase the available size of the RangeAlloc data structure. /// /// Precondition: new_size > self.limit @@ -410,14 +441,14 @@ mod tests { assert_eq!(allocator.used(), Sectors(100)); assert_eq!(allocator.available(), Sectors(28)); - let seg = allocator.alloc(Sectors(50)); + let seg = allocator.alloc_front(Sectors(50)); assert_eq!(seg.len(), 2); assert_eq!(seg.sum(), Sectors(28)); assert_eq!(allocator.used(), Sectors(128)); assert_eq!(allocator.available(), Sectors(0)); let available = allocator.available(); - allocator.alloc(available); + allocator.alloc_front(available); assert_eq!(allocator.available(), Sectors(0)); } @@ -488,7 +519,7 @@ mod tests { fn test_allocator_failures_range_overwrite() { let mut allocator = RangeAllocator::new(BlockdevSize::new(Sectors(128)), &[]).unwrap(); - let seg = allocator.alloc(Sectors(128)); + let seg = allocator.alloc_front(Sectors(128)); assert_eq!(allocator.used(), Sectors(128)); assert_eq!( seg.iter().collect::>(), diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index 8ac4f05334..e722e13dab 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -495,7 +495,14 @@ fn get_blockdev( // least the recorded size, so all segments should be // available to be allocated. If this fails, the most likely // conclusion is metadata corruption. - let segments = segment_table.get(&dev_uuid); + let segs = segment_table.get(&dev_uuid); + let mut segments = vec![]; + segments.extend(segs.unwrap_or(&vec![])); + let meta = data_map.get(&dev_uuid); + let raid = meta.map(|base| &base.1.raid_meta_allocs); + let integrity = meta.map(|base| &base.1.integrity_meta_allocs); + segments.extend(raid.unwrap_or(&vec![])); + segments.extend(integrity.unwrap_or(&vec![])); assert_eq!(info.luks, None); Ok(( @@ -503,7 +510,7 @@ fn get_blockdev( v2::StratBlockDev::new( info.dev_info.device_number, bda, - segments.unwrap_or(&vec![]), + &segments, bd_save.user_info.clone(), bd_save.hardware_info.clone(), devnode, diff --git a/src/engine/strat_engine/serde_structs.rs b/src/engine/strat_engine/serde_structs.rs index 1820425e65..340709d9e6 100644 --- a/src/engine/strat_engine/serde_structs.rs +++ b/src/engine/strat_engine/serde_structs.rs @@ -116,6 +116,12 @@ pub struct BaseDevSave { #[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct BaseBlockDevSave { pub uuid: DevUuid, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub raid_meta_allocs: Vec<(Sectors, Sectors)>, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub integrity_meta_allocs: Vec<(Sectors, Sectors)>, #[serde(skip_serializing_if = "Option::is_none")] #[serde(serialize_with = "serialize_option_string")] pub user_info: Option, From a07a6f7e3a3aa59ed3c93d3ae2e678cc94e7497e Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 24 Jan 2024 00:13:21 -0500 Subject: [PATCH 19/32] Allocate space for dm-integrity and md-raid --- .../strat_engine/backstore/blockdev/mod.rs | 11 +-- .../strat_engine/backstore/blockdev/v1.rs | 16 ++--- .../strat_engine/backstore/blockdev/v2.rs | 72 ++++++++++++++----- .../strat_engine/backstore/blockdevmgr.rs | 7 +- .../strat_engine/backstore/data_tier.rs | 42 ++++++++++- src/engine/strat_engine/backstore/devices.rs | 2 +- src/engine/strat_engine/liminal/setup.rs | 10 ++- src/engine/strat_engine/metadata/mod.rs | 2 +- 8 files changed, 116 insertions(+), 46 deletions(-) diff --git a/src/engine/strat_engine/backstore/blockdev/mod.rs b/src/engine/strat_engine/backstore/blockdev/mod.rs index 6d9c736acc..332e97b628 100644 --- a/src/engine/strat_engine/backstore/blockdev/mod.rs +++ b/src/engine/strat_engine/backstore/blockdev/mod.rs @@ -12,7 +12,7 @@ use crate::{ engine::{ strat_engine::{ backstore::{devices::BlockSizes, range_alloc::PerDevSegments}, - metadata::{BDAExtendedSize, BlockdevSize, MDADataSize, BDA}, + metadata::{BlockdevSize, MDADataSize, BDA}, }, types::{DevUuid, StratSigblockVersion}, }, @@ -71,12 +71,13 @@ pub trait InternalBlockDev { fn available(&self) -> Sectors; // ALL SIZE METHODS (except size(), which is in BlockDev impl.) - /// The number of Sectors on this device used by Stratis for metadata - fn metadata_size(&self) -> BDAExtendedSize; + /// The number of Sectors on this device used by Stratis and potentially other devicemapper + /// layers for metadata + fn metadata_size(&self) -> Sectors; - /// The maximum size of variable length metadata that can be accommodated. + /// The maximum size of variable length Stratis metadata that can be accommodated. /// self.max_metadata_size() < self.metadata_size() - fn max_metadata_size(&self) -> MDADataSize; + fn max_stratis_metadata_size(&self) -> MDADataSize; /// Whether or not the blockdev is in use by upper layers. It is if the /// sum of the blocks used exceeds the Stratis metadata size. diff --git a/src/engine/strat_engine/backstore/blockdev/v1.rs b/src/engine/strat_engine/backstore/blockdev/v1.rs index 10b717c097..f022eaba36 100644 --- a/src/engine/strat_engine/backstore/blockdev/v1.rs +++ b/src/engine/strat_engine/backstore/blockdev/v1.rs @@ -29,8 +29,8 @@ use crate::{ crypt::handle::v1::CryptHandle, device::blkdev_size, metadata::{ - disown_device, static_header, BDAExtendedSize, BlockdevSize, MDADataSize, - MetadataLocation, StaticHeader, BDA, + disown_device, static_header, BlockdevSize, MDADataSize, MetadataLocation, + StaticHeader, BDA, }, serde_structs::{BaseBlockDevSave, Recordable}, types::BDAResult, @@ -198,8 +198,8 @@ impl StratBlockDev { } /// The maximum size of variable length metadata that can be accommodated. - /// self.max_metadata_size() < self.metadata_size() - pub fn max_metadata_size(&self) -> MDADataSize { + /// self.max_stratis_metadata_size() > self.metadata_size() + pub fn max_stratis_metadata_size(&self) -> MDADataSize { self.bda.max_data_size() } @@ -361,16 +361,16 @@ impl InternalBlockDev for StratBlockDev { self.used.available() } - fn metadata_size(&self) -> BDAExtendedSize { - self.bda.extended_size() + fn metadata_size(&self) -> Sectors { + self.bda.extended_size().sectors() } - fn max_metadata_size(&self) -> MDADataSize { + fn max_stratis_metadata_size(&self) -> MDADataSize { self.bda.max_data_size() } fn in_use(&self) -> bool { - self.used.used() > self.metadata_size().sectors() + self.used.used() > self.metadata_size() } fn alloc(&mut self, size: Sectors) -> PerDevSegments { diff --git a/src/engine/strat_engine/backstore/blockdev/v2.rs b/src/engine/strat_engine/backstore/blockdev/v2.rs index 6640da8f36..efa9d82ea7 100644 --- a/src/engine/strat_engine/backstore/blockdev/v2.rs +++ b/src/engine/strat_engine/backstore/blockdev/v2.rs @@ -14,7 +14,7 @@ use std::{ use chrono::{DateTime, Utc}; use serde_json::Value; -use devicemapper::{Device, Sectors}; +use devicemapper::{Bytes, Device, Sectors, IEC}; use crate::{ engine::{ @@ -27,8 +27,8 @@ use crate::{ }, device::blkdev_size, metadata::{ - disown_device, static_header, BDAExtendedSize, BlockdevSize, MDADataSize, - MetadataLocation, StaticHeader, BDA, + disown_device, static_header, BlockdevSize, MDADataSize, MetadataLocation, + StaticHeader, BDA, }, serde_structs::{BaseBlockDevSave, Recordable}, types::BDAResult, @@ -41,6 +41,25 @@ use crate::{ stratis::{StratisError, StratisResult}, }; +/// Return the amount of space required for integrity for a device of the given size. +/// +/// This is a slight overestimation for the sake of simplicity. Because it uses the whole disk +/// size, once the integrity metadata size is calculated, the remaining data size is now smaller +/// than the metadata region could support for integrity. +pub fn integrity_meta_space(total_space: Sectors) -> Sectors { + Bytes(4096).sectors() + + Bytes::from(64 * IEC::Mi).sectors() + + Bytes::from((*total_space * 32u64 + 4095) & !4096).sectors() +} + +/// Return the amount of space required for RAID for a device of the given size. +/// +/// This is a slight overestimation for the sake of simplicity. The maximum metadata size is used +/// to leave adequate room for any sized device's metadata. +pub fn raid_meta_space() -> Sectors { + Bytes::from(129 * IEC::Mi).sectors() +} + #[derive(Debug)] pub struct StratBlockDev { dev: Device, @@ -77,16 +96,21 @@ impl StratBlockDev { /// /// Precondition: segments in other_segments do not overlap with Stratis /// metadata region. + #[allow(clippy::too_many_arguments)] pub fn new( dev: Device, bda: BDA, other_segments: &[(Sectors, Sectors)], + raid_meta_allocs: &[(Sectors, Sectors)], + integrity_meta_allocs: &[(Sectors, Sectors)], user_info: Option, hardware_info: Option, devnode: DevicePath, ) -> BDAResult { let mut segments = vec![(Sectors(0), bda.extended_size().sectors())]; segments.extend(other_segments); + segments.extend(raid_meta_allocs); + segments.extend(integrity_meta_allocs); let allocator = match RangeAllocator::new(bda.dev_size(), &segments) { Ok(a) => a, @@ -117,8 +141,8 @@ impl StratBlockDev { devnode, new_size: None, blksizes, - raid_meta_allocs: Vec::new(), - integrity_meta_allocs: Vec::new(), + raid_meta_allocs: raid_meta_allocs.to_owned(), + integrity_meta_allocs: integrity_meta_allocs.to_owned(), }) } @@ -164,16 +188,20 @@ impl StratBlockDev { Ok(blkdev_size(&File::open(physical_path)?)?.sectors()) } - /// Allocate room for metadata from the front of the device. - #[allow(dead_code)] - fn alloc_meta_front(&mut self, size: Sectors) -> PerDevSegments { - self.used.alloc_front(size) + /// Allocate room for RAID metadata from the back of the device. + pub fn alloc_raid_meta(&mut self, size: Sectors) { + let segs = self.used.alloc_front(size); + for (start, len) in segs.iter() { + self.raid_meta_allocs.push((*start, *len)); + } } - /// Allocate room for metadata from the back of the device. - #[allow(dead_code)] - fn alloc_meta_back(&mut self, size: Sectors) -> PerDevSegments { - self.used.alloc_back(size) + /// Allocate room for integrity metadata from the back of the device. + pub fn alloc_int_meta_back(&mut self, size: Sectors) { + let segs = self.used.alloc_back(size); + for (start, len) in segs.iter() { + self.integrity_meta_allocs.push((*start, *len)); + } } /// Set the newly detected size of a block device. @@ -231,16 +259,18 @@ impl InternalBlockDev for StratBlockDev { self.used.available() } - fn metadata_size(&self) -> BDAExtendedSize { - self.bda.extended_size() + fn metadata_size(&self) -> Sectors { + self.bda.extended_size().sectors() + + self.integrity_meta_allocs.iter().map(|(_, len)| *len).sum() + + self.raid_meta_allocs.iter().map(|(_, len)| *len).sum() } - fn max_metadata_size(&self) -> MDADataSize { + fn max_stratis_metadata_size(&self) -> MDADataSize { self.bda.max_data_size() } fn in_use(&self) -> bool { - self.used.used() > self.metadata_size().sectors() + self.used.used() > self.metadata_size() } fn alloc(&mut self, size: Sectors) -> PerDevSegments { @@ -285,6 +315,14 @@ impl InternalBlockDev for StratBlockDev { self.bda.header = h; self.used.increase_size(size.sectors()); + let integrity_grow = integrity_meta_space(size.sectors()) + - self + .integrity_meta_allocs + .iter() + .map(|(_, len)| *len) + .sum::(); + self.alloc_int_meta_back(integrity_grow); + Ok(true) } } diff --git a/src/engine/strat_engine/backstore/blockdevmgr.rs b/src/engine/strat_engine/backstore/blockdevmgr.rs index f36ee5846f..04525081c3 100644 --- a/src/engine/strat_engine/backstore/blockdevmgr.rs +++ b/src/engine/strat_engine/backstore/blockdevmgr.rs @@ -395,7 +395,7 @@ where let candidates = self .block_devs .iter_mut() - .filter(|b| b.max_metadata_size().bytes() >= data_size); + .filter(|b| b.max_stratis_metadata_size().bytes() >= data_size); debug!( "Writing {} of pool level metadata to devices in pool", @@ -464,10 +464,7 @@ where /// The number of sectors given over to Stratis metadata /// self.allocated_size() - self.metadata_size() >= self.avail_space() pub fn metadata_size(&self) -> Sectors { - self.block_devs - .iter() - .map(|bd| bd.metadata_size().sectors()) - .sum() + self.block_devs.iter().map(|bd| bd.metadata_size()).sum() } pub fn grow(&mut self, dev: DevUuid) -> StratisResult { diff --git a/src/engine/strat_engine/backstore/data_tier.rs b/src/engine/strat_engine/backstore/data_tier.rs index c8f6f7c281..840a9c707e 100644 --- a/src/engine/strat_engine/backstore/data_tier.rs +++ b/src/engine/strat_engine/backstore/data_tier.rs @@ -13,7 +13,11 @@ use crate::{ engine::{ strat_engine::{ backstore::{ - blockdev::{v1, v2, InternalBlockDev}, + blockdev::{ + v1, + v2::{self, integrity_meta_space, raid_meta_space}, + InternalBlockDev, + }, blockdevmgr::BlockDevMgr, devices::UnownedDevices, shared::{metadata_to_segment, AllocatedAbove, BlkDevSegment, BlockDevPartition}, @@ -101,7 +105,16 @@ impl DataTier { /// Initially 0 segments are allocated. /// /// WARNING: metadata changing event - pub fn new(block_mgr: BlockDevMgr) -> DataTier { + pub fn new(mut block_mgr: BlockDevMgr) -> DataTier { + for (_, bd) in block_mgr.blockdevs_mut() { + bd.alloc_raid_meta(raid_meta_space()); + bd.alloc_int_meta_back(integrity_meta_space( + // NOTE: Subtracting metadata size works here because the only metadata currently + // recorded in a newly created block device is the BDA. If this becomes untrue in + // the future, this code will no longer work. + bd.total_size().sectors() - bd.metadata_size(), + )); + } DataTier { block_mgr, segments: AllocatedAbove { inner: vec![] }, @@ -116,7 +129,30 @@ impl DataTier { pool_uuid: PoolUuid, devices: UnownedDevices, ) -> StratisResult> { - self.block_mgr.add(pool_uuid, devices) + let uuids = self.block_mgr.add(pool_uuid, devices)?; + let bds = self + .block_mgr + .blockdevs_mut() + .into_iter() + .filter_map(|(uuid, bd)| { + if uuids.contains(&uuid) { + Some(bd) + } else { + None + } + }) + .collect::>(); + assert_eq!(bds.len(), uuids.len()); + for bd in bds { + bd.alloc_raid_meta(raid_meta_space()); + bd.alloc_int_meta_back(integrity_meta_space( + // NOTE: Subtracting metadata size works here because the only metadata currently + // recorded in a newly created block device is the BDA. If this becomes untrue in + // the future, this code will no longer work. + bd.total_size().sectors() - bd.metadata_size(), + )); + } + Ok(uuids) } /// Lookup an immutable blockdev by its Stratis UUID. diff --git a/src/engine/strat_engine/backstore/devices.rs b/src/engine/strat_engine/backstore/devices.rs index b7ed570626..5d9aad2b49 100644 --- a/src/engine/strat_engine/backstore/devices.rs +++ b/src/engine/strat_engine/backstore/devices.rs @@ -837,7 +837,7 @@ pub fn initialize_devices( bda.initialize(&mut f)?; - v2::StratBlockDev::new(devno, bda, &[], None, hw_id, devnode).map_err(|(e, _)| e) + v2::StratBlockDev::new(devno, bda, &[], &[], &[], None, hw_id, devnode).map_err(|(e, _)| e) } /// Clean up an unencrypted device after initialization failure. diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index e722e13dab..bda98929f8 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -495,14 +495,10 @@ fn get_blockdev( // least the recorded size, so all segments should be // available to be allocated. If this fails, the most likely // conclusion is metadata corruption. - let segs = segment_table.get(&dev_uuid); - let mut segments = vec![]; - segments.extend(segs.unwrap_or(&vec![])); + let segments = segment_table.get(&dev_uuid); let meta = data_map.get(&dev_uuid); let raid = meta.map(|base| &base.1.raid_meta_allocs); let integrity = meta.map(|base| &base.1.integrity_meta_allocs); - segments.extend(raid.unwrap_or(&vec![])); - segments.extend(integrity.unwrap_or(&vec![])); assert_eq!(info.luks, None); Ok(( @@ -510,7 +506,9 @@ fn get_blockdev( v2::StratBlockDev::new( info.dev_info.device_number, bda, - &segments, + segments.unwrap_or(&vec![]), + raid.unwrap_or(&vec![]), + integrity.unwrap_or(&vec![]), bd_save.user_info.clone(), bd_save.hardware_info.clone(), devnode, diff --git a/src/engine/strat_engine/metadata/mod.rs b/src/engine/strat_engine/metadata/mod.rs index 5088c2832f..cfc1dc1411 100644 --- a/src/engine/strat_engine/metadata/mod.rs +++ b/src/engine/strat_engine/metadata/mod.rs @@ -17,7 +17,7 @@ mod static_header; pub use self::{ bda::BDA, - sizes::{BDAExtendedSize, BlockdevSize, MDADataSize}, + sizes::{BlockdevSize, MDADataSize}, static_header::{ device_identifiers, disown_device, static_header, MetadataLocation, StaticHeader, StaticHeaderResult, StratisIdentifiers, From 46bce40c364d5f5bcf18f788010d6346519980ac Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 18 Mar 2024 11:48:31 -0400 Subject: [PATCH 20/32] Add lower bound for filesystem snapshot tests --- src/engine/strat_engine/thinpool/thinpool.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/engine/strat_engine/thinpool/thinpool.rs b/src/engine/strat_engine/thinpool/thinpool.rs index 040590700d..1dcfb5e389 100644 --- a/src/engine/strat_engine/thinpool/thinpool.rs +++ b/src/engine/strat_engine/thinpool/thinpool.rs @@ -3113,6 +3113,7 @@ mod tests { None, ) .unwrap(); + warn!("Available: {}", backstore.available_in_backstore()); let mut pool = ThinPool::::new( pool_uuid, &ThinPoolSizeParams::new(backstore.available_in_backstore()).unwrap(), @@ -3207,7 +3208,7 @@ mod tests { #[test] fn loop_test_filesystem_snapshot() { loopbacked::test_with_spec( - &loopbacked::DeviceLimits::Range(2, 3, None), + &loopbacked::DeviceLimits::Range(2, 3, Some(Bytes::from(5 * IEC::Gi).sectors())), test_filesystem_snapshot, ); } @@ -3215,7 +3216,7 @@ mod tests { #[test] fn real_test_filesystem_snapshot() { real::test_with_spec( - &real::DeviceLimits::AtLeast(2, None, None), + &real::DeviceLimits::AtLeast(2, Some(Bytes::from(5 * IEC::Gi).sectors()), None), test_filesystem_snapshot, ); } From c40d40be1c900561f2cfd4f72b82d94ea610d050 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 22 May 2024 15:47:20 -0400 Subject: [PATCH 21/32] Add record of enabled features in metdata V2 --- src/engine/strat_engine/pool/v1.rs | 1 + src/engine/strat_engine/pool/v2.rs | 7 ++++++- src/engine/strat_engine/serde_structs.rs | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/engine/strat_engine/pool/v1.rs b/src/engine/strat_engine/pool/v1.rs index f739eb7ee4..932dbe3849 100644 --- a/src/engine/strat_engine/pool/v1.rs +++ b/src/engine/strat_engine/pool/v1.rs @@ -389,6 +389,7 @@ impl StratPool { flex_devs: self.thin_pool.record(), thinpool_dev: self.thin_pool.record(), started: Some(true), + features: vec![], } } diff --git a/src/engine/strat_engine/pool/v2.rs b/src/engine/strat_engine/pool/v2.rs index a77c6ec106..f1604c6f34 100644 --- a/src/engine/strat_engine/pool/v2.rs +++ b/src/engine/strat_engine/pool/v2.rs @@ -25,7 +25,7 @@ use crate::{ }, liminal::DeviceSet, metadata::{MDADataSize, BDA}, - serde_structs::{FlexDevsSave, PoolSave, Recordable}, + serde_structs::{FlexDevsSave, PoolFeatures, PoolSave, Recordable}, shared::tiers_to_bdas, thinpool::{StratFilesystem, ThinPool, ThinPoolSizeParams, DATA_BLOCK_SIZE}, types::BDARecordResult, @@ -343,12 +343,17 @@ impl StratPool { } pub fn record(&self, name: &str) -> PoolSave { + let mut features = vec![]; + if self.is_encrypted() { + features.push(PoolFeatures::Encryption); + } PoolSave { name: name.to_owned(), backstore: self.backstore.record(), flex_devs: self.thin_pool.record(), thinpool_dev: self.thin_pool.record(), started: Some(true), + features, } } diff --git a/src/engine/strat_engine/serde_structs.rs b/src/engine/strat_engine/serde_structs.rs index 340709d9e6..e9b395f5a3 100644 --- a/src/engine/strat_engine/serde_structs.rs +++ b/src/engine/strat_engine/serde_structs.rs @@ -71,6 +71,14 @@ pub trait Recordable { fn record(&self) -> T; } +/// List of optional features for pools. +#[derive(Debug, Deserialize, Eq, PartialEq, Serialize, Hash, Copy, Clone)] +pub enum PoolFeatures { + Raid, + Integrity, + Encryption, +} + // ALL structs that represent variable length metadata in pre-order // depth-first traversal order. Note that when organized by types rather than // values the structure is a DAG not a tree. This just means that there are @@ -85,6 +93,9 @@ pub struct PoolSave { // TODO: This data type should no longer be optional in Stratis 4.0 #[serde(skip_serializing_if = "Option::is_none")] pub started: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub features: Vec, } #[derive(Debug, Deserialize, Eq, PartialEq, Serialize)] From 42d73524a330fd0457c58cdcbc8108affb2bcbcd Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Wed, 29 May 2024 10:44:07 -0400 Subject: [PATCH 22/32] Add ability to specify passphrase over D-Bus API for locked pools --- src/bin/stratis-min/stratis-min.rs | 8 +- src/dbus_api/api/manager_3_2/methods.rs | 10 +- src/dbus_api/api/manager_3_4/methods.rs | 8 +- src/dbus_api/api/manager_3_7/api.rs | 25 + src/dbus_api/api/manager_3_7/methods.rs | 128 ++++ src/dbus_api/api/manager_3_7/mod.rs | 8 + src/dbus_api/api/mod.rs | 3 +- src/engine/engine.rs | 1 + src/engine/shared.rs | 4 +- src/engine/sim_engine/engine.rs | 19 + src/engine/sim_engine/keys.rs | 4 +- .../strat_engine/backstore/backstore/v2.rs | 65 +- src/engine/strat_engine/crypt/handle/v1.rs | 52 +- src/engine/strat_engine/crypt/handle/v2.rs | 51 +- src/engine/strat_engine/crypt/shared.rs | 180 +++-- src/engine/strat_engine/engine.rs | 15 +- src/engine/strat_engine/keys.rs | 4 +- src/engine/strat_engine/liminal/liminal.rs | 677 ++++++++++++------ src/engine/strat_engine/liminal/setup.rs | 2 +- src/engine/strat_engine/pool/v2.rs | 17 +- src/engine/types/mod.rs | 2 +- src/jsonrpc/server/key.rs | 34 +- src/jsonrpc/server/pool.rs | 17 +- 23 files changed, 887 insertions(+), 447 deletions(-) create mode 100644 src/dbus_api/api/manager_3_7/api.rs create mode 100644 src/dbus_api/api/manager_3_7/methods.rs create mode 100644 src/dbus_api/api/manager_3_7/mod.rs diff --git a/src/bin/stratis-min/stratis-min.rs b/src/bin/stratis-min/stratis-min.rs index 500b7d1646..afaf60d0ba 100644 --- a/src/bin/stratis-min/stratis-min.rs +++ b/src/bin/stratis-min/stratis-min.rs @@ -13,7 +13,7 @@ use stratisd::{ CLEVIS_TANG_TRUST_URL, }, jsonrpc::client::{filesystem, key, pool, report}, - stratis::{StratisError, VERSION}, + stratis::VERSION, }; fn parse_args() -> Command { @@ -244,12 +244,6 @@ fn main() -> Result<(), String> { None => None, }; let prompt = args.get_flag("prompt"); - if prompt && unlock_method == Some(UnlockMethod::Clevis) { - return Err(Box::new(StratisError::Msg( - "--prompt and an unlock_method of clevis are mutually exclusive" - .to_string(), - ))); - } pool::pool_start(id, unlock_method, prompt)?; Ok(()) } else if let Some(args) = subcommand.subcommand_matches("stop") { diff --git a/src/dbus_api/api/manager_3_2/methods.rs b/src/dbus_api/api/manager_3_2/methods.rs index ae90f345e5..0e89fd79a4 100644 --- a/src/dbus_api/api/manager_3_2/methods.rs +++ b/src/dbus_api/api/manager_3_2/methods.rs @@ -57,11 +57,11 @@ pub fn start_pool(m: &MethodInfo<'_, MTSync, TData>) -> MethodResult { } }; - let ret = match handle_action!(block_on( - dbus_context - .engine - .start_pool(PoolIdentifier::Uuid(pool_uuid), unlock_method) - )) { + let ret = match handle_action!(block_on(dbus_context.engine.start_pool( + PoolIdentifier::Uuid(pool_uuid), + unlock_method, + None + ))) { Ok(StartAction::Started(_)) => { let guard = match block_on( dbus_context diff --git a/src/dbus_api/api/manager_3_4/methods.rs b/src/dbus_api/api/manager_3_4/methods.rs index 949219d34b..27a6f551c9 100644 --- a/src/dbus_api/api/manager_3_4/methods.rs +++ b/src/dbus_api/api/manager_3_4/methods.rs @@ -63,9 +63,11 @@ pub fn start_pool(m: &MethodInfo<'_, MTSync, TData>) -> MethodResult { } }; - let ret = match handle_action!(block_on( - dbus_context.engine.start_pool(id.clone(), unlock_method) - )) { + let ret = match handle_action!(block_on(dbus_context.engine.start_pool( + id.clone(), + unlock_method, + None + ))) { Ok(StartAction::Started(_)) => { let guard = match block_on(dbus_context.engine.get_pool(id.clone())) { Some(g) => g, diff --git a/src/dbus_api/api/manager_3_7/api.rs b/src/dbus_api/api/manager_3_7/api.rs new file mode 100644 index 0000000000..3427ce854c --- /dev/null +++ b/src/dbus_api/api/manager_3_7/api.rs @@ -0,0 +1,25 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus_tree::{Factory, MTSync, Method}; + +use crate::dbus_api::{api::manager_3_7::methods::start_pool, types::TData}; + +pub fn start_pool_method(f: &Factory, TData>) -> Method, TData> { + f.method("StartPool", (), start_pool) + .in_arg(("id", "s")) + .in_arg(("id_type", "s")) + .in_arg(("unlock_method", "(bs)")) + .in_arg(("key_fd", "(bh)")) + // In order from left to right: + // b: true if the pool was newly started + // o: pool path + // oa: block device paths + // oa: filesystem paths + // + // Rust representation: bool + .out_arg(("result", "(b(oaoao))")) + .out_arg(("return_code", "q")) + .out_arg(("return_string", "s")) +} diff --git a/src/dbus_api/api/manager_3_7/methods.rs b/src/dbus_api/api/manager_3_7/methods.rs new file mode 100644 index 0000000000..de1367e63b --- /dev/null +++ b/src/dbus_api/api/manager_3_7/methods.rs @@ -0,0 +1,128 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::{arg::OwnedFd, Message, Path}; +use dbus_tree::{MTSync, MethodInfo, MethodResult}; +use futures::executor::block_on; + +use crate::{ + dbus_api::{ + blockdev::create_dbus_blockdev, + filesystem::create_dbus_filesystem, + pool::create_dbus_pool, + types::{DbusErrorEnum, TData, OK_STRING}, + util::{engine_to_dbus_err_tuple, get_next_arg, tuple_to_option}, + }, + engine::{Name, PoolIdentifier, PoolUuid, StartAction, UnlockMethod}, + stratis::StratisError, +}; + +pub fn start_pool(m: &MethodInfo<'_, MTSync, TData>) -> MethodResult { + let base_path = m.path.get_name(); + let message: &Message = m.msg; + let mut iter = message.iter_init(); + let dbus_context = m.tree.get_data(); + let default_return: ( + bool, + (Path<'static>, Vec>, Vec>), + ) = (false, (Path::default(), Vec::new(), Vec::new())); + let return_message = message.method_return(); + + let id_str: &str = get_next_arg(&mut iter, 0)?; + let id = { + let id_type_str: &str = get_next_arg(&mut iter, 1)?; + match id_type_str { + "uuid" => match PoolUuid::parse_str(id_str) { + Ok(u) => PoolIdentifier::Uuid(u), + Err(e) => { + let (rc, rs) = engine_to_dbus_err_tuple(&e); + return Ok(vec![return_message.append3(default_return, rc, rs)]); + } + }, + "name" => PoolIdentifier::Name(Name::new(id_str.to_string())), + _ => { + let (rc, rs) = engine_to_dbus_err_tuple(&StratisError::Msg(format!( + "ID type {id_type_str} not recognized" + ))); + return Ok(vec![return_message.append3(default_return, rc, rs)]); + } + } + }; + let unlock_method = { + let unlock_method_tup: (bool, &str) = get_next_arg(&mut iter, 2)?; + match tuple_to_option(unlock_method_tup) { + Some(unlock_method_str) => match UnlockMethod::try_from(unlock_method_str) { + Ok(um) => Some(um), + Err(e) => { + let (rc, rs) = engine_to_dbus_err_tuple(&e); + return Ok(vec![return_message.append3(default_return, rc, rs)]); + } + }, + None => None, + } + }; + let fd_opt: (bool, OwnedFd) = get_next_arg(&mut iter, 3)?; + let fd = tuple_to_option(fd_opt); + + let ret = match handle_action!(block_on(dbus_context.engine.start_pool( + id.clone(), + unlock_method, + fd.map(|f| f.into_fd()), + ))) { + Ok(StartAction::Started(_)) => { + let guard = match block_on(dbus_context.engine.get_pool(id.clone())) { + Some(g) => g, + None => { + let (rc, rs) = engine_to_dbus_err_tuple(&StratisError::Msg( + format!("Pool with {id:?} was successfully started but appears to have been removed before it could be exposed on the D-Bus") + )); + return Ok(vec![return_message.append3(default_return, rc, rs)]); + } + }; + + let (pool_name, pool_uuid, pool) = guard.as_tuple(); + let pool_path = + create_dbus_pool(dbus_context, base_path.clone(), &pool_name, pool_uuid, pool); + let mut bd_paths = Vec::new(); + for (bd_uuid, tier, bd) in pool.blockdevs() { + bd_paths.push(create_dbus_blockdev( + dbus_context, + pool_path.clone(), + bd_uuid, + tier, + bd, + )); + } + let mut fs_paths = Vec::new(); + for (name, fs_uuid, fs) in pool.filesystems() { + fs_paths.push(create_dbus_filesystem( + dbus_context, + pool_path.clone(), + &pool_name, + &name, + fs_uuid, + fs, + )); + } + + if pool.is_encrypted() { + dbus_context.push_locked_pools(block_on(dbus_context.engine.locked_pools())); + } + dbus_context.push_stopped_pools(block_on(dbus_context.engine.stopped_pools())); + + (true, (pool_path, bd_paths, fs_paths)) + } + Ok(StartAction::Identity) => default_return, + Err(e) => { + let (rc, rs) = engine_to_dbus_err_tuple(&e); + return Ok(vec![return_message.append3(default_return, rc, rs)]); + } + }; + + Ok(vec![return_message.append3( + ret, + DbusErrorEnum::OK as u16, + OK_STRING.to_string(), + )]) +} diff --git a/src/dbus_api/api/manager_3_7/mod.rs b/src/dbus_api/api/manager_3_7/mod.rs new file mode 100644 index 0000000000..f6871e56b4 --- /dev/null +++ b/src/dbus_api/api/manager_3_7/mod.rs @@ -0,0 +1,8 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +mod api; +mod methods; + +pub use api::start_pool_method; diff --git a/src/dbus_api/api/mod.rs b/src/dbus_api/api/mod.rs index 94bce8abf2..f299e006a1 100644 --- a/src/dbus_api/api/mod.rs +++ b/src/dbus_api/api/mod.rs @@ -14,6 +14,7 @@ mod manager_3_2; mod manager_3_4; mod manager_3_5; mod manager_3_6; +mod manager_3_7; pub mod prop_conv; mod report_3_0; mod shared; @@ -143,7 +144,7 @@ pub fn get_base_tree<'a>( .add_m(manager_3_0::list_keys_method(&f)) .add_m(manager_3_0::destroy_pool_method(&f)) .add_m(manager_3_0::engine_state_report_method(&f)) - .add_m(manager_3_4::start_pool_method(&f)) + .add_m(manager_3_7::start_pool_method(&f)) .add_m(manager_3_6::stop_pool_method(&f)) .add_m(manager_3_2::refresh_state_method(&f)) .add_p(manager_3_0::version_property(&f)) diff --git a/src/engine/engine.rs b/src/engine/engine.rs index ddcc1b405b..f9eefe6628 100644 --- a/src/engine/engine.rs +++ b/src/engine/engine.rs @@ -451,6 +451,7 @@ pub trait Engine: Debug + Report + Send + Sync { &self, pool_id: PoolIdentifier, unlock_method: Option, + passphrase_fd: Option, ) -> StratisResult>; /// Stop and tear down a pool, storing the information for it to be started diff --git a/src/engine/shared.rs b/src/engine/shared.rs index e1a8a0c88f..f32d427f25 100644 --- a/src/engine/shared.rs +++ b/src/engine/shared.rs @@ -121,9 +121,9 @@ where } } -/// Shared implementation of setting keys in the keyring for both the strat_engine +/// Shared implementation of reading keys from file descriptors in both the strat_engine /// and sim_engine. -pub fn set_key_shared(key_fd: RawFd, memory: &mut [u8]) -> StratisResult { +pub fn read_key_shared(key_fd: RawFd, memory: &mut [u8]) -> StratisResult { let mut key_file = unsafe { File::from_raw_fd(key_fd) }; let bytes_read = key_file.read(memory)?; diff --git a/src/engine/sim_engine/engine.rs b/src/engine/sim_engine/engine.rs index 68e544aa58..7c61fc685d 100644 --- a/src/engine/sim_engine/engine.rs +++ b/src/engine/sim_engine/engine.rs @@ -4,6 +4,7 @@ use std::{ collections::{hash_map::RandomState, HashMap, HashSet}, + os::fd::RawFd, path::Path, sync::Arc, }; @@ -291,6 +292,7 @@ impl Engine for SimEngine { &self, id: PoolIdentifier, unlock_method: Option, + passphrase_fd: Option, ) -> StratisResult> { if let Some(guard) = self.pools.read(id.clone()).await { let (_, pool_uuid, pool) = guard.as_tuple(); @@ -302,6 +304,10 @@ impl Engine for SimEngine { return Err(StratisError::Msg(format!( "Pool with UUID {pool_uuid} is not encrypted but an unlock method was provided" ))); + } else if !pool.is_encrypted() && passphrase_fd.is_some() { + return Err(StratisError::Msg(format!( + "Pool with UUID {pool_uuid} is not encrypted but a passphrase was provided" + ))); } else { Ok(StartAction::Identity) } @@ -330,6 +336,19 @@ impl Engine for SimEngine { }) .map(|(n, p)| (n, u, p))?, }; + if pool.is_encrypted() && unlock_method.is_none() { + return Err(StratisError::Msg(format!( + "Pool with UUID {pool_uuid} is encrypted but no unlock method was provided" + ))); + } else if !pool.is_encrypted() && unlock_method.is_some() { + return Err(StratisError::Msg(format!( + "Pool with UUID {pool_uuid} is not encrypted but an unlock method was provided" + ))); + } else if !pool.is_encrypted() && passphrase_fd.is_some() { + return Err(StratisError::Msg(format!( + "Pool with UUID {pool_uuid} is not encrypted but a passphrase was provided" + ))); + } self.pools.modify_all().await.insert(name, pool_uuid, pool); Ok(StartAction::Started(pool_uuid)) } diff --git a/src/engine/sim_engine/keys.rs b/src/engine/sim_engine/keys.rs index 4a9a66f293..2e14e763aa 100644 --- a/src/engine/sim_engine/keys.rs +++ b/src/engine/sim_engine/keys.rs @@ -9,7 +9,7 @@ use libcryptsetup_rs::SafeMemHandle; use crate::{ engine::{ engine::{KeyActions, MAX_STRATIS_PASS_SIZE}, - shared, + shared::read_key_shared, types::{Key, KeyDescription, MappingCreateAction, MappingDeleteAction, SizedKeyMemory}, }, stratis::StratisResult, @@ -52,7 +52,7 @@ impl KeyActions for SimKeyActions { key_fd: RawFd, ) -> StratisResult> { let mut memory = vec![0; MAX_STRATIS_PASS_SIZE]; - let size = shared::set_key_shared(key_fd, memory.as_mut_slice())?; + let size = read_key_shared(key_fd, memory.as_mut_slice())?; memory.truncate(size); match self.read(key_desc) { diff --git a/src/engine/strat_engine/backstore/backstore/v2.rs b/src/engine/strat_engine/backstore/backstore/v2.rs index c47c652e07..153d8e3e1a 100644 --- a/src/engine/strat_engine/backstore/backstore/v2.rs +++ b/src/engine/strat_engine/backstore/backstore/v2.rs @@ -27,14 +27,14 @@ use crate::{ dm::{get_dm, list_of_backstore_devices, remove_optional_devices, DEVICEMAPPER_PATH}, metadata::{MDADataSize, BDA}, names::{format_backstore_ids, CacheRole}, - serde_structs::{BackstoreSave, CapSave, Recordable}, + serde_structs::{BackstoreSave, CapSave, PoolFeatures, PoolSave, Recordable}, shared::bds_to_bdas, types::BDARecordResult, writing::wipe_sectors, }, types::{ ActionAvailability, BlockDevTier, DevUuid, EncryptionInfo, KeyDescription, PoolUuid, - UnlockMethod, + SizedKeyMemory, UnlockMethod, }, }, stratis::{StratisError, StratisResult}, @@ -302,13 +302,15 @@ impl Backstore { /// self.cache.is_some() <=> self.cache_tier.is_some() pub fn setup( pool_uuid: PoolUuid, - backstore_save: &BackstoreSave, + pool_save: &PoolSave, datadevs: Vec, cachedevs: Vec, last_update_time: DateTime, + unlock_method: Option, + passphrase: Option, ) -> BDARecordResult { let block_mgr = BlockDevMgr::new(datadevs, Some(last_update_time)); - let data_tier = DataTier::setup(block_mgr, &backstore_save.data_tier)?; + let data_tier = DataTier::setup(block_mgr, &pool_save.backstore.data_tier)?; let (dm_name, dm_uuid) = format_backstore_ids(pool_uuid, CacheRole::OriginSub); let origin = match LinearDev::setup( get_dm(), @@ -332,7 +334,7 @@ impl Backstore { let (placeholder, cache, cache_tier, origin) = if !cachedevs.is_empty() { let block_mgr = BlockDevMgr::new(cachedevs, Some(last_update_time)); - match backstore_save.cache_tier { + match pool_save.backstore.cache_tier { Some(ref cache_tier_save) => { let cache_tier = match CacheTier::setup(block_mgr, cache_tier_save) { Ok(ct) => ct, @@ -380,20 +382,41 @@ impl Backstore { (Some(placeholder), None, None, Some(origin)) }; - let enc = match CryptHandle::setup( - &once(DEVICEMAPPER_PATH) - .chain(once( - format_backstore_ids(pool_uuid, CacheRole::Cache) - .0 - .to_string() - .as_str(), - )) - .collect::(), - pool_uuid, - UnlockMethod::Any, - ) { - Ok(opt) => opt.map(Either::Right), - Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), + let metadata_enc_enabled = pool_save.features.contains(PoolFeatures::Encryption); + let crypt_physical_path = &once(DEVICEMAPPER_PATH) + .chain(once( + format_backstore_ids(pool_uuid, CacheRole::Cache) + .0 + .to_string() + .as_str(), + )) + .collect::(); + let enc = match (metadata_enc_enabled, unlock_method, passphrase.as_ref()) { + (true, Some(unlock_method), pass) => { + match CryptHandle::setup(crypt_physical_path, pool_uuid, unlock_method, pass) { + Ok(opt) => { + if let Some(h) = opt { + Some(Either::Right(h)) + } else { + return Err((StratisError::Msg("Metadata reported that encryption is enabled but no crypt header was found".to_string()), data_tier.block_mgr.into_bdas())); + } + } + Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), + } + } + (true, None, _) => { + unreachable!("Checked in liminal device code"); + } + (false, _, _) => match CryptHandle::load_metadata(crypt_physical_path, pool_uuid) { + Ok(opt) => { + if opt.is_some() { + return Err((StratisError::Msg("Metadata reported that encryption is not enabled but a crypt header was found".to_string()), data_tier.block_mgr.into_bdas())); + } else { + None + } + } + Err(e) => return Err((e, data_tier.block_mgr.into_bdas())), + }, }; Ok(Backstore { @@ -403,8 +426,8 @@ impl Backstore { cache, placeholder, enc, - allocs: backstore_save.cap.allocs.clone(), - crypt_meta_allocs: backstore_save.cap.crypt_meta_allocs.clone(), + allocs: pool_save.backstore.cap.allocs.clone(), + crypt_meta_allocs: pool_save.backstore.cap.crypt_meta_allocs.clone(), }) } diff --git a/src/engine/strat_engine/crypt/handle/v1.rs b/src/engine/strat_engine/crypt/handle/v1.rs index 336c162c65..ff2c2d5648 100644 --- a/src/engine/strat_engine/crypt/handle/v1.rs +++ b/src/engine/strat_engine/crypt/handle/v1.rs @@ -409,10 +409,11 @@ pub fn setup_crypt_device(physical_path: &Path) -> StratisResult, + passphrase: Option<&SizedKeyMemory>, ) -> StratisResult> { let metadata = match load_crypt_metadata(device, physical_path)? { Some(m) => m, @@ -429,6 +430,7 @@ pub fn setup_crypt_handle( device, metadata.encryption_info.key_description(), unlock, + passphrase, &metadata.activation_name, )? }; @@ -758,6 +760,7 @@ impl CryptHandle { device, encryption_info.key_description(), UnlockMethod::Any, + None, &activation_name, ) } @@ -793,9 +796,12 @@ impl CryptHandle { pub fn setup( physical_path: &Path, unlock_method: Option, + passphrase: Option<&SizedKeyMemory>, ) -> StratisResult> { match setup_crypt_device(physical_path)? { - Some(ref mut device) => setup_crypt_handle(device, physical_path, unlock_method), + Some(ref mut device) => { + setup_crypt_handle(device, physical_path, unlock_method, passphrase) + } None => Ok(None), } } @@ -845,7 +851,7 @@ impl CryptHandle { } /// Get the keyslot associated with the given token ID. - pub fn keyslots(&self, token_id: c_uint) -> StratisResult>> { + pub fn keyslot(&self, token_id: c_uint) -> StratisResult> { get_keyslot_number(&mut self.acquire_crypt_device()?, token_id) } @@ -890,18 +896,16 @@ impl CryptHandle { )); } - let keyslots = self.keyslots(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { + let keyslot = self.keyslot(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { StratisError::Msg(format!( "Token slot {CLEVIS_LUKS_TOKEN_ID} appears to be empty; could not determine keyslots" )) })?; - for keyslot in keyslots { - log_on_failure!( - clevis_luks_unbind(self.luks2_device_path(), keyslot), - "Failed to unbind device {} from Clevis", - self.luks2_device_path().display() - ); - } + log_on_failure!( + clevis_luks_unbind(self.luks2_device_path(), keyslot), + "Failed to unbind device {} from Clevis", + self.luks2_device_path().display() + ); self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_clevis_info(); Ok(()) } @@ -919,11 +923,9 @@ impl CryptHandle { } let mut device = self.acquire_crypt_device()?; - let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)? - .and_then(|vec| vec.into_iter().next()) - .ok_or_else(|| { - StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) - })?; + let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { + StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) + })?; clevis_luks_regen(self.luks2_device_path(), keyslot)?; // Need to reload LUKS2 metadata after Clevis metadata modification. @@ -984,15 +986,13 @@ impl CryptHandle { } let mut device = self.acquire_crypt_device()?; - let keyslots = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? + let keyslot = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? .ok_or_else(|| StratisError::Msg("No LUKS2 keyring token was found".to_string()))?; - for keyslot in keyslots { - log_on_failure!( - device.keyslot_handle().destroy(keyslot), - "Failed partway through the kernel keyring unbinding operation \ - which cannot be rolled back; manual intervention may be required" - ) - } + log_on_failure!( + device.keyslot_handle().destroy(keyslot), + "Failed partway through the kernel keyring unbinding operation \ + which cannot be rolled back; manual intervention may be required" + ); device .token_handle() .json_set(TokenInput::RemoveToken(LUKS2_TOKEN_ID))?; @@ -1375,7 +1375,7 @@ mod tests { handle.deactivate().unwrap(); - let handle = CryptHandle::setup(path, Some(UnlockMethod::Keyring)) + let handle = CryptHandle::setup(path, Some(UnlockMethod::Keyring), None) .unwrap() .unwrap_or_else(|| { panic!( @@ -1481,7 +1481,7 @@ mod tests { fn unlock_clevis(paths: &[&Path]) { let path = paths.first().copied().expect("Expected exactly one path"); - CryptHandle::setup(path, Some(UnlockMethod::Clevis)) + CryptHandle::setup(path, Some(UnlockMethod::Clevis), None) .unwrap() .unwrap(); } diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs index a7ee3d3425..3866229f0b 100644 --- a/src/engine/strat_engine/crypt/handle/v2.rs +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -174,11 +174,12 @@ pub fn setup_crypt_device(physical_path: &Path) -> StratisResult, ) -> StratisResult> { let metadata = match load_crypt_metadata(device, physical_path, pool_uuid)? { Some(m) => m, @@ -194,6 +195,7 @@ pub fn setup_crypt_handle( device, metadata.encryption_info.key_description(), unlock_method, + passphrase, &metadata.activation_name, )? } @@ -443,6 +445,7 @@ impl CryptHandle { device, encryption_info.key_description(), UnlockMethod::Any, + None, &activation_name, ) } @@ -476,17 +479,17 @@ impl CryptHandle { physical_path: &Path, pool_uuid: PoolUuid, unlock_method: UnlockMethod, + passphrase: Option<&SizedKeyMemory>, ) -> StratisResult> { match setup_crypt_device(physical_path)? { Some(ref mut device) => { - setup_crypt_handle(device, physical_path, pool_uuid, unlock_method) + setup_crypt_handle(device, physical_path, pool_uuid, unlock_method, passphrase) } None => Ok(None), } } /// Load the required information for Stratis from the LUKS2 metadata. - #[cfg(test)] pub fn load_metadata( physical_path: &Path, pool_uuid: PoolUuid, @@ -531,7 +534,7 @@ impl CryptHandle { } /// Get the keyslot associated with the given token ID. - pub fn keyslots(&self, token_id: c_uint) -> StratisResult>> { + pub fn keyslot(&self, token_id: c_uint) -> StratisResult> { get_keyslot_number(&mut self.acquire_crypt_device()?, token_id) } @@ -576,18 +579,16 @@ impl CryptHandle { )); } - let keyslots = self.keyslots(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { + let keyslot = self.keyslot(CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { StratisError::Msg(format!( "Token slot {CLEVIS_LUKS_TOKEN_ID} appears to be empty; could not determine keyslots" )) })?; - for keyslot in keyslots { - log_on_failure!( - clevis_luks_unbind(self.luks2_device_path(), keyslot), - "Failed to unbind device {} from Clevis", - self.luks2_device_path().display() - ); - } + log_on_failure!( + clevis_luks_unbind(self.luks2_device_path(), keyslot), + "Failed to unbind device {} from Clevis", + self.luks2_device_path().display() + ); self.metadata.encryption_info = self.metadata.encryption_info.clone().unset_clevis_info(); Ok(()) } @@ -605,11 +606,9 @@ impl CryptHandle { } let mut device = self.acquire_crypt_device()?; - let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)? - .and_then(|vec| vec.into_iter().next()) - .ok_or_else(|| { - StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) - })?; + let keyslot = get_keyslot_number(&mut device, CLEVIS_LUKS_TOKEN_ID)?.ok_or_else(|| { + StratisError::Msg("Clevis binding found but no keyslot was associated".to_string()) + })?; clevis_luks_regen(self.luks2_device_path(), keyslot)?; // Need to reload LUKS2 metadata after Clevis metadata modification. @@ -670,15 +669,13 @@ impl CryptHandle { } let mut device = self.acquire_crypt_device()?; - let keyslots = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? + let keyslot = get_keyslot_number(&mut device, LUKS2_TOKEN_ID)? .ok_or_else(|| StratisError::Msg("No LUKS2 keyring token was found".to_string()))?; - for keyslot in keyslots { - log_on_failure!( - device.keyslot_handle().destroy(keyslot), - "Failed partway through the kernel keyring unbinding operation \ - which cannot be rolled back; manual intervention may be required" - ) - } + log_on_failure!( + device.keyslot_handle().destroy(keyslot), + "Failed partway through the kernel keyring unbinding operation \ + which cannot be rolled back; manual intervention may be required" + ); device .token_handle() .json_set(TokenInput::RemoveToken(LUKS2_TOKEN_ID))?; @@ -961,7 +958,7 @@ mod tests { handle.deactivate().unwrap(); - let handle = CryptHandle::setup(path, pool_uuid, UnlockMethod::Keyring) + let handle = CryptHandle::setup(path, pool_uuid, UnlockMethod::Keyring, None) .unwrap() .unwrap_or_else(|| { panic!( @@ -1058,7 +1055,7 @@ mod tests { fn unlock_clevis(paths: &[&Path], pool_uuid: PoolUuid) { let path = paths.first().copied().expect("Expected exactly one path"); - CryptHandle::setup(path, pool_uuid, UnlockMethod::Clevis) + CryptHandle::setup(path, pool_uuid, UnlockMethod::Clevis, None) .unwrap() .unwrap(); } diff --git a/src/engine/strat_engine/crypt/shared.rs b/src/engine/strat_engine/crypt/shared.rs index 5fa3488c7e..31dd2d5126 100644 --- a/src/engine/strat_engine/crypt/shared.rs +++ b/src/engine/strat_engine/crypt/shared.rs @@ -4,7 +4,7 @@ use std::{ fs::OpenOptions, - io::Write, + io::{ErrorKind, Write}, mem::forget, path::{Path, PathBuf}, slice::from_raw_parts_mut, @@ -25,7 +25,7 @@ use libcryptsetup_rs::{ CryptDebugLevel, CryptLogLevel, CryptStatusInfo, CryptWipePattern, EncryptionFormat, }, }, - register, set_debug_level, set_log_callback, CryptDevice, CryptInit, + register, set_debug_level, set_log_callback, CryptDevice, CryptInit, LibcryptErr, }; use crate::{ @@ -423,7 +423,7 @@ fn device_is_active(device: Option<&mut CryptDevice>, device_name: &DmName) -> S Ok(CryptStatusInfo::Active) => Ok(()), Ok(CryptStatusInfo::Busy) => { info!( - "Newly activated device {} reported that it was busy; you may see \ + "Newly device {} reported that it was busy; you may see \ temporary failures due to the device being busy.", device_name, ); @@ -461,46 +461,80 @@ pub fn activate( device: &mut CryptDevice, key_desc: Option<&KeyDescription>, unlock_method: UnlockMethod, + passphrase: Option<&SizedKeyMemory>, name: &DmName, ) -> StratisResult<()> { - if let (Some(kd), UnlockMethod::Keyring | UnlockMethod::Any) = (key_desc, unlock_method) { - let key_description_missing = keys::search_key_persistent(kd) - .map_err(|_| { - StratisError::Msg(format!( - "Searching the persistent keyring for the key description {} failed.", - kd.as_application_str(), - )) - })? - .is_none(); - if key_description_missing { - warn!( - "Key description {} was not found in the keyring", - kd.as_application_str() - ); - } - if key_description_missing && unlock_method == UnlockMethod::Keyring { - return Err(StratisError::Msg(format!( - "The key description \"{}\" is not currently set.", - kd.as_application_str(), - ))); - } - } - log_on_failure!( - device.token_handle().activate_by_token::<()>( - Some(&name.to_string()), + if let Some(p) = passphrase { + let key_slot = if unlock_method == UnlockMethod::Keyring { - Some(LUKS2_TOKEN_ID) + Some(get_keyslot_number(device, LUKS2_TOKEN_ID)?.ok_or_else(|| { + StratisError::Msg("LUKS keyring keyslot not found".to_string()) + })?) } else if unlock_method == UnlockMethod::Clevis { - Some(CLEVIS_LUKS_TOKEN_ID) + Some( + get_keyslot_number(device, CLEVIS_LUKS_TOKEN_ID)? + .ok_or_else(|| StratisError::Msg("Clevis keyslot not found".to_string()))?, + ) } else { None - }, - None, - CryptActivate::empty(), - ), - "Failed to activate device with name {}", - name - ); + }; + log_on_failure!( + device.activate_handle().activate_by_passphrase( + Some(&name.to_string()), + key_slot, + p.as_ref(), + CryptActivate::empty(), + ), + "Failed to activate device with name {}", + name + ); + } else { + if let (Some(kd), UnlockMethod::Keyring | UnlockMethod::Any) = (key_desc, unlock_method) { + let key_description_missing = keys::search_key_persistent(kd) + .map_err(|_| { + StratisError::Msg(format!( + "Searching the persistent keyring for the key description {} failed.", + kd.as_application_str(), + )) + })? + .is_none(); + if key_description_missing { + warn!( + "Key description {} was not found in the keyring", + kd.as_application_str() + ); + } + if key_description_missing && unlock_method == UnlockMethod::Keyring { + return Err(StratisError::Msg(format!( + "The key description \"{}\" is not currently set.", + kd.as_application_str(), + ))); + } + } + device + .token_handle() + .activate_by_token::<()>( + Some(&name.to_string()), + if unlock_method == UnlockMethod::Keyring { + Some(LUKS2_TOKEN_ID) + } else if unlock_method == UnlockMethod::Clevis { + Some(CLEVIS_LUKS_TOKEN_ID) + } else { + None + }, + None, + CryptActivate::empty(), + ) + .map_err(|e| match e { + LibcryptErr::IOError(e) => match e.kind() { + ErrorKind::NotFound => StratisError::Msg(format!( + "Token slot for unlock method {unlock_method:?} was empty" + )), + _ => StratisError::from(e), + }, + _ => StratisError::from(e), + })?; + } // Check activation status. device_is_active(Some(device), name)?; @@ -514,7 +548,7 @@ pub fn activate( pub fn get_keyslot_number( device: &mut CryptDevice, token_id: c_uint, -) -> StratisResult>> { +) -> StratisResult> { let json = match device.token_handle().json_get(token_id) { Ok(j) => j, Err(_) => return Ok(None), @@ -523,32 +557,38 @@ pub fn get_keyslot_number( .get(TOKEN_KEYSLOTS_KEY) .and_then(|k| k.as_array()) .ok_or_else(|| StratisError::Msg("keyslots value was malformed".to_string()))?; - Ok(Some( - vec.iter() - .filter_map(|int_val| { - let as_str = int_val.as_str(); - if as_str.is_none() { - warn!( - "Discarding invalid value in LUKS2 token keyslot array: {}", - int_val - ); - } - let s = match as_str { - Some(s) => s, - None => return None, - }; - let as_c_uint = s.parse::(); - if let Err(ref e) = as_c_uint { - warn!( - "Discarding invalid value in LUKS2 token keyslot array: {}; \ + let mut keyslots = vec + .iter() + .filter_map(|int_val| { + let as_str = int_val.as_str(); + if as_str.is_none() { + warn!( + "Discarding invalid value in LUKS2 token keyslot array: {}", + int_val + ); + } + let s = match as_str { + Some(s) => s, + None => return None, + }; + let as_c_uint = s.parse::(); + if let Err(ref e) = as_c_uint { + warn!( + "Discarding invalid value in LUKS2 token keyslot array: {}; \ failed to convert it to an integer: {}", - s, e, - ); - } - as_c_uint.ok() - }) - .collect::>(), - )) + s, e, + ); + } + as_c_uint.ok() + }) + .collect::>(); + if keyslots.len() > 1 { + return Err(StratisError::Msg(format!( + "Each token should be associated with exactly one keyslot; found {}", + keyslots.len() + ))); + } + Ok(keyslots.pop()) } /// Deactivate an encrypted Stratis device but do not wipe it. This is not @@ -621,14 +661,12 @@ pub fn ensure_wiped( ensure_inactive(device, name)?; let keyslot_number = get_keyslot_number(device, LUKS2_TOKEN_ID); match keyslot_number { - Ok(Some(nums)) => { - for i in nums.iter() { - log_on_failure!( - device.keyslot_handle().destroy(*i), - "Failed to destroy keyslot at index {}", - i - ); - } + Ok(Some(keyslot)) => { + log_on_failure!( + device.keyslot_handle().destroy(keyslot), + "Failed to destroy keyslot at index {}", + keyslot + ); } Ok(None) => { info!( diff --git a/src/engine/strat_engine/engine.rs b/src/engine/strat_engine/engine.rs index 5546faabe7..ff801c4c89 100644 --- a/src/engine/strat_engine/engine.rs +++ b/src/engine/strat_engine/engine.rs @@ -4,6 +4,7 @@ use std::{ collections::{HashMap, HashSet}, + os::fd::RawFd, path::Path, sync::Arc, }; @@ -679,6 +680,7 @@ impl Engine for StratEngine { &pools, PoolIdentifier::Uuid(pool_uuid), Some(unlock_method), + None, )?; pools.insert(name, pool_uuid, pool); StratisResult::Ok(unlocked_uuids) @@ -783,6 +785,7 @@ impl Engine for StratEngine { &self, id: PoolIdentifier, unlock_method: Option, + passphrase_fd: Option, ) -> StratisResult> { if let Some(lock) = self.pools.read(id.clone()).await { let (_, pool_uuid, pool) = lock.as_tuple(); @@ -794,6 +797,10 @@ impl Engine for StratEngine { return Err(StratisError::Msg(format!( "Pool with UUID {pool_uuid} is not encrypted but an unlock method was provided" ))); + } else if !pool.is_encrypted() && passphrase_fd.is_some() { + return Err(StratisError::Msg(format!( + "Pool with UUID {pool_uuid} is not encrypted but a passphrase was provided" + ))); } else { Ok(StartAction::Identity) } @@ -801,7 +808,8 @@ impl Engine for StratEngine { let mut pools = self.pools.modify_all().await; let mut liminal = self.liminal_devices.write().await; let pool_uuid = spawn_blocking!({ - let (name, pool_uuid, pool, _) = liminal.start_pool(&pools, id, unlock_method)?; + let (name, pool_uuid, pool, _) = + liminal.start_pool(&pools, id, unlock_method, passphrase_fd)?; pools.insert(name, pool_uuid, pool); StratisResult::Ok(pool_uuid) })??; @@ -1093,7 +1101,8 @@ mod test { test_async!(engine.stop_pool(PoolIdentifier::Uuid(uuid), true)).unwrap(); - test_async!(engine.start_pool(PoolIdentifier::Uuid(uuid), Some(unlock_method))).unwrap(); + test_async!(engine.start_pool(PoolIdentifier::Uuid(uuid), Some(unlock_method), None)) + .unwrap(); test_async!(engine.destroy_pool(uuid)).unwrap(); cmd::udev_settle().unwrap(); engine.teardown().unwrap(); @@ -1492,7 +1501,7 @@ mod test { assert_eq!(test_async!(engine.pools()).len(), 0); assert!( - test_async!(engine.start_pool(PoolIdentifier::Uuid(uuid), None)) + test_async!(engine.start_pool(PoolIdentifier::Uuid(uuid), None, None)) .unwrap() .is_changed() ); diff --git a/src/engine/strat_engine/keys.rs b/src/engine/strat_engine/keys.rs index 4163729d77..6b202bc933 100644 --- a/src/engine/strat_engine/keys.rs +++ b/src/engine/strat_engine/keys.rs @@ -11,7 +11,7 @@ use libcryptsetup_rs::SafeMemHandle; use crate::{ engine::{ engine::{KeyActions, MAX_STRATIS_PASS_SIZE}, - shared, + shared::read_key_shared, strat_engine::names::KeyDescription, types::{Key, MappingCreateAction, MappingDeleteAction, SizedKeyMemory}, }, @@ -366,7 +366,7 @@ impl KeyActions for StratKeyActions { key_fd: RawFd, ) -> StratisResult> { let mut memory = SafeMemHandle::alloc(MAX_STRATIS_PASS_SIZE)?; - let bytes = shared::set_key_shared(key_fd, memory.as_mut())?; + let bytes = read_key_shared(key_fd, memory.as_mut())?; set_key_idem(key_desc, SizedKeyMemory::new(memory, bytes)) } diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index 7e2a16e9d7..a42baa726d 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -6,18 +6,21 @@ use std::{ collections::{HashMap, HashSet}, + os::fd::RawFd, path::PathBuf, }; use chrono::{DateTime, Utc}; use either::Either; +use libcryptsetup_rs::SafeMemHandle; use serde_json::{Map, Value}; use devicemapper::Sectors; use crate::{ engine::{ - engine::{DumpState, Pool, StateDiff}, + engine::{DumpState, Pool, StateDiff, MAX_STRATIS_PASS_SIZE}, + shared::read_key_shared, strat_engine::{ backstore::{blockdev::InternalBlockDev, find_stratis_devs_by_uuid}, crypt::handle::v1::CryptHandle, @@ -38,15 +41,15 @@ use crate::{ }, metadata::{StratisIdentifiers, BDA}, pool::{v1, v2, AnyPool}, - serde_structs::PoolSave, + serde_structs::{PoolFeatures, PoolSave}, shared::tiers_to_bdas, types::BDARecordResult, }, structures::Table, types::{ DevUuid, LockedPoolsInfo, MaybeInconsistent, Name, PoolEncryptionInfo, PoolIdentifier, - PoolUuid, StoppedPoolsInfo, StratBlockDevDiff, StratSigblockVersion, UdevEngineEvent, - UnlockMethod, UuidOrConflict, + PoolUuid, SizedKeyMemory, StoppedPoolsInfo, StratBlockDevDiff, StratSigblockVersion, + UdevEngineEvent, UnlockMethod, UuidOrConflict, }, BlockDevTier, }, @@ -95,9 +98,16 @@ impl LiminalDevices { pools: &Table, pool_uuid: PoolUuid, unlock_method: UnlockMethod, + passphrase: Option<&SizedKeyMemory>, ) -> StratisResult> { - fn handle_luks(luks_info: &LLuksInfo, unlock_method: UnlockMethod) -> StratisResult<()> { - if CryptHandle::setup(&luks_info.dev_info.devnode, Some(unlock_method))?.is_some() { + fn handle_luks( + luks_info: &LLuksInfo, + unlock_method: UnlockMethod, + passphrase: Option<&SizedKeyMemory>, + ) -> StratisResult<()> { + if CryptHandle::setup(&luks_info.dev_info.devnode, Some(unlock_method), passphrase)? + .is_some() + { Ok(()) } else { Err(StratisError::Msg(format!( @@ -134,10 +144,12 @@ impl LiminalDevices { for (dev_uuid, info) in map.iter() { match info { LInfo::Stratis(_) => (), - LInfo::Luks(ref luks_info) => match handle_luks(luks_info, unlock_method) { - Ok(()) => unlocked.push(*dev_uuid), - Err(e) => return Err(e), - }, + LInfo::Luks(ref luks_info) => { + match handle_luks(luks_info, unlock_method, passphrase) { + Ok(()) => unlocked.push(*dev_uuid), + Err(e) => return Err(e), + } + } } } unlocked @@ -164,48 +176,68 @@ impl LiminalDevices { } /// Start a pool, create the devicemapper devices, and return the fully constructed - /// pool. - pub fn start_pool( + /// legacy pool. + /// + /// Precondition: Pool was determined to be in stopped or partially constructed pools. + pub fn start_pool_legacy( &mut self, pools: &Table, - id: PoolIdentifier, + pool_uuid: PoolUuid, unlock_method: Option, + passphrase_fd: Option, ) -> StratisResult<(Name, PoolUuid, AnyPool, Vec)> { - let pool_uuid = match id { - PoolIdentifier::Uuid(u) => u, - PoolIdentifier::Name(n) => self - .name_to_uuid - .get(&n) - .ok_or_else(|| StratisError::Msg(format!("Could not find a pool with name {n}"))) - .and_then(|uc| uc.to_result())?, - }; - // Here we take a reference to entries in stopped pools because the call to unlock_pool - // below requires the pool being unlocked to still have its entry in stopped_pools. - // Removing it here would cause an error. - let encryption_info = self + fn start_pool_failure( + pools: &Table, + pool_uuid: PoolUuid, + luks_info: StratisResult<(Option, MaybeInconsistent>)>, + infos: &HashMap, + bdas: HashMap, + meta_res: StratisResult<(DateTime, PoolSave)>, + ) -> BDARecordResult<(Name, AnyPool)> { + let (timestamp, metadata) = match meta_res { + Ok(o) => o, + Err(e) => return Err((e, bdas)), + }; + + setup_pool_legacy( + pools, pool_uuid, luks_info, infos, bdas, timestamp, metadata, + ) + } + + let pool = self .stopped_pools .get(&pool_uuid) .or_else(|| self.partially_constructed_pools.get(&pool_uuid)) - .ok_or_else(|| { - StratisError::Msg(format!( - "Requested pool with UUID {pool_uuid} was not found in stopped or partially constructed pools" - )) - })? - .encryption_info(); - let unlocked_devices = match (encryption_info, unlock_method) { - (Ok(Some(_)), None) => { + .expect("Checked in caller"); + + // Here we take a reference to entries in stopped pools because the call to unlock_pool + // below requires the pool being unlocked to still have its entry in stopped_pools. + // Removing it here would cause an error. + let encryption_info = pool.encryption_info(); + let unlocked_devices = match (encryption_info, unlock_method, passphrase_fd) { + (Ok(None), None, None) => Vec::new(), + (Ok(None), _, _) => { return Err(StratisError::Msg(format!( - "Pool with UUID {pool_uuid} is encrypted but no unlock method was provided" + "Pool with UUID {pool_uuid} is not encrypted but an unlock method or passphrase was provided" ))); } - (Ok(None), None) => Vec::new(), - (Ok(Some(_)), Some(method)) => self.unlock_pool(pools, pool_uuid, method)?, - (Ok(None), Some(_)) => { + (Ok(Some(_)), None, _) => { return Err(StratisError::Msg(format!( - "Pool with UUID {pool_uuid} is not encrypted but an unlock method was provided" + "Pool with UUID {pool_uuid} is encrypted but no unlock method was provided" ))); } - (Err(e), _) => return Err(e), + (Ok(Some(_)), Some(method), passphrase_fd) => { + let passphrase = if let Some(fd) = passphrase_fd { + let mut memory = SafeMemHandle::alloc(MAX_STRATIS_PASS_SIZE)?; + let len = read_key_shared(fd, memory.as_mut())?; + Some(SizedKeyMemory::new(memory, len)) + } else { + None + }; + + self.unlock_pool(pools, pool_uuid, method, passphrase.as_ref())? + } + (Err(e), _, _) => return Err(e), }; let uuids = unlocked_devices.into_iter().collect::>(); @@ -239,10 +271,226 @@ impl LiminalDevices { warn!("Failed to scan for newly unlocked Stratis devices: {}", e); return Err(e); } + } + + assert!(pools.get_by_uuid(pool_uuid).is_none()); + assert!(!self.stopped_pools.contains_key(&pool_uuid)); + + let encryption_info = stopped_pool.encryption_info(); + let pool_name = stopped_pool.pool_name(); + let luks_info = encryption_info.and_then(|ei| pool_name.map(|pn| (ei, pn))); + let infos = match stopped_pool.into_opened_set() { + Either::Left(i) => i, + Either::Right(ds) => { + let err = StratisError::Msg(format!( + "Some of the devices in pool with UUID {pool_uuid} are unopened" + )); + info!("Attempt to set up pool failed, but it may be possible to set up the pool later, if the situation changes: {}", err); + self.handle_stopped_pool(pool_uuid, ds); + return Err(err); + } + }; + + let res = load_stratis_metadata(pool_uuid, stratis_infos_ref(&infos)); + let (infos, bdas) = split_stratis_infos(infos); + + match start_pool_failure(pools, pool_uuid, luks_info, &infos, bdas, res) { + Ok((name, pool)) => { + self.uuid_lookup = self + .uuid_lookup + .drain() + .filter(|(_, (p, _))| *p != pool_uuid) + .collect(); + self.name_to_uuid = self + .name_to_uuid + .drain() + .filter_map(|(n, mut maybe_conflict)| { + if maybe_conflict.remove(&pool_uuid) { + None + } else { + Some((n, maybe_conflict)) + } + }) + .collect(); + info!( + "Pool with name \"{}\" and UUID \"{}\" set up", + name, pool_uuid + ); + Ok((name, pool_uuid, pool, uuids)) + } + Err((err, bdas)) => { + info!("Attempt to set up pool failed, but it may be possible to set up the pool later, if the situation changes: {}", err); + let device_set = reconstruct_stratis_infos(infos, bdas); + self.handle_stopped_pool(pool_uuid, device_set); + Err(err) + } + } + } + + /// Start a pool, create the devicemapper devices, and return the fully constructed + /// metadata V2 pool. + /// + /// Precondition: Pool was determined to be in stopped or partially constructed pools. + pub fn start_pool_new( + &mut self, + pools: &Table, + pool_uuid: PoolUuid, + unlock_method: Option, + passphrase_fd: Option, + ) -> StratisResult<(Name, PoolUuid, AnyPool, Vec)> { + fn start_pool_failure( + pools: &Table, + pool_uuid: PoolUuid, + infos: &HashMap, + bdas: HashMap, + meta_res: StratisResult<(DateTime, PoolSave)>, + unlock_method: Option, + passphrase_fd: Option, + ) -> BDARecordResult<(Name, AnyPool)> { + let (timestamp, metadata) = match meta_res { + Ok(o) => o, + Err(e) => return Err((e, bdas)), + }; + + let passphrase = match ( + metadata.features.contains(PoolFeatures::Encryption), + unlock_method, + passphrase_fd, + ) { + (false, None, None) | (true, Some(_), None) => None, + (false, _, _) => { + return Err(( + StratisError::Msg(format!( + "Pool with UUID {pool_uuid} is not encrypted but an unlock method or passphrase was provided" + )), + bdas, + )); + } + (true, None, _) => return Err(( + StratisError::Msg( + "Metadata reported that encryption enabled but no unlock method was provided" + .to_string() + ), + bdas, + )), + (true, Some(_), Some(fd)) => { + let mut memory = match SafeMemHandle::alloc(MAX_STRATIS_PASS_SIZE) { + Ok(m) => m, + Err(e) => return Err((StratisError::from(e), bdas)), + }; + let len = match read_key_shared(fd, memory.as_mut()) { + Ok(l) => l, + Err(e) => return Err((e, bdas)), + }; + Some(SizedKeyMemory::new(memory, len)) + } + }; + + setup_pool( + pools, + pool_uuid, + infos, + bdas, + timestamp, + metadata, + unlock_method, + passphrase, + ) + } + + let stopped_pool = self + .stopped_pools + .remove(&pool_uuid) + .or_else(|| self.partially_constructed_pools.remove(&pool_uuid)) + .expect("Checked above"); + + assert!(pools.get_by_uuid(pool_uuid).is_none()); + assert!(!self.stopped_pools.contains_key(&pool_uuid)); + + let infos = stopped_pool + .into_opened_set() + .expect_left("Cannot fail in V2 of metadata"); + + let res = load_stratis_metadata(pool_uuid, stratis_infos_ref(&infos)); + let (infos, bdas) = split_stratis_infos(infos); + + match start_pool_failure( + pools, + pool_uuid, + &infos, + bdas, + res, + unlock_method, + passphrase_fd, + ) { + Ok((name, pool)) => { + self.uuid_lookup = self + .uuid_lookup + .drain() + .filter(|(_, (p, _))| *p != pool_uuid) + .collect(); + self.name_to_uuid = self + .name_to_uuid + .drain() + .filter_map(|(n, mut maybe_conflict)| { + if maybe_conflict.remove(&pool_uuid) { + None + } else { + Some((n, maybe_conflict)) + } + }) + .collect(); + info!( + "Pool with name \"{}\" and UUID \"{}\" set up", + name, pool_uuid + ); + Ok((name, pool_uuid, pool, Vec::new())) + } + Err((err, bdas)) => { + info!("Attempt to set up pool failed, but it may be possible to set up the pool later, if the situation changes: {}", err); + let device_set = reconstruct_stratis_infos(infos, bdas); + self.handle_stopped_pool(pool_uuid, device_set); + Err(err) + } + } + } + + /// Start a pool, create the devicemapper devices, and return the fully constructed + /// pool. + pub fn start_pool( + &mut self, + pools: &Table, + id: PoolIdentifier, + unlock_method: Option, + passphrase_fd: Option, + ) -> StratisResult<(Name, PoolUuid, AnyPool, Vec)> { + let pool_uuid = match id { + PoolIdentifier::Uuid(u) => u, + PoolIdentifier::Name(ref n) => self + .name_to_uuid + .get(n) + .ok_or_else(|| StratisError::Msg(format!("Could not find a pool with name {n}"))) + .and_then(|uc| uc.to_result())?, }; + let pool = self + .stopped_pools + .get(&pool_uuid) + .or_else(|| self.partially_constructed_pools.get(&pool_uuid)) + .ok_or_else(|| { + StratisError::Msg(format!( + "Requested pool with UUID {pool_uuid} was not found in stopped or partially constructed pools" + )) + })?; + let metadata_version = pool.metadata_version()?; - self.try_setup_pool(pools, pool_uuid, stopped_pool) - .map(|(name, pool)| (name, pool_uuid, pool, uuids)) + match metadata_version { + StratSigblockVersion::V1 => { + self.start_pool_legacy(pools, pool_uuid, unlock_method, passphrase_fd) + } + StratSigblockVersion::V2 => { + self.start_pool_new(pools, pool_uuid, unlock_method, passphrase_fd) + } + } } /// Stop a pool, tear down the devicemapper devices, and store the pool information @@ -585,89 +833,6 @@ impl LiminalDevices { .collect::>() } - /// Attempt to set up a pool, starting it if it is not already started. - /// - /// See documentation for setup_pool for more information. - /// - /// Precondition: pools.get_by_uuid(pool_uuid).is_none() && - /// self.stopped_pools.get(pool_uuid).is_none() - fn try_setup_pool( - &mut self, - pools: &Table, - pool_uuid: PoolUuid, - device_set: DeviceSet, - ) -> StratisResult<(Name, AnyPool)> { - fn try_setup_pool_failure( - pools: &Table, - pool_uuid: PoolUuid, - luks_info: StratisResult<(Option, MaybeInconsistent>)>, - infos: &HashMap, - bdas: HashMap, - meta_res: StratisResult<(DateTime, PoolSave)>, - ) -> BDARecordResult<(Name, AnyPool)> { - let (timestamp, metadata) = match meta_res { - Ok(o) => o, - Err(e) => return Err((e, bdas)), - }; - - setup_pool( - pools, pool_uuid, luks_info, infos, bdas, timestamp, metadata, - ) - } - - assert!(pools.get_by_uuid(pool_uuid).is_none()); - assert!(!self.stopped_pools.contains_key(&pool_uuid)); - - let encryption_info = device_set.encryption_info(); - let pool_name = device_set.pool_name(); - let luks_info = encryption_info.and_then(|ei| pool_name.map(|pn| (ei, pn))); - let infos = match device_set.into_opened_set() { - Either::Left(i) => i, - Either::Right(ds) => { - let err = StratisError::Msg(format!( - "Some of the devices in pool with UUID {pool_uuid} are unopened" - )); - info!("Attempt to set up pool failed, but it may be possible to set up the pool later, if the situation changes: {}", err); - self.handle_stopped_pool(pool_uuid, ds); - return Err(err); - } - }; - - let res = load_stratis_metadata(pool_uuid, stratis_infos_ref(&infos)); - let (infos, bdas) = split_stratis_infos(infos); - match try_setup_pool_failure(pools, pool_uuid, luks_info, &infos, bdas, res) { - Ok((name, pool)) => { - self.uuid_lookup = self - .uuid_lookup - .drain() - .filter(|(_, (p, _))| *p != pool_uuid) - .collect(); - self.name_to_uuid = self - .name_to_uuid - .drain() - .filter_map(|(n, mut maybe_conflict)| { - if maybe_conflict.remove(&pool_uuid) { - None - } else { - Some((n, maybe_conflict)) - } - }) - .collect(); - info!( - "Pool with name \"{}\" and UUID \"{}\" set up", - name, pool_uuid - ); - Ok((name, pool)) - } - Err((err, bdas)) => { - info!("Attempt to set up pool failed, but it may be possible to set up the pool later, if the situation changes: {}", err); - let device_set = reconstruct_stratis_infos(infos, bdas); - self.handle_stopped_pool(pool_uuid, device_set); - Err(err) - } - } - } - /// Variation on try_setup_pool that returns None if the pool is marked /// as stopped in its metadata. /// @@ -685,17 +850,42 @@ impl LiminalDevices { luks_info: StratisResult<(Option, MaybeInconsistent>)>, infos: &HashMap, bdas: HashMap, + metadata_version: StratisResult, meta_res: StratisResult<(DateTime, PoolSave)>, ) -> BDARecordResult>> { + let metadata_version = match metadata_version { + Ok(mv) => mv, + Err(e) => return Err((e, bdas)), + }; let (timestamp, metadata) = match meta_res { Ok(o) => o, Err(e) => return Err((e, bdas)), }; if let Some(true) | None = metadata.started { - setup_pool( - pools, pool_uuid, luks_info, infos, bdas, timestamp, metadata, - ) - .map(Either::Left) + match metadata_version { + StratSigblockVersion::V1 => setup_pool_legacy( + pools, pool_uuid, luks_info, infos, bdas, timestamp, metadata, + ) + .map(Either::Left), + StratSigblockVersion::V2 => { + let is_encrypted = metadata.features.contains(PoolFeatures::Encryption); + setup_pool( + pools, + pool_uuid, + infos, + bdas, + timestamp, + metadata, + if is_encrypted { + Some(UnlockMethod::Any) + } else { + None + }, + None, + ) + .map(Either::Left) + } + } } else { Ok(Either::Right(bdas)) } @@ -704,6 +894,7 @@ impl LiminalDevices { assert!(pools.get_by_uuid(pool_uuid).is_none()); assert!(!self.stopped_pools.contains_key(&pool_uuid)); + let metadata_version = device_set.metadata_version(); let encryption_info = device_set.encryption_info(); let pool_name = device_set.pool_name(); let luks_info = encryption_info.and_then(|ei| pool_name.map(|pn| (ei, pn))); @@ -721,7 +912,15 @@ impl LiminalDevices { let res = load_stratis_metadata(pool_uuid, stratis_infos_ref(&infos)); let (infos, bdas) = split_stratis_infos(infos); - match try_setup_started_pool_failure(pools, pool_uuid, luks_info, &infos, bdas, res) { + match try_setup_started_pool_failure( + pools, + pool_uuid, + luks_info, + &infos, + bdas, + metadata_version, + res, + ) { Ok(Either::Left((name, pool))) => { self.uuid_lookup = self .uuid_lookup @@ -1085,7 +1284,8 @@ fn load_stratis_metadata( /// /// If there is a name conflict between the set of devices in devices /// and some existing pool, return an error. -fn setup_pool( +#[allow(clippy::too_many_arguments)] +fn setup_pool_legacy( pools: &Table, pool_uuid: PoolUuid, luks_info: StratisResult<(Option, MaybeInconsistent>)>, @@ -1104,109 +1304,130 @@ fn setup_pool( )), bdas)); } - let bda = bdas.values().next(); - let metadata_version = bda.expect("Must have at least one BDA").sigblock_version(); + let (datadevs, cachedevs) = match get_blockdevs_legacy(&metadata.backstore, infos, bdas) { + Err((err, bdas)) => return Err( + (StratisError::Chained( + format!( + "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", + pool_uuid, + &metadata.name, + ), + Box::new(err) + ), bdas)), + Ok((datadevs, cachedevs)) => (datadevs, cachedevs), + }; - match metadata_version { - StratSigblockVersion::V1 => { - let (datadevs, cachedevs) = match get_blockdevs_legacy(&metadata.backstore, infos, bdas) { - Err((err, bdas)) => return Err( - (StratisError::Chained( - format!( - "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", - pool_uuid, - &metadata.name, - ), - Box::new(err) - ), bdas)), - Ok((datadevs, cachedevs)) => (datadevs, cachedevs), - }; + let (pool_einfo, pool_name) = match luks_info { + Ok(inner) => inner, + Err(_) => { + // NOTE: This is not actually a hopeless situation. It may be + // that a LUKS device owned by Stratis corresponding to a + // Stratis device has just not been discovered yet. If it + // is, the appropriate info will be updated, and setup may + // yet succeed. + return Err(( + StratisError::Msg(format!( + "Some data devices in the set belonging to pool with UUID {} and name {} appear to be encrypted devices managed by Stratis, and some do not", + pool_uuid, + &metadata.name + )), tiers_to_bdas(datadevs, cachedevs, None))); + } + }; - let (pool_einfo, pool_name) = match luks_info { - Ok(inner) => inner, - Err(_) => { - // NOTE: This is not actually a hopeless situation. It may be - // that a LUKS device owned by Stratis corresponding to a - // Stratis device has just not been discovered yet. If it - // is, the appropriate info will be updated, and setup may - // yet succeed. - return Err(( - StratisError::Msg(format!( - "Some data devices in the set belonging to pool with UUID {} and name {} appear to be encrypted devices managed by Stratis, and some do not", - pool_uuid, - &metadata.name - )), tiers_to_bdas(datadevs, cachedevs, None))); + v1::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata, pool_einfo) + .map(|(name, mut pool)| { + if matches!(pool_name, MaybeInconsistent::Yes | MaybeInconsistent::No(None)) || MaybeInconsistent::No(Some(&name)) != pool_name.as_ref() || pool.blockdevs().iter().map(|(_, _, bd)| { + bd.pool_name() + }).fold(false, |acc, next| { + match next { + Some(Some(name)) => { + if MaybeInconsistent::No(Some(name)) == pool_name.as_ref() { + acc + } else { + true + } + }, + Some(None) => true, + None => false, } - }; + }) { + if let Err(e) = pool.rename_pool(&name) { + warn!("Pool will not be able to be started by name; pool name metadata in LUKS2 token is not consistent across all devices: {}", e); + } + } + (name, AnyPool::V1(pool)) + }) + .map_err(|(err, bdas)| { + (StratisError::Chained( + format!( + "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" + ), + Box::new(err), + ), bdas) + }) +} - v1::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata, pool_einfo) - .map(|(name, mut pool)| { - if matches!(pool_name, MaybeInconsistent::Yes | MaybeInconsistent::No(None)) || MaybeInconsistent::No(Some(&name)) != pool_name.as_ref() || pool.blockdevs().iter().map(|(_, _, bd)| { - bd.pool_name() - }).fold(false, |acc, next| { - match next { - Some(Some(name)) => { - if MaybeInconsistent::No(Some(name)) == pool_name.as_ref() { - acc - } else { - true - } - }, - Some(None) => true, - None => false, - } - }) { - if let Err(e) = pool.rename_pool(&name) { - warn!("Pool will not be able to be started by name; pool name metadata in LUKS2 token is not consistent across all devices: {}", e); - } - } - (name, AnyPool::V1(pool)) - }) - .map_err(|(err, bdas)| { - (StratisError::Chained( - format!( - "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" - ), - Box::new(err), - ), bdas) - }) - } - StratSigblockVersion::V2 => { - let (datadevs, cachedevs) = match get_blockdevs(&metadata.backstore, infos, bdas) { - Err((err, bdas)) => return Err( - (StratisError::Chained( - format!( - "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", - pool_uuid, - &metadata.name, - ), - Box::new(err) - ), bdas)), - Ok((datadevs, cachedevs)) => (datadevs, cachedevs), - }; +/// Given a set of devices, try to set up a pool. +/// Return the pool information if a pool is set up. Otherwise, return +/// the pool information to the stopped pools data structure. +/// Do not attempt setup if the pool contains any unopened devices. +/// +/// If there is a name conflict between the set of devices in devices +/// and some existing pool, return an error. +#[allow(clippy::too_many_arguments)] +fn setup_pool( + pools: &Table, + pool_uuid: PoolUuid, + infos: &HashMap, + bdas: HashMap, + timestamp: DateTime, + metadata: PoolSave, + unlock_method: Option, + passphrase: Option, +) -> BDARecordResult<(Name, AnyPool)> { + if let Some((uuid, _)) = pools.get_by_name(&metadata.name) { + return Err(( + StratisError::Msg(format!( + "There is a pool name conflict. The devices currently being processed have been identified as belonging to the pool with UUID {} and name {}, but a pool with the same name and UUID {} is already active", + pool_uuid, + &metadata.name, + uuid + )), bdas)); + } - let dev = datadevs.first(); - if dev.is_none() { - return Err(( - StratisError::Msg(format!( - "There do not appear to be any data devices in the set with pool UUID {pool_uuid}" - )), - tiers_to_bdas(datadevs, cachedevs, None), - )); - } + let (datadevs, cachedevs) = match get_blockdevs(&metadata.backstore, infos, bdas) { + Err((err, bdas)) => return Err( + (StratisError::Chained( + format!( + "There was an error encountered when calculating the block devices for pool with UUID {} and name {}", + pool_uuid, + &metadata.name, + ), + Box::new(err) + ), bdas)), + Ok((datadevs, cachedevs)) => (datadevs, cachedevs), + }; - v2::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata) - .map(|(name, pool)| { - (name, AnyPool::V2(pool)) - }) - .map_err(|(err, bdas)| { - (StratisError::Chained( - format!( - "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" - ), - Box::new(err), - ), bdas) - }) - } + let dev = datadevs.first(); + if dev.is_none() { + return Err(( + StratisError::Msg(format!( + "There do not appear to be any data devices in the set with pool UUID {pool_uuid}" + )), + tiers_to_bdas(datadevs, cachedevs, None), + )); } + + v2::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata, unlock_method, passphrase) + .map(|(name, pool)| { + (name, AnyPool::V2(pool)) + }) + .map_err(|(err, bdas)| { + (StratisError::Chained( + format!( + "An attempt to set up pool with UUID {pool_uuid} from the assembled devices failed" + ), + Box::new(err), + ), bdas) + }) } diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index bda98929f8..ef84bb69da 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -401,7 +401,7 @@ fn get_blockdev_legacy( Some(luks) => &luks.dev_info.devnode, None => &info.dev_info.devnode, }; - let handle = match CryptHandle::setup(physical_path, None) { + let handle = match CryptHandle::setup(physical_path, None, None) { Ok(h) => h, Err(e) => return Err((e, bda)), }; diff --git a/src/engine/strat_engine/pool/v2.rs b/src/engine/strat_engine/pool/v2.rs index f1604c6f34..e82c6cfbca 100644 --- a/src/engine/strat_engine/pool/v2.rs +++ b/src/engine/strat_engine/pool/v2.rs @@ -34,8 +34,8 @@ use crate::{ ActionAvailability, BlockDevTier, Clevis, Compare, CreateAction, DeleteAction, DevUuid, Diff, EncryptionInfo, FilesystemUuid, GrowAction, Key, KeyDescription, Name, PoolDiff, PoolEncryptionInfo, PoolUuid, PropChangeAction, RegenAction, RenameAction, - SetCreateAction, SetDeleteAction, StratFilesystemDiff, StratPoolDiff, - StratSigblockVersion, + SetCreateAction, SetDeleteAction, SizedKeyMemory, StratFilesystemDiff, StratPoolDiff, + StratSigblockVersion, UnlockMethod, }, }, stratis::{StratisError, StratisResult}, @@ -221,13 +221,22 @@ impl StratPool { cachedevs: Vec, timestamp: DateTime, metadata: &PoolSave, + unlock_method: Option, + passphrase: Option, ) -> BDARecordResult<(Name, StratPool)> { if let Err(e) = check_metadata(metadata) { return Err((e, tiers_to_bdas(datadevs, cachedevs, None))); } - let backstore = - Backstore::setup(uuid, &metadata.backstore, datadevs, cachedevs, timestamp)?; + let backstore = Backstore::setup( + uuid, + metadata, + datadevs, + cachedevs, + timestamp, + unlock_method, + passphrase, + )?; let action_avail = backstore.action_availability(); let pool_name = &metadata.name; diff --git a/src/engine/types/mod.rs b/src/engine/types/mod.rs index 3ea5428053..5a0281f8c0 100644 --- a/src/engine/types/mod.rs +++ b/src/engine/types/mod.rs @@ -130,7 +130,7 @@ impl Display for StratisUuid { } /// Use Clevis or keyring to unlock LUKS volume. -#[derive(Serialize, Deserialize, Clone, Copy, Eq, PartialEq)] +#[derive(Serialize, Deserialize, Clone, Copy, Eq, PartialEq, Debug)] pub enum UnlockMethod { Clevis, Keyring, diff --git a/src/jsonrpc/server/key.rs b/src/jsonrpc/server/key.rs index 97f3f90c92..a2ec2d42f6 100644 --- a/src/jsonrpc/server/key.rs +++ b/src/jsonrpc/server/key.rs @@ -5,10 +5,8 @@ use std::{os::unix::io::RawFd, sync::Arc}; use crate::{ - engine::{ - Engine, KeyDescription, MappingCreateAction, MappingDeleteAction, PoolIdentifier, PoolUuid, - }, - stratis::{StratisError, StratisResult}, + engine::{Engine, KeyDescription, MappingCreateAction, MappingDeleteAction}, + stratis::StratisResult, }; // stratis-min key set @@ -38,31 +36,3 @@ pub async fn key_unset(engine: Arc, key_desc: &KeyDescription) -> St pub async fn key_list(engine: Arc) -> StratisResult> { Ok(engine.get_key_handler().await.list()?.into_iter().collect()) } - -pub async fn key_get_desc( - engine: Arc, - id: PoolIdentifier, -) -> StratisResult> { - let stopped = engine.stopped_pools().await; - let guard = engine.get_pool(id.clone()).await; - if let Some((_, _, pool)) = guard.as_ref().map(|guard| guard.as_tuple()) { - match pool.encryption_info() { - Some(ei) => ei.key_description().map(|opt| opt.cloned()), - None => Ok(None), - } - } else if let Some(info) = stopped.stopped.get(match id { - PoolIdentifier::Uuid(ref u) => u, - PoolIdentifier::Name(ref n) => stopped - .name_to_uuid - .get(n) - .ok_or_else(|| StratisError::Msg(format!("Pool with name {n} not found")))?, - }) { - if let Some(ref i) = info.info { - i.key_description().map(|opt| opt.cloned()) - } else { - Ok(None) - } - } else { - Err(StratisError::Msg(format!("Pool with {id} not found"))) - } -} diff --git a/src/jsonrpc/server/pool.rs b/src/jsonrpc/server/pool.rs index 78255d846d..104c9459ae 100644 --- a/src/jsonrpc/server/pool.rs +++ b/src/jsonrpc/server/pool.rs @@ -4,19 +4,15 @@ use std::{os::unix::io::RawFd, path::Path, sync::Arc}; -use tokio::task::block_in_place; - use serde_json::Value; +use tokio::task::block_in_place; use crate::{ engine::{ BlockDevTier, CreateAction, DeleteAction, EncryptionInfo, Engine, EngineAction, KeyDescription, Name, PoolIdentifier, PoolUuid, RenameAction, UnlockMethod, }, - jsonrpc::{ - interface::PoolListType, - server::key::{key_get_desc, key_set}, - }, + jsonrpc::interface::PoolListType, stratis::{StratisError, StratisResult}, }; @@ -27,11 +23,10 @@ pub async fn pool_start( unlock_method: Option, prompt: Option, ) -> StratisResult { - if let (Some(fd), Some(kd)) = (prompt, key_get_desc(Arc::clone(&engine), id.clone()).await?) { - key_set(engine.clone(), &kd, fd).await?; - } - - Ok(engine.start_pool(id, unlock_method).await?.is_changed()) + Ok(engine + .start_pool(id, unlock_method, prompt) + .await? + .is_changed()) } // stratis-min pool stop From c9d2318e0070555de62f738cdfd024fdb74b4517 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 30 May 2024 11:27:27 -0400 Subject: [PATCH 23/32] Update introspect data --- tests/client-dbus/src/stratisd_client_dbus/_introspect.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/client-dbus/src/stratisd_client_dbus/_introspect.py b/tests/client-dbus/src/stratisd_client_dbus/_introspect.py index 7e899437fe..0afe2924d4 100644 --- a/tests/client-dbus/src/stratisd_client_dbus/_introspect.py +++ b/tests/client-dbus/src/stratisd_client_dbus/_introspect.py @@ -48,6 +48,7 @@ + @@ -236,6 +237,9 @@ + + + From fb2c2aa39eb4251d9b1b746b515d1fccf489b49b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Thu, 30 May 2024 11:27:58 -0400 Subject: [PATCH 24/32] Fix StartPool usage in udev tests --- tests/client-dbus/tests/udev/test_udev.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/client-dbus/tests/udev/test_udev.py b/tests/client-dbus/tests/udev/test_udev.py index caf8564508..bcec8fd2db 100644 --- a/tests/client-dbus/tests/udev/test_udev.py +++ b/tests/client-dbus/tests/udev/test_udev.py @@ -267,6 +267,7 @@ def _simple_initial_discovery_test( "id": pool_uuid, "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "uuid", + "key_fd": (False, 0), }, ) if key_spec is None: @@ -367,6 +368,7 @@ def _simple_event_test(self, *, key_spec=None): # pylint: disable=too-many-loca "id": pool_uuid, "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "uuid", + "key_fd": (False, 0), }, ) # This should always fail because a pool cannot be successfully @@ -387,6 +389,7 @@ def _simple_event_test(self, *, key_spec=None): # pylint: disable=too-many-loca "id": pool_uuid, "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "uuid", + "key_fd": (False, 0), }, ) @@ -508,6 +511,7 @@ def test_duplicate_pool_name( "id": pool_uuid, "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "uuid", + "key_fd": (False, 0), }, ) @@ -543,6 +547,7 @@ def test_duplicate_pool_name( "id": pool_uuid, "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "uuid", + "key_fd": (False, 0), }, ) @@ -708,6 +713,7 @@ def _simple_start_by_name_test(self): "id": "encrypted", "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "name", + "key_fd": (False, 0), }, ) self.assertFalse(changed) @@ -723,6 +729,7 @@ def _simple_start_by_name_test(self): "id": "unencrypted", "unlock_method": (False, ""), "id_type": "name", + "key_fd": (False, 0), }, ) self.assertFalse(changed) @@ -743,6 +750,7 @@ def _simple_start_by_name_test(self): "id": "encrypted", "unlock_method": (True, str(EncryptionMethod.KEYRING)), "id_type": "name", + "key_fd": (False, 0), }, ) self.assertTrue(changed) @@ -757,6 +765,7 @@ def _simple_start_by_name_test(self): "id": "unencrypted", "unlock_method": (False, ""), "id_type": "name", + "key_fd": (False, 0), }, ) self.assertTrue(changed) From d4af956eb1fb53cb7ae114c366e5fbb0fbda02a3 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 31 May 2024 10:26:54 -0400 Subject: [PATCH 25/32] Fix Clevis tests by setting up udev and allocating from backstore --- .github/workflows/fedora.yml | 53 ++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index a8f57c38e1..967881615d 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -69,11 +69,6 @@ jobs: - task: make -f Makefile test toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo - - task: >- - TANG_URL=localhost - make -f Makefile test-clevis-loop-should-fail - toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN - components: cargo - task: make -f Makefile build toolchain: 1.79.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN components: cargo @@ -157,3 +152,51 @@ jobs: run: udevadm control --reload - name: Test ${{ matrix.task }} on ${{ matrix.toolchain }} toolchain run: ${{ matrix.task }} + + # TESTS WITH UDEV + checks_with_tang_should_fail: + strategy: + matrix: + include: + - task: >- + TANG_URL=localhost + make -f Makefile test-clevis-loop-should-fail + toolchain: 1.78.0 # CURRENT DEVELOPMENT RUST TOOLCHAIN + components: cargo + runs-on: ubuntu-22.04 + container: + image: fedora:40 # CURRENT DEVELOPMENT ENVIRONMENT + options: --privileged -v /dev:/dev -v /run/udev:/run/udev -v /usr/lib/udev:/usr/lib/udev --ipc=host + steps: + - uses: actions/checkout@v4 + - name: Install dependencies for Fedora + run: > + dnf install -y + asciidoc + clang + clevis + cryptsetup-devel + curl + dbus-devel + glibc-static + device-mapper-devel + device-mapper-persistent-data + libblkid-devel + make + ncurses + sudo + systemd-devel + systemd-udev + xfsprogs + - uses: dtolnay/rust-toolchain@master + with: + components: ${{ matrix.components }} + toolchain: ${{ matrix.toolchain }} + - name: Build stratisd + run: PROFILEDIR=debug make -f Makefile build-all + - name: Install stratisd + run: PROFILEDIR=debug make -f Makefile install + - name: Reload udev + run: udevadm control --reload + - name: Test ${{ matrix.task }} on ${{ matrix.toolchain }} toolchain + run: ${{ matrix.task }} From 36c0a8eea3b427350963096570d313fbd133de7b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 3 Jun 2024 14:44:45 -0400 Subject: [PATCH 26/32] Add test for passphrase unlock --- src/engine/strat_engine/crypt/handle/v1.rs | 45 +++++++++++++++++++- src/engine/strat_engine/crypt/handle/v2.rs | 48 +++++++++++++++++++++- src/engine/strat_engine/tests/crypt.rs | 10 +++-- 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/engine/strat_engine/crypt/handle/v1.rs b/src/engine/strat_engine/crypt/handle/v1.rs index ff2c2d5648..b31a716223 100644 --- a/src/engine/strat_engine/crypt/handle/v1.rs +++ b/src/engine/strat_engine/crypt/handle/v1.rs @@ -1486,7 +1486,7 @@ mod tests { .unwrap(); } - crypt::insert_and_remove_key(paths, both_initialize, unlock_clevis); + crypt::insert_and_remove_key(paths, both_initialize, |paths, _| unlock_clevis(paths)); } #[test] @@ -1682,4 +1682,47 @@ mod tests { test_clevis_sss_configs, ); } + + fn test_passphrase_unlock(paths: &[&Path]) { + fn init(paths: &[&Path], key_desc: &KeyDescription) { + let path = paths[0]; + + let handle = CryptHandle::initialize( + path, + PoolUuid::new_v4(), + DevUuid::new_v4(), + Name::new("pool_name".to_string()), + &EncryptionInfo::KeyDesc(key_desc.clone()), + None, + ) + .unwrap(); + handle.deactivate().unwrap(); + } + + fn unlock(paths: &[&Path], key: &SizedKeyMemory) { + let path = paths[0]; + + CryptHandle::setup(path, Some(UnlockMethod::Any), Some(key)) + .unwrap() + .unwrap(); + } + + crypt::insert_and_remove_key(paths, init, unlock); + } + + #[test] + fn real_test_passphrase_unlock() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_passphrase_unlock, + ); + } + + #[test] + fn loop_test_passphrase_unlock() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_passphrase_unlock, + ); + } } diff --git a/src/engine/strat_engine/crypt/handle/v2.rs b/src/engine/strat_engine/crypt/handle/v2.rs index 3866229f0b..e3b5a885ac 100644 --- a/src/engine/strat_engine/crypt/handle/v2.rs +++ b/src/engine/strat_engine/crypt/handle/v2.rs @@ -1064,7 +1064,7 @@ mod tests { crypt::insert_and_remove_key( paths, |paths, key_desc| both_initialize(paths, key_desc, pool_uuid), - |paths| unlock_clevis(paths, pool_uuid), + |paths, _| unlock_clevis(paths, pool_uuid), ); } @@ -1248,4 +1248,50 @@ mod tests { test_clevis_sss_configs, ); } + + fn test_passphrase_unlock(paths: &[&Path]) { + fn init(paths: &[&Path], pool_uuid: PoolUuid, key_desc: &KeyDescription) { + let path = paths[0]; + + let handle = CryptHandle::initialize( + path, + pool_uuid, + &EncryptionInfo::KeyDesc(key_desc.clone()), + None, + ) + .unwrap(); + handle.deactivate().unwrap(); + } + + fn unlock(paths: &[&Path], pool_uuid: PoolUuid, key: &SizedKeyMemory) { + let path = paths[0]; + + CryptHandle::setup(path, pool_uuid, UnlockMethod::Any, Some(key)) + .unwrap() + .unwrap(); + } + + let pool_uuid = PoolUuid::new_v4(); + crypt::insert_and_remove_key( + paths, + |paths, key_desc| init(paths, pool_uuid, key_desc), + |paths, key| unlock(paths, pool_uuid, key), + ); + } + + #[test] + fn real_test_passphrase_unlock() { + real::test_with_spec( + &real::DeviceLimits::Exactly(1, None, None), + test_passphrase_unlock, + ); + } + + #[test] + fn loop_test_passphrase_unlock() { + loopbacked::test_with_spec( + &loopbacked::DeviceLimits::Exactly(1, None), + test_passphrase_unlock, + ); + } } diff --git a/src/engine/strat_engine/tests/crypt.rs b/src/engine/strat_engine/tests/crypt.rs index 2ab40179e1..5f3e4d9e50 100644 --- a/src/engine/strat_engine/tests/crypt.rs +++ b/src/engine/strat_engine/tests/crypt.rs @@ -13,7 +13,10 @@ use libcryptsetup_rs::SafeMemHandle; use crate::engine::{ engine::{KeyActions, MAX_STRATIS_PASS_SIZE}, - strat_engine::{keys::StratKeyActions, names::KeyDescription}, + strat_engine::{ + keys::{read_key_persistent, StratKeyActions}, + names::KeyDescription, + }, types::SizedKeyMemory, }; @@ -67,19 +70,20 @@ where pub fn insert_and_remove_key(physical_paths: &[&Path], test_pre: F1, test_post: F2) where F1: FnOnce(&[&Path], &KeyDescription) + UnwindSafe, - F2: FnOnce(&[&Path]), + F2: FnOnce(&[&Path], &SizedKeyMemory), { let key_description = set_up_key("test-description-for-stratisd"); let result = catch_unwind(|| test_pre(physical_paths, &key_description)); + let (_, key) = read_key_persistent(&key_description).unwrap().unwrap(); StratKeyActions.unset(&key_description).unwrap(); if let Err(e) = result { resume_unwind(e) } - test_post(physical_paths) + test_post(physical_paths, &key) } /// Takes physical device paths from loopback or real tests and passes From a548926f4217d150cdcb09407c484e9752c1cc93 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 3 Jun 2024 10:46:12 -0400 Subject: [PATCH 27/32] Add metadata version to StoppedPools --- src/dbus_api/api/manager_3_2/props.rs | 2 +- src/dbus_api/api/manager_3_7/api.rs | 18 ++++++++++++++++-- src/dbus_api/api/manager_3_7/mod.rs | 3 ++- src/dbus_api/api/manager_3_7/props.rs | 18 ++++++++++++++++++ src/dbus_api/api/mod.rs | 2 +- src/dbus_api/api/prop_conv.rs | 11 ++++++++++- src/dbus_api/api/shared.rs | 4 ++-- src/dbus_api/tree.rs | 12 ++++++------ src/engine/sim_engine/engine.rs | 2 ++ src/engine/strat_engine/liminal/device_info.rs | 1 + src/engine/types/mod.rs | 1 + 11 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 src/dbus_api/api/manager_3_7/props.rs diff --git a/src/dbus_api/api/manager_3_2/props.rs b/src/dbus_api/api/manager_3_2/props.rs index 9e0ce6b77f..5e58ff844f 100644 --- a/src/dbus_api/api/manager_3_2/props.rs +++ b/src/dbus_api/api/manager_3_2/props.rs @@ -14,5 +14,5 @@ pub fn get_stopped_pools( i: &mut IterAppend<'_>, p: &PropInfo<'_, MTSync, TData>, ) -> Result<(), MethodErr> { - get_manager_property(i, p, |e| Ok(shared::stopped_pools_prop(e))) + get_manager_property(i, p, |e| Ok(shared::stopped_pools_prop(e, false))) } diff --git a/src/dbus_api/api/manager_3_7/api.rs b/src/dbus_api/api/manager_3_7/api.rs index 3427ce854c..7dc0cc6cce 100644 --- a/src/dbus_api/api/manager_3_7/api.rs +++ b/src/dbus_api/api/manager_3_7/api.rs @@ -2,9 +2,16 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -use dbus_tree::{Factory, MTSync, Method}; +use dbus_tree::{Access, EmitsChangedSignal, Factory, MTSync, Method, Property}; -use crate::dbus_api::{api::manager_3_7::methods::start_pool, types::TData}; +use crate::dbus_api::{ + api::{ + manager_3_7::{methods::start_pool, props::get_stopped_pools}, + prop_conv::StoppedOrLockedPools, + }, + consts, + types::TData, +}; pub fn start_pool_method(f: &Factory, TData>) -> Method, TData> { f.method("StartPool", (), start_pool) @@ -23,3 +30,10 @@ pub fn start_pool_method(f: &Factory, TData>) -> Method, TData>) -> Property, TData> { + f.property::(consts::STOPPED_POOLS_PROP, ()) + .access(Access::Read) + .emits_changed(EmitsChangedSignal::True) + .on_get(get_stopped_pools) +} diff --git a/src/dbus_api/api/manager_3_7/mod.rs b/src/dbus_api/api/manager_3_7/mod.rs index f6871e56b4..48fc8b4d99 100644 --- a/src/dbus_api/api/manager_3_7/mod.rs +++ b/src/dbus_api/api/manager_3_7/mod.rs @@ -4,5 +4,6 @@ mod api; mod methods; +mod props; -pub use api::start_pool_method; +pub use api::{start_pool_method, stopped_pools_property}; diff --git a/src/dbus_api/api/manager_3_7/props.rs b/src/dbus_api/api/manager_3_7/props.rs new file mode 100644 index 0000000000..543dba10dc --- /dev/null +++ b/src/dbus_api/api/manager_3_7/props.rs @@ -0,0 +1,18 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use dbus::arg::IterAppend; +use dbus_tree::{MTSync, MethodErr, PropInfo}; + +use crate::dbus_api::{ + api::shared::{self, get_manager_property}, + types::TData, +}; + +pub fn get_stopped_pools( + i: &mut IterAppend<'_>, + p: &PropInfo<'_, MTSync, TData>, +) -> Result<(), MethodErr> { + get_manager_property(i, p, |e| Ok(shared::stopped_pools_prop(e, true))) +} diff --git a/src/dbus_api/api/mod.rs b/src/dbus_api/api/mod.rs index f299e006a1..8a92d980f6 100644 --- a/src/dbus_api/api/mod.rs +++ b/src/dbus_api/api/mod.rs @@ -148,7 +148,7 @@ pub fn get_base_tree<'a>( .add_m(manager_3_6::stop_pool_method(&f)) .add_m(manager_3_2::refresh_state_method(&f)) .add_p(manager_3_0::version_property(&f)) - .add_p(manager_3_2::stopped_pools_property(&f)), + .add_p(manager_3_7::stopped_pools_property(&f)), ) .add( f.interface(consts::REPORT_INTERFACE_NAME_3_0, ()) diff --git a/src/dbus_api/api/prop_conv.rs b/src/dbus_api/api/prop_conv.rs index e53051fac1..f4bbc177fa 100644 --- a/src/dbus_api/api/prop_conv.rs +++ b/src/dbus_api/api/prop_conv.rs @@ -66,7 +66,7 @@ pub fn locked_pools_to_prop(pools: &LockedPoolsInfo) -> StoppedOrLockedPools { } /// Convert a stopped pool data structure to a property format. -pub fn stopped_pools_to_prop(pools: &StoppedPoolsInfo) -> StoppedOrLockedPools { +pub fn stopped_pools_to_prop(pools: &StoppedPoolsInfo, metadata: bool) -> StoppedOrLockedPools { pools .stopped .iter() @@ -111,6 +111,15 @@ pub fn stopped_pools_to_prop(pools: &StoppedPoolsInfo) -> StoppedOrLockedPools { .collect::>(), )), ); + if metadata { + map.insert( + "metadata_version".to_string(), + match stopped.metadata_version { + Some(m) => Variant(Box::new((true, m as u64))), + None => Variant(Box::new((false, 0))), + }, + ); + } (uuid_to_string!(u), map) }) .collect::>() diff --git a/src/dbus_api/api/shared.rs b/src/dbus_api/api/shared.rs index 825e1b3dda..e2132b6d88 100644 --- a/src/dbus_api/api/shared.rs +++ b/src/dbus_api/api/shared.rs @@ -163,6 +163,6 @@ pub fn locked_pools_prop(e: Arc) -> StoppedOrLockedPools { /// Generate D-Bus representation of stopped pools #[inline] -pub fn stopped_pools_prop(e: Arc) -> StoppedOrLockedPools { - prop_conv::stopped_pools_to_prop(&block_on(e.stopped_pools())) +pub fn stopped_pools_prop(e: Arc, metadata: bool) -> StoppedOrLockedPools { + prop_conv::stopped_pools_to_prop(&block_on(e.stopped_pools()), metadata) } diff --git a/src/dbus_api/tree.rs b/src/dbus_api/tree.rs index b44cd502d1..1fadb300a6 100644 --- a/src/dbus_api/tree.rs +++ b/src/dbus_api/tree.rs @@ -578,32 +578,32 @@ impl DbusTreeHandler { consts::MANAGER_INTERFACE_NAME_3_2 => { Vec::new(), consts::STOPPED_POOLS_PROP.to_string() => - box_variant!(stopped_pools_to_prop(&stopped_pools)) + box_variant!(stopped_pools_to_prop(&stopped_pools, false)) }, consts::MANAGER_INTERFACE_NAME_3_3 => { Vec::new(), consts::STOPPED_POOLS_PROP.to_string() => - box_variant!(stopped_pools_to_prop(&stopped_pools)) + box_variant!(stopped_pools_to_prop(&stopped_pools, false)) }, consts::MANAGER_INTERFACE_NAME_3_4 => { Vec::new(), consts::STOPPED_POOLS_PROP.to_string() => - box_variant!(stopped_pools_to_prop(&stopped_pools)) + box_variant!(stopped_pools_to_prop(&stopped_pools, false)) }, consts::MANAGER_INTERFACE_NAME_3_5 => { Vec::new(), consts::STOPPED_POOLS_PROP.to_string() => - box_variant!(stopped_pools_to_prop(&stopped_pools)) + box_variant!(stopped_pools_to_prop(&stopped_pools, false)) }, consts::MANAGER_INTERFACE_NAME_3_6 => { Vec::new(), consts::STOPPED_POOLS_PROP.to_string() => - box_variant!(stopped_pools_to_prop(&stopped_pools)) + box_variant!(stopped_pools_to_prop(&stopped_pools, false)) }, consts::MANAGER_INTERFACE_NAME_3_7 => { Vec::new(), consts::STOPPED_POOLS_PROP.to_string() => - box_variant!(stopped_pools_to_prop(&stopped_pools)) + box_variant!(stopped_pools_to_prop(&stopped_pools, true)) } }, ) diff --git a/src/engine/sim_engine/engine.rs b/src/engine/sim_engine/engine.rs index 7c61fc685d..65546d1c60 100644 --- a/src/engine/sim_engine/engine.rs +++ b/src/engine/sim_engine/engine.rs @@ -29,6 +29,7 @@ use crate::{ SetUnlockAction, StartAction, StopAction, StoppedPoolInfo, StoppedPoolsInfo, StratFilesystemDiff, UdevEngineEvent, UnlockMethod, }, + StratSigblockVersion, }, stratis::{StratisError, StratisResult}, }; @@ -254,6 +255,7 @@ impl Engine for SimEngine { uuid: dev_uuid, }) .collect::>(), + metadata_version: Some(StratSigblockVersion::V2), }, ); st diff --git a/src/engine/strat_engine/liminal/device_info.rs b/src/engine/strat_engine/liminal/device_info.rs index 561c188d53..a367a3255c 100644 --- a/src/engine/strat_engine/liminal/device_info.rs +++ b/src/engine/strat_engine/liminal/device_info.rs @@ -708,6 +708,7 @@ impl DeviceSet { } }) .collect::>(), + metadata_version: self.metadata_version().ok(), }) } diff --git a/src/engine/types/mod.rs b/src/engine/types/mod.rs index 5a0281f8c0..6f2c626ce5 100644 --- a/src/engine/types/mod.rs +++ b/src/engine/types/mod.rs @@ -246,6 +246,7 @@ pub struct LockedPoolsInfo { pub struct StoppedPoolInfo { pub info: Option, pub devices: Vec, + pub metadata_version: Option, } #[derive(Default, Debug, Eq, PartialEq)] From 7264bcf70d0c89bf1b6f9b7971272c60ee9e3de3 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 7 Jun 2024 10:15:01 -0400 Subject: [PATCH 28/32] Add ability to report features on stopped pools --- src/dbus_api/api/prop_conv.rs | 16 ++++ src/engine/sim_engine/engine.rs | 11 ++- .../strat_engine/backstore/backstore/v2.rs | 2 +- .../strat_engine/liminal/device_info.rs | 73 ++++++++++++------- src/engine/strat_engine/liminal/liminal.rs | 50 +++++-------- src/engine/strat_engine/liminal/setup.rs | 60 +++++++++++++-- src/engine/strat_engine/serde_structs.rs | 10 ++- src/engine/types/mod.rs | 6 ++ 8 files changed, 157 insertions(+), 71 deletions(-) diff --git a/src/dbus_api/api/prop_conv.rs b/src/dbus_api/api/prop_conv.rs index f4bbc177fa..96a1f66576 100644 --- a/src/dbus_api/api/prop_conv.rs +++ b/src/dbus_api/api/prop_conv.rs @@ -66,6 +66,9 @@ pub fn locked_pools_to_prop(pools: &LockedPoolsInfo) -> StoppedOrLockedPools { } /// Convert a stopped pool data structure to a property format. +/// +/// if metadata is true show pool V2 D-Bus attributes such as metadata version and enabled features +/// for the stopped pool. pub fn stopped_pools_to_prop(pools: &StoppedPoolsInfo, metadata: bool) -> StoppedOrLockedPools { pools .stopped @@ -119,6 +122,19 @@ pub fn stopped_pools_to_prop(pools: &StoppedPoolsInfo, metadata: bool) -> Stoppe None => Variant(Box::new((false, 0))), }, ); + map.insert( + "features".to_string(), + match stopped.features { + Some(ref f) => { + let mut feat = HashMap::new(); + if f.encryption { + feat.insert("encryption".to_string(), true); + } + Variant(Box::new((true, feat))) + } + None => Variant(Box::new((false, HashMap::::new()))), + }, + ); } (uuid_to_string!(u), map) }) diff --git a/src/engine/sim_engine/engine.rs b/src/engine/sim_engine/engine.rs index 65546d1c60..45f2a034a7 100644 --- a/src/engine/sim_engine/engine.rs +++ b/src/engine/sim_engine/engine.rs @@ -24,10 +24,10 @@ use crate::{ SomeLockWriteGuard, Table, }, types::{ - CreateAction, DeleteAction, DevUuid, EncryptionInfo, FilesystemUuid, LockedPoolsInfo, - Name, PoolDevice, PoolDiff, PoolIdentifier, PoolUuid, RenameAction, ReportType, - SetUnlockAction, StartAction, StopAction, StoppedPoolInfo, StoppedPoolsInfo, - StratFilesystemDiff, UdevEngineEvent, UnlockMethod, + CreateAction, DeleteAction, DevUuid, EncryptionInfo, Features, FilesystemUuid, + LockedPoolsInfo, Name, PoolDevice, PoolDiff, PoolIdentifier, PoolUuid, RenameAction, + ReportType, SetUnlockAction, StartAction, StopAction, StoppedPoolInfo, + StoppedPoolsInfo, StratFilesystemDiff, UdevEngineEvent, UnlockMethod, }, StratSigblockVersion, }, @@ -256,6 +256,9 @@ impl Engine for SimEngine { }) .collect::>(), metadata_version: Some(StratSigblockVersion::V2), + features: Some(Features { + encryption: pool.is_encrypted(), + }), }, ); st diff --git a/src/engine/strat_engine/backstore/backstore/v2.rs b/src/engine/strat_engine/backstore/backstore/v2.rs index 153d8e3e1a..83339bc298 100644 --- a/src/engine/strat_engine/backstore/backstore/v2.rs +++ b/src/engine/strat_engine/backstore/backstore/v2.rs @@ -382,7 +382,7 @@ impl Backstore { (Some(placeholder), None, None, Some(origin)) }; - let metadata_enc_enabled = pool_save.features.contains(PoolFeatures::Encryption); + let metadata_enc_enabled = pool_save.features.contains(&PoolFeatures::Encryption); let crypt_physical_path = &once(DEVICEMAPPER_PATH) .chain(once( format_backstore_ids(pool_uuid, CacheRole::Cache) diff --git a/src/engine/strat_engine/liminal/device_info.rs b/src/engine/strat_engine/liminal/device_info.rs index a367a3255c..ec89be4bd9 100644 --- a/src/engine/strat_engine/liminal/device_info.rs +++ b/src/engine/strat_engine/liminal/device_info.rs @@ -21,12 +21,12 @@ use crate::{ backstore::blockdev::{v1, v2}, liminal::{ identify::{DeviceInfo, LuksInfo, StratisDevInfo, StratisInfo}, - setup::get_name, + setup::{get_feature_set, get_name}, }, metadata::{StratisIdentifiers, BDA}, }, types::{ - DevUuid, EncryptionInfo, LockedPoolInfo, MaybeInconsistent, Name, PoolDevice, + DevUuid, EncryptionInfo, Features, LockedPoolInfo, MaybeInconsistent, Name, PoolDevice, PoolEncryptionInfo, PoolUuid, StoppedPoolInfo, StratSigblockVersion, }, }, @@ -611,11 +611,15 @@ impl DeviceSet { ) } - /// Get the name, if available, of the pool formed by the devices + /// Get the required pool level metadata information, if available, of the pool formed by the devices /// in this DeviceSet. - pub fn pool_name(&self) -> StratisResult>> { + pub fn pool_level_metadata_info( + &self, + ) -> StratisResult<(MaybeInconsistent>, Option)> { match self.as_opened_set() { - Some(set) => get_name(set).map(MaybeInconsistent::No), + Some(set) => get_name(&set).map(MaybeInconsistent::No).and_then(|name| { + get_feature_set(&set).map(|feat| (name, feat.map(Features::from))) + }), None => gather_pool_name( self.internal.len(), self.internal.values().map(|info| match info { @@ -623,7 +627,12 @@ impl DeviceSet { LInfo::Luks(l) => Some(l.pool_name.as_ref()), }), ) - .map(|opt| opt.expect("self.as_opened_set().is_some() if pool is unencrypted")), + .map(|opt| { + ( + opt.expect("self.as_opened_set().is_some() if pool is unencrypted"), + Some(Features { encryption: true }), + ) + }), } } @@ -688,27 +697,37 @@ impl DeviceSet { self.internal.values().map(|info| info.encryption_info()), ) .ok() - .map(|info| StoppedPoolInfo { - info, - devices: self - .internal - .iter() - .map(|(uuid, l)| { - let devnode = match l { - LInfo::Stratis(strat_info) => strat_info - .luks - .as_ref() - .map(|l| l.dev_info.devnode.clone()) - .unwrap_or_else(|| strat_info.dev_info.devnode.clone()), - LInfo::Luks(luks_info) => luks_info.dev_info.devnode.clone(), - }; - PoolDevice { - devnode, - uuid: *uuid, - } - }) - .collect::>(), - metadata_version: self.metadata_version().ok(), + .map(|info| { + let features = match self.pool_level_metadata_info() { + Ok((_, opt)) => opt, + Err(e) => { + warn!("Failed to read metadata for pool: {e}"); + None + } + }; + StoppedPoolInfo { + info, + devices: self + .internal + .iter() + .map(|(uuid, l)| { + let devnode = match l { + LInfo::Stratis(strat_info) => strat_info + .luks + .as_ref() + .map(|l| l.dev_info.devnode.clone()) + .unwrap_or_else(|| strat_info.dev_info.devnode.clone()), + LInfo::Luks(luks_info) => luks_info.dev_info.devnode.clone(), + }; + PoolDevice { + devnode, + uuid: *uuid, + } + }) + .collect::>(), + metadata_version: self.metadata_version().ok(), + features, + } }) } diff --git a/src/engine/strat_engine/liminal/liminal.rs b/src/engine/strat_engine/liminal/liminal.rs index a42baa726d..6dfdf75480 100644 --- a/src/engine/strat_engine/liminal/liminal.rs +++ b/src/engine/strat_engine/liminal/liminal.rs @@ -189,7 +189,7 @@ impl LiminalDevices { fn start_pool_failure( pools: &Table, pool_uuid: PoolUuid, - luks_info: StratisResult<(Option, MaybeInconsistent>)>, + luks_info: StratisResult>, infos: &HashMap, bdas: HashMap, meta_res: StratisResult<(DateTime, PoolSave)>, @@ -276,9 +276,7 @@ impl LiminalDevices { assert!(pools.get_by_uuid(pool_uuid).is_none()); assert!(!self.stopped_pools.contains_key(&pool_uuid)); - let encryption_info = stopped_pool.encryption_info(); - let pool_name = stopped_pool.pool_name(); - let luks_info = encryption_info.and_then(|ei| pool_name.map(|pn| (ei, pn))); + let luks_info = stopped_pool.encryption_info(); let infos = match stopped_pool.into_opened_set() { Either::Left(i) => i, Either::Right(ds) => { @@ -353,7 +351,7 @@ impl LiminalDevices { }; let passphrase = match ( - metadata.features.contains(PoolFeatures::Encryption), + metadata.features.contains(&PoolFeatures::Encryption), unlock_method, passphrase_fd, ) { @@ -768,8 +766,8 @@ impl LiminalDevices { info_map.process_info_add(info); } - match info_map.pool_name() { - Ok(MaybeInconsistent::No(Some(name))) => { + match info_map.pool_level_metadata_info() { + Ok((MaybeInconsistent::No(Some(name)), _)) => { if let Some(maybe_conflict) = self.name_to_uuid.get_mut(&name) { maybe_conflict.add(*pool_uuid); if let UuidOrConflict::Conflict(set) = maybe_conflict { @@ -847,7 +845,7 @@ impl LiminalDevices { fn try_setup_started_pool_failure( pools: &Table, pool_uuid: PoolUuid, - luks_info: StratisResult<(Option, MaybeInconsistent>)>, + luks_info: StratisResult>, infos: &HashMap, bdas: HashMap, metadata_version: StratisResult, @@ -868,7 +866,7 @@ impl LiminalDevices { ) .map(Either::Left), StratSigblockVersion::V2 => { - let is_encrypted = metadata.features.contains(PoolFeatures::Encryption); + let is_encrypted = metadata.features.contains(&PoolFeatures::Encryption); setup_pool( pools, pool_uuid, @@ -895,9 +893,7 @@ impl LiminalDevices { assert!(!self.stopped_pools.contains_key(&pool_uuid)); let metadata_version = device_set.metadata_version(); - let encryption_info = device_set.encryption_info(); - let pool_name = device_set.pool_name(); - let luks_info = encryption_info.and_then(|ei| pool_name.map(|pn| (ei, pn))); + let luks_info = device_set.encryption_info(); let infos = match device_set.into_opened_set() { Either::Left(i) => i, Either::Right(ds) => { @@ -1074,8 +1070,8 @@ impl LiminalDevices { .insert(device_path.to_path_buf(), (pool_uuid, device_uuid)); devices.process_info_add(info); - match devices.pool_name() { - Ok(MaybeInconsistent::No(Some(name))) => { + match devices.pool_level_metadata_info() { + Ok((MaybeInconsistent::No(Some(name)), _)) => { if let Some(maybe_conflict) = self.name_to_uuid.get_mut(&name) { maybe_conflict.add(pool_uuid); if let UuidOrConflict::Conflict(set) = maybe_conflict { @@ -1111,8 +1107,8 @@ impl LiminalDevices { devices.process_info_remove(device_path, pool_uuid, dev_uuid); self.uuid_lookup.remove(device_path); - match devices.pool_name() { - Ok(MaybeInconsistent::No(Some(name))) => { + match devices.pool_level_metadata_info() { + Ok((MaybeInconsistent::No(Some(name)), _)) => { if let Some(maybe_conflict) = self.name_to_uuid.get_mut(&name) { maybe_conflict.add(pool_uuid); if let UuidOrConflict::Conflict(set) = maybe_conflict { @@ -1261,7 +1257,7 @@ fn load_stratis_metadata( ))); } - match get_metadata(infos) { + match get_metadata(&infos) { Ok(opt) => opt .ok_or_else(|| { StratisError::Msg(format!( @@ -1288,7 +1284,7 @@ fn load_stratis_metadata( fn setup_pool_legacy( pools: &Table, pool_uuid: PoolUuid, - luks_info: StratisResult<(Option, MaybeInconsistent>)>, + luks_info: StratisResult>, infos: &HashMap, bdas: HashMap, timestamp: DateTime, @@ -1317,7 +1313,7 @@ fn setup_pool_legacy( Ok((datadevs, cachedevs)) => (datadevs, cachedevs), }; - let (pool_einfo, pool_name) = match luks_info { + let pool_einfo = match luks_info { Ok(inner) => inner, Err(_) => { // NOTE: This is not actually a hopeless situation. It may be @@ -1336,21 +1332,9 @@ fn setup_pool_legacy( v1::StratPool::setup(pool_uuid, datadevs, cachedevs, timestamp, &metadata, pool_einfo) .map(|(name, mut pool)| { - if matches!(pool_name, MaybeInconsistent::Yes | MaybeInconsistent::No(None)) || MaybeInconsistent::No(Some(&name)) != pool_name.as_ref() || pool.blockdevs().iter().map(|(_, _, bd)| { + if pool.blockdevs().iter().map(|(_, _, bd)| { bd.pool_name() - }).fold(false, |acc, next| { - match next { - Some(Some(name)) => { - if MaybeInconsistent::No(Some(name)) == pool_name.as_ref() { - acc - } else { - true - } - }, - Some(None) => true, - None => false, - } - }) { + }).any(|name| name != Some(Some(&Name::new(metadata.name.clone()))) || matches!(name, Some(None))) { if let Err(e) = pool.rename_pool(&name) { warn!("Pool will not be able to be started by name; pool name metadata in LUKS2 token is not consistent across all devices: {}", e); } diff --git a/src/engine/strat_engine/liminal/setup.rs b/src/engine/strat_engine/liminal/setup.rs index ef84bb69da..ff5360168a 100644 --- a/src/engine/strat_engine/liminal/setup.rs +++ b/src/engine/strat_engine/liminal/setup.rs @@ -26,7 +26,7 @@ use crate::{ device::blkdev_size, liminal::device_info::{LStratisDevInfo, LStratisInfo}, metadata::BDA, - serde_structs::{BackstoreSave, BaseBlockDevSave, PoolSave}, + serde_structs::{BackstoreSave, BaseBlockDevSave, PoolFeatures, PoolSave}, shared::{bds_to_bdas, tiers_to_bdas}, types::{BDARecordResult, BDAResult}, }, @@ -44,7 +44,7 @@ use crate::{ /// /// Precondition: infos and bdas have identical sets of keys pub fn get_metadata( - infos: HashMap, + infos: &HashMap, ) -> StratisResult, PoolSave)>> { // Try to read from all available devnodes that could contain most // recent metadata. In the event of errors, continue to try until all are @@ -82,9 +82,7 @@ pub fn get_metadata( /// metadata could be written. /// Returns an error if devices provided don't match the devices recorded in the /// metadata. -/// -/// Precondition: infos and bdas have identical sets of keys -pub fn get_name(infos: HashMap) -> StratisResult> { +pub fn get_name(infos: &HashMap) -> StratisResult> { let found_uuids = infos.keys().copied().collect::>(); match get_metadata(infos)? { Some((_, pool)) => { @@ -128,6 +126,58 @@ pub fn get_name(infos: HashMap) -> StratisResult, +) -> StratisResult>> { + let found_uuids = infos.keys().copied().collect::>(); + match get_metadata(infos)? { + Some((_, pool)) => { + let v = []; + let meta_uuids = pool + .backstore + .data_tier + .blockdev + .devs + .iter() + .map(|bd| bd.uuid) + .chain( + pool.backstore + .cache_tier + .as_ref() + .map(|ct| ct.blockdev.devs.iter()) + .unwrap_or_else(|| v.iter()) + .map(|bd| bd.uuid), + ) + .collect::>(); + + if found_uuids != meta_uuids { + return Err(StratisError::Msg(format!( + "UUIDs in metadata ({}) did not match UUIDs found ({})", + Itertools::intersperse( + meta_uuids.into_iter().map(|u| u.to_string()), + ", ".to_string(), + ) + .collect::(), + Itertools::intersperse( + found_uuids.into_iter().map(|u| u.to_string()), + ", ".to_string(), + ) + .collect::(), + ))); + } + + Ok(Some(pool.features)) + } + None => Ok(None), + } +} + /// Get all the blockdevs corresponding to this pool that can be obtained from /// the given devices. Sort the blockdevs in the order in which they were /// recorded in the metadata. diff --git a/src/engine/strat_engine/serde_structs.rs b/src/engine/strat_engine/serde_structs.rs index e9b395f5a3..794893cf84 100644 --- a/src/engine/strat_engine/serde_structs.rs +++ b/src/engine/strat_engine/serde_structs.rs @@ -16,7 +16,7 @@ use serde::{Serialize, Serializer}; use devicemapper::{Sectors, ThinDevId}; -use crate::engine::types::{DevUuid, FilesystemUuid}; +use crate::engine::types::{DevUuid, Features, FilesystemUuid}; const MAXIMUM_STRING_SIZE: usize = 255; @@ -79,6 +79,14 @@ pub enum PoolFeatures { Encryption, } +impl From> for Features { + fn from(v: Vec) -> Self { + Features { + encryption: v.contains(&PoolFeatures::Encryption), + } + } +} + // ALL structs that represent variable length metadata in pre-order // depth-first traversal order. Note that when organized by types rather than // values the structure is a DAG not a tree. This just means that there are diff --git a/src/engine/types/mod.rs b/src/engine/types/mod.rs index 6f2c626ce5..a763596558 100644 --- a/src/engine/types/mod.rs +++ b/src/engine/types/mod.rs @@ -247,6 +247,12 @@ pub struct StoppedPoolInfo { pub info: Option, pub devices: Vec, pub metadata_version: Option, + pub features: Option, +} + +#[derive(Debug, Eq, PartialEq)] +pub struct Features { + pub encryption: bool, } #[derive(Default, Debug, Eq, PartialEq)] From 664ebc560b7dc598042af20bfb7d16d0c4cbc331 Mon Sep 17 00:00:00 2001 From: mulhern Date: Fri, 5 Jul 2024 15:41:13 -0400 Subject: [PATCH 29/32] Run tests on both legacy and v2 pools Signed-off-by: mulhern --- tests-fmf/python.fmf | 17 +++- tests/client-dbus/tests/udev/_utils.py | 105 ++++++++++++++++++------- 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/tests-fmf/python.fmf b/tests-fmf/python.fmf index 3d2b37c06b..13634de2ef 100644 --- a/tests-fmf/python.fmf +++ b/tests-fmf/python.fmf @@ -17,12 +17,23 @@ environment: STRATISD: /usr/libexec/stratisd STRATIS_DUMPMETADATA: /usr/bin/stratis-dumpmetadata PYTHONPATH: ./src - LEGACY_POOL: /usr/local/bin/stratis-legacy-pool -/udev: +/legacy: + environment+: + LEGACY_POOL: /usr/local/bin/stratis-legacy-pool + +/legacy/udev: + summary: Run Python udev tests + test: make -f Makefile udev-tests + +/legacy/loop: + summary: Run Python tests that use loopbacked device framework + test: make -f Makefile tang-tests dump-metadata-tests + +/v2/udev: summary: Run Python udev tests test: make -f Makefile udev-tests -/loop: +/v2/loop: summary: Run Python tests that use loopbacked device framework test: make -f Makefile tang-tests dump-metadata-tests diff --git a/tests/client-dbus/tests/udev/_utils.py b/tests/client-dbus/tests/udev/_utils.py index 40f23a515b..7c5b31457d 100644 --- a/tests/client-dbus/tests/udev/_utils.py +++ b/tests/client-dbus/tests/udev/_utils.py @@ -16,6 +16,7 @@ """ # isort: STDLIB +import json import logging import os import random @@ -50,7 +51,7 @@ from ._loopback import LoopBackDevices _STRATISD = os.environ["STRATISD"] -_LEGACY_POOL = os.environ["LEGACY_POOL"] +_LEGACY_POOL = os.environ.get("LEGACY_POOL") CRYPTO_LUKS_FS_TYPE = "crypto_LUKS" STRATIS_FS_TYPE = "stratis" @@ -80,40 +81,86 @@ def create_pool( :rtype: bool * str * list of str :raises RuntimeError: if pool is not created """ - newly_created = False - if len(get_pools(name)) == 0: - cmdline = [_LEGACY_POOL, name] + devices - if key_description is not None: - cmdline.extend(["--key-desc", key_description]) - if clevis_info is not None: + def create_legacy_pool(): + newly_created = False + + if len(get_pools(name)) == 0: + cmdline = [_LEGACY_POOL, name] + devices + if key_description is not None: + cmdline.extend(["--key-desc", key_description]) + if clevis_info is not None: + (pin, (tang_url, thp)) = clevis_info + cmdline.extend(["--clevis", pin]) + if pin == "tang": + cmdline.extend(["--tang-url", tang_url]) + if thp is None: + cmdline.append("--trust-url") + else: + cmdline.extend(["--thumbprint", thp]) + + with subprocess.Popen( + cmdline, + text=True, + ) as output: + output.wait() + if output.returncode != 0: + raise RuntimeError( + f"Unable to create a pool {name} with devices {devices}: {output.stderr}" + ) + + newly_created = True + + i = 0 + while get_pools(name) == [] and i < 5: + i += 1 + time.sleep(1) + (pool_object_path, _) = next(iter(get_pools(name))) + bd_object_paths = [op for op, _ in get_blockdevs(pool_object_path)] + + return (newly_created, (pool_object_path, bd_object_paths)) + + def create_v2_pool(): + if clevis_info is None: + clevis_arg = None + else: (pin, (tang_url, thp)) = clevis_info - cmdline.extend(["--clevis", pin]) if pin == "tang": - cmdline.extend(["--tang-url", tang_url]) - if thp is None: - cmdline.append("--trust-url") - else: - cmdline.extend(["--thumbprint", thp]) - - with subprocess.Popen( - cmdline, - text=True, - ) as output: - output.wait() - if output.returncode != 0: - raise RuntimeError( - f"Unable to create a pool {name} with devices {devices}: {output.stderr}" + clevis_arg = ( + "tang", + json.dumps( + {"url": tang_url, "stratis:tang:trust_url": True} + if thp is None + else {"url": tang_url, "thp": thp} + ), ) + else: + clevis_arg = None + + (result, exit_code, error_str) = Manager.Methods.CreatePool( + get_object(TOP_OBJECT), + { + "name": name, + "devices": devices, + "key_desc": ( + (False, "") if key_description is None else (True, key_description) + ), + "clevis_info": ( + (False, ("", "")) if clevis_arg is None else (True, clevis_arg) + ), + }, + ) + + if exit_code != StratisdErrors.OK: + raise RuntimeError( + f"Unable to create a pool {name} with devices {devices}: {error_str}" + ) - newly_created = True + return result - i = 0 - while get_pools(name) == [] and i < 5: - i += 1 - time.sleep(1) - (pool_object_path, _) = next(iter(get_pools(name))) - bd_object_paths = [op for op, _ in get_blockdevs(pool_object_path)] + (newly_created, (pool_object_path, bd_object_paths)) = ( + create_v2_pool() if _LEGACY_POOL is None else create_legacy_pool() + ) if not overprovision: Pool.Properties.Overprovisioning.Set(get_object(pool_object_path), False) From 7df5a6b1867138695b7d5cc22807e44fe07ff574 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 15 Jul 2024 16:42:12 -0400 Subject: [PATCH 30/32] Allow test_encryption_simple_event to handle metadata V2 --- tests/client-dbus/tests/udev/test_udev.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/client-dbus/tests/udev/test_udev.py b/tests/client-dbus/tests/udev/test_udev.py index bcec8fd2db..a6978741e6 100644 --- a/tests/client-dbus/tests/udev/test_udev.py +++ b/tests/client-dbus/tests/udev/test_udev.py @@ -27,6 +27,7 @@ from ._dm import remove_stratis_setup from ._loopback import UDEV_ADD_EVENT, UDEV_REMOVE_EVENT from ._utils import ( + _LEGACY_POOL, CRYPTO_LUKS_FS_TYPE, STRATIS_FS_TYPE, OptionalKeyServiceContextManager, @@ -329,7 +330,11 @@ def _simple_event_test(self, *, key_spec=None): # pylint: disable=too-many-loca :type key_spec: (str, bytes) or NoneType """ num_devices = 3 - udev_wait_type = STRATIS_FS_TYPE if key_spec is None else CRYPTO_LUKS_FS_TYPE + udev_wait_type = ( + STRATIS_FS_TYPE + if key_spec is None or _LEGACY_POOL is None + else CRYPTO_LUKS_FS_TYPE + ) device_tokens = self._lb_mgr.create_devices(num_devices) devnodes = self._lb_mgr.device_files(device_tokens) key_spec = None if key_spec is None else [key_spec] @@ -393,7 +398,7 @@ def _simple_event_test(self, *, key_spec=None): # pylint: disable=too-many-loca }, ) - if key_spec is None: + if key_spec is None or _LEGACY_POOL is None: self.assertNotEqual(exit_code, StratisdErrors.OK) self.assertEqual(changed, False) else: From 0e91c93611b2633fcc71a96d29a9c211ed5b7a70 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Mon, 15 Jul 2024 16:56:22 -0400 Subject: [PATCH 31/32] Allow test_duplicate_pool_name to handle metadata V2 --- tests/client-dbus/tests/udev/test_udev.py | 55 ++++++++++++++++------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/tests/client-dbus/tests/udev/test_udev.py b/tests/client-dbus/tests/udev/test_udev.py index a6978741e6..eaa3047d66 100644 --- a/tests/client-dbus/tests/udev/test_udev.py +++ b/tests/client-dbus/tests/udev/test_udev.py @@ -492,17 +492,30 @@ def test_duplicate_pool_name( (luks_tokens, non_luks_tokens) = ( [ dev - for sublist in (pool_tokens[i] for i in encrypted_indices) + for sublist in ( + pool_tokens[i] + for i in (encrypted_indices if _LEGACY_POOL is not None else []) + ) for dev in sublist ], [ dev - for sublist in (pool_tokens[i] for i in unencrypted_indices) + for sublist in ( + pool_tokens[i] + for i in ( + unencrypted_indices + if _LEGACY_POOL is not None + else unencrypted_indices + encrypted_indices + ) + ) for dev in sublist ], ) - wait_for_udev(CRYPTO_LUKS_FS_TYPE, self._lb_mgr.device_files(luks_tokens)) + wait_for_udev( + CRYPTO_LUKS_FS_TYPE, + self._lb_mgr.device_files(luks_tokens), + ) wait_for_udev(STRATIS_FS_TYPE, self._lb_mgr.device_files(non_luks_tokens)) variant_pool_uuids = Manager.Properties.StoppedPools.Get( @@ -541,20 +554,28 @@ def test_duplicate_pool_name( get_object(object_path), {"name": random_string(10)} ) - self._lb_mgr.generate_synthetic_udev_events( - non_luks_tokens, UDEV_ADD_EVENT - ) - for pool_uuid, props in variant_pool_uuids.items(): - if "key_description" in props: - Manager.Methods.StartPool( - get_object(TOP_OBJECT), - { - "id": pool_uuid, - "unlock_method": (True, str(EncryptionMethod.KEYRING)), - "id_type": "uuid", - "key_fd": (False, 0), - }, - ) + if _LEGACY_POOL is not None: + self._lb_mgr.generate_synthetic_udev_events( + non_luks_tokens, UDEV_ADD_EVENT + ) + for pool_uuid, props in variant_pool_uuids.items(): + if "key_description" in props: + Manager.Methods.StartPool( + get_object(TOP_OBJECT), + { + "id": pool_uuid, + "unlock_method": ( + True, + str(EncryptionMethod.KEYRING), + ), + "id_type": "uuid", + "key_fd": (False, 0), + }, + ) + else: + self._lb_mgr.generate_synthetic_udev_events( + non_luks_tokens + luks_tokens, UDEV_ADD_EVENT + ) settle() From d69a78f8f55f730b59122dbca041923971d41ba1 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 19 Jul 2024 15:16:07 -0400 Subject: [PATCH 32/32] Reflect changes to encryption layering in sim pool --- src/engine/sim_engine/blockdev.rs | 55 ++----------------------------- src/engine/sim_engine/pool.rs | 52 ++++++++++++----------------- 2 files changed, 23 insertions(+), 84 deletions(-) diff --git a/src/engine/sim_engine/blockdev.rs b/src/engine/sim_engine/blockdev.rs index 4b3d474e9c..cd63aca3a7 100644 --- a/src/engine/sim_engine/blockdev.rs +++ b/src/engine/sim_engine/blockdev.rs @@ -12,7 +12,7 @@ use devicemapper::{Bytes, Sectors, IEC}; use crate::engine::{ engine::BlockDev, shared::now_to_timestamp, - types::{DevUuid, EncryptionInfo, KeyDescription, StratSigblockVersion}, + types::{DevUuid, StratSigblockVersion}, }; #[derive(Debug)] @@ -22,7 +22,6 @@ pub struct SimDev { user_info: Option, hardware_info: Option, initialization_time: DateTime, - encryption_info: Option, } impl SimDev { @@ -68,7 +67,7 @@ impl BlockDev for SimDev { impl SimDev { /// Generates a new device from any devnode. - pub fn new(devnode: &Path, encryption_info: Option<&EncryptionInfo>) -> (DevUuid, SimDev) { + pub fn new(devnode: &Path) -> (DevUuid, SimDev) { ( DevUuid::new_v4(), SimDev { @@ -76,7 +75,6 @@ impl SimDev { user_info: None, hardware_info: None, initialization_time: now_to_timestamp(), - encryption_info: encryption_info.cloned(), }, ) } @@ -87,37 +85,6 @@ impl SimDev { pub fn set_user_info(&mut self, user_info: Option<&str>) -> bool { set_blockdev_user_info!(self; user_info) } - - /// Set the clevis info for a block device. - pub fn set_clevis_info(&mut self, pin: &str, config: &Value) { - self.encryption_info = self - .encryption_info - .take() - .map(|ei| ei.set_clevis_info((pin.to_owned(), config.clone()))); - } - - /// Unset the clevis info for a block device. - pub fn unset_clevis_info(&mut self) { - self.encryption_info = self.encryption_info.take().map(|ei| ei.unset_clevis_info()); - } - - /// Set the key description for a block device. - pub fn set_key_desc(&mut self, key_desc: &KeyDescription) { - self.encryption_info = self - .encryption_info - .take() - .map(|ei| ei.set_key_desc(key_desc.clone())) - } - - /// Unset the key description for a block device. - pub fn unset_key_desc(&mut self) { - self.encryption_info = self.encryption_info.take().map(|ei| ei.unset_key_desc()) - } - - /// Get encryption information for this block device. - pub fn encryption_info(&self) -> Option<&EncryptionInfo> { - self.encryption_info.as_ref() - } } impl<'a> Into for &'a SimDev { @@ -128,24 +95,6 @@ impl<'a> Into for &'a SimDev { Value::from(self.devnode.display().to_string()), ); json.insert("size".to_string(), Value::from(self.size().to_string())); - if let Some(EncryptionInfo::Both(kd, (pin, config))) = self.encryption_info.as_ref() { - json.insert( - "key_description".to_string(), - Value::from(kd.as_application_str()), - ); - json.insert("clevis_pin".to_string(), Value::from(pin.to_owned())); - json.insert("clevis_config".to_string(), config.to_owned()); - } else if let Some(EncryptionInfo::KeyDesc(kd)) = self.encryption_info.as_ref() { - json.insert( - "key_description".to_string(), - Value::from(kd.as_application_str()), - ); - } else if let Some(EncryptionInfo::ClevisInfo((pin, config))) = - self.encryption_info.as_ref() - { - json.insert("clevis_pin".to_string(), Value::from(pin.to_owned())); - json.insert("clevis_config".to_string(), config.to_owned()); - } Value::from(json) } } diff --git a/src/engine/sim_engine/pool.rs b/src/engine/sim_engine/pool.rs index 1e9a017b29..45f45eb0f6 100644 --- a/src/engine/sim_engine/pool.rs +++ b/src/engine/sim_engine/pool.rs @@ -4,6 +4,7 @@ use std::{ collections::{hash_map::RandomState, HashMap, HashSet}, + iter::once, path::Path, vec::Vec, }; @@ -16,8 +17,8 @@ use crate::{ engine::{ engine::{BlockDev, Filesystem, Pool}, shared::{ - gather_encryption_info, init_cache_idempotent_or_err, validate_filesystem_size, - validate_filesystem_size_specs, validate_name, validate_paths, + init_cache_idempotent_or_err, validate_filesystem_size, validate_filesystem_size_specs, + validate_name, validate_paths, }, sim_engine::{blockdev::SimDev, filesystem::SimFilesystem}, structures::Table, @@ -39,6 +40,7 @@ pub struct SimPool { filesystems: Table, fs_limit: u64, enable_overprov: bool, + encryption_info: Option, } #[derive(Debug, Eq, PartialEq, Serialize)] @@ -51,7 +53,7 @@ pub struct PoolSave { impl SimPool { pub fn new(paths: &[&Path], enc_info: Option<&EncryptionInfo>) -> (PoolUuid, SimPool) { let devices: HashSet<_, RandomState> = HashSet::from_iter(paths); - let device_pairs = devices.iter().map(|p| SimDev::new(p, enc_info)); + let device_pairs = devices.iter().map(|p| SimDev::new(p)); ( PoolUuid::new_v4(), SimPool { @@ -60,6 +62,7 @@ impl SimPool { filesystems: Table::default(), fs_limit: 10, enable_overprov: true, + encryption_info: enc_info.cloned(), }, ) } @@ -86,35 +89,31 @@ impl SimPool { } fn encryption_info(&self) -> Option { - gather_encryption_info( - self.block_devs.len(), - self.block_devs.values().map(|bd| bd.encryption_info()), - ) - .expect("sim engine cannot create pools with encrypted and unencrypted devices together") + self.encryption_info + .as_ref() + .map(|p| PoolEncryptionInfo::from(once(p))) } fn add_clevis_info(&mut self, pin: &str, config: &Value) { - self.block_devs - .iter_mut() - .for_each(|(_, bd)| bd.set_clevis_info(pin, config)) + self.encryption_info = self + .encryption_info + .take() + .map(|ei| ei.set_clevis_info((pin.to_owned(), config.to_owned()))); } fn clear_clevis_info(&mut self) { - self.block_devs - .iter_mut() - .for_each(|(_, bd)| bd.unset_clevis_info()) + self.encryption_info = self.encryption_info.take().map(|ei| ei.unset_clevis_info()); } fn add_key_desc(&mut self, key_desc: &KeyDescription) { - self.block_devs - .iter_mut() - .for_each(|(_, bd)| bd.set_key_desc(key_desc)) + self.encryption_info = self + .encryption_info + .take() + .map(|ei| ei.set_key_desc(key_desc.to_owned())); } fn clear_key_desc(&mut self) { - self.block_devs - .iter_mut() - .for_each(|(_, bd)| bd.unset_key_desc()) + self.encryption_info = self.encryption_info.take().map(|ei| ei.unset_key_desc()); } /// Check the limit of filesystems on a pool and return an error if it has been passed. @@ -219,7 +218,7 @@ impl Pool for SimPool { "At least one blockdev path is required to initialize a cache.".to_string(), )); } - let blockdev_pairs: Vec<_> = blockdevs.iter().map(|p| SimDev::new(p, None)).collect(); + let blockdev_pairs: Vec<_> = blockdevs.iter().map(|p| SimDev::new(p)).collect(); let blockdev_uuids: Vec<_> = blockdev_pairs.iter().map(|(uuid, _)| *uuid).collect(); self.cache_devs.extend(blockdev_pairs); Ok(SetCreateAction::new(blockdev_uuids)) @@ -296,7 +295,6 @@ impl Pool for SimPool { } let devices: HashSet<_, RandomState> = HashSet::from_iter(paths); - let encryption_info = pool_enc_to_enc!(self.encryption_info()); let the_vec = match tier { BlockDevTier::Cache => &self.cache_devs, @@ -307,15 +305,7 @@ impl Pool for SimPool { let filtered_device_pairs: Vec<_> = devices .iter() - .map(|p| { - SimDev::new( - p, - match tier { - BlockDevTier::Data => encryption_info.as_ref(), - BlockDevTier::Cache => None, - }, - ) - }) + .map(|p| SimDev::new(p)) .filter(|(_, sd)| !filter.contains(&sd.devnode())) .collect();