From e1de4a5de64ee729cebfd6a9598ebd0539bad546 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 29 Jan 2021 18:19:33 -0500 Subject: [PATCH] extensions: Add support for development extensions In RHCOS, we ship kernel development-related packages as an extension. Those aren't really extensions that are meant to be layered onto the host. They're meant to be used in a build environment somewhere to compile kernel modules. This makes it very different from "OS extensions" in at least two drastic ways: 1. we don't want to do any depsolving (e.g. we don't want to pull in `gcc` or something) 2. some of those packages may be present in the base already, but we still want to redownload them Hesitated putting this functionality in rpm-ostree, but I think in the end it cuts from the benefit of moving this code to rpm-ostree if we can't entirely get rid of the Python script it obsoletes. Plus, being able to use the `match-base-evr` is still really useful for this use case. Let's add a new `kind` key to support this. The traditional extensions are called "OS extensions" and these new extensions are called "development extensions". The latter is not yet part of the state checksum, so change detection doesn't work there. I think that's fine for now though because the primary use case is the kernel, and there we want to match the base version. So if the kernel changes, the base would change too. (Though there's the corner case of adding a new package to the list while at the same version...) --- docs/extensions.md | 8 +++ rust/src/extensions.rs | 65 +++++++++++++++++++--- rust/src/lib.rs | 3 +- src/app/rpmostree-compose-builtin-tree.cxx | 40 ++++++++++++- tests/compose/test-basic-unified.sh | 23 +++++++- 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/docs/extensions.md b/docs/extensions.md index b8960f03cd..8a48f1e10d 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -36,6 +36,9 @@ extensions: # This can be whatever name you'd like. The name itself # isn't used by rpm-ostree. sooper-dooper-tracers: + # Optional; defaults to `os-extension`. An OS extension + # is an extension intended to be `rpm-ostree install`ed. + kind: os-extension # List of packages for this extension packages: - strace @@ -47,6 +50,11 @@ extensions: - x86_64 - aarch64 kernel-dev: + # A development extension lists packages useful for + # developing for the target OSTree, but won't be layered + # on top. A common example is kernel modules. No + # depsolving happens, packages listed are downloaded. + kind: development packages: - kernel-devel - kernel-headers diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs index a09928db3e..972ce87176 100644 --- a/rust/src/extensions.rs +++ b/rust/src/extensions.rs @@ -33,6 +33,21 @@ pub struct Extension { architectures: Option>, #[serde(skip_serializing_if = "Option::is_none")] match_base_evr: Option, + #[serde(default)] + kind: ExtensionKind, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum ExtensionKind { + OsExtension, + Development, +} + +impl Default for ExtensionKind { + fn default() -> Self { + ExtensionKind::OsExtension + } } fn extensions_load_stream( @@ -55,9 +70,11 @@ fn extensions_load_stream( .collect(); for (_, ext) in parsed.extensions.iter_mut() { - for pkg in &ext.packages { - if base_pkgs.contains_key(pkg.as_str()) { - bail!("package {} already present in base", pkg); + if ext.kind == ExtensionKind::OsExtension { + for pkg in &ext.packages { + if base_pkgs.contains_key(pkg.as_str()) { + bail!("package {} already present in base", pkg); + } } } if let Some(ref matched_base_pkg) = ext.match_base_evr { @@ -91,9 +108,18 @@ impl Extensions { self.repos.as_ref().map(|v| v.clone()).unwrap_or_default() } - pub(crate) fn get_packages(&self) -> Vec { + pub(crate) fn get_os_extension_packages(&self) -> Vec { self.extensions .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::OsExtension) + .flat_map(|(_, ext)| ext.packages.iter().cloned()) + .collect() + } + + pub(crate) fn get_development_packages(&self) -> Vec { + self.extensions + .iter() + .filter(|(_, ext)| ext.kind == ExtensionKind::Development) .flat_map(|(_, ext)| ext.packages.iter().cloned()) .collect() } @@ -156,7 +182,8 @@ extensions: let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); assert!(extensions.get_repos() == vec!["my-repo"]); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); + assert!(extensions.get_development_packages().is_empty()); } #[test] @@ -174,6 +201,21 @@ extensions: } } + #[test] + fn ext_in_devel() { + let buf = r###" +extensions: + foobar: + packages: + - foobar + kind: development +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_os_extension_packages().is_empty()); + assert!(extensions.get_development_packages() == vec!["foobar"]); + } + #[test] fn basearch_filter() { let buf = r###" @@ -192,10 +234,10 @@ extensions: "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["bazboo"]); + assert!(extensions.get_os_extension_packages() == vec!["bazboo"]); let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["dodo", "dada"]); + assert!(extensions.get_os_extension_packages() == vec!["dodo", "dada"]); } #[test] @@ -206,9 +248,16 @@ extensions: packages: - foobar-ext match-base-evr: foobar + kind: os-extension + devel: + packages: + - foobar-devel + match-base-evr: foobar + kind: development "###; let mut input = std::io::BufReader::new(buf.as_bytes()); let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); - assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_os_extension_packages() == vec!["foobar-ext-1.2-3"]); + assert!(extensions.get_development_packages() == vec!["foobar-devel-1.2-3"]); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f261f013cc..df90b3bb8b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -196,7 +196,8 @@ mod ffi { base_pkgs: &Vec, ) -> Result>; fn get_repos(&self) -> Vec; - fn get_packages(&self) -> Vec; + fn get_os_extension_packages(&self) -> Vec; + fn get_development_packages(&self) -> Vec; fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result; fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>; fn serialize_to_dir(&self, output_dir: &str) -> Result<()>; diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 978ef3bae6..ab6c27e7a4 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -1561,7 +1561,7 @@ rpmostree_compose_builtin_extensions (int argc, g_autoptr(RpmOstreeTreespec) spec = NULL; { g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free); - auto pkgs = extensions->get_packages(); + auto pkgs = extensions->get_os_extension_packages(); for (auto pkg : pkgs) g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str())); @@ -1622,6 +1622,44 @@ rpmostree_compose_builtin_extensions (int argc, return FALSE; } + /* This is hacky: for "development" extensions, we don't want any depsolving + * against the base OS. Rather than awkwardly teach the core about this, we + * just reuse its sack and keep all the functionality here. */ + + DnfContext *dnfctx = rpmostree_context_get_dnf (ctx); + DnfSack *sack = dnf_context_get_sack (dnfctx); + + /* disable the system repo; we always want to download, even if already in the base */ + dnf_sack_repo_enabled (sack, HY_SYSTEM_REPO_NAME, 0); + + auto pkgs = extensions->get_development_packages(); + g_autoptr(GPtrArray) devel_pkgs_to_download = + g_ptr_array_new_with_free_func (g_object_unref); + for (auto pkg : pkgs) + { + g_autoptr(GPtrArray) matches = rpmostree_get_matching_packages (sack, pkg.c_str()); + if (matches->len == 0) + return glnx_throw (error, "Package %s not found", pkg.c_str()); + DnfPackage *found_pkg = (DnfPackage*)matches->pdata[0]; + g_ptr_array_add (devel_pkgs_to_download, g_object_ref (found_pkg)); + } + + rpmostree_set_repos_on_packages (dnfctx, devel_pkgs_to_download); + + if (!rpmostree_download_packages (devel_pkgs_to_download, cancellable, error)) + return FALSE; + + for (guint i = 0; i < devel_pkgs_to_download->len; i++) + { + DnfPackage *pkg = (DnfPackage*)devel_pkgs_to_download->pdata[i]; + const char *src = dnf_package_get_filename (pkg); + const char *basename = glnx_basename (src); + if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd, + basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error)) + return FALSE; + } + + // XXX: account for development extensions extensions->update_state_checksum (state_checksum, opt_extensions_output_dir); extensions->serialize_to_dir (opt_extensions_output_dir); if (!process_touch_if_changed (error)) diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index 0578eb2392..9b511c1856 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -89,6 +89,16 @@ build_rpm dodo-base build_rpm dodo requires dodo-base build_rpm solitaire +# this is pretty terrible... need --json for `rpm-ostree db list` +kernel_vra=$(rpm-ostree db list --repo=${repo} ${treeref} kernel | tail -n1 | cut -d- -f2-) +kernel_v=$(cut -d- -f1 <<< "$kernel_vra") +kernel_ra=$(cut -d- -f2- <<< "$kernel_vra") +kernel_r=${kernel_ra%.x86_64} + +build_rpm kernel-core version ${kernel_v} release ${kernel_r} +build_rpm kernel-devel version ${kernel_v} release ${kernel_r} +build_rpm kernel-headers version ${kernel_v} release ${kernel_r} + cat > extensions.yaml << EOF extensions: extinct-birds: @@ -100,6 +110,13 @@ extensions: - nonexistent architectures: - badarch + kernel-devel: + kind: development + packages: + - kernel-core + - kernel-devel + - kernel-headers + match-base-evr: kernel EOF # we don't actually need root here, but in CI the cache may be in a qcow2 and @@ -110,10 +127,12 @@ runasroot rpm-ostree compose extensions --repo=${repo} \ --touch-if-changed extensions-changed ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm +ls extensions/kernel-{core,devel,headers}-${kernel_v}-${kernel_r}.x86_64.rpm test -f extensions-changed assert_jq extensions/extensions.json \ - '.extensions|length == 1' \ - '.extensions["extinct-birds"]' + '.extensions|length == 2' \ + '.extensions["extinct-birds"]' \ + '.extensions["kernel-devel"]' echo "ok extensions" rm extensions-changed